From 24886156468645e3449983de0e11d1af4a328ee9 Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Mon, 7 Aug 2023 02:00:22 +0200 Subject: [PATCH 1/6] Kaitai_of_data_encoding: support describe describe is used on all registered encodings (thus on all encodings which codec know about --- contrib/lib_kaitai_of_data_encoding/translate.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/lib_kaitai_of_data_encoding/translate.ml b/contrib/lib_kaitai_of_data_encoding/translate.ml index 79f23cfc2622..ce3d72b15eba 100644 --- a/contrib/lib_kaitai_of_data_encoding/translate.ml +++ b/contrib/lib_kaitai_of_data_encoding/translate.ml @@ -117,4 +117,5 @@ let rec from_data_encoding : in {(default_class_spec ~encoding_name) with seq; enums} | Conv {encoding; _} -> from_data_encoding ~encoding_name encoding + | Describe {encoding; _} -> from_data_encoding ~encoding_name encoding | _ -> failwith "Not implemented" -- GitLab From ee6cbee03144c4ac921c3ab20c8158dbd48eb4df Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Mon, 28 Aug 2023 14:49:05 +0200 Subject: [PATCH 2/6] Manifest: Make `octez_codec` depend on `kaitai` --- .gitlab/ci/jobs/packaging/opam_package.yml | 9 ++- .../lib_kaitai_of_data_encoding/dune | 6 +- .../lib_kaitai_of_data_encoding/ground.ml | 4 +- .../lib_kaitai_of_data_encoding/ground.mli | 12 ++-- .../lib_kaitai_of_data_encoding/test/dune | 6 +- .../test/test_translation_of_ground_types.ml | 8 +-- .../test/test_translation_of_tuples.ml | 16 ++--- .../lib_kaitai_of_data_encoding/translate.ml | 2 +- .../lib_kaitai_of_data_encoding/translate.mli | 2 +- contrib/kaitai-ocaml/src/dune | 4 +- contrib/kaitai-ocaml/test/basic.ml | 4 +- contrib/kaitai-ocaml/test/dune | 2 +- dune-project | 1 - manifest/main.ml | 72 ++++++++++--------- opam/kaitai-of-data-encoding.opam | 23 ------ opam/kaitai.opam | 1 + opam/octez-codec.opam | 1 + src/bin_codec/dune | 2 + 18 files changed, 80 insertions(+), 95 deletions(-) rename contrib/{ => kaitai-ocaml}/lib_kaitai_of_data_encoding/dune (57%) rename contrib/{ => kaitai-ocaml}/lib_kaitai_of_data_encoding/ground.ml (97%) rename contrib/{ => kaitai-ocaml}/lib_kaitai_of_data_encoding/ground.mli (90%) rename contrib/{ => kaitai-ocaml}/lib_kaitai_of_data_encoding/test/dune (78%) rename contrib/{ => kaitai-ocaml}/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml (92%) rename contrib/{ => kaitai-ocaml}/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml (89%) rename contrib/{ => kaitai-ocaml}/lib_kaitai_of_data_encoding/translate.ml (99%) rename contrib/{ => kaitai-ocaml}/lib_kaitai_of_data_encoding/translate.mli (96%) delete mode 100644 opam/kaitai-of-data-encoding.opam diff --git a/.gitlab/ci/jobs/packaging/opam_package.yml b/.gitlab/ci/jobs/packaging/opam_package.yml index 81b3c2f46274..79a0d37c6d30 100644 --- a/.gitlab/ci/jobs/packaging/opam_package.yml +++ b/.gitlab/ci/jobs/packaging/opam_package.yml @@ -416,9 +416,12 @@ opam:bls12-381: # Ignoring unreleased package internal-devtools. -# Ignoring unreleased package kaitai. - -# Ignoring unreleased package kaitai-of-data-encoding. +opam:kaitai: + extends: + - .opam_template + - .rules_template__trigger_all_opam_batch_7 + variables: + package: kaitai opam:octez-accuser-Proxford: extends: diff --git a/contrib/lib_kaitai_of_data_encoding/dune b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/dune similarity index 57% rename from contrib/lib_kaitai_of_data_encoding/dune rename to contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/dune index 44f3e5217a7a..cb7f41443a55 100644 --- a/contrib/lib_kaitai_of_data_encoding/dune +++ b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/dune @@ -2,9 +2,9 @@ ; Edit file manifest/main.ml instead. (library - (name kaitai_of_data_encoding) - (public_name kaitai-of-data-encoding) + (name kaitai_kaitai_of_data_encoding) + (public_name kaitai.kaitai-of-data-encoding) (libraries yaml data-encoding - kaitai)) + kaitai.ast)) diff --git a/contrib/lib_kaitai_of_data_encoding/ground.ml b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/ground.ml similarity index 97% rename from contrib/lib_kaitai_of_data_encoding/ground.ml rename to contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/ground.ml index 7a5a63f9273d..b6fcc87fafcf 100644 --- a/contrib/lib_kaitai_of_data_encoding/ground.ml +++ b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/ground.ml @@ -24,7 +24,7 @@ (* *) (*****************************************************************************) -open Kaitai.Types +open Kaitai_ast.Types let default_doc_spec = DocSpec.{summary = None; refs = []} @@ -32,7 +32,7 @@ let cond_no_cond = AttrSpec.ConditionalSpec.{ifExpr = None; repeat = RepeatSpec.NoRepeat} module Enum = struct - type map = (string * Kaitai.Types.EnumSpec.t) list + type map = (string * Kaitai_ast.Types.EnumSpec.t) list let add enums ((k, e) as enum) = let rec add = function diff --git a/contrib/lib_kaitai_of_data_encoding/ground.mli b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/ground.mli similarity index 90% rename from contrib/lib_kaitai_of_data_encoding/ground.mli rename to contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/ground.mli index eec3a1fc0650..cec50e38f046 100644 --- a/contrib/lib_kaitai_of_data_encoding/ground.mli +++ b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/ground.mli @@ -25,29 +25,29 @@ (*****************************************************************************) (** [default_doc_spec] is without summary and references. *) -val default_doc_spec : Kaitai.Types.DocSpec.t +val default_doc_spec : Kaitai_ast.Types.DocSpec.t (** [Enum] module defines enum definitions needed for describing data-encoding ground types. *) module Enum : sig (** [map] describes mapping of enum id (string) with the corresponding [EnumSpec.t]. *) - type map = (string * Kaitai.Types.EnumSpec.t) list + type map = (string * Kaitai_ast.Types.EnumSpec.t) list (** [bool] is a mapping for boolean type. *) - val bool : string * Kaitai.Types.EnumSpec.t + val bool : string * Kaitai_ast.Types.EnumSpec.t (** [add enums enum] returns a list of enum mappings. If [enums] don't contain [enum], then new list with it is returned, otherwise existing [enums] list is returned. *) - val add : map -> string * Kaitai.Types.EnumSpec.t -> map + val add : map -> string * Kaitai_ast.Types.EnumSpec.t -> map end (** [Attr] is module for getting [AttrSpec.t] of ground types. *) module Attr : sig (** [bool] returns [AttrSpec.t] definition of bool ground type. *) - val bool : Kaitai.Types.AttrSpec.t + val bool : Kaitai_ast.Types.AttrSpec.t (** [u1] returns [AttrSpec.t] definition of 8-bit unsigned integer. *) - val u1 : Kaitai.Types.AttrSpec.t + val u1 : Kaitai_ast.Types.AttrSpec.t end diff --git a/contrib/lib_kaitai_of_data_encoding/test/dune b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test/dune similarity index 78% rename from contrib/lib_kaitai_of_data_encoding/test/dune rename to contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test/dune index fe5cfcafcd33..472b9864cd6e 100644 --- a/contrib/lib_kaitai_of_data_encoding/test/dune +++ b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test/dune @@ -3,11 +3,11 @@ (library (name kaitai_of_data_encoding_test) - (package kaitai-of-data-encoding) + (package kaitai) (libraries yaml data-encoding - kaitai - kaitai-of-data-encoding) + kaitai.ast + kaitai.kaitai-of-data-encoding) (inline_tests (flags -verbose) (modes native)) (preprocess (pps ppx_expect))) diff --git a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml similarity index 92% rename from contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml rename to contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml index 3bb40a1110f5..99943459f6c4 100644 --- a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml +++ b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml @@ -29,11 +29,11 @@ let%expect_test "test uint8 translation" = let s = - Kaitai_of_data_encoding.Translate.from_data_encoding + Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding ~encoding_name:"ground_uint8" Data_encoding.uint8 in - print_endline (Kaitai.Print.print s) ; + print_endline (Kaitai_ast.Print.print s) ; [%expect {| meta: @@ -45,11 +45,11 @@ let%expect_test "test uint8 translation" = let%expect_test "test bool translation" = let s = - Kaitai_of_data_encoding.Translate.from_data_encoding + Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding ~encoding_name:"ground_bool" Data_encoding.bool in - print_endline (Kaitai.Print.print s) ; + print_endline (Kaitai_ast.Print.print s) ; [%expect {| meta: diff --git a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml similarity index 89% rename from contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml rename to contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml index 44eff71495b7..e01c009a88bb 100644 --- a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml +++ b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml @@ -26,11 +26,11 @@ let%expect_test "test tuple translation" = let s = - Kaitai_of_data_encoding.Translate.from_data_encoding + Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding ~encoding_name:"simple_tuple" Data_encoding.(tup2 bool uint8) in - print_endline (Kaitai.Print.print s) ; + print_endline (Kaitai_ast.Print.print s) ; [%expect {| meta: @@ -49,11 +49,11 @@ let%expect_test "test tuple translation" = let%expect_test "test long tuple translation" = let s = - Kaitai_of_data_encoding.Translate.from_data_encoding + Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding ~encoding_name:"simple_tuple" Data_encoding.(tup5 bool uint8 bool uint8 uint8) in - print_endline (Kaitai.Print.print s) ; + print_endline (Kaitai_ast.Print.print s) ; [%expect {| meta: @@ -78,11 +78,11 @@ let%expect_test "test long tuple translation" = let%expect_test "test tup1 tuple translation" = let s = - Kaitai_of_data_encoding.Translate.from_data_encoding + Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding ~encoding_name:"tup1" Data_encoding.(tup1 uint8) in - print_endline (Kaitai.Print.print s) ; + print_endline (Kaitai_ast.Print.print s) ; [%expect {| meta: @@ -94,12 +94,12 @@ let%expect_test "test tup1 tuple translation" = let%expect_test "test tuples with tup1 translation" = let s = - Kaitai_of_data_encoding.Translate.from_data_encoding + Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding ~encoding_name:"tup1tup" Data_encoding.( tup3 (tup1 bool) (tup2 uint8 bool) (tup2 (tup1 uint8) uint8)) in - print_endline (Kaitai.Print.print s) ; + print_endline (Kaitai_ast.Print.print s) ; [%expect {| meta: diff --git a/contrib/lib_kaitai_of_data_encoding/translate.ml b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/translate.ml similarity index 99% rename from contrib/lib_kaitai_of_data_encoding/translate.ml rename to contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/translate.ml index ce3d72b15eba..194d8cbdf489 100644 --- a/contrib/lib_kaitai_of_data_encoding/translate.ml +++ b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/translate.ml @@ -24,7 +24,7 @@ (* *) (*****************************************************************************) -open Kaitai.Types +open Kaitai_ast.Types (* We need to access the definition of data-encoding's [descr] type. For this reason we open the private/internal module [Data_encoding__Encoding] (rather diff --git a/contrib/lib_kaitai_of_data_encoding/translate.mli b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/translate.mli similarity index 96% rename from contrib/lib_kaitai_of_data_encoding/translate.mli rename to contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/translate.mli index 56eb68758c88..6cf70370855b 100644 --- a/contrib/lib_kaitai_of_data_encoding/translate.mli +++ b/contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/translate.mli @@ -29,4 +29,4 @@ @param [encoding_name] is added to the "meta" section of the class-spec. *) val from_data_encoding : - encoding_name:string -> 'a Data_encoding.t -> Kaitai.Types.ClassSpec.t + encoding_name:string -> 'a Data_encoding.t -> Kaitai_ast.Types.ClassSpec.t diff --git a/contrib/kaitai-ocaml/src/dune b/contrib/kaitai-ocaml/src/dune index cf5762d3d40e..94c9515d675a 100644 --- a/contrib/kaitai-ocaml/src/dune +++ b/contrib/kaitai-ocaml/src/dune @@ -2,8 +2,8 @@ ; Edit file manifest/main.ml instead. (library - (name kaitai) - (public_name kaitai) + (name kaitai_ast) + (public_name kaitai.ast) (instrumentation (backend bisect_ppx)) (libraries yaml)) diff --git a/contrib/kaitai-ocaml/test/basic.ml b/contrib/kaitai-ocaml/test/basic.ml index 05d7aeb096be..53de7a8e230f 100644 --- a/contrib/kaitai-ocaml/test/basic.ml +++ b/contrib/kaitai-ocaml/test/basic.ml @@ -39,9 +39,9 @@ seq: doc: The actual data. |} in - let k = Kaitai.Parse.parse s in + let k = Kaitai_ast.Parse.parse s in print_endline "Parsing ok" ; - let s = Kaitai.Print.print k in + let s = Kaitai_ast.Print.print k in print_endline "Serialization ok" ; print_endline s ; [%expect diff --git a/contrib/kaitai-ocaml/test/dune b/contrib/kaitai-ocaml/test/dune index 112ea428dcd9..2aba008f0869 100644 --- a/contrib/kaitai-ocaml/test/dune +++ b/contrib/kaitai-ocaml/test/dune @@ -6,6 +6,6 @@ (package kaitai) (instrumentation (backend bisect_ppx)) (libraries - kaitai) + kaitai.ast) (inline_tests (flags -verbose) (modes native)) (preprocess (pps ppx_expect))) diff --git a/dune-project b/dune-project index 900c5406cf4d..d961f8d41f65 100644 --- a/dune-project +++ b/dune-project @@ -5,7 +5,6 @@ (package (name bls12-381)) (package (name internal-devtools)) (package (name kaitai)) -(package (name kaitai-of-data-encoding)) (package (name octez-accuser-Proxford)) (package (name octez-accuser-PtNairob)) (package (name octez-accuser-alpha)) diff --git a/manifest/main.ml b/manifest/main.ml index 47f8b69a688b..03e885589bdb 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -7510,6 +7510,41 @@ let _octez_client = ]; ] +let kaitai = + public_lib + "kaitai.ast" + ~path:"contrib/kaitai-ocaml/src" + ~deps:[yaml] + ~synopsis:"OCaml library for reading Kaitai spec files" + +(* We use a private-lib with inline-tests in order to run the tests normally, + but without placing all the code for the tests within the main kaitai + library. *) +let _kaitai_test = + private_lib + "kaitai_test" + ~opam:"kaitai" + ~path:"contrib/kaitai-ocaml/test" + ~inline_tests:ppx_expect + ~deps:[kaitai] + +let kaitai_of_data_encoding = + public_lib + "kaitai.kaitai-of-data-encoding" + ~opam:"kaitai" + ~path:"contrib/kaitai-ocaml/lib_kaitai_of_data_encoding" + ~deps:[yaml; data_encoding; kaitai] + ~bisect_ppx:No + +let _kaitai_of_data_encoding_test = + private_lib + "kaitai_of_data_encoding_test" + ~opam:"kaitai" + ~path:"contrib/kaitai-ocaml/lib_kaitai_of_data_encoding/test" + ~deps:[yaml; data_encoding; kaitai; kaitai_of_data_encoding] + ~bisect_ppx:No + ~inline_tests:ppx_expect + let _octez_codec = public_exe "octez-codec" @@ -7530,6 +7565,8 @@ let _octez_codec = octez_event_logging |> open_; octez_signer_services; octez_version_value; + kaitai_of_data_encoding; + kaitai; ] @ Protocol.all_optionally @@ [ @@ -8090,41 +8127,6 @@ let octez_scoru_wasm_regressions = ] ~preprocess:[staged_pps [ppx_import; ppx_deriving_show]] -let kaitai = - public_lib - "kaitai" - ~path:"contrib/kaitai-ocaml/src" - ~deps:[yaml] - ~synopsis:"OCaml library for reading Kaitai spec files" - -(* We use a private-lib with inline-tests in order to run the tests normally, - but without placing all the code for the tests within the main kaitai - library. *) -let _kaitai_test = - private_lib - "kaitai_test" - ~opam:"kaitai" - ~path:"contrib/kaitai-ocaml/test" - ~inline_tests:ppx_expect - ~deps:[kaitai] - -let kaitai_of_data_encoding = - public_lib - "kaitai-of-data-encoding" - ~path:"contrib/lib_kaitai_of_data_encoding" - ~synopsis:"Kaitai spec generator for data-encoding library" - ~deps:[yaml; data_encoding; kaitai] - ~bisect_ppx:No - -let _kaitai_of_data_encoding_test = - private_lib - "kaitai_of_data_encoding_test" - ~opam:"kaitai-of-data-encoding" - ~path:"contrib/lib_kaitai_of_data_encoding/test" - ~deps:[yaml; data_encoding; kaitai; kaitai_of_data_encoding] - ~bisect_ppx:No - ~inline_tests:ppx_expect - (* Add entries to this function to declare that some dune and .opam files are not generated by the manifest on purpose. diff --git a/opam/kaitai-of-data-encoding.opam b/opam/kaitai-of-data-encoding.opam deleted file mode 100644 index 67e175d24d39..000000000000 --- a/opam/kaitai-of-data-encoding.opam +++ /dev/null @@ -1,23 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file manifest/main.ml instead. -opam-version: "2.0" -maintainer: "contact@tezos.com" -authors: ["Tezos devteam"] -homepage: "https://www.tezos.com/" -bug-reports: "https://gitlab.com/tezos/tezos/issues" -dev-repo: "git+https://gitlab.com/tezos/tezos.git" -license: "MIT" -depends: [ - "dune" { >= "3.0" } - "ocaml" { >= "4.14" } - "yaml" { >= "3.1.0" } - "data-encoding" { >= "0.7.1" & < "1.0.0" } - "kaitai" - "ppx_expect" -] -build: [ - ["rm" "-r" "vendors" "contrib"] - ["dune" "build" "-p" name "-j" jobs] - ["dune" "runtest" "-p" name "-j" jobs] {with-test} -] -synopsis: "Kaitai spec generator for data-encoding library" diff --git a/opam/kaitai.opam b/opam/kaitai.opam index 324d6c0fc7f1..6023810b594b 100644 --- a/opam/kaitai.opam +++ b/opam/kaitai.opam @@ -12,6 +12,7 @@ depends: [ "ocaml" { >= "4.14" } "yaml" { >= "3.1.0" } "ppx_expect" + "data-encoding" { >= "0.7.1" & < "1.0.0" } ] build: [ ["rm" "-r" "vendors" "contrib"] diff --git a/opam/octez-codec.opam b/opam/octez-codec.opam index 8fa0f003d280..cfafb2497f28 100644 --- a/opam/octez-codec.opam +++ b/opam/octez-codec.opam @@ -16,6 +16,7 @@ depends: [ "tezos-client-base" "tezos-signer-services" "tezos-version" + "kaitai" ] depopts: [ "tezos-client-005-PsBabyM1" diff --git a/src/bin_codec/dune b/src/bin_codec/dune index 17bb3128e240..6a2419f9674b 100644 --- a/src/bin_codec/dune +++ b/src/bin_codec/dune @@ -17,6 +17,8 @@ octez-libs.tezos-event-logging tezos-signer-services tezos-version.value + kaitai.kaitai-of-data-encoding + kaitai.ast (select void_for_linking-tezos-client-005-PsBabyM1 from (tezos-client-005-PsBabyM1 -> void_for_linking-tezos-client-005-PsBabyM1.empty) (-> void_for_linking-tezos-client-005-PsBabyM1.empty)) -- GitLab From 9a753fe80f17164f330b647fa5b901892d373ec1 Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Fri, 18 Aug 2023 17:26:35 +0200 Subject: [PATCH 3/6] Codec: Add new dump Kaitai spec file command --- src/bin_codec/commands.ml | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/bin_codec/commands.ml b/src/bin_codec/commands.ml index b92801a95bd3..cf3b3aa2c1e5 100644 --- a/src/bin_codec/commands.ml +++ b/src/bin_codec/commands.ml @@ -27,6 +27,12 @@ open Tezos_clic let group = {name = "encoding"; title = "Commands to handle encodings"} +let raw_id_parameter = + parameter (fun (cctxt : #Client_context.printer) id -> + match Data_encoding.Registration.find id with + | Some _ -> Lwt.return_ok id + | None -> cctxt#error "Unknown encoding id: %s" id) + let id_parameter = parameter (fun (cctxt : #Client_context.printer) id -> match Data_encoding.Registration.find id with @@ -416,4 +422,39 @@ let commands () = | Ok slices -> let* () = cctxt#message "%a\n" (pp_slices format) slices in Lwt_result_syntax.return_unit); + command + ~group + ~desc:"Dump a Kaitai Struct specification of a given registered encoding." + no_options + (prefix "dump" @@ prefix "kaitai" @@ prefix "for" + @@ param ~name:"id" ~desc:"Encoding identifier" raw_id_parameter + @@ stop) + (fun () encoding_id cctxt -> + let open Lwt_syntax in + let introspectable = + Data_encoding.Registration.find_introspectable encoding_id + in + let* kaitai_spec = + match introspectable with + | None -> cctxt#error "Error retrieving introspectable" + | Some (Any encoding) -> + let escape_encoding_id id = + id |> String.split '.' |> String.concat "__" + in + return + @@ Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:(escape_encoding_id encoding_id) + encoding + in + let* () = + cctxt#answer + "%a" + (fun fmt s -> + if s.[String.length s - 1] = '\n' then + let w, _ = Format.pp_get_formatter_output_functions fmt () in + w s 0 (String.length s - 1) + else Format.pp_print_string fmt s) + (Kaitai_ast.Print.print kaitai_spec) + in + Lwt_result_syntax.return_unit); ] -- GitLab From 885f590de6c4c806b07b7fe738262ae1a19486ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Proust?= Date: Fri, 1 Sep 2023 10:30:32 +0200 Subject: [PATCH 4/6] codec: add cram tests for kaitai file generation --- manifest/main.ml | 7 +++++++ src/bin_codec/dune | 2 ++ src/bin_codec/kaitai.t | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 src/bin_codec/kaitai.t diff --git a/manifest/main.ml b/manifest/main.ml index 03e885589bdb..66bca13bcc9d 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -7580,6 +7580,13 @@ let _octez_codec = if link then Protocol.client protocol else None); ]) ~linkall:true + ~dune: + Dune. + [ + S "cram" + :: G [S "deps" :: [S "codec.exe"]] + :: [S "package" :: [S "octez-codec"]]; + ] let _octez_proxy_server = public_exe diff --git a/src/bin_codec/dune b/src/bin_codec/dune index 6a2419f9674b..ecb1a9e1c0f7 100644 --- a/src/bin_codec/dune +++ b/src/bin_codec/dune @@ -96,3 +96,5 @@ (write-file void_for_linking-tezos-client-017-PtNairob.empty "") (write-file void_for_linking-tezos-client-018-Proxford.empty "") (write-file void_for_linking-tezos-client-alpha.empty "")))) + +(cram (deps codec.exe) (package octez-codec)) diff --git a/src/bin_codec/kaitai.t b/src/bin_codec/kaitai.t new file mode 100644 index 000000000000..750259db8a79 --- /dev/null +++ b/src/bin_codec/kaitai.t @@ -0,0 +1,19 @@ +ground.uint8 test + $ ./codec.exe dump kaitai for ground.uint8 + meta: + id: ground__uint8 + seq: + - id: uint8 + type: u1 +ground.bool test + $ ./codec.exe dump kaitai for ground.bool + meta: + id: ground__bool + enums: + bool: + 0: false + 255: true + seq: + - id: bool + type: u1 + enum: bool -- GitLab From 93be4148e2b59956eec6755e83ad6a757705c6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Proust?= Date: Mon, 4 Sep 2023 11:06:58 +0200 Subject: [PATCH 5/6] Move codec to contrib Codec is not core to the functioning of the chain; it is a useful tool to instrospect some data from the chain or the RPCs, to build scripts to wrangle data, etc. Codec depends on kaitai which is contrib. We can't have dependencies from src/ into contrib/. --- {src => contrib}/bin_codec/additional_registrations.ml | 0 {src => contrib}/bin_codec/codec.ml | 0 {src => contrib}/bin_codec/commands.ml | 0 {src => contrib}/bin_codec/commands.mli | 0 {src => contrib}/bin_codec/dune | 0 {src => contrib}/bin_codec/kaitai.t | 0 manifest/main.ml | 2 +- 7 files changed, 1 insertion(+), 1 deletion(-) rename {src => contrib}/bin_codec/additional_registrations.ml (100%) rename {src => contrib}/bin_codec/codec.ml (100%) rename {src => contrib}/bin_codec/commands.ml (100%) rename {src => contrib}/bin_codec/commands.mli (100%) rename {src => contrib}/bin_codec/dune (100%) rename {src => contrib}/bin_codec/kaitai.t (100%) diff --git a/src/bin_codec/additional_registrations.ml b/contrib/bin_codec/additional_registrations.ml similarity index 100% rename from src/bin_codec/additional_registrations.ml rename to contrib/bin_codec/additional_registrations.ml diff --git a/src/bin_codec/codec.ml b/contrib/bin_codec/codec.ml similarity index 100% rename from src/bin_codec/codec.ml rename to contrib/bin_codec/codec.ml diff --git a/src/bin_codec/commands.ml b/contrib/bin_codec/commands.ml similarity index 100% rename from src/bin_codec/commands.ml rename to contrib/bin_codec/commands.ml diff --git a/src/bin_codec/commands.mli b/contrib/bin_codec/commands.mli similarity index 100% rename from src/bin_codec/commands.mli rename to contrib/bin_codec/commands.mli diff --git a/src/bin_codec/dune b/contrib/bin_codec/dune similarity index 100% rename from src/bin_codec/dune rename to contrib/bin_codec/dune diff --git a/src/bin_codec/kaitai.t b/contrib/bin_codec/kaitai.t similarity index 100% rename from src/bin_codec/kaitai.t rename to contrib/bin_codec/kaitai.t diff --git a/manifest/main.ml b/manifest/main.ml index 66bca13bcc9d..20c1d401cf5d 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -7548,7 +7548,7 @@ let _kaitai_of_data_encoding_test = let _octez_codec = public_exe "octez-codec" - ~path:"src/bin_codec" + ~path:"contrib/bin_codec" ~internal_name:"codec" ~synopsis:"Tezos: `octez-codec` binary to encode and decode values" ~release_status:Released -- GitLab From 54ec9a28cd54bbd75099fa2068b124671b1e0732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Proust?= Date: Mon, 4 Sep 2023 11:08:57 +0200 Subject: [PATCH 6/6] Manifest: delete contrib and vendors directory selectively (Specifically: don't delete `contrib/` for code located in `contrib/`.) --- manifest/manifest.ml | 23 ++++++++++++++++++----- opam/kaitai.opam | 2 +- opam/octez-codec.opam | 2 +- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/manifest/manifest.ml b/manifest/manifest.ml index 6d218e5093ae..9ddb6a2b2f1c 100644 --- a/manifest/manifest.ml +++ b/manifest/manifest.ml @@ -2901,11 +2901,24 @@ let generate_opam ?release for_package (internals : Target.internal list) : |> String_set.of_list |> String_set.elements |> List.concat_map make_runtest in - { - Opam.command = [S "rm"; S "-r"; S "vendors"; S "contrib"]; - with_test = Never; - } - :: build :: runtests + let rms = + let active_paths = + internals + |> List.map (fun x -> x.Target.path) + |> List.map (fun p -> + let slash_index = String.index p '/' in + String.sub p 0 slash_index) + |> List.sort_uniq String.compare + in + let rms = + List.filter + (fun s -> not (List.mem s active_paths)) + List.["vendors"; "contrib"] + |> List.map (fun x -> Opam.S x) + in + match rms with [] -> [] | _ :: _ -> Opam.S "rm" :: Opam.S "-r" :: rms + in + {Opam.command = rms; with_test = Never} :: build :: runtests in let licenses = match diff --git a/opam/kaitai.opam b/opam/kaitai.opam index 6023810b594b..e4d9db8ae76b 100644 --- a/opam/kaitai.opam +++ b/opam/kaitai.opam @@ -15,7 +15,7 @@ depends: [ "data-encoding" { >= "0.7.1" & < "1.0.0" } ] build: [ - ["rm" "-r" "vendors" "contrib"] + ["rm" "-r" "vendors"] ["dune" "build" "-p" name "-j" jobs] ["dune" "runtest" "-p" name "-j" jobs] {with-test} ] diff --git a/opam/octez-codec.opam b/opam/octez-codec.opam index cfafb2497f28..71e62e2c525b 100644 --- a/opam/octez-codec.opam +++ b/opam/octez-codec.opam @@ -36,7 +36,7 @@ depopts: [ "tezos-client-alpha" ] build: [ - ["rm" "-r" "vendors" "contrib"] + ["rm" "-r" "vendors"] ["dune" "build" "-p" name "-j" jobs] ["dune" "runtest" "-p" name "-j" jobs] {with-test} ] -- GitLab