diff --git a/.gitlab/ci/jobs/packaging/opam_package.yml b/.gitlab/ci/jobs/packaging/opam_package.yml index c92a3456943bf87eea2f4cdc1a73f99c9aa1a317..87eab65c646a9d27df56f5467c456a17b8988ea3 100644 --- a/.gitlab/ci/jobs/packaging/opam_package.yml +++ b/.gitlab/ci/jobs/packaging/opam_package.yml @@ -416,6 +416,10 @@ opam:bls12-381: # Ignoring unreleased package internal-devtools. +# Ignoring unreleased package kaitai. + +# Ignoring unreleased package kaitai-of-data-encoding. + opam:octez-accuser-Proxford: extends: - .opam_template diff --git a/contrib/kaitai-ocaml/src/dune b/contrib/kaitai-ocaml/src/dune index b2b88e8fff05161e694a119641c9bf36af2d8a08..cf5762d3d40efe4c828797a6d7aaa8c0970fc284 100644 --- a/contrib/kaitai-ocaml/src/dune +++ b/contrib/kaitai-ocaml/src/dune @@ -3,6 +3,7 @@ (library (name kaitai) + (public_name kaitai) (instrumentation (backend bisect_ppx)) (libraries yaml)) diff --git a/contrib/kaitai-ocaml/src/print.ml b/contrib/kaitai-ocaml/src/print.ml index 91fde364bc6796d6d2a00e61a222749f45baec9d..0b35df9be074d27292e584bb5cf2567a0de07896 100644 --- a/contrib/kaitai-ocaml/src/print.ml +++ b/contrib/kaitai-ocaml/src/print.ml @@ -84,7 +84,18 @@ let to_yaml (t : ClassSpec.t) = "seq", sequence (t.seq - |> List.map (fun v -> mapping [("id", scalar v.AttrSpec.id)])) ); + |> List.map (fun v -> + mapping + (("id", scalar v.AttrSpec.id) + :: + (* We only add "type" to Yaml if not [AnyType]. + TODO: This is only correct if [AnyType] means no type? *) + (if v.AttrSpec.dataType = AnyType then [] + else + [ + ( "type", + scalar (DataType.to_string v.AttrSpec.dataType) ); + ])))) ); ]) let print t = diff --git a/contrib/kaitai-ocaml/src/types.ml b/contrib/kaitai-ocaml/src/types.ml index 1e0f13f456e9a99bca5f7741d8c54fcd9b3129d1..cf7c8821ccc105494a4766ccef21eafec5e03d04 100644 --- a/contrib/kaitai-ocaml/src/types.ml +++ b/contrib/kaitai-ocaml/src/types.ml @@ -110,7 +110,7 @@ end module DataType = struct (* https://github.com/kaitai-io/kaitai_struct_compiler/blob/master/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala *) type data_type = - | NumericType + | NumericType of numeric_type | BooleanType | BytesType of bytes_type | StrType of str_type @@ -119,6 +119,8 @@ module DataType = struct and int_width = W1 | W2 | W4 | W8 + and numeric_type = Int_type of int_type | Float_type of float_type + and int_type = | CalcIntType | Int1Type of {signed : bool} @@ -177,6 +179,11 @@ module DataType = struct } type t = data_type + + let to_string = function + | NumericType (Int_type (Int1Type {signed})) -> + if signed then "s1" else "u1" + | _ -> failwith "not supported" end module DocSpec = struct diff --git a/contrib/kaitai-ocaml/test/dune b/contrib/kaitai-ocaml/test/dune index 2da41d4d5ebb6941599d9e38d09fbb22d868684f..112ea428dcd968555434735b2fa5b27fd3a25815 100644 --- a/contrib/kaitai-ocaml/test/dune +++ b/contrib/kaitai-ocaml/test/dune @@ -3,6 +3,7 @@ (library (name kaitai_test) + (package kaitai) (instrumentation (backend bisect_ppx)) (libraries kaitai) diff --git a/contrib/lib_kaitai_of_data_encoding/dune b/contrib/lib_kaitai_of_data_encoding/dune new file mode 100644 index 0000000000000000000000000000000000000000..44f3e5217a7a6a3115f0c9a6e1ef2a298544d997 --- /dev/null +++ b/contrib/lib_kaitai_of_data_encoding/dune @@ -0,0 +1,10 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(library + (name kaitai_of_data_encoding) + (public_name kaitai-of-data-encoding) + (libraries + yaml + data-encoding + kaitai)) diff --git a/contrib/lib_kaitai_of_data_encoding/test/dune b/contrib/lib_kaitai_of_data_encoding/test/dune new file mode 100644 index 0000000000000000000000000000000000000000..fe5cfcafcd337ed170b34a44a96df6d29ce83b90 --- /dev/null +++ b/contrib/lib_kaitai_of_data_encoding/test/dune @@ -0,0 +1,13 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(library + (name kaitai_of_data_encoding_test) + (package kaitai-of-data-encoding) + (libraries + yaml + data-encoding + kaitai + kaitai-of-data-encoding) + (inline_tests (flags -verbose) (modes native)) + (preprocess (pps ppx_expect))) diff --git a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml new file mode 100644 index 0000000000000000000000000000000000000000..9a8e9216966508fb1a80fb9668b6d50f688181aa --- /dev/null +++ b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml @@ -0,0 +1,44 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Marigold, *) +(* Copyright (c) 2023 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(* This test suite is meant to test translation of ground encodings + to [Kaitai.Types.ClassSpec.t] *) + +let%expect_test "test uint8 translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_uint8" + Data_encoding.uint8 + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_uint8 + seq: + - id: uint8 + type: u1 + |}] diff --git a/contrib/lib_kaitai_of_data_encoding/translate.ml b/contrib/lib_kaitai_of_data_encoding/translate.ml new file mode 100644 index 0000000000000000000000000000000000000000..b9898b5a48623981458840712f1f96316435e17e --- /dev/null +++ b/contrib/lib_kaitai_of_data_encoding/translate.ml @@ -0,0 +1,82 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Marigold, *) +(* Copyright (c) 2023 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Kaitai.Types + +(* We need to access the definition of data-encoding's [descr] type. For this + reason we open the private/internal module [Data_encoding__Encoding] (rather + than the public module [Data_encoding.Encoding]. *) +open Data_encoding__Encoding + +let default_meta_spec ~encoding_name = + MetaSpec. + { + path = []; + isOpaque = false; + id = Some encoding_name; + endian = None; + bitEndian = None; + encoding = None; + forceDebug = false; + opaqueTypes = None; + zeroCopySubstream = None; + imports = []; + } + +let default_doc_spec = DocSpec.{summary = None; refs = []} + +let default_class_spec ~encoding_name = + ClassSpec. + { + fileName = None; + path = []; + meta = default_meta_spec ~encoding_name; + doc = default_doc_spec; + toStringExpr = None; + params = []; + seq = []; + types = []; + instances = []; + enums = []; + } + +let u1_attr_spec = + AttrSpec. + { + path = []; + id = "uint8"; + dataType = DataType.(NumericType (Int_type (Int1Type {signed = false}))); + cond = ConditionalSpec.{ifExpr = None; repeat = RepeatSpec.NoRepeat}; + valid = None; + doc = default_doc_spec; + } + +let from_data_encoding : + type a. encoding_name:string -> a Data_encoding.t -> ClassSpec.t = + fun ~encoding_name {encoding; json_encoding = _} -> + match encoding with + | Uint8 -> {(default_class_spec ~encoding_name) with seq = [u1_attr_spec]} + | _ -> failwith "Not implemented" diff --git a/contrib/lib_kaitai_of_data_encoding/translate.mli b/contrib/lib_kaitai_of_data_encoding/translate.mli new file mode 100644 index 0000000000000000000000000000000000000000..56eb68758c8884f9f8cf6cd1fbab5da8b05de12e --- /dev/null +++ b/contrib/lib_kaitai_of_data_encoding/translate.mli @@ -0,0 +1,32 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Marigold, *) +(* Copyright (c) 2023 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** [from_data_encoding ~encoding_name encoding] generates a formal description + of [encoding] as a kaitai [ClassSpec]. + + @param [encoding_name] is added to the "meta" section of the class-spec. *) +val from_data_encoding : + encoding_name:string -> 'a Data_encoding.t -> Kaitai.Types.ClassSpec.t diff --git a/dune-project b/dune-project index 226c2e8f4f8ad8967b3c095c2c3fa5e07871d224..0b8d168b52eccb90c1ac76fd360849996f68222e 100644 --- a/dune-project +++ b/dune-project @@ -4,6 +4,8 @@ (using ctypes 0.1) (package (name bls12-381)) (package (name internal-devtools)) +(package (name kaitai)) +(package (name kaitai-of-data-encoding)) (package (name octez-accuser-Proxford)) (package (name octez-accuser-PtNairob)) (package (name octez-accuser-alpha)) diff --git a/manifest/main.ml b/manifest/main.ml index 518f9290ab82b30377845bf22eeb5e904f36f76b..b5e5215bd22c07d1dd9d628da74b6d1c54a5b4b2 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -7997,19 +7997,40 @@ let octez_scoru_wasm_regressions = ~preprocess:[staged_pps [ppx_import; ppx_deriving_show]] let kaitai = - private_lib ~opam:"" "kaitai" ~path:"contrib/kaitai-ocaml/src" ~deps:[yaml] + public_lib + "kaitai" + ~path:"contrib/kaitai-ocaml/src" + ~deps:[yaml] + ~synopsis:"OCaml library for reading Kaitai spec files" (* We use a private-lib with inline-tests in order to run the tests normally, but without placing all the code for the tests within the main kaitai library. *) let _kaitai_test = private_lib - ~opam:"" "kaitai_test" + ~opam:"kaitai" ~path:"contrib/kaitai-ocaml/test" ~inline_tests:ppx_expect ~deps:[kaitai] +let kaitai_of_data_encoding = + public_lib + "kaitai-of-data-encoding" + ~path:"contrib/lib_kaitai_of_data_encoding" + ~synopsis:"Kaitai spec generator for data-encoding library" + ~deps:[yaml; data_encoding; kaitai] + ~bisect_ppx:No + +let _kaitai_of_data_encoding_test = + private_lib + "kaitai_of_data_encoding_test" + ~opam:"kaitai-of-data-encoding" + ~path:"contrib/lib_kaitai_of_data_encoding/test" + ~deps:[yaml; data_encoding; kaitai; kaitai_of_data_encoding] + ~bisect_ppx:No + ~inline_tests:ppx_expect + (* Add entries to this function to declare that some dune and .opam files are not generated by the manifest on purpose. diff --git a/opam/kaitai-of-data-encoding.opam b/opam/kaitai-of-data-encoding.opam new file mode 100644 index 0000000000000000000000000000000000000000..67e175d24d39615404d5710d1bc1b5cb9a5dc69a --- /dev/null +++ b/opam/kaitai-of-data-encoding.opam @@ -0,0 +1,23 @@ +# This file was automatically generated, do not edit. +# Edit file manifest/main.ml instead. +opam-version: "2.0" +maintainer: "contact@tezos.com" +authors: ["Tezos devteam"] +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +dev-repo: "git+https://gitlab.com/tezos/tezos.git" +license: "MIT" +depends: [ + "dune" { >= "3.0" } + "ocaml" { >= "4.14" } + "yaml" { >= "3.1.0" } + "data-encoding" { >= "0.7.1" & < "1.0.0" } + "kaitai" + "ppx_expect" +] +build: [ + ["rm" "-r" "vendors" "contrib"] + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +synopsis: "Kaitai spec generator for data-encoding library" diff --git a/opam/kaitai.opam b/opam/kaitai.opam new file mode 100644 index 0000000000000000000000000000000000000000..324d6c0fc7f197588adc7326f893dd1464d9a950 --- /dev/null +++ b/opam/kaitai.opam @@ -0,0 +1,21 @@ +# This file was automatically generated, do not edit. +# Edit file manifest/main.ml instead. +opam-version: "2.0" +maintainer: "contact@tezos.com" +authors: ["Tezos devteam"] +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +dev-repo: "git+https://gitlab.com/tezos/tezos.git" +license: "MIT" +depends: [ + "dune" { >= "3.0" } + "ocaml" { >= "4.14" } + "yaml" { >= "3.1.0" } + "ppx_expect" +] +build: [ + ["rm" "-r" "vendors" "contrib"] + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +synopsis: "OCaml library for reading Kaitai spec files"