From 653d5976df2ba21027d6ae26b5ea6990caf114fa Mon Sep 17 00:00:00 2001
From: Martin Tomazic <martin.tomazic97@gmail.com>
Date: Wed, 30 Aug 2023 18:41:25 +0200
Subject: [PATCH 1/4] Kaitai: Improve code quality

---
 contrib/kaitai-ocaml/src/print.ml             | 77 +++++++++----------
 contrib/lib_kaitai_of_data_encoding/ground.ml | 35 +++++----
 .../lib_kaitai_of_data_encoding/ground.mli    | 14 ++--
 .../lib_kaitai_of_data_encoding/translate.ml  | 12 ++-
 4 files changed, 70 insertions(+), 68 deletions(-)

diff --git a/contrib/kaitai-ocaml/src/print.ml b/contrib/kaitai-ocaml/src/print.ml
index b0f2759f26fa..74750f71b946 100644
--- a/contrib/kaitai-ocaml/src/print.ml
+++ b/contrib/kaitai-ocaml/src/print.ml
@@ -50,61 +50,58 @@ let mapping l =
       m_members = List.map (fun (k, v) -> (scalar k, v)) l;
     }
 
+let ( @? ) x xs = match x with None -> xs | Some x -> x :: xs
+
 let metaSpec (t : MetaSpec.t) =
-  mapping
-    (List.filter_map
-       (fun x -> x)
-       [(match t.id with None -> None | Some id -> Some ("id", scalar id))])
+  mapping @@ Option.map (fun id -> ("id", scalar id)) t.id @? []
 
 let classSpec _ = mapping [("test", scalar "test")]
 
 let instanceSpec _ = mapping [("test", scalar "test")]
 
+let types_spec types =
+  mapping (types |> List.map (fun (k, v) -> (k, classSpec v)))
+
+let instances_spec instances =
+  mapping (instances |> List.map (fun (k, v) -> (k, instanceSpec v)))
+
 let enumSpec enumspec =
   mapping
     (List.map
        (fun (v, EnumValueSpec.{name; _}) -> (string_of_int v, scalar name))
        enumspec.EnumSpec.map)
 
-let if_not_empty = function [] -> false | _ -> true
+let enums_spec enums =
+  mapping (enums |> List.map (fun (k, v) -> (k, enumSpec v)))
+
+(** We only add "type" to Yaml if not [AnyType].
+    TODO: This is only correct if [AnyType] means no type? *)
+let attr_type_if_not_any attr =
+  if attr.AttrSpec.dataType = AnyType then None
+  else Some ("type", scalar (DataType.to_string attr.AttrSpec.dataType))
+
+let attr_spec attr =
+  mapping
+    (Some ("id", scalar attr.AttrSpec.id)
+    @? attr_type_if_not_any attr
+    @? Option.map (fun enum -> ("enum", scalar enum)) attr.AttrSpec.enum
+    @? [])
+
+let seq_spec seq = sequence (List.map attr_spec seq)
+
+let not_empty = function [] -> false | _ -> true
+
+let spec_if_non_empty name args f =
+  if not_empty args then Some (name, f args) else None
 
 let to_yaml (t : ClassSpec.t) =
   mapping
-    (List.filter_map
-       (fun (b, n, v) -> if b then Some (n, v) else None)
-       [
-         (true, "meta", metaSpec t.meta);
-         ( if_not_empty t.types,
-           "types",
-           mapping (t.types |> List.map (fun (k, v) -> (k, classSpec v))) );
-         ( if_not_empty t.instances,
-           "instances",
-           mapping (t.instances |> List.map (fun (k, v) -> (k, instanceSpec v)))
-         );
-         ( if_not_empty t.enums,
-           "enums",
-           mapping (t.enums |> List.map (fun (k, v) -> (k, enumSpec v))) );
-         ( if_not_empty t.seq,
-           "seq",
-           sequence
-             (t.seq
-             |> List.map (fun v ->
-                    mapping
-                      (("id", scalar v.AttrSpec.id)
-                       ::
-                       (* We only add "type" to Yaml if not [AnyType].
-                          TODO: This is only correct if [AnyType] means no type? *)
-                       (if v.AttrSpec.dataType = AnyType then []
-                       else
-                         [
-                           ( "type",
-                             scalar (DataType.to_string v.AttrSpec.dataType) );
-                         ])
-                      @
-                      match v.AttrSpec.enum with
-                      | None -> []
-                      | Some enum -> [("enum", scalar enum)]))) );
-       ])
+  @@ Some ("meta", metaSpec t.meta)
+  @? spec_if_non_empty "types" t.types types_spec
+  @? spec_if_non_empty "instances" t.instances instances_spec
+  @? spec_if_non_empty "enums" t.enums enums_spec
+  @? spec_if_non_empty "seq" t.seq seq_spec
+  @? []
 
 let print t =
   let y = to_yaml t in
diff --git a/contrib/lib_kaitai_of_data_encoding/ground.ml b/contrib/lib_kaitai_of_data_encoding/ground.ml
index 7a5a63f9273d..15f3d9af04cd 100644
--- a/contrib/lib_kaitai_of_data_encoding/ground.ml
+++ b/contrib/lib_kaitai_of_data_encoding/ground.ml
@@ -31,6 +31,18 @@ let default_doc_spec = DocSpec.{summary = None; refs = []}
 let cond_no_cond =
   AttrSpec.ConditionalSpec.{ifExpr = None; repeat = RepeatSpec.NoRepeat}
 
+let default_attr_spec =
+  AttrSpec.
+    {
+      path = [];
+      id = "";
+      dataType = DataType.AnyType;
+      cond = cond_no_cond;
+      valid = None;
+      doc = default_doc_spec;
+      enum = None;
+    }
+
 module Enum = struct
   type map = (string * Kaitai.Types.EnumSpec.t) list
 
@@ -63,26 +75,19 @@ end
 
 module Attr = struct
   let bool =
-    AttrSpec.
-      {
-        path = [];
-        id = "bool";
-        dataType = DataType.(NumericType (Int_type (Int1Type {signed = false})));
-        cond = cond_no_cond;
-        valid = Some (ValidationAnyOf [IntNum 0; IntNum 255]);
-        doc = default_doc_spec;
-        enum = Some (fst Enum.bool);
-      }
+    {
+      default_attr_spec with
+      id = "bool";
+      dataType = DataType.(NumericType (Int_type (Int1Type {signed = false})));
+      valid = Some (ValidationAnyOf [IntNum 0; IntNum 255]);
+      enum = Some (fst Enum.bool);
+    }
 
   let u1 =
     AttrSpec.
       {
-        path = [];
+        default_attr_spec with
         id = "uint8";
         dataType = DataType.(NumericType (Int_type (Int1Type {signed = false})));
-        cond = cond_no_cond;
-        valid = None;
-        doc = default_doc_spec;
-        enum = None;
       }
 end
diff --git a/contrib/lib_kaitai_of_data_encoding/ground.mli b/contrib/lib_kaitai_of_data_encoding/ground.mli
index eec3a1fc0650..b6b4203c48cc 100644
--- a/contrib/lib_kaitai_of_data_encoding/ground.mli
+++ b/contrib/lib_kaitai_of_data_encoding/ground.mli
@@ -24,30 +24,32 @@
 (*                                                                           *)
 (*****************************************************************************)
 
+open Kaitai.Types
+
 (** [default_doc_spec] is without summary and references.  *)
-val default_doc_spec : Kaitai.Types.DocSpec.t
+val default_doc_spec : DocSpec.t
 
 (** [Enum] module defines enum definitions needed for describing data-encoding
     ground types. *)
 module Enum : sig
   (** [map] describes mapping of enum id (string) with the corresponding
       [EnumSpec.t]. *)
-  type map = (string * Kaitai.Types.EnumSpec.t) list
+  type map = (string * EnumSpec.t) list
 
   (** [bool] is a mapping for boolean type. *)
-  val bool : string * Kaitai.Types.EnumSpec.t
+  val bool : string * EnumSpec.t
 
   (** [add enums enum] returns a list of enum mappings. If [enums] don't contain
       [enum], then new list with it is returned, otherwise existing [enums] list
       is returned. *)
-  val add : map -> string * Kaitai.Types.EnumSpec.t -> map
+  val add : map -> string * EnumSpec.t -> map
 end
 
 (** [Attr] is module for getting [AttrSpec.t] of ground types. *)
 module Attr : sig
   (** [bool] returns [AttrSpec.t] definition of bool ground type. *)
-  val bool : Kaitai.Types.AttrSpec.t
+  val bool : AttrSpec.t
 
   (** [u1] returns [AttrSpec.t] definition of 8-bit unsigned integer. *)
-  val u1 : Kaitai.Types.AttrSpec.t
+  val u1 : AttrSpec.t
 end
diff --git a/contrib/lib_kaitai_of_data_encoding/translate.ml b/contrib/lib_kaitai_of_data_encoding/translate.ml
index 79f23cfc2622..463b42630baf 100644
--- a/contrib/lib_kaitai_of_data_encoding/translate.ml
+++ b/contrib/lib_kaitai_of_data_encoding/translate.ml
@@ -95,14 +95,12 @@ let rec seq_field_of_data_encoding :
 let rec from_data_encoding :
     type a. encoding_name:string -> a Data_encoding.t -> ClassSpec.t =
  fun ~encoding_name {encoding; json_encoding = _} ->
+  let class_spec_of_ground ?(enums = []) ground =
+    {(default_class_spec ~encoding_name) with seq = [ground]; enums}
+  in
   match encoding with
-  | Bool ->
-      {
-        (default_class_spec ~encoding_name) with
-        seq = [Ground.Attr.bool];
-        enums = [Ground.Enum.bool];
-      }
-  | Uint8 -> {(default_class_spec ~encoding_name) with seq = [Ground.Attr.u1]}
+  | Bool -> class_spec_of_ground ~enums:[Ground.Enum.bool] Ground.Attr.bool
+  | Uint8 -> class_spec_of_ground Ground.Attr.u1
   | Tup e ->
       (* Naked Tup likely due to [tup1]. We simply ignore this constructor. *)
       from_data_encoding ~encoding_name e
-- 
GitLab


From f29569bd8d1da33e58afed1b909b9c1d6df2476c Mon Sep 17 00:00:00 2001
From: Martin Tomazic <martin.tomazic97@gmail.com>
Date: Sat, 26 Aug 2023 13:45:09 +0200
Subject: [PATCH 2/4] Kaitai: Add support for e2e translation of ground numeric
 types

---
 contrib/bin_codec/kaitai.t                    |  78 +++++++++++
 contrib/kaitai-ocaml/src/print.ml             |   7 +-
 contrib/kaitai-ocaml/src/types.ml             |  22 +++-
 contrib/lib_kaitai_of_data_encoding/ground.ml |  58 ++++++++-
 .../lib_kaitai_of_data_encoding/ground.mli    |  22 ++++
 .../test/test_translation_of_ground_types.ml  | 121 ++++++++++++++++++
 .../test/test_translation_of_tuples.ml        |   4 +
 .../lib_kaitai_of_data_encoding/translate.ml  |   9 +-
 8 files changed, 310 insertions(+), 11 deletions(-)
 create mode 100644 contrib/bin_codec/kaitai.t

diff --git a/contrib/bin_codec/kaitai.t b/contrib/bin_codec/kaitai.t
new file mode 100644
index 000000000000..624d313b9efd
--- /dev/null
+++ b/contrib/bin_codec/kaitai.t
@@ -0,0 +1,78 @@
+ground.uint8 test
+  $ ./codec.exe dump kaitai for ground.uint8
+  meta:
+    id: ground__uint8
+    endian: be
+  seq:
+  - id: uint8
+    type: u1
+ground.bool test
+  $ ./codec.exe dump kaitai for ground.bool
+  meta:
+    id: ground__bool
+    endian: be
+  enums:
+    bool:
+      0: false
+      255: true
+  seq:
+  - id: bool
+    type: u1
+    enum: bool
+ground.int8 test
+  $ ./codec.exe dump kaitai for ground.int8
+  meta:
+    id: ground__int8
+    endian: be
+  seq:
+  - id: int8
+    type: s1
+ground.uint16 test
+  $ ./codec.exe dump kaitai for ground.uint16
+  meta:
+    id: ground__uint16
+    endian: be
+  seq:
+  - id: uint16
+    type: u2
+ground.int16 test
+  $ ./codec.exe dump kaitai for ground.int16
+  meta:
+    id: ground__int16
+    endian: be
+  seq:
+  - id: int16
+    type: s2
+ground.int32 test
+  $ ./codec.exe dump kaitai for ground.int32
+  meta:
+    id: ground__int32
+    endian: be
+  seq:
+  - id: int32
+    type: s4
+ground.int64 test
+  $ ./codec.exe dump kaitai for ground.int64
+  meta:
+    id: ground__int64
+    endian: be
+  seq:
+  - id: int64
+    type: s8
+ground.int31 test
+  $ ./codec.exe dump kaitai for ground.int31
+  meta:
+    id: ground__int31
+    endian: be
+  seq:
+  - id: int31
+    type: s4
+ground.float test
+  $ ./codec.exe dump kaitai for ground.float
+  meta:
+    id: ground__float
+    endian: be
+  seq:
+  - id: float
+    type: f8
+
diff --git a/contrib/kaitai-ocaml/src/print.ml b/contrib/kaitai-ocaml/src/print.ml
index 74750f71b946..c1dceab2e2a8 100644
--- a/contrib/kaitai-ocaml/src/print.ml
+++ b/contrib/kaitai-ocaml/src/print.ml
@@ -53,7 +53,12 @@ let mapping l =
 let ( @? ) x xs = match x with None -> xs | Some x -> x :: xs
 
 let metaSpec (t : MetaSpec.t) =
-  mapping @@ Option.map (fun id -> ("id", scalar id)) t.id @? []
+  mapping
+  @@ Option.map (fun id -> ("id", scalar id)) t.id
+  @? Option.map
+       (fun endian -> ("endian", scalar (Endianness.to_string endian)))
+       t.endian
+  @? []
 
 let classSpec _ = mapping [("test", scalar "test")]
 
diff --git a/contrib/kaitai-ocaml/src/types.ml b/contrib/kaitai-ocaml/src/types.ml
index 36f85100330e..20510b37a6c5 100644
--- a/contrib/kaitai-ocaml/src/types.ml
+++ b/contrib/kaitai-ocaml/src/types.ml
@@ -105,6 +105,11 @@ module Endianness = struct
   type cases = (Ast.expr * fixed_endian) list
 
   type t = [fixed_endian | `Calc of Ast.expr * cases | `Inherited]
+
+  let to_string = function
+    | `BE -> "be"
+    | `LE -> "le"
+    | `Calc _ | `Inherited -> failwith "not supported"
 end
 
 module DataType = struct
@@ -180,9 +185,22 @@ module DataType = struct
 
   type t = data_type
 
+  let width_to_int = function W1 -> 1 | W2 -> 2 | W4 -> 4 | W8 -> 8
+
   let to_string = function
-    | NumericType (Int_type (Int1Type {signed})) ->
-        if signed then "s1" else "u1"
+    | NumericType (Int_type int_type) -> (
+        match int_type with
+        | Int1Type {signed} -> if signed then "s1" else "u1"
+        | IntMultiType {signed; width; endian} ->
+            Printf.sprintf
+              "%s%d%s"
+              (if signed then "s" else "u")
+              (width_to_int width)
+              (endian
+              |> Option.map Endianness.to_string
+              |> Option.value ~default:"")
+        | _ -> failwith "not supported")
+    | NumericType (Float_type (FloatMultiType {width = _; endian = _})) -> "f8"
     | _ -> failwith "not supported"
 end
 
diff --git a/contrib/lib_kaitai_of_data_encoding/ground.ml b/contrib/lib_kaitai_of_data_encoding/ground.ml
index 15f3d9af04cd..517127f5e2a0 100644
--- a/contrib/lib_kaitai_of_data_encoding/ground.ml
+++ b/contrib/lib_kaitai_of_data_encoding/ground.ml
@@ -83,11 +83,55 @@ module Attr = struct
       enum = Some (fst Enum.bool);
     }
 
-  let u1 =
-    AttrSpec.
-      {
-        default_attr_spec with
-        id = "uint8";
-        dataType = DataType.(NumericType (Int_type (Int1Type {signed = false})));
-      }
+  let int1_type_attr_spec ~signed =
+    {
+      default_attr_spec with
+      id = (if signed then "int8" else "uint8");
+      dataType = DataType.(NumericType (Int_type (Int1Type {signed})));
+    }
+
+  let int_multi_type_atrr_spec ~id ~signed width =
+    {
+      default_attr_spec with
+      id;
+      dataType =
+        DataType.(
+          NumericType (Int_type (IntMultiType {signed; width; endian = None})));
+    }
+
+  let float_multi_type_attr_spec ~id =
+    {
+      default_attr_spec with
+      id;
+      dataType =
+        DataType.(
+          NumericType
+            (Float_type
+               (FloatMultiType
+                  {
+                    (* Data-encoding supports only 64-bit floats. *)
+                    width = DataType.W8;
+                    endian = None;
+                  })));
+    }
+
+  let u1 = int1_type_attr_spec ~signed:false
+
+  let s1 = int1_type_attr_spec ~signed:true
+
+  let u2 = int_multi_type_atrr_spec ~id:"uint16" ~signed:false DataType.W2
+
+  let s2 = int_multi_type_atrr_spec ~id:"int16" ~signed:true DataType.W2
+
+  let s4 = int_multi_type_atrr_spec ~id:"int32" ~signed:true DataType.W4
+
+  let s8 = int_multi_type_atrr_spec ~id:"int64" ~signed:true DataType.W8
+
+  let int31 =
+    (* TODO: https://gitlab.com/tezos/tezos/-/issues/6261
+             There should be a validation that [Int31] is in the appropriate
+             range. *)
+    int_multi_type_atrr_spec ~id:"int31" ~signed:true DataType.W4
+
+  let f8 = float_multi_type_attr_spec ~id:"float"
 end
diff --git a/contrib/lib_kaitai_of_data_encoding/ground.mli b/contrib/lib_kaitai_of_data_encoding/ground.mli
index b6b4203c48cc..1b8671508bfe 100644
--- a/contrib/lib_kaitai_of_data_encoding/ground.mli
+++ b/contrib/lib_kaitai_of_data_encoding/ground.mli
@@ -52,4 +52,26 @@ module Attr : sig
 
   (** [u1] returns [AttrSpec.t] definition of 8-bit unsigned integer. *)
   val u1 : AttrSpec.t
+
+  (** [s1] returns [AttrSpec.t] definition of 8-bit signed integer. *)
+  val s1 : AttrSpec.t
+
+  (** [u2] returns [AttrSpec.t] definition of 16-bit unsigned integer. *)
+  val u2 : AttrSpec.t
+
+  (** [s2] returns [AttrSpec.t] definition of 16-bit signed integer. *)
+  val s2 : AttrSpec.t
+
+  (** [s4] returns [AttrSpec.t] definition of 32-bit signed integer. *)
+  val s4 : AttrSpec.t
+
+  (** [s8] returns [AttrSpec.t] definition of 64-bit signed integer. *)
+  val s8 : AttrSpec.t
+
+  (** [int31] returns [AttrSpec.t] definition of 31-bit signed integer.
+      For more about this type see [Data_encoding.int31]. *)
+  val int31 : AttrSpec.t
+
+  (** [f8] returns [AttrSpec.t] definition of 64-bit float. *)
+  val f8 : AttrSpec.t
 end
diff --git a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml
index 3bb40a1110f5..4c26f57317aa 100644
--- a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml
+++ b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml
@@ -38,11 +38,131 @@ let%expect_test "test uint8 translation" =
     {|
     meta:
       id: ground_uint8
+      endian: be
     seq:
     - id: uint8
       type: u1
   |}]
 
+let%expect_test "test int8 translation" =
+  let s =
+    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+      ~encoding_name:"ground_int8"
+      Data_encoding.int8
+  in
+  print_endline (Kaitai_ast.Print.print s) ;
+  [%expect
+    {|
+    meta:
+      id: ground_int8
+      endian: be
+    seq:
+    - id: int8
+      type: s1
+  |}]
+
+let%expect_test "test uint16 translation" =
+  let s =
+    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+      ~encoding_name:"ground_uint16"
+      Data_encoding.uint16
+  in
+  print_endline (Kaitai_ast.Print.print s) ;
+  [%expect
+    {|
+    meta:
+      id: ground_uint16
+      endian: be
+    seq:
+    - id: uint16
+      type: u2
+  |}]
+
+let%expect_test "test int16 translation" =
+  let s =
+    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+      ~encoding_name:"ground_int16"
+      Data_encoding.int16
+  in
+  print_endline (Kaitai_ast.Print.print s) ;
+  [%expect
+    {|
+    meta:
+      id: ground_int16
+      endian: be
+    seq:
+    - id: int16
+      type: s2
+  |}]
+
+let%expect_test "test int32 translation" =
+  let s =
+    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+      ~encoding_name:"ground_int32"
+      Data_encoding.int32
+  in
+  print_endline (Kaitai_ast.Print.print s) ;
+  [%expect
+    {|
+    meta:
+      id: ground_int32
+      endian: be
+    seq:
+    - id: int32
+      type: s4
+  |}]
+
+let%expect_test "test int64 translation" =
+  let s =
+    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+      ~encoding_name:"ground_int64"
+      Data_encoding.int64
+  in
+  print_endline (Kaitai_ast.Print.print s) ;
+  [%expect
+    {|
+    meta:
+      id: ground_int64
+      endian: be
+    seq:
+    - id: int64
+      type: s8
+  |}]
+
+let%expect_test "test int31 translation" =
+  let s =
+    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+      ~encoding_name:"ground_int31"
+      Data_encoding.int31
+  in
+  print_endline (Kaitai_ast.Print.print s) ;
+  [%expect
+    {|
+    meta:
+      id: ground_int31
+      endian: be
+    seq:
+    - id: int31
+      type: s4
+  |}]
+
+let%expect_test "test float translation" =
+  let s =
+    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+      ~encoding_name:"ground_float"
+      Data_encoding.float
+  in
+  print_endline (Kaitai_ast.Print.print s) ;
+  [%expect
+    {|
+    meta:
+      id: ground_float
+      endian: be
+    seq:
+    - id: float
+      type: f8
+  |}]
+
 let%expect_test "test bool translation" =
   let s =
     Kaitai_of_data_encoding.Translate.from_data_encoding
@@ -54,6 +174,7 @@ let%expect_test "test bool translation" =
     {|
     meta:
       id: ground_bool
+      endian: be
     enums:
       bool:
         0: false
diff --git a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml
index 44eff71495b7..f95ca3b0be97 100644
--- a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml
+++ b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml
@@ -35,6 +35,7 @@ let%expect_test "test tuple translation" =
     {|
     meta:
       id: simple_tuple
+      endian: be
     enums:
       bool:
         0: false
@@ -58,6 +59,7 @@ let%expect_test "test long tuple translation" =
     {|
     meta:
       id: simple_tuple
+      endian: be
     enums:
       bool:
         0: false
@@ -87,6 +89,7 @@ let%expect_test "test tup1 tuple translation" =
     {|
     meta:
       id: tup1
+      endian: be
     seq:
     - id: uint8
       type: u1
@@ -104,6 +107,7 @@ let%expect_test "test tuples with tup1 translation" =
     {|
     meta:
       id: tup1tup
+      endian: be
     enums:
       bool:
         0: false
diff --git a/contrib/lib_kaitai_of_data_encoding/translate.ml b/contrib/lib_kaitai_of_data_encoding/translate.ml
index 463b42630baf..debdcdb234fb 100644
--- a/contrib/lib_kaitai_of_data_encoding/translate.ml
+++ b/contrib/lib_kaitai_of_data_encoding/translate.ml
@@ -37,7 +37,7 @@ let default_meta_spec ~encoding_name =
       path = [];
       isOpaque = false;
       id = Some encoding_name;
-      endian = None;
+      endian = Some `BE;
       bitEndian = None;
       encoding = None;
       forceDebug = false;
@@ -101,6 +101,13 @@ let rec from_data_encoding :
   match encoding with
   | Bool -> class_spec_of_ground ~enums:[Ground.Enum.bool] Ground.Attr.bool
   | Uint8 -> class_spec_of_ground Ground.Attr.u1
+  | Int8 -> class_spec_of_ground Ground.Attr.s1
+  | Uint16 -> class_spec_of_ground Ground.Attr.u2
+  | Int16 -> class_spec_of_ground Ground.Attr.s2
+  | Int32 -> class_spec_of_ground Ground.Attr.s4
+  | Int64 -> class_spec_of_ground Ground.Attr.s8
+  | Int31 -> class_spec_of_ground Ground.Attr.int31
+  | Float -> class_spec_of_ground Ground.Attr.f8
   | Tup e ->
       (* Naked Tup likely due to [tup1]. We simply ignore this constructor. *)
       from_data_encoding ~encoding_name e
-- 
GitLab


From 042685e8f640fb1b989f2d55204bb6e730186e37 Mon Sep 17 00:00:00 2001
From: Martin Tomazic <martin.tomazic97@gmail.com>
Date: Sat, 26 Aug 2023 15:41:37 +0200
Subject: [PATCH 3/4] Kaitai_of_data_encoding: e2e `bytes` and `string`
 translation

---
 contrib/bin_codec/kaitai.t                    | 20 ++++++
 contrib/kaitai-ocaml/src/print.ml             | 29 ++++++--
 contrib/kaitai-ocaml/src/types.ml             |  3 +
 contrib/lib_kaitai_of_data_encoding/ground.ml | 30 ++++++++
 .../lib_kaitai_of_data_encoding/ground.mli    |  6 ++
 .../test/test_translation_of_ground_types.ml  | 70 +++++++++++++++----
 .../lib_kaitai_of_data_encoding/translate.ml  |  5 ++
 7 files changed, 142 insertions(+), 21 deletions(-)

diff --git a/contrib/bin_codec/kaitai.t b/contrib/bin_codec/kaitai.t
index 624d313b9efd..721f77d5692c 100644
--- a/contrib/bin_codec/kaitai.t
+++ b/contrib/bin_codec/kaitai.t
@@ -75,4 +75,24 @@ ground.float test
   seq:
   - id: float
     type: f8
+ground.bytes test
+  $ ./codec.exe dump kaitai for ground.bytes
+  meta:
+    id: ground__bytes
+    endian: be
+  seq:
+  - id: size
+    type: u4
+  - id: fixed size (uint30) bytes
+    size: size
 
+ground.string test
+  $ ./codec.exe dump kaitai for ground.string
+  meta:
+    id: ground__string
+    endian: be
+  seq:
+  - id: size
+    type: u4
+  - id: fixed size (uint30) bytes
+    size: size
diff --git a/contrib/kaitai-ocaml/src/print.ml b/contrib/kaitai-ocaml/src/print.ml
index c1dceab2e2a8..ecaca247a950 100644
--- a/contrib/kaitai-ocaml/src/print.ml
+++ b/contrib/kaitai-ocaml/src/print.ml
@@ -85,14 +85,29 @@ let attr_type_if_not_any attr =
   if attr.AttrSpec.dataType = AnyType then None
   else Some ("type", scalar (DataType.to_string attr.AttrSpec.dataType))
 
-let attr_spec attr =
-  mapping
-    (Some ("id", scalar attr.AttrSpec.id)
-    @? attr_type_if_not_any attr
-    @? Option.map (fun enum -> ("enum", scalar enum)) attr.AttrSpec.enum
-    @? [])
+let size_header_mapping = mapping [("id", scalar "size"); ("type", scalar "u4")]
 
-let seq_spec seq = sequence (List.map attr_spec seq)
+let attr_spec attr =
+  match attr.AttrSpec.dataType with
+  (* [BytesType] attr require size header. *)
+  | BytesType (BytesLimitType {size; _}) ->
+      size_header_mapping
+      :: [
+           mapping
+             (Some ("id", scalar attr.AttrSpec.id)
+             @? Some ("size", scalar (Ast.to_string size))
+             @? []);
+         ]
+  | _ ->
+      [
+        mapping
+          (Some ("id", scalar attr.AttrSpec.id)
+          @? attr_type_if_not_any attr
+          @? Option.map (fun enum -> ("enum", scalar enum)) attr.AttrSpec.enum
+          @? []);
+      ]
+
+let seq_spec seq = sequence (List.concat_map attr_spec seq)
 
 let not_empty = function [] -> false | _ -> true
 
diff --git a/contrib/kaitai-ocaml/src/types.ml b/contrib/kaitai-ocaml/src/types.ml
index 20510b37a6c5..b3fdb076c661 100644
--- a/contrib/kaitai-ocaml/src/types.ml
+++ b/contrib/kaitai-ocaml/src/types.ml
@@ -87,6 +87,8 @@ module Ast = struct
     | List of t list
 
   type expr = t
+
+  let to_string = function Name name -> name | _ -> failwith "not implemented"
 end
 
 type processExpr =
@@ -201,6 +203,7 @@ module DataType = struct
               |> Option.value ~default:"")
         | _ -> failwith "not supported")
     | NumericType (Float_type (FloatMultiType {width = _; endian = _})) -> "f8"
+    | BytesType (BytesLimitType _) -> "fixed size (uint30) bytes"
     | _ -> failwith "not supported"
 end
 
diff --git a/contrib/lib_kaitai_of_data_encoding/ground.ml b/contrib/lib_kaitai_of_data_encoding/ground.ml
index 517127f5e2a0..486c5d2a4240 100644
--- a/contrib/lib_kaitai_of_data_encoding/ground.ml
+++ b/contrib/lib_kaitai_of_data_encoding/ground.ml
@@ -115,6 +115,23 @@ module Attr = struct
                   })));
     }
 
+  let bytes_limit_type_attr_spec ~id =
+    {
+      default_attr_spec with
+      id;
+      dataType =
+        DataType.(
+          BytesType
+            (BytesLimitType
+               {
+                 size = Name "size";
+                 terminator = None;
+                 include_ = false;
+                 padRight = None;
+                 process = None;
+               }));
+    }
+
   let u1 = int1_type_attr_spec ~signed:false
 
   let s1 = int1_type_attr_spec ~signed:true
@@ -134,4 +151,17 @@ module Attr = struct
     int_multi_type_atrr_spec ~id:"int31" ~signed:true DataType.W4
 
   let f8 = float_multi_type_attr_spec ~id:"float"
+
+  let bytes =
+    (* TODO:  https://gitlab.com/tezos/tezos/-/issues/6260
+              We fix size header to [`Uint30] for now. This corresponds to
+              size header of ground bytes encoding. Later on we want to add
+              support for [`Uint16], [`Uint8] and [`N]. *)
+    bytes_limit_type_attr_spec ~id:"fixed size (uint30) bytes"
+
+  let string =
+    (* TODO:  https://gitlab.com/tezos/tezos/-/issues/6260
+              Same as with [Bytes] above, i.e. we need to add support for [`Uint16],
+              [`Uint8] and [`N] size header as well. *)
+    bytes_limit_type_attr_spec ~id:"fixed size (uint30) bytes"
 end
diff --git a/contrib/lib_kaitai_of_data_encoding/ground.mli b/contrib/lib_kaitai_of_data_encoding/ground.mli
index 1b8671508bfe..c6e36b16082a 100644
--- a/contrib/lib_kaitai_of_data_encoding/ground.mli
+++ b/contrib/lib_kaitai_of_data_encoding/ground.mli
@@ -74,4 +74,10 @@ module Attr : sig
 
   (** [f8] returns [AttrSpec.t] definition of 64-bit float. *)
   val f8 : AttrSpec.t
+
+  (** [bytes] returns [AttrSpec.t] definition of [Data_encoding.bytes]. *)
+  val bytes : AttrSpec.t
+
+  (** [string] returns [AttrSpec.t] definition of [Data_encoding.string]. *)
+  val string : AttrSpec.t
 end
diff --git a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml
index 4c26f57317aa..da6a626ad4a1 100644
--- a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml
+++ b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml
@@ -46,11 +46,11 @@ let%expect_test "test uint8 translation" =
 
 let%expect_test "test int8 translation" =
   let s =
-    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+    Kaitai_of_data_encoding.Translate.from_data_encoding
       ~encoding_name:"ground_int8"
       Data_encoding.int8
   in
-  print_endline (Kaitai_ast.Print.print s) ;
+  print_endline (Kaitai.Print.print s) ;
   [%expect
     {|
     meta:
@@ -63,11 +63,11 @@ let%expect_test "test int8 translation" =
 
 let%expect_test "test uint16 translation" =
   let s =
-    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+    Kaitai_of_data_encoding.Translate.from_data_encoding
       ~encoding_name:"ground_uint16"
       Data_encoding.uint16
   in
-  print_endline (Kaitai_ast.Print.print s) ;
+  print_endline (Kaitai.Print.print s) ;
   [%expect
     {|
     meta:
@@ -80,11 +80,11 @@ let%expect_test "test uint16 translation" =
 
 let%expect_test "test int16 translation" =
   let s =
-    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+    Kaitai_of_data_encoding.Translate.from_data_encoding
       ~encoding_name:"ground_int16"
       Data_encoding.int16
   in
-  print_endline (Kaitai_ast.Print.print s) ;
+  print_endline (Kaitai.Print.print s) ;
   [%expect
     {|
     meta:
@@ -97,11 +97,11 @@ let%expect_test "test int16 translation" =
 
 let%expect_test "test int32 translation" =
   let s =
-    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+    Kaitai_of_data_encoding.Translate.from_data_encoding
       ~encoding_name:"ground_int32"
       Data_encoding.int32
   in
-  print_endline (Kaitai_ast.Print.print s) ;
+  print_endline (Kaitai.Print.print s) ;
   [%expect
     {|
     meta:
@@ -114,11 +114,11 @@ let%expect_test "test int32 translation" =
 
 let%expect_test "test int64 translation" =
   let s =
-    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+    Kaitai_of_data_encoding.Translate.from_data_encoding
       ~encoding_name:"ground_int64"
       Data_encoding.int64
   in
-  print_endline (Kaitai_ast.Print.print s) ;
+  print_endline (Kaitai.Print.print s) ;
   [%expect
     {|
     meta:
@@ -131,11 +131,11 @@ let%expect_test "test int64 translation" =
 
 let%expect_test "test int31 translation" =
   let s =
-    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+    Kaitai_of_data_encoding.Translate.from_data_encoding
       ~encoding_name:"ground_int31"
       Data_encoding.int31
   in
-  print_endline (Kaitai_ast.Print.print s) ;
+  print_endline (Kaitai.Print.print s) ;
   [%expect
     {|
     meta:
@@ -148,11 +148,11 @@ let%expect_test "test int31 translation" =
 
 let%expect_test "test float translation" =
   let s =
-    Kaitai_kaitai_of_data_encoding.Translate.from_data_encoding
+    Kaitai_of_data_encoding.Translate.from_data_encoding
       ~encoding_name:"ground_float"
       Data_encoding.float
   in
-  print_endline (Kaitai_ast.Print.print s) ;
+  print_endline (Kaitai.Print.print s) ;
   [%expect
     {|
     meta:
@@ -184,3 +184,45 @@ let%expect_test "test bool translation" =
       type: u1
       enum: bool
   |}]
+
+let%expect_test "test fixed size bytes translation" =
+  let s =
+    Kaitai_of_data_encoding.Translate.from_data_encoding
+      ~encoding_name:"ground_bytes"
+      Data_encoding.bytes
+  in
+  print_endline (Kaitai.Print.print s) ;
+  (* TODO: https://gitlab.com/tezos/tezos/-/issues/6258
+           Consider adding support for user defined types,
+           and using this feature to write a more kaitai
+           idiomatic spec file for [ground_bytes]. *)
+  [%expect
+    {|
+    meta:
+      id: ground_bytes
+      endian: be
+    seq:
+    - id: size
+      type: u4
+    - id: fixed size (uint30) bytes
+      size: size
+  |}]
+
+let%expect_test "test fixed size string translation" =
+  let s =
+    Kaitai_of_data_encoding.Translate.from_data_encoding
+      ~encoding_name:"ground_string"
+      Data_encoding.string
+  in
+  print_endline (Kaitai.Print.print s) ;
+  [%expect
+    {|
+    meta:
+      id: ground_string
+      endian: be
+    seq:
+    - id: size
+      type: u4
+    - id: fixed size (uint30) bytes
+      size: size
+  |}]
diff --git a/contrib/lib_kaitai_of_data_encoding/translate.ml b/contrib/lib_kaitai_of_data_encoding/translate.ml
index debdcdb234fb..d57d839ecbb2 100644
--- a/contrib/lib_kaitai_of_data_encoding/translate.ml
+++ b/contrib/lib_kaitai_of_data_encoding/translate.ml
@@ -108,6 +108,8 @@ let rec from_data_encoding :
   | Int64 -> class_spec_of_ground Ground.Attr.s8
   | Int31 -> class_spec_of_ground Ground.Attr.int31
   | Float -> class_spec_of_ground Ground.Attr.f8
+  | Bytes (_kind_length, _) -> class_spec_of_ground Ground.Attr.bytes
+  | String (_kind_length, _) -> class_spec_of_ground Ground.Attr.string
   | Tup e ->
       (* Naked Tup likely due to [tup1]. We simply ignore this constructor. *)
       from_data_encoding ~encoding_name e
@@ -122,4 +124,7 @@ let rec from_data_encoding :
       in
       {(default_class_spec ~encoding_name) with seq; enums}
   | Conv {encoding; _} -> from_data_encoding ~encoding_name encoding
+  | Describe {encoding; _} -> from_data_encoding ~encoding_name encoding
+  | Dynamic_size {kind = _; encoding} ->
+      from_data_encoding ~encoding_name encoding
   | _ -> failwith "Not implemented"
-- 
GitLab


From c6be7eef2a54753f6038c63b15f2b43ff452b0f3 Mon Sep 17 00:00:00 2001
From: Philippe Wang <philippe.wang@nomadic-labs.com>
Date: Mon, 11 Sep 2023 11:20:29 -0700
Subject: [PATCH 4/4] Kaitai: remove contrib/bin_codec/kaitai.t

---
 contrib/bin_codec/kaitai.t | 98 --------------------------------------
 1 file changed, 98 deletions(-)
 delete mode 100644 contrib/bin_codec/kaitai.t

diff --git a/contrib/bin_codec/kaitai.t b/contrib/bin_codec/kaitai.t
deleted file mode 100644
index 721f77d5692c..000000000000
--- a/contrib/bin_codec/kaitai.t
+++ /dev/null
@@ -1,98 +0,0 @@
-ground.uint8 test
-  $ ./codec.exe dump kaitai for ground.uint8
-  meta:
-    id: ground__uint8
-    endian: be
-  seq:
-  - id: uint8
-    type: u1
-ground.bool test
-  $ ./codec.exe dump kaitai for ground.bool
-  meta:
-    id: ground__bool
-    endian: be
-  enums:
-    bool:
-      0: false
-      255: true
-  seq:
-  - id: bool
-    type: u1
-    enum: bool
-ground.int8 test
-  $ ./codec.exe dump kaitai for ground.int8
-  meta:
-    id: ground__int8
-    endian: be
-  seq:
-  - id: int8
-    type: s1
-ground.uint16 test
-  $ ./codec.exe dump kaitai for ground.uint16
-  meta:
-    id: ground__uint16
-    endian: be
-  seq:
-  - id: uint16
-    type: u2
-ground.int16 test
-  $ ./codec.exe dump kaitai for ground.int16
-  meta:
-    id: ground__int16
-    endian: be
-  seq:
-  - id: int16
-    type: s2
-ground.int32 test
-  $ ./codec.exe dump kaitai for ground.int32
-  meta:
-    id: ground__int32
-    endian: be
-  seq:
-  - id: int32
-    type: s4
-ground.int64 test
-  $ ./codec.exe dump kaitai for ground.int64
-  meta:
-    id: ground__int64
-    endian: be
-  seq:
-  - id: int64
-    type: s8
-ground.int31 test
-  $ ./codec.exe dump kaitai for ground.int31
-  meta:
-    id: ground__int31
-    endian: be
-  seq:
-  - id: int31
-    type: s4
-ground.float test
-  $ ./codec.exe dump kaitai for ground.float
-  meta:
-    id: ground__float
-    endian: be
-  seq:
-  - id: float
-    type: f8
-ground.bytes test
-  $ ./codec.exe dump kaitai for ground.bytes
-  meta:
-    id: ground__bytes
-    endian: be
-  seq:
-  - id: size
-    type: u4
-  - id: fixed size (uint30) bytes
-    size: size
-
-ground.string test
-  $ ./codec.exe dump kaitai for ground.string
-  meta:
-    id: ground__string
-    endian: be
-  seq:
-  - id: size
-    type: u4
-  - id: fixed size (uint30) bytes
-    size: size
-- 
GitLab