diff --git a/contrib/kaitai-ocaml/src/print.ml b/contrib/kaitai-ocaml/src/print.ml index b0f2759f26fa57e6b1aed628207744b4729273d8..ecaca247a950957c38edcf6730d12c4ee7552a1a 100644 --- a/contrib/kaitai-ocaml/src/print.ml +++ b/contrib/kaitai-ocaml/src/print.ml @@ -50,61 +50,78 @@ let mapping l = m_members = List.map (fun (k, v) -> (scalar k, v)) l; } +let ( @? ) x xs = match x with None -> xs | Some x -> x :: xs + let metaSpec (t : MetaSpec.t) = mapping - (List.filter_map - (fun x -> x) - [(match t.id with None -> None | Some id -> Some ("id", scalar id))]) + @@ Option.map (fun id -> ("id", scalar id)) t.id + @? Option.map + (fun endian -> ("endian", scalar (Endianness.to_string endian))) + t.endian + @? [] let classSpec _ = mapping [("test", scalar "test")] let instanceSpec _ = mapping [("test", scalar "test")] +let types_spec types = + mapping (types |> List.map (fun (k, v) -> (k, classSpec v))) + +let instances_spec instances = + mapping (instances |> List.map (fun (k, v) -> (k, instanceSpec v))) + let enumSpec enumspec = mapping (List.map (fun (v, EnumValueSpec.{name; _}) -> (string_of_int v, scalar name)) enumspec.EnumSpec.map) -let if_not_empty = function [] -> false | _ -> true +let enums_spec enums = + mapping (enums |> List.map (fun (k, v) -> (k, enumSpec v))) + +(** We only add "type" to Yaml if not [AnyType]. + TODO: This is only correct if [AnyType] means no type? *) +let attr_type_if_not_any attr = + if attr.AttrSpec.dataType = AnyType then None + else Some ("type", scalar (DataType.to_string attr.AttrSpec.dataType)) + +let size_header_mapping = mapping [("id", scalar "size"); ("type", scalar "u4")] + +let attr_spec attr = + match attr.AttrSpec.dataType with + (* [BytesType] attr require size header. *) + | BytesType (BytesLimitType {size; _}) -> + size_header_mapping + :: [ + mapping + (Some ("id", scalar attr.AttrSpec.id) + @? Some ("size", scalar (Ast.to_string size)) + @? []); + ] + | _ -> + [ + mapping + (Some ("id", scalar attr.AttrSpec.id) + @? attr_type_if_not_any attr + @? Option.map (fun enum -> ("enum", scalar enum)) attr.AttrSpec.enum + @? []); + ] + +let seq_spec seq = sequence (List.concat_map attr_spec seq) + +let not_empty = function [] -> false | _ -> true + +let spec_if_non_empty name args f = + if not_empty args then Some (name, f args) else None let to_yaml (t : ClassSpec.t) = mapping - (List.filter_map - (fun (b, n, v) -> if b then Some (n, v) else None) - [ - (true, "meta", metaSpec t.meta); - ( if_not_empty t.types, - "types", - mapping (t.types |> List.map (fun (k, v) -> (k, classSpec v))) ); - ( if_not_empty t.instances, - "instances", - mapping (t.instances |> List.map (fun (k, v) -> (k, instanceSpec v))) - ); - ( if_not_empty t.enums, - "enums", - mapping (t.enums |> List.map (fun (k, v) -> (k, enumSpec v))) ); - ( if_not_empty t.seq, - "seq", - sequence - (t.seq - |> List.map (fun v -> - mapping - (("id", scalar v.AttrSpec.id) - :: - (* We only add "type" to Yaml if not [AnyType]. - TODO: This is only correct if [AnyType] means no type? *) - (if v.AttrSpec.dataType = AnyType then [] - else - [ - ( "type", - scalar (DataType.to_string v.AttrSpec.dataType) ); - ]) - @ - match v.AttrSpec.enum with - | None -> [] - | Some enum -> [("enum", scalar enum)]))) ); - ]) + @@ Some ("meta", metaSpec t.meta) + @? spec_if_non_empty "types" t.types types_spec + @? spec_if_non_empty "instances" t.instances instances_spec + @? spec_if_non_empty "enums" t.enums enums_spec + @? spec_if_non_empty "seq" t.seq seq_spec + @? [] let print t = let y = to_yaml t in diff --git a/contrib/kaitai-ocaml/src/types.ml b/contrib/kaitai-ocaml/src/types.ml index 36f85100330e47544ebdc50118c14dc1bec84015..b3fdb076c6617bfb6c765fc25199da776b9e2d35 100644 --- a/contrib/kaitai-ocaml/src/types.ml +++ b/contrib/kaitai-ocaml/src/types.ml @@ -87,6 +87,8 @@ module Ast = struct | List of t list type expr = t + + let to_string = function Name name -> name | _ -> failwith "not implemented" end type processExpr = @@ -105,6 +107,11 @@ module Endianness = struct type cases = (Ast.expr * fixed_endian) list type t = [fixed_endian | `Calc of Ast.expr * cases | `Inherited] + + let to_string = function + | `BE -> "be" + | `LE -> "le" + | `Calc _ | `Inherited -> failwith "not supported" end module DataType = struct @@ -180,9 +187,23 @@ module DataType = struct type t = data_type + let width_to_int = function W1 -> 1 | W2 -> 2 | W4 -> 4 | W8 -> 8 + let to_string = function - | NumericType (Int_type (Int1Type {signed})) -> - if signed then "s1" else "u1" + | NumericType (Int_type int_type) -> ( + match int_type with + | Int1Type {signed} -> if signed then "s1" else "u1" + | IntMultiType {signed; width; endian} -> + Printf.sprintf + "%s%d%s" + (if signed then "s" else "u") + (width_to_int width) + (endian + |> Option.map Endianness.to_string + |> Option.value ~default:"") + | _ -> failwith "not supported") + | NumericType (Float_type (FloatMultiType {width = _; endian = _})) -> "f8" + | BytesType (BytesLimitType _) -> "fixed size (uint30) bytes" | _ -> failwith "not supported" end diff --git a/contrib/lib_kaitai_of_data_encoding/ground.ml b/contrib/lib_kaitai_of_data_encoding/ground.ml index 7a5a63f9273dbb33c2616d4677a7e03cdb72c449..486c5d2a424098c5a49f6f8f38e9ac49de3e75bc 100644 --- a/contrib/lib_kaitai_of_data_encoding/ground.ml +++ b/contrib/lib_kaitai_of_data_encoding/ground.ml @@ -31,6 +31,18 @@ let default_doc_spec = DocSpec.{summary = None; refs = []} let cond_no_cond = AttrSpec.ConditionalSpec.{ifExpr = None; repeat = RepeatSpec.NoRepeat} +let default_attr_spec = + AttrSpec. + { + path = []; + id = ""; + dataType = DataType.AnyType; + cond = cond_no_cond; + valid = None; + doc = default_doc_spec; + enum = None; + } + module Enum = struct type map = (string * Kaitai.Types.EnumSpec.t) list @@ -63,26 +75,93 @@ end module Attr = struct let bool = - AttrSpec. - { - path = []; - id = "bool"; - dataType = DataType.(NumericType (Int_type (Int1Type {signed = false}))); - cond = cond_no_cond; - valid = Some (ValidationAnyOf [IntNum 0; IntNum 255]); - doc = default_doc_spec; - enum = Some (fst Enum.bool); - } - - let u1 = - AttrSpec. - { - path = []; - id = "uint8"; - dataType = DataType.(NumericType (Int_type (Int1Type {signed = false}))); - cond = cond_no_cond; - valid = None; - doc = default_doc_spec; - enum = None; - } + { + default_attr_spec with + id = "bool"; + dataType = DataType.(NumericType (Int_type (Int1Type {signed = false}))); + valid = Some (ValidationAnyOf [IntNum 0; IntNum 255]); + enum = Some (fst Enum.bool); + } + + let int1_type_attr_spec ~signed = + { + default_attr_spec with + id = (if signed then "int8" else "uint8"); + dataType = DataType.(NumericType (Int_type (Int1Type {signed}))); + } + + let int_multi_type_atrr_spec ~id ~signed width = + { + default_attr_spec with + id; + dataType = + DataType.( + NumericType (Int_type (IntMultiType {signed; width; endian = None}))); + } + + let float_multi_type_attr_spec ~id = + { + default_attr_spec with + id; + dataType = + DataType.( + NumericType + (Float_type + (FloatMultiType + { + (* Data-encoding supports only 64-bit floats. *) + width = DataType.W8; + endian = None; + }))); + } + + let bytes_limit_type_attr_spec ~id = + { + default_attr_spec with + id; + dataType = + DataType.( + BytesType + (BytesLimitType + { + size = Name "size"; + terminator = None; + include_ = false; + padRight = None; + process = None; + })); + } + + let u1 = int1_type_attr_spec ~signed:false + + let s1 = int1_type_attr_spec ~signed:true + + let u2 = int_multi_type_atrr_spec ~id:"uint16" ~signed:false DataType.W2 + + let s2 = int_multi_type_atrr_spec ~id:"int16" ~signed:true DataType.W2 + + let s4 = int_multi_type_atrr_spec ~id:"int32" ~signed:true DataType.W4 + + let s8 = int_multi_type_atrr_spec ~id:"int64" ~signed:true DataType.W8 + + let int31 = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/6261 + There should be a validation that [Int31] is in the appropriate + range. *) + int_multi_type_atrr_spec ~id:"int31" ~signed:true DataType.W4 + + let f8 = float_multi_type_attr_spec ~id:"float" + + let bytes = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/6260 + We fix size header to [`Uint30] for now. This corresponds to + size header of ground bytes encoding. Later on we want to add + support for [`Uint16], [`Uint8] and [`N]. *) + bytes_limit_type_attr_spec ~id:"fixed size (uint30) bytes" + + let string = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/6260 + Same as with [Bytes] above, i.e. we need to add support for [`Uint16], + [`Uint8] and [`N] size header as well. *) + bytes_limit_type_attr_spec ~id:"fixed size (uint30) bytes" end diff --git a/contrib/lib_kaitai_of_data_encoding/ground.mli b/contrib/lib_kaitai_of_data_encoding/ground.mli index eec3a1fc0650dfe41e9c6993332271d57b7f7648..c6e36b16082a7a151932e856c2ffb084e8e482d2 100644 --- a/contrib/lib_kaitai_of_data_encoding/ground.mli +++ b/contrib/lib_kaitai_of_data_encoding/ground.mli @@ -24,30 +24,60 @@ (* *) (*****************************************************************************) +open Kaitai.Types + (** [default_doc_spec] is without summary and references. *) -val default_doc_spec : Kaitai.Types.DocSpec.t +val default_doc_spec : DocSpec.t (** [Enum] module defines enum definitions needed for describing data-encoding ground types. *) module Enum : sig (** [map] describes mapping of enum id (string) with the corresponding [EnumSpec.t]. *) - type map = (string * Kaitai.Types.EnumSpec.t) list + type map = (string * EnumSpec.t) list (** [bool] is a mapping for boolean type. *) - val bool : string * Kaitai.Types.EnumSpec.t + val bool : string * EnumSpec.t (** [add enums enum] returns a list of enum mappings. If [enums] don't contain [enum], then new list with it is returned, otherwise existing [enums] list is returned. *) - val add : map -> string * Kaitai.Types.EnumSpec.t -> map + val add : map -> string * EnumSpec.t -> map end (** [Attr] is module for getting [AttrSpec.t] of ground types. *) module Attr : sig (** [bool] returns [AttrSpec.t] definition of bool ground type. *) - val bool : Kaitai.Types.AttrSpec.t + val bool : AttrSpec.t (** [u1] returns [AttrSpec.t] definition of 8-bit unsigned integer. *) - val u1 : Kaitai.Types.AttrSpec.t + val u1 : AttrSpec.t + + (** [s1] returns [AttrSpec.t] definition of 8-bit signed integer. *) + val s1 : AttrSpec.t + + (** [u2] returns [AttrSpec.t] definition of 16-bit unsigned integer. *) + val u2 : AttrSpec.t + + (** [s2] returns [AttrSpec.t] definition of 16-bit signed integer. *) + val s2 : AttrSpec.t + + (** [s4] returns [AttrSpec.t] definition of 32-bit signed integer. *) + val s4 : AttrSpec.t + + (** [s8] returns [AttrSpec.t] definition of 64-bit signed integer. *) + val s8 : AttrSpec.t + + (** [int31] returns [AttrSpec.t] definition of 31-bit signed integer. + For more about this type see [Data_encoding.int31]. *) + val int31 : AttrSpec.t + + (** [f8] returns [AttrSpec.t] definition of 64-bit float. *) + val f8 : AttrSpec.t + + (** [bytes] returns [AttrSpec.t] definition of [Data_encoding.bytes]. *) + val bytes : AttrSpec.t + + (** [string] returns [AttrSpec.t] definition of [Data_encoding.string]. *) + val string : AttrSpec.t end diff --git a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml index 3bb40a1110f50bf09563a31deb2455f63a01def3..da6a626ad4a1f99a84121089d9bc5ead91c6a6b9 100644 --- a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml +++ b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml @@ -38,11 +38,131 @@ let%expect_test "test uint8 translation" = {| meta: id: ground_uint8 + endian: be seq: - id: uint8 type: u1 |}] +let%expect_test "test int8 translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_int8" + Data_encoding.int8 + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_int8 + endian: be + seq: + - id: int8 + type: s1 + |}] + +let%expect_test "test uint16 translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_uint16" + Data_encoding.uint16 + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_uint16 + endian: be + seq: + - id: uint16 + type: u2 + |}] + +let%expect_test "test int16 translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_int16" + Data_encoding.int16 + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_int16 + endian: be + seq: + - id: int16 + type: s2 + |}] + +let%expect_test "test int32 translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_int32" + Data_encoding.int32 + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_int32 + endian: be + seq: + - id: int32 + type: s4 + |}] + +let%expect_test "test int64 translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_int64" + Data_encoding.int64 + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_int64 + endian: be + seq: + - id: int64 + type: s8 + |}] + +let%expect_test "test int31 translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_int31" + Data_encoding.int31 + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_int31 + endian: be + seq: + - id: int31 + type: s4 + |}] + +let%expect_test "test float translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_float" + Data_encoding.float + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_float + endian: be + seq: + - id: float + type: f8 + |}] + let%expect_test "test bool translation" = let s = Kaitai_of_data_encoding.Translate.from_data_encoding @@ -54,6 +174,7 @@ let%expect_test "test bool translation" = {| meta: id: ground_bool + endian: be enums: bool: 0: false @@ -63,3 +184,45 @@ let%expect_test "test bool translation" = type: u1 enum: bool |}] + +let%expect_test "test fixed size bytes translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_bytes" + Data_encoding.bytes + in + print_endline (Kaitai.Print.print s) ; + (* TODO: https://gitlab.com/tezos/tezos/-/issues/6258 + Consider adding support for user defined types, + and using this feature to write a more kaitai + idiomatic spec file for [ground_bytes]. *) + [%expect + {| + meta: + id: ground_bytes + endian: be + seq: + - id: size + type: u4 + - id: fixed size (uint30) bytes + size: size + |}] + +let%expect_test "test fixed size string translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_string" + Data_encoding.string + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_string + endian: be + seq: + - id: size + type: u4 + - id: fixed size (uint30) bytes + size: size + |}] diff --git a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml index 44eff71495b7723e2667e8a16b46866e393ce90c..f95ca3b0be975c11d4f98d09f25506c0914cc401 100644 --- a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml +++ b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_tuples.ml @@ -35,6 +35,7 @@ let%expect_test "test tuple translation" = {| meta: id: simple_tuple + endian: be enums: bool: 0: false @@ -58,6 +59,7 @@ let%expect_test "test long tuple translation" = {| meta: id: simple_tuple + endian: be enums: bool: 0: false @@ -87,6 +89,7 @@ let%expect_test "test tup1 tuple translation" = {| meta: id: tup1 + endian: be seq: - id: uint8 type: u1 @@ -104,6 +107,7 @@ let%expect_test "test tuples with tup1 translation" = {| meta: id: tup1tup + endian: be enums: bool: 0: false diff --git a/contrib/lib_kaitai_of_data_encoding/translate.ml b/contrib/lib_kaitai_of_data_encoding/translate.ml index 79f23cfc262256017a04a0ec9495412b4be38fba..d57d839ecbb2d611b7f14a03f0e87a0255a80d1b 100644 --- a/contrib/lib_kaitai_of_data_encoding/translate.ml +++ b/contrib/lib_kaitai_of_data_encoding/translate.ml @@ -37,7 +37,7 @@ let default_meta_spec ~encoding_name = path = []; isOpaque = false; id = Some encoding_name; - endian = None; + endian = Some `BE; bitEndian = None; encoding = None; forceDebug = false; @@ -95,14 +95,21 @@ let rec seq_field_of_data_encoding : let rec from_data_encoding : type a. encoding_name:string -> a Data_encoding.t -> ClassSpec.t = fun ~encoding_name {encoding; json_encoding = _} -> + let class_spec_of_ground ?(enums = []) ground = + {(default_class_spec ~encoding_name) with seq = [ground]; enums} + in match encoding with - | Bool -> - { - (default_class_spec ~encoding_name) with - seq = [Ground.Attr.bool]; - enums = [Ground.Enum.bool]; - } - | Uint8 -> {(default_class_spec ~encoding_name) with seq = [Ground.Attr.u1]} + | Bool -> class_spec_of_ground ~enums:[Ground.Enum.bool] Ground.Attr.bool + | Uint8 -> class_spec_of_ground Ground.Attr.u1 + | Int8 -> class_spec_of_ground Ground.Attr.s1 + | Uint16 -> class_spec_of_ground Ground.Attr.u2 + | Int16 -> class_spec_of_ground Ground.Attr.s2 + | Int32 -> class_spec_of_ground Ground.Attr.s4 + | Int64 -> class_spec_of_ground Ground.Attr.s8 + | Int31 -> class_spec_of_ground Ground.Attr.int31 + | Float -> class_spec_of_ground Ground.Attr.f8 + | Bytes (_kind_length, _) -> class_spec_of_ground Ground.Attr.bytes + | String (_kind_length, _) -> class_spec_of_ground Ground.Attr.string | Tup e -> (* Naked Tup likely due to [tup1]. We simply ignore this constructor. *) from_data_encoding ~encoding_name e @@ -117,4 +124,7 @@ let rec from_data_encoding : in {(default_class_spec ~encoding_name) with seq; enums} | Conv {encoding; _} -> from_data_encoding ~encoding_name encoding + | Describe {encoding; _} -> from_data_encoding ~encoding_name encoding + | Dynamic_size {kind = _; encoding} -> + from_data_encoding ~encoding_name encoding | _ -> failwith "Not implemented"