diff --git a/contrib/bin_codec_kaitai/kaitai.t b/contrib/bin_codec_kaitai/kaitai.t index 59b61eed6ff30029a2f578ad153b7d6a8c90ccd9..1bfaa2a6bfc43dfcfa3e5ea971b1f2fae7aa2f42 100644 --- a/contrib/bin_codec_kaitai/kaitai.t +++ b/contrib/bin_codec_kaitai/kaitai.t @@ -105,3 +105,46 @@ ground.string test seq: - id: fixed size (uint30) bytes type: fixed_bytes +ground.N test + $ ./codec.exe dump kaitai for ground.N + meta: + id: ground__N + endian: be + types: + group: + instances: + has_next: + value: ((b & 128) != 0) + value: + value: (b & 127) + seq: + - id: b + type: u1 + seq: + - id: groups + type: group + repeat: until + repeat-until: not (_.has_next) +ground.Z test + $ ./codec.exe dump kaitai for ground.Z + meta: + id: ground__Z + endian: be + types: + group: + instances: + has_next: + value: ((b & 128) != 0) + value: + value: (b & 127) + seq: + - id: b + type: u1 + instances: + is_negative: + value: (((groups[0].value) >> 6) == 1) + seq: + - id: groups + type: group + repeat: until + repeat-until: not (_.has_next) diff --git a/contrib/kaitai-ocaml/src/print.ml b/contrib/kaitai-ocaml/src/print.ml index 63cf7483536e02417a8bad664b8175b5a9dfc085..07bc62333d2ead596723c117a77ea38ca13e78f2 100644 --- a/contrib/kaitai-ocaml/src/print.ml +++ b/contrib/kaitai-ocaml/src/print.ml @@ -50,17 +50,24 @@ let mapping l = m_members = List.map (fun (k, v) -> (scalar k, v)) l; } -let ( @? ) x xs = match x with None -> xs | Some x -> x :: xs +let mapping_flatten l = mapping (List.flatten l) -let metaSpec (t : MetaSpec.t) = - mapping - @@ Option.map (fun id -> ("id", scalar id)) t.id - @? Option.map - (fun endian -> ("endian", scalar (Endianness.to_string endian))) - t.endian - @? [] +let map_list_of_option f = function None -> [] | Some x -> [f x] -let instanceSpec _ = mapping [("test", scalar "test")] +let metaSpec (t : MetaSpec.t) = + mapping_flatten + [ + map_list_of_option (fun id -> ("id", scalar id)) t.id; + map_list_of_option + (fun endian -> ("endian", scalar (Endianness.to_string endian))) + t.endian; + ] + +let instanceSpec InstanceSpec.{doc = _; descr} = + match descr with + | ValueInstanceSpec instance -> + mapping [("value", scalar (Ast.to_string instance.value))] + | ParseInstanceSpec -> failwith "not supported" let instances_spec instances = mapping (instances |> List.map (fun (k, v) -> (k, instanceSpec v))) @@ -77,8 +84,23 @@ let enums_spec enums = (** We only add "type" to Yaml if not [AnyType]. TODO: This is only correct if [AnyType] means no type? *) let attr_type_if_not_any attr = - if attr.AttrSpec.dataType = AnyType then None - else Some ("type", scalar (DataType.to_string attr.AttrSpec.dataType)) + match attr.AttrSpec.dataType with + | AnyType -> [] + | NumericType _ | BooleanType | BytesType _ | StrType _ | ComplexDataType _ -> + [("type", scalar (DataType.to_string attr.AttrSpec.dataType))] + +let repeat_spec = + let open RepeatSpec in + function + | NoRepeat -> [] + | RepeatUntil expr -> + [ + ("repeat", scalar "until"); ("repeat-until", scalar (Ast.to_string expr)); + ] + | _ -> failwith "not supported" + +let enum_spec attr = + map_list_of_option (fun enum -> ("enum", scalar enum)) attr.AttrSpec.enum let attr_spec attr = match attr.AttrSpec.dataType with @@ -86,34 +108,36 @@ let attr_spec attr = | BytesType (BytesLimitType {size; _}) -> [ mapping - (Some ("id", scalar attr.AttrSpec.id) - @? Some ("size", scalar (Ast.to_string size)) - @? []); + [ + ("id", scalar attr.AttrSpec.id); + ("size", scalar (Ast.to_string size)); + ]; ] | _ -> [ - mapping - (Some ("id", scalar attr.AttrSpec.id) - @? attr_type_if_not_any attr - @? Option.map (fun enum -> ("enum", scalar enum)) attr.AttrSpec.enum - @? []); + mapping_flatten + [ + [("id", scalar attr.AttrSpec.id)]; + attr_type_if_not_any attr; + repeat_spec attr.cond.repeat; + enum_spec attr; + ]; ] let seq_spec seq = sequence (List.concat_map attr_spec seq) -let not_empty = function [] -> false | _ -> true - let spec_if_non_empty name args f = - if not_empty args then Some (name, f args) else None + match args with [] -> [] | _ :: _ -> [(name, f args)] let rec to_yaml (t : ClassSpec.t) = - mapping - @@ (if t.isTopLevel then Some ("meta", metaSpec t.meta) else None) - @? spec_if_non_empty "types" t.types types_spec - @? spec_if_non_empty "instances" t.instances instances_spec - @? spec_if_non_empty "enums" t.enums enums_spec - @? spec_if_non_empty "seq" t.seq seq_spec - @? [] + mapping_flatten + [ + (if t.isTopLevel then [("meta", metaSpec t.meta)] else []); + spec_if_non_empty "types" t.types types_spec; + spec_if_non_empty "instances" t.instances instances_spec; + spec_if_non_empty "enums" t.enums enums_spec; + spec_if_non_empty "seq" t.seq seq_spec; + ] and types_spec types = mapping (types |> List.map (fun (k, v) -> (k, to_yaml v))) diff --git a/contrib/kaitai-ocaml/src/types.ml b/contrib/kaitai-ocaml/src/types.ml index 4ee1ea08c66978c3df8b76203aaa0bfdb4ed65a6..7d85178b69b7add24b8c6d16297b7e00af8f8838 100644 --- a/contrib/kaitai-ocaml/src/types.ml +++ b/contrib/kaitai-ocaml/src/types.ml @@ -45,10 +45,20 @@ module Ast = struct | BitXor | BitAnd + let operator_to_string = function + | BitAnd -> "&" + | RShift -> ">>" + | _ -> failwith "not implemented" + type unaryop = Invert | Not | Minus type cmpop = Eq | NotEq | Lt | LtE | Gt | GtE + let cmpop_to_string = function + | NotEq -> "!=" + | Eq -> "==" + | _ -> failwith "not implemented" + type t = | Raw of string | BoolOp of {op : boolop; values : t list} @@ -77,7 +87,29 @@ module Ast = struct type expr = t - let to_string = function Name name -> name | _ -> failwith "not implemented" + let rec to_string = function + | IntNum n -> Int.to_string n + | Name name -> name + | UnaryOp {op; operand} -> ( + match op with + | Not -> "not " ^ to_string operand + | _ -> failwith "unary operator not supported") + | BinOp {left; op; right} -> + Format.sprintf + "(%s %s %s)" + (to_string left) + (operator_to_string op) + (to_string right) + | Compare {left; ops; right} -> + Format.sprintf + "(%s %s %s)" + (to_string left) + (cmpop_to_string ops) + (to_string right) + | Attribute {value; attr} -> Format.sprintf "(%s.%s)" (to_string value) attr + | Subscript {value; idx} -> + Format.sprintf "%s[%s]" (to_string value) (to_string idx) + | _ -> failwith "not implemented" end type processExpr = diff --git a/contrib/kaitai-struct-files/007-PsDELPH1__gas__cost.ksy b/contrib/kaitai-struct-files/007-PsDELPH1__gas__cost.ksy new file mode 100644 index 0000000000000000000000000000000000000000..7e2222e3a5c488cba8a0512496c65b5109108f04 --- /dev/null +++ b/contrib/kaitai-struct-files/007-PsDELPH1__gas__cost.ksy @@ -0,0 +1,21 @@ +meta: + id: 007-PsDELPH1__gas__cost + endian: be +types: + group: + instances: + has_next: + value: ((b & 128) != 0) + value: + value: (b & 127) + seq: + - id: b + type: u1 +instances: + is_negative: + value: (((groups[0].value) >> 6) == 1) +seq: +- id: groups + type: group + repeat: until + repeat-until: not (_.has_next) diff --git a/contrib/kaitai-struct-files/008-PtEdo2Zk__gas__cost.ksy b/contrib/kaitai-struct-files/008-PtEdo2Zk__gas__cost.ksy new file mode 100644 index 0000000000000000000000000000000000000000..68b4ba542c007fbda39ff03ecfdbcf535cce9273 --- /dev/null +++ b/contrib/kaitai-struct-files/008-PtEdo2Zk__gas__cost.ksy @@ -0,0 +1,21 @@ +meta: + id: 008-PtEdo2Zk__gas__cost + endian: be +types: + group: + instances: + has_next: + value: ((b & 128) != 0) + value: + value: (b & 127) + seq: + - id: b + type: u1 +instances: + is_negative: + value: (((groups[0].value) >> 6) == 1) +seq: +- id: groups + type: group + repeat: until + repeat-until: not (_.has_next) diff --git a/contrib/kaitai-struct-files/ground__N.ksy b/contrib/kaitai-struct-files/ground__N.ksy new file mode 100644 index 0000000000000000000000000000000000000000..a98ec779c262fe0180257af492b58bf305944a1a --- /dev/null +++ b/contrib/kaitai-struct-files/ground__N.ksy @@ -0,0 +1,18 @@ +meta: + id: ground__N + endian: be +types: + group: + instances: + has_next: + value: ((b & 128) != 0) + value: + value: (b & 127) + seq: + - id: b + type: u1 +seq: +- id: groups + type: group + repeat: until + repeat-until: not (_.has_next) diff --git a/contrib/kaitai-struct-files/ground__Z.ksy b/contrib/kaitai-struct-files/ground__Z.ksy new file mode 100644 index 0000000000000000000000000000000000000000..bb1605a3ab2d0af4f7134031f46f4126e82e2821 --- /dev/null +++ b/contrib/kaitai-struct-files/ground__Z.ksy @@ -0,0 +1,21 @@ +meta: + id: ground__Z + endian: be +types: + group: + instances: + has_next: + value: ((b & 128) != 0) + value: + value: (b & 127) + seq: + - id: b + type: u1 +instances: + is_negative: + value: (((groups[0].value) >> 6) == 1) +seq: +- id: groups + type: group + repeat: until + repeat-until: not (_.has_next) diff --git a/contrib/lib_kaitai_of_data_encoding/ground.ml b/contrib/lib_kaitai_of_data_encoding/ground.ml index 02388eb136092d791c54f74638ee939154188733..b27df117b333ad779f5ace2250f97e9620882bcf 100644 --- a/contrib/lib_kaitai_of_data_encoding/ground.ml +++ b/contrib/lib_kaitai_of_data_encoding/ground.ml @@ -187,4 +187,85 @@ module Class = struct let bytes ~encoding_name = class_spec_of_attr ~encoding_name Attr.bytes let string ~encoding_name = class_spec_of_attr ~encoding_name Attr.string + + let byte_group = + { + (default_class_spec ~encoding_name:"group") with + seq = [{Attr.u1 with id = "b"}]; + instances = + [ + ( "has_next", + default_instance_spec + ~id:"has_next" + Ast.( + Compare + { + left = + BinOp {left = Name "b"; op = BitAnd; right = IntNum 128}; + ops = NotEq; + right = IntNum 0; + }) ); + ( "value", + default_instance_spec + ~id:"value" + (BinOp {left = Name "b"; op = BitAnd; right = IntNum 127}) ); + ]; + isTopLevel = false; + } + + let repeat_until_end_bytes_group_attr = + { + default_attr_spec with + id = "groups"; + dataType = DataType.(ComplexDataType (UserType byte_group)); + cond = + AttrSpec.ConditionalSpec. + { + ifExpr = None; + repeat = + RepeatSpec.RepeatUntil + Ast.( + UnaryOp + { + op = Not; + operand = Attribute {value = Name "_"; attr = "has_next"}; + }); + }; + } + + let n ~encoding_name = + class_spec_of_attr ~encoding_name repeat_until_end_bytes_group_attr + + let z ~encoding_name = + let instances = + [ + ( "is_negative", + default_instance_spec + ~id:"is_negative" + Ast.( + Compare + { + left = + BinOp + { + left = + Attribute + { + value = + Subscript + {value = Name "groups"; idx = IntNum 0}; + attr = "value"; + }; + op = RShift; + right = IntNum 6; + }; + ops = Eq; + right = IntNum 1; + }) ); + ] + in + class_spec_of_attr + ~encoding_name + repeat_until_end_bytes_group_attr + ~instances end diff --git a/contrib/lib_kaitai_of_data_encoding/ground.mli b/contrib/lib_kaitai_of_data_encoding/ground.mli index a2868362c025d8cec9f5f37d4476a0cff61c0a30..c37667850a0ae4ba6ad1d9b93d04b83007fd3891 100644 --- a/contrib/lib_kaitai_of_data_encoding/ground.mli +++ b/contrib/lib_kaitai_of_data_encoding/ground.mli @@ -109,4 +109,21 @@ module Class : sig (** [string] returns [ClassSpec.t] definition of [Data_encoding.string]. *) val string : encoding_name:string -> ClassSpec.t + + (** [byte_group] represents a user defined type for a variable-length sequence + of bytes encoding a Zarith natural number. It is used for describing + encoding such as [Data_encoding.Z] and [Data_encoding.N]. + + As from the [Data_encoding] documentation: "each byte has a running unary + size bit: the most significant bit of each byte indicates whether this is + the last byte in the sequence (0) or whether the sequence continues (1). + Size bits ignored, the data is the binary representation of the number + in little-endian order." *) + val byte_group : ClassSpec.t + + (** [n] returns [ClassSpec.t] for [Data_encoding.N]. *) + val n : encoding_name:string -> ClassSpec.t + + (** [z] returns [ClassSpec.t] for [Data_encoding.Z]. *) + val z : encoding_name:string -> ClassSpec.t end diff --git a/contrib/lib_kaitai_of_data_encoding/helpers.ml b/contrib/lib_kaitai_of_data_encoding/helpers.ml index bc3936f9e123ff7460f2e22b1e6720794fa661db..3e4abcb5ad414e8fd76bfe8915d1bfce635ff935 100644 --- a/contrib/lib_kaitai_of_data_encoding/helpers.ml +++ b/contrib/lib_kaitai_of_data_encoding/helpers.ml @@ -77,6 +77,21 @@ let types_field_from_attr_seq attributes = in List.fold_left add_uniq_assoc [] types -let class_spec_of_attr ~encoding_name ?(enums = []) attr = +let class_spec_of_attr ~encoding_name ?(enums = []) ?(instances = []) attr = let types = types_field_from_attr_seq [attr] in - {(default_class_spec ~encoding_name) with seq = [attr]; enums; types} + { + (default_class_spec ~encoding_name) with + seq = [attr]; + enums; + types; + instances; + } + +let default_instance_spec ~id value = + InstanceSpec. + { + doc = default_doc_spec; + descr = + InstanceSpec.ValueInstanceSpec + {id; path = []; value; ifExpr = None; dataTypeOpt = None}; + } diff --git a/contrib/lib_kaitai_of_data_encoding/helpers.mli b/contrib/lib_kaitai_of_data_encoding/helpers.mli index dbc9a4e5380b9b6fcf4b511036abbe24d9446b11..b05350bdb99227027164603ba4507b28568b24da 100644 --- a/contrib/lib_kaitai_of_data_encoding/helpers.mli +++ b/contrib/lib_kaitai_of_data_encoding/helpers.mli @@ -60,17 +60,23 @@ val default_class_spec : encoding_name:string -> ClassSpec.t as but a different value than [kv]. *) val add_uniq_assoc : (string * 'a) list -> string * 'a -> (string * 'a) list -(** [class_spec_of_attr ~encoding_name ?enums attr] returns a [ClassSpet.t] - for [attr]. +(** [class_spec_of_attr ~encoding_name ?enums ?instances attr] returns a + [ClassSpec.t] for [attr]. In case of [attr] being of [ComplexDataType UserType _] type, then [types] section of returned [ClassSpec.t] is automatically populated with an appropriate type. @param ~encoding_name is added to meta section as [id]. - @param ?enums is added to class specification if present. *) + @param ?enums is added to class specification if present. + @param ?instances is added to class specification if present. *) val class_spec_of_attr : encoding_name:string -> ?enums:(string * EnumSpec.t) list -> + ?instances:(string * InstanceSpec.t) list -> AttrSpec.t -> ClassSpec.t + +(** [default_instance_spec ~id expr] returns a default instance specification for + of a given [id] and [expr]. *) +val default_instance_spec : id:string -> Ast.t -> InstanceSpec.t diff --git a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml index 59348c1146b29ac51c372f0c0566204cc51c4434..bc07f8280463dbccc825a590f9ee75a611beecd8 100644 --- a/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml +++ b/contrib/lib_kaitai_of_data_encoding/test/test_translation_of_ground_types.ml @@ -232,3 +232,64 @@ let%expect_test "test fixed size string translation" = - id: fixed size (uint30) bytes type: fixed_bytes |}] + +let%expect_test "test big numbers translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_n" + Data_encoding.n + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_n + endian: be + types: + group: + instances: + has_next: + value: ((b & 128) != 0) + value: + value: (b & 127) + seq: + - id: b + type: u1 + seq: + - id: groups + type: group + repeat: until + repeat-until: not (_.has_next) + |}] + +let%expect_test "test big numbers translation" = + let s = + Kaitai_of_data_encoding.Translate.from_data_encoding + ~encoding_name:"ground_z" + Data_encoding.z + in + print_endline (Kaitai.Print.print s) ; + [%expect + {| + meta: + id: ground_z + endian: be + types: + group: + instances: + has_next: + value: ((b & 128) != 0) + value: + value: (b & 127) + seq: + - id: b + type: u1 + instances: + is_negative: + value: (((groups[0].value) >> 6) == 1) + seq: + - id: groups + type: group + repeat: until + repeat-until: not (_.has_next) + |}] diff --git a/contrib/lib_kaitai_of_data_encoding/translate.ml b/contrib/lib_kaitai_of_data_encoding/translate.ml index 1d3a798d6b7d52e9a604b48c1f7706ca176f05f5..b3d28993f46af185e5122977a1093639a5dd89c5 100644 --- a/contrib/lib_kaitai_of_data_encoding/translate.ml +++ b/contrib/lib_kaitai_of_data_encoding/translate.ml @@ -77,6 +77,8 @@ let rec from_data_encoding : | Float -> Ground.Class.float ~encoding_name | Bytes (_kind_length, _) -> Ground.Class.bytes ~encoding_name | String (_kind_length, _) -> Ground.Class.string ~encoding_name + | N -> Ground.Class.n ~encoding_name + | Z -> Ground.Class.z ~encoding_name | Tup e -> (* Naked Tup likely due to [tup1]. We simply ignore this constructor. *) from_data_encoding ~encoding_name e