From 7e3dda940f4d342cf8cb2d379eb1e3b6a7d99ab9 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 29 Jan 2025 14:54:21 +0100 Subject: [PATCH 01/31] Manifest: make Tobi depend on opam-file-format --- manifest/externals.ml | 2 ++ manifest/product_tobi.ml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/manifest/externals.ml b/manifest/externals.ml index 6dd6ccc54ba8..66036555fed8 100644 --- a/manifest/externals.ml +++ b/manifest/externals.ml @@ -204,6 +204,8 @@ let ocplib_endian_bigstring = let ocplib_ocamlres = external_lib ~opam:"ocp-ocamlres" "ocplib-ocamlres" V.(at_least "0.4") +let opam_file_format = external_lib "opam-file-format" V.(at_least "2.1.6") + let opentelemetry = external_lib "opentelemetry" V.True let opentelemetry_lwt = external_lib "opentelemetry-lwt" V.True diff --git a/manifest/product_tobi.ml b/manifest/product_tobi.ml index 03ce3eafbc98..22a8c56a30a1 100644 --- a/manifest/product_tobi.ml +++ b/manifest/product_tobi.ml @@ -20,6 +20,6 @@ let _ci_bin_main = ~internal_name:"main" ~synopsis:"CLI for Tobi, which allows to install components" ~path:"tobi/src" - ~deps:[unix; clap] + ~deps:[unix; clap; opam_file_format] ~release_status:Unreleased ~bisect_ppx:No -- GitLab From 8c3ef186aca1e15034bc75569b58380614fb36d6 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 29 Jan 2025 14:54:54 +0100 Subject: [PATCH 02/31] Manifest: generate --- opam/tobi.opam | 1 + opam/virtual/octez-deps.opam | 1 + tobi/src/dune | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/opam/tobi.opam b/opam/tobi.opam index 9e1cd7871654..44c0eb8e713c 100644 --- a/opam/tobi.opam +++ b/opam/tobi.opam @@ -12,6 +12,7 @@ depends: [ "ocaml" { >= "4.14" } "base-unix" "clap" { >= "0.3.0" } + "opam-file-format" { >= "2.1.6" } ] build: [ ["rm" "-r" "vendors" "contrib"] diff --git a/opam/virtual/octez-deps.opam b/opam/virtual/octez-deps.opam index 3ca158e61358..77446c0f4f08 100644 --- a/opam/virtual/octez-deps.opam +++ b/opam/virtual/octez-deps.opam @@ -81,6 +81,7 @@ depends: [ "ocamlgraph" "ocp-ocamlres" { >= "0.4" } "ocplib-endian" + "opam-file-format" { >= "2.1.6" } "opentelemetry" "optint" "ppx_deriving" diff --git a/tobi/src/dune b/tobi/src/dune index 94bda24bcb9f..e1941090e8ee 100644 --- a/tobi/src/dune +++ b/tobi/src/dune @@ -7,7 +7,8 @@ (package tobi) (libraries unix - clap) + clap + opam-file-format) (link_flags (:standard) (:include %{workspace_root}/static-link-flags.sexp))) -- GitLab From f4fd75f93c22d01bf0acc4fbc85e78ab8bca6314 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 29 Jan 2025 14:58:54 +0100 Subject: [PATCH 03/31] Tobi: add Misc.wrap_errors --- tobi/src/misc.ml | 4 ++++ tobi/src/misc.mli | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/tobi/src/misc.ml b/tobi/src/misc.ml index 7b040de02aae..a3ab49d72f66 100644 --- a/tobi/src/misc.ml +++ b/tobi/src/misc.ml @@ -49,6 +49,10 @@ let unit = Ok () let ( let* ) r f = match r with Ok x -> f x | Error _ as e -> e +let wrap_errors context = function + | Ok _ as x -> x + | Error {code; message} -> Error {code; message = context :: message} + let iter_r (type e) iter container f = let exception E of e error in try diff --git a/tobi/src/misc.mli b/tobi/src/misc.mli index 50a259a86308..629f70391064 100644 --- a/tobi/src/misc.mli +++ b/tobi/src/misc.mli @@ -109,6 +109,15 @@ val unit : (unit, 'a) result (** The bind operator of the error monad. *) val ( let* ) : ('a, 'b) result -> ('a -> ('c, 'b) result) -> ('c, 'b) result +(** {3 Transforming Errors} *) + +(** Wrap errors to give them more context. + + For instance, you can write [wrap_errors "failed to read file" @@ ...] + at the beginning of a function to prepend ["failed to read file"] to the [message] + of all errors. *) +val wrap_errors : string -> ('a, 'b error) result -> ('a, 'b error) result + (** {3 Error-Monad Versions of Standard Library Functions} *) (** Those functions are similar to their [Stdlib] counterpart, except that -- GitLab From efcf6eddb847b73850093261a7846fb8d8e828b9 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 29 Jan 2025 14:59:07 +0100 Subject: [PATCH 04/31] Tobi: add Misc.PP --- tobi/src/misc.ml | 50 +++++++++++++++++++++++++++++++++++++++++++++++ tobi/src/misc.mli | 27 +++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/tobi/src/misc.ml b/tobi/src/misc.ml index a3ab49d72f66..e6e5f50f62cf 100644 --- a/tobi/src/misc.ml +++ b/tobi/src/misc.ml @@ -73,3 +73,53 @@ let list_map_r list f = list_map_r ~acc:(new_head :: acc) tail f in list_map_r list f + +module PP = struct + type t = + | Bool of bool + | Char of char + | Int of int + | Float of float + | String of string + | List of t list + | Variant of string * t list + | Tuple of t list + | Record of (string * t) list + + let rec pp context_requires_parentheses fmt value = + let pp_par = pp true in + let pp = pp false in + let fp x = Format.fprintf fmt x in + match value with + | Bool b -> fp "%b" b + | Char c -> fp "%C" c + | Int i -> fp "%d" i + | Float f -> fp "%g" f + | String s -> fp "%S" s + | List [] -> fp "[]" + | List [item] -> fp "[ %a ]" pp item + | List (head :: tail) -> + fp "@[@[[@ %a" pp head ; + List.iter (fp ";@ %a" pp) tail ; + fp "@]@ ]@]" + | Variant (name, []) -> fp "%s" name + | Variant (name, items) -> + if context_requires_parentheses then + fp "@[(%s@ %a)@]" name pp (Tuple items) + else fp "@[%s@ %a@]" name pp (Tuple items) + | Tuple [] -> fp "()" + | Tuple [item] -> pp_par fmt item + | Tuple (head :: tail) -> + fp "@[(%a" pp head ; + List.iter (fp ",@ %a" pp) tail ; + fp ")@]" + | Record [] -> fp "{}" + | Record [(k, v)] -> fp "@[{ %s =@ %a }@]" k pp v + | Record (head :: tail) -> + let pp_item _fmt (k, v) = fp "@[%s =@ %a@]" k pp v in + fp "@[@[{@ %a" pp_item head ; + List.iter (fp ";@ %a" pp_item) tail ; + fp "@]@ }@]" + + let pp = pp false +end diff --git a/tobi/src/misc.mli b/tobi/src/misc.mli index 629f70391064..43c3fd45fa44 100644 --- a/tobi/src/misc.mli +++ b/tobi/src/misc.mli @@ -141,3 +141,30 @@ val list_iter_r : (** Error-monad version of [List.map]. *) val list_map_r : 'b list -> ('b -> ('a, 'c) result) -> ('a list, 'c) result + +(** {2 Pretty-Printing} *) + +module PP : sig + (** Pretty-printing values using OCaml syntax, with indentation. + + This is an alternative to the [Format] module. + With [PP] you only have to write functions to embed values; + you do not have to think about opening boxes at all. *) + + (** Values. *) + type t = + | Bool of bool + | Char of char + | Int of int + | Float of float + | String of string + | List of t list + | Variant of string * t list + | Tuple of t list + | Record of (string * t) list + + (** Pretty-print a value. + + The result is valid OCaml code, which can be convenient when debugging. *) + val pp : Format.formatter -> t -> unit +end -- GitLab From b14e847079cff282994d865ef4b6907908c0b9a0 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 29 Jan 2025 14:59:28 +0100 Subject: [PATCH 05/31] Tobi: add opam file parser --- tobi/src/opam.ml | 271 ++++++++++++++++++++++++++++++++++++++++++++++ tobi/src/opam.mli | 74 +++++++++++++ 2 files changed, 345 insertions(+) create mode 100644 tobi/src/opam.ml create mode 100644 tobi/src/opam.mli diff --git a/tobi/src/opam.ml b/tobi/src/opam.ml new file mode 100644 index 000000000000..e32d6520b901 --- /dev/null +++ b/tobi/src/opam.ml @@ -0,0 +1,271 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs *) +(* *) +(*****************************************************************************) + +open Misc + +(* First, we define pretty-printers for the types of the [opam-file-format] library. + This helps when debugging, in particular when encountering a construct that we + do not support yet. *) + +let pp_opam_relop (op : OpamParserTypes.relop) : PP.t = + match op with + | `Eq -> Variant ("`Eq", []) + | `Neq -> Variant ("`Neq", []) + | `Geq -> Variant ("`Geq", []) + | `Gt -> Variant ("`Gt", []) + | `Leq -> Variant ("`Leq", []) + | `Lt -> Variant ("`Lt", []) + +let pp_opam_logop (op : OpamParserTypes.logop) : PP.t = + match op with `And -> Variant ("`And", []) | `Or -> Variant ("`Or", []) + +let rec pp_opam_value (value : OpamParserTypes.FullPos.value) : PP.t = + match value.pelem with + | Bool b -> Variant ("Bool", [Bool b]) + | Int i -> Variant ("Int", [Int i]) + | String s -> Variant ("String", [String s]) + | Relop (op, a, b) -> + Variant + ("Relop", [pp_opam_relop op.pelem; pp_opam_value a; pp_opam_value b]) + | Prefix_relop (op, a) -> + Variant ("Prefix_relop", [pp_opam_relop op.pelem; pp_opam_value a]) + | Logop (op, a, b) -> + Variant + ("Logop", [pp_opam_logop op.pelem; pp_opam_value a; pp_opam_value b]) + | Pfxop _ -> Variant ("", []) + | Ident s -> Variant ("Ident", [String s]) + | List l -> Variant ("List", [List (List.map pp_opam_value l.pelem)]) + | Group _ -> Variant ("", []) + | Option (v, l) -> + Variant + ("Option", [pp_opam_value v; List (List.map pp_opam_value l.pelem)]) + | Env_binding _ -> Variant ("", []) + +let rec pp_opam_item (item : OpamParserTypes.FullPos.opamfile_item) : PP.t = + match item.pelem with + | Section {section_kind; section_name; section_items} -> + Variant + ( "Section", + [ + Record + [ + ("section_kind", String section_kind.pelem); + ( "section_name", + match section_name with + | None -> Variant ("None", []) + | Some name -> Variant ("Some", [String name.pelem]) ); + ( "section_items", + List (List.map pp_opam_item section_items.pelem) ); + ]; + ] ) + | Variable (name, value) -> + Variant ("Variable", [String name.pelem; pp_opam_value value]) + +(* Then we define our own AST, which is higher-level than the one from opam-file-format. + We also define pretty-printers for those, once again to help when debugging. *) + +type version_term = Version_var | Version_value of string + +let pp_version_term v : PP.t = + match v with + | Version_var -> Variant ("Version_var", []) + | Version_value vv -> Variant ("Version_value", [String vv]) + +type comparison_operator = EQ | NEQ | GEQ | GT | LEQ | LT + +let pp_comparison_operator op : PP.t = + match op with + | EQ -> Variant ("EQ", []) + | NEQ -> Variant ("NEQ", []) + | GEQ -> Variant ("GEQ", []) + | GT -> Variant ("GT", []) + | LEQ -> Variant ("LEQ", []) + | LT -> Variant ("LT", []) + +type dependency_condition = + | True + | With_test + | Comparison of comparison_operator * version_term + | And of dependency_condition * dependency_condition + | Or of dependency_condition * dependency_condition + +let rec pp_dependency_condition dc : PP.t = + match dc with + | True -> Variant ("True", []) + | With_test -> Variant ("With_test", []) + | Comparison (op, v) -> + Variant ("Comparison", [pp_comparison_operator op; pp_version_term v]) + | And (a, b) -> + Variant ("And", [pp_dependency_condition a; pp_dependency_condition b]) + | Or (a, b) -> + Variant ("Or", [pp_dependency_condition a; pp_dependency_condition b]) + +type dependency = {name : string; condition : dependency_condition} + +let pp_dependency {name; condition} : PP.t = + Record + [("name", String name); ("condition", pp_dependency_condition condition)] + +type command_item = Const of string | Var of string + +let pp_command_item item : PP.t = + match item with + | Const s -> Variant ("Const", [String s]) + | Var s -> Variant ("Var", [String s]) + +type build_condition = True | With_test + +let pp_build_condition condition : PP.t = + match condition with + | True -> Variant ("True", []) + | With_test -> Variant ("With_test", []) + +type build_instruction = { + command : command_item; + arguments : command_item list; + condition : build_condition; +} + +let pp_build_instruction {command; arguments; condition} : PP.t = + Record + [ + ("command", pp_command_item command); + ("arguments", List (List.map pp_command_item arguments)); + ("condition", pp_build_condition condition); + ] + +type t = { + depends : dependency list; + depopts : dependency list; + conflicts : dependency list; + build : build_instruction list; +} + +let pp {depends; depopts; conflicts; build} : PP.t = + Record + [ + ("depends", List (List.map pp_dependency depends)); + ("depopts", List (List.map pp_dependency depopts)); + ("conflicts", List (List.map pp_dependency conflicts)); + ("build", List (List.map pp_build_instruction build)); + ] + +(* This module does not implement all the opam file format. + When something is not recognized, we raise [Unsupported]. + + [value] is the value that is not supported, ready to be pretty-printed, + to make it easier to add support for the missing feature. + + We use exceptions instead of the error monad to make it easier + to write the conversion functions. But in the end the exception is converted + to the error monad by the main parsing function [parse_file]. *) +exception Unsupported of {function_name : string; value : PP.t} + +let unsupported function_name value = raise (Unsupported {function_name; value}) + +(* Finally we define conversion functions from the low-level AST of opam-file-format + to our higher-level AST. *) + +let dependency_condition_operator (op : OpamParserTypes.FullPos.relop) = + match op.pelem with + | `Eq -> EQ + | `Neq -> NEQ + | `Geq -> GEQ + | `Gt -> GT + | `Leq -> LEQ + | `Lt -> LT + +let as_version_term (value : OpamParserTypes.FullPos.value) = + match value.pelem with + | Ident "version" -> Version_var + | String version -> Version_value version + | _ -> unsupported "as_version_term" (pp_opam_value value) + +let rec as_dependency_condition (value : OpamParserTypes.FullPos.value) : + dependency_condition = + match value.pelem with + | Ident "with-test" -> With_test + | Prefix_relop (relop, version) -> + Comparison (dependency_condition_operator relop, as_version_term version) + | Logop ({pelem = `And; _}, a, b) -> + And (as_dependency_condition a, as_dependency_condition b) + | Logop ({pelem = `Or; _}, a, b) -> + Or (as_dependency_condition a, as_dependency_condition b) + | _ -> unsupported "as_dependency_condition" (pp_opam_value value) + +let as_dependency (value : OpamParserTypes.FullPos.value) = + match value.pelem with + | String name -> {name; condition = True} + | Option ({pelem = String name; _}, {pelem = [condition]; _}) -> + {name; condition = as_dependency_condition condition} + | _ -> unsupported "as_dependency" (pp_opam_value value) + +let as_build_instruction_item (value : OpamParserTypes.FullPos.value) = + match value.pelem with + | String s -> Const s + | Ident s -> Var s + | _ -> unsupported "as_build_instruction_item" (pp_opam_value value) + +let as_condition + (value : + OpamParserTypes.FullPos.value list OpamParserTypes.FullPos.with_pos) = + match value.pelem with + | [{pelem = Ident "with-test"; _}] -> With_test + | _ -> unsupported "as_condition" (List (List.map pp_opam_value value.pelem)) + +let as_build_instruction (value : OpamParserTypes.FullPos.value) = + match value.pelem with + | List {pelem = head :: tail; _} -> + let command = as_build_instruction_item head in + let arguments = List.map as_build_instruction_item tail in + let condition = True in + {command; arguments; condition} + | Option ({pelem = List {pelem = head :: tail; _}; _}, filter) -> + let command = as_build_instruction_item head in + let arguments = List.map as_build_instruction_item tail in + let condition = as_condition filter in + {command; arguments; condition} + | _ -> unsupported "as_build_instruction" (pp_opam_value value) + +let as_list as_item (value : OpamParserTypes.FullPos.value option) = + match value with + | None -> [] + | Some value -> ( + match value.pelem with + | List items -> List.map as_item items.pelem + | _ -> unsupported "as_list" (pp_opam_value value)) + +let find_variable name (items : OpamParserTypes.FullPos.opamfile_item list) = + Fun.flip List.find_map items @@ fun item -> + match item.pelem with + | Section _ -> unsupported "find_variable" (pp_opam_item item) + | Variable (name', value) -> if name'.pelem = name then Some value else None + +let parse_file filename = + try + if not (Sys.file_exists filename) then fail "file not found: %s" filename + else + (* Parse using the opam-file-format library. This gives a low-level AST. *) + let* items = + try Ok (OpamParser.FullPos.file filename).file_contents with + | OpamLexer.Error msg -> fail "failed to parse %s: %s" filename msg + | Parsing.Parse_error -> fail "failed to parse %s" filename + in + (* Give meaning to the low-level AST. *) + let depends = find_variable "depends" items |> as_list as_dependency in + let depopts = find_variable "depopts" items |> as_list as_dependency in + let conflicts = + find_variable "conflicts" items |> as_list as_dependency + in + let build = find_variable "build" items |> as_list as_build_instruction in + Ok {depends; depopts; conflicts; build} + with Unsupported {function_name; value} -> + fail + "failed to parse %s" + filename + ~reason: + [sf "in Opam.%s: unsupported case:@.%a@?" function_name PP.pp value] diff --git a/tobi/src/opam.mli b/tobi/src/opam.mli new file mode 100644 index 000000000000..41d8a543b2db --- /dev/null +++ b/tobi/src/opam.mli @@ -0,0 +1,74 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(** Opam file format parser. *) + +open Misc + +(** Version terms for dependency conditions. + + [Version_var] denotes the variable [version], typically used as [{= version}]. + + [Version_value] denotes a version constant, such as ["1.0"]. *) +type version_term = Version_var | Version_value of string + +(** Comparison operators. + + - [EQ]: [=] + - [NEQ]: [!=] + - [GEQ]: [>=] + - [GT]: [>] + - [LEQ]: [<=] + - [LT]: [<] *) +type comparison_operator = EQ | NEQ | GEQ | GT | LEQ | LT + +(** Dependency conditions. + + This is the part that is between braces next to dependencies, + such as [{>= "1.0" & with-test}]. *) +type dependency_condition = + | True + | With_test + | Comparison of comparison_operator * version_term + | And of dependency_condition * dependency_condition + | Or of dependency_condition * dependency_condition + +(** Dependency definitions. *) +type dependency = {name : string; condition : dependency_condition} + +(** Items in build instructions. + + [Const] denotes a constant expression, while [Var] denotes a variable + that opam will interpret (such as [jobs] or [name]). *) +type command_item = Const of string | Var of string + +(** Conditions of build instructions. + + This is the part that is between braces next to build instructions, + such as [{with-test}]. *) +type build_condition = True | With_test + +(** A single command of build instructions. *) +type build_instruction = { + command : command_item; + arguments : command_item list; + condition : build_condition; +} + +(** Opam package definitions. *) +type t = { + depends : dependency list; + depopts : dependency list; + conflicts : dependency list; + build : build_instruction list; +} + +(** Pretty-print an opam package definition. *) +val pp : t -> PP.t + +(** Parse a [.opam] file. *) +val parse_file : string -> (t, [> `failed]) r -- GitLab From 5b20afaa91ab9493d94cbd76aa8c9619f824582e Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 29 Jan 2025 14:59:44 +0100 Subject: [PATCH 06/31] Tobi: add Config.find_component_by_name --- tobi/src/config.ml | 7 +++++++ tobi/src/config.mli | 3 +++ 2 files changed, 10 insertions(+) diff --git a/tobi/src/config.ml b/tobi/src/config.ml index a08a30ed9116..138aa5184f59 100644 --- a/tobi/src/config.ml +++ b/tobi/src/config.ml @@ -15,6 +15,13 @@ let pervasive_paths config = config.pervasive_paths let components config = config.components +let find_component_by_name name config = + match + List.find_opt (fun component -> component.name = name) config.components + with + | None -> error `not_found "component not found in configuration" + | Some component -> Ok component + (* Read and parse [filename]. *) let load_file filename = match open_in filename with diff --git a/tobi/src/config.mli b/tobi/src/config.mli index b13359271de7..52b81d54d2b1 100644 --- a/tobi/src/config.mli +++ b/tobi/src/config.mli @@ -70,6 +70,9 @@ val pervasive_paths : t -> string list (** Get the list of components. *) val components : t -> component list +(** Find a component with a given name. *) +val find_component_by_name : string -> t -> (component, [> `not_found]) r + (** Load the configuration for a given version. This fetches file ["tobi/config"] for the given version and parses it. -- GitLab From 03e617f9e7144eb89e95e48c136a21c27152b90f Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 29 Jan 2025 15:00:05 +0100 Subject: [PATCH 07/31] Tobi: add Component module --- tobi/src/component.ml | 109 +++++++++++++++++++++++++++++++++++++++++ tobi/src/component.mli | 33 +++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 tobi/src/component.ml create mode 100644 tobi/src/component.mli diff --git a/tobi/src/component.ml b/tobi/src/component.ml new file mode 100644 index 000000000000..1fac02b3f831 --- /dev/null +++ b/tobi/src/component.ml @@ -0,0 +1,109 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs *) +(* *) +(*****************************************************************************) + +open Misc + +type dependency = + | Internal of {name : string; version : Version.t; with_test : bool} + | External of Opam.dependency + +type t = { + name : string; + version : Version.t; + paths : string * string list; + dependencies : dependency list; + build : Opam.build_instruction list; +} + +(* Convert a tree of [And] nodes into a list. + This allows to extract nodes like [With_test] more easily. *) +let rec flatten_opam_dependency_condition_conjunction + (condition : Opam.dependency_condition) : Opam.dependency_condition list = + match condition with + | And (a, b) -> + flatten_opam_dependency_condition_conjunction a + @ flatten_opam_dependency_condition_conjunction b + | x -> [x] + +(* Convert an [Opam.dependency] into a [dependency]. + Mainly we want to extract version constraints of internal dependencies + to make them easier to work with. *) +let convert_opam_dependency config dev_version (dependency : Opam.dependency) = + match Config.find_component_by_name dependency.name config with + | Error {code = `not_found; _} -> + (* Not an internal component, we don't really care about those for now. *) + Ok (External dependency) + | Ok component -> + (* Extract [with-test] annotations. *) + let conjunction = + flatten_opam_dependency_condition_conjunction dependency.condition + in + let with_test, conjunction = + if List.mem (With_test : Opam.dependency_condition) conjunction then + ( true, + List.filter + (( <> ) (With_test : Opam.dependency_condition)) + conjunction ) + else (false, conjunction) + in + (* The rest must either empty or a single term. *) + let* condition = + match conjunction with + | [] -> Ok (True : Opam.dependency_condition) + | [x] -> Ok x + | _ :: _ :: _ -> + fail + "unsupported version constraint on internal dependency %S" + dependency.name + in + (* The only supported version terms are of the form [= version] or [= "version"]. *) + let* version = + match condition with + | True | Comparison (EQ, Version_var) -> Ok dev_version + | Comparison (EQ, Version_value version) -> Ok (Version.Old version) + | _ -> + fail + "unsupported version constraint on internal dependency %S" + dependency.name + in + Ok (Internal {name = component.name; version; with_test}) + +let load_not_cached (name, version) = + wrap_errors + (sf "failed to load component %S version %S" name (Version.show version)) + @@ + (* Load the configuration so that we know which packages are internal components + and which ones are external, for the requested version. *) + let* config = Config.load version in + (* Check that the requested component actually exists in this configuration. *) + let* component = + match Config.find_component_by_name name config with + | Ok _ as x -> x + | Error {code = `not_found; message} -> Error {code = `failed; message} + in + (* Parse the opam file of the component, for the requested version. *) + let* opam = + match version with + | Dev -> Opam.parse_file component.opam + | Old version -> + Git.with_checkout_into_tmp + ~git_reference:version + ~path:component.opam + (fun checkout -> Opam.parse_file checkout.tmp_path) + in + (* Extract dependencies and their version constraints from the opam file. *) + let* dependencies = + list_map_r opam.depends (convert_opam_dependency config version) + in + Ok {name; version; paths = component.paths; dependencies; build = opam.build} + +(* Memoization makes it easy to load each component only once for a given version. *) +let load_cached = memoize load_not_cached + +(* The [memoize] function is made for functions with one argument, not two. + We just have to uncurry. *) +let load name version = load_cached (name, version) diff --git a/tobi/src/component.mli b/tobi/src/component.mli new file mode 100644 index 000000000000..318bf5c489a3 --- /dev/null +++ b/tobi/src/component.mli @@ -0,0 +1,33 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(** Component definitions. *) + +open Misc + +(** Component dependencies. + + Internal dependencies are dependencies on other components from the same repository. + External dependencies are regular opam packages. *) +type dependency = + | Internal of {name : string; version : Version.t; with_test : bool} + | External of Opam.dependency + +(** Component definitions. + + [paths] are read from Tobi's configuration. + [dependencies] and [build] instructions are read from the opam file. *) +type t = { + name : string; + version : Version.t; + paths : string * string list; + dependencies : dependency list; + build : Opam.build_instruction list; +} + +(** Load the definition of a component for a given version. *) +val load : string -> Version.t -> (t, [> `failed]) r -- GitLab From ab01d079a0cb3d97cd51d8fb93d14767d6352a5a Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 29 Jan 2025 15:00:30 +0100 Subject: [PATCH 08/31] Tobi: list internal component dependencies --- tobi/src/cmd_list.ml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tobi/src/cmd_list.ml b/tobi/src/cmd_list.ml index db6d1d7c59cb..0eccdebcbb4b 100644 --- a/tobi/src/cmd_list.ml +++ b/tobi/src/cmd_list.ml @@ -22,4 +22,14 @@ let run ~verbose version = List.iter (echo " - %s") (path :: other_paths)) ; (* Output opam file path. *) echo "- opam: %s" component.opam ; + (* Output dependencies. *) + let* component = Component.load component.name version in + (Fun.flip List.iter component.dependencies @@ function + | Internal {name; version; with_test} -> + echo + "- depends on component: %s.%s%s" + name + (Version.show version) + (if with_test then " (only for tests)" else "") + | External _ -> ()) ; unit -- GitLab From 3947bece3e96d0e0afd50dfeae7e96ffebc1f02d Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 24 Feb 2025 10:53:11 +0100 Subject: [PATCH 09/31] Opam: update lock file --- opam/virtual/octez-deps.opam.locked | 1 + 1 file changed, 1 insertion(+) diff --git a/opam/virtual/octez-deps.opam.locked b/opam/virtual/octez-deps.opam.locked index cc3bbfb83d8c..6bfa050c0ad8 100644 --- a/opam/virtual/octez-deps.opam.locked +++ b/opam/virtual/octez-deps.opam.locked @@ -174,6 +174,7 @@ depends: [ "ocp-ocamlres" {= "0.4"} "ocplib-endian" {= "1.2"} "ohex" {= "0.2.0"} + "opam-file-format" {= "2.1.6"} "opentelemetry" {= "0.10"} "optint" {= "0.3.0"} "ordering" {= "3.17.2"} -- GitLab From 902e13eb2cae841fa259be6caf9d942716a7bead Mon Sep 17 00:00:00 2001 From: Neelay Sant Date: Sat, 15 Feb 2025 20:06:58 +0000 Subject: [PATCH 10/31] RISC-V: Deduplicate LD OpCode --- .../lib/src/interpreter/common_memory.rs | 13 ++++- src/riscv/lib/src/interpreter/rv64i.rs | 22 ++++++++- .../lib/src/machine_state/instruction.rs | 21 ++++++-- .../machine_state/instruction/constructors.rs | 48 ++++++++++++++++++- .../instruction/tagged_instruction.rs | 4 +- src/riscv/lib/src/parser/instruction.rs | 2 + .../lib/tests/expected/jstz/state_hash_final | 2 +- 7 files changed, 102 insertions(+), 10 deletions(-) diff --git a/src/riscv/lib/src/interpreter/common_memory.rs b/src/riscv/lib/src/interpreter/common_memory.rs index baf49b1d75db..5ea920313d0c 100644 --- a/src/riscv/lib/src/interpreter/common_memory.rs +++ b/src/riscv/lib/src/interpreter/common_memory.rs @@ -7,7 +7,7 @@ use crate::{ machine_state::{ AccessType, MachineCoreState, memory::{self, Memory, OutOfBounds}, - registers::XRegister, + registers::{NonZeroXRegister, XRegister}, }, state_backend as backend, traps::Exception, @@ -40,6 +40,17 @@ where self.read_from_address(address) } + /// Generic read function for loading `mem::size_of` bytes from address val(rs1) + imm + /// where `rs1` is known not be x0. + pub(super) fn read_from_bus_nz( + &mut self, + imm: i64, + rs1: NonZeroXRegister, + ) -> Result { + let address = self.hart.xregisters.read_nz(rs1).wrapping_add(imm as u64); + self.read_from_address(address) + } + /// Generic store-operation for writing `mem::size_of` bytes starting at `address` pub(super) fn write_to_address( &mut self, diff --git a/src/riscv/lib/src/interpreter/rv64i.rs b/src/riscv/lib/src/interpreter/rv64i.rs index 99b3c3528dd9..764d5001c210 100644 --- a/src/riscv/lib/src/interpreter/rv64i.rs +++ b/src/riscv/lib/src/interpreter/rv64i.rs @@ -242,9 +242,27 @@ where MC: memory::MemoryConfig, M: backend::ManagerReadWrite, { - /// `LD` I-type instruction + /// Loads a double-word (8 bytes) starting from address given by: `val(rs1) + imm` + /// where `rs1` and `rd` are NonZeroXRegisters. /// - /// Loads a double-word (8 bytes) starting from address given by: val(rs1) + imm + /// Relevant opcodes: + /// - `LD` + pub fn run_ldnz( + &mut self, + imm: i64, + rs1: NonZeroXRegister, + rd: NonZeroXRegister, + ) -> Result<(), Exception> { + let value: i64 = self.read_from_bus_nz(imm, rs1)?; + // i64 as u64 is a no-op + self.hart.xregisters.write_nz(rd, value as u64); + Ok(()) + } + + /// Loads a double-word (8 bytes) starting from address given by: `imm`. + /// + /// Relevant opcodes: + /// - `LD` pub fn run_ld(&mut self, imm: i64, rs1: XRegister, rd: XRegister) -> Result<(), Exception> { let value: i64 = self.read_from_bus(imm, rs1)?; // i64 as u64 is a no-op diff --git a/src/riscv/lib/src/machine_state/instruction.rs b/src/riscv/lib/src/machine_state/instruction.rs index b3d4ef9c5ee1..be580e67bbd5 100644 --- a/src/riscv/lib/src/machine_state/instruction.rs +++ b/src/riscv/lib/src/machine_state/instruction.rs @@ -370,6 +370,8 @@ pub enum OpCode { JalrAbsolute, /// Same as `Jr` but jumps to `val(rs1) + imm`. JrImm, + /// Same as Ld but only using NonZeroXRegisters. + Ldnz, } impl OpCode { @@ -420,6 +422,7 @@ impl OpCode { Self::Lhu => Args::run_lhu, Self::Lwu => Args::run_lwu, Self::Ld => Args::run_ld, + Self::Ldnz => Args::run_ldnz, Self::Sb => Args::run_sb, Self::Sh => Args::run_sh, Self::Sw => Args::run_sw, @@ -779,6 +782,18 @@ macro_rules! impl_load_type { .map(|_| Next(self.width)) } }; + + ($fn: ident, non_zero) => { + /// SAFETY: This function must only be called on an `Args` belonging + /// to the same OpCode as the OpCode used to derive this function. + unsafe fn $fn( + &self, + core: &mut MachineCoreState, + ) -> Result { + core.$fn(self.imm, self.rs1.nzx, self.rd.nzx) + .map(|_| Next(self.width)) + } + }; } macro_rules! impl_cload_sp_type { ($fn: ident) => { @@ -1162,6 +1177,7 @@ impl Args { impl_load_type!(run_lhu); impl_load_type!(run_lwu); impl_load_type!(run_ld); + impl_load_type!(run_ldnz, non_zero); // RV64I S-type instructions impl_store_type!(run_sb); @@ -1519,10 +1535,7 @@ impl From<&InstrCacheable> for Instruction { opcode: OpCode::Lwu, args: args.to_args(InstrWidth::Uncompressed), }, - InstrCacheable::Ld(args) => Instruction { - opcode: OpCode::Ld, - args: args.to_args(InstrWidth::Uncompressed), - }, + InstrCacheable::Ld(args) => Instruction::from_ic_ld(args), // RV64I S-type instructions InstrCacheable::Sb(args) => Instruction { opcode: OpCode::Sb, diff --git a/src/riscv/lib/src/machine_state/instruction/constructors.rs b/src/riscv/lib/src/machine_state/instruction/constructors.rs index 52a68d6dc0cb..8e0a63b1cc1b 100644 --- a/src/riscv/lib/src/machine_state/instruction/constructors.rs +++ b/src/riscv/lib/src/machine_state/instruction/constructors.rs @@ -5,7 +5,7 @@ use super::{Args, Instruction, OpCode}; use crate::{ default::ConstDefault, - machine_state::registers::{NonZeroXRegister, nz}, + machine_state::registers::{NonZeroXRegister, XRegister, nz}, parser::{ XRegisterParsed, instruction::{ @@ -522,6 +522,39 @@ impl Instruction { }, } } + + /// Create a new [`Instruction`] with the appropriate [`super::ArgsShape`] for [`OpCode::Ld`]. + pub(crate) fn new_ld(rd: XRegister, rs1: XRegister, imm: i64, width: InstrWidth) -> Self { + Self { + opcode: OpCode::Ld, + args: Args { + rd: rd.into(), + rs1: rs1.into(), + imm, + width, + ..Args::DEFAULT + }, + } + } + + /// Create a new [`Instruction`] with the appropriate [`super::ArgsShape`] for [`OpCode::Ldnz`]. + pub(crate) fn new_ldnz( + rd: NonZeroXRegister, + rs1: NonZeroXRegister, + imm: i64, + width: InstrWidth, + ) -> Self { + Self { + opcode: OpCode::Ldnz, + args: Args { + rd: rd.into(), + rs1: rs1.into(), + imm, + width, + ..Args::DEFAULT + }, + } + } } impl Instruction { @@ -974,4 +1007,17 @@ impl Instruction { } } } + + /// Convert [`InstrCacheable::Ld`] according to whether register is non-zero. + /// + /// [`InstrCacheable::Ld`]: crate::parser::instruction::InstrCacheable::Ld + pub(super) fn from_ic_ld(args: &ITypeArgs) -> Instruction { + use XRegisterParsed as X; + match (split_x0(args.rd), split_x0(args.rs1)) { + (X::NonZero(rd), X::NonZero(rs1)) => { + Instruction::new_ldnz(rd, rs1, args.imm, InstrWidth::Uncompressed) + } + _ => Instruction::new_ld(args.rd, args.rs1, args.imm, InstrWidth::Uncompressed), + } + } } diff --git a/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs b/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs index 7e76b87282fd..e92e5619bb7d 100644 --- a/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs +++ b/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs @@ -355,7 +355,9 @@ pub fn opcode_to_argsshape(opcode: &OpCode) -> ArgsShape { Addi | Andi | Ori | Xori | Slli | Srli | Srai | Add | Sub | Mv | Neg | And | Or | Xor | Sll | Srl | Sra | Jal | J | JrImm | JAbsolute | JalrAbsolute | Jr | Jalr | CAddiw - | Li | CLdsp | CLwsp | Nop | Beq | Beqz | Bne | Bnez | JalrImm => ArgsShape::NZXSrcNZXDest, + | Li | CLdsp | CLwsp | Nop | Beq | Beqz | Bne | Bnez | JalrImm | Ldnz => { + ArgsShape::NZXSrcNZXDest + } Addiw | Addw | Subw | Sllw | Srlw | Sraw | Slti | Sltiu | Slliw | Srliw | Sraiw | Slt | Sltu | Auipc => ArgsShape::XSrcNZXDest, diff --git a/src/riscv/lib/src/parser/instruction.rs b/src/riscv/lib/src/parser/instruction.rs index 5ba29ff65dcf..b4c636f02309 100644 --- a/src/riscv/lib/src/parser/instruction.rs +++ b/src/riscv/lib/src/parser/instruction.rs @@ -341,6 +341,8 @@ pub enum InstrCacheable { Lbu(ITypeArgs), Lhu(ITypeArgs), Lwu(ITypeArgs), + /// `LD` - Loads a double-word (8 bytes) starting + /// from address given by: `val(rs1) + imm`. Ld(ITypeArgs), // RV64I S-type instructions diff --git a/src/riscv/lib/tests/expected/jstz/state_hash_final b/src/riscv/lib/tests/expected/jstz/state_hash_final index f231cdf88b74..96c591b6c361 100644 --- a/src/riscv/lib/tests/expected/jstz/state_hash_final +++ b/src/riscv/lib/tests/expected/jstz/state_hash_final @@ -1 +1 @@ -Hash { digest: [144, 216, 233, 51, 249, 204, 185, 104, 32, 152, 233, 232, 64, 237, 200, 214, 93, 113, 48, 143, 235, 84, 217, 214, 38, 165, 193, 80, 11, 176, 143, 166] } +Hash { digest: [48, 237, 65, 151, 92, 36, 254, 123, 30, 201, 191, 24, 187, 57, 17, 9, 255, 117, 36, 116, 26, 107, 122, 254, 237, 166, 0, 202, 217, 82, 109, 76] } -- GitLab From e0000e3b24723b323869bce87764b35f43c6bc05 Mon Sep 17 00:00:00 2001 From: Neelay Sant Date: Mon, 17 Feb 2025 09:46:30 +0000 Subject: [PATCH 11/31] RISC-V: Deduplicate C.LD OpCode --- src/riscv/lib/src/interpreter/rv64c.rs | 12 ----------- src/riscv/lib/src/interpreter/rv64i.rs | 1 + .../lib/src/machine_state/instruction.rs | 11 ++++------ .../instruction/tagged_instruction.rs | 2 +- src/riscv/lib/src/parser.rs | 20 ++++++++++++++++--- src/riscv/lib/src/parser/instruction.rs | 15 +++++++++++++- .../lib/tests/expected/jstz/state_hash_final | 2 +- 7 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/riscv/lib/src/interpreter/rv64c.rs b/src/riscv/lib/src/interpreter/rv64c.rs index 333312d8a1a1..3bc37ba3f36d 100644 --- a/src/riscv/lib/src/interpreter/rv64c.rs +++ b/src/riscv/lib/src/interpreter/rv64c.rs @@ -70,18 +70,6 @@ where MC: memory::MemoryConfig, M: backend::ManagerReadWrite, { - /// `C.LD` CL-type compressed instruction - /// - /// Loads a 64-bit value from memory into register `rd`. It computes - /// an effective address by adding the immediate to the base address - /// in register `rs1`. - /// The immediate is obtained by zero-extending and scaling by 8 the - /// offset encoded in the instruction (see U:C-16.3). - pub fn run_cld(&mut self, imm: i64, rs1: XRegister, rd: XRegister) -> Result<(), Exception> { - debug_assert!(imm >= 0 && imm % 8 == 0); - self.run_ld(imm, rs1, rd) - } - /// `C.LDSP` CI-type compressed instruction /// /// Loads a 64-bit value from memory into register `rd`. It computes diff --git a/src/riscv/lib/src/interpreter/rv64i.rs b/src/riscv/lib/src/interpreter/rv64i.rs index 764d5001c210..67a7af944bdd 100644 --- a/src/riscv/lib/src/interpreter/rv64i.rs +++ b/src/riscv/lib/src/interpreter/rv64i.rs @@ -247,6 +247,7 @@ where /// /// Relevant opcodes: /// - `LD` + /// - `C.LD` pub fn run_ldnz( &mut self, imm: i64, diff --git a/src/riscv/lib/src/machine_state/instruction.rs b/src/riscv/lib/src/machine_state/instruction.rs index be580e67bbd5..05143b6ca8c4 100644 --- a/src/riscv/lib/src/machine_state/instruction.rs +++ b/src/riscv/lib/src/machine_state/instruction.rs @@ -342,7 +342,6 @@ pub enum OpCode { CSubw, // RV64C compressed instructions - CLd, CLdsp, CSd, CSdsp, @@ -556,7 +555,6 @@ impl OpCode { Self::CAddw => Args::run_caddw, Self::CSubw => Args::run_csubw, Self::Nop => Args::run_nop, - Self::CLd => Args::run_cld, Self::CLdsp => Args::run_cldsp, Self::CSd => Args::run_csd, Self::CSdsp => Args::run_csdsp, @@ -1417,7 +1415,6 @@ impl Args { // RV64C compressed instructions impl_store_type!(run_csd); impl_css_type!(run_csdsp); - impl_load_type!(run_cld); impl_cload_sp_type!(run_cldsp); impl_ci_type!(run_caddiw, non_zero); impl_cr_type!(run_caddw); @@ -2068,10 +2065,10 @@ impl From<&InstrCacheable> for Instruction { InstrCacheable::CNop => Instruction::new_nop(InstrWidth::Compressed), // RV64C compressed instructions - InstrCacheable::CLd(args) => Instruction { - opcode: OpCode::CLd, - args: args.to_args(InstrWidth::Compressed), - }, + InstrCacheable::CLd(args) => { + debug_assert!(args.imm >= 0 && args.imm % 8 == 0); + Instruction::new_ldnz(args.rd, args.rs1, args.imm, InstrWidth::Compressed) + } InstrCacheable::CLdsp(args) => Instruction { opcode: OpCode::CLdsp, args: args.into(), diff --git a/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs b/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs index e92e5619bb7d..1b67968d0391 100644 --- a/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs +++ b/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs @@ -338,7 +338,7 @@ pub fn opcode_to_argsshape(opcode: &OpCode) -> ArgsShape { | Amomaxuw | Lrd | Scd | Amoswapd | Amoaddd | Amoxord | Amoandd | Amoord | Amomind | Amomaxd | Amominud | Amomaxud | Rem | Remu | Remw | Remuw | Div | Divu | Divw | Divuw | Mul | Mulh | Mulhsu | Mulhu | Mulw | Csrrw | Csrrs | Csrrc | Csrrwi | Csrrsi | Csrrci - | CLw | CSw | CSwsp | CAddw | CSubw | CLd | CSd | CSdsp | Unknown => ArgsShape::XSrcXDest, + | CLw | CSw | CSwsp | CAddw | CSubw | CSd | CSdsp | Unknown => ArgsShape::XSrcXDest, Fadds | Fsubs | Fmuls | Fdivs | Fsqrts | Fmins | Fmaxs | Fsgnjs | Fsgnjns | Fsgnjxs | Fmadds | Fmsubs | Fnmsubs | Fnmadds | Faddd | Fsubd | Fmuld | Fdivd | Fsqrtd | Fmind diff --git a/src/riscv/lib/src/parser.rs b/src/riscv/lib/src/parser.rs index ad4f058f4a5b..0cad8f6fedb1 100644 --- a/src/riscv/lib/src/parser.rs +++ b/src/riscv/lib/src/parser.rs @@ -1004,11 +1004,25 @@ const fn c_rs1p(instr: u16) -> XRegister { parse_xregister(c_reg_prime(instr, 7)) } +#[inline(always)] +const fn c_rs1p_nz(instr: u16) -> NonZeroXRegister { + // This cannot panic as parsing can only return a register from x8 to x15, + // which are all non-zero. + NonZeroXRegister::assert_from(parse_xregister(c_reg_prime(instr, 7))) +} + #[inline(always)] const fn c_rdp_rs2p(instr: u16) -> XRegister { parse_xregister(c_reg_prime(instr, 2)) } +#[inline(always)] +const fn c_rdp_rs2p_nz(instr: u16) -> NonZeroXRegister { + // This cannot panic as parsing can only return a register from x8 to x15, + // which are all non-zero. + NonZeroXRegister::assert_from(parse_xregister(c_reg_prime(instr, 2))) +} + #[inline(always)] const fn c_f_rdp_rs2p(instr: u16) -> FRegister { parse_fregister(c_reg_prime(instr, 2)) @@ -1214,9 +1228,9 @@ const fn parse_compressed_instruction_inner(instr: u16) -> Instr { rs1: c_rs1p(instr), imm: clw_imm(instr), }), - C_F3_3 => CLd(ITypeArgs { - rd: c_rdp_rs2p(instr), - rs1: c_rs1p(instr), + C_F3_3 => CLd(NonZeroITypeArgs { + rd: c_rdp_rs2p_nz(instr), + rs1: c_rs1p_nz(instr), imm: cld_imm(instr), }), C_F3_5 => CFsd(FStoreArgs { diff --git a/src/riscv/lib/src/parser/instruction.rs b/src/riscv/lib/src/parser/instruction.rs index b4c636f02309..17ef9664e1fc 100644 --- a/src/riscv/lib/src/parser/instruction.rs +++ b/src/riscv/lib/src/parser/instruction.rs @@ -40,6 +40,13 @@ pub struct ITypeArgs { pub imm: i64, } +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, serde::Serialize, serde::Deserialize)] +pub struct NonZeroITypeArgs { + pub rd: NonZeroXRegister, + pub rs1: NonZeroXRegister, + pub imm: i64, +} + /// Intermediate representation of Args for I-type instructions with parsed split of registers. #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, serde::Serialize, serde::Deserialize)] pub struct SplitITypeArgs { @@ -577,7 +584,13 @@ pub enum InstrCacheable { CNop, // RV64C compressed instructions - CLd(ITypeArgs), + /// `C.LD` - Loads a 64-bit value from memory into register `rd`. + /// It computes an effective address by adding the immediate to the base address + /// in register `rs1`. + /// + /// The immediate is obtained by zero-extending and scaling by 8 the + /// offset encoded in the instruction (see U:C-16.3). + CLd(NonZeroITypeArgs), CLdsp(CIBNZTypeArgs), CSd(SBTypeArgs), CSdsp(CSSTypeArgs), diff --git a/src/riscv/lib/tests/expected/jstz/state_hash_final b/src/riscv/lib/tests/expected/jstz/state_hash_final index 96c591b6c361..c5c2870ebf1c 100644 --- a/src/riscv/lib/tests/expected/jstz/state_hash_final +++ b/src/riscv/lib/tests/expected/jstz/state_hash_final @@ -1 +1 @@ -Hash { digest: [48, 237, 65, 151, 92, 36, 254, 123, 30, 201, 191, 24, 187, 57, 17, 9, 255, 117, 36, 116, 26, 107, 122, 254, 237, 166, 0, 202, 217, 82, 109, 76] } +Hash { digest: [65, 103, 123, 191, 90, 185, 66, 122, 221, 159, 228, 88, 159, 151, 244, 51, 242, 210, 9, 86, 129, 173, 45, 5, 97, 52, 232, 224, 22, 61, 36, 214] } -- GitLab From a2ce9a6be31e323d068fab31b863fd603ba709fc Mon Sep 17 00:00:00 2001 From: Neelay Sant Date: Mon, 17 Feb 2025 10:35:36 +0000 Subject: [PATCH 12/31] RISC-V: Deduplicate C.LDSP OpCode --- src/riscv/lib/src/interpreter/rv64c.rs | 16 +--------------- src/riscv/lib/src/interpreter/rv64i.rs | 1 + src/riscv/lib/src/machine_state/instruction.rs | 11 ++++------- .../instruction/tagged_instruction.rs | 4 +--- src/riscv/lib/src/parser/instruction.rs | 5 +++++ .../lib/tests/expected/jstz/state_hash_final | 2 +- 6 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/riscv/lib/src/interpreter/rv64c.rs b/src/riscv/lib/src/interpreter/rv64c.rs index 3bc37ba3f36d..49e4f24e68cd 100644 --- a/src/riscv/lib/src/interpreter/rv64c.rs +++ b/src/riscv/lib/src/interpreter/rv64c.rs @@ -70,20 +70,6 @@ where MC: memory::MemoryConfig, M: backend::ManagerReadWrite, { - /// `C.LDSP` CI-type compressed instruction - /// - /// Loads a 64-bit value from memory into register `rd`. It computes - /// an effective address by adding the immediate to the stack pointer. - /// The immediate is obtained by zero-extending and scaling by 8 the - /// offset encoded in the instruction (see U:C-16.3). - pub fn run_cldsp(&mut self, imm: i64, rd_rs1: NonZeroXRegister) -> Result<(), Exception> { - debug_assert!(imm >= 0 && imm % 8 == 0); - let value: i64 = self.read_from_bus(imm, sp)?; - // i64 as u64 is a no-op - self.hart.xregisters.write_nz(rd_rs1, value as u64); - Ok(()) - } - /// `C.SD` CS-type compressed instruction /// /// Stores a 64-bit value in register `rs2` to memory. It computes @@ -166,7 +152,7 @@ mod tests { state.run_sd(0, t0, a4)?; state.run_sw(8, t0, a3)?; - state.run_cldsp(offset as i64, nz::t4)?; + state.run_ldnz(offset as i64, nz::sp, nz::t4)?; state.run_clwsp((offset + 8) as i64, nz::t3)?; assert_eq!(state.hart.xregisters.read_nz(nz::t4), v_d); assert_eq!(state.hart.xregisters.read_nz(nz::t3), v_w as i32 as u64); diff --git a/src/riscv/lib/src/interpreter/rv64i.rs b/src/riscv/lib/src/interpreter/rv64i.rs index 67a7af944bdd..5daf411324cb 100644 --- a/src/riscv/lib/src/interpreter/rv64i.rs +++ b/src/riscv/lib/src/interpreter/rv64i.rs @@ -248,6 +248,7 @@ where /// Relevant opcodes: /// - `LD` /// - `C.LD` + /// - `C.LDSP` pub fn run_ldnz( &mut self, imm: i64, diff --git a/src/riscv/lib/src/machine_state/instruction.rs b/src/riscv/lib/src/machine_state/instruction.rs index 05143b6ca8c4..07d503bb6d83 100644 --- a/src/riscv/lib/src/machine_state/instruction.rs +++ b/src/riscv/lib/src/machine_state/instruction.rs @@ -342,7 +342,6 @@ pub enum OpCode { CSubw, // RV64C compressed instructions - CLdsp, CSd, CSdsp, CAddiw, @@ -555,7 +554,6 @@ impl OpCode { Self::CAddw => Args::run_caddw, Self::CSubw => Args::run_csubw, Self::Nop => Args::run_nop, - Self::CLdsp => Args::run_cldsp, Self::CSd => Args::run_csd, Self::CSdsp => Args::run_csdsp, Self::CAddiw => Args::run_caddiw, @@ -1415,7 +1413,6 @@ impl Args { // RV64C compressed instructions impl_store_type!(run_csd); impl_css_type!(run_csdsp); - impl_cload_sp_type!(run_cldsp); impl_ci_type!(run_caddiw, non_zero); impl_cr_type!(run_caddw); impl_cr_type!(run_csubw); @@ -2069,10 +2066,10 @@ impl From<&InstrCacheable> for Instruction { debug_assert!(args.imm >= 0 && args.imm % 8 == 0); Instruction::new_ldnz(args.rd, args.rs1, args.imm, InstrWidth::Compressed) } - InstrCacheable::CLdsp(args) => Instruction { - opcode: OpCode::CLdsp, - args: args.into(), - }, + InstrCacheable::CLdsp(args) => { + debug_assert!(args.imm >= 0 && args.imm % 8 == 0); + Instruction::new_ldnz(args.rd_rs1, nz::sp, args.imm, InstrWidth::Compressed) + } InstrCacheable::CSd(args) => Instruction { opcode: OpCode::CSd, args: args.to_args(InstrWidth::Compressed), diff --git a/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs b/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs index 1b67968d0391..14d447bd1417 100644 --- a/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs +++ b/src/riscv/lib/src/machine_state/instruction/tagged_instruction.rs @@ -355,9 +355,7 @@ pub fn opcode_to_argsshape(opcode: &OpCode) -> ArgsShape { Addi | Andi | Ori | Xori | Slli | Srli | Srai | Add | Sub | Mv | Neg | And | Or | Xor | Sll | Srl | Sra | Jal | J | JrImm | JAbsolute | JalrAbsolute | Jr | Jalr | CAddiw - | Li | CLdsp | CLwsp | Nop | Beq | Beqz | Bne | Bnez | JalrImm | Ldnz => { - ArgsShape::NZXSrcNZXDest - } + | Li | CLwsp | Nop | Beq | Beqz | Bne | Bnez | JalrImm | Ldnz => ArgsShape::NZXSrcNZXDest, Addiw | Addw | Subw | Sllw | Srlw | Sraw | Slti | Sltiu | Slliw | Srliw | Sraiw | Slt | Sltu | Auipc => ArgsShape::XSrcNZXDest, diff --git a/src/riscv/lib/src/parser/instruction.rs b/src/riscv/lib/src/parser/instruction.rs index 17ef9664e1fc..a708c8361c72 100644 --- a/src/riscv/lib/src/parser/instruction.rs +++ b/src/riscv/lib/src/parser/instruction.rs @@ -591,6 +591,11 @@ pub enum InstrCacheable { /// The immediate is obtained by zero-extending and scaling by 8 the /// offset encoded in the instruction (see U:C-16.3). CLd(NonZeroITypeArgs), + /// `C.LDSP` - Loads a 64-bit value from memory into register `rd`. It computes + /// an effective address by adding the immediate to the stack pointer. + /// + /// The immediate is obtained by zero-extending and scaling by 8 the + /// offset encoded in the instruction (see U:C-16.3). CLdsp(CIBNZTypeArgs), CSd(SBTypeArgs), CSdsp(CSSTypeArgs), diff --git a/src/riscv/lib/tests/expected/jstz/state_hash_final b/src/riscv/lib/tests/expected/jstz/state_hash_final index c5c2870ebf1c..32f4745c89d0 100644 --- a/src/riscv/lib/tests/expected/jstz/state_hash_final +++ b/src/riscv/lib/tests/expected/jstz/state_hash_final @@ -1 +1 @@ -Hash { digest: [65, 103, 123, 191, 90, 185, 66, 122, 221, 159, 228, 88, 159, 151, 244, 51, 242, 210, 9, 86, 129, 173, 45, 5, 97, 52, 232, 224, 22, 61, 36, 214] } +Hash { digest: [115, 238, 144, 217, 172, 162, 146, 218, 254, 68, 199, 211, 26, 17, 67, 122, 62, 107, 33, 78, 253, 253, 20, 216, 188, 97, 94, 7, 34, 202, 76, 8] } -- GitLab From 8d5fcf9ac52d31b2ba6e793c1ef0e8e9ea62d676 Mon Sep 17 00:00:00 2001 From: Julien Sagot Date: Fri, 21 Feb 2025 12:35:17 +0100 Subject: [PATCH 13/31] Tezt/P2P: check that clique cluster is properly set up before running tests --- tezt/tests/p2p.ml | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/tezt/tests/p2p.ml b/tezt/tests/p2p.ml index a6cc22f75e18..1d8210dc74d3 100644 --- a/tezt/tests/p2p.ml +++ b/tezt/tests/p2p.ml @@ -991,6 +991,16 @@ let trusted_ring () = in unit +let check_connected_points expected node = + let* points = Node.RPC.(call node @@ get_network_points) in + let connected_points = List.filter point_is_running points in + Check.( + (List.length connected_points = expected) + int + ~__LOC__ + ~error_msg:"Expected %R connected points, got %L") ; + unit + (* This test sets up a clique between a set of [M] nodes [N_1; ...; N_M]. @@ -1020,6 +1030,7 @@ let expected_peer_id () = let nodes = Cluster.create num_nodes [] in Cluster.clique nodes ; let* () = Cluster.start ~wait_connections:true nodes in + let* () = Lwt_list.iter_s (check_connected_points (num_nodes - 1)) nodes in let point_id_of_node node = let addr, port = Node.point node in sf "%s:%d" addr port @@ -1071,15 +1082,9 @@ let expected_peer_id () = ~error_msg: ("Expected the expected_peer_id of neighbor of " ^ Node.name node ^ " to be %R, got %L")) ; - let* points = Node.RPC.(call node @@ get_network_points) in - let connected_points = List.filter point_is_running points in - Check.( - (List.length connected_points = num_nodes - 1) - int - ~__LOC__ - ~error_msg:"Expected %R connected points, got %L") ; - unit + check_connected_points (num_nodes - 1) node in + Log.info "Set wrong [expected_peer_id]" ; (* For all nodes [node], we set [expected_peer_id] on its connection to its neighbor [next(node)] to the peer id of @@ -1126,15 +1131,8 @@ let expected_peer_id () = (* Each node should now have two connected peers less *) let* () = iter_p nodes @@ fun node -> - let* points = Node.RPC.(call node @@ get_network_points) in - let connected_points = List.filter point_is_running points in (* We lost two connections *) - Check.( - (List.length connected_points = num_nodes - 3) - int - ~__LOC__ - ~error_msg:"Expected %R connected points, got %L") ; - unit + check_connected_points (num_nodes - 3) node in unit -- GitLab From 6c35fbd157738ff59c4553c58a828b5d5d7d8dd9 Mon Sep 17 00:00:00 2001 From: Thomas Letan Date: Wed, 26 Feb 2025 11:49:07 +0100 Subject: [PATCH 14/31] EVM Node: Uniform log configuration --- etherlink/CHANGES_NODE.md | 2 + etherlink/bin_node/main.ml | 183 ++++++++++++------------------------- 2 files changed, 62 insertions(+), 123 deletions(-) diff --git a/etherlink/CHANGES_NODE.md b/etherlink/CHANGES_NODE.md index cdbb8f0a84f6..bcbb7b754ede 100644 --- a/etherlink/CHANGES_NODE.md +++ b/etherlink/CHANGES_NODE.md @@ -19,6 +19,8 @@ snapshot. (!16963) - **experimental feature** Adds a configuration for the `tx_queue`. (!16903) +- Commands emitting logs now systematically comply with the `verbosity` + configuration option. (!16975) ### RPCs changes diff --git a/etherlink/bin_node/main.ml b/etherlink/bin_node/main.ml index 8e3dc856c948..f2021b0db64a 100644 --- a/etherlink/bin_node/main.ml +++ b/etherlink/bin_node/main.ml @@ -831,6 +831,48 @@ let websocket_checks config = Internal_event.Simple.emit Event.buggy_dream_websocket () |> Lwt_result.ok | _ -> Lwt_result_syntax.return_unit +let make_event_config ~verbosity ?daily_logs_path () = + let open Tezos_event_logging.Internal_event in + let open Tezos_base_unix.Internal_event_unix in + let open Tezos_base.Internal_event_config in + let config = make_with_defaults ~verbosity () in + match daily_logs_path with + | Some daily_logs_path -> + (* Show only above Info rpc_server events, they are not + relevant as we do not have a REST-API server. If not + set, the daily logs are polluted with these + uninformative logs. *) + let daily_logs_section_prefixes = + [ + ("rpc_server", Some Notice); + ("rpc_server", Some Warning); + ("rpc_server", Some Error); + ("rpc_server", Some Fatal); + ] + in + let uri = + make_config_uri + ~create_dirs:true + ~daily_logs:7 + ~level:Info + ~format:"pp-rfc5424" + ~chmod:0o640 + ~section_prefixes:daily_logs_section_prefixes + (`Path Filename.Infix.(daily_logs_path // "daily.log")) + in + add_uri_to_config uri config + | None -> config + +let init_logs ~daily_logs ~data_dir configuration = + let open Tezos_base_unix.Internal_event_unix in + let daily_logs_path = + if daily_logs then Some Filename.Infix.(data_dir // "daily_logs") else None + in + let config = + make_event_config ~verbosity:configuration.verbose ?daily_logs_path () + in + init ~config () + let start_proxy ~data_dir ~keep_alive ?rpc_addr ?rpc_port ?rpc_batch_limit ?cors_origins ?cors_headers ?log_filter_max_nb_blocks ?log_filter_max_nb_logs ?log_filter_chunk_size ?rollup_node_endpoint @@ -875,10 +917,7 @@ let start_proxy ~data_dir ~keep_alive ?rpc_addr ?rpc_port ?rpc_batch_limit }; } in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - init ~config:(make_with_defaults ~verbosity:config.verbose ()) () - in + let*! () = init_logs ~daily_logs:true ~data_dir config in let*! () = Internal_event.Simple.emit Event.event_starting "proxy" in let* () = Evm_node_lib_dev.Proxy.main config in let wait, _resolve = Lwt.wait () in @@ -911,24 +950,6 @@ let sequencer_disable_native_execution configuration = } | Never -> return configuration -let make_event_config ~verbosity ~daily_logs_path - ?(daily_logs_section_prefixes = []) () = - let open Tezos_event_logging.Internal_event in - let open Tezos_base_unix.Internal_event_unix in - let open Tezos_base.Internal_event_config in - let config = make_with_defaults ~verbosity () in - let uri = - make_config_uri - ~create_dirs:true - ~daily_logs:7 - ~level:Info - ~format:"pp-rfc5424" - ~chmod:0o640 - ~section_prefixes:daily_logs_section_prefixes - (`Path Filename.Infix.(daily_logs_path // "daily.log")) - in - add_uri_to_config uri config - let start_sequencer ?password_filename ~wallet_dir ~data_dir ?rpc_addr ?rpc_port ?rpc_batch_limit ?cors_origins ?cors_headers ?tx_pool_timeout_limit ?tx_pool_addr_limit ?tx_pool_tx_per_addr_limit ~keep_alive @@ -983,27 +1004,7 @@ let start_sequencer ?password_filename ~wallet_dir ~data_dir ?rpc_addr ?rpc_port ~finalized_view () in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = - make_event_config - ~verbosity:configuration.verbose - ~daily_logs_path:Filename.Infix.(data_dir // "daily_logs") - (* Show only above Info rpc_server events, they are not - relevant as we do not have a REST-API server. If not - set, the daily logs are polluted with these - uninformative logs. *) - ~daily_logs_section_prefixes: - [ - ("rpc_server", Some Notice); - ("rpc_server", Some Warning); - ("rpc_server", Some Error); - ("rpc_server", Some Fatal); - ] - () - in - init ~config () - in + let*! () = init_logs ~daily_logs:true ~data_dir configuration in let*! configuration = match sandbox_key with | None -> @@ -1134,27 +1135,7 @@ let rpc_command = read_write_config ~finalized_view in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = - make_event_config - ~verbosity:config.verbose - ~daily_logs_path:Filename.Infix.(data_dir // "daily_logs") - (* Show only above Info rpc_server events, they are not - relevant as we do not have a REST-API server. If not - set, the daily logs are polluted with these - uninformative logs. *) - ~daily_logs_section_prefixes: - [ - ("rpc_server", Some Notice); - ("rpc_server", Some Warning); - ("rpc_server", Some Error); - ("rpc_server", Some Fatal); - ] - () - in - init ~config () - in + let*! () = init_logs ~daily_logs:true ~data_dir config in let* () = websocket_checks config in let*! () = Internal_event.Simple.emit Event.event_starting "rpc" in Evm_node_lib_dev.Rpc.main @@ -1207,27 +1188,7 @@ let start_observer ~data_dir ~keep_alive ?rpc_addr ?rpc_port ?rpc_batch_limit ?network () in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = - make_event_config - ~verbosity:config.verbose - ~daily_logs_path:Filename.Infix.(data_dir // "daily_logs") - (* Show only above Info rpc_server events, they are not - relevant as we do not have a REST-API server. If not - set, the daily logs are polluted with these - uninformative logs. *) - ~daily_logs_section_prefixes: - [ - ("rpc_server", Some Notice); - ("rpc_server", Some Warning); - ("rpc_server", Some Error); - ("rpc_server", Some Fatal); - ] - () - in - init ~config () - in + let*! () = init_logs ~daily_logs:true ~data_dir config in let* () = websocket_checks config in let*! () = Internal_event.Simple.emit Event.event_starting "observer" in Evm_node_lib_dev.Observer.main @@ -1442,6 +1403,7 @@ let init_from_rollup_node_command = (fun (data_dir, omit_delayed_tx_events) rollup_node_data_dir () -> let open Lwt_result_syntax in let* configuration = Cli.create_or_read_config ~data_dir () in + let*! () = init_logs ~daily_logs:false ~data_dir configuration in Evm_node_lib_dev.Evm_context.init_from_rollup_node ~configuration ~omit_delayed_tx_events @@ -1550,15 +1512,6 @@ let replay_command = l2_level () -> let open Lwt_result_syntax in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = - if kernel_verbosity = Some Evm_node_lib_dev.Events.Debug then - Some (make_with_defaults ~verbosity:Debug ()) - else None - in - init ?config () - in let* configuration = Cli.create_or_read_config ~data_dir @@ -1567,6 +1520,7 @@ let replay_command = ?native_execution_policy () in + let*! () = init_logs ~daily_logs:false ~data_dir configuration in Evm_node_lib_dev.Replay.main ~profile ?kernel_path @@ -1598,12 +1552,8 @@ let patch_kernel_command = (fun (data_dir, block_number, force) kernel_path () -> let open Lwt_result_syntax in let open Evm_node_lib_dev in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = make_with_defaults ~verbosity:Warning () in - init ~config () - in let* configuration = Cli.create_or_read_config ~data_dir () in + let*! () = init_logs ~daily_logs:false ~data_dir configuration in (* We remove the [observer] configuration. This [patch] should not need to interact with an upstream EVM node. *) let configuration = {configuration with observer = None} in @@ -1770,11 +1720,7 @@ mode.|} ?history_mode () in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = make_with_defaults ~verbosity:Warning () in - init ~config () - in + let*! () = init_logs ~daily_logs:false ~data_dir config in let* () = websocket_checks config in Configuration.save ~force ~data_dir config) @@ -1806,11 +1752,7 @@ let check_config_command = config_path | None -> load ?network ~data_dir () in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = make_with_defaults ~verbosity:Warning () in - init ~config () - in + let*! () = init_logs ~daily_logs:false ~data_dir config in let* () = websocket_checks config in if print_config then Format.printf "%a\n" (Configuration.pp_print_json ~data_dir) config @@ -2401,6 +2343,9 @@ let export_snapshot (data_dir, snapshot_file, compress_on_the_fly, uncompressed) = let open Lwt_result_syntax in let open Evm_node_lib_dev.Snapshots in + let* configuration = Configuration.Cli.create_or_read_config ~data_dir () in + let*! () = init_logs ~daily_logs:false ~data_dir configuration in + let compression = match (compress_on_the_fly, uncompressed) with | true, true -> @@ -2441,11 +2386,10 @@ let import_snapshot_command = (prefixes ["snapshot"; "import"] @@ Params.snapshot_file_or_url @@ stop) (fun (data_dir, force) snapshot_file () -> let open Lwt_result_syntax in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = make_with_defaults ~verbosity:Notice () in - init ~config () + let* configuration = + Configuration.Cli.create_or_read_config ~data_dir () in + let*! () = init_logs ~daily_logs:false ~data_dir configuration in Evm_node_lib_dev.Snapshots.import_from ~cancellable:true ~force @@ -2552,11 +2496,7 @@ let switch_history_mode_command = let* () = Evm_store.use store @@ fun conn -> let* config = Cli.create_or_read_config ~data_dir ~history_mode () in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = make_with_defaults ~verbosity:Warning () in - init ~config () - in + let*! () = init_logs ~daily_logs:false ~data_dir config in let* store_history_mode = Evm_store.Metadata.find_history_mode conn in let* history_mode = @@ -2599,12 +2539,8 @@ let patch_state_command = @@ stop) (fun (data_dir, block_number, force) key value () -> let open Evm_node_lib_dev in - let*! () = - let open Tezos_base_unix.Internal_event_unix in - let config = make_with_defaults ~verbosity:Warning () in - init ~config () - in let* configuration = Cli.create_or_read_config ~data_dir () in + let*! () = init_logs ~daily_logs:false ~data_dir configuration in if force then (* We remove the [observer] configuration. This [patch] should not need to interact with an upstream EVM node. *) @@ -2642,6 +2578,7 @@ let preemptive_kernel_download_command = let* configuration = Cli.create_or_read_config ~data_dir ?preimages ?preimages_endpoint () in + let*! () = init_logs ~daily_logs:false ~data_dir configuration in let kernel_execution_config = configuration.kernel_execution in let*? preimages_endpoint = Option.either -- GitLab From 98ad9595a25dc83dc2fed09d5b25ebd0bace4a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Kr=C3=BCger?= Date: Mon, 24 Feb 2025 22:47:06 +0000 Subject: [PATCH 15/31] RISC-V: Disallow EnrichedCells in Layouts --- src/riscv/lib/src/machine_state.rs | 5 +- .../lib/src/machine_state/block_cache.rs | 68 ++++++---------- .../src/machine_state/block_cache/bcall.rs | 14 +++- src/riscv/lib/src/pvm/common.rs | 24 +++--- src/riscv/lib/src/pvm/node_pvm.rs | 20 +++-- src/riscv/lib/src/state_backend.rs | 59 ++++++-------- .../lib/src/state_backend/owned_backend.rs | 60 +++++--------- .../lib/src/state_backend/proof_backend.rs | 79 ++++++------------- src/riscv/lib/src/state_backend/region.rs | 27 ++++--- src/riscv/lib/src/state_backend/trans.rs | 13 +-- .../lib/src/state_backend/verify_backend.rs | 37 +++++---- 11 files changed, 165 insertions(+), 241 deletions(-) diff --git a/src/riscv/lib/src/machine_state.rs b/src/riscv/lib/src/machine_state.rs index f0cc8c3cf7b4..216fd86a26a6 100644 --- a/src/riscv/lib/src/machine_state.rs +++ b/src/riscv/lib/src/machine_state.rs @@ -244,7 +244,10 @@ impl, M: backend::Ma pub fn bind( space: backend::AllocatedOf, M>, block_builder: B::BlockBuilder, - ) -> Self { + ) -> Self + where + M::ManagerRoot: ManagerReadWrite, + { Self { core: MachineCoreState::bind(space.0), block_cache: BlockCache::bind(space.1, block_builder), diff --git a/src/riscv/lib/src/machine_state/block_cache.rs b/src/riscv/lib/src/machine_state/block_cache.rs index 11396a04fd5b..5e17ba7dd1ba 100644 --- a/src/riscv/lib/src/machine_state/block_cache.rs +++ b/src/riscv/lib/src/machine_state/block_cache.rs @@ -90,7 +90,9 @@ use super::address_translation::PAGE_OFFSET_WIDTH; use super::instruction::Instruction; use super::{MachineCoreState, memory::MemoryConfig}; use super::{ProgramCounterUpdate, memory::Address}; +use crate::default::ConstDefault; use crate::machine_state::address_translation::PAGE_SIZE; +use crate::machine_state::instruction::Args; use crate::parser::instruction::InstrWidth; use crate::state_backend::{ self, AllocatedOf, Atom, Cell, EnrichedCell, EnrichedValue, ManagerBase, ManagerClone, @@ -102,8 +104,6 @@ use crate::{ cache_utils::Sizes, storage::{Hash, HashError}, }; -use crate::{default::ConstDefault, state_backend::verify_backend}; -use crate::{machine_state::instruction::Args, storage::binary}; /// Mask for getting the offset within a page const PAGE_OFFSET_MASK: usize = (1 << PAGE_OFFSET_WIDTH) - 1; @@ -117,11 +117,11 @@ pub struct ICallLayout { } impl state_backend::Layout for ICallLayout { - type Allocated = EnrichedCell, M>; + type Allocated = Cell; fn allocate(backend: &mut M) -> Self::Allocated { - let value = backend.allocate_enriched_cell(Instruction::DEFAULT); - EnrichedCell::bind(value) + let value = backend.allocate_region([Instruction::DEFAULT]); + Cell::bind(value) } } @@ -135,25 +135,11 @@ impl state_backend::ProofLayout for ICallLayout { fn to_merkle_tree( state: state_backend::RefProofGenOwnedAlloc, ) -> Result { - let serialised = binary::serialise(&state)?; - proof_backend::merkle::MerkleTree::make_merkle_leaf( - serialised, - state.cell_ref().get_access_info(), - ) + Atom::to_merkle_tree(state) } fn from_proof(proof: state_backend::ProofTree) -> state_backend::FromProofResult { - let leaf = proof.into_leaf()?; - - let cell = match leaf { - state_backend::ProofPart::Present(data) => { - let value = binary::deserialise(data)?; - verify_backend::EnrichedCell::Present(value) - } - state_backend::ProofPart::Absent => verify_backend::EnrichedCell::Absent, - }; - - Ok(EnrichedCell::bind(cell)) + Atom::from_proof(proof) } } @@ -258,7 +244,10 @@ pub struct Cached, M: ManagerBase> { } impl, M: ManagerBase> Cached { - fn bind(space: AllocatedOf, M>) -> Self { + fn bind(space: AllocatedOf, M>) -> Self + where + M::ManagerRoot: ManagerReadWrite, + { Self { address: space.0, fence_counter: space.1, @@ -394,12 +383,15 @@ pub trait BlockCacheLayout: state_backend::CommitmentLayout + state_backend::Pro type Sizes; - fn bind, M: state_backend::ManagerBase>( + fn bind( space: state_backend::AllocatedOf, block_builder: B::BlockBuilder, ) -> BlockCache where - Self: Sized; + Self: Sized, + M: state_backend::ManagerBase, + M::ManagerRoot: ManagerReadWrite, + B: Block; fn entry, M: ManagerBase>( entries: &Self::Entries, @@ -447,7 +439,10 @@ impl BlockCacheLayout fn bind, M: ManagerBase>( space: AllocatedOf, block_builder: B::BlockBuilder, - ) -> BlockCache { + ) -> BlockCache + where + M::ManagerRoot: ManagerReadWrite, + { BlockCache { current_block_addr: space.0, next_instr_addr: space.1, @@ -535,7 +530,10 @@ impl, B: Block, MC: MemoryConfig /// Bind the block cache to the given allocated state and the given [block builder]. /// /// [block builder]: Block::BlockBuilder - pub fn bind(space: AllocatedOf, block_builder: B::BlockBuilder) -> Self { + pub fn bind(space: AllocatedOf, block_builder: B::BlockBuilder) -> Self + where + M::ManagerRoot: ManagerReadWrite, + { BCL::bind(space, block_builder) } @@ -908,7 +906,7 @@ mod tests { mode::Mode, registers::{XRegister, a1, nz, t0, t1}, }, - state_backend::{CommitmentLayout, owned_backend::Owned}, + state_backend::owned_backend::Owned, }; pub type TestLayout = Layout; @@ -1210,22 +1208,6 @@ mod tests { ); }); - /// Tests that a layout which contains an [`EnrichedCell`] is hashed identically as - /// a layout which contains a [`Cell`] of the same value. - #[test] - fn test_enriched_cell_hashing() { - let instr = Instruction::DEFAULT; - - let ec_value = (instr, ICall::::from(&instr)); - let ec: EnrichedCell, Ref<'_, Owned>> = EnrichedCell::bind(&ec_value); - let ec_hash = as CommitmentLayout>::state_hash(ec).unwrap(); - - let c_value = [instr; 1]; - let c: Cell> = Cell::bind(&c_value); - let c_hash = as CommitmentLayout>::state_hash(c).unwrap(); - assert_eq!(ec_hash, c_hash); - } - /// The initialised block cache must not return any blocks. This is especially important for /// blocks at address 0 which at one point were accidentally valid but empty which caused loops. #[test] diff --git a/src/riscv/lib/src/machine_state/block_cache/bcall.rs b/src/riscv/lib/src/machine_state/block_cache/bcall.rs index 4418b6605962..56db7e26f5ce 100644 --- a/src/riscv/lib/src/machine_state/block_cache/bcall.rs +++ b/src/riscv/lib/src/machine_state/block_cache/bcall.rs @@ -66,7 +66,9 @@ pub trait Block { type BlockBuilder: Default; /// Bind the block to the given allocated state. - fn bind(allocated: AllocatedOf, M>) -> Self; + fn bind(allocated: AllocatedOf, M>) -> Self + where + M::ManagerRoot: ManagerReadWrite; /// Given a manager morphism `f : &M -> N`, return the layout's allocated structure containing /// the constituents of `N` that were produced from the constituents of `&M`. @@ -233,8 +235,14 @@ impl Block for Interpreted { self.len_instr.write(0); } - fn bind((len_instr, instr): AllocatedOf, M>) -> Self { - Self { len_instr, instr } + fn bind(space: AllocatedOf, M>) -> Self + where + M::ManagerRoot: ManagerReadWrite, + { + Self { + len_instr: space.0, + instr: space.1.map(EnrichedCell::bind), + } } fn struct_ref<'a, F: FnManager>>( diff --git a/src/riscv/lib/src/pvm/common.rs b/src/riscv/lib/src/pvm/common.rs index 2cd4b92d3758..80fe4ee57790 100644 --- a/src/riscv/lib/src/pvm/common.rs +++ b/src/riscv/lib/src/pvm/common.rs @@ -18,7 +18,7 @@ use crate::{ pvm::sbi, state_backend::{ self, Atom, Cell, - proof_backend::{ProofDynRegion, ProofEnrichedCell, ProofGen, ProofRegion}, + proof_backend::{ProofDynRegion, ProofGen, ProofRegion}, }, traps::EnvironException, }; @@ -153,7 +153,10 @@ impl< pub fn bind( space: state_backend::AllocatedOf, M>, block_builder: B::BlockBuilder, - ) -> Self { + ) -> Self + where + M::ManagerRoot: state_backend::ManagerReadWrite, + { Self { version: space.0, machine_state: machine_state::MachineState::bind(space.1, block_builder), @@ -180,7 +183,10 @@ impl< } /// Generate a proof-generating version of this PVM. - pub fn start_proof(&self) -> PvmProofGen<'_, MC, CL, M> { + pub fn start_proof(&self) -> PvmProofGen<'_, MC, CL, M> + where + M: state_backend::ManagerRead, + { enum ProofWrapper {} impl state_backend::FnManager for ProofWrapper { @@ -197,18 +203,10 @@ impl< ) -> as state_backend::ManagerBase>::DynRegion { ProofDynRegion::bind(input) } - - fn map_enriched_cell( - input: ::EnrichedCell, - ) -> as state_backend::ManagerBase>::EnrichedCell { - ProofEnrichedCell::bind(input) - } } - Pvm::bind( - self.struct_ref::(), - bcall::InterpretedBlockBuilder, - ) + let space = self.struct_ref::(); + Pvm::bind(space, bcall::InterpretedBlockBuilder) } /// Reset the PVM state. diff --git a/src/riscv/lib/src/pvm/node_pvm.rs b/src/riscv/lib/src/pvm/node_pvm.rs index a74bf16fa158..b5c2b5756fdc 100644 --- a/src/riscv/lib/src/pvm/node_pvm.rs +++ b/src/riscv/lib/src/pvm/node_pvm.rs @@ -19,9 +19,7 @@ use crate::{ state_backend::{ self, AllocatedOf, CommitmentLayout, ProofLayout, ProofTree, Ref, owned_backend::Owned, - proof_backend::{ - ProofDynRegion, ProofEnrichedCell, ProofGen, ProofRegion, proof::MerkleProof, - }, + proof_backend::{ProofDynRegion, ProofGen, ProofRegion, proof::MerkleProof}, verify_backend::Verifier, }, storage::{self, Hash, Repo}, @@ -44,7 +42,10 @@ pub struct State { } impl State { - pub fn bind(space: state_backend::AllocatedOf) -> Self { + pub fn bind(space: state_backend::AllocatedOf) -> Self + where + M::ManagerRoot: state_backend::ManagerReadWrite, + { Self { pvm: Pvm::::bind(space.0, InterpretedBlockBuilder), level_is_set: space.1, @@ -69,7 +70,10 @@ impl State { } /// Generate a proof-generating version of this state. - pub fn start_proof(&self) -> State>> { + pub fn start_proof(&self) -> State>> + where + M: state_backend::ManagerRead, + { enum ProofWrapper {} impl state_backend::FnManager for ProofWrapper { @@ -86,12 +90,6 @@ impl State { ) -> as state_backend::ManagerBase>::DynRegion { ProofDynRegion::bind(input) } - - fn map_enriched_cell( - input: ::EnrichedCell, - ) -> as state_backend::ManagerBase>::EnrichedCell { - ProofEnrichedCell::bind(input) - } } State::bind(self.struct_ref::()) diff --git a/src/riscv/lib/src/state_backend.rs b/src/riscv/lib/src/state_backend.rs index 9e660ed52498..ad408d52e1c2 100644 --- a/src/riscv/lib/src/state_backend.rs +++ b/src/riscv/lib/src/state_backend.rs @@ -140,6 +140,14 @@ pub trait ManagerBase: Sized { /// /// [`Owned`]: owned_backend::Owned type ManagerRoot: ManagerBase; + + /// Upgrade a single-element region to an enriched cell. + fn enrich_cell>( + cell: Self::Region, + ) -> Self::EnrichedCell; + + /// Obtain a reference to the underlying region of an enriched cell. + fn as_devalued_cell(cell: &Self::EnrichedCell) -> &Self::Region; } /// Manager with allocation capabilities @@ -155,11 +163,6 @@ pub trait ManagerAlloc: 'static + ManagerReadWrite { /// Allocate a dynamic region in the state storage. fn allocate_dyn_region(&mut self) -> Self::DynRegion; - - /// Allocate an enriched cell. - fn allocate_enriched_cell(&mut self, init_value: V::E) -> Self::EnrichedCell - where - V: EnrichedValueLinked; } /// Manager with read capabilities @@ -259,16 +262,6 @@ pub trait ManagerSerialise: ManagerRead { region: &Self::DynRegion, serializer: S, ) -> Result; - - /// Serialise the contents of the enriched cell. - fn serialise_enriched_cell( - cell: &Self::EnrichedCell, - serializer: S, - ) -> Result - where - V: EnrichedValue, - V::E: serde::Serialize, - S: serde::Serializer; } /// Manager with the ability to deserialise regions @@ -287,14 +280,6 @@ pub trait ManagerDeserialise: ManagerBase { fn deserialise_dyn_region<'de, const LEN: usize, D: serde::Deserializer<'de>>( deserializer: D, ) -> Result, D::Error>; - - /// Deserialise an enriched cell. - fn deserialise_enriched_cell<'de, V, D: serde::Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> - where - V: EnrichedValueLinked, - V::E: serde::Deserialize<'de>; } /// Manager with the ability to clone regions @@ -323,9 +308,19 @@ impl<'backend, M: ManagerBase> ManagerBase for Ref<'backend, M> { type DynRegion = &'backend M::DynRegion; - type EnrichedCell = &'backend M::EnrichedCell; + type EnrichedCell = &'backend M::Region; type ManagerRoot = M::ManagerRoot; + + fn enrich_cell>( + cell: Self::Region, + ) -> Self::EnrichedCell { + cell + } + + fn as_devalued_cell(cell: &Self::EnrichedCell) -> &Self::Region { + cell + } } impl ManagerSerialise for Ref<'_, M> { @@ -342,16 +337,6 @@ impl ManagerSerialise for Ref<'_, M> { ) -> Result { M::serialise_dyn_region(region, serializer) } - - fn serialise_enriched_cell( - cell: &Self::EnrichedCell, - serializer: S, - ) -> Result - where - V::E: serde::Serialize, - { - M::serialise_enriched_cell(cell, serializer) - } } impl ManagerRead for Ref<'_, M> { @@ -387,14 +372,14 @@ impl ManagerRead for Ref<'_, M> { V: EnrichedValue, V::E: Copy, { - M::enriched_cell_read_stored(cell) + M::region_read(cell, 0) } fn enriched_cell_ref_stored(cell: &Self::EnrichedCell) -> &V::E where V: EnrichedValue, { - M::enriched_cell_ref_stored(cell) + M::region_ref(cell, 0) } fn enriched_cell_read_derived>( @@ -403,7 +388,7 @@ impl ManagerRead for Ref<'_, M> { where V::D: Copy, { - M::enriched_cell_read_derived(cell) + V::derive(M::region_ref(cell, 0)) } } diff --git a/src/riscv/lib/src/state_backend/owned_backend.rs b/src/riscv/lib/src/state_backend/owned_backend.rs index 3c50e0c4ac6a..5940e5277991 100644 --- a/src/riscv/lib/src/state_backend/owned_backend.rs +++ b/src/riscv/lib/src/state_backend/owned_backend.rs @@ -4,7 +4,7 @@ // SPDX-License-Identifier: MIT use std::{ - fmt, + array, fmt, marker::PhantomData, mem::{self, MaybeUninit}, }; @@ -36,6 +36,18 @@ impl ManagerBase for Owned { type EnrichedCell = (V::E, V::D); type ManagerRoot = Self; + + fn enrich_cell>( + cell: Self::Region, + ) -> Self::EnrichedCell { + let [value] = cell; + let derived = V::derive(&value); + (value, derived) + } + + fn as_devalued_cell(cell: &Self::EnrichedCell) -> &Self::Region { + array::from_ref(&cell.0) + } } impl ManagerAlloc for Owned { @@ -55,14 +67,6 @@ impl ManagerAlloc for Owned { Box::from_raw(alloc.cast()) } } - - fn allocate_enriched_cell>( - &mut self, - value: V::E, - ) -> Self::EnrichedCell { - let derived = V::derive(&value); - (value, derived) - } } impl ManagerRead for Owned { @@ -239,17 +243,6 @@ impl ManagerSerialise for Owned { ) -> Result { serializer.serialize_bytes(region.as_slice()) } - - fn serialise_enriched_cell( - cell: &Self::EnrichedCell, - serializer: S, - ) -> Result - where - V::E: serde::Serialize, - { - use serde::Serialize; - cell.0.serialize(serializer) - } } impl ManagerDeserialise for Owned { @@ -288,8 +281,7 @@ impl ManagerDeserialise for Owned { where A: serde::de::SeqAccess<'de>, { - let mut values: [MaybeUninit; LEN] = - std::array::from_fn(|_| MaybeUninit::uninit()); + let mut values: [MaybeUninit; LEN] = array::from_fn(|_| MaybeUninit::uninit()); for value in values.iter_mut() { value.write(seq.next_element::()?.ok_or_else(|| { @@ -320,19 +312,6 @@ impl ManagerDeserialise for Owned { vec.try_into() .map_err(|_err| serde::de::Error::custom("Dynamic region of mismatching length")) } - - fn deserialise_enriched_cell<'de, V, D: serde::Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> - where - V: EnrichedValueLinked, - V::E: serde::Deserialize<'de>, - { - use serde::Deserialize; - let value = V::E::deserialize(deserializer)?; - let derived = V::derive(&value); - Ok((value, derived)) - } } impl ManagerClone for Owned { @@ -361,7 +340,7 @@ impl ManagerClone for Owned { pub mod test_helpers { use super::*; use crate::state_backend::{ - Cell, Cells, DynCells, EnrichedCell, Ref, + Cell, Cells, DynCells, EnrichedCell, FnManagerIdent, Ref, proof_backend::{ProofDynRegion, ProofGen, ProofRegion}, test_helpers::TestBackendFactory, }; @@ -469,7 +448,8 @@ pub mod test_helpers { } proptest::proptest!(|(value: u64)| { - let mut cell: EnrichedCell = EnrichedCell::bind((0u64, T::from(&0))); + let cell = Cell::bind([0u64]); + let mut cell: EnrichedCell = EnrichedCell::bind(cell); cell.write(value); let read_value = cell.read_ref_stored(); @@ -488,8 +468,7 @@ pub mod test_helpers { assert_eq!(derived.0, derived_after.0); // Serialisation is consistent with that of the `ProofGen` backend. - let proof_cell: EnrichedCell> = - EnrichedCell::bind(cell.cell_ref()); + let proof_cell: EnrichedCell> = EnrichedCell::bind(cell.struct_ref::()); let proof_bytes = bincode::serialize(&proof_cell).unwrap(); assert_eq!(bytes, proof_bytes); }); @@ -513,7 +492,8 @@ pub mod test_helpers { } proptest::proptest!(|(value: u64)| { - let mut ecell: EnrichedCell = EnrichedCell::bind((0u64, Fun::from(&0))); + let cell = Cell::bind([0u64]); + let mut ecell: EnrichedCell = EnrichedCell::bind(cell); let mut cell: Cell = Cell::bind([0; 1]); ecell.write(value); cell.write(value); diff --git a/src/riscv/lib/src/state_backend/proof_backend.rs b/src/riscv/lib/src/state_backend/proof_backend.rs index 16e3b90b1416..33355ab8713d 100644 --- a/src/riscv/lib/src/state_backend/proof_backend.rs +++ b/src/riscv/lib/src/state_backend/proof_backend.rs @@ -21,7 +21,7 @@ use std::{ }; use merkle::AccessInfo; -use serde::{Serialize, ser::SerializeTuple}; +use serde::ser::SerializeTuple; use super::{ EnrichedValue, EnrichedValueLinked, ManagerBase, ManagerRead, ManagerReadWrite, @@ -45,6 +45,16 @@ impl ManagerBase for ProofGen { type EnrichedCell = ProofEnrichedCell; type ManagerRoot = Self; + + fn enrich_cell>( + underlying: Self::Region, + ) -> Self::EnrichedCell { + ProofEnrichedCell { underlying } + } + + fn as_devalued_cell(cell: &Self::EnrichedCell) -> &Self::Region { + &cell.underlying + } } /// Implementation of [`ManagerRead`] which wraps another manager and @@ -97,11 +107,7 @@ impl ManagerRead for ProofGen { V: EnrichedValue, V::E: Copy, { - cell.set_read(); - match &cell.written { - None => M::enriched_cell_read_stored(&cell.source), - Some(value) => *value, - } + Self::region_read(&cell.underlying, 0) } fn enriched_cell_read_derived(cell: &Self::EnrichedCell) -> V::D @@ -116,8 +122,7 @@ impl ManagerRead for ProofGen { where V: EnrichedValue, { - cell.set_read(); - cell.unrecorded_ref_stored() + Self::region_ref(&cell.underlying, 0) } } @@ -179,8 +184,7 @@ impl ManagerWrite for ProofGen { where V: EnrichedValueLinked, { - cell.set_write(); - cell.written = Some(value); + Self::region_write(&mut cell.underlying, 0, value); } } @@ -229,19 +233,6 @@ impl ManagerSerialise for ProofGen { region.unrecorded_read_all(0, &mut values); serializer.serialize_bytes(values.as_slice()) } - - fn serialise_enriched_cell( - cell: &Self::EnrichedCell, - serializer: S, - ) -> Result - where - V: EnrichedValue, - V::E: serde::Serialize, - S: serde::Serializer, - { - let elem = cell.unrecorded_ref_stored(); - elem.serialize(serializer) - } } /// Proof region which wraps a region managed by another manager. @@ -388,53 +379,35 @@ impl ProofDynRegion { /// The underlying cell is never mutated, but written values are recorded /// in order to preserve the integrity of subsequent reads. pub struct ProofEnrichedCell { - source: M::EnrichedCell, - written: Option, - access: Cell, + underlying: ProofRegion, } impl ProofEnrichedCell { /// Bind a pre-existing enriched cell. - pub fn bind(source: M::EnrichedCell) -> Self { + pub fn bind(source: M::Region) -> Self { Self { - source, - written: None, - access: Cell::new(AccessInfo::NoAccess), + underlying: ProofRegion::bind(source), } } /// Get a copy of the access log. pub fn get_access_info(&self) -> AccessInfo { - self.access.get() + self.underlying.access.get() } /// Set the access log to `Read` or, if previously `Write`, to `ReadWrite`. pub fn set_read(&self) { - self.access.set(AccessInfo::and_read(self.access.get())) + self.underlying.set_read() } /// Set the access log to `Write` or, if previously `Read`, to `ReadWrite`. pub fn set_write(&mut self) { - self.access.set(AccessInfo::and_write(self.access.get())) + self.underlying.set_write() } /// Set the access log to `ReadWrite`. pub fn set_read_write(&self) { - self.access.set(AccessInfo::ReadWrite) - } -} - -impl ProofEnrichedCell { - /// Version of [`ManagerRead::enriched_cell_ref_stored`] which does not - /// record the access as a read. - fn unrecorded_ref_stored(&self) -> &V::E - where - V: EnrichedValue, - { - match &self.written { - None => M::enriched_cell_ref_stored(&self.source), - Some(value) => value, - } + self.underlying.set_read_write() } } @@ -720,9 +693,8 @@ mod tests { proptest!(|(value_before: u64, value_after: u64)| { // A read followed by a write - let cell = (value_before, T::from(&value_before)); - let mut proof_cell: ProofEnrichedCell> = - ProofEnrichedCell::bind(&cell); + let value = [value_before]; + let mut proof_cell: ProofEnrichedCell> = ProofEnrichedCell::bind(&value); prop_assert_eq!(proof_cell.get_access_info(), AccessInfo::NoAccess); let value = ProofGen::>::enriched_cell_read_stored(&proof_cell); prop_assert_eq!(value, value_before); @@ -733,9 +705,8 @@ mod tests { prop_assert_eq!(proof_cell.get_access_info(), AccessInfo::ReadWrite); // A write followed by a read - let cell = (value_before, T::from(&value_before)); - let mut proof_cell: ProofEnrichedCell> = - ProofEnrichedCell::bind(&cell); + let value = [value_before]; + let mut proof_cell: ProofEnrichedCell> = ProofEnrichedCell::bind(&value); prop_assert_eq!(proof_cell.get_access_info(), AccessInfo::NoAccess); ProofGen::>::enriched_cell_write(&mut proof_cell, value_after); prop_assert_eq!(proof_cell.get_access_info(), AccessInfo::Write); diff --git a/src/riscv/lib/src/state_backend/region.rs b/src/riscv/lib/src/state_backend/region.rs index 94121ffc0f0f..b4d2f362375d 100644 --- a/src/riscv/lib/src/state_backend/region.rs +++ b/src/riscv/lib/src/state_backend/region.rs @@ -3,8 +3,6 @@ // // SPDX-License-Identifier: MIT -use std::borrow::Borrow; - use super::{ Elem, EnrichedValue, EnrichedValueLinked, FnManager, ManagerBase, ManagerClone, ManagerDeserialise, ManagerRead, ManagerReadWrite, ManagerSerialise, ManagerWrite, Ref, @@ -25,7 +23,12 @@ pub struct EnrichedCell { impl EnrichedCell { /// Bind this state to the enriched cell. - pub fn bind(cell: M::EnrichedCell) -> Self { + pub fn bind(cell: Cell) -> Self + where + V: EnrichedValueLinked, + { + let region = cell.into_region(); + let cell = M::enrich_cell(region); Self { cell } } @@ -36,10 +39,11 @@ impl EnrichedCell { /// Given a manager morphism `f : &M -> N`, return the layout's allocated structure containing /// the constituents of `N` that were produced from the constituents of `&M`. - pub fn struct_ref<'a, F: FnManager>>(&'a self) -> EnrichedCell { - EnrichedCell { - cell: F::map_enriched_cell(self.cell.borrow()), - } + pub fn struct_ref<'a, F: FnManager>>(&'a self) -> Cell { + let cell = self.cell_ref(); + let region = M::as_devalued_cell(cell); + let region = F::map_region(region); + Cell::bind(region) } /// Write the value back to the enriched cell. @@ -401,7 +405,9 @@ where where S: serde::Serializer, { - M::serialise_enriched_cell(&self.cell, serializer) + let cell = self.cell_ref(); + let region = M::as_devalued_cell(cell); + M::serialise_region(region, serializer) } } @@ -429,14 +435,15 @@ impl<'de, E: serde::Deserialize<'de>, const LEN: usize, M: ManagerDeserialise> impl<'de, V, M: ManagerDeserialise> serde::Deserialize<'de> for EnrichedCell where - V: EnrichedValueLinked, + V: EnrichedValueLinked, V::E: serde::Deserialize<'de>, { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { - let cell = M::deserialise_enriched_cell(deserializer)?; + let region = M::deserialise_region(deserializer)?; + let cell = M::enrich_cell(region); Ok(Self { cell }) } } diff --git a/src/riscv/lib/src/state_backend/trans.rs b/src/riscv/lib/src/state_backend/trans.rs index 823fd4c1bd16..72e809f31a15 100644 --- a/src/riscv/lib/src/state_backend/trans.rs +++ b/src/riscv/lib/src/state_backend/trans.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -use super::{EnrichedValue, ManagerBase}; +use super::ManagerBase; /// Transformation from a manager `I` to another manager pub trait FnManager { @@ -18,11 +18,6 @@ pub trait FnManager { fn map_dyn_region( input: I::DynRegion, ) -> ::DynRegion; - - /// Transform the enriched cell of manager `I` to one of manager `O`. - fn map_enriched_cell( - input: I::EnrichedCell, - ) -> ::EnrichedCell; } /// Identity transformation for [`FnManager`] @@ -38,10 +33,4 @@ impl FnManager for FnManagerIdent { fn map_dyn_region(input: M::DynRegion) -> M::DynRegion { input } - - fn map_enriched_cell( - input: ::EnrichedCell, - ) -> ::EnrichedCell { - input - } } diff --git a/src/riscv/lib/src/state_backend/verify_backend.rs b/src/riscv/lib/src/state_backend/verify_backend.rs index 41204f331423..826d3aac3107 100644 --- a/src/riscv/lib/src/state_backend/verify_backend.rs +++ b/src/riscv/lib/src/state_backend/verify_backend.rs @@ -55,6 +55,16 @@ impl ManagerBase for Verifier { type EnrichedCell = EnrichedCell; type ManagerRoot = Self; + + fn enrich_cell>( + underlying: Self::Region, + ) -> Self::EnrichedCell { + EnrichedCell { underlying } + } + + fn as_devalued_cell(cell: &Self::EnrichedCell) -> &Self::Region { + &cell.underlying + } } impl ManagerRead for Verifier { @@ -136,10 +146,7 @@ impl ManagerRead for Verifier { where V: EnrichedValue, { - match cell { - EnrichedCell::Absent => not_found(), - EnrichedCell::Present(cell) => cell, - } + Self::region_ref(&cell.underlying, 0) } } @@ -205,10 +212,7 @@ impl ManagerWrite for Verifier { where V: super::EnrichedValueLinked, { - match cell { - EnrichedCell::Absent => *cell = EnrichedCell::Present(value), - EnrichedCell::Present(stored) => *stored = value, - } + Self::region_write(&mut cell.underlying, 0, value); } } @@ -458,9 +462,8 @@ impl Default for DynRegion { - Absent, - Present(V::E), +pub struct EnrichedCell { + underlying: Region, } impl Cell { @@ -482,9 +485,8 @@ where V::E: Clone, { fn clone(&self) -> Self { - match self { - Self::Absent => Self::Absent, - Self::Present(value) => Self::Present(value.clone()), + Self { + underlying: self.underlying.clone(), } } } @@ -584,13 +586,14 @@ mod tests { } /// Construct a [`state_backend::EnrichedCell`] from a proptest value. - fn arb_to_enriched_cell( + fn arb_to_enriched_cell>( value: Option, ) -> state_backend::EnrichedCell { let cell = match value { - Some(value) => EnrichedCell::Present(value), - None => EnrichedCell::Absent, + Some(value) => Region::Partial(Box::new([Some(value)])), + None => Region::Absent, }; + let cell = Cell::bind(cell); state_backend::EnrichedCell::bind(cell) } -- GitLab From 35838c132e4aa2357e5e634f3bfa3c5a46b660c2 Mon Sep 17 00:00:00 2001 From: Kurtis Charnock Date: Fri, 14 Feb 2025 10:14:21 +0000 Subject: [PATCH 16/31] doc: correct Rust syntax error in example --- docs/shell/smart_rollup_node.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/shell/smart_rollup_node.rst b/docs/shell/smart_rollup_node.rst index 488d3e885705..fba7806d9f83 100644 --- a/docs/shell/smart_rollup_node.rst +++ b/docs/shell/smart_rollup_node.rst @@ -1525,7 +1525,7 @@ Define these functions in the ``lib.rs`` as follows: mod host; use crate::host::read_input; - use crate::host:ReadInputMessageInfo; + use crate::host::ReadInputMessageInfo; pub const MAX_MESSAGE_SIZE: u32 = 4096u32; -- GitLab From fb0f54f74910750690f5c0988cf3acae876445d3 Mon Sep 17 00:00:00 2001 From: Nic Volanschi Date: Tue, 18 Feb 2025 15:10:39 +0100 Subject: [PATCH 17/31] doc: update links to tezos.gitlab.io --- CHANGES.rst | 2 +- README.md | 18 +++++------ docs/alpha/adaptive_issuance.rst | 2 +- docs/alpha/dal_support.rst | 1 + docs/developer/error_monad_p3_advanced.rst | 3 +- docs/developer/images/rpc.excalidraw | 4 +-- docs/developer/python_environment.rst | 2 +- docs/developer/rollup_metrics.csv | 35 ++++++++++++++++++++++ docs/protocols/020_paris.rst | 11 ++++--- docs/protocols/021_quebec.rst | 2 +- docs/quebec/adaptive_issuance.rst | 2 +- docs/releases/version-20.rst | 2 +- docs/rio/adaptive_issuance.rst | 2 +- docs/scripts/extract_content | 2 +- docs/shell/storage.rst | 10 +++---- 15 files changed, 66 insertions(+), 32 deletions(-) create mode 100644 docs/developer/rollup_metrics.csv diff --git a/CHANGES.rst b/CHANGES.rst index ccf6be7ffc23..a16ebcdaff6f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,7 @@ Development Changelog ''''''''''''''''''''' -**NB:** The changelog for releases can be found at: https://tezos.gitlab.io/CHANGES.html +**NB:** The changelog for releases can be found at: https://octez.tezos.com/docs/CHANGES.html This file lists the changes added to each version of octez-node, diff --git a/README.md b/README.md index 6c20216b90b2..29192061024b 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,11 @@ For more information about the project, see https://tezos.com. ## Getting started Instructions to -[install](https://tezos.gitlab.io/introduction/howtoget.html), [start -using](https://tezos.gitlab.io/introduction/howtouse.html), and +[install](https://octez.tezos.com/docs/introduction/howtoget.html), [start +using](https://octez.tezos.com/docs/introduction/howtouse.html), and [taking part in the -consensus](https://tezos.gitlab.io/introduction/howtorun.html) are -available at https://tezos.gitlab.io/. +consensus](https://octez.tezos.com/docs/introduction/howtorun.html) are +available at https://octez.tezos.com/docs/. ## The Tezos software @@ -28,11 +28,11 @@ In more detail, this git repository contains: The Tezos software may run either on the nodes of the main Tezos network (mainnet) or on [various Tezos test -networks](https://tezos.gitlab.io/introduction/test_networks.html). +networks](https://octez.tezos.com/docs/introduction/test_networks.html). The documentation for developers, including developers of the Tezos software and developer of Tezos applications and tools, is available -online at https://tezos.gitlab.io/. This documentation is always in +online at https://octez.tezos.com/docs/. This documentation is always in sync with the master branch which may however be slightly desynchronized with the code running on the live networks. @@ -47,7 +47,7 @@ All development of the Tezos code happens on GitLab at https://gitlab.com/tezos/tezos. Merge requests (https://gitlab.com/tezos/tezos/-/merge_requests) should usually target the `master` branch; see [the contribution -instructions](https://tezos.gitlab.io/developer/contributing.html). +instructions](https://octez.tezos.com/docs/developer/contributing.html). The issue tracker at https://gitlab.com/tezos/tezos/issues can be used to report bugs and to request new simple features. The [Tezos Agora @@ -71,11 +71,11 @@ The core of the Tezos software that implements the economic ruleset is called the *protocol*. Unlike the rest of the source code, updates to the protocol must be further adopted through the [Tezos on-chain voting -procedure](https://tezos.gitlab.io/whitedoc/voting.html). Protocol +procedure](https://octez.tezos.com/docs/active/voting.html). Protocol contributors are encouraged to synchronize their contributions to minimize the number of protocol proposals that the stakeholders have to study and to maximize the throughput of the voting procedure. ## Community -Links to community websites are gathered at . +Links to community websites are gathered at . diff --git a/docs/alpha/adaptive_issuance.rst b/docs/alpha/adaptive_issuance.rst index 708d3f6d99ad..47ce7d3309b5 100644 --- a/docs/alpha/adaptive_issuance.rst +++ b/docs/alpha/adaptive_issuance.rst @@ -433,7 +433,7 @@ Where: The `RPC endpoint -`__, +`__, ``/issuance/expected_issuance`` reports the precomputed values of all participation rewards for the provided block and the next ``issuance_modification_delay`` cycles. diff --git a/docs/alpha/dal_support.rst b/docs/alpha/dal_support.rst index adf1d28f0e0c..ab89063c65ae 100644 --- a/docs/alpha/dal_support.rst +++ b/docs/alpha/dal_support.rst @@ -148,6 +148,7 @@ Penalties A baker that is correctly accused, through an accusation operation included in a block, loses their DAL rewards for the cycle containing the block. +.. _daL_rollups_integration: Smart Rollups integration ========================= diff --git a/docs/developer/error_monad_p3_advanced.rst b/docs/developer/error_monad_p3_advanced.rst index 2a46f600198a..00c5eb479306 100644 --- a/docs/developer/error_monad_p3_advanced.rst +++ b/docs/developer/error_monad_p3_advanced.rst @@ -213,8 +213,7 @@ you are registering: - ``Permanent``: is for irremediable failures, i.e., failures that happen and will always happen whatever the context. E.g., - `originating a - contract `__ + :ref:`originating a contract ` that does not type-check is a permanent error. This is used by the shell to mark the data as invalid. diff --git a/docs/developer/images/rpc.excalidraw b/docs/developer/images/rpc.excalidraw index 741e896b2a2b..ec8ba50a2f09 100644 --- a/docs/developer/images/rpc.excalidraw +++ b/docs/developer/images/rpc.excalidraw @@ -2094,11 +2094,11 @@ "locked": false, "fontSize": 28, "fontFamily": 3, - "text": "Images used in https://tezos.gitlab.io/developer/rpc.html", + "text": "Images used in https://octez.tezos.com/docsab.io/developer/rpc.html", "textAlign": "left", "verticalAlign": "top", "containerId": null, - "originalText": "Images used in https://tezos.gitlab.io/developer/rpc.html", + "originalText": "Images used in https://octez.tezos.com/docsab.io/developer/rpc.html", "autoResize": true, "lineHeight": 1.2 }, diff --git a/docs/developer/python_environment.rst b/docs/developer/python_environment.rst index 96110e8a36fa..89b973a2ddab 100644 --- a/docs/developer/python_environment.rst +++ b/docs/developer/python_environment.rst @@ -1,7 +1,7 @@ Python Environment ================== -Octez uses Python to build this documentation website (:doc:`https://tezos.gitlab.io <../index>`) and for a limited set of +Octez uses Python to build this documentation website (:doc:`https://octez.tezos.com/docs/ <../index>`) and for a limited set of utility scripts (although this latter usage of Python is deprecated). This page contains installation instructions for the Python environment. diff --git a/docs/developer/rollup_metrics.csv b/docs/developer/rollup_metrics.csv new file mode 100644 index 000000000000..f37ff059bc1f --- /dev/null +++ b/docs/developer/rollup_metrics.csv @@ -0,0 +1,35 @@ +Name,Type,Description,Labels +octez_sc_rollup_node_batcher_batches_size,Gauge,"Batcher batches sent", +octez_sc_rollup_node_batcher_get_time,Gauge,"Time to fetch batches", +octez_sc_rollup_node_batcher_inject_time,Gauge,"Time to inject batches", +octez_sc_rollup_node_batcher_last_batch_level,Gauge,"Last batch level", +octez_sc_rollup_node_batcher_last_batch_time,Gauge,"Last batch time", +octez_sc_rollup_node_batcher_message_queue_size,Gauge,"Batcher message queue size", +octez_sc_rollup_node_batcher_messages_size,Gauge,"Batcher messages size in batches", +octez_sc_rollup_node_block_timeout,Gauge,"Number of block before player timeout",opponent;start_level +octez_sc_rollup_node_challenge_window,Gauge,"Current challenge window", +octez_sc_rollup_node_commitment_period,Gauge,"Current commitment period", +octez_sc_rollup_node_dal_attestation_lag,Gauge,"DAL attestation lag", +octez_sc_rollup_node_dal_batcher_queue_length,Gauge,"Number of messages waiting for publication on the DAL", +octez_sc_rollup_node_dal_enabled,Gauge,"DAL enabled in protocol", +octez_sc_rollup_node_dal_injections_queue_length,Gauge,"Number of recently published DAL slots, who have not yet been forgotten", +octez_sc_rollup_node_dal_number_of_slots,Gauge,"DAL number of slots", +octez_sc_rollup_node_gc_oldest_available_level,Gauge,"Oldest Available Level after GC", +octez_sc_rollup_node_gc_process_time,Gauge,"GC processing time", +octez_sc_rollup_node_inbox_external_messages_number,Gauge,"Number of external messages in inbox", +octez_sc_rollup_node_inbox_fetch_time,Gauge,"The time the rollup node spent fetching the inbox", +octez_sc_rollup_node_inbox_internal_messages_number,Gauge,"Number of internal messages in inbox", +octez_sc_rollup_node_inbox_level,Gauge,"Level of last inbox", +octez_sc_rollup_node_inbox_process_time,Gauge,"The time the rollup node spent processing the head", +octez_sc_rollup_node_inbox_total_time,Gauge,"The total time the rollup node spent handling the inbox", +octez_sc_rollup_node_lcc_level_l1,Gauge,"Last cemented commitment level on L1", +octez_sc_rollup_node_lcc_level_local,Gauge,"Last cemented commitment level locally", +octez_sc_rollup_node_lpc_level_l1,Gauge,"Last published commitment on L1", +octez_sc_rollup_node_lpc_level_local,Gauge,"Last published commitment by operator", +octez_sc_rollup_node_node_info,Counter,"General information on the node",version;commit_hash;commit_date +octez_sc_rollup_node_number_of_conflicts,Gauge,"Number of conflicts", +octez_sc_rollup_node_protocol,Counter,"Rollup node current protocol",protocol +octez_sc_rollup_node_rollup_node_info,Counter,"Rollup node info",rollup_address;mode;genesis_level;genesis_hash;pvm_kind;history_mode +octez_sc_rollup_node_state_of_refutation_game,Gauge,"State of refutation game",opponent;start_level + + diff --git a/docs/protocols/020_paris.rst b/docs/protocols/020_paris.rst index 656559a4aa19..1a604eb547b1 100644 --- a/docs/protocols/020_paris.rst +++ b/docs/protocols/020_paris.rst @@ -38,8 +38,8 @@ Data Availability Layer network. Do note that this operation itself does not contain the data themselves. Instead, it contains the slot index, the commitment over the data, and a proof regarding the size for those data. The data must be sent using the - DAL node. For more information on how it works, please read the `DAL - documentation `_. + DAL node. For more information on how it works, please read the :doc:`DAL + documentation <../shell/dal>`. - Optimize the DAL commitment publication operation by memoizing the cryptobox. (MR :gl:`!11594`) @@ -50,8 +50,7 @@ Data Availability Layer enables kernel to read the DAL parameters. This should ease the writing of smart rollups kernels to make them generic over the values of those parameters. For more information on how it works, - please read the `DAL smart rollup integration - `_. + please read :ref:`dal_rollups_integration`. Adaptive Issuance @@ -117,8 +116,8 @@ Breaking Changes - A new ``dal_attestation`` field has been added to the ``block_metadata`` indicating the attested slots. The slots being attested are the slots that were published ``attestation_lag`` levels - ago (MRs :gl:`!11903`, :gl:`!12063`) (see `DAL documentation - `_ for more context). + ago (MRs :gl:`!11903`, :gl:`!12063`) (see :doc:`DAL documentation + <../shell/dal>` for more context). - The protocol no longer relies on stake snapshots to compute rights. Instead: diff --git a/docs/protocols/021_quebec.rst b/docs/protocols/021_quebec.rst index 8b2be0a6e5b6..5424402bff65 100644 --- a/docs/protocols/021_quebec.rst +++ b/docs/protocols/021_quebec.rst @@ -89,7 +89,7 @@ RPC Changes - In the following paths, ``../`` is short for ``/chains//blocks//context/delegates//``. - See https://tezos.gitlab.io/quebec/baking_power.html for more + See :doc:`../active/baking_power` for more details on the renamed and new RPCs. * Renamed RPC ``GET ../current_frozen_deposits`` to ``GET diff --git a/docs/quebec/adaptive_issuance.rst b/docs/quebec/adaptive_issuance.rst index a0dbdc9579e3..d7d09cc16b4f 100644 --- a/docs/quebec/adaptive_issuance.rst +++ b/docs/quebec/adaptive_issuance.rst @@ -437,7 +437,7 @@ Where: The `RPC endpoint -`__, +`__, ``/issuance/expected_issuance`` reports the precomputed values of all participation rewards for the provided block and the next ``consensus_rights_delay`` cycles. diff --git a/docs/releases/version-20.rst b/docs/releases/version-20.rst index ca748609e347..07da9fefcdfc 100644 --- a/docs/releases/version-20.rst +++ b/docs/releases/version-20.rst @@ -54,7 +54,7 @@ Some deprecated RPCs have been removed. Please check `the changelog <../CHANGES. The RPC ``/health/ready`` has been introduced to get the status of the RPC server. -Starting from Octez v20, we strongly advise nodes and bakers operators to synchronise their clocks using NTP as issued in `the node section of getting started `__. +Starting from Octez v20, we strongly advise nodes and bakers operators to synchronise their clocks using NTP as issued in :ref:`start_node`. Smart rollup ~~~~~~~~~~~~ diff --git a/docs/rio/adaptive_issuance.rst b/docs/rio/adaptive_issuance.rst index 8f728cce48e0..8463021a09f7 100644 --- a/docs/rio/adaptive_issuance.rst +++ b/docs/rio/adaptive_issuance.rst @@ -433,7 +433,7 @@ Where: The `RPC endpoint -`__, +`__, ``/issuance/expected_issuance`` reports the precomputed values of all participation rewards for the provided block and the next ``issuance_modification_delay`` cycles. diff --git a/docs/scripts/extract_content b/docs/scripts/extract_content index 9b5cf98a8972..3e5fe42364f8 100755 --- a/docs/scripts/extract_content +++ b/docs/scripts/extract_content @@ -26,7 +26,7 @@ if ($0 !~ /