diff --git a/.gitlab/ci/templates.yml b/.gitlab/ci/templates.yml index ce70230152e731906362b9b6e848bd8a8a37308f..ef475f94bec85df0a5cbdb7346262c4f01bf5dfb 100644 --- a/.gitlab/ci/templates.yml +++ b/.gitlab/ci/templates.yml @@ -2,7 +2,7 @@ variables: # /!\ CI_REGISTRY is overriden to use a private Docker registry mirror in AWS ECR # in GitLab namespaces `nomadic-labs` and `tezos` ## This value MUST be the same as `opam_repository_tag` in `scripts/version.sh` - build_deps_image_version: 418eefa51c9ed24b495f76dc52fa1eb30ab6ce65 + build_deps_image_version: 268fef41228dab880a2d4446a3d421ca9b9e03e3 build_deps_image_name: "${CI_REGISTRY}/tezos/opam-repository" GIT_STRATEGY: fetch GIT_DEPTH: "1" diff --git a/CHANGES.rst b/CHANGES.rst index cae2f57145b3e36b653d7d9ceaa4e73498768932..7f8f68ec51e7d3984bb1334e0a62e82ac64c52ea 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -71,6 +71,15 @@ Node - Added metrics to track the pending requests of chain validator, block validator and prevalidator workers. +- **Breaking change**: The node context storage format was + upgraded. To this end, a new storage version was introduced: 1.0 + (previously 0.8). Backward compatibility is preserved: upgrading + from 0.6, 0.7 (Octez 12.x) or 0.8 (Octez 13.0) is done through the + ``tezos-node upgrade storage`` command. This upgrade is + instantaneous. However, be careful that there is no forward + compatibility: previous versions of Octez will refuse to run on an + upgraded data directory. + Client ------ diff --git a/manifest/main.ml b/manifest/main.ml index b07bee7fbcb3572c5016d288765bd32e6551f288..c407d36ae4b381b95db3a6d13bd7033c47f119a3 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -174,10 +174,10 @@ let ipaddr = let ipaddr_unix = external_sublib ipaddr "ipaddr.unix" -let irmin = external_lib "irmin" V.(at_least "3.2.2" && less_than "3.3.0") +let irmin = external_lib "irmin" V.(at_least "3.3.1" && less_than "3.4.0") let irmin_pack = - external_lib "irmin-pack" V.(at_least "3.2.2" && less_than "3.3.0") + external_lib "irmin-pack" V.(at_least "3.3.1" && less_than "3.4.0") let irmin_pack_unix = external_sublib irmin_pack "irmin-pack.unix" diff --git a/opam/tezos-context.opam b/opam/tezos-context.opam index c37e208c8a228730ba9292df60fca847d5c9d183..4db2365d64f57cbd63fa1103ac221a0b0f79c362 100644 --- a/opam/tezos-context.opam +++ b/opam/tezos-context.opam @@ -12,8 +12,8 @@ depends: [ "tezos-base" "tezos-stdlib" "tezos-shell-services" - "irmin" { >= "3.2.2" & < "3.3.0" } - "irmin-pack" { >= "3.2.2" & < "3.3.0" } + "irmin" { >= "3.3.1" & < "3.4.0" } + "irmin-pack" { >= "3.3.1" & < "3.4.0" } "tezos-stdlib-unix" "fmt" { >= "0.8.7" } "bigstringaf" { >= "0.2.0" } diff --git a/opam/tezos-sc-rollup-node-013-PtJakart.opam b/opam/tezos-sc-rollup-node-013-PtJakart.opam index 94a02df3d8c2226b564169c2a07912a697d5adc2..da4ef95a5418740f0a20a5758b3735ea939ac9d0 100644 --- a/opam/tezos-sc-rollup-node-013-PtJakart.opam +++ b/opam/tezos-sc-rollup-node-013-PtJakart.opam @@ -24,8 +24,8 @@ depends: [ "tezos-shell-services" "tezos-sc-rollup-013-PtJakart" "data-encoding" { >= "0.5.3" & < "0.6" } - "irmin-pack" { >= "3.2.2" & < "3.3.0" } - "irmin" { >= "3.2.2" & < "3.3.0" } + "irmin-pack" { >= "3.3.1" & < "3.4.0" } + "irmin" { >= "3.3.1" & < "3.4.0" } "ringo" { >= "0.9" } "ringo-lwt" { >= "0.9" } "tezos-injector-013-PtJakart" diff --git a/opam/tezos-sc-rollup-node-alpha.opam b/opam/tezos-sc-rollup-node-alpha.opam index c180872d39b81fe9570a498840b1e14fc6ae0e65..f81cfc11a17751c98606c6c215ca2b382911d69b 100644 --- a/opam/tezos-sc-rollup-node-alpha.opam +++ b/opam/tezos-sc-rollup-node-alpha.opam @@ -24,8 +24,8 @@ depends: [ "tezos-shell-services" "tezos-sc-rollup-alpha" "data-encoding" { >= "0.5.3" & < "0.6" } - "irmin-pack" { >= "3.2.2" & < "3.3.0" } - "irmin" { >= "3.2.2" & < "3.3.0" } + "irmin-pack" { >= "3.3.1" & < "3.4.0" } + "irmin" { >= "3.3.1" & < "3.4.0" } "ringo" { >= "0.9" } "ringo-lwt" { >= "0.9" } "tezos-injector-alpha" diff --git a/opam/tezos-store.opam b/opam/tezos-store.opam index fdf36de0fb85c10be142a6e38449fe841314678e..51e232b9d924121a502c0c408b02bfb4cab89af4 100644 --- a/opam/tezos-store.opam +++ b/opam/tezos-store.opam @@ -13,7 +13,7 @@ depends: [ "tezos-base" "tezos-version" "index" { >= "1.6.0" & < "1.7.0" } - "irmin-pack" { >= "3.2.2" & < "3.3.0" } + "irmin-pack" { >= "3.3.1" & < "3.4.0" } "tezos-context" "tezos-shell-context" "tezos-validation" diff --git a/scripts/version.sh b/scripts/version.sh index 7b44c34534b14812e36c64fd200c31cf5a792fbc..434672c8e430dc10ee184904233096da9da52421 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -20,12 +20,12 @@ export recommended_node_version=14.12.0 ## full_opam_repository is a commit hash of the public OPAM repository, i.e. ## https://github.com/ocaml/opam-repository -export full_opam_repository_tag=11fd9cf290f6549b89cbaac9ba0085813c58c035 +export full_opam_repository_tag=6edc78d95cd1fa558d4cecaae906f286ad1f8881 ## opam_repository is an additional, tezos-specific opam repository. ## This value MUST be the same as `build_deps_image_version` in `.gitlab/ci/templates.yml export opam_repository_url=https://gitlab.com/tezos/opam-repository -export opam_repository_tag=418eefa51c9ed24b495f76dc52fa1eb30ab6ce65 +export opam_repository_tag=268fef41228dab880a2d4446a3d421ca9b9e03e3 export opam_repository_git=$opam_repository_url.git export opam_repository=$opam_repository_git\#$opam_repository_tag diff --git a/src/bin_node/node_data_version.ml b/src/bin_node/node_data_version.ml index 6c2cc507c4769451edbb4334f8be5e9ea55dad21..71abef6b73c02337ac0b62eca1b46b7bce780008 100644 --- a/src/bin_node/node_data_version.ml +++ b/src/bin_node/node_data_version.ml @@ -95,17 +95,20 @@ end * - (0.)0.5 : never released (but used in 10.0~rc1 and 10.0~rc2) * - (0.)0.6 : store upgrade (switching from LMDB) * - (0.)0.7 : new store metadata representation - * - (0.)0.8 : context upgrade (upgrade to irmin.3.0) *) + * - (0.)0.8 : context upgrade (upgrade to irmin.3.0) + * - 1.0 : context upgrade (upgrade to irmin.3.3) *) (* FIXME https://gitlab.com/tezos/tezos/-/issues/2861 We should enable the semantic versioning instead of applying hardcoded rules.*) -let current_version = Version.make ~major:0 ~minor:8 +let current_version = Version.make ~major:1 ~minor:0 let v_0_6 = Version.make ~major:0 ~minor:6 let v_0_7 = Version.make ~major:0 ~minor:7 +let v_0_8 = Version.make ~major:0 ~minor:8 + (* List of upgrade functions from each still supported previous version to the current [data_version] above. If this list grows too much, an idea would be to have triples (version, version, @@ -113,9 +116,24 @@ let v_0_7 = Version.make ~major:0 ~minor:7 statically. *) let upgradable_data_version = let open Lwt_result_syntax in + let v_1_0_upgrade ~data_dir = + let context_root = context_dir data_dir in + (* The upgrade function consist in letting irmin doing its own + file renaming. To do so, it must be done using a RW instance.*) + let*! ctxt = Context.init ~readonly:false context_root in + let*! () = Context.close ctxt in + return_unit + in [ - (v_0_6, fun ~data_dir:_ _ ~chain_name:_ ~sandbox_parameters:_ -> return_unit); - (v_0_7, fun ~data_dir:_ _ ~chain_name:_ ~sandbox_parameters:_ -> return_unit); + ( v_0_6, + fun ~data_dir _ ~chain_name:_ ~sandbox_parameters:_ -> + v_1_0_upgrade ~data_dir ); + ( v_0_7, + fun ~data_dir _ ~chain_name:_ ~sandbox_parameters:_ -> + v_1_0_upgrade ~data_dir ); + ( v_0_8, + fun ~data_dir _ ~chain_name:_ ~sandbox_parameters:_ -> + v_1_0_upgrade ~data_dir ); ] type error += Invalid_data_dir_version of Version.t * Version.t @@ -390,12 +408,6 @@ let ensure_data_dir ?(bare = false) data_dir = let* o = ensure_data_dir bare data_dir in match o with | None -> return_unit - (* Here, we enable the automatic upgrade from "0.6" and "0.7" to - "0.8". This should be removed as soon as the "0.8" version or - above is mandatory. *) - | Some (v, _upgrade) when Version.(equal v_0_6 v) || Version.(equal v_0_7 v) - -> - upgrade_data_dir ~data_dir () ~chain_name:() ~sandbox_parameters:() | Some (version, _) -> tzfail (Data_dir_needs_upgrade {expected = current_version; actual = version}) diff --git a/src/bin_node/node_storage_command.ml b/src/bin_node/node_storage_command.ml index baffb9931f00926e207dc1da09d3c8a7875442a8..cc35fd437f819e73fca206fdbf60b9dda2a6a0d7 100644 --- a/src/bin_node/node_storage_command.ml +++ b/src/bin_node/node_storage_command.ml @@ -65,7 +65,7 @@ module Term = struct (Node_data_version.Invalid_data_dir {data_dir = context_dir; msg = None}) else - let pack = context_dir // "store.pack" in + let pack = context_dir // "store.0.suffix" in let*! b = Lwt_unix.file_exists pack in if not b then tzfail diff --git a/src/lib_context/disk/context.ml b/src/lib_context/disk/context.ml index a13ffce340635d71ed3b9ac142806b1574edcbcf..0a0d7625323dc2e53a66610626ff8ca93b1d983d 100644 --- a/src/lib_context/disk/context.ml +++ b/src/lib_context/disk/context.ml @@ -185,11 +185,11 @@ module Indexing_strategy : sig val get : unit -> t - type irmin_t := Irmin_pack.Pack_store.Indexing_strategy.t + type irmin_t := Irmin_pack.Indexing_strategy.t val to_irmin : t -> irmin_t end = struct - module I = Irmin_pack.Pack_store.Indexing_strategy + module I = Irmin_pack.Indexing_strategy let singleton = ref `Minimal @@ -317,7 +317,7 @@ module Make (Encoding : module type of Tezos_context_encoding.Context) = struct ["predecessor_ops_metadata_hash"] let sync index = - if index.readonly then Store.sync index.repo ; + if index.readonly then Store.reload index.repo ; Lwt.return () let exists index key = @@ -1015,7 +1015,7 @@ module Make (Encoding : module type of Tezos_context_encoding.Context) = struct let* key = Snapshot.Import.save_elt import snapshot in Store.Tree.of_key idx.repo (`Node key) - let close_import import = Snapshot.Import.close import + let close_import import index = Snapshot.Import.close import index.repo let make_context index = empty index diff --git a/src/lib_context/dump/context_dump.ml b/src/lib_context/dump/context_dump.ml index c02ffb535f7c27e4923705a2525ae746a1444842..ea494beabf95682db54e4000a3149ba7fc9af948 100644 --- a/src/lib_context/dump/context_dump.ml +++ b/src/lib_context/dump/context_dump.ml @@ -539,7 +539,7 @@ module Make (I : Dump_interface) = struct let* e = get_command rbuf in match e with | Eof -> - I.close_import import_t ; + I.close_import import_t index ; return_unit | _ -> tzfail Inconsistent_context_dump in diff --git a/src/lib_context/dump/context_dump_intf.ml b/src/lib_context/dump/context_dump_intf.ml index f441ecc2ef005ddaff320b9ead339a3f4ab8f145..77dd9d436f81293fa6f2a6b68a2856e0de3e449f 100644 --- a/src/lib_context/dump/context_dump_intf.ml +++ b/src/lib_context/dump/context_dump_intf.ml @@ -174,7 +174,7 @@ module type Dump_interface = sig val save_inode : index -> import -> Snapshot.t -> tree option Lwt.t (** [close importer] close the [importer] instance.*) - val close_import : import -> unit + val close_import : import -> index -> unit end module type S = sig diff --git a/src/lib_shell/test/test_node.ml b/src/lib_shell/test/test_node.ml index 49c6eb9fc1adaf951c0058c7168a57b8f5efaf99..a1c894d940c5835b8ed9c2ce94dbaaa1f31e2903 100644 --- a/src/lib_shell/test/test_node.ml +++ b/src/lib_shell/test/test_node.ml @@ -41,6 +41,7 @@ let init_config (* (f : 'a -> unit -> unit Lwt.t) *) f test_dir switch () : unit Lwt.t = let sandbox_parameters : Data_encoding.json = `Null in let config : Node.config = + let open Filename.Infix in { genesis = Shell_test_helpers.genesis; chain_name = Distributed_db_version.Name.zero; @@ -49,9 +50,9 @@ let init_config (* (f : 'a -> unit -> unit Lwt.t) *) f test_dir switch () : user_activated_protocol_overrides = []; operation_metadata_size_limit = None; data_dir = test_dir; - store_root = test_dir; - context_root = test_dir; - protocol_root = test_dir; + store_root = test_dir // "store"; + context_root = test_dir // "context"; + protocol_root = test_dir // "protocol"; patch_context = None; p2p = None; target = None; diff --git a/src/lib_store/test/test_utils.ml b/src/lib_store/test/test_utils.ml index 8e3a12480617d658a8752d48847bd85b1ac63dbf..367cacd6ce74049c3cd2c0043337f3a4c947b63c 100644 --- a/src/lib_store/test/test_utils.ml +++ b/src/lib_store/test/test_utils.ml @@ -287,7 +287,16 @@ let wrap_simple_store_init ?(patch_context = dummy_patch_context) Lwt.finalize (fun () -> k (store_dir, context_dir) store) (fun () -> - let*! _ = Store.close_store store in + let*! () = + Lwt.catch + (fun () -> Store.close_store store) + (fun _ -> + (* FIXME https://gitlab.com/tezos/tezos/-/issues/3305 + + Avoid to re-raise Pack_error: "Double_close" + when closing the Context multiple times.*) + Lwt.return_unit) + in Lwt.return_unit))) in match r with diff --git a/src/proto_alpha/bin_sc_rollup_node/configuration.ml b/src/proto_alpha/bin_sc_rollup_node/configuration.ml index af7f25564bbd73ad9522c1b9a1f5256e46146c32..5dad522a9eb5cd4a11b10d0329b54385196547f6 100644 --- a/src/proto_alpha/bin_sc_rollup_node/configuration.ml +++ b/src/proto_alpha/bin_sc_rollup_node/configuration.ml @@ -38,6 +38,10 @@ type t = { let default_data_dir = Filename.concat (Sys.getenv "HOME") ".tezos-sc-rollup-node" +let storage_dir = "storage" + +let default_storage_dir data_dir = Filename.concat data_dir storage_dir + let relative_filename data_dir = Filename.concat data_dir "config.json" let filename config = relative_filename config.data_dir diff --git a/src/proto_alpha/bin_sc_rollup_node/configuration.mli b/src/proto_alpha/bin_sc_rollup_node/configuration.mli index 23bb4576d681ad4a6bbe55911e573a3bd2f29692..5c5c41e6b069c86b5aaa6c41c028806c885380ae 100644 --- a/src/proto_alpha/bin_sc_rollup_node/configuration.mli +++ b/src/proto_alpha/bin_sc_rollup_node/configuration.mli @@ -36,6 +36,10 @@ type t = { (** [default_data_dir] is the default value for [data_dir]. *) val default_data_dir : string +(** [default_storage_dir] returns the default value of the storage dir + given a [data_dir]. *) +val default_storage_dir : string -> string + (** [default_rpc_addr] is the default value for [rpc_addr]. *) val default_rpc_addr : string diff --git a/src/proto_alpha/bin_sc_rollup_node/store.ml b/src/proto_alpha/bin_sc_rollup_node/store.ml index 16e30f0ac0daeafed8069ce5733d03bc2b821126..0454490ca46f3b9529a7c98e4d4bc434a966b61b 100644 --- a/src/proto_alpha/bin_sc_rollup_node/store.ml +++ b/src/proto_alpha/bin_sc_rollup_node/store.ml @@ -40,7 +40,10 @@ type path = string list let load configuration = let open Lwt_syntax in let open Configuration in - let* repo = IStore.Repo.v (Irmin_pack.config configuration.data_dir) in + let* repo = + IStore.Repo.v + (Irmin_pack.config (default_storage_dir configuration.data_dir)) + in IStore.main repo let flush store = IStore.flush (IStore.repo store) diff --git a/src/proto_alpha/lib_protocol/sc_rollup_outbox_message_repr.ml b/src/proto_alpha/lib_protocol/sc_rollup_outbox_message_repr.ml index ab6e68e2ed5d2bf96b7e5c3269d0ca34786339ba..609b4c8cd60494032252939278a6627981275823 100644 --- a/src/proto_alpha/lib_protocol/sc_rollup_outbox_message_repr.ml +++ b/src/proto_alpha/lib_protocol/sc_rollup_outbox_message_repr.ml @@ -83,14 +83,19 @@ let encoding = (fun transactions -> Atomic_transaction_batch {transactions}) (obj1 (req "transactions" (list transaction_encoding)))) -let pp_transaction fmt {destination; entrypoint; unparsed_parameters = _} = +let pp_transaction fmt {destination; entrypoint; unparsed_parameters} = + let json = + Data_encoding.Json.construct Script_repr.expr_encoding unparsed_parameters + in Format.fprintf fmt - "@[%a@;%a@]" + "@[%a@;%a@;%a@]" Contract_hash.pp destination Entrypoint_repr.pp entrypoint + Data_encoding.Json.pp + json let pp fmt (Atomic_transaction_batch {transactions}) = Format.pp_print_list diff --git a/src/proto_alpha/lib_protocol/test/unit/test_sc_rollup_arith.ml b/src/proto_alpha/lib_protocol/test/unit/test_sc_rollup_arith.ml index 106b7e319e5598d535f7fa9b25e58a6f3116a45d..652c3b942fd6880d52991ee51e2a958d50641787 100644 --- a/src/proto_alpha/lib_protocol/test/unit/test_sc_rollup_arith.ml +++ b/src/proto_alpha/lib_protocol/test/unit/test_sc_rollup_arith.ml @@ -339,7 +339,13 @@ let test_output_messages_proofs ~valid ~inbox_level (source, expected_outputs) = let*! proof_is_valid = verify_output_proof proof in fail_when proof_is_valid - (Exn (Failure "A wrong output proof is valid.")) + (Exn + (Failure + (Format.asprintf + "A wrong output proof is valid: %s -> %a" + source + Sc_rollup_PVM_sem.pp_output + output))) | Error _ -> return () in List.iter_es check_output expected_outputs @@ -394,14 +400,14 @@ let test_invalid_output_messages () = ("1 out", [make_output ~outbox_level ~message_index:1 1]); ( "1 out 1 1 + out", [ - make_output ~outbox_level ~message_index:0 1; + make_output ~outbox_level ~message_index:0 0; make_output ~outbox_level ~message_index:3 2; ] ); ( "1 out 1 1 + out out", [ - make_output ~outbox_level ~message_index:0 1; - make_output ~outbox_level ~message_index:1 2; - make_output ~outbox_level ~message_index:2 3; + make_output ~outbox_level ~message_index:0 42; + make_output ~outbox_level ~message_index:1 32; + make_output ~outbox_level ~message_index:2 13; ] ); ] |> List.iter_es (test_output_messages_proofs ~valid:false ~inbox_level) diff --git a/tezt/tests/proxy_server_test.ml b/tezt/tests/proxy_server_test.ml index 014243eb698a129490b7ac8351fdfa16a85e8024..5f12006a06c62c7b97d43e125fb0d1064426efa7 100644 --- a/tezt/tests/proxy_server_test.ml +++ b/tezt/tests/proxy_server_test.ml @@ -214,7 +214,7 @@ let test_wrong_data_dir = let args = ["--data-dir"; wrong_data_dir] in let process = Proxy_server.spawn ~args node in let* stderr = Process.check_and_read_stderr ~expect_failure:true process in - let re_str = "error reading data-dir.*" in + let re_str = "No_such_file_or_directory" in let good_match = stderr =~ rex re_str in if not good_match then Test.fail