From b58e9c144da79002fdf4a784f51ecaa3b6b5b4d5 Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Tue, 22 Dec 2020 17:48:06 +0100 Subject: [PATCH 1/7] Storage: fix build for irmin 2.3.0 --- src/lib_storage/context.ml | 52 ++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/src/lib_storage/context.ml b/src/lib_storage/context.ml index 1ed1f1a3c351..1a1b2231a8a3 100644 --- a/src/lib_storage/context.ml +++ b/src/lib_storage/context.ml @@ -106,16 +106,25 @@ end = struct Error_monad.pp_print_error err)) - let short_hash t = Irmin.Type.(short_hash string (H.to_raw_string t)) + let short_hash_string = Irmin.Type.(unstage (short_hash string)) + + let short_hash_staged = + Irmin.Type.stage + @@ fun ?seed t -> short_hash_string ?seed (H.to_raw_string t) let t : t Irmin.Type.t = Irmin.Type.map - ~cli:(pp, of_string) + ~pp + ~of_string Irmin.Type.(string_of (`Fixed H.digest_size)) - ~short_hash + ~short_hash:short_hash_staged H.of_raw_string H.to_raw_string + let short_hash = + let f = short_hash_string ?seed:None in + fun t -> f (H.to_raw_string t) + let hash_size = H.digest_size let hash = H.digesti_string @@ -131,9 +140,13 @@ module Node = struct type entry = {kind : kind; name : M.step; node : Hash.t} + let s = Irmin.Type.(string_of `Int64) + + let pre_hash_v = Irmin.Type.(unstage (pre_hash s)) + (* Irmin 1.4 uses int64 to store string lengths *) let step_t = - let pre_hash = Irmin.Type.(pre_hash (string_of `Int64)) in + let pre_hash = Irmin.Type.(stage @@ fun x -> pre_hash_v x) in Irmin.Type.like M.step_t ~pre_hash let metadata_t = @@ -177,14 +190,16 @@ module Node = struct let import t = List.map import_entry (M.list t) - let pre_hash entries = Irmin.Type.pre_hash entries_t entries + let pre_hash_entries = Irmin.Type.(unstage (pre_hash entries_t)) + + let pre_hash entries = pre_hash_entries entries end include M let pre_hash_v1 x = V1.pre_hash (V1.import x) - let t = Irmin.Type.(like t ~pre_hash:pre_hash_v1) + let t = Irmin.Type.(like t ~pre_hash:(stage @@ fun x -> pre_hash_v1 x)) end module Commit = struct @@ -192,19 +207,23 @@ module Commit = struct module V1 = Irmin.Private.Commit.V1 (M) include M - let pre_hash_v1 t = Irmin.Type.pre_hash V1.t (V1.import t) + let pre_hash_v1_t = Irmin.Type.(unstage (pre_hash V1.t)) - let t = Irmin.Type.like t ~pre_hash:pre_hash_v1 + let pre_hash_v1 t = pre_hash_v1_t (V1.import t) + + let t = Irmin.Type.(like t ~pre_hash:(stage @@ fun x -> pre_hash_v1 x)) end module Contents = struct type t = string - let pre_hash_v1 x = - let ty = Irmin.Type.(pair (string_of `Int64) unit) in - Irmin.Type.(pre_hash ty) (x, ()) + let ty = Irmin.Type.(pair (string_of `Int64) unit) + + let pre_hash_ty = Irmin.Type.(unstage (pre_hash ty)) - let t = Irmin.Type.(like ~pre_hash:pre_hash_v1 string) + let pre_hash_v1 x = pre_hash_ty (x, ()) + + let t = Irmin.Type.(like string ~pre_hash:(stage @@ fun x -> pre_hash_v1 x)) let merge = Irmin.Merge.(idempotent (Irmin.Type.option t)) end @@ -216,12 +235,19 @@ module Conf = struct end module Store = - Irmin_pack.Make_ext (Conf) (Irmin.Metadata.None) (Contents) + Irmin_pack.Make_ext + (struct + let io_version = `V1 + end) + (Conf) + (Irmin.Metadata.None) + (Contents) (Irmin.Path.String_list) (Irmin.Branch.String) (Hash) (Node) (Commit) + module P = Store.Private type index = { -- GitLab From 298a1e656ee7aa649cfd22c4fbfc941bd1fe0832 Mon Sep 17 00:00:00 2001 From: Craig Ferguson Date: Tue, 12 Jan 2021 21:12:47 +0100 Subject: [PATCH 2/7] Storage: adapt to change to Tree.list in Irmin 2.3.0 --- src/lib_storage/context.ml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/lib_storage/context.ml b/src/lib_storage/context.ml index 1a1b2231a8a3..b8a60a1c4eb9 100644 --- a/src/lib_storage/context.ml +++ b/src/lib_storage/context.ml @@ -315,10 +315,10 @@ let unshallow context = P.Repo.batch context.index.repo (fun x y _ -> List.iter_s (fun (s, k) -> - match k with - | `Contents -> + match Store.Tree.destruct k with + | `Contents _ -> Lwt.return () - | `Node -> + | `Node _ -> Store.Tree.get_tree context.tree [s] >>= fun tree -> Store.save_tree ~clear:true context.index.repo x y tree @@ -398,12 +398,12 @@ let fold ctxt key ~init ~f = Store.Tree.list ctxt.tree (data_key key) >>= fun keys -> List.fold_left_s - (fun acc (name, kind) -> + (fun acc (name, t) -> let key = - match kind with - | `Contents -> + match Store.Tree.destruct t with + | `Contents _ -> `Key (key @ [name]) - | `Node -> + | `Node _ -> `Dir (key @ [name]) in f key acc) @@ -852,11 +852,14 @@ module Dumpable_context = struct >>= fun keys -> keys |> List.sort (fun (a, _) (b, _) -> String.compare a b) - |> List.map_s (fun (key, value_kind) -> - Store.Tree.get_tree tree [key] - >|= fun value -> - let value_hash = tree_hash value in - {key; value; value_kind; value_hash}) + |> List.map_s (fun (key, value) -> + Store.Tree.kind value [] + >|= function + | None -> + assert false (* The value must exist in the tree *) + | Some value_kind -> + let value_hash = tree_hash value in + {key; value; value_kind; value_hash}) >|= fun bindings -> Store.Tree.clear tree ; bindings module Hashtbl = Hashtbl.MakeSeeded (struct -- GitLab From b8a4533d95d9390f35bb04ed1600c391e0bd112e Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Tue, 22 Dec 2020 17:38:45 +0100 Subject: [PATCH 3/7] Storage: sync read-only instances before exists/checkout calls --- src/lib_storage/context.ml | 15 ++++++++++++--- src/lib_storage/context.mli | 4 ++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/lib_storage/context.ml b/src/lib_storage/context.ml index b8a60a1c4eb9..e7fe16d25605 100644 --- a/src/lib_storage/context.ml +++ b/src/lib_storage/context.ml @@ -254,6 +254,7 @@ type index = { path : string; repo : Store.Repo.t; patch_context : (context -> context tzresult Lwt.t) option; + readonly : bool; } and context = {index : index; parents : Store.Commit.t list; tree : Store.tree} @@ -288,11 +289,19 @@ let restore_integrity ?ppf index = "unable to fix the corrupted context: %d bad entries detected" n) +let sync index = + if index.readonly then Store.sync index.repo ; + Lwt.return () + let exists index key = + sync index + >>= fun () -> Store.Commit.of_hash index.repo (Hash.of_context_hash key) >|= function None -> false | Some _ -> true let checkout index key = + sync index + >>= fun () -> Store.Commit.of_hash index.repo (Hash.of_context_hash key) >>= function | None -> @@ -499,11 +508,11 @@ let set_predecessor_ops_metadata_hash v hash = (*-- Initialisation ----------------------------------------------------------*) -let init ?patch_context ?mapsize:_ ?readonly root = +let init ?patch_context ?mapsize:_ ?(readonly = false) root = Store.Repo.v - (Irmin_pack.config ?readonly ?index_log_size:!index_log_size root) + (Irmin_pack.config ~readonly ?index_log_size:!index_log_size root) >>= fun repo -> - let v = {path = root; repo; patch_context} in + let v = {path = root; repo; patch_context; readonly} in Lwt.return v let close index = Store.Repo.close index.repo diff --git a/src/lib_storage/context.mli b/src/lib_storage/context.mli index ef5428c93f5b..e6cfaa40994e 100644 --- a/src/lib_storage/context.mli +++ b/src/lib_storage/context.mli @@ -47,6 +47,10 @@ val init : (** Close the index. Does not fail when the context is already closed. *) val close : index -> unit Lwt.t +(** Sync the context with disk. Only useful for read-only instances. + Does not fail when the context is not in read-only mode. *) +val sync : index -> unit Lwt.t + val compute_testchain_chain_id : Block_hash.t -> Chain_id.t val compute_testchain_genesis : Block_hash.t -> Block_hash.t -- GitLab From 7d896b0283a47945b62069a12d824c1298e8e420 Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Thu, 14 Jan 2021 11:56:06 +0100 Subject: [PATCH 4/7] Storage: update opam dependencies --- src/lib_storage/tezos-storage.opam | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib_storage/tezos-storage.opam b/src/lib_storage/tezos-storage.opam index b7fa94c07519..983ea05ce697 100644 --- a/src/lib_storage/tezos-storage.opam +++ b/src/lib_storage/tezos-storage.opam @@ -10,8 +10,9 @@ depends: [ "dune" { >= "2.0" } "tezos-base" "tezos-lmdb" - "irmin" { >= "2.2.0" } - "irmin-pack" + "irmin" { >= "2.3.0" } + "irmin-pack" { >= "2.3.0" } + "irmin-mem" { >= "2.3.0" } "digestif" {>= "0.7.3"} "tezos-shell-services" "tezos-stdlib-unix" -- GitLab From fddefe0bad6249ff5349296f06f93b91daf7b294 Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Fri, 15 Jan 2021 17:43:24 +0100 Subject: [PATCH 5/7] tests_python: increase timeout for 'many bakers' tests --- tests_python/tests_008/test_many_bakers.py | 2 +- tests_python/tests_alpha/test_many_bakers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests_python/tests_008/test_many_bakers.py b/tests_python/tests_008/test_many_bakers.py index 61e0a0cba48c..8038c6cf8067 100644 --- a/tests_python/tests_008/test_many_bakers.py +++ b/tests_python/tests_008/test_many_bakers.py @@ -23,7 +23,7 @@ class TestManyBakers: sandbox.add_baker(i, f'bootstrap{i + 1}', proto=protocol.DAEMON) def test_wait(self): - time.sleep(5) + time.sleep(10) def test_progress(self, sandbox: Sandbox): min_level = min( diff --git a/tests_python/tests_alpha/test_many_bakers.py b/tests_python/tests_alpha/test_many_bakers.py index 61e0a0cba48c..8038c6cf8067 100644 --- a/tests_python/tests_alpha/test_many_bakers.py +++ b/tests_python/tests_alpha/test_many_bakers.py @@ -23,7 +23,7 @@ class TestManyBakers: sandbox.add_baker(i, f'bootstrap{i + 1}', proto=protocol.DAEMON) def test_wait(self): - time.sleep(5) + time.sleep(10) def test_progress(self, sandbox: Sandbox): min_level = min( -- GitLab From 6796f9fa785c7673ee0f8158035b853cc959289f Mon Sep 17 00:00:00 2001 From: Craig Ferguson Date: Tue, 12 Jan 2021 14:49:55 +0100 Subject: [PATCH 6/7] CI: bump up version of opam-repository to use Irmin 2.3 packages --- .gitlab-ci.yml | 2 +- scripts/version.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4b090ef312a8..d272ac21c34e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ include: variables: ## Please update `scripts/version.sh` accordingly - build_deps_image_version: 4eb9728016e05758054c600ddc66c7e295c27a26 + build_deps_image_version: 2743a159dc15cea7782f2581c08080ae0c6ce140 build_deps_image_name: registry.gitlab.com/tezos/opam-repository public_docker_image_name: docker.io/${CI_PROJECT_PATH} GIT_STRATEGY: fetch diff --git a/scripts/version.sh b/scripts/version.sh index 1028a4a82929..7754952825b3 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -10,10 +10,10 @@ recommended_rust_version=1.44.0 ## Please update `.gitlab-ci.yml` accordingly ## full_opam_repository is a commit hash of the public OPAM repository, i.e. ## https://github.com/ocaml/opam-repository -full_opam_repository_tag=bf94421703ae6d95113e5b24890f304701e47b78 +full_opam_repository_tag=d3e4998c62c36c7d131960f2306cf8ddf9898e3e ## opam_repository is an additional, tezos-specific opam repository. -opam_repository_tag=4eb9728016e05758054c600ddc66c7e295c27a26 +opam_repository_tag=2743a159dc15cea7782f2581c08080ae0c6ce140 opam_repository_url=https://gitlab.com/tezos/opam-repository.git opam_repository=$opam_repository_url\#$opam_repository_tag -- GitLab From 091a0a1b4f5eb22fa82154fe182a40221222189e Mon Sep 17 00:00:00 2001 From: Thomas Gazagnaire Date: Fri, 15 Jan 2021 20:34:47 +0100 Subject: [PATCH 7/7] TMP: use samoht/opam-repository fork --- .gitlab-ci.yml | 2 +- scripts/version.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d272ac21c34e..fc7f92d0f253 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ include: variables: ## Please update `scripts/version.sh` accordingly build_deps_image_version: 2743a159dc15cea7782f2581c08080ae0c6ce140 - build_deps_image_name: registry.gitlab.com/tezos/opam-repository + build_deps_image_name: registry.gitlab.com/samoht/opam-repository public_docker_image_name: docker.io/${CI_PROJECT_PATH} GIT_STRATEGY: fetch GIT_DEPTH: "1" diff --git a/scripts/version.sh b/scripts/version.sh index 7754952825b3..845e4ba59f74 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -14,7 +14,7 @@ full_opam_repository_tag=d3e4998c62c36c7d131960f2306cf8ddf9898e3e ## opam_repository is an additional, tezos-specific opam repository. opam_repository_tag=2743a159dc15cea7782f2581c08080ae0c6ce140 -opam_repository_url=https://gitlab.com/tezos/opam-repository.git +opam_repository_url=https://gitlab.com/samoht/opam-repository.git opam_repository=$opam_repository_url\#$opam_repository_tag ## Other variables, used both in Makefile and scripts -- GitLab