From 5b01c7c8bef84a8ff6467a987e1c9cd56cdcd4cb Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Thu, 2 Feb 2023 17:50:19 +0100 Subject: [PATCH 1/6] Python: run make fmt --- tests_python/client/client.py | 4 ---- tests_python/client/client_output.py | 5 ----- tests_python/daemons/node.py | 1 - tests_python/pytest_plugins/job_selection.py | 6 +++--- tests_python/tools/utils.py | 3 +-- 5 files changed, 4 insertions(+), 15 deletions(-) diff --git a/tests_python/client/client.py b/tests_python/client/client.py index ac384f598fa6..7bce9812311b 100644 --- a/tests_python/client/client.py +++ b/tests_python/client/client.py @@ -1670,7 +1670,6 @@ class Client: def sapling_gen_key( self, key_name: str, force: bool = False, args: List[str] = None ) -> client_output.SaplingGenKeyResult: - cmd = ['sapling', 'gen', 'key', key_name, '--unencrypted'] args = args or [] if force: @@ -1714,7 +1713,6 @@ class Client: force: bool = False, args: List[str] = None, ) -> None: - mnemonic_str = " ".join(mnemonic) cmd = [ 'sapling', @@ -1739,7 +1737,6 @@ class Client: index: int, force: bool = False, ) -> client_output.SaplingDeriveKeyResult: - cmd = [ 'sapling', 'derive', @@ -1761,7 +1758,6 @@ class Client: def sapling_get_balance( self, key_name: str, contract_name: str, args: List[str] = None ) -> client_output.SaplingGetBalanceResult: - cmd = [ 'sapling', 'get', diff --git a/tests_python/client/client_output.py b/tests_python/client/client_output.py index 15a2d33f3e78..9310a9f53eb2 100644 --- a/tests_python/client/client_output.py +++ b/tests_python/client/client_output.py @@ -95,7 +95,6 @@ class GetAddressesResult: """Result of 'list known addresses' operation.""" def __init__(self, client_output: str): - pattern = re.compile(r"^(\w+):\s*(\w+).*$", re.MULTILINE) self.wallet = dict(re.findall(pattern, client_output)) @@ -236,7 +235,6 @@ class HashResult: """Result of a 'hash data' command.""" def __init__(self, client_output: str): - pattern = r'''Raw packed data: ?(0x[0-9a-f]*) Script-expression-ID-Hash: ?(\w*) Raw Script-expression-ID-Hash: ?(\w*) @@ -257,7 +255,6 @@ class SignBytesResult: """Result of a 'sign bytes ...' command.""" def __init__(self, client_output: str): - pattern = r'Signature: ?(\w*)\n' match = re.search(pattern, client_output) if match is None: @@ -269,7 +266,6 @@ class SignMessageResult: """Result of a 'sign message ...' command.""" def __init__(self, client_output: str): - pattern = r'Signature: ?(\w*)\n' match = re.search(pattern, client_output) if match is None: @@ -538,7 +534,6 @@ class CheckSignMessageResult: """Result of a 'check that message...' command.""" def __init__(self, client_output: str): - pattern = r'Signature check successful *\n' match = re.search(pattern, client_output) if match is None: diff --git a/tests_python/daemons/node.py b/tests_python/daemons/node.py index ced1ff8eb9a6..61b9d014da06 100644 --- a/tests_python/daemons/node.py +++ b/tests_python/daemons/node.py @@ -76,7 +76,6 @@ class Node: singleprocess: bool = False, env: Dict[str, str] = None, ): - """Creates a new Popen instance for a octez-node, and manages context. args: diff --git a/tests_python/pytest_plugins/job_selection.py b/tests_python/pytest_plugins/job_selection.py index 0e5ee07a2d74..3fd437a8267b 100644 --- a/tests_python/pytest_plugins/job_selection.py +++ b/tests_python/pytest_plugins/job_selection.py @@ -105,10 +105,10 @@ def knapsack(items: List[Tuple[Any, float]], bag_count: int) -> List[Bag]: for _ in range(0, bag_count): knapsack.append({'total_weight': 0.0, 'items': []}) - for (item, weight) in items: + for item, weight in items: min_index = 0 min_total_weight = knapsack[0]['total_weight'] - for (index, bag) in enumerate(knapsack[1:]): + for index, bag in enumerate(knapsack[1:]): if bag['total_weight'] < min_total_weight: min_total_weight = bag['total_weight'] min_index = index + 1 @@ -169,7 +169,7 @@ def tabulate( for row in [headers_s] + rows_s: width = cell_width[0] print(f'{{0: <{width}}}'.format(str(row[0])), end="") - for (col_idx, col) in enumerate(row[1:]): + for col_idx, col in enumerate(row[1:]): width = cell_width[col_idx + 1] print(f'{{0: >{width}}}'.format(str(col)), end="") print() diff --git a/tests_python/tools/utils.py b/tests_python/tools/utils.py index afafa8f117aa..d26390ce11f9 100644 --- a/tests_python/tools/utils.py +++ b/tests_python/tools/utils.py @@ -428,7 +428,6 @@ def assert_run_script_success( def assert_run_script_failwith( client: Client, contract: str, param: str, storage: str ) -> None: - pattern = 'script reached FAILWITH instruction' with assert_run_failure(pattern): client.run_script(contract, param, storage, trace_stack=True) @@ -454,7 +453,7 @@ def assert_typecheck_failure( def suball(replacements: List[Tuple[Pattern, str]], string: str) -> str: """Apply all the replacements in `replacements`, in order, to `string` using re.sub""" - for (pattern, replacement) in replacements: + for pattern, replacement in replacements: string = re.sub(pattern, replacement, string) return string -- GitLab From 664a00e724f2760c3ab9b39dbee4ad1ac0d101c6 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Thu, 2 Feb 2023 17:50:33 +0100 Subject: [PATCH 2/6] Sapling: update binding to match librustzcash.v5.0.0 --- .../bindings/rustzcash_ctypes_bindings.ml | 25 +++++++++---------- src/lib_sapling/rustzcash.ml | 25 +++++++------------ 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/lib_sapling/bindings/rustzcash_ctypes_bindings.ml b/src/lib_sapling/bindings/rustzcash_ctypes_bindings.ml index e62ddbd64d16..78b2943b19bf 100644 --- a/src/lib_sapling/bindings/rustzcash_ctypes_bindings.ml +++ b/src/lib_sapling/bindings/rustzcash_ctypes_bindings.ml @@ -25,9 +25,8 @@ module Bindings (F : Cstubs.FOREIGN) = struct let init_zksnark_params = foreign "librustzcash_init_zksnark_params" - (ocaml_bytes @-> size_t @-> ocaml_string @-> ocaml_bytes @-> size_t - @-> ocaml_string @-> ptr uchar @-> size_t @-> ocaml_string - @-> returning void) + (ocaml_bytes @-> size_t @-> ocaml_bytes @-> size_t @-> ptr uchar + @-> size_t @-> returning void) let nsk_to_nk = foreign @@ -63,14 +62,14 @@ module Bindings (F : Cstubs.FOREIGN) = struct let sapling_compute_cm = foreign - "librustzcash_sapling_compute_cm" - (ocaml_bytes @-> ocaml_bytes @-> uint64_t @-> ocaml_bytes @-> ocaml_bytes - @-> returning bool) + "librustzcash_sapling_compute_cmu" + (bool @-> ocaml_bytes @-> ocaml_bytes @-> uint64_t @-> ocaml_bytes + @-> ocaml_bytes @-> returning bool) let sapling_ka_agree = foreign "librustzcash_sapling_ka_agree" - (ocaml_bytes @-> ocaml_bytes @-> ocaml_bytes @-> returning bool) + (bool @-> ocaml_bytes @-> ocaml_bytes @-> ocaml_bytes @-> returning bool) let sapling_ka_derivepublic = foreign @@ -96,23 +95,23 @@ module Bindings (F : Cstubs.FOREIGN) = struct (* ZIP32 functions *) let zip32_xsk_master = foreign - "librustzcash_zip32_xsk_master" + "librustzcash_zip32_sapling_xsk_master" (ocaml_bytes @-> size_t @-> ocaml_bytes @-> returning void) let zip32_xfvk_address = foreign - "librustzcash_zip32_xfvk_address" + "librustzcash_zip32_find_sapling_address" (ocaml_bytes @-> ocaml_bytes @-> ocaml_bytes @-> ocaml_bytes - @-> returning bool) + @-> ocaml_bytes @-> returning bool) let zip32_xsk_derive = foreign - "librustzcash_zip32_xsk_derive" + "librustzcash_zip32_sapling_xsk_derive" (ocaml_bytes @-> uint32_t @-> ocaml_bytes @-> returning void) let zip32_xfvk_derive = foreign - "librustzcash_zip32_xfvk_derive" + "librustzcash_zip32_sapling_xfvk_derive" (ocaml_bytes @-> uint32_t @-> ocaml_bytes @-> returning bool) (* Prover *) @@ -146,7 +145,7 @@ module Bindings (F : Cstubs.FOREIGN) = struct let verification_ctx_init = foreign "librustzcash_sapling_verification_ctx_init" - (void @-> returning (ptr void)) + (bool @-> returning (ptr void)) let verification_ctx_free = foreign diff --git a/src/lib_sapling/rustzcash.ml b/src/lib_sapling/rustzcash.ml index 84945c332d73..fe23e428b8c1 100644 --- a/src/lib_sapling/rustzcash.ml +++ b/src/lib_sapling/rustzcash.ml @@ -394,7 +394,7 @@ let valid_position pos = Parameters of type Rust `usize` are converted to OCaml `int` because they are only file paths. NULL is a void pointer. *) -let init_zksnark_params ~spend_path ~spend_hash ~output_path ~output_hash = +let init_zksnark_params ~spend_path ~output_path = let spend_path = Bytes.of_string spend_path in let output_path = Bytes.of_string output_path in let spend_path_len = Bytes.length spend_path in @@ -402,17 +402,12 @@ let init_zksnark_params ~spend_path ~spend_hash ~output_path ~output_hash = RS.init_zksnark_params (Ctypes.ocaml_bytes_start spend_path) (Unsigned.Size_t.of_int spend_path_len) - (Ctypes.ocaml_string_start spend_hash) (Ctypes.ocaml_bytes_start output_path) (Unsigned.Size_t.of_int output_path_len) - (Ctypes.ocaml_string_start output_hash) (* Getting a NULL pointer of type uchar. Causing the warning saying we convert a void * to unsigned char* *) Ctypes.(from_voidp uchar null) Unsigned.Size_t.zero - (* Any value can be passed. The Rust code does check if the path is a NULL - pointer (i.e. see previous comment) *) - (Ctypes.ocaml_string_start "Undefined") let nsk_to_nk nsk = let nk = Bytes.create 32 in @@ -480,6 +475,7 @@ let compute_cm diversifier pk_d ~amount rcm = let cm = Bytes.create 32 in let res = RS.sapling_compute_cm + false (Ctypes.ocaml_bytes_start (of_diversifier diversifier)) (Ctypes.ocaml_bytes_start (of_pkd pk_d)) (Unsigned.UInt64.of_int64 amount) @@ -493,6 +489,7 @@ let ka_agree (p : Bytes.t) (sk : Bytes.t) = let ka = Bytes.create 32 in let res = RS.sapling_ka_agree + false (Ctypes.ocaml_bytes_start p) (Ctypes.ocaml_bytes_start sk) (Ctypes.ocaml_bytes_start ka) @@ -535,7 +532,7 @@ let proving_ctx_free ctx = RS.proving_ctx_free ctx type verification_ctx = unit Ctypes_static.ptr -let verification_ctx_init () = RS.verification_ctx_init () +let verification_ctx_init () = RS.verification_ctx_init false let verification_ctx_free ctx = RS.verification_ctx_free ctx @@ -654,10 +651,12 @@ let zip32_xsk_master seed = let zip32_xfvk_address xfvk j = let j_ret = Bytes.create 11 in let addr = Bytes.create 43 in - let bytes_xfvk = of_zip32_full_viewing_key xfvk in + let fvk = of_full_viewing_key xfvk.fvk in + let dk = xfvk.dk in let res = RS.zip32_xfvk_address - (Ctypes.ocaml_bytes_start bytes_xfvk) + (Ctypes.ocaml_bytes_start fvk) + (Ctypes.ocaml_bytes_start dk) (Ctypes.ocaml_bytes_start (of_diversifier_index j)) (Ctypes.ocaml_bytes_start j_ret) (Ctypes.ocaml_bytes_start addr) @@ -779,13 +778,7 @@ let find_params ?(getenv_opt = Sys.getenv_opt) ?(getcwd = Sys.getcwd) let init_params () = let {spend_path; output_path} = find_params () in - let spend_hash = - "8270785a1a0d0bc77196f000ee6d221c9c9894f55307bd9357c3f0105d31ca63991ab91324160d8f53e2bbd3c2633a6eb8bdf5205d822e7f3f73edac51b2b70c\000" - in - let output_hash = - "657e3d38dbb5cb5e7dd2970e8b03d69b4787dd907285b5a7f0790dcc8072f60bf593b32cc2d1c030e00ff5ae64bf84c5c3beb84ddc841d48264b4a171744d028\000" - in - init_zksnark_params ~spend_path ~spend_hash ~output_path ~output_hash + init_zksnark_params ~spend_path ~output_path let init_params_lazy = Lazy.from_fun init_params -- GitLab From 6d3816ace9674358945f76a4a1036d234fdff625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Proust?= Date: Fri, 10 Feb 2023 11:08:00 +0100 Subject: [PATCH 3/6] Build: update tezos-rust-libs opam dependency --- .gitlab-ci.yml | 2 +- manifest/main.ml | 4 ++-- opam/tezos-sapling.opam | 2 +- opam/tezos-wasmer.opam | 2 +- opam/virtual/octez-deps.opam | 2 +- scripts/version.sh | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 457479cff029..295a41c733a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,7 +33,7 @@ variables: # /!\ CI_REGISTRY is overriden to use a private Docker registry mirror in AWS ECR # in GitLab namespaces `nomadic-labs` and `tezos` ## This value MUST be the same as `opam_repository_tag` in `scripts/version.sh` - build_deps_image_version: e86fb42578f762ced5052c92ef484833686d6158 + build_deps_image_version: 25dac742bf06804f9fe91a8b38f98e12c9a4e80e build_deps_image_name: "${CI_REGISTRY}/tezos/opam-repository" GIT_STRATEGY: fetch GIT_DEPTH: "1" diff --git a/manifest/main.ml b/manifest/main.ml index 26f297d23f33..591b2d4290ef 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -329,10 +329,10 @@ let tar = external_lib "tar" V.True let tar_unix = external_lib "tar-unix" V.(at_least "2.0.1" && less_than "3.0.0") let tezos_rust_lib = - opam_only ~can_vendor:false "tezos-rust-libs" V.(exactly "1.3") + opam_only ~can_vendor:false "tezos-rust-libs" V.(exactly "1.4") let tezos_rust_lib_sapling = - opam_only ~can_vendor:false "tezos-rust-libs" V.(at_least "1.1") + opam_only ~can_vendor:false "tezos-rust-libs" V.(exactly "1.4") let tls = external_lib "tls" V.(at_least "0.13.0") diff --git a/opam/tezos-sapling.opam b/opam/tezos-sapling.opam index 2fef032f3b5e..4dd1d42f3cef 100644 --- a/opam/tezos-sapling.opam +++ b/opam/tezos-sapling.opam @@ -19,7 +19,7 @@ depends: [ "tezos-stdlib" "tezos-crypto" "tezos-error-monad" - "tezos-rust-libs" { >= "1.1" } + "tezos-rust-libs" { = "1.4" } "tezos-lwt-result-stdlib" "tezos-base" {with-test} "tezos-stdlib-unix" {with-test} diff --git a/opam/tezos-wasmer.opam b/opam/tezos-wasmer.opam index 2b182944646e..1789f05dd0f4 100644 --- a/opam/tezos-wasmer.opam +++ b/opam/tezos-wasmer.opam @@ -14,7 +14,7 @@ depends: [ "ctypes" { >= "0.18.0" } "ctypes-foreign" { >= "0.18.0" } "lwt" { >= "5.6.0" } - "tezos-rust-libs" { = "1.3" } + "tezos-rust-libs" { = "1.4" } ] x-opam-monorepo-opam-provided: [ "tezos-rust-libs" diff --git a/opam/virtual/octez-deps.opam b/opam/virtual/octez-deps.opam index b787b32f5500..14bd6d5c0877 100644 --- a/opam/virtual/octez-deps.opam +++ b/opam/virtual/octez-deps.opam @@ -92,7 +92,7 @@ depends: [ "tar-unix" { >= "2.0.1" & < "3.0.0" } "tezos-plompiler" { >= "1.0.1" & < "2.0.0" } "tezos-plonk" { >= "1.0.1" & < "2.0.0" } - "tezos-rust-libs" { = "1.3" & >= "1.1" } + "tezos-rust-libs" { = "1.4" } "tezt" { >= "3.0.0" } "tls" { >= "0.13.0" } "uri" { >= "3.1.0" } diff --git a/scripts/version.sh b/scripts/version.sh index de58c4757029..c337d6abbfad 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -20,12 +20,12 @@ export recommended_node_version=16.18.1 ## full_opam_repository is a commit hash of the public OPAM repository, i.e. ## https://github.com/ocaml/opam-repository -export full_opam_repository_tag=6698d7801c195f2f1e5ef81997bb09a75d71f6c0 +export full_opam_repository_tag=4d8e7321c83290c76d178bd18b7bd73b332168e3 ## opam_repository is an additional, tezos-specific opam repository. ## This value MUST be the same as `build_deps_image_version` in `.gitlab/ci/templates.yml export opam_repository_url=https://gitlab.com/tezos/opam-repository -export opam_repository_tag="${OPAM_REPOSITORY_TAG:-e86fb42578f762ced5052c92ef484833686d6158}" +export opam_repository_tag="${OPAM_REPOSITORY_TAG:-25dac742bf06804f9fe91a8b38f98e12c9a4e80e}" export opam_repository_git="$opam_repository_url.git" export opam_repository="$opam_repository_git"\#"$opam_repository_tag" -- GitLab From 542c39953584da401747013e3e5960cadd1d5cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Thu, 16 Feb 2023 07:51:35 +0100 Subject: [PATCH 4/6] Tezt/DAL: Fix flaky test in DAL attestation test Co-authored-by: Eugen Zlinescu --- tezt/tests/dal.ml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index 4537ff40bee7..4e292fe8d4e8 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -2040,14 +2040,22 @@ let test_attestor ~with_baker_daemon protocol parameters cryptobox node client (slot_idx level = slot_index) int ~error_msg:"Expected index %L (got %R)") ; - let expected_status = - if level < first_not_attested_published_level then "attested" - else "unattested" - in - Check.( - (expected_status = status) - string - ~error_msg:"Expected status %L (got %R)") ; + (if + level < intermediary_level || level >= first_not_attested_published_level + then + (* We cannot know for sure the status of a slot between + [intermediary_level] and + [first_not_attested_published_level]. Before + [intermediary_level], it should be [attested], and above + [first_not_attested_published_level], it should be + [unattested]. *) + let expected_status = + if level < intermediary_level then "attested" else "unattested" + in + Check.( + (expected_status = status) + string + ~error_msg:"Expected status %L (got %R)")) ; check_attestations (level + 1) in check_attestations first_level -- GitLab From fb8c5b58e9e15a39648b1759e5579e7de80eb145 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Thu, 16 Feb 2023 11:03:51 +0100 Subject: [PATCH 5/6] CI: add BISECT_FILE variable in .tezt_template --- .gitlab/ci/jobs/test/tezt.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/ci/jobs/test/tezt.yml b/.gitlab/ci/jobs/test/tezt.yml index 4226d422a6f5..f24f51e77d35 100644 --- a/.gitlab/ci/jobs/test/tezt.yml +++ b/.gitlab/ci/jobs/test/tezt.yml @@ -9,6 +9,9 @@ include: .gitlab/ci/jobs/test/common.yml .tezt_template: variables: JUNIT: "tezt-junit.xml" + # if we do not set BISECT_FILE here, it will be empty, and include the + # full CI_PROJECT_DIR in artifacts as per the template .tezt_template. + BISECT_FILE: "$CI_PROJECT_DIR/_coverage_output/" artifacts: reports: junit: $JUNIT -- GitLab From 024b32c03a5176cac658d9254263175168abed21 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Fri, 17 Feb 2023 15:21:43 +0100 Subject: [PATCH 6/6] CI: strip binaries in .build --- .gitlab/ci/jobs/build/common.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/ci/jobs/build/common.yml b/.gitlab/ci/jobs/build/common.yml index 4029dba148ff..81de3f40e2ea 100644 --- a/.gitlab/ci/jobs/build/common.yml +++ b/.gitlab/ci/jobs/build/common.yml @@ -11,6 +11,8 @@ GIT_VERSION: "dev" script: - make -C ${CI_PROJECT_DIR}/.gitlab/ci build-full-unreleased + - chmod +w ${CI_PROJECT_DIR}/octez-* + - strip -s ${CI_PROJECT_DIR}/octez-* cache: key: "$CI_COMMIT_REF_SLUG" paths: -- GitLab