From 2de3254f4caf3c7e6bf170861388d08dc243c5f0 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Fri, 12 Jul 2024 11:01:28 +0200 Subject: [PATCH 1/4] alpha: introduce stitching from alpha --- src/proto_alpha/lib_protocol/init_storage.ml | 19 ++ src/proto_alpha/lib_protocol/raw_context.ml | 297 ++++++++++++++++++- src/proto_alpha/lib_protocol/raw_context.mli | 5 +- 3 files changed, 318 insertions(+), 3 deletions(-) diff --git a/src/proto_alpha/lib_protocol/init_storage.ml b/src/proto_alpha/lib_protocol/init_storage.ml index 04a32a02a4db..06ecdc454fa5 100644 --- a/src/proto_alpha/lib_protocol/init_storage.ml +++ b/src/proto_alpha/lib_protocol/init_storage.ml @@ -195,6 +195,24 @@ let prepare_first_block chain_id ctxt ~typecheck_smart_contract let* ctxt = Sc_rollup_inbox_storage.init_inbox ~predecessor ctxt in let* ctxt = Adaptive_issuance_storage.init ctxt in return (ctxt, commitments_balance_updates @ bootstrap_balance_updates) + (* Start of Alpha stitching. Used for automatic protocol snapshot *) + | Alpha -> + (* Please update [next_protocol] and [previous_protocol] in + [tezt/lib_tezos/protocol.ml] when you update this value. *) + (* TODO (#2704): possibly handle attestations for migration block (in bakers); + if that is done, do not set Storage.Tenderbake.First_level_of_protocol. + /!\ this storage is also use to add the smart rollup + inbox migration message. see `sc_rollup_inbox_storage`. *) + let* ctxt = + Storage.Tenderbake.First_level_of_protocol.update ctxt level + in + (* Migration of refutation games needs to be kept for each protocl. *) + let* ctxt = + Sc_rollup_refutation_storage.migrate_clean_refutation_games ctxt + in + return (ctxt, []) + (* End of Alpha stitching. Used for automatic protocol snapshot *) + (* Start of alpha predecessor stitching. Used for automatic protocol snapshot *) | ParisC_020 (* Please update [next_protocol] and [previous_protocol] in [tezt/lib_tezos/protocol.ml] when you update this value. *) -> @@ -218,6 +236,7 @@ let prepare_first_block chain_id ctxt ~typecheck_smart_contract let*! ctxt = Pending_denunciations_storage.clear ctxt in let*! ctxt = Raw_context.remove ctxt ["last_snapshot"] in return (ctxt, []) + (* End of alpha predecessor stitching. Used for automatic protocol snapshot *) in let* ctxt = List.fold_left_es patch_script ctxt Legacy_script_patches.addresses_to_patch diff --git a/src/proto_alpha/lib_protocol/raw_context.ml b/src/proto_alpha/lib_protocol/raw_context.ml index 0ff7eb24dd68..8a84e42aab28 100644 --- a/src/proto_alpha/lib_protocol/raw_context.ml +++ b/src/proto_alpha/lib_protocol/raw_context.ml @@ -881,7 +881,10 @@ let prepare ~level ~predecessor_timestamp ~timestamp ~adaptive_issuance_enable }; } -type previous_protocol = Genesis of Parameters_repr.t | ParisC_020 +type previous_protocol = + | Genesis of Parameters_repr.t + | Alpha + | (* Alpha predecessor *) ParisC_020 (* Alpha predecessor *) let check_and_update_protocol_version ctxt = let open Lwt_result_syntax in @@ -897,7 +900,9 @@ let check_and_update_protocol_version ctxt = else if Compare.String.(s = "genesis") then let+ param, ctxt = get_proto_param ctxt in (Genesis param, ctxt) - else if Compare.String.(s = "paris_020") then return (ParisC_020, ctxt) + else if Compare.String.(s = "alpha_current") then return (Alpha, ctxt) + else if (* Alpha predecessor *) Compare.String.(s = "paris_020") then + return (ParisC_020, ctxt) (* Alpha predecessor *) else Lwt.return @@ storage_error (Incompatible_protocol_version s) in let*! ctxt = @@ -925,6 +930,7 @@ let[@warning "-32"] get_previous_protocol_constants ctxt = context." | Some constants -> return constants) +(* Start of code to remove at next automatic protocol snapshot *) let update_block_time_related_constants (c : Constants_parametric_repr.t) = let divide_period p = Period_repr.of_seconds_exn @@ -985,6 +991,7 @@ let update_cycle_eras ctxt level ~prev_blocks_per_cycle ~blocks_per_cycle in let*? new_cycle_eras = Level_repr.add_cycle_era new_cycle_era cycle_eras in set_cycle_eras ctxt new_cycle_eras +(* End of code to remove at next automatic protocol snapshot *) (* You should ensure that if the type `Constants_parametric_repr.t` is different from `Constants_parametric_previous_repr.t` or the value of these @@ -1016,6 +1023,290 @@ let prepare_first_block ~level ~timestamp _chain_id ctxt = let* ctxt = set_cycle_eras ctxt cycle_eras in let*! result = add_constants ctxt param.constants in return (result, None) + (* Start of Alpha stitching. Used for automatic protocol snapshot *) + | Alpha -> + let module Previous = Constants_parametric_repr in + let* c = get_constants ctxt in + let dal = + let ({ + feature_enable; + incentives_enable; + number_of_slots; + attestation_lag; + attestation_threshold; + cryptobox_parameters; + } + : Previous.dal) = + c.dal + in + { + Constants_parametric_repr.feature_enable; + incentives_enable; + number_of_slots; + attestation_lag; + attestation_threshold; + cryptobox_parameters; + } + in + let reveal_activation_level = + let ({ + raw_data; + metadata; + dal_page; + dal_parameters; + dal_attested_slots_validity_lag; + } + : Previous.sc_rollup_reveal_activation_level) = + c.sc_rollup.reveal_activation_level + in + let raw_data = + Constants_parametric_repr.{blake2B = raw_data.blake2B} + in + { + Constants_parametric_repr.raw_data; + metadata; + dal_page; + dal_parameters; + dal_attested_slots_validity_lag; + } + in + let sc_rollup = + let ({ + arith_pvm_enable; + origination_size; + challenge_window_in_blocks; + stake_amount; + commitment_period_in_blocks; + max_lookahead_in_blocks; + max_active_outbox_levels; + max_outbox_messages_per_level; + number_of_sections_in_dissection; + timeout_period_in_blocks; + max_number_of_stored_cemented_commitments; + max_number_of_parallel_games; + reveal_activation_level = _; + private_enable; + riscv_pvm_enable; + } + : Previous.sc_rollup) = + c.sc_rollup + in + Constants_parametric_repr. + { + arith_pvm_enable; + origination_size; + challenge_window_in_blocks; + stake_amount; + commitment_period_in_blocks; + max_lookahead_in_blocks; + max_active_outbox_levels; + max_outbox_messages_per_level; + number_of_sections_in_dissection; + timeout_period_in_blocks; + max_number_of_stored_cemented_commitments; + max_number_of_parallel_games; + reveal_activation_level; + private_enable; + riscv_pvm_enable; + } + in + let zk_rollup = + let ({ + enable; + origination_size; + min_pending_to_process; + max_ticket_payload_size; + } + : Previous.zk_rollup) = + c.zk_rollup + in + Constants_parametric_repr. + { + enable; + origination_size; + min_pending_to_process; + max_ticket_payload_size; + } + in + let adaptive_rewards_params = + let ({ + issuance_ratio_final_min; + issuance_ratio_final_max; + issuance_ratio_initial_min; + issuance_ratio_initial_max; + initial_period; + transition_period; + max_bonus; + growth_rate; + center_dz; + radius_dz; + } + : Previous.adaptive_rewards_params) = + c.adaptive_issuance.adaptive_rewards_params + in + Constants_parametric_repr. + { + issuance_ratio_final_min; + issuance_ratio_final_max; + issuance_ratio_initial_min; + issuance_ratio_initial_max; + initial_period; + transition_period; + max_bonus; + growth_rate; + center_dz; + radius_dz; + } + in + let adaptive_issuance = + let ({ + global_limit_of_staking_over_baking; + edge_of_staking_over_delegation; + launch_ema_threshold; + adaptive_rewards_params = _; + activation_vote_enable; + autostaking_enable; + force_activation; + ns_enable; + } + : Previous.adaptive_issuance) = + c.adaptive_issuance + in + Constants_parametric_repr. + { + global_limit_of_staking_over_baking; + edge_of_staking_over_delegation; + launch_ema_threshold; + adaptive_rewards_params; + activation_vote_enable; + autostaking_enable; + force_activation; + ns_enable; + } + in + let issuance_weights = + let ({ + base_total_issued_per_minute; + baking_reward_fixed_portion_weight; + baking_reward_bonus_weight; + attesting_reward_weight; + seed_nonce_revelation_tip_weight; + vdf_revelation_tip_weight; + } + : Previous.issuance_weights) = + c.issuance_weights + in + { + Constants_parametric_repr.base_total_issued_per_minute; + baking_reward_fixed_portion_weight; + baking_reward_bonus_weight; + attesting_reward_weight; + seed_nonce_revelation_tip_weight; + vdf_revelation_tip_weight; + } + in + let constants = + let ({ + consensus_rights_delay; + blocks_preservation_cycles; + delegate_parameters_activation_delay; + blocks_per_cycle; + blocks_per_commitment; + nonce_revelation_threshold; + cycles_per_voting_period; + hard_gas_limit_per_operation; + hard_gas_limit_per_block; + proof_of_work_threshold; + minimal_stake; + minimal_frozen_stake; + vdf_difficulty; + origination_size; + max_operations_time_to_live; + issuance_weights = _; + cost_per_byte; + hard_storage_limit_per_operation; + quorum_min; + quorum_max; + min_proposal_quorum; + liquidity_baking_subsidy; + liquidity_baking_toggle_ema_threshold; + minimal_block_delay; + delay_increment_per_round; + consensus_committee_size; + consensus_threshold; + minimal_participation_ratio; + limit_of_delegation_over_baking; + percentage_of_frozen_deposits_slashed_per_double_baking; + percentage_of_frozen_deposits_slashed_per_double_attestation; + max_slashing_per_block; + max_slashing_threshold; + (* The `testnet_dictator` should absolutely be None on mainnet *) + testnet_dictator; + initial_seed; + cache_script_size; + cache_stake_distribution_cycles; + cache_sampler_state_cycles; + dal = _; + sc_rollup = _; + zk_rollup = _; + adaptive_issuance = _; + direct_ticket_spending_enable; + } + : Previous.t) = + c + in + { + Constants_parametric_repr.consensus_rights_delay; + blocks_preservation_cycles; + delegate_parameters_activation_delay; + blocks_per_cycle; + blocks_per_commitment; + nonce_revelation_threshold; + cycles_per_voting_period; + hard_gas_limit_per_operation; + hard_gas_limit_per_block; + proof_of_work_threshold; + minimal_stake; + minimal_frozen_stake; + vdf_difficulty; + origination_size; + max_operations_time_to_live; + issuance_weights; + cost_per_byte; + hard_storage_limit_per_operation; + quorum_min; + quorum_max; + min_proposal_quorum; + liquidity_baking_subsidy; + liquidity_baking_toggle_ema_threshold; + minimal_block_delay; + delay_increment_per_round; + consensus_committee_size; + consensus_threshold; + minimal_participation_ratio; + limit_of_delegation_over_baking; + percentage_of_frozen_deposits_slashed_per_double_baking; + percentage_of_frozen_deposits_slashed_per_double_attestation; + max_slashing_per_block; + max_slashing_threshold; + (* The `testnet_dictator` should absolutely be None on mainnet *) + testnet_dictator; + initial_seed; + cache_script_size; + cache_stake_distribution_cycles; + cache_sampler_state_cycles; + dal; + sc_rollup; + zk_rollup; + adaptive_issuance; + direct_ticket_spending_enable; + } + in + let*! ctxt = add_constants ctxt constants in + let*! c = get_previous_protocol_constants ctxt in + return (ctxt, Some c) + (* End of Alpha stitching. Used for automatic protocol snapshot *) + (* Start of alpha predecessor stitching. Used for automatic protocol snapshot *) | ParisC_020 -> let*! c = get_previous_protocol_constants ctxt in @@ -1235,7 +1526,9 @@ let prepare_first_block ~level ~timestamp _chain_id ctxt = in let*! ctxt = add_constants ctxt constants in return (ctxt, Some c) + (* End of alpha predecessor stitching. Used for automatic protocol snapshot *) in + let+ ctxt = prepare ctxt diff --git a/src/proto_alpha/lib_protocol/raw_context.mli b/src/proto_alpha/lib_protocol/raw_context.mli index e1215c20854f..af33f199b310 100644 --- a/src/proto_alpha/lib_protocol/raw_context.mli +++ b/src/proto_alpha/lib_protocol/raw_context.mli @@ -93,7 +93,10 @@ val prepare : Context.t -> t tzresult Lwt.t -type previous_protocol = Genesis of Parameters_repr.t | ParisC_020 +type previous_protocol = + | Genesis of Parameters_repr.t + | Alpha + | (* Alpha predecessor *) ParisC_020 (* Alpha predecessor *) val prepare_first_block : level:int32 -> -- GitLab From add9579785367875ae11d8334be8c5b9e6afa371 Mon Sep 17 00:00:00 2001 From: mbourgoin Date: Thu, 4 Jul 2024 09:40:13 +0200 Subject: [PATCH 2/4] manifest: rename Alpha into Dev - to handle both proto_alpha and protocols in stabilization --- manifest/main.ml | 4 +-- manifest/product_client_libs.ml | 2 +- manifest/product_octez.ml | 55 ++++++++++++++++----------------- manifest/product_octez.mli | 2 +- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/manifest/main.ml b/manifest/main.ml index cccbf3a0dbed..d67806a132c0 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -161,7 +161,7 @@ let () = let () = let write_protocol fmt protocol = match Protocol.number protocol with - | Alpha | V _ -> Format.fprintf fmt "%s\n" (Protocol.name_dash protocol) + | Dev | V _ -> Format.fprintf fmt "%s\n" (Protocol.name_dash protocol) | Other -> () in write "script-inputs/active_protocol_versions" @@ fun fmt -> @@ -171,7 +171,7 @@ let () = let () = let write_protocol fmt protocol = match Protocol.number protocol with - | Alpha | V _ -> Format.fprintf fmt "%s\n" (Protocol.short_hash protocol) + | Dev | V _ -> Format.fprintf fmt "%s\n" (Protocol.short_hash protocol) | Other -> () in write "script-inputs/active_protocol_versions_without_number" @@ fun fmt -> diff --git a/manifest/product_client_libs.ml b/manifest/product_client_libs.ml index 510d50258b73..eb972b1d4e42 100644 --- a/manifest/product_client_libs.ml +++ b/manifest/product_client_libs.ml @@ -89,7 +89,7 @@ let _octez_codec_kaitai = (fun protocol -> let link = match Protocol.number protocol with - | Alpha -> true + | Dev -> true | V number -> number >= 005 | Other -> false in diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index 0551a82aa444..a9cc04a3f257 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -4869,7 +4869,7 @@ let _octez_scoru_wasm_fast_tests = (* PROTOCOL PACKAGES *) module Protocol : sig - type number = Alpha | V of int | Other + type number = Dev | V of int | Other type status = Active | Frozen | Overridden | Not_mainnet @@ -4941,13 +4941,13 @@ module Protocol : sig val all_optionally : (t -> target option) list -> target list end = struct - type number = Alpha | V of int | Other + type number = Dev | V of int | Other module Name : sig type t - (** [alpha] is a protocol name with protocol number [Alpha] *) - val alpha : t + (** [dev] is a named protocol name with protocol number [Dev] *) + val dev : string -> t (** [v name num] constuct a protocol name with protocol number [V num] *) val v : string -> int -> t @@ -4987,7 +4987,7 @@ end = struct name) ; let make_full_name sep name = match number with - | Alpha | Other -> name + | Dev | Other -> name | V number -> sf "%03d%c%s" number sep name in let name_dash = make_full_name '-' name in @@ -4998,7 +4998,7 @@ end = struct let v name number = make name (V number) - let alpha = make "alpha" Alpha + let dev name = make name Dev let other name = make name Other @@ -5130,10 +5130,10 @@ end = struct let test_helpers_exn p = mandatory "test_helpers" p p.test_helpers - (* N as in "protocol number in the Alpha family". *) + (* N as in "protocol number in the Dev family". *) module N = struct (* This function is asymmetrical on purpose: we don't want to compare - numbers with [Alpha] because such comparisons would break when snapshotting. + numbers with [Dev] because such comparisons would break when snapshotting. So the left-hand side is the number of the protocol being built, but the right-hand side is an integer. @@ -5145,7 +5145,7 @@ end = struct meaning the opposite? *) let compare_asymmetric a b = match a with - | Alpha -> 1 + | Dev -> 1 | V a -> Int.compare a b | Other -> invalid_arg "cannot use N.compare_asymmetric on Other protocols" @@ -5605,10 +5605,10 @@ end = struct in let disable_warnings = match number with - (* [Other] and [Alpha] protocols can be edited and should be + (* [Other] and [Dev] protocols can be edited and should be fixed whenever a warning that we care about triggers. We only want to disable a limited set of warnings *) - | Other | Alpha -> [] + | Other | Dev -> [] (* [V _] protocols can't be edited to accomodate warnings, we need to disable warnings instead. *) | V _ as number -> if N.(number >= 014) then [] @@ -5677,7 +5677,7 @@ include Tezos_protocol_environment.V%d.Make(Name)() sf "Tezos/Protocol: %s economic-protocol definition" name_underscore - | Alpha | V _ -> "Tezos/Protocol: economic-protocol definition") + | Dev | V _ -> "Tezos/Protocol: economic-protocol definition") ~modules:["Protocol"; sf "Tezos_protocol_%s" name_underscore] ~flags:(Flags.standard ~nopervasives:true ~disable_warnings ()) ~deps: @@ -5800,7 +5800,7 @@ let hash = Protocol.hash "Tezos/Protocol: %s (economic-protocol definition \ parameterized by its environment implementation)" name_underscore - | Alpha | V _ -> + | Dev | V _ -> "Tezos/Protocol: economic-protocol definition parameterized by \ its environment implementation") ~modules:["Functor"] @@ -5850,7 +5850,7 @@ let hash = Protocol.hash "Tezos/Protocol: %s (economic-protocol definition, embedded \ in `octez-node`)" name_underscore - | Alpha | V _ -> + | Dev | V _ -> "Tezos/Protocol: economic-protocol definition, embedded in \ `octez-node`") ~modules:["Registerer"] @@ -5862,7 +5862,7 @@ let hash = Protocol.hash (* Contrary to client libs and protocol plugin registerers, embedded protocols are useful even when the protocol was overridden. *) Released - | V _, Not_mainnet | (Alpha | Other), _ -> + | V _, Not_mainnet | (Dev | Other), _ -> (* Ideally we would not release the opam packages but this would require removing the dependencies when releasing, both from .opam files and dune files. *) @@ -5980,7 +5980,7 @@ let hash = Protocol.hash match (number, status) with | V _, (Active | Frozen) -> Released | V _, (Overridden | Not_mainnet) -> Unreleased - | Alpha, _ -> Experimental + | Dev, _ -> Experimental | Other, _ -> Unreleased in let optional_library_release_status = @@ -5989,7 +5989,7 @@ let hash = Protocol.hash (* Put explicit dependency in meta-package octez.opam to force the optional dependency to be installed. *) Released - | V _, (Overridden | Not_mainnet) | (Alpha | Other), _ -> + | V _, (Overridden | Not_mainnet) | (Dev | Other), _ -> (* Ideally we would not release the opam packages but this would require removing the dependencies when releasing, both from .opam files and dune files. *) @@ -7007,7 +7007,7 @@ let hash = Protocol.hash let _020_PsParisC = active (Name.v "PsParisC" 020) - let alpha = active Name.alpha + let alpha = active (Name.dev "alpha") let all = List.rev !all_rev @@ -7584,7 +7584,7 @@ let _octez_node = that results in inconsistent hashes. Once this bug is fixed, this exception can be removed. *) false - | (Frozen | Overridden | Not_mainnet), _ | Active, (Alpha | Other) -> + | (Frozen | Overridden | Not_mainnet), _ | Active, (Dev | Other) -> (* Other protocols are optional. *) true in @@ -7650,8 +7650,7 @@ let _octez_client = let is_optional = match (Protocol.status protocol, Protocol.number protocol) with | Active, V _ -> false - | (Frozen | Overridden | Not_mainnet), _ | Active, (Alpha | Other) -> - true + | (Frozen | Overridden | Not_mainnet), _ | Active, (Dev | Other) -> true in let targets = List.filter_map @@ -7745,7 +7744,7 @@ let _octez_codec = (fun protocol -> let link = match Protocol.number protocol with - | Alpha -> true + | Dev -> true | V number -> number >= 005 | Other -> false in @@ -7972,7 +7971,7 @@ let _octez_dal_node = that results in inconsistent hashes. Once this bug is fixed, this exception can be removed. *) false - | (Frozen | Overridden | Not_mainnet), _ | Active, (Alpha | Other) -> + | (Frozen | Overridden | Not_mainnet), _ | Active, (Dev | Other) -> (* Other protocols are optional. *) true in @@ -8036,7 +8035,7 @@ let _octez_dac_node = that results in inconsistent hashes. Once this bug is fixed, this exception can be removed. *) false - | (Frozen | Overridden | Not_mainnet), _ | Active, (Alpha | Other) -> + | (Frozen | Overridden | Not_mainnet), _ | Active, (Dev | Other) -> (* Other protocols are optional. *) true in @@ -8089,7 +8088,7 @@ let _octez_dac_client = that results in inconsistent hashes. Once this bug is fixed, this exception can be removed. *) false - | (Frozen | Overridden | Not_mainnet), _ | Active, (Alpha | Other) -> + | (Frozen | Overridden | Not_mainnet), _ | Active, (Dev | Other) -> (* Other protocols are optional. *) true in @@ -8126,8 +8125,7 @@ let _octez_smart_rollup_node = let is_optional = match (Protocol.status protocol, Protocol.number protocol) with | Active, V _ -> false - | (Frozen | Overridden | Not_mainnet), _ | Active, (Alpha | Other) -> - true + | (Frozen | Overridden | Not_mainnet), _ | Active, (Dev | Other) -> true in let targets = List.filter_map Fun.id [Protocol.octez_sc_rollup_node protocol] @@ -8164,8 +8162,7 @@ let _octez_smart_rollup_node_lib_tests = let is_optional = match (Protocol.status protocol, Protocol.number protocol) with | Active, V _ -> false - | (Frozen | Overridden | Not_mainnet), _ | Active, (Alpha | Other) -> - true + | (Frozen | Overridden | Not_mainnet), _ | Active, (Dev | Other) -> true in let targets = List.filter_map Fun.id [Protocol.octez_sc_rollup_node protocol] diff --git a/manifest/product_octez.mli b/manifest/product_octez.mli index 5140658bc8f3..0e02bdf3e263 100644 --- a/manifest/product_octez.mli +++ b/manifest/product_octez.mli @@ -84,7 +84,7 @@ val tezt_wrapper : Manifest.target module Protocol : sig type t - type number = Alpha | V of int | Other + type number = Dev | V of int | Other (** Status of the protocol on Mainnet. -- GitLab From c862a53ac701d2015f1d08fce4f77a54b5328731 Mon Sep 17 00:00:00 2001 From: mbourgoin Date: Mon, 8 Jul 2024 19:29:12 +0200 Subject: [PATCH 3/4] scripts/proto_manager: auto protocol stabilisation - currently only focuses on the automatic protocol stabilisation. - future MRs will add protocol snapshotting, preparation for user activated protocol override, and user activated upgrades. --- .gitignore | 4 +- .gitlab/ci/pipelines/before_merging.yml | 52 -- .../ci/pipelines/schedule_extended_test.yml | 38 - ci/bin/code_verification.ml | 22 +- ci/bin/common.ml | 13 - scripts/proto_manager.sh | 721 ++++++++++++++++++ tezt/lib_tezos/protocol.ml | 6 +- 7 files changed, 730 insertions(+), 126 deletions(-) create mode 100644 scripts/proto_manager.sh diff --git a/.gitignore b/.gitignore index e6bd913f1944..74356d86b8fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ - ## /!\ /!\ Update .dockerignore accordingly /!\ /!\ .DS_Store __pycache__ *.pyc +# temporary files created during snapshotting +*.old + **/_build /_build_rust /_opam diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index fb0cb137d8f2..c7d10e2659fb 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -1750,58 +1750,6 @@ oc.script:test-gen-genesis: script: - dune build gen_genesis.exe -oc.script:snapshot_alpha_and_link: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - scripts/snapshot_alpha.sh - - scripts/snapshot_alpha_and_link.sh - - scripts/user_activated_upgrade.sh - - src/proto_alpha/**/* - when: on_success - needs: - - job: oc.docker:ci:amd64 - - job: trigger - - job: oc.build_x86_64-released - optional: true - - job: oc.build_x86_64-exp-dev-extra - optional: true - dependencies: - - oc.docker:ci:amd64 - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/script:snapshot_alpha_and_link.sh - after_script: - - sccache --stop-server || true - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - oc.script:test_octez_release_versions: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index f025c47034b5..9b16b327ac9a 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -1419,44 +1419,6 @@ oc.script:test-gen-genesis: script: - dune build gen_genesis.exe -oc.script:snapshot_alpha_and_link: - image: ${ci_image_name}/build:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - when: always - dependencies: - - oc.docker:ci:amd64 - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: dune-build-cache-$CI_PIPELINE_ID - paths: - - $CI_PROJECT_DIR/_dune_cache - policy: pull - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/script:snapshot_alpha_and_link.sh - after_script: - - sccache --stop-server || true - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - DUNE_CACHE: enabled - DUNE_CACHE_STORAGE_MODE: hardlink - DUNE_CACHE_ROOT: $CI_PROJECT_DIR/_dune_cache - oc.script:test_octez_release_versions: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index d7e104287137..605c435b27d1 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -772,7 +772,7 @@ let jobs pipeline_type = (otherwise, they run immediately after [job_start]). In [Scheduled_extended_test] we are not in a hurry and we let them be [Staged []]. *) - let order_after_build = + let _order_after_build = make_dependencies ~before_merging:(fun job_start -> Dependent @@ -1150,25 +1150,6 @@ let jobs pipeline_type = (before_script ~eval_opam:true ["cd scripts/gen-genesis"]) ["dune build gen_genesis.exe"] in - let job_oc_script_snapshot_alpha_and_link : tezos_job = - job - ~__POS__ - ~name:"oc.script:snapshot_alpha_and_link" - ~stage:Stages.test - ~image:Images.CI.build - ~dependencies:order_after_build - (* Since the above dependencies are only for ordering, we do not set [dependent] *) - ~rules:(make_rules ~changes:changeset_script_snapshot_alpha_and_link ()) - ~before_script: - (before_script - ~take_ownership:true - ~source_version:true - ~eval_opam:true - []) - ["./scripts/ci/script:snapshot_alpha_and_link.sh"] - |> enable_cargo_cache |> enable_sccache - |> enable_dune_cache ~key:build_cache_key ~policy:Pull - in let job_oc_script_test_release_versions : tezos_job = job ~__POS__ @@ -1566,7 +1547,6 @@ let jobs pipeline_type = job_semgrep; job_oc_integration_compiler_rejections; job_oc_script_test_gen_genesis; - job_oc_script_snapshot_alpha_and_link; job_oc_script_test_release_versions; job_oc_script_b58_prefix; job_oc_test_liquidity_baking_scripts; diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 665b15eb640a..4ce4ba2b51e1 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -491,19 +491,6 @@ let changeset_semgrep_files = changeset_base @ make ["src/**/*"; "tezt/**/*"; "devtools/**/*"; "scripts/semgrep/**/*"]) -(* We only need to run the [oc.script:snapshot_alpha_and_link] job if - protocol Alpha or if the scripts changed. *) -let changeset_script_snapshot_alpha_and_link = - Changeset.( - changeset_base - @ make - [ - "src/proto_alpha/**/*"; - "scripts/snapshot_alpha_and_link.sh"; - "scripts/snapshot_alpha.sh"; - "scripts/user_activated_upgrade.sh"; - ]) - let changeset_script_b58_prefix = Changeset.( changeset_base diff --git a/scripts/proto_manager.sh b/scripts/proto_manager.sh new file mode 100644 index 000000000000..1dbd2c9b4848 --- /dev/null +++ b/scripts/proto_manager.sh @@ -0,0 +1,721 @@ +#!/usr/bin/env bash + +############################################################################## +# # +# SPDX-License-Identifier: MIT # +# SPDX-FileCopyrightText: 2024 Nomadic Labs # +# # +############################################################################## +#!/usr/bin/env bash + +set -e + +## The script may create commits. Keep track of the number of commits to clean up in case of an error +commits=0 + +## Log colors: +if [[ -t 1 ]]; then + yellow="\e[33m" + red="\e[31m" + blue="\e[34m" + green="\e[32m" + cyan="\e[36m" + magenta="\e[35m" + reset="\e[0m" +else + yellow="" + red="" + blue="" + green="" + cyan="" + reset="" + magenta="" +fi + +function error() { + echo -e "${red}Error: $*${reset}" 1>&2 +} + +function warning() { + echo -e "${yellow}Warning: $*${reset}" 1>&2 +} + +function log_blue() { + echo -e "${blue}$*${reset}" +} + +function log_green() { + echo -e "${green}$*${reset}" +} + +function log_cyan() { + echo -e "${cyan}$*${reset}" +} + +function log_magenta() { + echo -e "${magenta}$*${reset}" +} + +function print_and_exit() { + if [[ $# -ne 2 ]]; then + error "Exiting because of an error at $0" + else + error "Exiting because of an error at $0:$1" + fi + if [[ ${commits} -gt 0 ]]; then + log_green "To clean up git commits run:" + echo "git reset --hard HEAD~${commits}" + fi + + log_green "To cleanup other created files" + echo "rm -rf src/proto_${protocol_target}" + echo "rm -rf docs/${label}" + echo "rm -rf src/proto_${version}_${short_hash}" + echo "rm -rf src/proto_${version}" + echo "exiting..." + exit 1 +} + +function commit() { + + if [[ -n ${git} ]]; then + git add "${script_dir}/../" + git commit -m "${capitalized_label}/$1" + echo -e "${blue}Created commit:${cyan} ${capitalized_label}/$1${reset}" + commits=$((commits + 1)) + fi +} + +## Automatically clean and exit on error +trap print_and_exit ERR + +function usage() { + # colored option strings + f="${green}-f${reset}" + t="${green}-t${reset}" + l="${green}-l${reset}" + g="${green}-g${reset}" + h="${green}-h${reset}" + s="${green}-s${reset}" + from="${green}--from${reset}" + to="${green}--to${reset}" + link="${green}--link${reset}" + git="${green}--git${reset}" + help="${green}--help${reset}" + stabilise="${green}--stabilise${reset}" + ## colored vars + alpha="${red}alpha${reset}" + beta="${red}beta${reset}" + _023_stockholm="${red}023_stockholm${reset}" + script=${blue}$0${reset} + + echo -e " +${yellow}Usage: ${blue}$0${reset} ${red}[options]${reset} +${red}Options:${reset} + ${h} , ${help} + Print this help message. + ${s}, ${stabilise} + Stabilise a protocol. + ${f}, ${from} ${red}${reset} + The source protocol to stabilise. + ${t}, ${to} ${red}${reset} + The target protocol to stabilise. + ${l}, ${link} + Link the protocol in the node, client and codec. + ${g}, ${git} + Create git commits for the changes. + + +${yellow}tl;dr:${reset} +- To stabilise protocol alpha into beta and link it in the node, client and codec: + ${script} ${stabilise} ${from} ${alpha} ${to} ${beta} ${link} ${git} + or + ${script} ${s} ${f} ${alpha} ${t} ${beta} ${l} ${g} +" +} + +# Read command line arguments +while true; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + -s | --stabilise) + command="stabilise" + shift + ;; + -f | --from) + protocol_source="$2" + shift 2 + ;; + -t | --to) + protocol_target="$2" + shift 2 + ;; + -l | --link) + link=true + shift + ;; + -g | --git) + git=true + shift + ;; + --) + shift + break + ;; + *) + break + ;; + esac +done + +# Check if commands are valid +# if command = stabilise then protocol_source and protocol_target are required + +if [[ -z ${command} ]]; then + echo "No command specified" 1>&2 + usage 1>&2 + print_and_exit 1 "${LINENO}" +fi +if [[ ${command} == "stabilise" ]]; then + if [[ -z ${protocol_target} ]]; then + error "No protocol target specified" 1>&2 + usage 1>&2 + print_and_exit 1 "${LINENO}" + fi + if [[ -z ${protocol_source} ]]; then + error "No protocol source specified" 1>&2 + usage 1>&2 + print_and_exit 1 "${LINENO}" + fi + if [[ ${protocol_source} == "${protocol_target}" ]]; then + error "Protocol source and target should be different" 1>&2 + usage 1>&2 + print_and_exit 1 "${LINENO}" + fi + # + msg="Will ${command} protocol from 'src/proto_${protocol_source}' into 'src/proto_${protocol_target}'" + +fi + +if [[ -z ${link} ]]; then + link=false +else + msg="${msg} and link protocol ${protocol_target} in the node, client and codec" +fi + +log_blue "${msg}." + +# Check if the protocol source exists +if [[ ! -d "src/proto_${protocol_source}" ]]; then + error "the directory 'src/proto_${protocol_source}' does not exist" 1>&2 + print_and_exit 1 "${LINENO}" +fi + +# if stabilise command is used, protocol_target should be of the form [a-z]+[0-9]* +if [[ ${command} == "stabilise" ]]; then + if ! [[ ${protocol_source} =~ ^[a-z]+[0-9]*$ ]]; then + error "To ${red}stabilise${reset}, protocol_source should be of the form [a-z]+[0-9]+" 1>&2 + print_and_exit 1 "${LINENO}" + fi + if ! [[ ${protocol_target} =~ ^[a-z]+[0-9]*$ ]]; then + error "To ${red}stabilise${reset} protocol_target should be of the form [a-z]+[0-9]+" 1>&2 + print_and_exit 1 "${LINENO}" + fi +fi + +# check if ./octez-protocol-compiler is present +if [[ ! -f "./octez-protocol-compiler" ]]; then + error "octez-protocol-compiler not found, compile it before running this script" 1>&2 + print_and_exit 1 "${LINENO}" +fi + +rm -rf /tmp/tezos_proto_snapshot + +script_dir="$(cd "$(dirname "$0")" && pwd -P)" +cd "${script_dir}"/.. + +function snapshot_protocol() { + capitalized_label=$(tr '[:lower:]' '[:upper:]' <<< "${label:0:1}")${label:1} + + capitalized_source=$(tr '[:lower:]' '[:upper:]' <<< "${protocol_source:0:1}")${protocol_source:1} + + #replace _ with - inn protocol_source + tezos_protocol_source=$(echo "${protocol_source}" | tr '_' '-') + + # e.g. Pt8PY9P47nYw7WgPqpr49JZX5iU511ZJ9UPrBKu1CuYtBsLy7q7 (set below) + long_hash= + # e.g. Pt8PY9P4 (set below) + short_hash= + + if ! [[ -d "src/proto_${protocol_source}" ]]; then + error "'src/proto_${protocol_source}' does not exist" 1>&2 + print_and_exgxit 1 "${LINENO}" + fi + + if ! [[ -d "docs/${protocol_source}" ]]; then + error "'docs/${protocol_source}' does not exist" 1>&2 + print_and_exit 1 "${LINENO}" + fi + + if [[ -d "src/proto_${version}" ]]; then + error "'src/proto_${version}' already exists, you should remove it." + print_and_exit 1 "${LINENO}" + fi + + if [[ -d "docs/${label}" ]]; then + error "'docs/${label}' already exists, you should remove it." + print_and_exit 1 "${LINENO}" + fi + + # Create proto_beta source-code + + # create a temporary directory until the hash is known + # this is equivalent to `cp src/proto_${protocol_source}/ src/proto_${version}` but only for versioned files + echo "Copying src/proto_${protocol_source} to src/proto_${version}" + mkdir /tmp/tezos_proto_snapshot + git archive HEAD "src/proto_${protocol_source}/" | tar -x -C /tmp/tezos_proto_snapshot + mv "/tmp/tezos_proto_snapshot/src/proto_${protocol_source}" "src/proto_${version}" + rm -rf /tmp/tezos_proto_snapshot + #delete all dune files in src/proto_${version} containing the line "; This file was automatically generated, do not edit." + find "src/proto_${version}" -name dune -exec grep -q "; This file was automatically generated, do not edit." {} \; -exec rm {} \; + commit "src: copy from ${protocol_source}" + + # set current version + # Starting from 018 the version value moved to `constants_repr`. To be + # able to snapshot older protocol the `raw_context` file is kept even + # if it is not strictly needed anymore. + echo "Setting current version in raw_context and proxy" + + sed -i.old.old -e "s/let version_value = \"${protocol_source}_current\"/let version_value = \"${protocol_target}\"/" \ + "src/proto_${version}/lib_protocol/constants_repr.ml" \ + "src/proto_${version}/lib_protocol/raw_context.ml" \ + "src/proto_${version}/lib_client/proxy.ml" + + commit "src: set current version" + + log_cyan "Computing hash" + long_hash=$(./octez-protocol-compiler -hash-only "src/proto_${version}/lib_protocol") + short_hash=$(echo "${long_hash}" | head -c 8) + log_magenta "Hash computed: ${long_hash}" + log_magenta "Short hash: ${short_hash}" + + new_protocol_name="${version}" + new_tezos_protocol="${version}" + new_versioned_name="${label}" + + cd "src/proto_${new_protocol_name}" + # rename main_*.ml{,i} files of the binaries + find . -name main_\*_"${protocol_source}".ml -or -name main_\*_"${protocol_source}".mli | while read -r file; do + new_file=${file//_"${protocol_source}"/_"${new_protocol_name}"} + if [[ -n ${git} ]]; then + git mv "${file}" "${new_file}" + else + mv "${file}" "${new_file}" + fi + done + commit "src: rename binaries main_*.ml{,i} files" + + cd lib_protocol + + # replace fake hash with real hash, this file doesn't influence the hash + sed -i.old -e 's/"hash": "[^"]*",/"hash": "'"${long_hash}"'",/' \ + TEZOS_PROTOCOL + commit "src: replace fake hash with real hash" + + # We use `--print0` and `xargs -0` instead of just passing the result + # of find to sed in order to support spaces in filenames. + find . -type f -print0 | xargs -0 \ + sed -i.old -e "s/protocol_${protocol_source}/protocol_${new_protocol_name}/" \ + -e "s/protocol-${tezos_protocol_source}/protocol-${new_tezos_protocol}/" \ + -e "s/protocol-functor-${tezos-protocol-source}/protocol-functor-${new_tezos_protocol}/" + # add this protocol to the immutable list + printf %s "${long_hash}" >> ../../lib_protocol_compiler/final_protocol_versions + commit "src: replace protocol_${protocol_source} with protocol_${new_protocol_name}" + + cd ../../.. + + sed -i "s/${protocol_source}/${label}/g" "src/proto_${label}/README.md" + + commit "src: rename protocol in the README" + + if [[ ${link} == true ]]; then + echo -e "\e[33mLinking protocol in the node, client and codec\e[0m" + sed "/let *${protocol_source} = active (Name.dev \"${protocol_source}\")/i \ let _${label} = active (Name.dev \"${label}\")\n" -i manifest/product_octez.ml + ocamlformat -i manifest/product_octez.ml + commit "manifest: link protocol in the node, client and codec" + + log_blue "Make manifest" + make -C manifest + commit "manifest: make manifest" + + fi + + # Update protocol tests + + # Replace test invocation headers that mention protocol_source + find "src/proto_${new_protocol_name}" -type f -path \*/test/\*.ml \ + -exec sed -i "s@Invocation:\(.*\)/proto_${protocol_source}/\(.*\)@Invocation:\1/proto_${new_protocol_name}/\2@" \{\} \; + commit "tests: fix test invocation headers" + + #Replace all occurences of \[capitalized_protocol_source\] with \[capitalized_label\] in src_proto_${new_protocol_name} + find "src/proto_${new_protocol_name}" -type f -exec sed -i "s/\\[${capitalized_source}\\]/\\[${capitalized_label}\\]/g" {} \; + commit "tests: fix tests registrations" + + #update scoru_wasm protocol_migratiton tests + # add proto_${label} to proto_name before Proto_alpha -> "Proto_alpha" + sed "/Proto_${protocol_source} -> \"Proto_${protocol_source}\"/i \ | ${capitalized_label} -> \"${capitalized_label}\"" -i.old src/lib_scoru_wasm/test/test_protocol_migration.ml + sed -r "s/\((V.*, V.*,) Proto_${protocol_source}\);/ \(\1 Proto_${protocol_source}\); \(\1 ${capitalized_label});/" -i.old src/lib_scoru_wasm/test/test_protocol_migration.ml + ocamlformat -i src/lib_scoru_wasm/test/test_protocol_migration.ml + + sed "/let proto_${protocol_source}_name = .*/i \let proto_${label}_name = \"${label}\"" -i.old src/lib_scoru_wasm/constants.ml + ocamlformat -i src/lib_scoru_wasm/constants.ml + + sed -r "s/(type protocol = .*)/\1 | ${capitalized_label}/" -i.old src/lib_scoru_wasm/pvm_input_kind.ml + sed -r "s/(type protocol = .*)/\1 | ${capitalized_label}/" -i.old src/lib_scoru_wasm/pvm_input_kind.mli + sed "/| payload when String.equal payload Constants.proto_${protocol_source}_name ->/i \ | payload when String.equal payload Constants.proto_${label}_name -> Some (Protocol_migration $capitalized_label)" -i.old src/lib_scoru_wasm/pvm_input_kind.ml + sed -r "s/(Data_encoding.\(Binary.to_string_exn string Constants.proto_${protocol_source}_name\))/\1 | ${capitalized_label} ->Data_encoding.(Binary.to_string_exn string Constants.proto_beta_name)/" -i.old src/lib_scoru_wasm/pvm_input_kind.ml + ocamlformat -i src/lib_scoru_wasm/pvm_input_kind.ml + ocamlformat -i src/lib_scoru_wasm/pvm_input_kind.mli + + sed -r "s/Proto_${protocol_source} -> (V.*)/ Proto_${protocol_source} -> \1 | ${capitalized_label} -> \1/" -i.old src/lib_scoru_wasm/wasm_vm.ml + ocamlformat -i src/lib_scoru_wasm/wasm_vm.ml + commit "scoru: update scoru_wasm protocol_migration" + + sed -e "s/Proto_${protocol_source}/${capitalized_label}/g" \ + -e "s/Tezos_scoru_wasm.Constants.proto_${protocol_source}_name/Tezos_scoru_wasm.Constants.proto_${label}_name/g" \ + -i.old "src/proto_${label}/lib_protocol/test/unit/test_sc_rollup_wasm.ml" + ocamlformat -i "src/proto_${label}/lib_protocol/test/unit/test_sc_rollup_wasm.ml" + commit "sc_rollup: update proto_${label}/test/unit/test_sc_rollup_wasm" + + # Update alpha + + log_blue "update raw_context.ml" + # add "else if Compare.String.(s = "$label") then return ($capitalized_label, ctxt)" before else Lwt.return @@ storage_error (Incompatible_protocol_version s) + #sed "/else Lwt.return @@ storage_error (Incompatible_protocol_version s)/i \ else if Compare.String.(s = \"${label}\") then return (${capitalized_label}, ctxt)" -i.old "src/proto_${protocol_source}/lib_protocol/raw_context.ml" + + start_source="\(\* Start of ${capitalized_source} stitching. Used for automatic protocol snapshot \*\)" + end_source="\(\* End of ${capitalized_source} stitching. Used for automatic protocol snapshot \*\)" + start_predecessor="\(\* Start of alpha predecessor stitching. Used for automatic protocol snapshot \*\)" + end_predecessor="\(\* End of alpha predecessor stitching. Used for automatic protocol snapshot \*\)" + type_to_remove="\(\* Alpha predecessor \*\)" + + start_remove="\(\* Start of code to remove at next automatic protocol snapshot \*\)" + end_remove="\(\* End of code to remove at next automatic protocol snapshot \*\)" + + start_typechecker="\(\* This line is only here to please the typechecker\," + end_typechecker="let\*\! c = get_previous_protocol_constants ctxt in" + + log_blue "fix prepare_first_block" + prepare_first_block=$(sed -n "/${start_source}/,/${end_source}/p" "src/proto_${protocol_source}/lib_protocol/raw_context.ml") + + # shellcheck disable=SC2001 + prepare_first_block_patched=$(sed "s/${capitalized_source}/${capitalized_label}/g" <<< "${prepare_first_block}") + # shellcheck disable=SC2001 + prepare_first_block_patched=$(sed "s/let module Previous = Constants_parametric_repr in/let module Previous = Constants_parametric_previous_repr in/" <<< "${prepare_first_block_patched}") + prepare_first_block_patched=$(perl -0777 -pe "s/${start_typechecker}.*${end_typechecker}//s" <<< "${prepare_first_block_patched}") + # shellcheck disable=SC2001 + prepare_first_block_patched=$(sed -e "s/let\* c = get_constants ctxt in/let*! c = get_previous_protocol_constants ctxt in/" <<< "${prepare_first_block_patched}") + escaped_prepare_first_block=$(printf '%s\n' "$prepare_first_block_patched" | sed 's/[`~!@#$%^&*()-_=+{}\|;:",<.>/?]/\\&/g') + #replace all multiline code between $start_predecessor and $end_predecessor with the content of prepare_first_block_patched in src/proto_${protocol_source}/lib_protocol/raw_context.ml using perl + perl -0777 -pe "s/${start_predecessor}.*${end_predecessor}/${escaped_prepare_first_block}/s" -i "src/proto_${protocol_source}/lib_protocol/raw_context.ml" + # remove all code between $start_remove and $end_remove in src/proto_${protocol_source}/lib_protocol/raw_context.ml + perl -0777 -pe "s/${start_remove}.*${end_remove}//s" -i "src/proto_${protocol_source}/lib_protocol/raw_context.ml" + #replace code between "$type_to_remove' and '$type_to_remove' with capitalized_label in src/proto_${protocol_source}/lib_protocol/raw_context.ml + perl -0777 -pe "s/${type_to_remove}.*${type_to_remove}/${capitalized_label}/" -i "src/proto_${protocol_source}/lib_protocol/raw_context.ml" + perl -0777 -pe "s/${type_to_remove}.*${type_to_remove}/${capitalized_label}/" -i "src/proto_${protocol_source}/lib_protocol/raw_context.mli" + replace_string="Compare.String.(s = \"${label}\") then return (${capitalized_label}, ctxt)" + #replace code between "$type_to_remove' and '$type_to_remove' with LOL in src/proto_${protocol_source}/lib_protocol/raw_context.ml + perl -0777 -pe "s/${type_to_remove}.*${type_to_remove}/$replace_string/s" -i "src/proto_${protocol_source}/lib_protocol/raw_context.ml" + + ocamlformat -i "src/proto_${protocol_source}/lib_protocol/raw_context.ml" + ocamlformat -i "src/proto_${protocol_source}/lib_protocol/raw_context.mli" + + prepare_first_block=$(sed -n "/${start_source}/,/${end_source}/p" "src/proto_${protocol_source}/lib_protocol/init_storage.ml") + # shellcheck disable=SC2001 + prepare_first_block_patched=$(echo "${prepare_first_block}" | sed "s/${capitalized_source}/${capitalized_label}/g") + escaped_prepare_first_block=$(printf '%s\n' "$prepare_first_block_patched" | sed 's/[`~!@#$%^&*()-_=+{}\|;:",<.>/?]/\\&/g') + #replace all code between '(* Start of alpha predecessor stitching. Used for automatic protocol snapshot *)' and '(* End of alpha predecessor stitching. Used for automatic protocol snapshot *)' with the content of prepare_first_block_patched in src/proto_${protocol_source}/lib_protocol/raw_context.ml + perl -0777 -pe "s/${start_predecessor}.*${end_predecessor}/${escaped_prepare_first_block}/s" -i "src/proto_${protocol_source}/lib_protocol/init_storage.ml" + ocamlformat -i "src/proto_${protocol_source}/lib_protocol/init_storage.ml" + commit "alpha: add ${capitalized_label} as Alpha previous protocol" + + make + + # Update tezt tests + + # automatically add the new protocol tag to alcotezt + + temp_file=$(mktemp) + tac tezt/lib_alcotezt/alcotezt_utils.ml | tail +2 | tac > "${temp_file}" ## The script may create commits. Keep track of the number of commits to clean up in case of an error + echo " | Some \"${new_protocol_name}\" -> [\"${label}\"]" >> "${temp_file}" + echo " | Some _ -> assert false" >> "${temp_file}" + mv "${temp_file}" tezt/lib_alcotezt/alcotezt_utils.ml + commit "tezt: add new protocol tag to alcotezt" + + cd "${script_dir}"/.. + + # Adapt tezt/lib_tezos/protocol.ml + + log_blue "Adapt tezt/lib_tezos/protocol.ml" + + #replace "| capitalized_source$" with "| capitalized_source | capitalized_label$" + # replace capitalized_source -> "capitalized_source" with capitalized_source -> "capitalized_source" | capitalized_label -> "capitalized_label" + # replace capitalized_source -> "protocol_source" with "capitalized_source -> "protocol_source" | capitalized_label -> "label" + + sed -i.old -e "s/type t = / type t = ${capitalized_label} | /g" \ + -e "s/${capitalized_source} -> \"${capitalized_source}\"/${capitalized_source} -> \"${capitalized_source}\" | ${capitalized_label} -> \"${capitalized_label}\"/g" \ + -e "s/${capitalized_source} -> \"${protocol_source}\"/${capitalized_source} -> \"${protocol_source}\" | ${capitalized_label} -> \"${label}\"/g" \ + tezt/lib_tezos/protocol.ml + sed -i.old -e "s/let all = \[/let all = [ ${capitalized_label};/" tezt/lib_tezos/protocol.ml + + #source=Alpha; echo "let number = function ParisC -> 020 | Alpha -> 021" |sed -r 's/(.*) '$source' -> ([0-9][0-9][0-9])/printf "\1 Beta -> \2 | '$source' -> %03i" "$(echo \2+1 | bc)"/ge' + # shellcheck disable=SC2086 + # shellcheck disable=SC2016 + sed -r 's/(.*) '${capitalized_source}' -> ([0-9][0-9][0-9])/printf "\1 '${capitalized_label}' -> \2 | '${capitalized_source}' -> %03i" "$(echo \2+1 | bc)"/ge' -i.old tezt/lib_tezos/protocol.ml + + sed "/| ${capitalized_source} -> \"proto_${protocol_source}\"/i | ${capitalized_label} -> \"proto_${label}\"\n" -i.old tezt/lib_tezos/protocol.ml + + # add $(capitalized_label) -> "$long_hash" before "(* DO NOT REMOVE, AUTOMATICALLY ADD STABILISED PROTOCOL HASH HERE *)" + sed "/\(\* DO NOT REMOVE, AUTOMATICALLY ADD STABILISED PROTOCOL HASH HERE \*\)/i \ | ${capitalized_label} -> \"${long_hash}\"" -i.old tezt/lib_tezos/protocol.ml + + sed -r "s/(.*) ${capitalized_source} -> Some (.*)/\1 ${capitalized_source} -> Some ${capitalized_label} | ${capitalized_label} -> Some \2/g" -i.old tezt/lib_tezos/protocol.ml + sed -i.old -e "s/type t = / type t = ${capitalized_label} | /g" tezt/lib_tezos/protocol.mli + ocamlformat -i tezt/lib_tezos/protocol.ml + ocamlformat -i tezt/lib_tezos/protocol.mli + commit "tezt: adapt lib_tezos/protocol.ml" + + log_blue "generate regression test" + # shellcheck disable=SC2312 + # shellcheck disable=SC1003 + printf ' +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) %s Nomadic Labs *) +(* *) +(*****************************************************************************) + +(* Testing + ------- + Component: Protocol + Invocation: dune exec tezt/tests/main.exe -- --file check_proto_%s_changes.ml + Subject: Ensure protocol_%s has not changed +*) + +let register () = + Regression.register + ~__FILE__ + ~title:"Check that the %s protocol has not changed" + ~tags:[Tag.layer1; "protocol"; "%s"] + ~uses_node:false + ~uses_client:false + ~uses_admin_client:false + @@ fun _protocol -> + (* Check that md5sum of all files in src/proto_%s is consistent *) + let _, _ = + Unix.open_process + "mkdir /tmp/tezos_proto_snapshot && git archive HEAD src/proto_%s | tar -x -C /tmp/tezos_proto_snapshot" + in + let ic, _ = + Unix.open_process + "find /tmp/tezos_proto_snapshot -type f -exec md5sum {} %s; | sort -k 2 | md5sum" + in + let _, _ = Unix.open_process "rm -rf /tmp/tezos_proto_snapshot" in + + let output = input_line ic in + Regression.capture output ; + return ()' "$(date +%Y)" "${label}" "${label}" "${label}" "${label}" "${label}" "${label}" '\\' > "tezt/tests/check_proto_${label}_changes.ml" + ocamlformat -i "tezt/tests/check_proto_${label}_changes.ml" + if [[ -n ${git} ]]; then + git add "tezt/tests/check_proto_${label}_changes.ml" + fi + + sed -i.old -e "s/let register_protocol_independent_tests () =/let register_protocol_independent_tests () =\n Check_proto_${label}_changes.register ();/" tezt/tests/main.ml + ocamlformat -i tezt/tests/main.ml + commit "tezt: generate regression test" + + #fix other tests: + sed -r "s/(.*) Protocol.${capitalized_source} -> (.*)/ \1 Protocol.${capitalized_source} -> \2 | Protocol.${capitalized_label} -> \2/g" -i tezt/tests/*.ml + ocamlformat -i tezt/tests/*.ml + commit "tezt: fix other tests" + + mkdir -p "tezt/tests/encoding_samples/${label}" + cp -r tezt/tests/encoding_samples/"${protocol_source}"/* tezt/tests/encoding_samples/"${label}" + commit "tezt: copy ${protocol_source} encoding samples to ${label}" + + # for regression files, protocol_name should be at least 5 character long, if not add enough trailing '-' at the end + regression_protocol_name=${capitalized_label} + while [[ ${#regression_protocol_name} -lt 5 ]]; do + regression_protocol_name="${regression_protocol_name}-" + done + + # extract hash from src/${protocol_source}/TEZOS_PROTOCOL in line "hash": "..." + source_hash=$(grep -oP '(?<="hash": ")[^"]*' "src/proto_${protocol_source}/lib_protocol/TEZOS_PROTOCOL") + + # shellcheck disable=SC2312 + # shellcheck disable=SC2001 + find . -type f -name "*${capitalized_source}*.out" | while read -r FILE; do + NEW_FILENAME=$(echo "${FILE}" | sed "s/${capitalized_source}/${regression_protocol_name}/g") + + # Create the directory structure for the new file if it doesn't exist + mkdir -p "$(dirname "${NEW_FILENAME}")" + + # Preserve the file permissions + cp -p "${FILE}" "${NEW_FILENAME}" + + #replace all occurences of protocol_source with label + sed -i.old -e "s/${protocol_source}/${label}/g" "${NEW_FILENAME}" + + #replace all occurences of old hash with new hash + sed -i.old -e "s/${source_hash}/${long_hash}/g" "${NEW_FILENAME}" + done + commit "tezt: copy ${protocol_source} regression files" + + mkdir -p "tezt/tests/expected/check_proto_${label}_changes.ml" + rm -rf /tmp/tezos_proto_snapshot + mkdir -p /tmp/tezos_proto_snapshot + git archive HEAD "src/proto_${label}/" | tar -x -C /tmp/tezos_proto_snapshot + find /tmp/tezos_proto_snapshot -type f -exec md5sum {} \; | sort -k 2 | md5sum > "tezt/tests/expected/check_proto_${label}_changes.ml/Check that the ${label} protocol has not changed.out" + rm -rf /tmp/tezos_proto_snapshot + if [[ -n ${git} ]]; then + git add "tezt/tests/expected/check_proto_${label}_changes.ml" + fi + commit "tezt: add expected output for stabilisation regression test" + + dune exec tezt/tests/main.exe -- --on-unknown-regression-files delete + commit "tezt: delete unknown regression files" + + dune exec tezt/tests/main.exe -- --title 'meta: list runtime dependencies' --reset-regressions + commit "tezt: reset runtime dependencies regressions" + + dune exec tezt/tests/main.exe -- --file tezt/tests/protocol_migration.ml --title 'Alpha: weeklynet regression test' --reset-regressions + commit "tezt: reset weeklynet regression test" + + dune exec tezt/tests/main.exe -- --file "tezt/tests/check_proto_${label}_changes.ml" --title "Check that the ${label} protocol has not changed" --reset-regressions + commit "tezt: reset beta regression test" + + log_blue "Update kaitai structs" + make kaitai-struct-files-update + commit "kaitai: update structs" + + # Misc. updates + + log_blue "add octez-activate-${label} command to client sandbox" + + sed "/parameters_file=\"\$bin_dir.*/i \ ${label}_parameters_file=\"\$bin_dir/../../_build/default/src/proto_${label}/lib_parameters/sandbox-parameters.json\"" -i src/bin_client/octez-init-sandboxed-client.sh + sed "/alias octez-activate-${protocol_source}=.*/i \alias octez-activate-${label}=\"\$client -block genesis activate protocol ${long_hash} with fitness 1 and key activator and parameters \$${label}_parameters_file\";" -i src/bin_client/octez-init-sandboxed-client.sh + commit "sandbox: add octez-activate-${label} command to client sandbox" + + sed "/let protocol_${protocol_source}_parameters_template =/i \ let protocol_${label}_parameters_template =\n Filename.current_dir_name \/\/ \"src\" \/\/ \"proto_${label}\" \/\/ \"parameters\"\n \/\/ \"mainnet_parameters.json\"" -i.old devtools/testnet_experiment_tools/testnet_experiment_tools.ml + sed "/| Tezt_tezos.Protocol.Alpha ->/i \ | Tezt_tezos.Protocol.${capitalized_label} ->\n Some protocol_${label}_parameters_template" -i.old devtools/testnet_experiment_tools/testnet_experiment_tools.ml + ocamlformat -i devtools/testnet_experiment_tools/testnet_experiment_tools.ml + commit "devtools: update testnet_experiment_tools" + + # update linter to allow reformating of beta protocol + sed -i.old -e "s/-not -name \"proto_${protocol_source}\"/-name \"proto_${protocol_source}\" -name \"proto_${label}\"/" scripts/lint.sh + commit "scripts: update linter to allow reformating of beta protocol" + + log_blue "Update ci" + make -C ci + commit "ci: regenerate ci" + # Create proto_beta docs + + echo "Copying docs/${protocol_source} to docs/${label}" + mkdir /tmp/tezos_proto_doc_snapshot + git archive HEAD "docs/${protocol_source}/" | tar -x -C /tmp/tezos_proto_doc_snapshot + mv "/tmp/tezos_proto_doc_snapshot/docs/${protocol_source}" "docs/${label}" + rm -rf /tmp/tezos_proto_doc_snapshot commit "copy from docs/${protocol_source}" + commit "docs: copy from docs/${protocol_source}" + + # fix versioned links (in labels, references, and paths) in docs + echo "Fixing versioned links in docs" + cd "docs/${label}" + find . -name \*.rst -exec \ + sed -i.old \ + -e "s,src/proto_${protocol_source},src/proto_${new_protocol_name},g" \ + -e "s,tezos-protocol-${tezos_protocol_source}/,tezos-protocol-${new_tezos_protocol}/,g" \ + -e "s,raw_protocol_${protocol_source}/,raw_protocol_${new_protocol_name}/,g" \ + -e "s/_${protocol_source}:/_${label}:/g" \ + -e "s/_${protocol_source}>/_${label}>/g" \ + -e "s/_${protocol_source}\`/_${label}\`/g" \ + -e "s/-${protocol_source}.html/-${label}.html/g" \ + \{\} \; + commit "docs: fix versioned links" + + cd ../.. + + # generate docs/protocols/${new_versioned_name}.rst from docs/protocols/${protocol_source}.rst + echo "Copying docs/protocols/${protocol_source}.rst to docs/protocols/${new_versioned_name}.rst" + cp "docs/protocols/${protocol_source}.rst" "docs/protocols/${new_versioned_name}.rst" + commit "docs: copy docs/protocols/${protocol_source}.rst to docs/protocols/${new_versioned_name}.rst" + sed -e "s/^Protocol ${capitalized_source}/Protocol ${capitalized_label}/" \ + -e "s/protocol ${capitalized_source}/protocol ${capitalized_label}/" \ + -e "s,src/proto_${protocol_source},src/proto_${new_protocol_name},g" \ + -i "docs/protocols/${new_protocol_name}.rst" + commit "docs: fix docs/protocols/${new_protocol_name}.rst" + + # add entries in the doc index for the snaptshotted protocol by + # pattern-matching some existing lines and inserting variations thereof + echo "Add entries in the doc index" + doc_index="docs/index.rst" + + awk -v PATTERN1="${capitalized_source} Dev Protocol Reference <${protocol_source}/index>" \ + -v REPLACEMENT1="${capitalized_label} Protocol Reference <${label}/index>" \ + -v PATTERN2="protocols/${protocol_source}" \ + -v REPLACEMENT2="protocols/${new_versioned_name}" '{ + if ($0 ~ PATTERN1) { + x=$0 + sub(PATTERN1,REPLACEMENT1) + print + print x + } else if ($0 ~ PATTERN2) { + x=$0 + sub(PATTERN2,REPLACEMENT2) + print + print x + } else { + print + } + }' < "${doc_index}" > "${doc_index}.tmp" + mv "${doc_index}.tmp" "${doc_index}" + + commit "docs: add entries in the doc index" + + # update docs Makefile + sed -i.old -r "s/(PROTOCOLS .*) ${protocol_source}/\1 ${protocol_source} ${label}/" docs/Makefile + sed "/${protocol_source}_long .*/i \ ${label}_long = ${long_hash}" -i docs/Makefile + sed "/${protocol_source}_short .*/i \ ${label}_short = ${label}" -i docs/Makefile + sed -i.old -e "s/${protocol_source}\/rpc\.rst/${protocol_source}\/rpc.rst ${label}\/rpc.rst/g" docs/Makefile + commit "docs: update docs Makefile" + + make -C docs "${label}/rpc.rst" + if [[ -n ${git} ]]; then + git add "docs/${label}/rpc.rst" + fi + commit "docs: generate ${label}/rpc.rst" + +} + +#run command +case ${command} in +stabilise) + label=${protocol_target} + version=${protocol_target} + snapshot_protocol "${protocol_source}" "${protocol_target}" + + ;; + +esac + +# remove files generated by sed +find . -name '*.old' -exec rm {} \; + +if [[ -n ${git} ]]; then + echo "${commits} commits created" + echo "You can review them and squash them if needed" + echo "if you want to remove them you can run" + echo "git reset --hard HEAD~${commits}" +fi diff --git a/tezt/lib_tezos/protocol.ml b/tezt/lib_tezos/protocol.ml index f8b1a22b24af..826b4d94417d 100644 --- a/tezt/lib_tezos/protocol.ml +++ b/tezt/lib_tezos/protocol.ml @@ -57,6 +57,7 @@ let tag protocol = String.lowercase_ascii (name protocol) let hash = function | Alpha -> "ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK" | ParisC -> "PsParisCZo7KAh1Z1smVd9ZMZ1HHn5gkzbM94V3PLCpknFWhUAi" +(* DO NOT REMOVE, AUTOMATICALLY ADD STABILISED PROTOCOL HASH HERE *) let genesis_hash = "ProtoGenesisGenesisGenesisGenesisGenesisGenesk612im" @@ -321,7 +322,10 @@ let iter_on_supported_protocols ~title ~protocols ?(supports = Any_protocol) f = let add_to_test_parameters ?(additional_tags = fun _ -> []) protocol title tags uses = let uses = match uses with None -> [] | Some uses -> uses protocol in - ( name protocol ^ ": " ^ title, + (* To help automatic generation of regression files during stabilisation we ensure + protocol_name is always of at least size 5 by adding '-' trailing chars to it if needed *) + let protocol_name = name protocol ^ (String.make (max 0 (5 - String.length (name protocol))) '-') in + ( protocol_name ^ ": " ^ title, (tag protocol :: tags) @ additional_tags protocol, uses ) -- GitLab From 08b319b1ac878dfd6ba928afa870b602c8f867c6 Mon Sep 17 00:00:00 2001 From: mbourgoin Date: Tue, 16 Jul 2024 14:05:36 +0200 Subject: [PATCH 4/4] WIP: introduce automatic snapshotting --- scripts/proto_manager.sh | 242 ++++++++++++++++++++++++++++----------- 1 file changed, 174 insertions(+), 68 deletions(-) diff --git a/scripts/proto_manager.sh b/scripts/proto_manager.sh index 1dbd2c9b4848..3433d9d61a6b 100644 --- a/scripts/proto_manager.sh +++ b/scripts/proto_manager.sh @@ -97,12 +97,14 @@ function usage() { g="${green}-g${reset}" h="${green}-h${reset}" s="${green}-s${reset}" + p="${green}-p${reset}" from="${green}--from${reset}" to="${green}--to${reset}" link="${green}--link${reset}" git="${green}--git${reset}" help="${green}--help${reset}" stabilise="${green}--stabilise${reset}" + snapshot="${green}--snapshot${reset}" ## colored vars alpha="${red}alpha${reset}" beta="${red}beta${reset}" @@ -116,10 +118,12 @@ ${red}Options:${reset} Print this help message. ${s}, ${stabilise} Stabilise a protocol. + ${p}, ${snapshot} + Snapshot a protocol. ${f}, ${from} ${red}${reset} - The source protocol to stabilise. + The source protocol to stabilise or snapshot. ${t}, ${to} ${red}${reset} - The target protocol to stabilise. + The target protocol to stabilise or snapshot. ${l}, ${link} Link the protocol in the node, client and codec. ${g}, ${git} @@ -131,6 +135,11 @@ ${yellow}tl;dr:${reset} ${script} ${stabilise} ${from} ${alpha} ${to} ${beta} ${link} ${git} or ${script} ${s} ${f} ${alpha} ${t} ${beta} ${l} ${g} + +- To snapshot protocol alpha into 023_stockholm and link it in the node, client and codec: + ${script} ${snapshot} ${from} ${beta} ${to} ${_023_stockholm} ${link} ${git} + or + ${script} ${p} ${f} ${beta} ${t} ${_023_stockholm} ${l} ${g} " } @@ -145,6 +154,10 @@ while true; do command="stabilise" shift ;; + -p | --snapshot) + command="snapshot" + shift + ;; -f | --from) protocol_source="$2" shift 2 @@ -179,7 +192,7 @@ if [[ -z ${command} ]]; then usage 1>&2 print_and_exit 1 "${LINENO}" fi -if [[ ${command} == "stabilise" ]]; then +if [[ ${command} == "stabilise" ]] || [[ ${command} == "snapshot" ]]; then if [[ -z ${protocol_target} ]]; then error "No protocol target specified" 1>&2 usage 1>&2 @@ -226,6 +239,18 @@ if [[ ${command} == "stabilise" ]]; then fi fi +# if snapshot_command is used, protocol_target should be of the form [a-z]+_[0-9][0-9][0-9] +if [[ ${command} == "snapshot" ]]; then + if ! [[ ${protocol_source} =~ ^[a-z]+[0-9]*$ ]]; then + error "To ${red}snapshot${reset}, protocol_source should be of the form [a-z]+[0-9]+" 1>&2 + print_and_exit 1 "${LINENO}" + fi + if ! [[ ${protocol_target} =~ ^[a-z]+_[0-9][0-9][0-9]$ ]]; then + error "To ${red}snapshot${reset}, protocol_target should be of the form [a-z]+_[0-9][0-9][0-9]" 1>&2 + clean_and_exit 1 "${LINENO}" + fi +fi + # check if ./octez-protocol-compiler is present if [[ ! -f "./octez-protocol-compiler" ]]; then error "octez-protocol-compiler not found, compile it before running this script" 1>&2 @@ -250,6 +275,12 @@ function snapshot_protocol() { # e.g. Pt8PY9P4 (set below) short_hash= + if [[ ${version} != "${label}" ]]; then + is_snapshot=true + else + is_snapshot=false + fi + if ! [[ -d "src/proto_${protocol_source}" ]]; then error "'src/proto_${protocol_source}' does not exist" 1>&2 print_and_exgxit 1 "${LINENO}" @@ -289,11 +320,17 @@ function snapshot_protocol() { # if it is not strictly needed anymore. echo "Setting current version in raw_context and proxy" - sed -i.old.old -e "s/let version_value = \"${protocol_source}_current\"/let version_value = \"${protocol_target}\"/" \ - "src/proto_${version}/lib_protocol/constants_repr.ml" \ - "src/proto_${version}/lib_protocol/raw_context.ml" \ - "src/proto_${version}/lib_client/proxy.ml" - + if [[ ${is_snapshot} == true ]]; then + sed -i.old.old -e "s/let version_value = \"${protocol_source}\"/let version_value = \"${protocol_target}\"/" \ + "src/proto_${version}/lib_protocol/constants_repr.ml" \ + "src/proto_${version}/lib_protocol/raw_context.ml" \ + "src/proto_${version}/lib_client/proxy.ml" + else + sed -i.old.old -e "s/let version_value = \"${protocol_source}_current\"/let version_value = \"${protocol_target}\"/" \ + "src/proto_${version}/lib_protocol/constants_repr.ml" \ + "src/proto_${version}/lib_protocol/raw_context.ml" \ + "src/proto_${version}/lib_client/proxy.ml" + fi commit "src: set current version" log_cyan "Computing hash" @@ -302,9 +339,24 @@ function snapshot_protocol() { log_magenta "Hash computed: ${long_hash}" log_magenta "Short hash: ${short_hash}" - new_protocol_name="${version}" - new_tezos_protocol="${version}" - new_versioned_name="${label}" + if [[ ${is_snapshot} == true ]]; then + echo "Renaming src/proto_${version} to src/proto_${version}_${short_hash}" + new_protocol_name="${version}_${short_hash}" + new_tezos_protocol="${version}-${short_hash}" + new_versioned_name="${version}_${label}" + + if [[ -d "src/proto_${new_protocol_name}" ]]; then + error "'src/proto_${new_protocol_name}' already exists, you should remove it" + print_and_exit 1 "${LINENO}" + fi + + git mv "src/proto_${version}" "src/proto_${new_protocol_name}" + commit "src: rename proto_${version} to proto_${new_protocol_name}" + else + new_protocol_name="${version}" + new_tezos_protocol="${version}" + new_versioned_name="${label}" + fi cd "src/proto_${new_protocol_name}" # rename main_*.ml{,i} files of the binaries @@ -337,9 +389,14 @@ function snapshot_protocol() { cd ../../.. - sed -i "s/${protocol_source}/${label}/g" "src/proto_${label}/README.md" + if [[ ${is_snapshot} == true ]]; then + rm "src/proto_${new_protocol_name}/README.md" + commit "src: remove README" + else + sed -i "s/${protocol_source}/${label}/g" "src/proto_${label}/README.md" - commit "src: rename protocol in the README" + commit "src: rename protocol in the README" + fi if [[ ${link} == true ]]; then echo -e "\e[33mLinking protocol in the node, client and codec\e[0m" @@ -350,7 +407,9 @@ function snapshot_protocol() { log_blue "Make manifest" make -C manifest commit "manifest: make manifest" - + if [[ ${command} == "snapshot" ]]; then + warning "${protocol_source} has been unlinked in the manifest, you can safely remove it from the source code" + fi fi # Update protocol tests @@ -458,39 +517,42 @@ function snapshot_protocol() { cd "${script_dir}"/.. # Adapt tezt/lib_tezos/protocol.ml + if [[ ${is_snapshot} == true ]]; then + echo "Unimplemented yet for snapshots" + else - log_blue "Adapt tezt/lib_tezos/protocol.ml" + log_blue "Adapt tezt/lib_tezos/protocol.ml" - #replace "| capitalized_source$" with "| capitalized_source | capitalized_label$" - # replace capitalized_source -> "capitalized_source" with capitalized_source -> "capitalized_source" | capitalized_label -> "capitalized_label" - # replace capitalized_source -> "protocol_source" with "capitalized_source -> "protocol_source" | capitalized_label -> "label" + #replace "| capitalized_source$" with "| capitalized_source | capitalized_label$" + # replace capitalized_source -> "capitalized_source" with capitalized_source -> "capitalized_source" | capitalized_label -> "capitalized_label" + # replace capitalized_source -> "protocol_source" with "capitalized_source -> "protocol_source" | capitalized_label -> "label" - sed -i.old -e "s/type t = / type t = ${capitalized_label} | /g" \ - -e "s/${capitalized_source} -> \"${capitalized_source}\"/${capitalized_source} -> \"${capitalized_source}\" | ${capitalized_label} -> \"${capitalized_label}\"/g" \ - -e "s/${capitalized_source} -> \"${protocol_source}\"/${capitalized_source} -> \"${protocol_source}\" | ${capitalized_label} -> \"${label}\"/g" \ - tezt/lib_tezos/protocol.ml - sed -i.old -e "s/let all = \[/let all = [ ${capitalized_label};/" tezt/lib_tezos/protocol.ml + sed -i.old -e "s/type t = / type t = ${capitalized_label} | /g" \ + -e "s/${capitalized_source} -> \"${capitalized_source}\"/${capitalized_source} -> \"${capitalized_source}\" | ${capitalized_label} -> \"${capitalized_label}\"/g" \ + -e "s/${capitalized_source} -> \"${protocol_source}\"/${capitalized_source} -> \"${protocol_source}\" | ${capitalized_label} -> \"${label}\"/g" \ + tezt/lib_tezos/protocol.ml + sed -i.old -e "s/let all = \[/let all = [ ${capitalized_label};/" tezt/lib_tezos/protocol.ml - #source=Alpha; echo "let number = function ParisC -> 020 | Alpha -> 021" |sed -r 's/(.*) '$source' -> ([0-9][0-9][0-9])/printf "\1 Beta -> \2 | '$source' -> %03i" "$(echo \2+1 | bc)"/ge' - # shellcheck disable=SC2086 - # shellcheck disable=SC2016 - sed -r 's/(.*) '${capitalized_source}' -> ([0-9][0-9][0-9])/printf "\1 '${capitalized_label}' -> \2 | '${capitalized_source}' -> %03i" "$(echo \2+1 | bc)"/ge' -i.old tezt/lib_tezos/protocol.ml + #source=Alpha; echo "let number = function ParisC -> 020 | Alpha -> 021" |sed -r 's/(.*) '$source' -> ([0-9][0-9][0-9])/printf "\1 Beta -> \2 | '$source' -> %03i" "$(echo \2+1 | bc)"/ge' + # shellcheck disable=SC2086 + # shellcheck disable=SC2016 + sed -r 's/(.*) '${capitalized_source}' -> ([0-9][0-9][0-9])/printf "\1 '${capitalized_label}' -> \2 | '${capitalized_source}' -> %03i" "$(echo \2+1 | bc)"/ge' -i.old tezt/lib_tezos/protocol.ml - sed "/| ${capitalized_source} -> \"proto_${protocol_source}\"/i | ${capitalized_label} -> \"proto_${label}\"\n" -i.old tezt/lib_tezos/protocol.ml + sed "/| ${capitalized_source} -> \"proto_${protocol_source}\"/i | ${capitalized_label} -> \"proto_${label}\"\n" -i.old tezt/lib_tezos/protocol.ml - # add $(capitalized_label) -> "$long_hash" before "(* DO NOT REMOVE, AUTOMATICALLY ADD STABILISED PROTOCOL HASH HERE *)" - sed "/\(\* DO NOT REMOVE, AUTOMATICALLY ADD STABILISED PROTOCOL HASH HERE \*\)/i \ | ${capitalized_label} -> \"${long_hash}\"" -i.old tezt/lib_tezos/protocol.ml + # add $(capitalized_label) -> "$long_hash" before "(* DO NOT REMOVE, AUTOMATICALLY ADD STABILISED PROTOCOL HASH HERE *)" + sed "/\(\* DO NOT REMOVE, AUTOMATICALLY ADD STABILISED PROTOCOL HASH HERE \*\)/i \ | ${capitalized_label} -> \"${long_hash}\"" -i.old tezt/lib_tezos/protocol.ml - sed -r "s/(.*) ${capitalized_source} -> Some (.*)/\1 ${capitalized_source} -> Some ${capitalized_label} | ${capitalized_label} -> Some \2/g" -i.old tezt/lib_tezos/protocol.ml - sed -i.old -e "s/type t = / type t = ${capitalized_label} | /g" tezt/lib_tezos/protocol.mli - ocamlformat -i tezt/lib_tezos/protocol.ml - ocamlformat -i tezt/lib_tezos/protocol.mli - commit "tezt: adapt lib_tezos/protocol.ml" + sed -r "s/(.*) ${capitalized_source} -> Some (.*)/\1 ${capitalized_source} -> Some ${capitalized_label} | ${capitalized_label} -> Some \2/g" -i.old tezt/lib_tezos/protocol.ml + sed -i.old -e "s/type t = / type t = ${capitalized_label} | /g" tezt/lib_tezos/protocol.mli + ocamlformat -i tezt/lib_tezos/protocol.ml + ocamlformat -i tezt/lib_tezos/protocol.mli + commit "tezt: adapt lib_tezos/protocol.ml" - log_blue "generate regression test" - # shellcheck disable=SC2312 - # shellcheck disable=SC1003 - printf ' + log_blue "generate regression test" + # shellcheck disable=SC2312 + # shellcheck disable=SC1003 + printf ' (*****************************************************************************) (* *) (* SPDX-License-Identifier: MIT *) @@ -528,19 +590,20 @@ let register () = let output = input_line ic in Regression.capture output ; return ()' "$(date +%Y)" "${label}" "${label}" "${label}" "${label}" "${label}" "${label}" '\\' > "tezt/tests/check_proto_${label}_changes.ml" - ocamlformat -i "tezt/tests/check_proto_${label}_changes.ml" - if [[ -n ${git} ]]; then - git add "tezt/tests/check_proto_${label}_changes.ml" - fi + ocamlformat -i "tezt/tests/check_proto_${label}_changes.ml" + if [[ -n ${git} ]]; then + git add "tezt/tests/check_proto_${label}_changes.ml" + fi - sed -i.old -e "s/let register_protocol_independent_tests () =/let register_protocol_independent_tests () =\n Check_proto_${label}_changes.register ();/" tezt/tests/main.ml - ocamlformat -i tezt/tests/main.ml - commit "tezt: generate regression test" + sed -i.old -e "s/let register_protocol_independent_tests () =/let register_protocol_independent_tests () =\n Check_proto_${label}_changes.register ();/" tezt/tests/main.ml + ocamlformat -i tezt/tests/main.ml + commit "tezt: generate regression test" - #fix other tests: - sed -r "s/(.*) Protocol.${capitalized_source} -> (.*)/ \1 Protocol.${capitalized_source} -> \2 | Protocol.${capitalized_label} -> \2/g" -i tezt/tests/*.ml - ocamlformat -i tezt/tests/*.ml - commit "tezt: fix other tests" + #fix other tests: + sed -r "s/(.*) Protocol.${capitalized_source} -> (.*)/ \1 Protocol.${capitalized_source} -> \2 | Protocol.${capitalized_label} -> \2/g" -i tezt/tests/*.ml + ocamlformat -i tezt/tests/*.ml + commit "tezt: fix other tests" + fi mkdir -p "tezt/tests/encoding_samples/${label}" cp -r tezt/tests/encoding_samples/"${protocol_source}"/* tezt/tests/encoding_samples/"${label}" @@ -574,16 +637,18 @@ let register () = done commit "tezt: copy ${protocol_source} regression files" - mkdir -p "tezt/tests/expected/check_proto_${label}_changes.ml" - rm -rf /tmp/tezos_proto_snapshot - mkdir -p /tmp/tezos_proto_snapshot - git archive HEAD "src/proto_${label}/" | tar -x -C /tmp/tezos_proto_snapshot - find /tmp/tezos_proto_snapshot -type f -exec md5sum {} \; | sort -k 2 | md5sum > "tezt/tests/expected/check_proto_${label}_changes.ml/Check that the ${label} protocol has not changed.out" - rm -rf /tmp/tezos_proto_snapshot - if [[ -n ${git} ]]; then - git add "tezt/tests/expected/check_proto_${label}_changes.ml" + if [[ ${is_snapshot} == false ]]; then + mkdir -p "tezt/tests/expected/check_proto_${label}_changes.ml" + rm -rf /tmp/tezos_proto_snapshot + mkdir -p /tmp/tezos_proto_snapshot + git archive HEAD "src/proto_${label}/" | tar -x -C /tmp/tezos_proto_snapshot + find /tmp/tezos_proto_snapshot -type f -exec md5sum {} \; | sort -k 2 | md5sum > "tezt/tests/expected/check_proto_${label}_changes.ml/Check that the ${label} protocol has not changed.out" + rm -rf /tmp/tezos_proto_snapshot + if [[ -n ${git} ]]; then + git add "tezt/tests/expected/check_proto_${label}_changes.ml" + fi + commit "tezt: add expected output for stabilisation regression test" fi - commit "tezt: add expected output for stabilisation regression test" dune exec tezt/tests/main.exe -- --on-unknown-regression-files delete commit "tezt: delete unknown regression files" @@ -605,9 +670,15 @@ let register () = log_blue "add octez-activate-${label} command to client sandbox" - sed "/parameters_file=\"\$bin_dir.*/i \ ${label}_parameters_file=\"\$bin_dir/../../_build/default/src/proto_${label}/lib_parameters/sandbox-parameters.json\"" -i src/bin_client/octez-init-sandboxed-client.sh - sed "/alias octez-activate-${protocol_source}=.*/i \alias octez-activate-${label}=\"\$client -block genesis activate protocol ${long_hash} with fitness 1 and key activator and parameters \$${label}_parameters_file\";" -i src/bin_client/octez-init-sandboxed-client.sh - commit "sandbox: add octez-activate-${label} command to client sandbox" + if [[ ${is_snapshot} == true ]]; then + sed -i.old -e "s@${protocol_source}_parameters_file=\"\$bin_dir.*@${label}_parameters_file=\"\$bin_dir/../../_build/default/src/proto_${new_protocol_name}/lib_parameters/sandbox-parameters.json\"@" \ + -e "s/alias octez-activate-${protocol_source}=.*/alias octez-activate-${label}=\"\$client -block genesis activate protocol ${long_hash} with fitness 1 and key activator and parameters \$${label}_parameters_file\";/" \ + src/bin_client/octez-init-sandboxed-client.sh + else + sed "/parameters_file=\"\$bin_dir.*/i \ ${label}_parameters_file=\"\$bin_dir/../../_build/default/src/proto_${label}/lib_parameters/sandbox-parameters.json\"" -i src/bin_client/octez-init-sandboxed-client.sh + sed "/alias octez-activate-${protocol_source}=.*/i \alias octez-activate-${label}=\"\$client -block genesis activate protocol ${long_hash} with fitness 1 and key activator and parameters \$${label}_parameters_file\";" -i src/bin_client/octez-init-sandboxed-client.sh + commit "sandbox: add octez-activate-${label} command to client sandbox" + fi sed "/let protocol_${protocol_source}_parameters_template =/i \ let protocol_${label}_parameters_template =\n Filename.current_dir_name \/\/ \"src\" \/\/ \"proto_${label}\" \/\/ \"parameters\"\n \/\/ \"mainnet_parameters.json\"" -i.old devtools/testnet_experiment_tools/testnet_experiment_tools.ml sed "/| Tezt_tezos.Protocol.Alpha ->/i \ | Tezt_tezos.Protocol.${capitalized_label} ->\n Some protocol_${label}_parameters_template" -i.old devtools/testnet_experiment_tools/testnet_experiment_tools.ml @@ -662,10 +733,14 @@ let register () = echo "Add entries in the doc index" doc_index="docs/index.rst" - awk -v PATTERN1="${capitalized_source} Dev Protocol Reference <${protocol_source}/index>" \ - -v REPLACEMENT1="${capitalized_label} Protocol Reference <${label}/index>" \ - -v PATTERN2="protocols/${protocol_source}" \ - -v REPLACEMENT2="protocols/${new_versioned_name}" '{ + if [[ ${is_snapshot} == true ]]; then + # we are snapshotting, simply replace all occurences of ${protocol_source} with ${label} in the index + sed -i.old -e "s/${protocol_source}/${label}/g" "${doc_index}" + else + awk -v PATTERN1="${capitalized_source} Dev Protocol Reference <${protocol_source}/index>" \ + -v REPLACEMENT1="${capitalized_label} Protocol Reference <${label}/index>" \ + -v PATTERN2="protocols/${protocol_source}" \ + -v REPLACEMENT2="protocols/${new_versioned_name}" '{ if ($0 ~ PATTERN1) { x=$0 sub(PATTERN1,REPLACEMENT1) @@ -680,8 +755,8 @@ let register () = print } }' < "${doc_index}" > "${doc_index}.tmp" - mv "${doc_index}.tmp" "${doc_index}" - + mv "${doc_index}.tmp" "${doc_index}" + fi commit "docs: add entries in the doc index" # update docs Makefile @@ -708,6 +783,37 @@ stabilise) ;; +snapshot) + # e.g. stockholm + label=$(echo "${protocol_target}" | cut -d'_' -f1) + # e.g. 022 + version=$(echo "${protocol_target}" | cut -d'_' -f2) + if ! { [[ ${label} =~ ^[a-z]+$ ]] && [[ ${version} =~ ^[0-9][0-9][0-9]$ ]]; }; then + error "Wrong protocol version." + error "Name should be a lowercase alphabetic word." + error "Number should be a 3-digit number." + echo + usage + print_and_exit 1 "${LINENO}" + fi + snapshot_protocol "${protocol_source}" "${protocol_target}" + # remove the README because it is specific to Experimental/Stabilised protocols + rm "src/proto_${version}_${short_hash}/README.md" + commit "src: remove README" + + if [[ ${link} == true ]]; then + + log_blue "Linking protocol in the node, client and codec" + sed "s/let _${protocol_source} = active .*/ let _${version}_${short_hash} = active (Name.v \"${short_hash}\" ${version})\n/" -i manifest/product_octez.ml + ocamlformat -i manifest/product_octez.ml + log_blue "Removing dune files from src/proto_${protocol_source}" + # find all dune files in src/proto_${protocol_source} and remove them + find "src/proto_${protocol_source}" -name dune -exec grep -q "; This file was automatically generated, do not edit." {} \; -exec rm {} \; + # remove all proto_${protocol_source} related opam files + find . -name "*-${protocol_source}*.opam" -exec rm {} \; + commit "manifest: link protocol in the node, client and codec" + fi + ;; esac # remove files generated by sed -- GitLab