diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 5059dd487681e37f747e46a25dbfbcf6f6927623..0a5d2fc80b4427f04019f0e06bf8de82f0c419b8 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -18,7 +18,7 @@ - diff poetry.lock /home/tezos/poetry.lock - diff pyproject.toml /home/tezos/pyproject.toml # 2. Actually build - - make all build-sandbox build-unreleased + - make all build-unreleased # 3. Also build the tps evaluation tool which is not part of the default build. # NOTE: We add $COVERAGE_OPTIONS to all dune build commands to enable reuse of # build artifacts. diff --git a/.gitlab/ci/templates.yml b/.gitlab/ci/templates.yml index 92b02ce1686a30ab74f5f81f4f3f0d776dcb3975..b5c0e61b4fb0197c54e53a98c97972370ebb9d7d 100644 --- a/.gitlab/ci/templates.yml +++ b/.gitlab/ci/templates.yml @@ -2,7 +2,7 @@ variables: # /!\ CI_REGISTRY is overriden to use a private Docker registry mirror in AWS ECR # in GitLab namespaces `nomadic-labs` and `tezos` ## This value MUST be the same as `opam_repository_tag` in `scripts/version.sh` - build_deps_image_version: 98a3fd9095e9008f5f687266da8faf798db2240c + build_deps_image_version: 7b619e8f294911d4ee5930b3eb5c334dcf710e65 build_deps_image_name: "${CI_REGISTRY}/tezos/opam-repository" GIT_STRATEGY: fetch GIT_DEPTH: "1" diff --git a/.gitlab/ci/test/integration.yml b/.gitlab/ci/test/integration.yml index 3445203f1e36a209d31473a35588bb30b34f803f..14480248fc8a997b23d22efb005e8fab22e3b58a 100644 --- a/.gitlab/ci/test/integration.yml +++ b/.gitlab/ci/test/integration.yml @@ -1,11 +1,6 @@ -# integration:proto:sandbox and integration:compiler-rejections do not +# integration:compiler-rejections does not # require access to the binaries like the "true" integration tests -# below. Therefore, they do not extend the .integration_template. - -integration:proto:sandbox: - extends: .test_template - script: - - dune build @runtest_sandbox +# below. Therefore, it does not extend the .integration_template. integration:compiler-rejections: extends: .test_template @@ -60,21 +55,6 @@ script:test_release_versions: - sudo chown -R $(id -u):$(id -g) $CI_PROJECT_DIR - ./scripts/test_release_version.sh -############################################################ -## Stage: run OCaml integration tests ## -############################################################ - -integration:sandboxes:acc-endorsement: - extends: .integration_template - script: - - TMP=$PWD make -f sandbox.Makefile accusations_simple_double_endorsing - artifacts: - paths: - - flextesa-acc-sde - - $BISECT_FILE - expire_in: 1 day - when: always - ############################################################ ## Stage: run python integration tests ## ############################################################ @@ -84,10 +64,10 @@ integration:sandboxes:acc-endorsement: .pytest_template: extends: - .test_template - needs: - - build_x86_64 - dependencies: - - build_x86_64 + dependencies: ["build_x86_64"] + # Start immediately after 'build_x86_64' and don't wait for + # intermediate stages to succeed + needs: ["build_x86_64"] before_script: # Load the environment poetry previously created in the docker image. # Give access to the Python dependencies/executables diff --git a/.gitlab/ci/test/templates.yml b/.gitlab/ci/test/templates.yml index 3f3fa12e995dbb66790d2d2ef309a628ccc499a5..66b0219baf2b59851fee33f7d5a30e17477de42d 100644 --- a/.gitlab/ci/test/templates.yml +++ b/.gitlab/ci/test/templates.yml @@ -25,7 +25,7 @@ - .template__coverage_files dependencies: ["build_x86_64"] # Start immediately after 'build_x86_64' and don't wait for - # intermediate stages to succeeds + # intermediate stages to succeed needs: ["build_x86_64"] before_script: - - if [ ! -f tezos-node ] || [ ! -f tezos-client ] || [ ! -f tezos-codec ] || [ ! -f tezos-sandbox ]; then make all build-unreleased; fi + - if [ ! -f tezos-node ] || [ ! -f tezos-client ] || [ ! -f tezos-codec ]; then make all build-unreleased; fi diff --git a/.gitlab/ci/test/tezt.yml b/.gitlab/ci/test/tezt.yml index be9efdc212c8636e2eb2fec6f42ab47dea068b18..6becf1c068cbf9e063febae566b683450fbf22ff 100644 --- a/.gitlab/ci/test/tezt.yml +++ b/.gitlab/ci/test/tezt.yml @@ -73,7 +73,7 @@ tezt:self_tests: # - cat mainnet.roll.* | xz -d -v -T0 > mainnet.rolling # - scripts/prepare_migration_test.sh auto mainnet.rolling "$block_hash" # script: -# - if [ ! -f tezos-node ] || [ ! -f tezos-client ] || [ ! -f tezos-codec ] || [ ! -f tezos-sandbox ]; then make; fi +# - if [ ! -f tezos-node ] || [ ! -f tezos-client ] || [ ! -f tezos-codec ] ; then make; fi # - dune exec ./tezt/manual_tests/main.exe -- migration --color --log-buffer-size 5000 --log-file tezt-migration.log # artifacts: # when: always diff --git a/.gitlab/ci/test_coverage.yml b/.gitlab/ci/test_coverage.yml index ca4b28ebcd3dbc6963f6af2b237702cba5d08aa5..478de6e1fcd506ad82049f9a510d5a35469589cf 100644 --- a/.gitlab/ci/test_coverage.yml +++ b/.gitlab/ci/test_coverage.yml @@ -20,7 +20,6 @@ unified_coverage: - "integration:pytest 7/9" - "integration:pytest 8/9" - "integration:pytest 9/9" - - "integration:sandboxes:acc-endorsement" - "test-liquidity-baking-scripts" - "tezt 1/9" - "tezt 2/9" diff --git a/Makefile b/Makefile index 38f592492d50fdfae43fb1ffb6cfa1be5e029b7c..4c606b0c68a39d79dfb400230b179b3787bd8bb7 100644 --- a/Makefile +++ b/Makefile @@ -137,11 +137,6 @@ coverage-report-cobertura: enable-time-measurement: @$(MAKE) build PROFILE=dev DUNE_INSTRUMENT_WITH=tezos-time-measurement -.PHONY: build-sandbox -build-sandbox: - @dune build --profile=$(PROFILE) $(COVERAGE_OPTIONS) src/bin_sandbox/main.exe - @cp -f _build/default/src/bin_sandbox/main.exe tezos-sandbox - .PHONY: test-protocol-compile test-protocol-compile: @dune build --profile=$(PROFILE) $(COVERAGE_OPTIONS) @runtest_compile_protocol @@ -185,10 +180,6 @@ test-python-alpha: all test-python-tenderbake: all @$(MAKE) -C tests_python tenderbake -.PHONY: test-flextesa -test-flextesa: - @$(MAKE) -f sandbox.Makefile - # TODO: https://gitlab.com/tezos/tezos/-/issues/3018 # Disable verbose once the log file bug in Alcotest is fixed. .PHONY: test-js @@ -214,11 +205,11 @@ test-tezt-coverage: @dune exec --profile=$(PROFILE) $(COVERAGE_OPTIONS) tezt/tests/main.exe -- --keep-going --test-timeout 1800 .PHONY: test-code -test-code: test-protocol-compile test-unit test-flextesa test-python test-tezt +test-code: test-protocol-compile test-unit test-python test-tezt -# This is `make test-code` except for flextesa (which doesn't -# play well with coverage). We allow failure (prefix "-") because we still want -# the coverage report even if an individual test happens to fail. +# This is as `make test-code` except we allow failure (prefix "-") +# because we still want the coverage report even if an individual +# test happens to fail. .PHONY: test-coverage test-coverage: -@$(MAKE) test-protocol-compile @@ -394,7 +385,7 @@ coverage-clean: .PHONY: clean clean: coverage-clean @-dune clean - @-rm -f ${TEZOS_BIN} ${UNRELEASED_TEZOS_BIN} tezos-sandbox + @-rm -f ${TEZOS_BIN} ${UNRELEASED_TEZOS_BIN} @-${MAKE} -C docs clean @-${MAKE} -C tests_python clean @-rm -f docs/api/tezos-{baker,endorser,accuser}-alpha.html docs/api/tezos-{admin-,}client.html docs/api/tezos-signer.html diff --git a/docs/developer/contributing.rst b/docs/developer/contributing.rst index e8da5bc5d7e3a745d33e64d0b32c2f8451d0fcf5..b9e8483fecfdaeaa698bdb4534b04dfd42c70d8a 100644 --- a/docs/developer/contributing.rst +++ b/docs/developer/contributing.rst @@ -324,7 +324,6 @@ any subitems represent the longer description of that commit):: * Doc: mark bug #13 as fixed * Test_python: add test for p2p bug #13 - * Flextesa: add test for p2p bug #13 * Shell: fix p2p bug #13 - fix bug in the shell - fix relative unit test diff --git a/docs/developer/flextesa.rst b/docs/developer/flextesa.rst deleted file mode 100644 index efc0e593f556da98129fe806030544cef55dbdb8..0000000000000000000000000000000000000000 --- a/docs/developer/flextesa.rst +++ /dev/null @@ -1,289 +0,0 @@ -Flexible Network Sandboxes -========================== - -The binary ``tezos-sandbox`` uses the Flextesa library to provide -sandbox networks with baker and endorser daemons, with various test -scenarios. - -Some of those scenarios run in the CI, see ``./src/bin_sandbox/dune`` (although -it is not recommended to experiment with sandboxes using ``dune`` which can -leave unkilled processes easily, see issue -`#2445 `__). - - -Build ------ - -The application is not built by default, one needs: - -:: - - make build-sandbox - -Usage ------ - -See ``./tezos-sandbox --help`` and all the examples below. - -When running (semi-)interactive tests, it is recommended to wrap the -call with ``rlwrap`` or ``ledit``. - -MacOSX Users ------------- - -At runtime, sandboxes usually depend on a couple of linux utilities. - -If you are on Mac OS X, you can do ``brew install coreutils util-linux``. Then run -the tests with: - -:: - - export PATH="/usr/local/opt/coreutils/libexec/gnubin:/usr/local/opt/util-linux/bin:$PATH" - - -See Also --------- - -``tezos-sandbox`` based on the Flextesa library which is being developed -at `gitlab.com/tezos/flextesa `__. -One can find more instructions there, including how to use isolated -sandboxes using Docker. TQ Tezos' assets portal also shows how to start -a sandbox and interact with it using a separate ``tezos-client``: -https://assets.tqtezos.com/docs/setup/2-sandbox/. - - -Concepts --------- - -Let’s clear a couple of things up: - -- *full* Vs *manual* sandbox: we call “full” a sandbox that uses - baker/endorser/accuser daemons and hence advances by itself. A - “manual” sandbox only has nodes, they require successive calls to - ``bake for`` tezos-client commands (or ``bake`` in the interactive - prompt if any). -- Each sandbox scenario has a *root path* where all logs and generated - files go (usually exposed with the ``--root-path`` option). -- Some sandboxes can start an interactive command-line interface. The - command-line parsing uses the Sexplib library; see - https://github.com/janestreet/sexplib#about for the lexical - conventions. Try the ``help`` command for instance. -- By default, sandboxed nodes get assigned successive port numbers for - their RPC and P2P services; with the option ``--base-port PORT``, - ``PORT`` will be used for the RPC of node 0, ``PORT + 1`` for its - P2P, ``PORT + 2`` for the RPC of node 1, etc. -- Sandboxes like the ``mini-network`` also provide a shell-environment - file at ``$ROOT_PATH/shell.env`` which provides aliases to - ``tezos-client`` commands compatible with the sandbox (see also - the ``help-env`` interactive command). - - -Examples --------- - -Interactive Mini-Network -~~~~~~~~~~~~~~~~~~~~~~~~ - -One can run a mini-network advancing fast with accusers, bakers, and -endorsers: - -:: - - rlwrap ./tezos-sandbox mini-network \ - --root-path /tmp/zz-mininet-test \ - --size 2 \ - --number-of-bootstrap-accounts 2 \ - --tezos-node-binary ./tezos-node \ - --tezos-baker-alpha-binary ./tezos-baker-alpha \ - --tezos-accuser-alpha-binary ./tezos-accuser-alpha \ - --tezos-client-binary ./tezos-client - -Once the network is started this test scenario becomes interactive: - -:: - - Flextesa.mininet: Please enter command: - -Just try ``h`` (or ``help``) to see the available commands, or ``q`` to kill the -sandbox and quit. - -The ``mini-network`` has many options, ``./tezos-sandbox mini --help``. - -Mini-Network with User Activated Upgrade -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This example runs another full sandbox (3 nodes, 2 “bakers”), for a -limited amount of time (60 blocks, no interactivity), and performs a -user-activated-upgrade (a.k.a. a protocol “hard-fork”) between Babylon -and Carthage (as built on the ``master`` branch). - -We also set the base-port to 3000 and add some random traffic; -i.e. create contract originations and contract calls (for now the -setting also requires also the ``--until-level`` option): - -:: - - ./tezos-sandbox mini-network \ - --root-path /tmp/hard-fork-mininet \ - --size 3 \ - --base-port 3_000 \ - --number-of-bootstrap-accounts 2 \ - --protocol-hash PsBabyM1eUXZseaJdmXFApDSBqj8YBfwELoxZHHW77EMcAbbwAS \ - --protocol-kind Babylon \ - --until-level 60 \ - --random-traffic any \ - --tezos-baker-alpha-binary ./tezos-baker-005-PsBabyM1 \ - --tezos-endorser-alpha-binary ./tezos-endorser-005-PsBabyM1 \ - --tezos-accuser-alpha-binary ./tezos-accuser-005-PsBabyM1 \ - --hard-fork 20:PsCARTHAGazKbHtnKfLzQg3kms52kSRpgnDY982a9oYsSXRLQEb \ - --hard-fork-baker-alpha-binary ./tezos-baker-006-PsCARTHA \ - --hard-fork-endorser-alpha-binary ./tezos-endorser-006-PsCARTHA \ - --hard-fork-accuser-alpha-binary ./tezos-accuser-006-PsCARTHA \ - --tezos-node-binary ./tezos-node \ - --tezos-client-binary ./tezos-client - - -Manual Mini-Network With An Archive Node -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -An interactive Carthage sandbox with 3 nodes, one of which running in -``archive`` mode, and no baking daemons: - -:: - - rlwrap ./tezos-sandbox mini-network \ - --root-path /tmp/manual-mininet \ - --size 3 \ - --set-history-mode N000:archive \ - --no-baking \ - --protocol-hash PsCARTHAGazKbHtnKfLzQg3kms52kSRpgnDY982a9oYsSXRLQEb \ - --protocol-kind Carthage \ - --tezos-baker-alpha-binary ./tezos-baker-006-PsCARTHA \ - --tezos-endorser-alpha-binary ./tezos-endorser-006-PsCARTHA \ - --tezos-accuser-alpha-binary ./tezos-accuser-006-PsCARTHA \ - --tezos-node-binary ./tezos-node \ - --tezos-client-binary ./tezos-client - -Once the network is started, we enter the interactive mode, and we can -use the ``bake`` command to create blocks (``bake`` can take an -argument: which client to bake with, e.g. \ ``bake 2``). - -We can check that ``N000`` is indeed an archive node: -``c2 rpc get /chains/main/checkpoint`` (``help`` tells us that ``c2`` is -the client for the node ``N000``). - -Double Endorsement Accusation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are 3 “accusation scenarios” so far, see -``./tezos-sandbox accusation --help``. For instance, the following command -starts a small 3-node network, forces one baker to endorse two -concurrent branches, and then makes another baker inject (and bake) the -double-endorsement-evidence operation. The option ``--pause-at-end=true`` tells -``tezos-sandbox`` to enter the interactive mode (command prompt) at the end of -the test to give a chance to explore the sandbox before killing all the nodes. - -:: - - PATH=.:$PATH rlwrap ./tezos-sandbox accusations simple-double-endorsing \ - --root $PWD/double-endorsing-test \ - --pause-at-end=true - - -Voting With a Ledger Nano Device -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The voting test tries to do a full round of voting and protocol switch, -including baking on the test-chain, see documentation in -``./tezos-sandbox voting --help``. - -The test can run in a simpler-to-setup, or “degraded,” mode of operation -(cf. call in ``./src/bin_flextesa/dune`` for the version which -run in Gitlab-CI pipelines). In this example, we run instead a full test -with a Ledger Nano device as one of the bakers/voters. The test automatically -becomes **interactive** because the user has to press buttons on the -device, including for changing between apps. - -Get an URI for your ledger (the test requires both the Wallet and -Baking apps): - -:: - - tezos-client list connected ledgers - -And use the URI (no need to import it) for the ``--with-ledger`` option: - -:: - - rlwrap ./tezos-sandbox voting \ - ./src/proto_006_PsCARTHA/lib_protocol/TEZOS_PROTOCOL \ - ./src/proto_demo_noops/lib_protocol/TEZOS_PROTOCOL \ - --with-ledger "ledger://crouching-tiger-hidden-dragon/ed25519/0'/0'" \ - --serialize-proposals \ - --base-port=20_000 \ - --current-node-binary ./tezos-node \ - --current-client-binary ./tezos-client \ - --winner-client-binary ./tezos-client \ - --current-admin-client-binary ./tezos-admin-client \ - --pause-on-error=true - -- The first path argument has to be the path to a valid protocol which - can be switched to from the current (``proto_alpha``) one. -- The second protocol, the looser, only needs to be valid for the - protocol compilation. -- The option ``--serialize-proposals`` tells the test to call - ``tezos-client submit proposals for ...`` one proposal at a time - which is the only method the ledger Wallet app can really understand. -- The ``*-binary`` options allow to set the paths to the executables - for the different protocols: ``current`` and ``winner``. - -The test becomes interactive and guides you through the interactions -with the ledger, e.g.: - -:: - - Flextesa.voting: - Ledger-prompt - - Setting up "ledger://crouching-tiger-hidden-dragon/ed25519/0'/0'" for - baking. The ledger should be showing the setup parameters (Address, - Main chain, HWMs). - - Please hit “✔” on the ledger. - -Implementation Considerations ------------------------------ - -``Running_processes`` is very high-level (actually agnostic to Tezos). -Most processes are actually calls to ``sh -c