From ae64c9a904243d032bca766e701d93ccc2626c36 Mon Sep 17 00:00:00 2001 From: Brahima Dibassi Date: Wed, 19 Feb 2025 10:21:27 +0100 Subject: [PATCH 1/2] Etherlink/EVM Evaluation: support Dencun PySpec File Modification Script --- .../evm_evaluation/scripts/ethereum_editor.py | 374 ++++++++++++++++++ 1 file changed, 374 insertions(+) create mode 100644 etherlink/kernel_evm/evm_evaluation/scripts/ethereum_editor.py diff --git a/etherlink/kernel_evm/evm_evaluation/scripts/ethereum_editor.py b/etherlink/kernel_evm/evm_evaluation/scripts/ethereum_editor.py new file mode 100644 index 000000000000..18475abf43f6 --- /dev/null +++ b/etherlink/kernel_evm/evm_evaluation/scripts/ethereum_editor.py @@ -0,0 +1,374 @@ +""" +Script for managing Ethereum test files replacement. +Provides functionality to: +- Open and modify Ethereum test JSON files +- Update references between generated fillers and existing tests + +requires: +- fs : Use pip install fs +""" + +import time +from fs import open_fs +from fs.base import FS +import json +import argparse +import logging + +# Configure logging +logging.basicConfig( + level=logging.WARNING, format="%(asctime)s - %(levelname)s - %(message)s" +) + +FORKS = ["Shanghai", "Cancun"] + + +def check_forks(fork_list: list[str]) -> bool: + """Given a list of forks, check if they are all valid""" + return all(fork in FORKS for fork in fork_list) + + +# Filesystem initialization +fs: FS +ethereum_fs: FS +ethereum_tests_fs: FS +generated_fillers_fs: FS + +# JSON Helpers +TEST_KEY: str = "testName" +INFO_KEY: str = "_info" +SOURCE_KEY: str = "source" +JSON_EXT: str = "json" +PYSPEC_FILLER_NAME_PREFIX: str = "py_" +PYSPEC_FILLER_NAME_SUFFIX: str = "Filler_" +POST_KEY: str = "post" + + +def get_filler_from_test(testData: dict, test_name: str) -> str: + return testData[test_name][INFO_KEY][SOURCE_KEY] + + +def set_filler_for_test( + testData: dict, test_name: str, filler_path: str +) -> None: + testData[test_name][INFO_KEY][SOURCE_KEY] = filler_path + + +def load_json_file(filesystem: FS, file_path: str) -> dict: + """Load JSON data from a filesystem path.""" + with filesystem.open(file_path) as f: + return json.load(f) + + +def save_json_file(filesystem: FS, file_path: str, data: dict) -> None: + """Save JSON data to a filesystem path.""" + with filesystem.open(file_path, "w") as f: + json.dump(data, f, indent=4) + + +# File path manipulation +def extract_filename(file_path: str) -> str: + """Extracts the base filename without extension from a path.""" + filename = file_path.split("/")[-1] + if "." in filename: + return filename.split(".")[0] + return filename + + +def rename_file(file_path: str, new_name: str) -> str: + """Changes a file's name to the specified new one.""" + path, _ = file_path.rsplit("/", 1) + ext = get_file_extension(file_path) + return f"{path}/{new_name}.{ext}" + + +def get_file_extension(file_path: str) -> str: + """Extracts the file extension from a path.""" + return file_path.split("/")[-1].split(".")[-1] + + +def set_file_extension(file_path: str, new_extension: str) -> str: + """Changes a file's extension to the specified new one.""" + return f"{file_path.rsplit('.', 1)[0]}.{new_extension}" + + +def set_file_name_extension( + file_path: str, new_name: str, new_extension: str +) -> str: + """Changes a file's name and extension to the specified new ones.""" + return f"{file_path.rsplit('/', 1)[0]}/{new_name}.{new_extension}" + + +def get_filler_index(filler_path: str) -> int: + filler_name: str = extract_filename(filler_path) + num: str = filler_name.split(PYSPEC_FILLER_NAME_SUFFIX)[-1] + return int(num) + + +def get_parent_path(file_path: str) -> str: + return file_path.rsplit("/", 1)[0] + + +def remove_file(fs: FS, file_path: str, log: bool = True) -> None: + if fs.exists(file_path): + return fs.remove(file_path) + if log: + logging.warning(f"File not found: {file_path}") + + +class FillerManager: + """ + Handles filler file updates and dependency management. + Attributes: + generated_filler_path (str): Path to generated filler file + filler_data (dict): Content of generated filler + linked_test_path (str): Path to related test file + test_data (dict): Ethereum test data + target_filler_path (str): Final destination path for filler + """ + + generated_filler_path: str + filler_data: dict + linked_test_path: str + linked_test_data: dict + target_filler_path: str + + def __init__(self, generated_filler_path: str) -> None: + """ + Initializes filler processor from generated file. + Args: + generated_filler_path: Relative path to generated filler + """ + self.generated_filler_path = generated_filler_path + self.filler_data = load_json_file( + generated_fillers_fs, self.generated_filler_path + ) + self.linked_test_path = self.filler_data[TEST_KEY] + test_name = extract_filename(self.linked_test_path) + self.linked_test_data = load_json_file( + ethereum_fs, self.linked_test_path + ) + self.target_filler_path = get_filler_from_test( + self.linked_test_data, test_name + ) + + def __str__(self) -> str: + return ( + f"Generated filler: {self.generated_filler_path}\n" + f"Linked test: {self.linked_test_path}\n" + f"Target path: {self.target_filler_path}" + ) + + def _update_filler_references(self) -> None: + """Updates filler references and removes old version.""" + # Remove temporary metadata + self.filler_data.pop(TEST_KEY) + # Remove old filler file + remove_file(ethereum_tests_fs, self.target_filler_path) + # Update extension if different + source_ext = get_file_extension(self.generated_filler_path) + if source_ext != get_file_extension(self.target_filler_path): + self.target_filler_path = set_file_extension( + self.target_filler_path, source_ext + ) + # Update test metadata + test_key = extract_filename(self.linked_test_path) + set_filler_for_test( + self.linked_test_data, test_key, self.target_filler_path + ) + + def apply_update(self) -> None: + """Executes full filler update process.""" + self._update_filler_references() + save_json_file( + ethereum_tests_fs, self.target_filler_path, self.filler_data + ) + save_json_file( + ethereum_fs, self.linked_test_path, self.linked_test_data + ) + + +class PySpecTestManager: + initial_test_data: list[dict] + generated_fillers: list[dict] + test_path: str + + def __init__(self, test_file_path: str) -> None: + raw_initial_test_data = load_json_file(ethereum_fs, test_file_path) + self.test_path = test_file_path + self.generated_fillers = [] + + self.initial_test_data = [ + single_test_content + for _, single_test_content in raw_initial_test_data.items() + if check_forks(single_test_content[POST_KEY]) + ] + + def add_filler(self, index: int, filler_data: dict) -> None: + self.generated_fillers.append((index, filler_data)) + + def update_fillers_and_tests(self) -> None: + logging.info(f"Updating test and fillers for: {self.test_path}") + + # Ensure the number of test scenarios and generated fillers are the same + if len(self.initial_test_data) != len(self.generated_fillers): + logging.warning( + f"PySpec Test : {self.test_path} processing was canceled\nMismatch between the number of test scenarios and generated fillers there are {len(self.initial_test_data)} scenarios and {len( + self.generated_fillers, + )} fillers." + ) + return + + # Sort fillers by index so they are applied in order + self.generated_fillers.sort(key=lambda x: x[0]) + + for test_content, (index, filler) in zip( + self.initial_test_data, self.generated_fillers + ): + # Test Modification + new_test_name = extract_filename(self.test_path) + f"_{index}" + new_test_path = rename_file(self.test_path, new_test_name) + new_test_content = {new_test_name: test_content} + + # Filler Modification + old_filler_path = get_filler_from_test( + new_test_content, new_test_name + ) + new_filler_name = new_test_name + f"{PYSPEC_FILLER_NAME_SUFFIX}" + new_filler_path = set_file_name_extension( + old_filler_path, new_filler_name, JSON_EXT + ) + + # Update Filler Reference in Test + set_filler_for_test( + new_test_content, new_test_name, new_filler_path + ) + + # Save new files and remove old ones + save_json_file(ethereum_fs, new_test_path, new_test_content) + save_json_file(ethereum_tests_fs, new_filler_path, filler) + + # Remove old filler file + remove_file(ethereum_tests_fs, old_filler_path, False) + + # Remove original test file + remove_file(ethereum_fs, self.test_path) + + +def categorize_fillers() -> tuple[list[str], list[str]]: + """ + Categorizes fillers based on naming patterns. + Returns: + Tuple containing (legacy_fillers, pyspec_fillers) + """ + legacy_fillers: list[str] = [] + pyspec_fillers: list[str] = [] + for filler_path in generated_fillers_fs.walk.files(): + filler_name = extract_filename(filler_path) + if not filler_name.startswith(PYSPEC_FILLER_NAME_PREFIX): + legacy_fillers.append(filler_path) + else: + pyspec_fillers.append(filler_path) + return legacy_fillers, pyspec_fillers + + +def collect_pyspec_tests( + pyspec_fillers: list[str], +) -> dict[str, PySpecTestManager]: + pyspec_test_managers: dict[str, PySpecTestManager] = {} + for pyspec_filler_path in pyspec_fillers: + filler_data = load_json_file(generated_fillers_fs, pyspec_filler_path) + pyspec_test_path: str = filler_data.pop(TEST_KEY) + if pyspec_test_path not in pyspec_test_managers: + if not ethereum_fs.exists(pyspec_test_path): + logging.info( + f"PySpec Test : {pyspec_test_path} does not exist, skipping PySpec Test Manager creation." + ) + continue + logging.info(f"Creating PySpec Test Manager : {pyspec_test_path}") + pyspec_test_managers[pyspec_test_path] = PySpecTestManager( + pyspec_test_path + ) + + pyspec_test_managers[pyspec_test_path].add_filler( + get_filler_index(pyspec_filler_path), filler_data + ) + return pyspec_test_managers + + +# Main execution +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=""" +Ethereum Test Editor\n +A utility to manage Ethereum test files, including automatic updates and synchronization between generated filler files and their corresponding test JSON files.\n +Requirements: +\t- Python package 'fs' (install via pip: pip install fs).""", + formatter_class=argparse.RawTextHelpFormatter, + epilog=""" +Example: +\tpython ethereum_editor.py --fillers_path /path/to/generated_fillers --tests_path /path/to/ethereum_tests --proccess_legacy --proccess_pyspec""", + ) + + parser.add_argument( + "--verbose", "-v", action="store_true", help="Enable verbose logging" + ) + parser.add_argument( + "--fillers_path", help="Path to legacy fillers directory" + ) + parser.add_argument("--tests_path", help="Path to Ethereum directory") + parser.add_argument( + "--proccess_legacy", + "-p_leg", + help="Process legacy fillers", + action="store_true", + ) + parser.add_argument( + "--proccess_pyspec", + "-p_pyspec", + help="Process pyspec fillers", + action="store_true", + ) + args = parser.parse_args() + if args.verbose: + logging.getLogger().setLevel(logging.INFO) + if args.fillers_path: + try: + generated_fillers_fs = open_fs(args.fillers_path) + except OSError: + logging.exception("While opening fillers path") + exit(1) + if args.tests_path: + try: + ethereum_tests_fs = open_fs(args.tests_path) + ethereum_fs = open_fs(get_parent_path(args.tests_path)) + except OSError: + logging.exception("While opening tests path") + exit(1) + + logging.info("Starting filler replacement process...") + logging.info(f"Forks to process: {FORKS}") + start_time = time.perf_counter() + + legacy, pyspec = categorize_fillers() + + if args.proccess_legacy: + logging.info("Processing legacy fillers...") + # Process legacy fillers + for legacy_filler_path in legacy: + filler_manager = FillerManager(legacy_filler_path) + logging.info( + f"Processing Legacy Filler: {filler_manager.generated_filler_path}" + ) + filler_manager.apply_update() + print(f"New pyspec fillers: {len(pyspec)}") + + if args.proccess_pyspec: + # Process PySpec fillers + pyspec_tests = collect_pyspec_tests(pyspec) + for _, data in pyspec_tests.items(): + data.update_fillers_and_tests() + print(f"Migrated legacy fillers: {len(legacy)}") + + end_time = time.perf_counter() + print(f"Process completed in {end_time - start_time:.2f} seconds.") -- GitLab From 6bd631c2c6408fda42e8caf85fcef68f06381845 Mon Sep 17 00:00:00 2001 From: Rodi-Can Bozman Date: Thu, 13 Mar 2025 15:45:04 +0100 Subject: [PATCH 2/2] Etherlink/README: procedure to set up the EVM test-suite --- .../evm_evaluation/scripts/README.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 etherlink/kernel_evm/evm_evaluation/scripts/README.md diff --git a/etherlink/kernel_evm/evm_evaluation/scripts/README.md b/etherlink/kernel_evm/evm_evaluation/scripts/README.md new file mode 100644 index 000000000000..3028f8fd7f2f --- /dev/null +++ b/etherlink/kernel_evm/evm_evaluation/scripts/README.md @@ -0,0 +1,41 @@ +# Evaluation Scripts + +## Context + +The EVM evaluation was written for version `v13`/`v13.1` of [ethereum/tests](https://github.com/ethereum/tests/) which is the official test-suite of Ethereum. Those versions were meant for configuration up to Shanghai. + +Right after Shanghai, Cancun was proposed and accepted on Ethereum. With this configuration came `v14`/`v14.1` which introduced a new framework called **Pyspec**. This framework was created to generate more scenarios from a single starting point without having to write anything manually. + +Even though this is a big improvement on their side, it caused issues on our end as our EVM evaluator relies on something they've called "filler files" which was written manually by a developer who introduced a scenario. Since that framework generates tests, nobody took the time to write those associated filler files. + +We were relying on those filler files because we can't use the test-suite like other VMs because we use Irmin as a backend and not Patricia Merkle Tree: we can't compare root hashes. Those filler files contain the Patricia Merkle State Trie before and after the scenario was executed, which we can use because it provides details on the account, balance, nonce, and storage slot after execution. + +The solution to that problem was to generate those missing filler files thanks to an EVM of reference. We chose [revm](https://github.com/bluealloy/revm) to do that. Once the fillers are generated, we pass the directory of generated filler files to a script that will dispatch the tests into the right directories on the test-suite repository. + +## Set up a new branch for testing for upcoming Ethereum forks + +Quick disclaimer before starting, this method is a bit tedious, but it's the fastest one we found at that time, feel free to enhance the procedure if you have any ideas. + +### Filler generation + +First generate the filler files.\ +Clone [this repository](https://github.com/rodibozman/revm/), pull the latest changes from the upstream **revm**.\ +Use the branch from [revm-fork!1](https://github.com/rodibozman/revm/pull/1) and add the new spec. config in the code (like it was done for Shanghai and Cancun). Finally generate the filler files: + +```bash +cd bins/revme +cargo run --release -- statetest tests/GeneralStateTests/ tests/LegacyTests/Constantinople/GeneralStateTests +``` + +This should create a directory called `output_jsons/` which will contain all the generated filler files. + +### Filler dispatch + +Dispatch the filler files from `output_jsons/` thanks to the script at `tezos/etherlink/kernel_evm/evm_evaluation/scripts/ethereum_editor.py`. + +``` +python3 ethereum_editor.py --fillers_path --tests_path --proccess_legacy --proccess_pyspec +``` + +For the CI, since we can't use the [ethereum/tests](https://github.com/ethereum/tests/) repository, we forked it [here](https://github.com/functori/tests/) and we introduced a new branch called `v14.1@etherlink` for Cancun.\ +For upcoming configuration, create a new branch and push the changes there so that the CI can pull the appropriate tests. See the job called `test_evm_compatibility`. -- GitLab