diff --git a/Makefile b/Makefile index 63bcc90f83..9af1c29233 100644 --- a/Makefile +++ b/Makefile @@ -49,11 +49,11 @@ download-json-fixtures: cd tests/execution-spec-tests && bash get_execution_spec_tests.sh test-fixtures: download-json-fixtures - $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k test_execution_spec_tests.py -k test_bchain" + $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k 'test_execution_spec_tests.py and test_bchain'" fixtures-failing-list: download-json-fixtures cat /dev/null > tests/execution-spec-tests/failing.llvm - - $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k test_execution_spec_tests.py -k test_bchain --save-failing --maxfail=10000" + - $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k 'test_execution_spec_tests.py and test_bchain' --save-failing --maxfail=10000" LC_ALL=en_US.UTF-8 sort -f -d -o tests/execution-spec-tests/failing.llvm tests/execution-spec-tests/failing.llvm if [ "$(shell uname)" = "Darwin" ]; then \ sed -i '' '1{/^[[:space:]]*$$/d;}' tests/execution-spec-tests/failing.llvm ;\ diff --git a/kevm-pyk/src/kevm_pyk/kproj/evm-semantics/state-utils.md b/kevm-pyk/src/kevm_pyk/kproj/evm-semantics/state-utils.md index c50606be91..3a5c872e24 100644 --- a/kevm-pyk/src/kevm_pyk/kproj/evm-semantics/state-utils.md +++ b/kevm-pyk/src/kevm_pyk/kproj/evm-semantics/state-utils.md @@ -262,6 +262,10 @@ The `"rlp"` key loads the block information. _ => #asWord(RR) + // fallback rule to catch invalid rlp encodings + rule load "rlp" : _ => .K ... + _ => EVMC_INVALID_BLOCK [owise] + rule load "genesisRLP": [ [ HP, HO, HC, HR, HT, HE, HB, HD, HI, HL, HG, HS, HX, HM, HN, HF, WR, UB, EB, BR, RR, .JSONs ], _, _, _, .JSONs ] => .K ... .List => ListItem(#blockHeaderHash(HP, HO, HC, HR, HT, HE, HB, HD, HI, HL, HG, HS, HX, HM, HN, HF, WR, UB, EB, BR, RR)) ListItem(#asWord(HP)) 0 => #asWord(EB) diff --git a/kevm-pyk/src/tests/utils.py b/kevm-pyk/src/tests/utils.py index 2602a18a1e..9a83fc5d7f 100644 --- a/kevm-pyk/src/tests/utils.py +++ b/kevm-pyk/src/tests/utils.py @@ -43,18 +43,18 @@ def _assert_exit_code_zero(pattern: Pattern) -> None: assert pretty == GOLDEN -def _skipped_tests(test_dir: Path, slow_tests_file: Path, failing_tests_file: Path) -> dict[Path, list[str]]: +def _skipped_tests(test_dir: Path, slow_tests_file: Path, failing_tests_file: Path) -> dict[Path, frozenset[str]]: try: slow_tests = read_csv_file(slow_tests_file) except FileNotFoundError as e: _LOGGER.warning(e) slow_tests = () failing_tests = read_csv_file(failing_tests_file) - skipped: dict[Path, list[str]] = {} + skipped: dict[Path, set[str]] = {} for test_file, test in slow_tests + failing_tests: test_file = test_dir / test_file - skipped.setdefault(test_file, []).append(test) - return skipped + skipped.setdefault(test_file, set()).add(test) + return {k: frozenset(v) for k, v in skipped.items()} def read_csv_file(csv_file: Path) -> tuple[tuple[Path, str], ...]: @@ -71,25 +71,27 @@ def _test( usegas: bool, save_failing: bool, compute_chain_id: Callable[[str], int], - skipped_tests: dict[Path, list[str]], + skipped_tests: dict[Path, frozenset[str]], test_dir: Path, failing_tests_file: Path, ) -> None: - skipped_gst_tests = skipped_tests.get(gst_file, []) + skipped_gst_tests = skipped_tests.get(gst_file, frozenset()) if '*' in skipped_gst_tests: pytest.skip() - failing_tests: list[str] = [] gst_file_relative_path: Final[str] = str(gst_file.relative_to(test_dir)) with gst_file.open() as f: gst_data = json.load(f) + failing_tests: list[str] = [] + + chain_id = compute_chain_id(gst_file_relative_path) + for test_name, test in gst_data.items(): _LOGGER.info(f'Running test: {gst_file} - {test_name}') if test_name in skipped_gst_tests: continue - chain_id = compute_chain_id(gst_file_relative_path) res = interpret({test_name: test}, schedule, mode, chain_id, usegas, check=False) try: @@ -99,14 +101,14 @@ def _test( raise failing_tests.append(test_name) - if not failing_tests: + if len(failing_tests) == 0: return - if save_failing: - with failing_tests_file.open('a', newline='') as ff: - writer = csv.writer(ff) - if len(failing_tests) == len(gst_data): - writer.writerow([gst_file_relative_path, '*']) - else: - for test_name in sorted(failing_tests): - writer.writerow([gst_file_relative_path, test_name]) + + with failing_tests_file.open('a', newline='') as ff: + writer = csv.writer(ff) + if len(failing_tests) == len(gst_data): + writer.writerow([gst_file_relative_path, '*']) + else: + for test_name in sorted(failing_tests): + writer.writerow([gst_file_relative_path, test_name]) raise AssertionError(f'Found failing tests in GST file {gst_file_relative_path}: {failing_tests}') diff --git a/tests/bchain.0.chainId b/tests/bchain.0.chainId index 0055987496..3ffaf7077d 100644 --- a/tests/bchain.0.chainId +++ b/tests/bchain.0.chainId @@ -1,14 +1,8 @@ -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/create_selfdestruct_same_tx.json BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/delegatecall_from_new_contract_to_pre_existing_contract.json BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/delegatecall_from_pre_existing_contract_to_new_contract.json BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision_multi_tx.json BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision_two_different_transactions.json BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision.json -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/recreate_self_destructed_contract_different_txs.json BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/reentrancy_selfdestruct_revert.json -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode_create_tx.json -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode.json BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_in_same_tx_with_revert.json -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_same_block_different_tx.json -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_not_created_in_same_tx_with_revert.json -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_pre_existing.json \ No newline at end of file +BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_not_created_in_same_tx_with_revert.json \ No newline at end of file diff --git a/tests/execution-spec-tests/failing.llvm b/tests/execution-spec-tests/failing.llvm index c7f232f241..41e851fa4c 100644 --- a/tests/execution-spec-tests/failing.llvm +++ b/tests/execution-spec-tests/failing.llvm @@ -3,8 +3,6 @@ blockchain_tests/cancun/eip4844_blobs/blob_txs_full/reject_valid_full_blob_in_bl blockchain_tests/cancun/eip4844_blobs/blob_txs/blob_type_tx_pre_fork.json,tests/cancun/eip4844_blobs/test_blob_txs.py::test_blob_type_tx_pre_fork[fork_ShanghaiToCancunAtTime15k-blockchain_test_from_state_test-one_blob_tx] blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/fork_transition_excess_blob_gas_at_blob_genesis.json,* blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/fork_transition_excess_blob_gas_post_blob_genesis.json,* -blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/invalid_post_fork_block_without_blob_fields.json,* -blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/invalid_pre_fork_block_with_blob_fields.json,* blockchain_tests/cancun/eip4844_blobs/excess_blob_gas/invalid_blob_gas_used_in_header.json,* blockchain_tests/cancun/eip4844_blobs/point_evaluation_precompile/precompile_before_fork.json,* blockchain_tests/cancun/eip4844_blobs/point_evaluation_precompile/precompile_during_fork.json,* diff --git a/tests/failing.llvm b/tests/failing.llvm index fdd390007e..7a18195128 100644 --- a/tests/failing.llvm +++ b/tests/failing.llvm @@ -2,16 +2,8 @@ BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4788_beacon_root/beacon_root BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_type_tx_pre_fork.json,src/GeneralStateTestsFiller/Pyspecs/cancun/eip4844_blobs/test_blob_txs.py::test_blob_type_tx_pre_fork[fork_ShanghaiToCancunAtTime15k-blockchain_test-one_blob_tx] BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/fork_transition_excess_blob_gas.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/invalid_blob_gas_used_in_header.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/invalid_post_fork_block_without_blob_fields.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/invalid_pre_fork_block_with_blob_fields.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/precompile_before_fork.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/precompile_during_fork.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/reject_valid_full_blob_in_block_rlp.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/create_selfdestruct_same_tx.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/recreate_self_destructed_contract_different_txs.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_same_block_different_tx.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode_create_tx.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_pre_existing.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip7516_blobgasfee/blobbasefee_before_fork.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip7516_blobgasfee/blobbasefee_during_fork.json,*