Minor tweaks to the test harness

Makefile

@@ -49,11 +49,11 @@ download-json-fixtures:
 	cd tests/execution-spec-tests && bash get_execution_spec_tests.sh
 
 test-fixtures: download-json-fixtures
-	$(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k test_execution_spec_tests.py -k test_bchain"
+	$(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k 'test_execution_spec_tests.py and test_bchain'"
 
 fixtures-failing-list: download-json-fixtures
 	cat /dev/null > tests/execution-spec-tests/failing.llvm
-	- $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k test_execution_spec_tests.py -k test_bchain --save-failing --maxfail=10000"
+	- $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k 'test_execution_spec_tests.py and test_bchain' --save-failing --maxfail=10000"
 	LC_ALL=en_US.UTF-8 sort -f -d -o tests/execution-spec-tests/failing.llvm tests/execution-spec-tests/failing.llvm
 	if [ "$(shell uname)" = "Darwin" ]; then \
 		sed -i '' '1{/^[[:space:]]*$$/d;}' tests/execution-spec-tests/failing.llvm ;\

kevm-pyk/src/kevm_pyk/kproj/evm-semantics/state-utils.md

@@ -262,6 +262,10 @@ The `"rlp"` key loads the block information.
          </k>
          <requestsRoot> _ => #asWord(RR) </requestsRoot>
 
+    // fallback rule to catch invalid rlp encodings
+    rule <k> load "rlp" : _ => .K ... </k>
+         <statusCode> _ => EVMC_INVALID_BLOCK </statusCode>  [owise]
+
     rule <k> load "genesisRLP": [ [ HP, HO, HC, HR, HT, HE, HB, HD, HI, HL, HG, HS, HX, HM, HN, HF, WR, UB, EB, BR, RR, .JSONs ], _, _, _, .JSONs ] => .K ... </k>
          <blockhashes> .List => ListItem(#blockHeaderHash(HP, HO, HC, HR, HT, HE, HB, HD, HI, HL, HG, HS, HX, HM, HN, HF, WR, UB, EB, BR, RR)) ListItem(#asWord(HP)) </blockhashes>
          <previousExcessBlobGas> 0 => #asWord(EB) </previousExcessBlobGas>

kevm-pyk/src/tests/utils.py

@@ -43,18 +43,18 @@ def _assert_exit_code_zero(pattern: Pattern) -> None:
     assert pretty == GOLDEN
 
 
-def _skipped_tests(test_dir: Path, slow_tests_file: Path, failing_tests_file: Path) -> dict[Path, list[str]]:
+def _skipped_tests(test_dir: Path, slow_tests_file: Path, failing_tests_file: Path) -> dict[Path, frozenset[str]]:
     try:
         slow_tests = read_csv_file(slow_tests_file)
     except FileNotFoundError as e:
         _LOGGER.warning(e)
         slow_tests = ()
     failing_tests = read_csv_file(failing_tests_file)
-    skipped: dict[Path, list[str]] = {}
+    skipped: dict[Path, set[str]] = {}
     for test_file, test in slow_tests + failing_tests:
         test_file = test_dir / test_file
-        skipped.setdefault(test_file, []).append(test)
-    return skipped
+        skipped.setdefault(test_file, set()).add(test)
+    return {k: frozenset(v) for k, v in skipped.items()}
 
 
 def read_csv_file(csv_file: Path) -> tuple[tuple[Path, str], ...]:
@@ -71,25 +71,27 @@ def _test(
     usegas: bool,
     save_failing: bool,
     compute_chain_id: Callable[[str], int],
-    skipped_tests: dict[Path, list[str]],
+    skipped_tests: dict[Path, frozenset[str]],
     test_dir: Path,
     failing_tests_file: Path,
 ) -> None:
-    skipped_gst_tests = skipped_tests.get(gst_file, [])
+    skipped_gst_tests = skipped_tests.get(gst_file, frozenset())
     if '*' in skipped_gst_tests:
         pytest.skip()
 
-    failing_tests: list[str] = []
     gst_file_relative_path: Final[str] = str(gst_file.relative_to(test_dir))
 
     with gst_file.open() as f:
         gst_data = json.load(f)
 
+    failing_tests: list[str] = []
+
+    chain_id = compute_chain_id(str(gst_file.relative_to(test_dir)))
+
     for test_name, test in gst_data.items():
         _LOGGER.info(f'Running test: {gst_file} - {test_name}')
         if test_name in skipped_gst_tests:
             continue
-        chain_id = compute_chain_id(gst_file_relative_path)
         res = interpret({test_name: test}, schedule, mode, chain_id, usegas, check=False)
 
         try:
@@ -99,14 +101,14 @@ def _test(
                 raise
             failing_tests.append(test_name)
 
-    if not failing_tests:
+    if len(failing_tests) == 0:
         return
-    if save_failing:
-        with failing_tests_file.open('a', newline='') as ff:
-            writer = csv.writer(ff)
-            if len(failing_tests) == len(gst_data):
-                writer.writerow([gst_file_relative_path, '*'])
-            else:
-                for test_name in sorted(failing_tests):
-                    writer.writerow([gst_file_relative_path, test_name])
+
+    with failing_tests_file.open('a', newline='') as ff:
+        writer = csv.writer(ff)
+        if len(failing_tests) == len(gst_data):
+            writer.writerow([gst_file_relative_path, '*'])
+        else:
+            for test_name in sorted(failing_tests):
+                writer.writerow([gst_file_relative_path, test_name])
     raise AssertionError(f'Found failing tests in GST file {gst_file_relative_path}: {failing_tests}')

tests/bchain.0.chainId

@@ -1,14 +1,8 @@
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/create_selfdestruct_same_tx.json
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/delegatecall_from_new_contract_to_pre_existing_contract.json
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/delegatecall_from_pre_existing_contract_to_new_contract.json
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision_multi_tx.json
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision_two_different_transactions.json
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision.json
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/recreate_self_destructed_contract_different_txs.json
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/reentrancy_selfdestruct_revert.json
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode_create_tx.json
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode.json
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_in_same_tx_with_revert.json
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_same_block_different_tx.json
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_not_created_in_same_tx_with_revert.json
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_pre_existing.json
+BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_not_created_in_same_tx_with_revert.json

tests/execution-spec-tests/failing.llvm

@@ -3,8 +3,6 @@ blockchain_tests/cancun/eip4844_blobs/blob_txs_full/reject_valid_full_blob_in_bl
 blockchain_tests/cancun/eip4844_blobs/blob_txs/blob_type_tx_pre_fork.json,tests/cancun/eip4844_blobs/test_blob_txs.py::test_blob_type_tx_pre_fork[fork_ShanghaiToCancunAtTime15k-blockchain_test_from_state_test-one_blob_tx]
 blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/fork_transition_excess_blob_gas_at_blob_genesis.json,*
 blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/fork_transition_excess_blob_gas_post_blob_genesis.json,*
-blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/invalid_post_fork_block_without_blob_fields.json,*
-blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/invalid_pre_fork_block_with_blob_fields.json,*
 blockchain_tests/cancun/eip4844_blobs/excess_blob_gas/invalid_blob_gas_used_in_header.json,*
 blockchain_tests/cancun/eip4844_blobs/point_evaluation_precompile/precompile_before_fork.json,*
 blockchain_tests/cancun/eip4844_blobs/point_evaluation_precompile/precompile_during_fork.json,*

tests/failing.llvm

@@ -2,16 +2,8 @@ BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4788_beacon_root/beacon_root
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_type_tx_pre_fork.json,src/GeneralStateTestsFiller/Pyspecs/cancun/eip4844_blobs/test_blob_txs.py::test_blob_type_tx_pre_fork[fork_ShanghaiToCancunAtTime15k-blockchain_test-one_blob_tx]
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/fork_transition_excess_blob_gas.json,*
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/invalid_blob_gas_used_in_header.json,*
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/invalid_post_fork_block_without_blob_fields.json,*
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/invalid_pre_fork_block_with_blob_fields.json,*
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/precompile_before_fork.json,*
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/precompile_during_fork.json,*
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/reject_valid_full_blob_in_block_rlp.json,*
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/create_selfdestruct_same_tx.json,*
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/recreate_self_destructed_contract_different_txs.json,*
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_same_block_different_tx.json,*
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode_create_tx.json,*
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode.json,*
-BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_pre_existing.json,*
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip7516_blobgasfee/blobbasefee_before_fork.json,*
 BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip7516_blobgasfee/blobbasefee_during_fork.json,*