Skip to content

Commit 306ef67

Browse files
anvacaruanvacaru
authored
Minor tweaks to the test harness (#2819)
* minor optimizations to _test * add fallback rlp decoding rule * update Makefile * remove comments * add endl * add endl * format bchain.0.chainid * update skipped_tests to use frozenset instead of lists * check if test is skipped in loop * Refactor chain_id computation for clarity
1 parent 21fde76 commit 306ef67

File tree

6 files changed

+26
-36
lines changed

6 files changed

+26
-36
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,11 @@ download-json-fixtures:
4949
cd tests/execution-spec-tests && bash get_execution_spec_tests.sh
5050

5151
test-fixtures: download-json-fixtures
52-
$(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k test_execution_spec_tests.py -k test_bchain"
52+
$(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k 'test_execution_spec_tests.py and test_bchain'"
5353

5454
fixtures-failing-list: download-json-fixtures
5555
cat /dev/null > tests/execution-spec-tests/failing.llvm
56-
- $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k test_execution_spec_tests.py -k test_bchain --save-failing --maxfail=10000"
56+
- $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k 'test_execution_spec_tests.py and test_bchain' --save-failing --maxfail=10000"
5757
LC_ALL=en_US.UTF-8 sort -f -d -o tests/execution-spec-tests/failing.llvm tests/execution-spec-tests/failing.llvm
5858
if [ "$(shell uname)" = "Darwin" ]; then \
5959
sed -i '' '1{/^[[:space:]]*$$/d;}' tests/execution-spec-tests/failing.llvm ;\

‎kevm-pyk/src/kevm_pyk/kproj/evm-semantics/state-utils.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,10 @@ The `"rlp"` key loads the block information.
262262
</k>
263263
<requestsRoot> _ => #asWord(RR) </requestsRoot>
264264
265+
// fallback rule to catch invalid rlp encodings
266+
rule <k> load "rlp" : _ => .K ... </k>
267+
<statusCode> _ => EVMC_INVALID_BLOCK </statusCode> [owise]
268+
265269
rule <k> load "genesisRLP": [ [ HP, HO, HC, HR, HT, HE, HB, HD, HI, HL, HG, HS, HX, HM, HN, HF, WR, UB, EB, BR, RR, .JSONs ], _, _, _, .JSONs ] => .K ... </k>
266270
<blockhashes> .List => ListItem(#blockHeaderHash(HP, HO, HC, HR, HT, HE, HB, HD, HI, HL, HG, HS, HX, HM, HN, HF, WR, UB, EB, BR, RR)) ListItem(#asWord(HP)) </blockhashes>
267271
<previousExcessBlobGas> 0 => #asWord(EB) </previousExcessBlobGas>

‎kevm-pyk/src/tests/utils.py‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -43,18 +43,18 @@ def _assert_exit_code_zero(pattern: Pattern) -> None:
4343
assert pretty == GOLDEN
4444

4545

46-
def _skipped_tests(test_dir: Path, slow_tests_file: Path, failing_tests_file: Path) -> dict[Path, list[str]]:
46+
def _skipped_tests(test_dir: Path, slow_tests_file: Path, failing_tests_file: Path) -> dict[Path, frozenset[str]]:
4747
try:
4848
slow_tests = read_csv_file(slow_tests_file)
4949
except FileNotFoundError as e:
5050
_LOGGER.warning(e)
5151
slow_tests = ()
5252
failing_tests = read_csv_file(failing_tests_file)
53-
skipped: dict[Path, list[str]] = {}
53+
skipped: dict[Path, set[str]] = {}
5454
for test_file, test in slow_tests + failing_tests:
5555
test_file = test_dir / test_file
56-
skipped.setdefault(test_file, []).append(test)
57-
return skipped
56+
skipped.setdefault(test_file, set()).add(test)
57+
return {k: frozenset(v) for k, v in skipped.items()}
5858

5959

6060
def read_csv_file(csv_file: Path) -> tuple[tuple[Path, str], ...]:
@@ -71,25 +71,27 @@ def _test(
7171
usegas: bool,
7272
save_failing: bool,
7373
compute_chain_id: Callable[[str], int],
74-
skipped_tests: dict[Path, list[str]],
74+
skipped_tests: dict[Path, frozenset[str]],
7575
test_dir: Path,
7676
failing_tests_file: Path,
7777
) -> None:
78-
skipped_gst_tests = skipped_tests.get(gst_file, [])
78+
skipped_gst_tests = skipped_tests.get(gst_file, frozenset())
7979
if '*' in skipped_gst_tests:
8080
pytest.skip()
8181

82-
failing_tests: list[str] = []
8382
gst_file_relative_path: Final[str] = str(gst_file.relative_to(test_dir))
8483

8584
with gst_file.open() as f:
8685
gst_data = json.load(f)
8786

87+
failing_tests: list[str] = []
88+
89+
chain_id = compute_chain_id(gst_file_relative_path)
90+
8891
for test_name, test in gst_data.items():
8992
_LOGGER.info(f'Running test: {gst_file} - {test_name}')
9093
if test_name in skipped_gst_tests:
9194
continue
92-
chain_id = compute_chain_id(gst_file_relative_path)
9395
res = interpret({test_name: test}, schedule, mode, chain_id, usegas, check=False)
9496

9597
try:
@@ -99,14 +101,14 @@ def _test(
99101
raise
100102
failing_tests.append(test_name)
101103

102-
if not failing_tests:
104+
if len(failing_tests) == 0:
103105
return
104-
if save_failing:
105-
with failing_tests_file.open('a', newline='') as ff:
106-
writer = csv.writer(ff)
107-
if len(failing_tests) == len(gst_data):
108-
writer.writerow([gst_file_relative_path, '*'])
109-
else:
110-
for test_name in sorted(failing_tests):
111-
writer.writerow([gst_file_relative_path, test_name])
106+
107+
with failing_tests_file.open('a', newline='') as ff:
108+
writer = csv.writer(ff)
109+
if len(failing_tests) == len(gst_data):
110+
writer.writerow([gst_file_relative_path, '*'])
111+
else:
112+
for test_name in sorted(failing_tests):
113+
writer.writerow([gst_file_relative_path, test_name])
112114
raise AssertionError(f'Found failing tests in GST file {gst_file_relative_path}: {failing_tests}')

‎tests/bchain.0.chainId‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,8 @@
1-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/create_selfdestruct_same_tx.json
21
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/delegatecall_from_new_contract_to_pre_existing_contract.json
32
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/delegatecall_from_pre_existing_contract_to_new_contract.json
43
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision_multi_tx.json
54
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision_two_different_transactions.json
65
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision.json
7-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/recreate_self_destructed_contract_different_txs.json
86
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/reentrancy_selfdestruct_revert.json
9-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode_create_tx.json
10-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode.json
117
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_in_same_tx_with_revert.json
12-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_same_block_different_tx.json
13-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_not_created_in_same_tx_with_revert.json
14-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_pre_existing.json
8+
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_not_created_in_same_tx_with_revert.json

‎tests/execution-spec-tests/failing.llvm‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@ blockchain_tests/cancun/eip4844_blobs/blob_txs_full/reject_valid_full_blob_in_bl
33
blockchain_tests/cancun/eip4844_blobs/blob_txs/blob_type_tx_pre_fork.json,tests/cancun/eip4844_blobs/test_blob_txs.py::test_blob_type_tx_pre_fork[fork_ShanghaiToCancunAtTime15k-blockchain_test_from_state_test-one_blob_tx]
44
blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/fork_transition_excess_blob_gas_at_blob_genesis.json,*
55
blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/fork_transition_excess_blob_gas_post_blob_genesis.json,*
6-
blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/invalid_post_fork_block_without_blob_fields.json,*
7-
blockchain_tests/cancun/eip4844_blobs/excess_blob_gas_fork_transition/invalid_pre_fork_block_with_blob_fields.json,*
86
blockchain_tests/cancun/eip4844_blobs/excess_blob_gas/invalid_blob_gas_used_in_header.json,*
97
blockchain_tests/cancun/eip4844_blobs/point_evaluation_precompile/precompile_before_fork.json,*
108
blockchain_tests/cancun/eip4844_blobs/point_evaluation_precompile/precompile_during_fork.json,*

‎tests/failing.llvm‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,8 @@ BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4788_beacon_root/beacon_root
22
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_type_tx_pre_fork.json,src/GeneralStateTestsFiller/Pyspecs/cancun/eip4844_blobs/test_blob_txs.py::test_blob_type_tx_pre_fork[fork_ShanghaiToCancunAtTime15k-blockchain_test-one_blob_tx]
33
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/fork_transition_excess_blob_gas.json,*
44
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/invalid_blob_gas_used_in_header.json,*
5-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/invalid_post_fork_block_without_blob_fields.json,*
6-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/invalid_pre_fork_block_with_blob_fields.json,*
75
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/precompile_before_fork.json,*
86
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/precompile_during_fork.json,*
97
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/reject_valid_full_blob_in_block_rlp.json,*
10-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/create_selfdestruct_same_tx.json,*
11-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/recreate_self_destructed_contract_different_txs.json,*
12-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_same_block_different_tx.json,*
13-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode_create_tx.json,*
14-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode.json,*
15-
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_pre_existing.json,*
168
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip7516_blobgasfee/blobbasefee_before_fork.json,*
179
BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip7516_blobgasfee/blobbasefee_during_fork.json,*

0 commit comments

Comments
 (0)