Skip to content

Commit

Permalink
Tweaks to the conformance harness (#2505)
Browse files Browse the repository at this point in the history 
* tweaks to the conformance harness

* Set Version: 1.0.621

* apply review suggestions

* bchain test ids relative to BlockchainTests/GeneralStateTests/

---------

Co-authored-by: devops <[email protected]>
  • Loading branch information
@anvacaru
anvacaru and devops authored Jun 27, 2024
1 parent 9443a79 commit 9b848e2
Show file tree
Hide file tree
Showing 5 changed files with 21 additions and 110 deletions.
2 changes: 1 addition & 1 deletion kevm-pyk/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "kevm-pyk"
version = "1.0.620"
version = "1.0.621"
description = ""
authors = [
"Runtime Verification, Inc. <[email protected]>",
Expand Down
2 changes: 1 addition & 1 deletion kevm-pyk/src/kevm_pyk/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
if TYPE_CHECKING:
from typing import Final

VERSION: Final = '1.0.620'
VERSION: Final = '1.0.621'
33 changes: 16 additions & 17 deletions kevm-pyk/src/tests/integration/test_conformance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,16 @@


def _test(gst_file: Path, schedule: str, mode: str, chainid: int, usegas: bool) -> None:
skipped_gst_tests = SKIPPED_TESTS.get(gst_file, [])
if '*' in skipped_gst_tests:
pytest.skip()

with gst_file.open() as f:
gst_data = json.load(f)

for test_name, test in gst_data.items():
_LOGGER.info(f'Running test: {gst_file} - {test_name}')
if test_name in SKIPPED_TESTS.get(gst_file, []):
if test_name in skipped_gst_tests:
continue
res = interpret({test_name: test}, schedule, mode, chainid, usegas, check=False)
_assert_exit_code_zero(res)
Expand Down Expand Up @@ -79,9 +83,8 @@ def read_csv_file(csv_file: Path) -> tuple[tuple[Path, str], ...]:
SKIPPED_TESTS: Final = _skipped_tests()

VM_TEST_DIR: Final = TEST_DIR / 'BlockchainTests/GeneralStateTests/VMTests'
ALL_VM_TESTS: Final = tuple(VM_TEST_DIR.glob('*/*.json'))
VM_TESTS: Final = tuple(test_file for test_file in ALL_VM_TESTS if test_file not in SKIPPED_TESTS)
REST_VM_TESTS: Final = tuple(test_file for test_file in ALL_VM_TESTS if test_file in SKIPPED_TESTS)
VM_TESTS: Final = tuple(VM_TEST_DIR.glob('*/*.json'))
SKIPPED_VM_TESTS: Final = tuple(test_file for test_file in VM_TESTS if test_file in SKIPPED_TESTS)


@pytest.mark.parametrize(
Expand All @@ -96,27 +99,23 @@ def test_vm(test_file: Path) -> None:
@pytest.mark.skip(reason='failing / slow VM tests')
@pytest.mark.parametrize(
'test_file',
REST_VM_TESTS,
ids=[str(test_file.relative_to(VM_TEST_DIR)) for test_file in REST_VM_TESTS],
SKIPPED_VM_TESTS,
ids=[str(test_file.relative_to(VM_TEST_DIR)) for test_file in SKIPPED_VM_TESTS],
)
def test_rest_vm(test_file: Path) -> None:
_test(test_file, 'DEFAULT', 'VMTESTS', 1, True)


BCHAIN_TEST_DIR: Final = TEST_DIR / 'BlockchainTests/GeneralStateTests'
ALL_BCHAIN_TESTS: Final = tuple(BCHAIN_TEST_DIR.glob('**/*.json'))
BCHAIN_TESTS: Final = tuple(
test_file for test_file in ALL_BCHAIN_TESTS if test_file not in (list(SKIPPED_TESTS) + list(ALL_VM_TESTS))
)
REST_BCHAIN_TESTS: Final = tuple(
test_file for test_file in ALL_BCHAIN_TESTS if test_file in SKIPPED_TESTS and test_file not in ALL_VM_TESTS
)
ALL_TEST_DIR: Final = TEST_DIR / 'BlockchainTests/GeneralStateTests'
ALL_TESTS: Final = tuple(ALL_TEST_DIR.glob('**/*.json'))
BCHAIN_TESTS: Final = tuple(test_file for test_file in ALL_TESTS if test_file not in set(VM_TESTS))
SKIPPED_BCHAIN_TESTS: Final = tuple(test_file for test_file in BCHAIN_TESTS if test_file in SKIPPED_TESTS)


@pytest.mark.parametrize(
'test_file',
BCHAIN_TESTS,
ids=[str(test_file.relative_to(TEST_DIR)) for test_file in BCHAIN_TESTS],
ids=[str(test_file.relative_to(ALL_TEST_DIR)) for test_file in BCHAIN_TESTS],
)
def test_bchain(test_file: Path) -> None:
_test(test_file, 'SHANGHAI', 'NORMAL', 1, True)
Expand All @@ -125,8 +124,8 @@ def test_bchain(test_file: Path) -> None:
@pytest.mark.skip(reason='failing / slow blockchain tests')
@pytest.mark.parametrize(
'test_file',
REST_BCHAIN_TESTS,
ids=[str(test_file.relative_to(TEST_DIR)) for test_file in REST_BCHAIN_TESTS],
SKIPPED_BCHAIN_TESTS,
ids=[str(test_file.relative_to(ALL_TEST_DIR)) for test_file in SKIPPED_BCHAIN_TESTS],
)
def test_rest_bchain(test_file: Path) -> None:
_test(test_file, 'SHANGHAI', 'NORMAL', 1, True)
2 changes: 1 addition & 1 deletion package/version
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.0.620
1.0.621
92 changes: 2 additions & 90 deletions tests/slow.llvm
View file
Original file line number Diff line number Diff line change
@@ -1,90 +1,2 @@
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_Berlin
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_Istanbul
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_London
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_Merge
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_Shanghai
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,*
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,*

0 comments on commit 9b848e2

Please sign in to comment.