diff --git a/Makefile b/Makefile index 5c84b9965c..432d8a02cf 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,17 @@ test: test-integration test-conformance test-prove test-interactive test-conformance: poetry $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k test_conformance.py" +conformance-failing-list: poetry + cat /dev/null > tests/failing.llvm + - $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k test_conformance.py --save-failing --maxfail=10000" + LC_ALL=en_US.UTF-8 sort -f -d -o tests/failing.llvm tests/failing.llvm + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' '1{/^[[:space:]]*$$/d;}' tests/failing.llvm ;\ + echo >> tests/failing.llvm ;\ + else \ + sed -i '1{/^[[:space:]]*$$/d;}' tests/failing.llvm ;\ + fi + test-vm: poetry $(MAKE) -C kevm-pyk/ test-integration PYTEST_ARGS+="-k test_vm" diff --git a/tests/failing.llvm b/tests/failing.llvm index d39064cc84..c3907c58c2 100644 --- a/tests/failing.llvm +++ b/tests/failing.llvm @@ -17,16 +17,16 @@ BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4788_beacon_root/no_beacon_r BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4788_beacon_root/tx_to_beacon_root_contract.json,src/GeneralStateTestsFiller/Pyspecs/cancun/eip4788_beacon_root/test_beacon_root_contract.py::test_tx_to_beacon_root_contract[fork_Cancun-tx_type_3-blockchain_test-call_beacon_root_contract_True-auto_access_list_False] BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4788_beacon_root/tx_to_beacon_root_contract.json,src/GeneralStateTestsFiller/Pyspecs/cancun/eip4788_beacon_root/test_beacon_root_contract.py::test_tx_to_beacon_root_contract[fork_Cancun-tx_type_3-blockchain_test-call_beacon_root_contract_True-auto_access_list_True] BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_gas_subtraction_tx.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_tx_attribute_calldata_opcodes.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_tx_attribute_gasprice_opcode.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_tx_attribute_opcodes.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_tx_attribute_value_opcode.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_type_tx_pre_fork.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blobhash_gas_cost.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blobhash_invalid_blob_index.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blobhash_multiple_txs_in_block.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blobhash_opcode_contexts.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blobhash_scenarios.json,* +BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_tx_attribute_calldata_opcodes.json,* +BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_tx_attribute_gasprice_opcode.json,* +BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_tx_attribute_opcodes.json,* +BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_tx_attribute_value_opcode.json,* +BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/blob_type_tx_pre_fork.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/call_opcode_types.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/correct_decreasing_blob_gas_costs.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/correct_excess_blob_gas_calculation.json,* @@ -58,8 +58,8 @@ BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/point_evaluation_ BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/precompile_before_fork.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/precompile_during_fork.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/reject_valid_full_blob_in_block_rlp.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/sufficient_balance_blob_tx_pre_fund_tx.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/sufficient_balance_blob_tx.json,* +BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/sufficient_balance_blob_tx_pre_fund_tx.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/tx_entry_point.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/valid_blob_tx_combinations.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip4844_blobs/valid_inputs.json,* @@ -67,9 +67,9 @@ BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/create_sel BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision_multi_tx.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/dynamic_create2_selfdestruct_collision_two_different_transactions.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/recreate_self_destructed_contract_different_txs.json,* +BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_same_block_different_tx.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode_create_tx.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/self_destructing_initcode.json,* -BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_created_same_block_different_tx.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip6780_selfdestruct/selfdestruct_pre_existing.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip7516_blobgasfee/blobbasefee_before_fork.json,* BlockchainTests/GeneralStateTests/Pyspecs/cancun/eip7516_blobgasfee/blobbasefee_during_fork.json,* @@ -1010,8 +1010,8 @@ BlockchainTests/GeneralStateTests/stEIP1559/valCausesOOF.json,valCausesOOF_d0g2v BlockchainTests/GeneralStateTests/stEIP1559/valCausesOOF.json,valCausesOOF_d1g0v1_Cancun BlockchainTests/GeneralStateTests/stEIP1559/valCausesOOF.json,valCausesOOF_d1g2v0_Cancun BlockchainTests/GeneralStateTests/stEIP1559/valCausesOOF.json,valCausesOOF_d1g2v1_Cancun -BlockchainTests/GeneralStateTests/stEIP3607/transactionCollidingWithNonEmptyAccount_calls.json,* BlockchainTests/GeneralStateTests/stEIP3607/transactionCollidingWithNonEmptyAccount_callsItself.json,* +BlockchainTests/GeneralStateTests/stEIP3607/transactionCollidingWithNonEmptyAccount_calls.json,* BlockchainTests/GeneralStateTests/stEIP3607/transactionCollidingWithNonEmptyAccount_init_Paris.json,* BlockchainTests/GeneralStateTests/stEIP3607/transactionCollidingWithNonEmptyAccount_send_Paris.json,* BlockchainTests/GeneralStateTests/stExample/invalidTr.json,* @@ -1042,8 +1042,8 @@ BlockchainTests/GeneralStateTests/stSpecialTest/eoaEmptyParis.json,eoaEmptyParis BlockchainTests/GeneralStateTests/stSpecialTest/eoaEmptyParis.json,eoaEmptyParis_d1g0v1_Cancun BlockchainTests/GeneralStateTests/stSpecialTest/failed_tx_xcf416c53_Paris.json,* BlockchainTests/GeneralStateTests/stTransactionTest/HighGasPriceParis.json,* -BlockchainTests/GeneralStateTests/stTransactionTest/NoSrcAccount.json,* BlockchainTests/GeneralStateTests/stTransactionTest/NoSrcAccount1559.json,* -BlockchainTests/GeneralStateTests/stTransactionTest/NoSrcAccountCreate.json,* BlockchainTests/GeneralStateTests/stTransactionTest/NoSrcAccountCreate1559.json,* +BlockchainTests/GeneralStateTests/stTransactionTest/NoSrcAccountCreate.json,* +BlockchainTests/GeneralStateTests/stTransactionTest/NoSrcAccount.json,* BlockchainTests/GeneralStateTests/stTransactionTest/ValueOverflowParis.json,*