From 910136c2a3ef6be8fdc976cb0c6eadc3e4862d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 07:46:01 +0200 Subject: [PATCH 01/11] test: Add the evmone test command One command runs both fixture formats, deciding which each test case is when it runs it: "_info.fixture-format" names it, and a case without one is recognised by the shape only that format has. A format the tool does not run is skipped, as is a file holding no test at all, which is what EEST's shared pre-allocation is. Anything else it cannot recognise is a fault in the fixture. The two test tools are untouched and keep working. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 141 ++++++++++++++---- .../blockchaintest/eip7778_block_gas.json | 5 +- .../testcmd/one_unsupported_case.json | 116 ++++++++++++++ .../integration/testcmd_broken/truncated.json | 3 + .../testcmd_skipped/all_unsupported.json | 9 ++ test/integration/testcmd_skipped/empty.json | 1 + .../testcmd_skipped/partial_state_test.json | 20 +++ .../testcmd_skipped/pre_alloc.json | 27 ++++ test/utils/test_collector.cpp | 54 +++++++ test/utils/test_collector.hpp | 6 + test/utils/test_driver.cpp | 119 +++++++++++++++ test/utils/test_driver.hpp | 39 +++++ tools/evmone/main.cpp | 58 +++++++ 13 files changed, 565 insertions(+), 33 deletions(-) create mode 100644 test/integration/testcmd/one_unsupported_case.json create mode 100644 test/integration/testcmd_broken/truncated.json create mode 100644 test/integration/testcmd_skipped/all_unsupported.json create mode 100644 test/integration/testcmd_skipped/empty.json create mode 100644 test/integration/testcmd_skipped/partial_state_test.json create mode 100644 test/integration/testcmd_skipped/pre_alloc.json diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 4abbe17ece..87aa3c3210 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -82,39 +82,116 @@ DUP1,4 {\"pc\":6,\"op\":3,\"gas\":\"0xf4234\",\"gasCost\":\"0x3\",\"memSize\":0,\"stack\":\\[\"0x0\",\"0x4\"\\],\"depth\":1,\"refund\":0,\"opName\":\"SUB\"} ") -endif() + # One command runs both fixture formats, deciding per test case which one it is. + add_test(NAME ${PREFIX}/test_both_formats COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/test1.json + ${CMAKE_CURRENT_SOURCE_DIR}/blockchaintest/eip7778_block_gas.json) + set_tests_properties( + ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 2 passed in") + + # JSON which is not a test at all, as fixture directories hold beside their fixtures, is + # skipped rather than guessed at. + add_test(NAME ${PREFIX}/test_not_a_fixture COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/index.json) + set_tests_properties( + ${PREFIX}/test_not_a_fixture PROPERTIES PASS_REGULAR_EXPRESSION + "index\\.json::[^\n]* - not a test.*0 passed, 2 skipped in") + + # Over a directory a test is a whole file, so a case the loader refuses must not take the + # rest of the file with it: the file passes on the case beside it, the refused one first. + add_test(NAME ${PREFIX}/test_directory_with_unsupported_case COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd) + set_tests_properties( + ${PREFIX}/test_directory_with_unsupported_case PROPERTIES + PASS_REGULAR_EXPRESSION "= 1 passed in") + + # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these two assert on nothing + # else: a sound fixture directory succeeds, and a fault in one fails. + add_test(NAME ${PREFIX}/test_exit_code_success COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd) + + add_test(NAME ${PREFIX}/test_exit_code_failure COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) + set_tests_properties(${PREFIX}/test_exit_code_failure PROPERTIES WILL_FAIL TRUE) + + # WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero + # too, so the counts are what distinguish the two. + add_test(NAME ${PREFIX}/test_fault_is_not_a_skip COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) + set_tests_properties( + ${PREFIX}/test_fault_is_not_a_skip PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") -# A file holding a case which is not a fixture at all, beside one which runs. -add_test(NAME ${PREFIX}/fixture_fault COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) -set_tests_properties(${PREFIX}/fixture_fault PROPERTIES WILL_FAIL TRUE) - -# WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero too, -# so the counts are what distinguish the two. -add_test(NAME ${PREFIX}/fixture_fault_is_not_a_skip COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) -set_tests_properties( - ${PREFIX}/fixture_fault_is_not_a_skip PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - -# Selecting only the case which is not a fixture must still fault. -add_test(NAME ${PREFIX}/fixture_fault_survives_filter COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault -k b_not_a_fixture) -set_tests_properties( - ${PREFIX}/fixture_fault_survives_filter PROPERTIES - PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - -# A case whose load throws takes the rest of the file with it: the whole file is loaded before -# any of it runs, so the case after it is never reached and the failure is named after the file -# rather than the case it came from. FAILED pins that as a failure: a PASS_REGULAR_EXPRESSION -# makes CTest ignore the exit code, and a fault downgraded to a skip names the file too. -add_test(NAME ${PREFIX}/case_after_exception COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_cases) -set_tests_properties( - ${PREFIX}/case_after_exception PROPERTIES - PASS_REGULAR_EXPRESSION - "collected 1 test.*FAILED[^\n]*case_after_exception\\.json[^\n]*exception" - FAIL_REGULAR_EXPRESSION "b_wrong_state_root" -) + # Whether a file is a fixture file is a property of the file, not of what -k selected from + # it: selecting only the case which is not a fixture must still fault. + add_test(NAME ${PREFIX}/test_fault_survives_filter COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault -k b_not_a_fixture) + set_tests_properties( + ${PREFIX}/test_fault_survives_filter PROPERTIES + PASS_REGULAR_EXPRESSION "1 failed, 0 passed") + + # A case whose load throws must not abandon the cases after it, so both faults are reported. + add_test(NAME ${PREFIX}/test_case_after_exception COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_cases) + set_tests_properties( + ${PREFIX}/test_case_after_exception PROPERTIES + # "collected 1 test" keeps this on the directory form, where the whole file is one test + # and the per-case catch is what keeps the second fault reachable. + PASS_REGULAR_EXPRESSION + "collected 1 test.*a_load_error:\n exception.*b_wrong_state_root:.*state root") + + # Collected from a directory a file is one test, so a file with nothing to run is skipped + # whole, named by the first reason which explains it. All three reasons live here. + add_test(NAME ${PREFIX}/test_directory_skips_whole_files COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) + set_tests_properties( + ${PREFIX}/test_directory_skips_whole_files PROPERTIES + PASS_REGULAR_EXPRESSION + "all_unsupported\\.json - unsupported fixture format[^\n]*\nSKIPPED[^\n]*empty\\.json - no test cases.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") + + # Nothing to run is not nothing to report: the run fails rather than passing empty. + add_test(NAME ${PREFIX}/test_directory_skips_whole_files_exit_code COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) + set_tests_properties( + ${PREFIX}/test_directory_skips_whole_files_exit_code PROPERTIES WILL_FAIL TRUE) + + # Naming the file instead of the directory makes each case its own test, and a fault in one + # is still a fault: the case beside it runs. + add_test(NAME ${PREFIX}/test_file_fault COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault/unrecognised_case.json) + set_tests_properties( + ${PREFIX}/test_file_fault PROPERTIES PASS_REGULAR_EXPRESSION + "FAILED[^\n]*unrecognised_case\\.json::b_not_a_fixture.*1 failed, 1 passed in") + + # -k selects among the cases of a named file too. + add_test(NAME ${PREFIX}/test_file_filter COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault/unrecognised_case.json -k a_runs) + set_tests_properties( + ${PREFIX}/test_file_filter PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") + + # Naming a file whose cases this tool does not run skips them one by one, where collecting + # the same file from a directory would skip it whole. + add_test(NAME ${PREFIX}/test_file_declined_cases COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd/one_unsupported_case.json) + set_tests_properties( + ${PREFIX}/test_file_declined_cases PROPERTIES PASS_REGULAR_EXPRESSION + "SKIPPED[^\n]*c_declared_format_wins - unsupported fixture format.*= 1 passed, 2 skipped in") + + # --trace reaches the runner through the app-level flag, and turns the progress row into a + # line naming each test so the trace which follows it can be told apart. + add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace + ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/test1.json) + set_tests_properties( + ${PREFIX}/test_trace PROPERTIES PASS_REGULAR_EXPRESSION + "test1\\.json::test1\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in") + + # A named file which does not parse names itself, because no case in it can be. + add_test(NAME ${PREFIX}/test_file_does_not_parse COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_broken/truncated.json) + set_tests_properties( + ${PREFIX}/test_file_does_not_parse PROPERTIES PASS_REGULAR_EXPRESSION + "FAILED[^\n]*truncated\\.json - exception[^\n]*parse error.*1 failed, 0 passed in") + +endif() add_subdirectory(blockchaintest) add_subdirectory(export) diff --git a/test/integration/blockchaintest/eip7778_block_gas.json b/test/integration/blockchaintest/eip7778_block_gas.json index 326c57ef0b..f7bf7042c1 100644 --- a/test/integration/blockchaintest/eip7778_block_gas.json +++ b/test/integration/blockchaintest/eip7778_block_gas.json @@ -130,6 +130,9 @@ }, "genesisRLP": "0x", "sealEngine": "NoProof", - "_info": {} + "_info": { + "fixture-format": "blockchain_test", + "comment": "The format every EEST blockchain fixture declares." + } } } diff --git a/test/integration/testcmd/one_unsupported_case.json b/test/integration/testcmd/one_unsupported_case.json new file mode 100644 index 0000000000..20b93aafb7 --- /dev/null +++ b/test/integration/testcmd/one_unsupported_case.json @@ -0,0 +1,116 @@ +{ + "a_unsupported_unsupported_rlp": { + "_info": { + "comment": "Declines to load: expectException without rlp_decoded. Nothing of this case runs, so the case beside it is what the file reports." + }, + "network": "Cancun", + "genesisBlockHeader": { + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "stateRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "number": "0x00", + "gasLimit": "0x01000000", + "gasUsed": "0x00", + "timestamp": "0x00", + "extraData": "0x00", + "baseFeePerGas": "0x10", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "blobGasUsed": "0x00", + "excessBlobGas": "0x00", + "hash": "0x0000000000000000000000000000000000000000000000000000000000000001" + }, + "pre": { + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "code": "0x", + "nonce": "0x00", + "balance": "0x02540be400" + } + }, + "blocks": [ + { + "expectException": "TransactionException.INVALID_SIGNATURE_VRS", + "rlp": "0x" + } + ], + "lastblockhash": "0x0000000000000000000000000000000000000000000000000000000000000001", + "postState": { + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "code": "0x", + "nonce": "0x00", + "balance": "0x02540be400" + } + } + }, + "b_runs_state_test": { + "_info": { + "comment": "A state test beside blockchain ones, which is the whole point of deciding the format per case." + }, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + }, + "c_declared_format_wins": { + "_info": { + "fixture-format": "blockchain_test_engine", + "comment": "Carries blocks, so the shape alone would run it; the declared format is what decides, and this tool does not run that one." + }, + "blocks": [] + } +} diff --git a/test/integration/testcmd_broken/truncated.json b/test/integration/testcmd_broken/truncated.json new file mode 100644 index 0000000000..0c72185851 --- /dev/null +++ b/test/integration/testcmd_broken/truncated.json @@ -0,0 +1,3 @@ +{ + "a_case": { + "_info": {"comment": "The file ends here, mid-object, so it does not parse at all."} diff --git a/test/integration/testcmd_skipped/all_unsupported.json b/test/integration/testcmd_skipped/all_unsupported.json new file mode 100644 index 0000000000..400df5dbae --- /dev/null +++ b/test/integration/testcmd_skipped/all_unsupported.json @@ -0,0 +1,9 @@ +{ + "a_only_case_declines": { + "_info": { + "fixture-format": "blockchain_test_engine", + "comment": "The only case, in a format this tool does not run, so the file has nothing left to run and is skipped rather than passed." + }, + "blocks": [] + } +} diff --git a/test/integration/testcmd_skipped/empty.json b/test/integration/testcmd_skipped/empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/integration/testcmd_skipped/empty.json @@ -0,0 +1 @@ +{} diff --git a/test/integration/testcmd_skipped/partial_state_test.json b/test/integration/testcmd_skipped/partial_state_test.json new file mode 100644 index 0000000000..64d800acd4 --- /dev/null +++ b/test/integration/testcmd_skipped/partial_state_test.json @@ -0,0 +1,20 @@ +{ + "a_transaction_without_post": { + "comment": "Carries one of the two fields a state test is recognised by, so the shape does not name it and nothing else does either.", + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + } +} diff --git a/test/integration/testcmd_skipped/pre_alloc.json b/test/integration/testcmd_skipped/pre_alloc.json new file mode 100644 index 0000000000..355c895850 --- /dev/null +++ b/test/integration/testcmd_skipped/pre_alloc.json @@ -0,0 +1,27 @@ +{ + "testIds": [ + "tests/ported_static/vmIOandFlowOperations/test_mload.py::test_mload[fork_Osaka-blockchain_test_engine_x]" + ], + "environment": { + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01" + }, + "network": "Osaka", + "chainId": 1, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "comment": "EEST keeps the pre-allocation its fixtures share beside them, 35715 files of it in one release, and none of it is a test. Several entries, so the reason the file is skipped for is recorded once rather than per entry." +} diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index ab30751b91..55f26b1d7e 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -4,6 +4,8 @@ #include "test_collector.hpp" #include +#include +#include #include namespace evmone::test @@ -54,4 +56,56 @@ void ignore_test_files(std::vector& files, std::span i ignored, [&relative](const fs::path& prefix) { return is_under(relative, prefix); }); }); } + +bool collect_tests( + std::vector& cases, const fs::path& root, const TestSettings& settings, evmc::VM& vm) +{ + if (is_directory(root)) + { + auto files = collect_test_files(root); + ignore_test_files(files, settings.ignored); + cases.reserve(cases.size() + files.size()); + for (const auto& file : files) + { + // Loaded when the test runs: loading a whole tree up front costs far more. + cases.push_back( + {file.path.string(), [path = file.path, &settings, &vm](TestReport& report) { + run_fixture_file(path, settings, vm, report); + }}); + } + return true; + } + + // Naming a file loads it now, to name the fixtures in it. One which cannot be loaded + // becomes a single test reporting why. + json::json file; + try + { + std::ifstream f{root}; + file = json::json::parse(f); + } + catch (const std::exception& ex) + { + // Also reported here: --collect-only never runs the test. + std::cerr << root.string() << ": " << ex.what() << '\n'; + cases.push_back({root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + return false; + } + + // Whether the file is a fixture file at all decides what an unrecognised fixture in it means, + // exactly as it does for a whole file collected from a directory. + const auto file_holds_fixtures = is_fixture_file(file); + + for (const auto& [name, fixture] : file.items()) + { + if (!settings.selects(name)) + continue; + cases.push_back({root.string() + "::" + name, + [name, fixture, file_holds_fixtures, &settings, &vm](TestReport& report) { + run_fixture(name, fixture, file_holds_fixtures, settings, vm, report); + }}); + } + return true; +} } // namespace evmone::test diff --git a/test/utils/test_collector.hpp b/test/utils/test_collector.hpp index f116d6639e..596b802fd3 100644 --- a/test/utils/test_collector.hpp +++ b/test/utils/test_collector.hpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include #include #include #include @@ -32,4 +33,9 @@ struct TestFile /// "bc4895-withdrawals". void ignore_test_files( std::vector& files, std::span ignored); + +/// Adds to @p cases every test under @p root: one per file for a directory, one per fixture in +/// the file when the file itself is named. Returns whether every test was collected. +bool collect_tests(std::vector& cases, const std::filesystem::path& root, + const TestSettings& settings, evmc::VM& vm); } // namespace evmone::test diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 5ad47d8382..0827f86b6d 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -3,13 +3,18 @@ // SPDX-License-Identifier: Apache-2.0 #include "test_driver.hpp" +#include +#include #include +#include #include #include #include namespace evmone::test { +namespace fs = std::filesystem; + namespace { /// The report is laid out like pytest's. @@ -65,6 +70,49 @@ class Progress m_column = 0; } }; + +/// The fixture formats this tool runs. EEST names the format in each fixture's "_info", and a +/// heuristic covers the hand-written and pre-EEST files which have no "_info" at all. +enum class Format +{ + state_test, + blockchain_test, +}; + +/// What this tool makes of one fixture. +struct Classification +{ + /// How to run it, when this tool runs it. + std::optional format; + /// A fixture, whether or not this tool runs it. + bool is_fixture = false; + /// Why it is not run, when it is not. + std::string reason; +}; + +Classification classify(const json::json& fixture) +{ + if (const auto info = fixture.find("_info"); info != fixture.end()) + { + if (const auto format = info->find("fixture-format"); format != info->end()) + { + if (*format == "state_test") + return {.format = Format::state_test, .is_fixture = true}; + if (*format == "blockchain_test") + return {.format = Format::blockchain_test, .is_fixture = true}; + return {.is_fixture = true, .reason = "unsupported fixture format: " + format->dump()}; + } + } + // Nothing declares the format: a hand-written or pre-EEST file, or an "_info" without one. + // Each shape is named by what only it carries; anything else is not a test at all, as EEST's + // shared pre-allocation kept beside the fixtures is not. + if (fixture.contains("blocks")) + return {.format = Format::blockchain_test, .is_fixture = true}; + if (fixture.contains("transaction") && fixture.contains("post")) + return {.format = Format::state_test, .is_fixture = true}; + return {.reason = "not a test"}; +} + } // namespace int run_tests(std::span cases, std::ostream& out, const RunOptions& options) @@ -185,4 +233,75 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio // case of its own still counts as passed, which this does not change. return passed == 0 ? NOTHING_VERIFIED : SUCCESS; } + +bool is_fixture_file(const json::json& contents) +{ + return std::ranges::any_of( + contents.items(), [](const auto& item) { return classify(item.value()).is_fixture; }); +} + +void run_fixture_file( + const fs::path& path, const TestSettings& settings, evmc::VM& vm, TestReport& report) +{ + std::ifstream f{path}; + // Named, because items() only borrows: iterating a temporary dangles. + const auto contents = json::json::parse(f); + const auto file_holds_fixtures = is_fixture_file(contents); + + std::optional declined; // The reason for the first fixture this tool declined. + bool any_ran = false; + for (const auto& [name, fixture] : contents.items()) + { + if (!settings.selects(name)) + continue; + try + { + run_fixture(name, fixture, file_holds_fixtures, settings, vm, report); + } + catch (const UnsupportedTestFeature& ex) + { + // This tool's own limit, whether the fixture declared a format it does not run or + // the file holds no test for its shape to be read against. + if (!declined) + declined = ex.what(); + continue; + } + catch (const std::exception& ex) + { + // The fixture is a test and it went wrong, which is this file's verdict but not the + // end of it: the fixtures after it are still worth running. + report.fail(concat("exception: ", ex.what())); + } + any_ran = true; + } + + // A file in which this tool ran nothing it was asked for is skipped, not passed. + // TODO: A file whose cases -k all deselected still passes, as it did before this command + // existed, so a filter which matches nothing reports a tree of passing tests. Skip it + // instead, and an empty selection reaches NOTHING_VERIFIED on its own. + if (!any_ran) + { + if (declined) + throw UnsupportedTestFeature{*declined}; + if (!file_holds_fixtures) + throw UnsupportedTestFeature{"no test cases"}; + } +} + +void run_fixture(const std::string& name, const json::json& fixture, bool file_holds_fixtures, + const TestSettings& settings, evmc::VM& vm, TestReport& report) +{ + report.start_case(name); // Names whatever the load itself reports. + const auto [format, is_fixture, reason] = classify(fixture); + if (format == Format::state_test) + run_state_test(make_state_test(name, fixture), vm, settings.trace_summary, report); + else if (format == Format::blockchain_test) + run_blockchain_test(make_blockchain_test(name, fixture), vm, report); + // Not recognising a fixture at all is a fault in the file, once the rest of it shows the + // file to be a fixture file. Anything else is this tool's own limit. + else if (file_holds_fixtures && !is_fixture) + report.fail(reason); + else + throw UnsupportedTestFeature{reason}; +} } // namespace evmone::test diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index 2694b48680..8431b7e270 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -3,10 +3,16 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include +#include #include +#include +#include namespace evmone::test { +namespace json = nlohmann; + /// Nothing failed and something passed. constexpr int SUCCESS = 0; @@ -43,4 +49,37 @@ struct RunOptions /// got. [[nodiscard]] int run_tests( std::span cases, std::ostream& out, const RunOptions& options = {}); + +/// What the tests are run with. +struct TestSettings +{ + /// Run only the fixtures whose name contains this. + std::optional name_filter; + + /// Paths, relative to a test directory, not to collect tests from. + std::vector ignored; + + /// Report each test's execution summary on the trace stream. + bool trace_summary = false; + + /// Whether the name filter, if any, keeps the fixture called @p name. + [[nodiscard]] bool selects(const std::string& name) const noexcept + { + return !name_filter.has_value() || name.find(*name_filter) != std::string::npos; + } +}; + +/// Whether the file holds fixtures at all. A property of the file, never of what the name +/// filter selected out of it, so both ways of naming a test agree. +[[nodiscard]] bool is_fixture_file(const json::json& contents); + +/// Runs every selected fixture of one fixture file, which together are one test. Throws +/// UnsupportedTestFeature for a file this tool has nothing to run in. +void run_fixture_file(const std::filesystem::path& path, const TestSettings& settings, evmc::VM& vm, + TestReport& report); + +/// Runs one fixture of a file named on its own. Whether the file holds fixtures at all decides +/// what an unrecognised one in it means, so @p file_holds_fixtures is asked of the whole file. +void run_fixture(const std::string& name, const json::json& fixture, bool file_holds_fixtures, + const TestSettings& settings, evmc::VM& vm, TestReport& report); } // namespace evmone::test diff --git a/tools/evmone/main.cpp b/tools/evmone/main.cpp index e5e6c24ece..9bd76d374a 100644 --- a/tools/evmone/main.cpp +++ b/tools/evmone/main.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -151,6 +152,57 @@ int exec_t8n_cmd(evmc::VM& vm, const T8nOptions& opts) evmone::tooling::t8n(vm, args); return 0; } + +/// The options of the "test" command. +struct TestOptions +{ + std::vector paths; + evmone::test::TestSettings settings; + evmone::test::RunOptions run; +}; + +const CLI::App& setup_test_cmd(CLI::App& app, TestOptions& opts) +{ + auto& cmd = *app.add_subcommand("test", "Run Ethereum tests")->fallthrough(); + cmd.add_option("path", opts.paths, + "Test file or directory. Every .json file under a directory, except index.json, is a " + "test; naming a file makes each test case in it a test of its own.") + ->required() + ->check(CLI::ExistingPath); + cmd.add_option( + "-k", opts.settings.name_filter, "Run only the test cases whose name contains this."); + cmd.add_option("--ignore", opts.settings.ignored, + "Path, relative to a test directory, not to collect tests from. May be given more than " + "once. Whole path components are matched, so --ignore bc4895 keeps bc4895-withdrawals.") + // Without this the option is variadic and swallows the positional paths after it. + ->allow_extra_args(false); + cmd.add_flag("--collect-only", opts.run.collect_only, + "List each collected test, one per line, and exit."); + cmd.add_flag("--trace-summary", opts.settings.trace_summary, + "Report each state test's execution summary, as --trace also does. Blockchain tests " + "have no summary to report."); + return cmd; +} + +int exec_test_cmd(evmc::VM& vm, TestOptions opts, bool trace) +{ + // main() has switched the tracer on already. Its line per instruction is worth + // unsynchronising the streams for, and would run into the progress row, as a summary would. + if (trace) + std::ios::sync_with_stdio(false); + opts.settings.trace_summary |= trace; + opts.run.progress = !opts.settings.trace_summary; + + std::vector cases; + bool all_collected = true; + for (const auto& p : opts.paths) + all_collected &= collect_tests(cases, p, opts.settings, vm); + + const auto exit_code = evmone::test::run_tests(cases, std::cout, opts.run); + // A file which could not be loaded fails the listing too, not only a run of it. + return all_collected ? exit_code : evmone::test::TESTS_FAILED; +} + } // namespace int main(int argc, const char* const* argv) noexcept @@ -198,6 +250,9 @@ int main(int argc, const char* const* argv) noexcept T8nOptions t8n_opts; const auto& t8n_cmd = setup_t8n_cmd(app, t8n_opts); + TestOptions test_opts; + const auto& test_cmd = setup_test_cmd(app, test_opts); + try { app.parse(argc, argv); @@ -218,6 +273,9 @@ int main(int argc, const char* const* argv) noexcept if (t8n_cmd) return exec_t8n_cmd(vm, t8n_opts); + if (test_cmd) + return exec_test_cmd(vm, test_opts, trace); + return 0; } catch (const CLI::ParseError& e) From 8b12a8f0257f51e6c27c4ca8947f182cdf357e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 09:08:52 +0200 Subject: [PATCH 02/11] test: Settle that a file is not a test before running any of it An entry the tool did not recognise was a fault or a skip depending on whether the rest of its file holds fixtures, a bool threaded from the collector through a lambda into run_fixture to feed one branch. Loading the file settles it instead: a file with no fixture in it is skipped whole and nothing in it reaches execution, so an unrecognised entry in a fixture file is simply a fault. Named on the command line, such a file is one skipped test rather than one per entry. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 8 +++---- test/utils/test_collector.cpp | 27 +++++++++++---------- test/utils/test_driver.cpp | 42 +++++++++++++++------------------ test/utils/test_driver.hpp | 14 +++++------ 4 files changed, 44 insertions(+), 47 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 87aa3c3210..8ef2915b83 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -90,12 +90,12 @@ DUP1,4 ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 2 passed in") # JSON which is not a test at all, as fixture directories hold beside their fixtures, is - # skipped rather than guessed at. + # skipped whole rather than guessed at, even named on its own. add_test(NAME ${PREFIX}/test_not_a_fixture COMMAND evmone-cli test ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/index.json) set_tests_properties( ${PREFIX}/test_not_a_fixture PROPERTIES PASS_REGULAR_EXPRESSION - "index\\.json::[^\n]* - not a test.*0 passed, 2 skipped in") + "SKIPPED[^\n]*index\\.json - not a test.*0 passed, 1 skipped in") # Over a directory a test is a whole file, so a case the loader refuses must not take the # rest of the file with it: the file passes on the case beside it, the refused one first. @@ -140,13 +140,13 @@ DUP1,4 "collected 1 test.*a_load_error:\n exception.*b_wrong_state_root:.*state root") # Collected from a directory a file is one test, so a file with nothing to run is skipped - # whole, named by the first reason which explains it. All three reasons live here. + # whole, named by the first reason which explains it. Both reasons live here. add_test(NAME ${PREFIX}/test_directory_skips_whole_files COMMAND evmone-cli test ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) set_tests_properties( ${PREFIX}/test_directory_skips_whole_files PROPERTIES PASS_REGULAR_EXPRESSION - "all_unsupported\\.json - unsupported fixture format[^\n]*\nSKIPPED[^\n]*empty\\.json - no test cases.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") + "all_unsupported\\.json - unsupported fixture format[^\n]*\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") # Nothing to run is not nothing to report: the run fails rather than passing empty. add_test(NAME ${PREFIX}/test_directory_skips_whole_files_exit_code COMMAND evmone-cli test diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index 55f26b1d7e..53d5df257a 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -4,7 +4,6 @@ #include "test_collector.hpp" #include -#include #include #include @@ -76,13 +75,19 @@ bool collect_tests( return true; } - // Naming a file loads it now, to name the fixtures in it. One which cannot be loaded - // becomes a single test reporting why. + // Naming a file loads it now, to name the fixtures in it. One which cannot be loaded, or + // which is not a test, becomes a single test reporting why. json::json file; try { - std::ifstream f{root}; - file = json::json::parse(f); + file = load_fixture_file(root); + } + catch (const UnsupportedTestFeature&) + { + // A skip, as when collected from a directory, not a broken collection. + cases.push_back({root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + return true; } catch (const std::exception& ex) { @@ -93,18 +98,14 @@ bool collect_tests( return false; } - // Whether the file is a fixture file at all decides what an unrecognised fixture in it means, - // exactly as it does for a whole file collected from a directory. - const auto file_holds_fixtures = is_fixture_file(file); - for (const auto& [name, fixture] : file.items()) { if (!settings.selects(name)) continue; - cases.push_back({root.string() + "::" + name, - [name, fixture, file_holds_fixtures, &settings, &vm](TestReport& report) { - run_fixture(name, fixture, file_holds_fixtures, settings, vm, report); - }}); + cases.push_back( + {root.string() + "::" + name, [name, fixture, &settings, &vm](TestReport& report) { + run_fixture(name, fixture, settings, vm, report); + }}); } return true; } diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 0827f86b6d..e8ab9de929 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -234,19 +234,23 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio return passed == 0 ? NOTHING_VERIFIED : SUCCESS; } -bool is_fixture_file(const json::json& contents) +json::json load_fixture_file(const fs::path& path) { - return std::ranges::any_of( - contents.items(), [](const auto& item) { return classify(item.value()).is_fixture; }); + std::ifstream f{path}; + auto contents = json::json::parse(f); + // Not one fixture in it: EEST keeps its shared pre-allocation and an index of the fixtures + // beside them, and neither is a test. + if (std::ranges::none_of( + contents.items(), [](const auto& item) { return classify(item.value()).is_fixture; })) + throw UnsupportedTestFeature{"not a test"}; + return contents; } void run_fixture_file( const fs::path& path, const TestSettings& settings, evmc::VM& vm, TestReport& report) { - std::ifstream f{path}; // Named, because items() only borrows: iterating a temporary dangles. - const auto contents = json::json::parse(f); - const auto file_holds_fixtures = is_fixture_file(contents); + const auto contents = load_fixture_file(path); std::optional declined; // The reason for the first fixture this tool declined. bool any_ran = false; @@ -256,12 +260,11 @@ void run_fixture_file( continue; try { - run_fixture(name, fixture, file_holds_fixtures, settings, vm, report); + run_fixture(name, fixture, settings, vm, report); } catch (const UnsupportedTestFeature& ex) { - // This tool's own limit, whether the fixture declared a format it does not run or - // the file holds no test for its shape to be read against. + // This tool's own limit: a format it does not run, or a fixture its loader refuses. if (!declined) declined = ex.what(); continue; @@ -279,17 +282,12 @@ void run_fixture_file( // TODO: A file whose cases -k all deselected still passes, as it did before this command // existed, so a filter which matches nothing reports a tree of passing tests. Skip it // instead, and an empty selection reaches NOTHING_VERIFIED on its own. - if (!any_ran) - { - if (declined) - throw UnsupportedTestFeature{*declined}; - if (!file_holds_fixtures) - throw UnsupportedTestFeature{"no test cases"}; - } + if (!any_ran && declined) + throw UnsupportedTestFeature{*declined}; } -void run_fixture(const std::string& name, const json::json& fixture, bool file_holds_fixtures, - const TestSettings& settings, evmc::VM& vm, TestReport& report) +void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings, + evmc::VM& vm, TestReport& report) { report.start_case(name); // Names whatever the load itself reports. const auto [format, is_fixture, reason] = classify(fixture); @@ -297,11 +295,9 @@ void run_fixture(const std::string& name, const json::json& fixture, bool file_h run_state_test(make_state_test(name, fixture), vm, settings.trace_summary, report); else if (format == Format::blockchain_test) run_blockchain_test(make_blockchain_test(name, fixture), vm, report); - // Not recognising a fixture at all is a fault in the file, once the rest of it shows the - // file to be a fixture file. Anything else is this tool's own limit. - else if (file_holds_fixtures && !is_fixture) - report.fail(reason); + else if (is_fixture) + throw UnsupportedTestFeature{reason}; // A format this tool does not run. else - throw UnsupportedTestFeature{reason}; + report.fail(reason); // The rest of the file holds fixtures, so this one is broken. } } // namespace evmone::test diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index 8431b7e270..27cd9fce65 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -69,17 +69,17 @@ struct TestSettings } }; -/// Whether the file holds fixtures at all. A property of the file, never of what the name -/// filter selected out of it, so both ways of naming a test agree. -[[nodiscard]] bool is_fixture_file(const json::json& contents); +/// Parses the fixture file at @p path. Throws UnsupportedTestFeature for a file with no fixture +/// in it, which is nothing to run: fixture directories hold other JSON beside the fixtures. +[[nodiscard]] json::json load_fixture_file(const std::filesystem::path& path); /// Runs every selected fixture of one fixture file, which together are one test. Throws /// UnsupportedTestFeature for a file this tool has nothing to run in. void run_fixture_file(const std::filesystem::path& path, const TestSettings& settings, evmc::VM& vm, TestReport& report); -/// Runs one fixture of a file named on its own. Whether the file holds fixtures at all decides -/// what an unrecognised one in it means, so @p file_holds_fixtures is asked of the whole file. -void run_fixture(const std::string& name, const json::json& fixture, bool file_holds_fixtures, - const TestSettings& settings, evmc::VM& vm, TestReport& report); +/// Runs one fixture of a fixture file. One this tool does not recognise is a fault in the file; +/// one in a format it does not run is skipped. +void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings, + evmc::VM& vm, TestReport& report); } // namespace evmone::test From b01df2a73c744acb955b1498265b5e756f2fd3a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 14:21:44 +0200 Subject: [PATCH 03/11] test: Pin the filter, the trace summary and the first decline Three of the command's ctests were named for behaviour they could not observe: deleting the -k filter from a directory run, hardcoding the trace summary off, or taking the last decline reason rather than the first all left the integration suite green. One run with -k and --trace-summary over a directory holding a failing case covers the first two, and a second declining case in all_unsupported names which reason wins. The two fault tests differed only in the filter, so they fold into one. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 93 +++++++++---------- .../testcmd/one_unsupported_case.json | 6 +- .../testcmd_cases/case_after_exception.json | 2 +- .../testcmd_fault/unrecognised_case.json | 2 +- .../testcmd_skipped/all_unsupported.json | 10 +- .../testcmd_skipped/partial_state_test.json | 2 +- test/utils/test_driver.cpp | 4 +- 7 files changed, 61 insertions(+), 58 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 8ef2915b83..c384300a21 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -82,113 +82,110 @@ DUP1,4 {\"pc\":6,\"op\":3,\"gas\":\"0xf4234\",\"gasCost\":\"0x3\",\"memSize\":0,\"stack\":\\[\"0x0\",\"0x4\"\\],\"depth\":1,\"refund\":0,\"opName\":\"SUB\"} ") + set(DATA ${CMAKE_CURRENT_SOURCE_DIR}) + # One command runs both fixture formats, deciding per test case which one it is. add_test(NAME ${PREFIX}/test_both_formats COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/test1.json - ${CMAKE_CURRENT_SOURCE_DIR}/blockchaintest/eip7778_block_gas.json) + ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json) set_tests_properties( ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 2 passed in") # JSON which is not a test at all, as fixture directories hold beside their fixtures, is # skipped whole rather than guessed at, even named on its own. - add_test(NAME ${PREFIX}/test_not_a_fixture COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/index.json) + add_test(NAME ${PREFIX}/test_not_a_test COMMAND evmone-cli test + ${DATA}/statetest/tests1/SuiteA/index.json) set_tests_properties( - ${PREFIX}/test_not_a_fixture PROPERTIES PASS_REGULAR_EXPRESSION + ${PREFIX}/test_not_a_test PROPERTIES PASS_REGULAR_EXPRESSION "SKIPPED[^\n]*index\\.json - not a test.*0 passed, 1 skipped in") # Over a directory a test is a whole file, so a case the loader refuses must not take the # rest of the file with it: the file passes on the case beside it, the refused one first. - add_test(NAME ${PREFIX}/test_directory_with_unsupported_case COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd) + add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) set_tests_properties( - ${PREFIX}/test_directory_with_unsupported_case PROPERTIES - PASS_REGULAR_EXPRESSION "= 1 passed in") + ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these two assert on nothing # else: a sound fixture directory succeeds, and a fault in one fails. - add_test(NAME ${PREFIX}/test_exit_code_success COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd) + add_test(NAME ${PREFIX}/test_exit_ok COMMAND evmone-cli test ${DATA}/testcmd) - add_test(NAME ${PREFIX}/test_exit_code_failure COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) - set_tests_properties(${PREFIX}/test_exit_code_failure PROPERTIES WILL_FAIL TRUE) + add_test(NAME ${PREFIX}/test_exit_fail COMMAND evmone-cli test ${DATA}/testcmd_fault) + set_tests_properties(${PREFIX}/test_exit_fail PROPERTIES WILL_FAIL TRUE) # WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero - # too, so the counts are what distinguish the two. - add_test(NAME ${PREFIX}/test_fault_is_not_a_skip COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) + # too, so the counts are what distinguish the two. Whether a file is a fixture file is a + # property of the file, not of what -k selected: the case which is not a test still faults. + add_test(NAME ${PREFIX}/test_dir_fault COMMAND evmone-cli test + ${DATA}/testcmd_fault -k b_not_a_test) set_tests_properties( - ${PREFIX}/test_fault_is_not_a_skip PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") + ${PREFIX}/test_dir_fault PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - # Whether a file is a fixture file is a property of the file, not of what -k selected from - # it: selecting only the case which is not a fixture must still fault. - add_test(NAME ${PREFIX}/test_fault_survives_filter COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault -k b_not_a_fixture) + # Over a directory -k selects within the file's test, so the summary is what proves a case + # ran: a count alone would hold just as well if the filter had dropped every case. The + # summary appears only with --trace-summary, which nothing else here asks for. + add_test(NAME ${PREFIX}/test_dir_filter COMMAND evmone-cli test + ${DATA}/statetest/filter -k passing_test_case --trace-summary) set_tests_properties( - ${PREFIX}/test_fault_survives_filter PROPERTIES - PASS_REGULAR_EXPRESSION "1 failed, 0 passed") + ${PREFIX}/test_dir_filter PROPERTIES + PASS_REGULAR_EXPRESSION "\"pass\":true" + FAIL_REGULAR_EXPRESSION "failing_test_case") # A case whose load throws must not abandon the cases after it, so both faults are reported. - add_test(NAME ${PREFIX}/test_case_after_exception COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_cases) + add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_cases) set_tests_properties( - ${PREFIX}/test_case_after_exception PROPERTIES + ${PREFIX}/test_after_exception PROPERTIES # "collected 1 test" keeps this on the directory form, where the whole file is one test # and the per-case catch is what keeps the second fault reachable. PASS_REGULAR_EXPRESSION - "collected 1 test.*a_load_error:\n exception.*b_wrong_state_root:.*state root") + "collected 1 test.*a_load_error:\n exception.*b_bad_root:.*state root") # Collected from a directory a file is one test, so a file with nothing to run is skipped - # whole, named by the first reason which explains it. Both reasons live here. - add_test(NAME ${PREFIX}/test_directory_skips_whole_files COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) + # whole, named by the first reason which explains it: all_unsupported declines twice, and + # the format named is the first one's. + add_test(NAME ${PREFIX}/test_dir_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) set_tests_properties( - ${PREFIX}/test_directory_skips_whole_files PROPERTIES + ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION - "all_unsupported\\.json - unsupported fixture format[^\n]*\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") + "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") # Nothing to run is not nothing to report: the run fails rather than passing empty. - add_test(NAME ${PREFIX}/test_directory_skips_whole_files_exit_code COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) - set_tests_properties( - ${PREFIX}/test_directory_skips_whole_files_exit_code PROPERTIES WILL_FAIL TRUE) + add_test(NAME ${PREFIX}/test_dir_skipped_exit COMMAND evmone-cli test ${DATA}/testcmd_skipped) + set_tests_properties(${PREFIX}/test_dir_skipped_exit PROPERTIES WILL_FAIL TRUE) # Naming the file instead of the directory makes each case its own test, and a fault in one # is still a fault: the case beside it runs. add_test(NAME ${PREFIX}/test_file_fault COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault/unrecognised_case.json) + ${DATA}/testcmd_fault/unrecognised_case.json) set_tests_properties( ${PREFIX}/test_file_fault PROPERTIES PASS_REGULAR_EXPRESSION - "FAILED[^\n]*unrecognised_case\\.json::b_not_a_fixture.*1 failed, 1 passed in") + "FAILED[^\n]*unrecognised_case\\.json::b_not_a_test.*1 failed, 1 passed in") # -k selects among the cases of a named file too. add_test(NAME ${PREFIX}/test_file_filter COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault/unrecognised_case.json -k a_runs) + ${DATA}/testcmd_fault/unrecognised_case.json -k a_runs) set_tests_properties( ${PREFIX}/test_file_filter PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") # Naming a file whose cases this tool does not run skips them one by one, where collecting # the same file from a directory would skip it whole. - add_test(NAME ${PREFIX}/test_file_declined_cases COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd/one_unsupported_case.json) + add_test(NAME ${PREFIX}/test_file_declined COMMAND evmone-cli test + ${DATA}/testcmd/one_unsupported_case.json) set_tests_properties( - ${PREFIX}/test_file_declined_cases PROPERTIES PASS_REGULAR_EXPRESSION - "SKIPPED[^\n]*c_declared_format_wins - unsupported fixture format.*= 1 passed, 2 skipped in") + ${PREFIX}/test_file_declined PROPERTIES PASS_REGULAR_EXPRESSION + "SKIPPED[^\n]*c_engine - unsupported fixture format.*= 1 passed, 2 skipped in") # --trace reaches the runner through the app-level flag, and turns the progress row into a # line naming each test so the trace which follows it can be told apart. add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace - ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/test1.json) + ${DATA}/statetest/tests1/SuiteA/test1.json) set_tests_properties( ${PREFIX}/test_trace PROPERTIES PASS_REGULAR_EXPRESSION "test1\\.json::test1\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in") # A named file which does not parse names itself, because no case in it can be. - add_test(NAME ${PREFIX}/test_file_does_not_parse COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_broken/truncated.json) + add_test(NAME ${PREFIX}/test_file_unparsed COMMAND evmone-cli test + ${DATA}/testcmd_broken/truncated.json) set_tests_properties( - ${PREFIX}/test_file_does_not_parse PROPERTIES PASS_REGULAR_EXPRESSION + ${PREFIX}/test_file_unparsed PROPERTIES PASS_REGULAR_EXPRESSION "FAILED[^\n]*truncated\\.json - exception[^\n]*parse error.*1 failed, 0 passed in") endif() diff --git a/test/integration/testcmd/one_unsupported_case.json b/test/integration/testcmd/one_unsupported_case.json index 20b93aafb7..35eea677ad 100644 --- a/test/integration/testcmd/one_unsupported_case.json +++ b/test/integration/testcmd/one_unsupported_case.json @@ -1,5 +1,5 @@ { - "a_unsupported_unsupported_rlp": { + "a_bad_rlp": { "_info": { "comment": "Declines to load: expectException without rlp_decoded. Nothing of this case runs, so the case beside it is what the file reports." }, @@ -44,7 +44,7 @@ } } }, - "b_runs_state_test": { + "b_state": { "_info": { "comment": "A state test beside blockchain ones, which is the whole point of deciding the format per case." }, @@ -106,7 +106,7 @@ ] } }, - "c_declared_format_wins": { + "c_engine": { "_info": { "fixture-format": "blockchain_test_engine", "comment": "Carries blocks, so the shape alone would run it; the declared format is what decides, and this tool does not run that one." diff --git a/test/integration/testcmd_cases/case_after_exception.json b/test/integration/testcmd_cases/case_after_exception.json index 0b8f8e37f7..daac56ce62 100644 --- a/test/integration/testcmd_cases/case_after_exception.json +++ b/test/integration/testcmd_cases/case_after_exception.json @@ -5,7 +5,7 @@ "comment": "A state test with no pre state, so loading it throws." } }, - "b_wrong_state_root": { + "b_bad_root": { "_info": { "comment": "Runs and fails on the state root. Only reported if the case before it did not abandon the file." }, diff --git a/test/integration/testcmd_fault/unrecognised_case.json b/test/integration/testcmd_fault/unrecognised_case.json index e389b69953..2e3a02ea41 100644 --- a/test/integration/testcmd_fault/unrecognised_case.json +++ b/test/integration/testcmd_fault/unrecognised_case.json @@ -59,7 +59,7 @@ ] } }, - "b_not_a_fixture": { + "b_not_a_test": { "_info": { "comment": "Not a test: no fixture fields at all. Beside a case which runs, so this is unmistakably a fixture file with one broken case in it." } diff --git a/test/integration/testcmd_skipped/all_unsupported.json b/test/integration/testcmd_skipped/all_unsupported.json index 400df5dbae..2a9fb9915b 100644 --- a/test/integration/testcmd_skipped/all_unsupported.json +++ b/test/integration/testcmd_skipped/all_unsupported.json @@ -1,8 +1,14 @@ { - "a_only_case_declines": { + "a_engine": { "_info": { "fixture-format": "blockchain_test_engine", - "comment": "The only case, in a format this tool does not run, so the file has nothing left to run and is skipped rather than passed." + "comment": "The file is named by this reason, the first collected, not the one below." + }, + "blocks": [] + }, + "b_engine_x": { + "_info": { + "fixture-format": "blockchain_test_engine_x" }, "blocks": [] } diff --git a/test/integration/testcmd_skipped/partial_state_test.json b/test/integration/testcmd_skipped/partial_state_test.json index 64d800acd4..77e71ca03c 100644 --- a/test/integration/testcmd_skipped/partial_state_test.json +++ b/test/integration/testcmd_skipped/partial_state_test.json @@ -1,5 +1,5 @@ { - "a_transaction_without_post": { + "a_no_post": { "comment": "Carries one of the two fields a state test is recognised by, so the shape does not name it and nothing else does either.", "transaction": { "data": [ diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index e8ab9de929..ced3eeeeed 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -280,8 +280,8 @@ void run_fixture_file( // A file in which this tool ran nothing it was asked for is skipped, not passed. // TODO: A file whose cases -k all deselected still passes, as it did before this command - // existed, so a filter which matches nothing reports a tree of passing tests. Skip it - // instead, and an empty selection reaches NOTHING_VERIFIED on its own. + // existed, so a filter which matches nothing turns a failing tree green. Skip it instead, + // and an empty selection reaches NOTHING_VERIFIED on its own. if (!any_ran && declined) throw UnsupportedTestFeature{*declined}; } From ec2fab4b73e4445870999c45731333f96ba8f8f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 14:53:39 +0200 Subject: [PATCH 04/11] test: Name a fixture by its input, not by what it expects A fixture was taken for a blockchain test on a "blocks" key alone, so any JSON sidecar carrying one was run as a test and failed the whole run with "key 'genesisBlockHeader' not found", while the symmetric miss on the state shape was a quiet skip. Each shape now needs the state it starts from and what is applied to it: "pre" with "blocks", or "pre" with "transaction". Neither reads an expectation, so a fixture whose expectations are missing is still run rather than passing as no test at all, and "pre" is in every fixture of every format. classify() returns the format alone, the two ways of not running one among its values, so run_fixture switches over it exhaustively and no return site can label a fixture wrongly. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 8 ++- .../testcmd_skipped/partial_shapes.json | 24 +++++++ .../testcmd_skipped/partial_state_test.json | 20 ------ test/utils/test_driver.cpp | 63 +++++++++---------- 4 files changed, 60 insertions(+), 55 deletions(-) create mode 100644 test/integration/testcmd_skipped/partial_shapes.json delete mode 100644 test/integration/testcmd_skipped/partial_state_test.json diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index c384300a21..b79f2f0b66 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -84,11 +84,13 @@ DUP1,4 set(DATA ${CMAKE_CURRENT_SOURCE_DIR}) - # One command runs both fixture formats, deciding per test case which one it is. + # One command runs both fixture formats, deciding per test case which one it is: the state + # test and the last one by shape, the blockchain test by the format its "_info" declares. add_test(NAME ${PREFIX}/test_both_formats COMMAND evmone-cli test - ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json) + ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json + ${DATA}/blockchaintest/unrecovered_sender_blob_gas.json) set_tests_properties( - ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 2 passed in") + ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 3 passed in") # JSON which is not a test at all, as fixture directories hold beside their fixtures, is # skipped whole rather than guessed at, even named on its own. diff --git a/test/integration/testcmd_skipped/partial_shapes.json b/test/integration/testcmd_skipped/partial_shapes.json new file mode 100644 index 0000000000..5d85ea8afa --- /dev/null +++ b/test/integration/testcmd_skipped/partial_shapes.json @@ -0,0 +1,24 @@ +{ + "a_blocks_no_pre": { + "comment": "Carries one of the two keys a blockchain test is named by, so the shape does not name it and nothing else does either.", + "blocks": [] + }, + "b_tx_no_pre": { + "comment": "The same for a state test: a transaction to apply, but no state to apply it to.", + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + } +} diff --git a/test/integration/testcmd_skipped/partial_state_test.json b/test/integration/testcmd_skipped/partial_state_test.json deleted file mode 100644 index 77e71ca03c..0000000000 --- a/test/integration/testcmd_skipped/partial_state_test.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "a_no_post": { - "comment": "Carries one of the two fields a state test is recognised by, so the shape does not name it and nothing else does either.", - "transaction": { - "data": [ - "0x" - ], - "gasLimit": [ - "0x061a80" - ], - "gasPrice": "0x0a", - "nonce": "0x00", - "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value": [ - "0x0186a0" - ] - } - } -} diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index ced3eeeeed..979231ab7e 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -71,46 +71,38 @@ class Progress } }; -/// The fixture formats this tool runs. EEST names the format in each fixture's "_info", and a +/// What this tool makes of one fixture. EEST names the format in each fixture's "_info", and a /// heuristic covers the hand-written and pre-EEST files which have no "_info" at all. enum class Format { state_test, blockchain_test, + unsupported, ///< A fixture, in a format this tool does not run. + not_a_test, ///< Not a fixture at all. }; -/// What this tool makes of one fixture. -struct Classification -{ - /// How to run it, when this tool runs it. - std::optional format; - /// A fixture, whether or not this tool runs it. - bool is_fixture = false; - /// Why it is not run, when it is not. - std::string reason; -}; - -Classification classify(const json::json& fixture) +Format classify(const json::json& fixture) { if (const auto info = fixture.find("_info"); info != fixture.end()) { if (const auto format = info->find("fixture-format"); format != info->end()) { if (*format == "state_test") - return {.format = Format::state_test, .is_fixture = true}; + return Format::state_test; if (*format == "blockchain_test") - return {.format = Format::blockchain_test, .is_fixture = true}; - return {.is_fixture = true, .reason = "unsupported fixture format: " + format->dump()}; + return Format::blockchain_test; + return Format::unsupported; } } // Nothing declares the format: a hand-written or pre-EEST file, or an "_info" without one. - // Each shape is named by what only it carries; anything else is not a test at all, as EEST's - // shared pre-allocation kept beside the fixtures is not. - if (fixture.contains("blocks")) - return {.format = Format::blockchain_test, .is_fixture = true}; - if (fixture.contains("transaction") && fixture.contains("post")) - return {.format = Format::state_test, .is_fixture = true}; - return {.reason = "not a test"}; + // Each shape is named by the state it starts from and what is applied to it, never by what + // it expects, so a fixture whose expectations are missing is still a test and is run. + // Anything else is not a test at all, as EEST's shared pre-allocation is not. + if (fixture.contains("pre") && fixture.contains("blocks")) + return Format::blockchain_test; + if (fixture.contains("pre") && fixture.contains("transaction")) + return Format::state_test; + return Format::not_a_test; } } // namespace @@ -240,8 +232,8 @@ json::json load_fixture_file(const fs::path& path) auto contents = json::json::parse(f); // Not one fixture in it: EEST keeps its shared pre-allocation and an index of the fixtures // beside them, and neither is a test. - if (std::ranges::none_of( - contents.items(), [](const auto& item) { return classify(item.value()).is_fixture; })) + if (std::ranges::none_of(contents.items(), + [](const auto& i) { return classify(i.value()) != Format::not_a_test; })) throw UnsupportedTestFeature{"not a test"}; return contents; } @@ -290,14 +282,21 @@ void run_fixture(const std::string& name, const json::json& fixture, const TestS evmc::VM& vm, TestReport& report) { report.start_case(name); // Names whatever the load itself reports. - const auto [format, is_fixture, reason] = classify(fixture); - if (format == Format::state_test) + switch (classify(fixture)) + { + case Format::state_test: run_state_test(make_state_test(name, fixture), vm, settings.trace_summary, report); - else if (format == Format::blockchain_test) + break; + case Format::blockchain_test: run_blockchain_test(make_blockchain_test(name, fixture), vm, report); - else if (is_fixture) - throw UnsupportedTestFeature{reason}; // A format this tool does not run. - else - report.fail(reason); // The rest of the file holds fixtures, so this one is broken. + break; + case Format::unsupported: + throw UnsupportedTestFeature{ + "unsupported fixture format: " + fixture.at("_info").at("fixture-format").dump()}; + case Format::not_a_test: + // The rest of the file holds fixtures, so this one is broken. + report.fail("not a test"); + break; + } } } // namespace evmone::test From d398d3bec2bec09b27dffd8705bc7d7beb83b354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:24:31 +0200 Subject: [PATCH 05/11] test: Refuse a file which is not an object, and one which will not open items() walks an array by index, so a top-level JSON array was collected as fixtures named "0", "1", ... and run. A document which is not an object holds no fixture and is not a test. An ifstream which never opened reads as EOF, so every I/O failure was reported as a JSON syntax error in a file which has none, a 0-byte file and an unreadable one giving the same message. Check the stream. The help and the collector both still described a named file as one test per case in it, which a file holding no fixture at all has not been since it started being skipped whole. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 2 +- .../integration/testcmd_skipped/not_an_object.json | 7 +++++++ test/utils/test_collector.hpp | 5 +++-- test/utils/test_driver.cpp | 14 ++++++++++---- tools/evmone/main.cpp | 5 +++-- 5 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 test/integration/testcmd_skipped/not_an_object.json diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index b79f2f0b66..5ba1bd7922 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -147,7 +147,7 @@ DUP1,4 set_tests_properties( ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION - "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") + "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") # Nothing to run is not nothing to report: the run fails rather than passing empty. add_test(NAME ${PREFIX}/test_dir_skipped_exit COMMAND evmone-cli test ${DATA}/testcmd_skipped) diff --git a/test/integration/testcmd_skipped/not_an_object.json b/test/integration/testcmd_skipped/not_an_object.json new file mode 100644 index 0000000000..723839b091 --- /dev/null +++ b/test/integration/testcmd_skipped/not_an_object.json @@ -0,0 +1,7 @@ +[ + { + "comment": "A top-level array, so it holds no named fixture. Its one element would be taken for a blockchain test if the array were walked by index.", + "pre": {}, + "blocks": [] + } +] diff --git a/test/utils/test_collector.hpp b/test/utils/test_collector.hpp index 596b802fd3..2c193d57e3 100644 --- a/test/utils/test_collector.hpp +++ b/test/utils/test_collector.hpp @@ -34,8 +34,9 @@ struct TestFile void ignore_test_files( std::vector& files, std::span ignored); -/// Adds to @p cases every test under @p root: one per file for a directory, one per fixture in -/// the file when the file itself is named. Returns whether every test was collected. +/// Adds to @p cases every test under @p root: one per file for a directory, one per fixture for +/// a named file, or one for a named file holding no fixture. Returns whether every test was +/// collected. bool collect_tests(std::vector& cases, const std::filesystem::path& root, const TestSettings& settings, evmc::VM& vm); } // namespace evmone::test diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 979231ab7e..ecca6e1d2e 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -229,11 +229,17 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio json::json load_fixture_file(const fs::path& path) { std::ifstream f{path}; - auto contents = json::json::parse(f); + // A stream which never opened reads as EOF, which parses as a syntax error in a file which + // has none. + if (!f) + throw std::runtime_error{"cannot open the file"}; + const auto contents = json::json::parse(f); // Not one fixture in it: EEST keeps its shared pre-allocation and an index of the fixtures - // beside them, and neither is a test. - if (std::ranges::none_of(contents.items(), - [](const auto& i) { return classify(i.value()) != Format::not_a_test; })) + // beside them, and neither is a test. Nor is a document which is not an object at all: + // items() would walk an array by index, naming its elements "0", "1", ... + if (!contents.is_object() || std::ranges::none_of(contents.items(), [](const auto& i) { + return classify(i.value()) != Format::not_a_test; + })) throw UnsupportedTestFeature{"not a test"}; return contents; } diff --git a/tools/evmone/main.cpp b/tools/evmone/main.cpp index 9bd76d374a..3cd6cf0778 100644 --- a/tools/evmone/main.cpp +++ b/tools/evmone/main.cpp @@ -165,8 +165,9 @@ const CLI::App& setup_test_cmd(CLI::App& app, TestOptions& opts) { auto& cmd = *app.add_subcommand("test", "Run Ethereum tests")->fallthrough(); cmd.add_option("path", opts.paths, - "Test file or directory. Every .json file under a directory, except index.json, is a " - "test; naming a file makes each test case in it a test of its own.") + "Test file or directory. Under a directory every .json file except index.json is " + "one test. A named file gives one test per fixture, or one for the file when it holds " + "no fixture.") ->required() ->check(CLI::ExistingPath); cmd.add_option( From 413cea4f96fade9970b2a43a2fcfbee00afa29ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:29:40 +0200 Subject: [PATCH 06/11] test: Say what the pre-allocation fixture is for Its comment described being skipped one entry at a time, which stopped being how a file with no fixture in it is read, and pinned a file count from one release into a checked-in fixture. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/testcmd_skipped/pre_alloc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/testcmd_skipped/pre_alloc.json b/test/integration/testcmd_skipped/pre_alloc.json index 355c895850..015cd24e1d 100644 --- a/test/integration/testcmd_skipped/pre_alloc.json +++ b/test/integration/testcmd_skipped/pre_alloc.json @@ -23,5 +23,5 @@ "storage": {} } }, - "comment": "EEST keeps the pre-allocation its fixtures share beside them, 35715 files of it in one release, and none of it is a test. Several entries, so the reason the file is skipped for is recorded once rather than per entry." + "comment": "EEST keeps the pre-allocation its fixtures share beside them, and none of it is a test. Several entries, none of which names a format or carries a shape." } From 309534940537641122d59fcc2edbaef711e40c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:30:20 +0200 Subject: [PATCH 07/11] test: Report a fixture's exception as every other failure is The per-fixture catch built one string where the two-argument fail() splits the check from its detail, and only the detail is indented line by line, so a what() of several lines broke the report's columns. It also caught nothing but std::exception, where the run loop around it catches everything. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/utils/test_driver.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index ecca6e1d2e..9599760339 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -270,8 +270,13 @@ void run_fixture_file( catch (const std::exception& ex) { // The fixture is a test and it went wrong, which is this file's verdict but not the - // end of it: the fixtures after it are still worth running. - report.fail(concat("exception: ", ex.what())); + // end of it: the fixtures after it are still worth running. Reported as run_tests + // reports one, so a what() of several lines keeps the indent of the rest. + report.fail("exception", ex.what()); + } + catch (...) + { + report.fail("exception", "not derived from std::exception"); } any_ran = true; } From c9252a9125e60efb75b8ebf9010ac10c185d303a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:31:40 +0200 Subject: [PATCH 08/11] test: Keep the histogram off the progress row --trace turned the progress row into a line per test so its output could be told apart, but --histogram wrote its table into the row mid-line. Both write per test, so both decide the same way. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 11 +++++++++-- tools/evmone/main.cpp | 9 +++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 5ba1bd7922..bd562cd457 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -175,8 +175,15 @@ DUP1,4 ${PREFIX}/test_file_declined PROPERTIES PASS_REGULAR_EXPRESSION "SKIPPED[^\n]*c_engine - unsupported fixture format.*= 1 passed, 2 skipped in") - # --trace reaches the runner through the app-level flag, and turns the progress row into a - # line naming each test so the trace which follows it can be told apart. + # --trace and --histogram reach the runner through the app-level flags, and turn the progress + # row into a line naming each test so what each writes can be told apart. + add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram + ${DATA}/statetest/tests1/SuiteA/test1.json) + set_tests_properties( + ${PREFIX}/test_histogram PROPERTIES PASS_REGULAR_EXPRESSION + "test1\\.json::test1 +--- # HISTOGRAM.*= 1 passed in") + add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace ${DATA}/statetest/tests1/SuiteA/test1.json) set_tests_properties( diff --git a/tools/evmone/main.cpp b/tools/evmone/main.cpp index 3cd6cf0778..2dfafc0fe9 100644 --- a/tools/evmone/main.cpp +++ b/tools/evmone/main.cpp @@ -185,14 +185,15 @@ const CLI::App& setup_test_cmd(CLI::App& app, TestOptions& opts) return cmd; } -int exec_test_cmd(evmc::VM& vm, TestOptions opts, bool trace) +int exec_test_cmd(evmc::VM& vm, TestOptions opts, bool trace, bool histogram) { // main() has switched the tracer on already. Its line per instruction is worth - // unsynchronising the streams for, and would run into the progress row, as a summary would. + // unsynchronising the streams for, and anything it writes per test would run into the + // progress row, as a summary would. if (trace) std::ios::sync_with_stdio(false); opts.settings.trace_summary |= trace; - opts.run.progress = !opts.settings.trace_summary; + opts.run.progress = !(opts.settings.trace_summary || histogram); std::vector cases; bool all_collected = true; @@ -275,7 +276,7 @@ int main(int argc, const char* const* argv) noexcept return exec_t8n_cmd(vm, t8n_opts); if (test_cmd) - return exec_test_cmd(vm, test_opts, trace); + return exec_test_cmd(vm, test_opts, trace, histogram); return 0; } From a1f11e14f43435d4e490ffc2dbde37d553f5bcce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:32:10 +0200 Subject: [PATCH 09/11] test: Say that a collected test outlives the call which made it The tests hold the settings and the VM by reference, which only the caller can honour, and dropping the return value turned a tree which could not be collected from a failure into a silent success. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/utils/test_collector.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/utils/test_collector.hpp b/test/utils/test_collector.hpp index 2c193d57e3..72abe1c132 100644 --- a/test/utils/test_collector.hpp +++ b/test/utils/test_collector.hpp @@ -36,7 +36,7 @@ void ignore_test_files( /// Adds to @p cases every test under @p root: one per file for a directory, one per fixture for /// a named file, or one for a named file holding no fixture. Returns whether every test was -/// collected. -bool collect_tests(std::vector& cases, const std::filesystem::path& root, +/// collected. The tests hold @p settings and @p vm by reference, so both must outlive them. +[[nodiscard]] bool collect_tests(std::vector& cases, const std::filesystem::path& root, const TestSettings& settings, evmc::VM& vm); } // namespace evmone::test From abcda18c0ada379e0d3d7c156804128c349fcaba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:33:48 +0200 Subject: [PATCH 10/11] test: Share one document between the tests of a named file Each test captured the fixture it was made from, and the capture copies the subtree it names, so a file's whole document was held once more over again, and a listing paid for every copy to read nothing but the name. Hold the document once and index it when the test runs: peak memory on a 3MB file drops by about a quarter. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/utils/test_collector.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index 53d5df257a..36f3e25cc5 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -5,6 +5,7 @@ #include "test_collector.hpp" #include #include +#include #include namespace evmone::test @@ -98,13 +99,16 @@ bool collect_tests( return false; } - for (const auto& [name, fixture] : file.items()) + // One document shared by every test of it. Capturing the fixture itself would copy its + // subtree into each test, which a listing pays for in full to read nothing but the name. + const auto doc = std::make_shared(std::move(file)); + for (const auto& [name, fixture] : doc->items()) { if (!settings.selects(name)) continue; cases.push_back( - {root.string() + "::" + name, [name, fixture, &settings, &vm](TestReport& report) { - run_fixture(name, fixture, settings, vm, report); + {root.string() + "::" + name, [doc, name = name, &settings, &vm](TestReport& report) { + run_fixture(name, doc->at(name), settings, vm, report); }}); } return true; From 5fa8e79e161f3d6e77bb6597d1b60d8484259013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:36:49 +0200 Subject: [PATCH 11/11] test: Name the fixtures a file declined but ran beside Collected from a directory a whole file is one test, so a file which ran anything reported nothing about the fixtures it declined: two of three were dropped from a run which said "1 passed" and exited 0. The day a fixture stops being one this tool runs, its coverage would go with it and the count would not move. Name each on the error stream, which stays empty on a release whose files are each of one format. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 4 +++- test/utils/test_driver.cpp | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index bd562cd457..8abcba389e 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -102,9 +102,11 @@ DUP1,4 # Over a directory a test is a whole file, so a case the loader refuses must not take the # rest of the file with it: the file passes on the case beside it, the refused one first. + # The file's own verdict says nothing about what it declined, so each declined case is named. add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) set_tests_properties( - ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") + ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION + "a_bad_rlp: tests with invalidly rlp-encoded blocks.*c_engine: unsupported fixture format.*= 1 passed in") # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these two assert on nothing # else: a sound fixture directory succeeds, and a fault in one fails. diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 9599760339..6753451f34 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -250,7 +250,7 @@ void run_fixture_file( // Named, because items() only borrows: iterating a temporary dangles. const auto contents = load_fixture_file(path); - std::optional declined; // The reason for the first fixture this tool declined. + std::vector> declined; // Fixture name and reason. bool any_ran = false; for (const auto& [name, fixture] : contents.items()) { @@ -263,8 +263,7 @@ void run_fixture_file( catch (const UnsupportedTestFeature& ex) { // This tool's own limit: a format it does not run, or a fixture its loader refuses. - if (!declined) - declined = ex.what(); + declined.emplace_back(name, ex.what()); continue; } catch (const std::exception& ex) @@ -285,8 +284,13 @@ void run_fixture_file( // TODO: A file whose cases -k all deselected still passes, as it did before this command // existed, so a filter which matches nothing turns a failing tree green. Skip it instead, // and an empty selection reaches NOTHING_VERIFIED on its own. - if (!any_ran && declined) - throw UnsupportedTestFeature{*declined}; + if (!any_ran && !declined.empty()) + throw UnsupportedTestFeature{declined.front().second}; + + // The file ran, so its own verdict says nothing about what it declined. Name those, or a + // fixture this tool stops running disappears from a passing run rather than being missed. + for (const auto& [name, reason] : declined) + std::cerr << path.string() << "::" << name << ": " << reason << '\n'; } void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings,