diff --git a/test/blockchaintest/blockchaintest.cpp b/test/blockchaintest/blockchaintest.cpp index 8026c70ba8..f409b5d017 100644 --- a/test/blockchaintest/blockchaintest.cpp +++ b/test/blockchaintest/blockchaintest.cpp @@ -15,6 +15,13 @@ using evmone::test::TestCase; namespace { +/// A test which is one case: the driver takes an array of results, and this is the one. +TestCase one_case(std::string name, std::function run) +{ + return {name, + [name, run = std::move(run)] { return std::vector{evmone::test::run_one(name, run)}; }}; +} + /// Adds to @p cases every test under @p root: one per file for a directory, one per test case in /// the file when the file itself is named. Returns whether every test was collected. bool collect_tests(std::vector& cases, const fs::path& root, @@ -28,12 +35,28 @@ bool collect_tests(std::vector& cases, const fs::path& root, for (const auto& file : files) { // Loaded when the test runs: loading a whole tree up front costs far more. A - // load which throws over an unsupported fixture reaches the driver, which skips. + // load which throws over an unsupported fixture is the file's one result, which + // the driver reads as a skip. cases.push_back( - {file.path.string(), [path = file.path, &vm](evmone::test::TestReport& report) { - std::ifstream f{path}; - for (const auto& test : evmone::test::load_blockchain_tests(f)) - evmone::test::run_blockchain_test(test, vm, report); + {file.path.string(), [path = file.path, &vm] { + std::vector tests; + if (auto loaded = evmone::test::run_one(path.string(), + [&](auto&) { + std::ifstream f{path}; + tests = evmone::test::load_blockchain_tests(f); + }); + loaded.outcome != evmone::test::Outcome::passed) + return std::vector{std::move(loaded)}; + + std::vector results; + for (const auto& test : tests) + { + results.push_back(evmone::test::run_one(path.string() + "::" + test.name, + [&](evmone::test::TestReport& report) { + evmone::test::run_blockchain_test(test, vm, report); + })); + } + return results; }}); } } @@ -50,25 +73,25 @@ bool collect_tests(std::vector& cases, const fs::path& root, catch (const evmone::test::UnsupportedTestFeature&) { // An unsupported fixture is a skip, not a broken collection. - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + cases.push_back(one_case(root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); })); return true; } catch (const std::exception& ex) { // Also reported here: --collect-only never runs the test. std::cerr << root.string() << ": " << ex.what() << '\n'; - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + cases.push_back(one_case(root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); })); return false; } for (const auto& test : tests) { - cases.push_back( - {root.string() + "::" + test.name, [test, &vm](evmone::test::TestReport& report) { - evmone::test::run_blockchain_test(test, vm, report); - }}); + cases.push_back(one_case( + root.string() + "::" + test.name, [test, &vm](evmone::test::TestReport& report) { + evmone::test::run_blockchain_test(test, vm, report); + })); } } return true; diff --git a/test/integration/evmone-cli/test/statetest/CMakeLists.txt b/test/integration/evmone-cli/test/statetest/CMakeLists.txt index b5bd5fec53..82db1c10f6 100644 --- a/test/integration/evmone-cli/test/statetest/CMakeLists.txt +++ b/test/integration/evmone-cli/test/statetest/CMakeLists.txt @@ -70,6 +70,18 @@ set_tests_properties( PASS_REGULAR_EXPRESSION "test_case_1.*test_case_2" ) +# A file collected from a directory reports each of its fixtures, where the file alone would say +# only that something in it failed. +add_test( + NAME ${PREFIX}/directory_names_each_case + COMMAND evmone-statetest ${TESTS1}/SuiteA +) +set_tests_properties( + ${PREFIX}/directory_names_each_case PROPERTIES + PASS_REGULAR_EXPRESSION + "test2_multi\\.json::test_case_1 - state root\n[^\n]*test2_multi\\.json::test_case_2 - state root" +) + add_test( NAME ${PREFIX}/trace COMMAND evmone-statetest ${TESTS1}/SuiteA/test1.json --trace diff --git a/test/statetest/statetest.cpp b/test/statetest/statetest.cpp index 56a747e698..9a799e8fd6 100644 --- a/test/statetest/statetest.cpp +++ b/test/statetest/statetest.cpp @@ -15,6 +15,13 @@ using evmone::test::TestCase; namespace { +/// A test which is one case: the driver takes an array of results, and this is the one. +TestCase one_case(std::string name, std::function run) +{ + return {name, + [name, run = std::move(run)] { return std::vector{evmone::test::run_one(name, run)}; }}; +} + /// Adds to @p cases every test under @p root: one per file for a directory, one per test case in /// the file when the file itself is named. Returns whether every test was collected. bool collect_tests(std::vector& cases, const fs::path& root, @@ -35,15 +42,31 @@ bool collect_tests(std::vector& cases, const fs::path& root, for (const auto& file : files) { // Loaded when the test runs: loading a whole tree up front costs far more. - cases.push_back({file.path.string(), - [path = file.path, selected, &vm, trace](evmone::test::TestReport& report) { - std::ifstream f{path}; - for (const auto& test : evmone::test::load_state_tests(f)) - { - if (selected(test)) - evmone::test::run_state_test(test, vm, trace, report); - } - }}); + cases.push_back( + {file.path.string(), [path = file.path, selected, &vm, trace] { + std::vector tests; + // The whole file is loaded before any of it runs, so a load + // which throws is all the file has to report. + if (auto loaded = evmone::test::run_one(path.string(), + [&](auto&) { + std::ifstream f{path}; + tests = evmone::test::load_state_tests(f); + }); + loaded.outcome != evmone::test::Outcome::passed) + return std::vector{std::move(loaded)}; + + std::vector results; + for (const auto& test : tests) + { + if (!selected(test)) + continue; + results.push_back(evmone::test::run_one(path.string() + "::" + test.name, + [&](evmone::test::TestReport& report) { + evmone::test::run_state_test(test, vm, trace, report); + })); + } + return results; + }}); } } else // Treat as a file. @@ -60,8 +83,8 @@ bool collect_tests(std::vector& cases, const fs::path& root, { // Also reported here: --collect-only never runs the test. std::cerr << root.string() << ": " << ex.what() << '\n'; - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + cases.push_back(one_case(root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); })); return false; } @@ -69,10 +92,10 @@ bool collect_tests(std::vector& cases, const fs::path& root, { if (!selected(test)) continue; - cases.push_back({root.string() + "::" + test.name, + cases.push_back(one_case(root.string() + "::" + test.name, [test, &vm, trace](evmone::test::TestReport& report) { evmone::test::run_state_test(test, vm, trace, report); - }}); + })); } } return true; diff --git a/test/unittests/test_driver_test.cpp b/test/unittests/test_driver_test.cpp index d2208aebd9..c6e016dd29 100644 --- a/test/unittests/test_driver_test.cpp +++ b/test/unittests/test_driver_test.cpp @@ -22,6 +22,21 @@ Run run(std::span cases, const RunOptions& options = {}) const auto exit_code = run_tests(cases, out, options); return {exit_code, std::move(out).str()}; } + +/// A test holding one case, which is what a fixture file with one fixture in it comes to. +TestCase one(std::string name, std::function run) +{ + return {name, [name, run = std::move(run)] { return std::vector{run_one(name, run)}; }}; +} + +/// A test holding cases which only report the outcome given, running nothing. +TestCase holding(std::string name, std::initializer_list outcomes) +{ + std::vector results; + for (const auto outcome : outcomes) + results.push_back({name + "::case", outcome, "the reason", {}}); + return {std::move(name), [results = std::move(results)] { return results; }}; +} } // namespace TEST(test_driver, nothing_collected) @@ -35,7 +50,7 @@ TEST(test_driver, nothing_collected) TEST(test_driver, collect_only_lists_without_running) { bool ran = false; - const std::vector cases{{"a name", [&ran](TestReport&) { ran = true; }}}; + const std::vector cases{one("a name", [&ran](TestReport&) { ran = true; })}; const auto [exit_code, output] = run(cases, {.collect_only = true}); EXPECT_FALSE(ran); @@ -54,10 +69,10 @@ TEST(test_driver, exception_fails_only_its_own_test) { bool last_ran = false; const std::vector cases{ - {"ok", [](TestReport&) {}}, - {"throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }}, - {"unknown", [](TestReport&) { throw 42; }}, // NOLINT(hicpp-exception-baseclass) - {"last", [&last_ran](TestReport&) { last_ran = true; }}, + one("ok", [](TestReport&) {}), + one("throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }), + one("unknown", [](TestReport&) { throw 42; }), // NOLINT(hicpp-exception-baseclass) + one("last", [&last_ran](TestReport&) { last_ran = true; }), }; const auto [exit_code, output] = run(cases); @@ -71,8 +86,8 @@ TEST(test_driver, exception_fails_only_its_own_test) TEST(test_driver, unsupported_feature_skips) { const std::vector cases{ - {"ok", [](TestReport&) {}}, - {"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }}, + one("ok", [](TestReport&) {}), + one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }), }; const auto [exit_code, output] = run(cases); @@ -84,7 +99,7 @@ TEST(test_driver, unsupported_feature_skips) TEST(test_driver, everything_skipped_verifies_nothing) { const std::vector cases{ - {"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }}}; + one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, NOTHING_VERIFIED); @@ -94,7 +109,7 @@ TEST(test_driver, everything_skipped_verifies_nothing) TEST(test_driver, summary_names_the_check_which_failed) { const std::vector cases{ - {"mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); }}}; + one("mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, 1); @@ -103,10 +118,10 @@ TEST(test_driver, summary_names_the_check_which_failed) TEST(test_driver, failure_outranks_a_later_exception) { - const std::vector cases{{"both", [](TestReport& report) { - report.check_eq("a value", 1, 2); - throw std::runtime_error{"gave up afterwards"}; - }}}; + const std::vector cases{one("both", [](TestReport& report) { + report.check_eq("a value", 1, 2); + throw std::runtime_error{"gave up afterwards"}; + })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, TESTS_FAILED); @@ -116,10 +131,10 @@ TEST(test_driver, failure_outranks_a_later_exception) TEST(test_driver, failure_outranks_a_later_skip) { - const std::vector cases{{"both", [](TestReport& report) { - report.check_eq("a value", 1, 2); - throw UnsupportedTestFeature{"gave up afterwards"}; - }}}; + const std::vector cases{one("both", [](TestReport& report) { + report.check_eq("a value", 1, 2); + throw UnsupportedTestFeature{"gave up afterwards"}; + })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, 1); @@ -127,3 +142,44 @@ TEST(test_driver, failure_outranks_a_later_skip) // The summary names the check which failed, not what the test then gave up on. EXPECT_NE(output.find("FAILED both - a value"), std::string::npos); } + +TEST(test_driver, a_file_counts_once_however_many_fixtures_it_holds) +{ + const std::vector cases{ + holding("a file", {Outcome::passed, Outcome::passed, Outcome::passed})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, SUCCESS); + EXPECT_NE(output.find("collected 1 test"), std::string::npos); + EXPECT_NE(output.find("1 passed"), std::string::npos); +} + +TEST(test_driver, a_declined_fixture_is_named_though_its_file_passed) +{ + const std::vector cases{holding("a file", {Outcome::passed, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, SUCCESS); + // The file's own verdict says nothing about what it declined, so the fixture is named. + EXPECT_NE(output.find("1 passed in"), std::string::npos); + EXPECT_NE(output.find("SKIPPED a file::case - the reason"), std::string::npos); +} + +TEST(test_driver, one_failed_fixture_fails_its_file) +{ + const std::vector cases{ + holding("a file", {Outcome::passed, Outcome::failed, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, TESTS_FAILED); + EXPECT_NE(output.find("1 failed, 0 passed"), std::string::npos); +} + +TEST(test_driver, a_file_is_skipped_only_when_nothing_in_it_ran) +{ + const std::vector cases{holding("a file", {Outcome::skipped, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, NOTHING_VERIFIED); + EXPECT_NE(output.find("0 passed, 1 skipped"), std::string::npos); +} diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 5ad47d8382..b63a73a0e1 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -16,14 +16,6 @@ namespace constexpr int LINE_WIDTH = 72; constexpr int PROGRESS_WIDTH = 60; -/// The outcome of one test, spelled as the progress character for it. -enum class Outcome : char -{ - passed = '.', - failed = 'F', - skipped = 's', -}; - void banner(std::ostream& out, std::string_view title, char fill = '=') { const auto padding = LINE_WIDTH - static_cast(title.size()) - 2; @@ -32,13 +24,12 @@ void banner(std::ostream& out, std::string_view title, char fill = '=') << std::string(static_cast(std::max(padding - left, 1)), fill) << '\n'; } -/// A test which did not pass: what the summary says about it and what it recorded. +/// A file with something to report: how it counts, and every fixture of it which did not pass. struct Note { - Outcome outcome; std::string name; - std::string reason; - std::vector failures; + Outcome outcome; + std::vector results; }; /// One progress character per test, wrapped, each line ending in the percentage done. @@ -67,6 +58,46 @@ class Progress }; } // namespace +Result run_one(std::string name, const std::function& run) +{ + Result result{.name = std::move(name)}; + TestReport report{[&result](const Failure& failure) { result.failures.push_back(failure); }}; + report.start_case(result.name); // Names whatever the run itself reports. + + std::string exception_reason; + try + { + run(report); + } + catch (const UnsupportedTestFeature& ex) + { + result.outcome = Outcome::skipped; + result.reason = ex.what(); + } + catch (const std::exception& ex) + { + // One unloadable fixture in a tree of thousands fails its own test, not the run. + report.fail("exception", ex.what()); + exception_reason = concat("exception: ", ex.what()); + } + catch (...) + { + report.fail("exception", "not derived from std::exception"); + exception_reason = "exception not derived from std::exception"; + } + + // A recorded failure outranks giving up afterwards, in the summary too: the exception is + // the reason only when nothing failed before it threw. + if (!result.failures.empty()) + { + result.outcome = Outcome::failed; + result.reason = result.failures.size() == 1 && !exception_reason.empty() ? + std::move(exception_reason) : + result.failures.front().what; + } + return result; +} + int run_tests(std::span cases, std::ostream& out, const RunOptions& options) { if (options.collect_only) @@ -83,64 +114,55 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio std::vector notes; Progress row{out, cases.size()}; + size_t failed = 0; + size_t skipped = 0; + size_t passed = 0; for (const auto& test : cases) { - // Held until the run ends, as pytest holds them, so nothing interleaves. - std::vector failures; - TestReport report{[&failures](const Failure& failure) { failures.push_back(failure); }}; - report.start_case(test.name); - - auto outcome = Outcome::passed; - std::string reason; - std::string exception_reason; if (!options.progress) out << test.name << '\n'; // The only thing naming what the test prints next. out << std::flush; + + // Held until the run ends, as pytest holds them, so nothing interleaves. + std::vector results; try { - test.run(report); - } - catch (const UnsupportedTestFeature& ex) - { - outcome = Outcome::skipped; - reason = ex.what(); - } - catch (const std::exception& ex) - { - // One unloadable fixture in a tree of thousands fails its own test, not the run. - report.fail("exception", ex.what()); - exception_reason = concat("exception: ", ex.what()); + results = test.run(); } catch (...) { - report.fail("exception", "not derived from std::exception"); - exception_reason = "exception not derived from std::exception"; + // A test which throws rather than reporting is the one result which says so. + const auto error = std::current_exception(); + results.push_back( + run_one(test.name, [&error](TestReport&) { std::rethrow_exception(error); })); } // A test writes its own output, an EVM trace above all, to another stream. std::clog << std::flush; - // A recorded failure outranks giving up afterwards, in the summary too: the exception - // is the reason only when nothing failed before it threw. - if (!failures.empty()) - { + // The file counts once, for the worst its fixtures reached. It is skipped only when + // nothing in it ran at all, so one fixture running is enough to give it a verdict. + static constexpr auto is = [](Outcome outcome) { + return [outcome](const Result& result) { return result.outcome == outcome; }; + }; + auto outcome = Outcome::passed; + if (std::ranges::any_of(results, is(Outcome::failed))) outcome = Outcome::failed; - reason = failures.size() == 1 && !exception_reason.empty() ? - std::move(exception_reason) : - failures.front().what; - } - if (outcome != Outcome::passed) - notes.push_back({outcome, test.name, std::move(reason), std::move(failures)}); + else if (!results.empty() && std::ranges::none_of(results, is(Outcome::passed))) + outcome = Outcome::skipped; + + ++(outcome == Outcome::failed ? failed : outcome == Outcome::skipped ? skipped : passed); + + // Every fixture which did not pass is named, including one declined by a file which + // passed on the fixtures beside it. Otherwise it would vanish from a green run. + std::erase_if(results, is(Outcome::passed)); + if (!results.empty()) + notes.push_back({test.name, outcome, std::move(results)}); if (options.progress) row.advance(outcome); } - // Every test which did not pass left exactly one note, so the counts follow from them. - const auto failed = std::ranges::count(notes, Outcome::failed, &Note::outcome); - const auto skipped = std::ranges::count(notes, Outcome::skipped, &Note::outcome); - const auto passed = cases.size() - notes.size(); - if (failed != 0) { out << '\n'; @@ -150,8 +172,11 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio if (note.outcome != Outcome::failed) continue; banner(out, note.name, '_'); - for (const auto& failure : note.failures) - out << failure << '\n'; + for (const auto& result : note.results) + { + for (const auto& failure : result.failures) + out << failure << '\n'; + } } } @@ -161,10 +186,13 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio banner(out, "short test summary info"); for (const auto& note : notes) { - out << (note.outcome == Outcome::failed ? "FAILED " : "SKIPPED ") << note.name; - if (!note.reason.empty()) - out << " - " << note.reason; - out << '\n'; + for (const auto& result : note.results) + { + out << (result.outcome == Outcome::failed ? "FAILED " : "SKIPPED ") << result.name; + if (!result.reason.empty()) + out << " - " << result.reason; + out << '\n'; + } } } diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index 2694b48680..be525520ef 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -17,15 +17,42 @@ constexpr int TESTS_FAILED = 1; /// value for the first of those; a test skipped has verified no more than a missing one. constexpr int NOTHING_VERIFIED = 5; +/// How one fixture ended, spelled as the character the progress row marks it with. +enum class Outcome : char +{ + passed = '.', + failed = 'F', + skipped = 's', +}; + +/// What running one fixture produced. +struct Result +{ + /// The fixture, as "::", or the file alone when it never got as far as one. + std::string name; + + Outcome outcome = Outcome::passed; + + /// Why it did not pass. Empty when it did. + std::string reason; + + std::vector failures; +}; + /// A single test: its name and how to run it. struct TestCase { std::string name; - /// Executes the test, recording what did not hold in the report. - std::function run; + /// Executes the test, returning what each of its fixtures produced. A test which never got + /// as far as a fixture returns the one result which says so, rather than throwing. + std::function()> run; }; +/// Runs @p run under a report of its own and says what it produced. What the run recorded +/// outranks how it ended: an exception is the reason only when nothing failed before it threw. +[[nodiscard]] Result run_one(std::string name, const std::function& run); + /// How to run and what to report. struct RunOptions {