Skip to content

test: Report a file's fixtures one result each #1700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
chfast wants to merge 1 commit into from
Closed

test: Report a file's fixtures one result each #1700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 36 additions & 13 deletions test/blockchaintest/blockchaintest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ using evmone::test::TestCase;

namespace
{
/// A test which is one case: the driver takes an array of results, and this is the one.
TestCase one_case(std::string name, std::function<void(evmone::test::TestReport&)> run)
{
return {name,
[name, run = std::move(run)] { return std::vector{evmone::test::run_one(name, run)}; }};
}

/// Adds to @p cases every test under @p root: one per file for a directory, one per test case in
/// the file when the file itself is named. Returns whether every test was collected.
bool collect_tests(std::vector<TestCase>& cases, const fs::path& root,
Expand All @@ -28,12 +35,28 @@ bool collect_tests(std::vector<TestCase>& cases, const fs::path& root,
for (const auto& file : files)
{
// Loaded when the test runs: loading a whole tree up front costs far more. A
// load which throws over an unsupported fixture reaches the driver, which skips.
// load which throws over an unsupported fixture is the file's one result, which
// the driver reads as a skip.
cases.push_back(
{file.path.string(), [path = file.path, &vm](evmone::test::TestReport& report) {
std::ifstream f{path};
for (const auto& test : evmone::test::load_blockchain_tests(f))
evmone::test::run_blockchain_test(test, vm, report);
{file.path.string(), [path = file.path, &vm] {
std::vector<evmone::test::BlockchainTest> tests;
if (auto loaded = evmone::test::run_one(path.string(),
[&](auto&) {
std::ifstream f{path};
tests = evmone::test::load_blockchain_tests(f);
});
loaded.outcome != evmone::test::Outcome::passed)
return std::vector{std::move(loaded)};

std::vector<evmone::test::Result> results;
for (const auto& test : tests)
{
results.push_back(evmone::test::run_one(path.string() + "::" + test.name,
[&](evmone::test::TestReport& report) {
evmone::test::run_blockchain_test(test, vm, report);
}));
}
return results;
}});
}
}
Expand All @@ -50,25 +73,25 @@ bool collect_tests(std::vector<TestCase>& cases, const fs::path& root,
catch (const evmone::test::UnsupportedTestFeature&)
{
// An unsupported fixture is a skip, not a broken collection.
cases.push_back({root.string(),
[error = std::current_exception()](auto&) { std::rethrow_exception(error); }});
cases.push_back(one_case(root.string(),
[error = std::current_exception()](auto&) { std::rethrow_exception(error); }));
return true;
}
catch (const std::exception& ex)
{
// Also reported here: --collect-only never runs the test.
std::cerr << root.string() << ": " << ex.what() << '\n';
cases.push_back({root.string(),
[error = std::current_exception()](auto&) { std::rethrow_exception(error); }});
cases.push_back(one_case(root.string(),
[error = std::current_exception()](auto&) { std::rethrow_exception(error); }));
return false;
}

for (const auto& test : tests)
{
cases.push_back(
{root.string() + "::" + test.name, [test, &vm](evmone::test::TestReport& report) {
evmone::test::run_blockchain_test(test, vm, report);
}});
cases.push_back(one_case(
root.string() + "::" + test.name, [test, &vm](evmone::test::TestReport& report) {
evmone::test::run_blockchain_test(test, vm, report);
}));
}
}
return true;
Expand Down
12 changes: 12 additions & 0 deletions test/integration/evmone-cli/test/statetest/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,18 @@ set_tests_properties(
PASS_REGULAR_EXPRESSION "test_case_1.*test_case_2"
)

# A file collected from a directory reports each of its fixtures, where the file alone would say
# only that something in it failed.
add_test(
NAME ${PREFIX}/directory_names_each_case
COMMAND evmone-statetest ${TESTS1}/SuiteA
)
set_tests_properties(
${PREFIX}/directory_names_each_case PROPERTIES
PASS_REGULAR_EXPRESSION
"test2_multi\\.json::test_case_1 - state root\n[^\n]*test2_multi\\.json::test_case_2 - state root"
)

add_test(
NAME ${PREFIX}/trace
COMMAND evmone-statetest ${TESTS1}/SuiteA/test1.json --trace
Expand Down
49 changes: 36 additions & 13 deletions test/statetest/statetest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ using evmone::test::TestCase;

namespace
{
/// A test which is one case: the driver takes an array of results, and this is the one.
TestCase one_case(std::string name, std::function<void(evmone::test::TestReport&)> run)
{
return {name,
[name, run = std::move(run)] { return std::vector{evmone::test::run_one(name, run)}; }};
}

/// Adds to @p cases every test under @p root: one per file for a directory, one per test case in
/// the file when the file itself is named. Returns whether every test was collected.
bool collect_tests(std::vector<TestCase>& cases, const fs::path& root,
Expand All @@ -35,15 +42,31 @@ bool collect_tests(std::vector<TestCase>& cases, const fs::path& root,
for (const auto& file : files)
{
// Loaded when the test runs: loading a whole tree up front costs far more.
cases.push_back({file.path.string(),
[path = file.path, selected, &vm, trace](evmone::test::TestReport& report) {
std::ifstream f{path};
for (const auto& test : evmone::test::load_state_tests(f))
{
if (selected(test))
evmone::test::run_state_test(test, vm, trace, report);
}
}});
cases.push_back(
{file.path.string(), [path = file.path, selected, &vm, trace] {
std::vector<evmone::test::StateTransitionTest> tests;
// The whole file is loaded before any of it runs, so a load
// which throws is all the file has to report.
if (auto loaded = evmone::test::run_one(path.string(),
[&](auto&) {
std::ifstream f{path};
tests = evmone::test::load_state_tests(f);
});
loaded.outcome != evmone::test::Outcome::passed)
return std::vector{std::move(loaded)};

std::vector<evmone::test::Result> results;
for (const auto& test : tests)
{
if (!selected(test))
continue;
results.push_back(evmone::test::run_one(path.string() + "::" + test.name,
[&](evmone::test::TestReport& report) {
evmone::test::run_state_test(test, vm, trace, report);
}));
}
return results;
}});
}
}
else // Treat as a file.
Expand All @@ -60,19 +83,19 @@ bool collect_tests(std::vector<TestCase>& cases, const fs::path& root,
{
// Also reported here: --collect-only never runs the test.
std::cerr << root.string() << ": " << ex.what() << '\n';
cases.push_back({root.string(),
[error = std::current_exception()](auto&) { std::rethrow_exception(error); }});
cases.push_back(one_case(root.string(),
[error = std::current_exception()](auto&) { std::rethrow_exception(error); }));
return false;
}

for (const auto& test : tests)
{
if (!selected(test))
continue;
cases.push_back({root.string() + "::" + test.name,
cases.push_back(one_case(root.string() + "::" + test.name,
[test, &vm, trace](evmone::test::TestReport& report) {
evmone::test::run_state_test(test, vm, trace, report);
}});
}));
}
}
return true;
Expand Down
90 changes: 73 additions & 17 deletions test/unittests/test_driver_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,21 @@ Run run(std::span<const TestCase> cases, const RunOptions& options = {})
const auto exit_code = run_tests(cases, out, options);
return {exit_code, std::move(out).str()};
}

/// A test holding one case, which is what a fixture file with one fixture in it comes to.
TestCase one(std::string name, std::function<void(TestReport&)> run)
{
return {name, [name, run = std::move(run)] { return std::vector{run_one(name, run)}; }};
}

/// A test holding cases which only report the outcome given, running nothing.
TestCase holding(std::string name, std::initializer_list<Outcome> outcomes)
{
std::vector<Result> results;
for (const auto outcome : outcomes)
results.push_back({name + "::case", outcome, "the reason", {}});
return {std::move(name), [results = std::move(results)] { return results; }};
}
} // namespace

TEST(test_driver, nothing_collected)
Expand All @@ -35,7 +50,7 @@ TEST(test_driver, nothing_collected)
TEST(test_driver, collect_only_lists_without_running)
{
bool ran = false;
const std::vector<TestCase> cases{{"a name", [&ran](TestReport&) { ran = true; }}};
const std::vector<TestCase> cases{one("a name", [&ran](TestReport&) { ran = true; })};

const auto [exit_code, output] = run(cases, {.collect_only = true});
EXPECT_FALSE(ran);
Expand All @@ -54,10 +69,10 @@ TEST(test_driver, exception_fails_only_its_own_test)
{
bool last_ran = false;
const std::vector<TestCase> cases{
{"ok", [](TestReport&) {}},
{"throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }},
{"unknown", [](TestReport&) { throw 42; }}, // NOLINT(hicpp-exception-baseclass)
{"last", [&last_ran](TestReport&) { last_ran = true; }},
one("ok", [](TestReport&) {}),
one("throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }),
one("unknown", [](TestReport&) { throw 42; }), // NOLINT(hicpp-exception-baseclass)
one("last", [&last_ran](TestReport&) { last_ran = true; }),
};

const auto [exit_code, output] = run(cases);
Expand All @@ -71,8 +86,8 @@ TEST(test_driver, exception_fails_only_its_own_test)
TEST(test_driver, unsupported_feature_skips)
{
const std::vector<TestCase> cases{
{"ok", [](TestReport&) {}},
{"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }},
one("ok", [](TestReport&) {}),
one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }),
};

const auto [exit_code, output] = run(cases);
Expand All @@ -84,7 +99,7 @@ TEST(test_driver, unsupported_feature_skips)
TEST(test_driver, everything_skipped_verifies_nothing)
{
const std::vector<TestCase> cases{
{"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }}};
one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; })};

const auto [exit_code, output] = run(cases);
EXPECT_EQ(exit_code, NOTHING_VERIFIED);
Expand All @@ -94,7 +109,7 @@ TEST(test_driver, everything_skipped_verifies_nothing)
TEST(test_driver, summary_names_the_check_which_failed)
{
const std::vector<TestCase> cases{
{"mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); }}};
one("mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); })};

const auto [exit_code, output] = run(cases);
EXPECT_EQ(exit_code, 1);
Expand All @@ -103,10 +118,10 @@ TEST(test_driver, summary_names_the_check_which_failed)

TEST(test_driver, failure_outranks_a_later_exception)
{
const std::vector<TestCase> cases{{"both", [](TestReport& report) {
report.check_eq("a value", 1, 2);
throw std::runtime_error{"gave up afterwards"};
}}};
const std::vector<TestCase> cases{one("both", [](TestReport& report) {
report.check_eq("a value", 1, 2);
throw std::runtime_error{"gave up afterwards"};
})};

const auto [exit_code, output] = run(cases);
EXPECT_EQ(exit_code, TESTS_FAILED);
Expand All @@ -116,14 +131,55 @@ TEST(test_driver, failure_outranks_a_later_exception)

TEST(test_driver, failure_outranks_a_later_skip)
{
const std::vector<TestCase> cases{{"both", [](TestReport& report) {
report.check_eq("a value", 1, 2);
throw UnsupportedTestFeature{"gave up afterwards"};
}}};
const std::vector<TestCase> cases{one("both", [](TestReport& report) {
report.check_eq("a value", 1, 2);
throw UnsupportedTestFeature{"gave up afterwards"};
})};

const auto [exit_code, output] = run(cases);
EXPECT_EQ(exit_code, 1);
EXPECT_NE(output.find("1 failed"), std::string::npos);
// The summary names the check which failed, not what the test then gave up on.
EXPECT_NE(output.find("FAILED both - a value"), std::string::npos);
}

TEST(test_driver, a_file_counts_once_however_many_fixtures_it_holds)
{
const std::vector<TestCase> cases{
holding("a file", {Outcome::passed, Outcome::passed, Outcome::passed})};

const auto [exit_code, output] = run(cases);
EXPECT_EQ(exit_code, SUCCESS);
EXPECT_NE(output.find("collected 1 test"), std::string::npos);
EXPECT_NE(output.find("1 passed"), std::string::npos);
}

TEST(test_driver, a_declined_fixture_is_named_though_its_file_passed)
{
const std::vector<TestCase> cases{holding("a file", {Outcome::passed, Outcome::skipped})};

const auto [exit_code, output] = run(cases);
EXPECT_EQ(exit_code, SUCCESS);
// The file's own verdict says nothing about what it declined, so the fixture is named.
EXPECT_NE(output.find("1 passed in"), std::string::npos);
EXPECT_NE(output.find("SKIPPED a file::case - the reason"), std::string::npos);
}

TEST(test_driver, one_failed_fixture_fails_its_file)
{
const std::vector<TestCase> cases{
holding("a file", {Outcome::passed, Outcome::failed, Outcome::skipped})};

const auto [exit_code, output] = run(cases);
EXPECT_EQ(exit_code, TESTS_FAILED);
EXPECT_NE(output.find("1 failed, 0 passed"), std::string::npos);
}

TEST(test_driver, a_file_is_skipped_only_when_nothing_in_it_ran)
{
const std::vector<TestCase> cases{holding("a file", {Outcome::skipped, Outcome::skipped})};

const auto [exit_code, output] = run(cases);
EXPECT_EQ(exit_code, NOTHING_VERIFIED);
EXPECT_NE(output.find("0 passed, 1 skipped"), std::string::npos);
}
Loading