ipsilon · chfast · Sep 2, 2026 · Sep 2, 2026 · Sep 2, 2026 · Sep 2, 2026
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
@@ -82,39 +82,124 @@ DUP1,4
 {\"pc\":6,\"op\":3,\"gas\":\"0xf4234\",\"gasCost\":\"0x3\",\"memSize\":0,\"stack\":\\[\"0x0\",\"0x4\"\\],\"depth\":1,\"refund\":0,\"opName\":\"SUB\"}
 ")
 
-endif()
+    set(DATA ${CMAKE_CURRENT_SOURCE_DIR})
+
+    # One command runs both fixture formats, deciding per test case which one it is: the state
+    # test and the last one by shape, the blockchain test by the format its "_info" declares.
+    add_test(NAME ${PREFIX}/test_both_formats COMMAND evmone-cli test
+        ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json
+        ${DATA}/blockchaintest/unrecovered_sender_blob_gas.json)
+    set_tests_properties(
+        ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 3 passed in")
+
+    # JSON which is not a test at all, as fixture directories hold beside their fixtures, is
+    # skipped whole rather than guessed at, even named on its own.
+    add_test(NAME ${PREFIX}/test_not_a_test COMMAND evmone-cli test
+        ${DATA}/statetest/tests1/SuiteA/index.json)
+    set_tests_properties(
+        ${PREFIX}/test_not_a_test PROPERTIES PASS_REGULAR_EXPRESSION
+        "SKIPPED[^\n]*index\\.json - not a test.*0 passed, 1 skipped in")
+
+    # Over a directory a test is a whole file, so a case the loader refuses must not take the
+    # rest of the file with it: the file passes on the case beside it, the refused one first.
+    # The file's own verdict says nothing about what it declined, so each declined case is named.
+    add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd)
+    set_tests_properties(
+        ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION
+        "a_bad_rlp: tests with invalidly rlp-encoded blocks.*c_engine: unsupported fixture format.*= 1 passed in")
+
+    # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these two assert on nothing
+    # else: a sound fixture directory succeeds, and a fault in one fails.
+    add_test(NAME ${PREFIX}/test_exit_ok COMMAND evmone-cli test ${DATA}/testcmd)
+
+    add_test(NAME ${PREFIX}/test_exit_fail COMMAND evmone-cli test ${DATA}/testcmd_fault)
+    set_tests_properties(${PREFIX}/test_exit_fail PROPERTIES WILL_FAIL TRUE)
+
+    # WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero
+    # too, so the counts are what distinguish the two. Whether a file is a fixture file is a
+    # property of the file, not of what -k selected: the case which is not a test still faults.
+    add_test(NAME ${PREFIX}/test_dir_fault COMMAND evmone-cli test
+        ${DATA}/testcmd_fault -k b_not_a_test)
+    set_tests_properties(
+        ${PREFIX}/test_dir_fault PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed")
+
+    # Over a directory -k selects within the file's test, so the summary is what proves a case
+    # ran: a count alone would hold just as well if the filter had dropped every case. The
+    # summary appears only with --trace-summary, which nothing else here asks for.
+    add_test(NAME ${PREFIX}/test_dir_filter COMMAND evmone-cli test
+        ${DATA}/statetest/filter -k passing_test_case --trace-summary)
+    set_tests_properties(
+        ${PREFIX}/test_dir_filter PROPERTIES
+        PASS_REGULAR_EXPRESSION "\"pass\":true"
+        FAIL_REGULAR_EXPRESSION "failing_test_case")
+
+    # A case whose load throws must not abandon the cases after it, so both faults are reported.
+    add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_cases)
+    set_tests_properties(
+        ${PREFIX}/test_after_exception PROPERTIES
+        # "collected 1 test" keeps this on the directory form, where the whole file is one test
+        # and the per-case catch is what keeps the second fault reachable.
+        PASS_REGULAR_EXPRESSION
+        "collected 1 test.*a_load_error:\n    exception.*b_bad_root:.*state root")
+
+    # Collected from a directory a file is one test, so a file with nothing to run is skipped
+    # whole, named by the first reason which explains it: all_unsupported declines twice, and
+    # the format named is the first one's.
+    add_test(NAME ${PREFIX}/test_dir_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped)
+    set_tests_properties(
+        ${PREFIX}/test_dir_skipped PROPERTIES
+        PASS_REGULAR_EXPRESSION
+        "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in")
+
+    # Nothing to run is not nothing to report: the run fails rather than passing empty.
+    add_test(NAME ${PREFIX}/test_dir_skipped_exit COMMAND evmone-cli test ${DATA}/testcmd_skipped)
+    set_tests_properties(${PREFIX}/test_dir_skipped_exit PROPERTIES WILL_FAIL TRUE)
+
+    # Naming the file instead of the directory makes each case its own test, and a fault in one
+    # is still a fault: the case beside it runs.
+    add_test(NAME ${PREFIX}/test_file_fault COMMAND evmone-cli test
+        ${DATA}/testcmd_fault/unrecognised_case.json)
+    set_tests_properties(
+        ${PREFIX}/test_file_fault PROPERTIES PASS_REGULAR_EXPRESSION
+        "FAILED[^\n]*unrecognised_case\\.json::b_not_a_test.*1 failed, 1 passed in")
+
+    # -k selects among the cases of a named file too.
+    add_test(NAME ${PREFIX}/test_file_filter COMMAND evmone-cli test
+        ${DATA}/testcmd_fault/unrecognised_case.json -k a_runs)
+    set_tests_properties(
+        ${PREFIX}/test_file_filter PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in")
+
+    # Naming a file whose cases this tool does not run skips them one by one, where collecting
+    # the same file from a directory would skip it whole.
+    add_test(NAME ${PREFIX}/test_file_declined COMMAND evmone-cli test
+        ${DATA}/testcmd/one_unsupported_case.json)
+    set_tests_properties(
+        ${PREFIX}/test_file_declined PROPERTIES PASS_REGULAR_EXPRESSION
+        "SKIPPED[^\n]*c_engine - unsupported fixture format.*= 1 passed, 2 skipped in")
 
-# A file holding a case which is not a fixture at all, beside one which runs.
-add_test(NAME ${PREFIX}/fixture_fault COMMAND evmone-statetest
-    ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault)
-set_tests_properties(${PREFIX}/fixture_fault PROPERTIES WILL_FAIL TRUE)
-
-# WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero too,
-# so the counts are what distinguish the two.
-add_test(NAME ${PREFIX}/fixture_fault_is_not_a_skip COMMAND evmone-statetest
-    ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault)
-set_tests_properties(
-    ${PREFIX}/fixture_fault_is_not_a_skip PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed")
-
-# Selecting only the case which is not a fixture must still fault.
-add_test(NAME ${PREFIX}/fixture_fault_survives_filter COMMAND evmone-statetest
-    ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault -k b_not_a_fixture)
-set_tests_properties(
-    ${PREFIX}/fixture_fault_survives_filter PROPERTIES
-    PASS_REGULAR_EXPRESSION "1 failed, 0 passed")
-
-# A case whose load throws takes the rest of the file with it: the whole file is loaded before
-# any of it runs, so the case after it is never reached and the failure is named after the file
-# rather than the case it came from. FAILED pins that as a failure: a PASS_REGULAR_EXPRESSION
-# makes CTest ignore the exit code, and a fault downgraded to a skip names the file too.
-add_test(NAME ${PREFIX}/case_after_exception COMMAND evmone-statetest
-    ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_cases)
-set_tests_properties(
-    ${PREFIX}/case_after_exception PROPERTIES
-    PASS_REGULAR_EXPRESSION
-    "collected 1 test.*FAILED[^\n]*case_after_exception\\.json[^\n]*exception"
-    FAIL_REGULAR_EXPRESSION "b_wrong_state_root"
-)
+    # --trace and --histogram reach the runner through the app-level flags, and turn the progress
+    # row into a line naming each test so what each writes can be told apart.
+    add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram
+        ${DATA}/statetest/tests1/SuiteA/test1.json)
+    set_tests_properties(
+        ${PREFIX}/test_histogram PROPERTIES PASS_REGULAR_EXPRESSION
+        "test1\\.json::test1
+--- # HISTOGRAM.*= 1 passed in")
+
+    add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace
+        ${DATA}/statetest/tests1/SuiteA/test1.json)
+    set_tests_properties(
+        ${PREFIX}/test_trace PROPERTIES PASS_REGULAR_EXPRESSION
+        "test1\\.json::test1\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in")
+
+    # A named file which does not parse names itself, because no case in it can be.
+    add_test(NAME ${PREFIX}/test_file_unparsed COMMAND evmone-cli test
+        ${DATA}/testcmd_broken/truncated.json)
+    set_tests_properties(
+        ${PREFIX}/test_file_unparsed PROPERTIES PASS_REGULAR_EXPRESSION
+        "FAILED[^\n]*truncated\\.json - exception[^\n]*parse error.*1 failed, 0 passed in")
+
+endif()
 
 add_subdirectory(blockchaintest)
 add_subdirectory(export)

diff --git a/test/integration/blockchaintest/eip7778_block_gas.json b/test/integration/blockchaintest/eip7778_block_gas.json
@@ -130,6 +130,9 @@
         },
         "genesisRLP": "0x",
         "sealEngine": "NoProof",
-        "_info": {}
+        "_info": {
+            "fixture-format": "blockchain_test",
+            "comment": "The format every EEST blockchain fixture declares."
+        }
     }
 }
diff --git a/test/integration/testcmd/one_unsupported_case.json b/test/integration/testcmd/one_unsupported_case.json
@@ -0,0 +1,116 @@
+{
+ "a_bad_rlp": {
+  "_info": {
+   "comment": "Declines to load: expectException without rlp_decoded. Nothing of this case runs, so the case beside it is what the file reports."
+  },
+  "network": "Cancun",
+  "genesisBlockHeader": {
+   "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+   "coinbase": "0x0000000000000000000000000000000000000000",
+   "stateRoot": "0x0000000000000000000000000000000000000000000000000000000000000000",
+   "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+   "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+   "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+   "number": "0x00",
+   "gasLimit": "0x01000000",
+   "gasUsed": "0x00",
+   "timestamp": "0x00",
+   "extraData": "0x00",
+   "baseFeePerGas": "0x10",
+   "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+   "blobGasUsed": "0x00",
+   "excessBlobGas": "0x00",
+   "hash": "0x0000000000000000000000000000000000000000000000000000000000000001"
+  },
+  "pre": {
+   "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+    "code": "0x",
+    "nonce": "0x00",
+    "balance": "0x02540be400"
+   }
+  },
+  "blocks": [
+   {
+    "expectException": "TransactionException.INVALID_SIGNATURE_VRS",
+    "rlp": "0x"
+   }
+  ],
+  "lastblockhash": "0x0000000000000000000000000000000000000000000000000000000000000001",
+  "postState": {
+   "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+    "code": "0x",
+    "nonce": "0x00",
+    "balance": "0x02540be400"
+   }
+  }
+ },
+ "b_state": {
+  "_info": {
+   "comment": "A state test beside blockchain ones, which is the whole point of deciding the format per case."
+  },
+  "env": {
+   "currentBaseFee": "0x0a",
+   "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba",
+   "currentDifficulty": "0x020000",
+   "currentGasLimit": "0xff112233445566",
+   "currentNumber": "0x01",
+   "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000",
+   "currentTimestamp": "0x03e8"
+  },
+  "post": {
+   "London": [
+    {
+     "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530",
+     "indexes": {
+      "data": 0,
+      "gas": 0,
+      "value": 0
+     },
+     "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"
+    }
+   ]
+  },
+  "pre": {
+   "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": {
+    "balance": "0x0de0b6b3a7640000",
+    "code": "0x600160010160005500",
+    "nonce": "0x00",
+    "storage": {}
+   },
+   "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": {
+    "balance": "0x00",
+    "code": "0x",
+    "nonce": "0x01",
+    "storage": {}
+   },
+   "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+    "balance": "0x0de0b6b3a7640000",
+    "code": "0x",
+    "nonce": "0x00",
+    "storage": {}
+   }
+  },
+  "transaction": {
+   "data": [
+    "0x"
+   ],
+   "gasLimit": [
+    "0x061a80"
+   ],
+   "gasPrice": "0x0a",
+   "nonce": "0x00",
+   "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+   "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87",
+   "value": [
+    "0x0186a0"
+   ]
+  }
+ },
+ "c_engine": {
+  "_info": {
+   "fixture-format": "blockchain_test_engine",
+   "comment": "Carries blocks, so the shape alone would run it; the declared format is what decides, and this tool does not run that one."
+  },
+  "blocks": []
+ }
+}
diff --git a/test/integration/testcmd_broken/truncated.json b/test/integration/testcmd_broken/truncated.json
@@ -0,0 +1,3 @@
+{
+ "a_case": {
+  "_info": {"comment": "The file ends here, mid-object, so it does not parse at all."}
diff --git a/test/integration/testcmd_cases/case_after_exception.json b/test/integration/testcmd_cases/case_after_exception.json
@@ -5,7 +5,7 @@
    "comment": "A state test with no pre state, so loading it throws."
   }
  },
- "b_wrong_state_root": {
+ "b_bad_root": {
   "_info": {
    "comment": "Runs and fails on the state root. Only reported if the case before it did not abandon the file."
   },

diff --git a/test/integration/testcmd_fault/unrecognised_case.json b/test/integration/testcmd_fault/unrecognised_case.json
@@ -59,7 +59,7 @@
    ]
   }
  },
- "b_not_a_fixture": {
+ "b_not_a_test": {
   "_info": {
    "comment": "Not a test: no fixture fields at all. Beside a case which runs, so this is unmistakably a fixture file with one broken case in it."
   }

diff --git a/test/integration/testcmd_skipped/all_unsupported.json b/test/integration/testcmd_skipped/all_unsupported.json
@@ -0,0 +1,15 @@
+{
+ "a_engine": {
+  "_info": {
+   "fixture-format": "blockchain_test_engine",
+   "comment": "The file is named by this reason, the first collected, not the one below."
+  },
+  "blocks": []
+ },
+ "b_engine_x": {
+  "_info": {
+   "fixture-format": "blockchain_test_engine_x"
+  },
+  "blocks": []
+ }
+}
diff --git a/test/integration/testcmd_skipped/empty.json b/test/integration/testcmd_skipped/empty.json
@@ -0,0 +1 @@
+{}
diff --git a/test/integration/testcmd_skipped/not_an_object.json b/test/integration/testcmd_skipped/not_an_object.json
@@ -0,0 +1,7 @@
+[
+ {
+  "comment": "A top-level array, so it holds no named fixture. Its one element would be taken for a blockchain test if the array were walked by index.",
+  "pre": {},
+  "blocks": []
+ }
+]
diff --git a/test/integration/testcmd_skipped/partial_shapes.json b/test/integration/testcmd_skipped/partial_shapes.json
@@ -0,0 +1,24 @@
+{
+ "a_blocks_no_pre": {
+  "comment": "Carries one of the two keys a blockchain test is named by, so the shape does not name it and nothing else does either.",
+  "blocks": []
+ },
+ "b_tx_no_pre": {
+  "comment": "The same for a state test: a transaction to apply, but no state to apply it to.",
+  "transaction": {
+   "data": [
+    "0x"
+   ],
+   "gasLimit": [
+    "0x061a80"
+   ],
+   "gasPrice": "0x0a",
+   "nonce": "0x00",
+   "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+   "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87",
+   "value": [
+    "0x0186a0"
+   ]
+  }
+ }
+}
diff --git a/test/integration/testcmd_skipped/pre_alloc.json b/test/integration/testcmd_skipped/pre_alloc.json
@@ -0,0 +1,27 @@
+{
+ "testIds": [
+  "tests/ported_static/vmIOandFlowOperations/test_mload.py::test_mload[fork_Osaka-blockchain_test_engine_x]"
+ ],
+ "environment": {
+  "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba",
+  "currentGasLimit": "0xff112233445566",
+  "currentNumber": "0x01"
+ },
+ "network": "Osaka",
+ "chainId": 1,
+ "pre": {
+  "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": {
+   "balance": "0x0de0b6b3a7640000",
+   "code": "0x600160010160005500",
+   "nonce": "0x00",
+   "storage": {}
+  },
+  "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+   "balance": "0x0de0b6b3a7640000",
+   "code": "0x",
+   "nonce": "0x00",
+   "storage": {}
+  }
+ },
+ "comment": "EEST keeps the pre-allocation its fixtures share beside them, and none of it is a test. Several entries, none of which names a format or carries a shape."
+}