fix: recover SBOM components from monolithic non-ELF firmware

src/binver.cpp

@@ -15,17 +15,26 @@ namespace {
 // A curated version-banner signature. `anchors` seed the Aho-Corasick prefilter;
 // on a hit, `pattern` runs on a bounded window and its first capture group is the
 // version. `cpe_vendor`/`cpe_product` co-derive the CPE for the CVE join.
+//
+// `raw_safe` marks a signature whose anchor + pattern is specific enough to run on
+// arbitrary (non-ELF) bytes without false positives, used for monolithic non-ELF
+// firmware (see scan_binver's scan_nonelf path). A signature is raw_safe when its
+// anchor is a distinctive product string (not a common English word) and the
+// pattern pins a full version right after it. zlib is the one exclusion: its
+// anchors ("inflate "/"deflate ") are ordinary words that occur in prose and
+// code, so it stays ELF-gated until it gets its own raw-bytes FP check.
 struct VersionSig {
     std::string name;
     std::vector<std::string> anchors;
     std::string pattern;  // ECMAScript regex, one capture group = version
     std::string cpe_vendor;
     std::string cpe_product;
+    bool raw_safe;  // safe to match on non-ELF bytes (monolithic-image scan)
 };
 
 const std::vector<VersionSig>& sigs() {
     static const std::vector<VersionSig> s = {
-        {"busybox", {"BusyBox v"}, R"(BusyBox v(\d+\.\d+\.\d+))", "busybox", "busybox"},
+        {"busybox", {"BusyBox v"}, R"(BusyBox v(\d+\.\d+\.\d+))", "busybox", "busybox", true},
         // openssl: the authoritative banner is OPENSSL_VERSION_TEXT, "OpenSSL
         // <ver>[<suffix>]  <build date>" (e.g. "OpenSSL 1.1.1l  24 Aug 2021").
         // Requiring the trailing date rejects requirement/help text ("OpenSSL
@@ -38,17 +47,21 @@ const std::vector<VersionSig>& sigs() {
          {"OpenSSL "},
          R"(OpenSSL (\d+\.\d+\.\d+[a-z]?)(?:[-\s][A-Za-z][\w-]*)?\s+(?:\d{1,2} [A-Z][a-z]{2}|[A-Z][a-z]{2} +\d{1,2}) \d{4})",
          "openssl",
-         "openssl"},
+         "openssl",
+         true},
         {"dropbear",
          {"Dropbear v", "Dropbear-", "dropbear_"},
          R"([Dd]ropbear[ _v-]*(\d{4}\.\d+))",
          "dropbear_ssh_project",
-         "dropbear_ssh"},
-        {"dnsmasq", {"Dnsmasq version "}, R"(Dnsmasq version (\d+\.\d+))", "thekelleys", "dnsmasq"},
-        {"curl", {"libcurl/", "curl/"}, R"(curl/(\d+\.\d+\.\d+))", "haxx", "libcurl"},
-        {"zlib", {"inflate ", "deflate "}, R"((?:in|de)flate (\d+\.\d+\.\d+))", "zlib", "zlib"},
-        {"lighttpd", {"lighttpd/"}, R"(lighttpd/(\d+\.\d+\.\d+))", "lighttpd", "lighttpd"},
-        {"wget", {"GNU Wget "}, R"(GNU Wget (\d+\.\d+(?:\.\d+)?))", "gnu", "wget"},
+         "dropbear_ssh",
+         true},
+        {"dnsmasq", {"Dnsmasq version "}, R"(Dnsmasq version (\d+\.\d+))", "thekelleys", "dnsmasq",
+         true},
+        {"curl", {"libcurl/", "curl/"}, R"(curl/(\d+\.\d+\.\d+))", "haxx", "libcurl", true},
+        // zlib stays ELF-gated: "inflate "/"deflate " are ordinary words (raw_safe=false).
+        {"zlib", {"inflate ", "deflate "}, R"((?:in|de)flate (\d+\.\d+\.\d+))", "zlib", "zlib", false},
+        {"lighttpd", {"lighttpd/"}, R"(lighttpd/(\d+\.\d+\.\d+))", "lighttpd", "lighttpd", true},
+        {"wget", {"GNU Wget "}, R"(GNU Wget (\d+\.\d+(?:\.\d+)?))", "gnu", "wget", true},
         // NOTE: no OpenSSH banner rule — dropbear/openssh embed bug-compatibility
         // version lists ("OpenSSH_3.0", "OpenSSH_7.x", …) that a banner match reads
         // as dozens of bogus versions. dropbear (the SSH in most firmware) is above.
@@ -58,21 +71,21 @@ const std::vector<VersionSig>& sigs() {
         // ("v0.8.x_rtw_r24", "v2.10_ATBM_0.2") still resolves to the base version
         // because the optional third part requires a dot + digits (issue #10).
         {"wpa_supplicant", {"wpa_supplicant v"}, R"(wpa_supplicant v(\d+\.\d+(?:\.\d+)?))", "w1.fi",
-         "wpa_supplicant"},
-        {"hostapd", {"hostapd v"}, R"(hostapd v(\d+\.\d+(?:\.\d+)?))", "w1.fi", "hostapd"},
+         "wpa_supplicant", true},
+        {"hostapd", {"hostapd v"}, R"(hostapd v(\d+\.\d+(?:\.\d+)?))", "w1.fi", "hostapd", true},
         {"mosquitto", {"mosquitto version "}, R"(mosquitto version (\d+\.\d+\.\d+))", "eclipse",
-         "mosquitto"},
-        {"glibc", {"GNU C Library"}, R"(release version (\d+\.\d+(?:\.\d+)?))", "gnu", "glibc"},
-        {"musl", {"musl libc"}, R"(Version (\d+\.\d+\.\d+))", "musl-libc", "musl"},
+         "mosquitto", true},
+        {"glibc", {"GNU C Library"}, R"(release version (\d+\.\d+(?:\.\d+)?))", "gnu", "glibc", true},
+        {"musl", {"musl libc"}, R"(Version (\d+\.\d+\.\d+))", "musl-libc", "musl", true},
         // TLS/crypto libraries: single-value runtime banners in the .so/binary.
-        {"mbedtls", {"mbed TLS "}, R"(mbed TLS (\d+\.\d+\.\d+))", "arm", "mbed_tls"},
-        {"gnutls", {"GnuTLS "}, R"(GnuTLS (\d+\.\d+\.\d+))", "gnu", "gnutls"},
+        {"mbedtls", {"mbed TLS "}, R"(mbed TLS (\d+\.\d+\.\d+))", "arm", "mbed_tls", true},
+        {"gnutls", {"GnuTLS "}, R"(GnuTLS (\d+\.\d+\.\d+))", "gnu", "gnutls", true},
         // OpenVPN prints one version banner ("OpenVPN 2.4.7 arm-... [SSL (OpenSSL)]").
-        {"openvpn", {"OpenVPN "}, R"(OpenVPN (\d+\.\d+\.\d+))", "openvpn", "openvpn"},
+        {"openvpn", {"OpenVPN "}, R"(OpenVPN (\d+\.\d+\.\d+))", "openvpn", "openvpn", true},
         // Lua's LUA_RELEASE ("Lua 5.1.5") — require all three parts so the 2-part
         // LUA_VERSION ("Lua 5.1") in the same binary is not a second component.
         // Anchor "Lua 5." excludes LuaJIT ("LuaJIT 2.x").
-        {"lua", {"Lua 5."}, R"(Lua (5\.\d+\.\d+))", "lua", "lua"},
+        {"lua", {"Lua 5."}, R"(Lua (5\.\d+\.\d+))", "lua", "lua", true},
         // NOTE: no strongswan/expat banner rule — both embed symbol-version tables
         // (a full list of every release), the same trap that keeps OpenSSH out.
     };
@@ -194,21 +207,30 @@ std::vector<std::pair<std::string, std::string>> binver_cpe_products() {
     return out;
 }
 
-std::vector<Component> scan_binver(std::span<const uint8_t> data, const std::string& origin_path) {
+std::vector<Component> scan_binver(std::span<const uint8_t> data, const std::string& origin_path,
+                                   bool scan_nonelf) {
     std::vector<Component> out;
-    if (!is_elf(data)) return out;
+    const bool elf = is_elf(data);
+    // ELF input scans the full table (unchanged). Non-ELF input is scanned only
+    // when the caller allows it (a monolithic tree) and then only with the
+    // raw_safe signatures; otherwise it is skipped as before.
+    if (!elf && !scan_nonelf) return out;
 
     const auto& S = sigs();
     const auto& RE = compiled();
 
-    // One automaton over all anchors; anchor id -> sig index.
+    // One automaton over all active anchors; anchor id -> sig index. On non-ELF
+    // bytes, only the raw_safe signatures are armed.
     AhoCorasick ac;
     std::vector<uint32_t> id_to_sig;
-    for (uint32_t si = 0; si < S.size(); ++si)
+    for (uint32_t si = 0; si < S.size(); ++si) {
+        if (!elf && !S[si].raw_safe) continue;
         for (const auto& a : S[si].anchors) {
             ac.add(std::vector<uint8_t>(a.begin(), a.end()), static_cast<uint32_t>(id_to_sig.size()));
             id_to_sig.push_back(si);
         }
+    }
+    if (id_to_sig.empty()) return out;  // nothing armed (non-ELF with no raw_safe sig)
     ac.build();
 
     // Bounded window for the regex (M3): version banners are short.

src/binver.hpp

@@ -17,10 +17,21 @@
 
 namespace ft {
 
-// Scan an ELF file's bytes for known version banners. Returns one Component per
+// Scan a file's bytes for known version banners. Returns one Component per
 // distinct (name, version) found, source "binary-version", with purl + CPE from
-// the curated table. Non-ELF input returns {} (the banners are noise in text).
-std::vector<Component> scan_binver(std::span<const uint8_t> data, const std::string& origin_path);
+// the curated table.
+//
+// By default the scan runs only on ELF input; non-ELF input returns {} because a
+// version banner in arbitrary bytes (release notes, help text, a decompressed
+// container blob that mirrors an ELF already in the tree) is noise. Set
+// `scan_nonelf` to also scan non-ELF input, restricted to the distinctive-anchor
+// signatures that are false-positive-safe on arbitrary bytes (VersionSig::raw_safe).
+// The caller passes this only for a monolithic tree (no ELF member anywhere), so
+// statically-linked components in proprietary-OS / RTOS single-image firmware are
+// recovered without changing the result on any filesystem-based image. ELF input
+// always scans the full signature table regardless of the flag.
+std::vector<Component> scan_binver(std::span<const uint8_t> data, const std::string& origin_path,
+                                   bool scan_nonelf = false);
 
 // Scan any file for the Linux kernel banner ("Linux version X.Y.Z ...") and
 // return a linux_kernel component (source "kernel-banner", CPE

src/walk.cpp

@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <atomic>
 #include <filesystem>
+#include <fstream>
 #include <iterator>
 #include <optional>
 #include <string_view>
@@ -74,8 +75,17 @@ void parse_os_release(std::span<const uint8_t> data, std::string& id, std::strin
     }
 }
 
+// Peek the first 4 bytes for the ELF magic without mmapping the whole file. Used
+// to decide whether a tree is monolithic (no ELF member anywhere).
+bool path_is_elf(const fs::path& path) {
+    std::ifstream f(path, std::ios::binary);
+    char b[4] = {0, 0, 0, 0};
+    if (!f.read(b, 4)) return false;
+    return b[0] == 0x7f && b[1] == 'E' && b[2] == 'L' && b[3] == 'F';
+}
+
 FileHits scan_one(const fs::path& path, const fs::path& root, const Passes& passes,
-                  const Engine* engine) {
+                  const Engine* engine, bool monolithic) {
     FileHits fh;
     std::error_code ec;
     fh.path = fs::relative(path, root, ec).string();
@@ -109,7 +119,10 @@ FileHits scan_one(const fs::path& path, const fs::path& root, const Passes& pass
     }
     if (passes.sbom) {
         fh.components = scan_sbom(fh.path, fm.span());  // M4 package DBs
-        auto bv = scan_binver(fm.span(), fh.path);      // binary version strings (ELF only)
+        // Binary version banners. ELF members always; non-ELF bytes only when the
+        // whole tree is monolithic (no ELF anywhere), so proprietary-OS / RTOS
+        // single-image firmware gets an SBOM while filesystem trees are untouched.
+        auto bv = scan_binver(fm.span(), fh.path, monolithic);
         fh.components.insert(fh.components.end(), std::make_move_iterator(bv.begin()),
                              std::make_move_iterator(bv.end()));
         auto lm = scan_langmanifest(fh.path, fm.span());  // language manifests/lockfiles
@@ -250,14 +263,29 @@ Report scan_path(const std::string& root, const Passes& passes, unsigned nthread
     const size_t n = paths.size();
     rep.file_count = n;
 
+    // A monolithic image has no ELF member anywhere (proprietary-OS / RTOS single
+    // blob). Only then does binver scan non-ELF bytes for statically-linked
+    // component banners; a tree with any ELF keeps the ELF-only behavior, so
+    // filesystem images are unaffected. Cheap 4-byte peek, short-circuited on the
+    // first ELF. Only needed for the SBOM/CVE passes.
+    bool monolithic = false;
+    if (passes.sbom || passes.cve) {
+        monolithic = true;
+        for (const auto& p : paths)
+            if (path_is_elf(p)) {
+                monolithic = false;
+                break;
+            }
+    }
+
     std::vector<FileHits> results(n);
     if (nthreads == 0) nthreads = 1;
     nthreads = std::min<unsigned>(nthreads, std::max<size_t>(1, n));
 
     std::atomic<size_t> next{0};
     auto worker = [&] {
         for (size_t i = next.fetch_add(1); i < n; i = next.fetch_add(1))
-            results[i] = scan_one(paths[i], rel_root, passes, eng);
+            results[i] = scan_one(paths[i], rel_root, passes, eng, monolithic);
     };
     std::vector<std::thread> pool;
     for (unsigned t = 0; t < nthreads; ++t) pool.emplace_back(worker);

tests/test_sbom.py

@@ -146,10 +146,67 @@ def wfile(rel, body):
             print(f"FAIL: u-boot precision: expected ['2019.04'], got {ub}")
             fails += 1
 
+        # --- monolithic non-ELF firmware (binver raw-bytes scan) ---
+        # A proprietary-OS / RTOS image is one statically-linked non-ELF blob with
+        # no ELF member and no package DB. binver recovers its banners only when the
+        # whole tree is monolithic (no ELF anywhere), so filesystem images are
+        # untouched. Each case gets its own directory so the no-ELF condition holds.
+        def mono_dir(name, body):
+            p = os.path.join(d, name)
+            os.makedirs(p, exist_ok=True)
+            with open(os.path.join(p, "fw.bin"), "wb") as f:
+                f.write(body)
+            return p
+
+        def ossl_bb(rep, tree):
+            return (sorted(c["version"] for c in rep.get("components", []) if c["name"] == "openssl"),
+                    sorted(c["version"] for c in rep.get("components", []) if c["name"] == "busybox"))
+
+        # 1) Monolithic tree with clean banners -> recovered.
+        mono = mono_dir("mono", b"rtos image \x01\x02 BusyBox v1.31.1 junk "
+                                b"OpenSSL 0.9.7b 10 Apr 2003 \xaa\xbb end")
+        r = json.loads(subprocess.check_output([binary, "-j", "--sbom", mono],
+                                                stderr=subprocess.DEVNULL))
+        if ossl_bb(r, mono) != (["0.9.7b"], ["1.31.1"]):
+            print(f"FAIL: monolithic recovery: expected openssl 0.9.7b + busybox 1.31.1, "
+                  f"got {ossl_bb(r, mono)}")
+            fails += 1
+
+        # 2) The SAME blob next to an ELF (a filesystem tree) is NOT raw-scanned:
+        #    only the ELF's banner is reported, so filesystem images are unchanged.
+        mixed = mono_dir("mixed", b"rtos image BusyBox v1.31.1 OpenSSL 0.9.7b 10 Apr 2003 end")
+        with open(os.path.join(mixed, "prog"), "wb") as f:
+            f.write(elf + b" realbox BusyBox v1.36.0 x")
+        r = json.loads(subprocess.check_output([binary, "-j", "--sbom", mixed],
+                                                stderr=subprocess.DEVNULL))
+        if ossl_bb(r, mixed) != ([], ["1.36.0"]):
+            print(f"FAIL: ELF-present tree must skip the raw blob: expected busybox 1.36.0 only, "
+                  f"got {ossl_bb(r, mixed)}")
+            fails += 1
+
+        # 3) A corrupted build-date banner stays rejected (strict issue-#7 pattern).
+        bad = mono_dir("mono_bad", b"blob OpenSSL 0.9.7b 10 Apr 200\x228 more")
+        r = json.loads(subprocess.check_output([binary, "-j", "--sbom", bad],
+                                                stderr=subprocess.DEVNULL))
+        if any(c["name"] == "openssl" for c in r.get("components", [])):
+            print("FAIL: malformed openssl date must not be emitted on the raw path")
+            fails += 1
+
+        # 4) zlib's ordinary-word anchors stay ELF-gated (raw_safe=False), so a
+        #    monolithic blob does not get a zlib component from "inflate X.Y.Z".
+        zl = mono_dir("mono_zlib", b"payload inflate 1.2.11 Copyright 1995 blah")
+        r = json.loads(subprocess.check_output([binary, "-j", "--sbom", zl],
+                                                stderr=subprocess.DEVNULL))
+        if any(c["name"] == "zlib" for c in r.get("components", [])):
+            print("FAIL: zlib must stay ELF-gated on the raw path")
+            fails += 1
+
         if fails == 0:
             print(f"PASS: {len(names)} components (not-installed excluded), "
                   f"CycloneDX + SPDX emitted and consistent; binary-version precision "
-                  f"(issue #7) rejects requirement/dev/tool strings, keeps real banners")
+                  f"(issue #7) rejects requirement/dev/tool strings, keeps real banners; "
+                  f"monolithic non-ELF image recovered (busybox+openssl), filesystem trees "
+                  f"and malformed/zlib banners unaffected")
             return 0
         return 1
 

tests/unit/unit_tests.cpp

@@ -468,6 +468,14 @@ static std::vector<ft::Component> binver(const std::string& content) {
                            "bin/prog");
 }
 
+// Same, but with scan_nonelf=true: the monolithic-image path that also scans
+// non-ELF bytes (with the raw_safe signature subset).
+static std::vector<ft::Component> binver_raw(const std::string& content) {
+    return ft::scan_binver(std::span<const uint8_t>(
+                               reinterpret_cast<const uint8_t*>(content.data()), content.size()),
+                           "image.bin", /*scan_nonelf=*/true);
+}
+
 static void test_binver() {
     // "\x7f" "ELF" via explicit bytes (\x7f\x45\x4c\x46); banners separated by
     // spaces (is_elf only checks the first 4 bytes, the rest is arbitrary).
@@ -495,10 +503,42 @@ static void test_binver() {
         if (c.name == "busybox") ++bb_count;
     CHECK(bb_count == 1);
 
-    // Non-ELF content: version banners are ignored (noise in text/docs).
+    // Non-ELF content: version banners are ignored (noise in text/docs) on the
+    // default (filesystem) path.
     std::string txt = "Release notes: built with BusyBox v1.36.1 and OpenSSL 1.1.1n.\n";
     CHECK(binver(txt).empty());
 
+    // Monolithic non-ELF path (scan_nonelf=true): a statically-linked image with
+    // no ELF member still yields its component banners. A proprietary-OS image
+    // carries the banners as literals; the strict patterns pin real versions.
+    std::string mono = "rtos image \x01\x02 BusyBox v1.31.1 junk "
+                       "OpenSSL 0.9.7b 10 Apr 2003 \xaa\xbb Dropbear v2019.78 end";
+    auto mr = binver_raw(mono);
+    auto mby = [&](const std::string& n) -> const ft::Component* {
+        for (const auto& c : mr)
+            if (c.name == n) return &c;
+        return nullptr;
+    };
+    CHECK(mby("busybox") && mby("busybox")->version == "1.31.1");
+    CHECK(mby("openssl") && mby("openssl")->version == "0.9.7b");
+    CHECK(mby("dropbear") && mby("dropbear")->version == "2019.78");
+    // The same bytes on the default path stay empty (filesystem images unchanged).
+    CHECK(binver(mono).empty());
+
+    // raw_safe subset: zlib's anchors ("inflate "/"deflate ") are ordinary words,
+    // so zlib stays ELF-gated and is NOT matched on non-ELF bytes...
+    std::string zraw = "payload inflate 1.2.11 Copyright blah";
+    CHECK(binver_raw(zraw).empty());
+    // ...but inside an ELF the full table still runs and zlib is found.
+    std::string zelf = std::string("\x7f\x45\x4c\x46", 4) + " inflate 1.2.11 Copyright blah";
+    auto zc = binver(zelf);
+    CHECK(zc.size() == 1 && zc[0].name == "zlib" && zc[0].version == "1.2.11");
+
+    // The strict date requirement (issue #7) is preserved on the non-ELF path: a
+    // corrupted build-date banner is still rejected, not emitted with a bogus date.
+    std::string bad = "blob OpenSSL 0.9.7b 10 Apr 200\x22 8 more";
+    CHECK(binver_raw(bad).empty());
+
     // glibc + musl content banners.
     std::string glibc = std::string("\x7f\x45\x4c\x46", 4) +
                         " GNU C Library (GNU libc) stable release version 2.31. ";