From 51a527f9aef5d8ef5790e17adf38fad5a668807d Mon Sep 17 00:00:00 2001 From: fcostaoliveira Date: Fri, 2 Oct 2026 11:19:00 +0100 Subject: [PATCH 1/3] build: amalgamate.py --with-log and --include-prefix, for embedders beyond the core Redis and Valkey vendor only the core, but memtier_benchmark also uses the log codec (hdr_log_*) and the time helpers, and Node.js keeps the upstream src/ + include/hdr/ layout. The release assets could not replace either: - the log files include private headers (hdr_tests.h, hdr_endian.h, hdr_encoding.h, hdr_histogram_internal.h) that the assets do not ship, and includes that a flat directory does not have; - the generated .c included "hdr_histogram.h", which does not resolve when the header lives in include/hdr/. --with-log also emits hdr_histogram_log.c, hdr_encoding.c and hdr_time.c with their private headers inlined, plus their public headers rewritten for a flat layout. --include-prefix hdr/ makes the generated files include "hdr/hdr_histogram.h" and leaves the header byte-identical to the upstream one. Without the new flags the output is byte-identical to before. script/check-amalgamation.sh verifies both variants (per-file preprocessor equivalence with the sources, -Werror builds, a log encode/decode round trip, and the hdr/ layout) and is run by the amalgamation CI job and by the release job before anything is attached. The release now also attaches a hdr_histogram-amalgamation-with-log-.zip. docs/AMALGAMATION.md gains a section on migrating an existing vendored copy. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/amalgamation.yml | 5 + .github/workflows/release-amalgamation.yml | 11 +- docs/AMALGAMATION.md | 111 +++++++++++++---- script/amalgamate.py | 138 ++++++++++++++------- script/check-amalgamation.sh | 73 +++++++++++ 5 files changed, 272 insertions(+), 66 deletions(-) create mode 100755 script/check-amalgamation.sh diff --git a/.github/workflows/amalgamation.yml b/.github/workflows/amalgamation.yml index 2388362..c532c80 100644 --- a/.github/workflows/amalgamation.yml +++ b/.github/workflows/amalgamation.yml @@ -50,3 +50,8 @@ jobs: #define hdr_free free EOT cc -std=c99 -Wall -Wextra -Os -Iout-malloc -c out-malloc/hdr_histogram.c -o out-malloc/hdr_histogram.o + + - name: Optional variants (--with-log, --include-prefix) + run: | + sudo apt-get update && sudo apt-get install -y zlib1g-dev + sh script/check-amalgamation.sh "$RUNNER_TEMP/amalgamation-variants" diff --git a/.github/workflows/release-amalgamation.yml b/.github/workflows/release-amalgamation.yml index dd565b6..6ad5319 100644 --- a/.github/workflows/release-amalgamation.yml +++ b/.github/workflows/release-amalgamation.yml @@ -31,6 +31,7 @@ jobs: echo "::warning::release tag '$tag' does not match HDR_HISTOGRAM_VERSION '$version'" fi python3 script/amalgamate.py --output out + python3 script/amalgamate.py --output out-log --with-log - name: Check it before publishing run: | @@ -40,15 +41,19 @@ jobs: cmp orig.txt amal.txt cc -std=c99 -Wall -Wextra -Os -Iout -c out/hdr_histogram.c -o out/hdr_histogram.o rm out/hdr_histogram.o + sudo apt-get update && sudo apt-get install -y zlib1g-dev + sh script/check-amalgamation.sh "$RUNNER_TEMP/amalgamation-variants" - name: Package run: | set -euo pipefail name="hdr_histogram-amalgamation-${VERSION}" - mkdir -p "dist/$name" + logname="hdr_histogram-amalgamation-with-log-${VERSION}" + mkdir -p "dist/$name" "dist/$logname" cp out/hdr_histogram.c out/hdr_histogram.h out/hdr_malloc.h docs/AMALGAMATION.md "dist/$name/" - (cd dist && zip -qr "$name.zip" "$name" && rm -r "$name") - cp out/hdr_histogram.c out/hdr_histogram.h out/hdr_malloc.h dist/ + cp out-log/* docs/AMALGAMATION.md "dist/$logname/" + (cd dist && zip -qr "$name.zip" "$name" && zip -qr "$logname.zip" "$logname" && rm -r "$name" "$logname") + cp out-log/* dist/ (cd dist && sha256sum * > SHA256SUMS) ls -l dist diff --git a/docs/AMALGAMATION.md b/docs/AMALGAMATION.md index 6aef1f1..ffe783e 100644 --- a/docs/AMALGAMATION.md +++ b/docs/AMALGAMATION.md @@ -1,9 +1,10 @@ -# Amalgamated HdrHistogram_c core +# Amalgamated HdrHistogram_c -For projects that embed the library, `script/amalgamate.py` builds a self-contained -copy of the **core** (the record path, the value/percentile read path and the -iterators). It has no zlib, threads, logging-codec, interval-recorder or phaser -dependency, only the C standard library and libm. +For projects that embed the library, `script/amalgamate.py` builds a self-contained copy of the +**core** (the record path, the value/percentile read path and the iterators). It has no zlib, +threads, logging-codec, interval-recorder or phaser dependency, only the C standard library and +libm. Each generated `.c` file has its private headers inlined, so there are no private headers +to carry around. | file | description | |------|-------------| @@ -11,10 +12,18 @@ dependency, only the C standard library and libm. | `hdr_histogram.h` | the public API (unchanged copy) | | `hdr_malloc.h` | the default libc allocator hook (unchanged copy) | -The output is **generated, not committed**. Each GitHub release has these files attached -(`hdr_histogram-amalgamation-.zip` and the three files on their own), built by -`.github/workflows/release-amalgamation.yml`. This is the same model SQLite uses for its -amalgamation, and it means the files never go stale in the tree. +With `--with-log` it also produces the log codec (needs zlib): + +| file | description | +|------|-------------| +| `hdr_histogram_log.c`, `hdr_encoding.c`, `hdr_time.c` | the codec, time helpers and base64/varint code, private headers inlined | +| `hdr_histogram_log.h`, `hdr_time.h` | their public headers | + +The output is **generated, not committed**. Each GitHub release has these files attached: +`hdr_histogram-amalgamation-.zip` (core) and +`hdr_histogram-amalgamation-with-log-.zip` (core plus log codec), also as individual +files with a `SHA256SUMS`. They are built by `.github/workflows/release-amalgamation.yml`. This is +the same model SQLite uses for its amalgamation, and it means the files never go stale in the tree. ## Getting it @@ -23,7 +32,8 @@ From a release: download the zip (or the individual files) from the release page From any checkout or release tarball: ```sh -script/amalgamate.py --output path/to/dir +script/amalgamate.py --output path/to/dir # core +script/amalgamate.py --output path/to/dir --with-log # core + log codec ``` `--output` is relative to the current directory, so you can run the script from an unpacked @@ -31,12 +41,15 @@ release tarball into your own source tree. ## Using it -Copy the files into your project and build `hdr_histogram.c` with your own sources: +Copy the files into your project and build `hdr_histogram.c` (and, with `--with-log`, the other +`.c` files) with your own sources: ```sh cc -c -Ipath/to/amalgamated path/to/amalgamated/hdr_histogram.c ``` +### Allocator + To route allocations through your own allocator, provide a header that defines `hdr_malloc` / `hdr_calloc` / `hdr_realloc` / `hdr_free`. Either compile with it: @@ -50,11 +63,10 @@ or generate the amalgamation with it already baked in, so no compiler flag is ne script/amalgamate.py --output deps/hdr_histogram --malloc-include my_alloc.h ``` -`--malloc-include` only tells the generated `hdr_histogram.c` that `my_alloc.h` exists and -should be its default allocator header (one line changes: the default of -`HDR_MALLOC_INCLUDE`). The script then writes just two files, `hdr_histogram.c` and -`hdr_histogram.h`. It does not write `hdr_malloc.h`, and it does not create `my_alloc.h`: -that file is yours, kept next to the generated ones. +`--malloc-include` only tells the generated files that `my_alloc.h` exists and should be their +default allocator header (one line changes: the default of `HDR_MALLOC_INCLUDE`). The script then +does not write `hdr_malloc.h`, and it does not create `my_alloc.h`: that file is yours, kept next +to the generated ones. ``` deps/hdr_histogram/ @@ -65,11 +77,68 @@ deps/hdr_histogram/ A `-DHDR_MALLOC_INCLUDE` on the compiler command line still takes precedence. -`-DHDR_DISABLE_AVX2` also works on the amalgamated file: it drops the runtime AVX2 percentile -scan and always uses the scalar one, exactly as in the full build. +### Other switches + +`-DHDR_DISABLE_AVX2` also works on the amalgamated file: it drops the runtime AVX2 percentile scan +and always uses the scalar one, exactly as in the full build. + +### Header layout + +By default the generated `.c` files include the public headers as `"hdr_histogram.h"`, which +suits a flat directory. If your tree keeps them in a `hdr/` directory, as the upstream layout does +(`src/` plus `include/hdr/`), generate with `--include-prefix hdr/`: the `.c` files then include +`"hdr/hdr_histogram.h"`, and the generated `hdr_histogram.h` is byte-identical to the one in +`include/hdr/`. + +## Migrating an existing vendored copy + +Most embedders carry a flattened copy of the library with a few local edits. This is what it takes +to switch to the generated files. Each recipe below was applied to a copy of the project, which +was then built and tested. + +First, check whether your local edits are already upstream: + +| local patch | upstream now | +|-------------|--------------| +| `hdr_iter_linear_set_value_units_per_bucket` | yes: `hdr_iter_linear_set_value_units_per_bucket` (a no-op on non-linear iterators) | +| `hdr_record_value_capped` | yes, but it clamps to `[0, highest_trackable_value]`: 0 and values below `lowest_discernible_value` are recorded as they are, instead of being raised | +| an atomic variant of the above | yes: `hdr_record_value_capped_atomic` | +| a `total_count` getter | yes: `hdr_total_count` (NULL-safe, atomic load) | +| `hdr_string_write(&out, h)` | yes: `hdr_log_encode(h, &out)` (same body, arguments swapped) | +| the struct field `lowest_trackable_value` | renamed upstream to `lowest_discernible_value` | + +Then the mechanical steps: + +* **Redis** (`deps/hdr_histogram`, built with `-std=c99 -Wall -Os`): run + `script/amalgamate.py --output deps/hdr_histogram --malloc-include hdr_redis_malloc.h`, then delete + `hdr_atomic.h` and `hdr_tests.h`. Keep your `hdr_redis_malloc.h`. The `Makefile` needs no change + (its `-DHDR_MALLOC_INCLUDE` is now redundant but harmless). The iterator patch can be dropped. +* **Valkey**: the same command with your shim's name. Valkey also builds the dependency with CMake, + and its `CMakeLists.txt` lists `hdr_atomic.h` among the sources: remove that one entry, or CMake + stops with "Cannot find source file". +* **memtier_benchmark** uses the log codec and the time helpers as well, so generate with + `--with-log` (no `--malloc-include` needed, it will use `hdr_malloc.h`): + `script/amalgamate.py --output deps/hdr_histogram --with-log`. Delete the other files in that + directory (`hdr_atomic.h`, `hdr_encoding.h`, `hdr_endian.h`, `byteorder.h`) and list the new ones + in `Makefile.am`: `hdr_histogram.c/.h`, `hdr_histogram_log.c/.h`, `hdr_encoding.c`, `hdr_time.c/.h`, + `hdr_malloc.h`. Then drop the local `hdr_record_value_capped_atomic`, rename + `lowest_trackable_value` to `lowest_discernible_value`, and replace `hdr_string_write` as above. + The memtier includes (`"deps/hdr_histogram/hdr_histogram.h"`) are unchanged. +* **Node.js** (`deps/histogram`, which keeps `src/` plus `include/hdr/`): generate with + `script/amalgamate.py --output --include-prefix hdr/`, copy `hdr_histogram.c` to + `src/hdr_histogram.c` and `hdr_histogram.h` to `include/hdr/hdr_histogram.h`, and delete + `src/hdr_atomic.h` and `src/hdr_tests.h`. `histogram.gyp`, `BUILD.gn` and `unofficial.gni` list only + those two files, so they need no change. + +One behaviour change to know about when upgrading from an older copy: bucket counts are now +required to be non-negative. The record functions reject a negative count, and decoding a log +with a negative bucket count fails with `HDR_NEGATIVE_COUNT_INVALID`. Code that writes `counts[]` +directly should not store negative values. ## How it stays correct -The amalgamated `hdr_histogram.c` preprocesses to the exact same translation unit as the -normal multi-file build, so it behaves identically. CI (`.github/workflows/amalgamation.yml`) -generates it on every change and checks that, so the script cannot drift from the sources. +Each amalgamated `.c` file preprocesses to the exact same translation unit as the normal +multi-file build, so it behaves identically. CI (`.github/workflows/amalgamation.yml`) generates +every variant on each change, checks that equivalence, builds with `-Wall -Wextra -Werror`, and +round-trips a histogram through the amalgamated log codec. The release job runs the same checks +before it attaches anything. You can run them by hand with `sh script/check-amalgamation.sh`. diff --git a/script/amalgamate.py b/script/amalgamate.py index 8e7dd11..729e85b 100755 --- a/script/amalgamate.py +++ b/script/amalgamate.py @@ -14,12 +14,23 @@ /hdr_malloc.h the default libc allocator hook (copied as-is; not written with --malloc-include) +--with-log also emits the log codec, for projects that use the hdr_log_* API (it +needs zlib and the core above). Each file has its private headers inlined, so no +private header is ever needed: + + /hdr_histogram_log.c /hdr_encoding.c /hdr_time.c + /hdr_histogram_log.h /hdr_time.h + Override the allocator exactly as for the normal build, by compiling with -DHDR_MALLOC_INCLUDE='"my_alloc.h"' or bake it in with --malloc-include, which makes my_alloc.h the default instead -of hdr_malloc.h. Only hdr_histogram.c and hdr_histogram.h are written: hdr_malloc.h -is not, and my_alloc.h is not created (it is yours, kept alongside the output). -A -DHDR_MALLOC_INCLUDE on the compiler command line still wins. +of hdr_malloc.h. hdr_malloc.h is then not written, and my_alloc.h is not created +(it is yours, kept alongside the output). A -DHDR_MALLOC_INCLUDE on the compiler +command line still wins. + +--include-prefix hdr/ is for trees that keep the public headers in a hdr/ +directory (as the upstream layout does): the generated .c files then include +"hdr/hdr_histogram.h" instead of "hdr_histogram.h". The default is no prefix. --output is relative to the current directory, so the script can be run from a release tarball into your own tree. @@ -27,6 +38,7 @@ Usage: script/amalgamate.py --output DIR script/amalgamate.py --output deps/hdr_histogram --malloc-include hdr_redis_malloc.h + script/amalgamate.py --output deps/hdr_histogram --with-log """ import argparse import os @@ -36,22 +48,29 @@ HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) -# The core translation unit and the search path for its local includes. -ROOT_TU = "src/hdr_histogram.c" -PUBLIC_HEADER = "hdr_histogram.h" # shipped alongside, referenced by name +# Translation units and the search path for their local (private) includes. +CORE_TUS = ["src/hdr_histogram.c"] +LOG_TUS = ["src/hdr_histogram_log.c", "src/hdr_encoding.c", "src/hdr_time.c"] INCLUDE_DIRS = ["src", "include/hdr"] -# Files copied verbatim next to the amalgamated .c. -COPY_FILES = { - PUBLIC_HEADER: "include/hdr/hdr_histogram.h", - "hdr_malloc.h": "src/hdr_malloc.h", + +# Public headers are shipped alongside and referenced by name, never inlined. +PUBLIC_HEADERS = { + "hdr_histogram.h": "include/hdr/hdr_histogram.h", + "hdr_histogram_log.h": "include/hdr/hdr_histogram_log.h", + "hdr_time.h": "include/hdr/hdr_time.h", } +CORE_HEADERS = ["hdr_histogram.h"] +LOG_HEADERS = ["hdr_histogram_log.h", "hdr_time.h"] +MALLOC_HEADER = "hdr_malloc.h" -# Matches #include "foo.h" or #include (not #include SOME_MACRO). MALLOC_DEFAULT = '#define HDR_MALLOC_INCLUDE "hdr_malloc.h"' HEADER_NAME_RE = re.compile(r'^[A-Za-z0-9_][A-Za-z0-9_./+-]*$') +PREFIX_RE = re.compile(r'^([A-Za-z0-9_][A-Za-z0-9_.+-]*/)*$') -# a trailing /* ... */ comment is allowed (it is dropped when the header is inlined) +# Matches #include "foo.h" or #include (not #include SOME_MACRO). +# A trailing /* ... */ comment is allowed (it is dropped when the header is inlined). INCLUDE_RE = re.compile(r'^\s*#\s*include\s*(?:"([^"]+)"|<([^>]+)>)\s*(?:/\*.*\*/\s*)?$') +HDR_ANGLE_RE = re.compile(r'^(\s*#\s*include\s*)(.*)$') def version(): @@ -82,8 +101,8 @@ def resolve(name, angle): return None -def inline(path, seen, out): - """Append `path`'s content to `out`, recursively inlining local includes.""" +def inline(path, seen, out, prefix): + """Append `path`'s content to `out`, recursively inlining private includes.""" with open(os.path.join(ROOT, path)) as f: lines = f.readlines() out.append("/* ---- begin %s ---- */\n" % path) @@ -92,52 +111,69 @@ def inline(path, seen, out): if m: name = m.group(1) or m.group(2) angle = m.group(1) is None - if os.path.basename(name) == PUBLIC_HEADER: - if PUBLIC_HEADER not in seen: - seen.add(PUBLIC_HEADER) - out.append('#include "%s"\n' % PUBLIC_HEADER) + base = os.path.basename(name) + if base in PUBLIC_HEADERS and (not angle or name.startswith("hdr/")): + if base not in seen: + seen.add(base) + out.append('#include "%s%s"\n' % (prefix, base)) continue target = resolve(name, angle) if target is not None: if target not in seen: seen.add(target) - inline(target, seen, out) + inline(target, seen, out, prefix) continue # system include, the HDR_MALLOC_INCLUDE macro hook, or plain code out.append(line) out.append("/* ---- end %s ---- */\n" % path) -def generate(malloc_include=None): +def generate(tu, malloc_include, prefix): + name = os.path.basename(tu) + kind = "core" if tu in CORE_TUS else "log codec" banner = ( "/*\n" - " * HdrHistogram_c core -- amalgamated single-file build.\n" + " * HdrHistogram_c %s -- amalgamated (%s, private headers inlined).\n" " *\n" " * GENERATED by script/amalgamate.py from HdrHistogram_c %s. DO NOT EDIT;\n" " * change the sources and regenerate. Public domain (see hdr_histogram.h).\n" - " */\n" % version() + " */\n" % (kind, name, version()) ) + if tu in CORE_TUS: # keep the core banner exactly as it has always been + banner = ( + "/*\n" + " * HdrHistogram_c core -- amalgamated single-file build.\n" + " *\n" + " * GENERATED by script/amalgamate.py from HdrHistogram_c %s. DO NOT EDIT;\n" + " * change the sources and regenerate. Public domain (see hdr_histogram.h).\n" + " */\n" % version() + ) out = [banner] - # Empty seen-set: the public header include in hdr_histogram.c is emitted once - # at its natural position; later references (e.g. from hdr_tests.h) are deduped. - inline(ROOT_TU, set(), out) - body = "".join(out) - if malloc_include: - if body.count(MALLOC_DEFAULT) != 1: - sys.exit("cannot find the HDR_MALLOC_INCLUDE default in %s" % ROOT_TU) - body = body.replace( + # Empty seen-set: each public header include is emitted once, at its natural + # position; later references (e.g. from hdr_tests.h) are deduped. + inline(tu, set(), out, prefix) + text = "".join(out) + if malloc_include and MALLOC_DEFAULT in text: + if text.count(MALLOC_DEFAULT) != 1: + sys.exit("unexpected HDR_MALLOC_INCLUDE default in %s" % tu) + text = text.replace( MALLOC_DEFAULT, '#define HDR_MALLOC_INCLUDE "%s"' % malloc_include) - return body + return text -def copies(malloc_include=None): - result = {} - for dst, src in COPY_FILES.items(): - if malloc_include and dst == "hdr_malloc.h": - continue - with open(os.path.join(ROOT, src)) as f: - result[dst] = f.read() - return result +def public_header(name, prefix): + """Copy a public header; point its includes at the chosen layout.""" + with open(os.path.join(ROOT, PUBLIC_HEADERS[name])) as f: + lines = f.readlines() + if prefix == "hdr/": + return "".join(lines) # the upstream layout: verbatim + out = [] + for line in lines: + m = HDR_ANGLE_RE.match(line) + if m and m.group(2) in PUBLIC_HEADERS: + line = '%s"%s%s"%s\n' % (m.group(1), prefix, m.group(2), m.group(3)) + out.append(line) + return "".join(out) def main(): @@ -148,12 +184,30 @@ def main(): ap.add_argument("--malloc-include", metavar="HEADER", help="make HEADER the default allocator header instead of hdr_malloc.h " "(you supply it; e.g. hdr_redis_malloc.h)") + ap.add_argument("--with-log", action="store_true", + help="also emit the log codec (hdr_histogram_log.c, hdr_encoding.c, " + "hdr_time.c and their public headers); needs zlib") + ap.add_argument("--include-prefix", metavar="DIR/", default="", + help='how the generated files include the public headers, e.g. "hdr/" ' + "for a tree that keeps them in a hdr/ directory (default: none)") args = ap.parse_args() if args.malloc_include is not None and not HEADER_NAME_RE.match(args.malloc_include): ap.error("--malloc-include must be a plain header file name") - - files = {"hdr_histogram.c": generate(args.malloc_include)} - files.update(copies(args.malloc_include)) + if not PREFIX_RE.match(args.include_prefix): + ap.error('--include-prefix must be empty or a relative directory ending in "/"') + + tus = CORE_TUS + (LOG_TUS if args.with_log else []) + headers = CORE_HEADERS + (LOG_HEADERS if args.with_log else []) + files = {} + for tu in tus: + files[os.path.basename(tu)] = generate(tu, args.malloc_include, args.include_prefix) + if args.malloc_include and ('"%s"' % args.malloc_include) not in files["hdr_histogram.c"]: + sys.exit("cannot find the HDR_MALLOC_INCLUDE default in src/hdr_histogram.c") + for h in headers: + files[h] = public_header(h, args.include_prefix) + if not args.malloc_include: + with open(os.path.join(ROOT, "src", MALLOC_HEADER)) as f: + files[MALLOC_HEADER] = f.read() dst = os.path.abspath(args.output) os.makedirs(dst, exist_ok=True) diff --git a/script/check-amalgamation.sh b/script/check-amalgamation.sh new file mode 100755 index 0000000..7ed8305 --- /dev/null +++ b/script/check-amalgamation.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# Check the optional amalgamation variants (see docs/AMALGAMATION.md): +# --with-log the log codec files, each with its private headers inlined +# --include-prefix a tree that keeps the public headers in a hdr/ directory +# +# Run from the repository root. Needs a C compiler, python3 and zlib. +# Everything is written under the directory given as $1 (default: a temp dir). +set -eu + +CC=${CC:-cc} +WORK=${1:-$(mktemp -d)} +mkdir -p "$WORK" +SRC_FLAGS="-Iinclude -Isrc" +CFLAGS="-std=c99 -Wall -Wextra -Wmissing-prototypes -Os -D_GNU_SOURCE" + +same_tu() { # $1 = file name, $2 = amalgamated directory + $CC -E -P $SRC_FLAGS "src/$1" | grep -v '^[[:space:]]*$' > "$WORK/orig.txt" + $CC -E -P -I"$2" "$2/$1" | grep -v '^[[:space:]]*$' > "$WORK/amal.txt" + cmp "$WORK/orig.txt" "$WORK/amal.txt" + echo " $1: preprocesses to the same translation unit as src/$1" +} + +echo "== --with-log" +python3 script/amalgamate.py --output "$WORK/log" --with-log +for f in hdr_histogram.c hdr_histogram_log.c hdr_encoding.c hdr_time.c; do + same_tu "$f" "$WORK/log" + $CC $CFLAGS -Werror -I"$WORK/log" -c "$WORK/log/$f" -o "$WORK/log/$f.o" +done +# no private header, and no include, may be left behind +if grep -nE '#[[:space:]]*include[[:space:]]*[<"](hdr/|hdr_tests|hdr_endian|hdr_encoding|hdr_atomic|hdr_histogram_internal)' \ + "$WORK"/log/*.c "$WORK"/log/*.h; then + echo "ERROR: unresolved private or hdr/ include in the --with-log output" >&2 + exit 1 +fi + +cat > "$WORK/roundtrip.c" <<'EOF' +#include "hdr_histogram.h" +#include "hdr_histogram_log.h" +#include +#include +int main(void) +{ + struct hdr_histogram *h = NULL, *d = NULL; + char *enc = NULL; + int i, ok; + hdr_timespec ts; + if (hdr_init(1, 3600000000LL, 3, &h)) return 1; + for (i = 1; i <= 50000; i++) hdr_record_value(h, (int64_t) i * 37 % 100000 + 1); + if (hdr_log_encode(h, &enc)) return 2; + if (hdr_log_decode(&d, enc, strlen(enc))) return 3; + hdr_gettime(&ts); + ok = h->total_count == d->total_count && + hdr_value_at_percentile(h, 99) == hdr_value_at_percentile(d, 99) && ts.tv_sec > 0; + hdr_close(h); hdr_close(d); free(enc); + return ok ? 0 : 4; +} +EOF +$CC $CFLAGS -I"$WORK/log" "$WORK/roundtrip.c" "$WORK/log/hdr_histogram.c" "$WORK/log/hdr_histogram_log.c" \ + "$WORK/log/hdr_encoding.c" "$WORK/log/hdr_time.c" -lz -lm -o "$WORK/roundtrip" +"$WORK/roundtrip" +echo " log codec round trip through the amalgamated files: ok" + +echo "== --include-prefix hdr/ (public headers kept in a hdr/ directory)" +python3 script/amalgamate.py --output "$WORK/gen" --include-prefix hdr/ +mkdir -p "$WORK/tree/src" "$WORK/tree/include/hdr" +cp "$WORK/gen/hdr_histogram.c" "$WORK/gen/hdr_malloc.h" "$WORK/tree/src/" +cp "$WORK/gen/hdr_histogram.h" "$WORK/tree/include/hdr/" +cmp "$WORK/tree/include/hdr/hdr_histogram.h" include/hdr/hdr_histogram.h +echo " hdr_histogram.h is byte-identical to include/hdr/hdr_histogram.h" +$CC $CFLAGS -Werror -I"$WORK/tree/src" -I"$WORK/tree/include" -c "$WORK/tree/src/hdr_histogram.c" -o "$WORK/tree/hdr_histogram.o" +echo " compiles with -Isrc -Iinclude" + +echo "ok" From baef7f1e442ab090f20c9b82dfbc759211479d42 Mon Sep 17 00:00:00 2001 From: fcostaoliveira Date: Fri, 2 Oct 2026 11:50:28 +0100 Subject: [PATCH 2/3] build: check --with-log together with --malloc-include (core and codec use the caller's allocator) Co-Authored-By: Claude Sonnet 5.5 --- script/check-amalgamation.sh | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/script/check-amalgamation.sh b/script/check-amalgamation.sh index 7ed8305..7a108d4 100755 --- a/script/check-amalgamation.sh +++ b/script/check-amalgamation.sh @@ -60,6 +60,50 @@ $CC $CFLAGS -I"$WORK/log" "$WORK/roundtrip.c" "$WORK/log/hdr_histogram.c" "$WORK "$WORK/roundtrip" echo " log codec round trip through the amalgamated files: ok" +echo "== --with-log --malloc-include (the core and the codec both use the caller's allocator)" +python3 script/amalgamate.py --output "$WORK/alloc" --with-log --malloc-include my_alloc.h +test ! -e "$WORK/alloc/hdr_malloc.h" +if grep -l 'hdr_malloc\.h' "$WORK"/alloc/*; then + echo "ERROR: hdr_malloc.h is still referenced" >&2 + exit 1 +fi +cat > "$WORK/alloc/my_alloc.h" <<'EOF' +#include +extern int my_allocs; +static inline void *my_m(size_t n) { my_allocs++; return malloc(n); } +static inline void *my_c(size_t a, size_t b) { my_allocs++; return calloc(a, b); } +#define hdr_malloc my_m +#define hdr_calloc my_c +#define hdr_realloc realloc +#define hdr_free free +EOF +cat > "$WORK/alloc/t.c" <<'EOF' +#include "hdr_histogram.h" +#include "hdr_histogram_log.h" +#include +#include +int my_allocs; +int main(void) +{ + struct hdr_histogram *h = NULL, *d = NULL; + char *e = NULL; + int a0, a1, a2; + hdr_init(1, 1000000, 3, &h); + hdr_record_value(h, 500); + a0 = my_allocs; + hdr_log_encode(h, &e); + a1 = my_allocs; + hdr_log_decode(&d, e, strlen(e)); + a2 = my_allocs; + hdr_close(h); hdr_close(d); free(e); + return (a0 > 0 && a1 > a0 && a2 > a1) ? 0 : 1; +} +EOF +$CC $CFLAGS -I"$WORK/alloc" "$WORK/alloc/t.c" "$WORK/alloc/hdr_histogram.c" "$WORK/alloc/hdr_histogram_log.c" \ + "$WORK/alloc/hdr_encoding.c" "$WORK/alloc/hdr_time.c" -lz -lm -o "$WORK/alloc/t" +"$WORK/alloc/t" +echo " init, encode and decode all allocate through my_alloc.h" + echo "== --include-prefix hdr/ (public headers kept in a hdr/ directory)" python3 script/amalgamate.py --output "$WORK/gen" --include-prefix hdr/ mkdir -p "$WORK/tree/src" "$WORK/tree/include/hdr" From a83b484026ac25fa5eb48b89b8f1ab3c3b3aae20 Mon Sep 17 00:00:00 2001 From: fcostaoliveira Date: Fri, 2 Oct 2026 15:30:41 +0100 Subject: [PATCH 3/3] docs: mention --with-log and --include-prefix in the README Co-Authored-By: Claude Sonnet 5.5 --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8869602..aeb777e 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,10 @@ Features unlikely to be implemented Embedding the library (minimal static core without zlib/threads, custom allocator, disabling the AVX2 scan): see [docs/EMBEDDING.md](docs/EMBEDDING.md). -To vendor just the core as a single `hdr_histogram.c` and `hdr_histogram.h` instead, an -amalgamated copy is attached to each release and generated by `script/amalgamate.py`; see +To vendor the library as a few generated files instead of the whole tree, an amalgamation is +attached to each release and generated by `script/amalgamate.py`: the core on its own +(`hdr_histogram.c` and `hdr_histogram.h`), or with the log codec (`--with-log`), and for trees +that keep the headers in a `hdr/` directory (`--include-prefix hdr/`); see [docs/AMALGAMATION.md](docs/AMALGAMATION.md). # Simple Tutorial