run_all now sets a deterministic outputs and copies things where needed

configs/plots/composability.toml

@@ -1,18 +1,18 @@
 # CSV selections for plot_composability.jl combined.
-# Relative CSV paths are resolved under results_root; override it with
-# --results-root=DIR. Relative results_root is relative to the repository.
+# CSV paths are relative to results_root (override with --results-root=DIR),
+# which is relative to results/paper.
 # Figure sizing and fonts remain in figures.toml.
-results_root = "results-brev/composability"
+results_root = "composability"
 
 [krylov]
-cuda = "KRYLOV_FINAL/single_dagger_cuda/krylov/results.csv"
-cunumeric-single = "KRYLOV_FINAL/single_cunumeric/krylov/results.csv"
-cunumeric-multi = "KRYLOV_FINAL/multi_cunumeric/results.csv"
-dagger-single = "KRYLOV_FINAL/single_dagger_cuda/krylov/results.csv"
-dagger-multi = "KRYLOV_FINAL/multi_dagger/results.csv"
+cuda = "single/krylov/results.csv"
+cunumeric-single = "single/krylov/results.csv"
+cunumeric-multi = "multi/krylov/results.csv"
+dagger-single = "single/krylov/results.csv"
+dagger-multi = "multi/krylov/results.csv"
 
 [ode]
-cuda = "ODE_FINAL/single/results.csv"
-cunumeric-single = "ODE_FINAL/single/results.csv"
-cunumeric-multi = "ODE_FINAL/multi/results.csv"
-dagger-single = "ODE_FINAL/single/results.csv"
+cuda = "single/ordinarydiffeq/results.csv"
+cunumeric-single = "single/ordinarydiffeq/results.csv"
+cunumeric-multi = "multi/ordinarydiffeq/results.csv"
+dagger-single = "single/ordinarydiffeq/results.csv"

configs/plots/figures.toml

@@ -13,7 +13,7 @@ legend_scale = 1.0
 
 [[figure]]   # Gray-Scott @accelerate forms
 config = "configs/multi_gpu/grayscott_forms.toml"
-results = "results-brev/20261001-042330-0ADaZK"
+results = "grayscott_forms"
 fusion = ["on", "off"]
 # Grids left to the GC per step (function 0, let 0, begin 4, expr 6, plain 22)
 # × ~3.14 GB per grid per GPU (weak scaling keeps it fixed).
@@ -24,15 +24,15 @@ log = ["time"]   # forms span ~12 ms to ~840 ms
 
 [[figure]]   # Gray-Scott: cuNumeric (fused/unfused), cuPyNumeric, ImplicitGlobalGrid
 config = "configs/multi_gpu/grayscott.toml"
-results = "results-brev/grayscott-compare"
+results = "grayscott_compare"
 fusion = "both"
 name = "grayscott_compare"
 metrics = ["throughput", "time"]
 title = ""
 
 [[figure]]   # Monte Carlo: fused mapreduce vs broadcast + sum (naive), fused and unfused
 config = "configs/multi_gpu/montecarlo_forms.toml"
-results = "results-brev/20261005-224227-evLB1b"
+results = "montecarlo_forms"
 fusion = "both"
 name = "montecarlo_forms"
 metrics = ["throughput", "time"]
@@ -44,6 +44,6 @@ legend_columns = 2
 panel_size = [400, 245]   # shorter: less empty space between the line groups
 ylabel = "Time/step (ms)"
 panels = [
-    { config = "configs/multi_gpu/grayscott.toml", results = "results-brev/grayscott-compare", title = "Gray-Scott", log = true },
-    { config = "configs/multi_gpu/montecarlo_forms.toml", results = "results-brev/20261005-224227-evLB1b", title = "Monte Carlo", hide = ["PIE.jl (naive)"], relabel = { "PIE.jl (naive, unfused)" = "PIE.jl (unfused)" } },
+    { config = "configs/multi_gpu/grayscott.toml", results = "grayscott_compare", title = "Gray-Scott", log = true },
+    { config = "configs/multi_gpu/montecarlo_forms.toml", results = "montecarlo_forms", title = "Monte Carlo", hide = ["PIE.jl (naive)"], relabel = { "PIE.jl (naive, unfused)" = "PIE.jl (unfused)" } },
 ]

configs/plots/grid.toml

@@ -15,31 +15,31 @@ legend_columns = 2
 
 [[panel]]
 benchmark = "gemm"
-results = "results-brev/20261001-023020-EgnbwV"
+results = "gemm"
 note = { text = "JACC omitted:\nno BLAS-3 GEMM", model = "JACC.jl" }
 
 [[panel]]
 benchmark = "montecarlo"
-results = "results-brev/20261001-025503-B66XYH"
+results = "montecarlo"
 
 [[panel]]
 benchmark = "grayscott"
-results = "results-brev/20260930-231831-1QsvHo"
+results = "grayscott"
 
 [[panel]]
 benchmark = "cg"
-results = "results-brev/20260930-232808-LygubK"
+results = "cg"
 
 [[panel]]
 benchmark = "nas_ep"
-results = "results-brev/20260930-234926-SAIP3B"
+results = "nas_ep"
 log = true
 
 [[panel]]
 benchmark = "nas_ft"
-results = "results-brev/20261001-001153-TBPDeB"
+results = "nas_ft"
 split = { time = 350 }   # Dagger at 2+ GPUs
 
 [[panel]]
 benchmark = "nas_mg"
-results = "results-brev/20261001-003034-E0lZWa"
+results = "nas_mg"

configs/plots/grid_strong.toml

@@ -10,9 +10,9 @@ font_size = 6
 
 [[panel]]
 benchmark = "nas_ep"
-results = "results-brev/20261001-073858-GgV9Iq"
+results = "nas_ep_strong"
 log = true
 
 [[panel]]
 benchmark = "nas_ft"
-results = "results-brev/20261001-075222-pAw6qj"
+results = "nas_ft_strong"

configs/plots/sweeps.toml

@@ -15,14 +15,14 @@ font_size = 6
 [[panel]]
 benchmark = "grayscott"
 members = ["grayscott_function_accelerated"]
-results = "results-brev/20261005-235912-5SOKy7"
+results = "sweep_grayscott"
 xlabel = "Grid size N (N×N)"
 labels = { "PIE.jl (function accelerated)" = "PIE.jl", "PIE.jl (function accelerated, unfused)" = "PIE.jl (unfused)" }
 
 [[panel]]
 benchmark = "montecarlo"
 members = ["montecarlo", "montecarlo_naive"]
-results = "results-brev/20261006-001716-XNYg0g"
+results = "sweep_montecarlo"
 xlabel = "Samples"
 hide = ["PIE.jl (naive)"]
 labels = { "PIE.jl (naive, unfused)" = "PIE.jl (unfused)" }

configs/plots/tunes.toml

@@ -1,6 +1,6 @@
 # Dagger blocks_per_gpu sensitivity from the tune sweep (plot_tunes.jl).
 
-tunes = "results-brev/tunes"
+tunes = "tunes"
 # benchmarks = ["gemm", "montecarlo", "grayscott", "cg", "nas_ep", "nas_ft", "nas_mg"]
 benchmarks = ["cg", "nas_ep"]
 

plot_all.sh

@@ -1,13 +1,9 @@
 #!/bin/bash
-# Every paper figure: each configs/plots/grid*.toml -> plots/<name>/, then
-# figures, complexity, tunes and size sweeps, and last the composability figure
-# (its CSVs are under configs/plots/composability.toml's results_root).
+# Every paper figure, from results/paper (written by run_all.sh).
 set -euo pipefail
 cd "$(dirname "$0")"
 julia=${JULIA:-${CUNUMERIC_BENCH_JULIA:-julia}}
-for config in configs/plots/grid*.toml; do
-    "$julia" --project=. plotter/plot_grid.jl --config="$config" --out="plots/$(basename "$config" .toml)"
-done
+"$julia" --project=. plotter/plot_grid.jl --config=configs/plots/grid.toml --out=plots/grid
 "$julia" --project=. plotter/plot_figures.jl
 "$julia" --project=. plotter/plot_complexity.jl
 "$julia" --project=. plotter/plot_tunes.jl

plotter/plot_composability.jl

@@ -253,8 +253,7 @@ function combined_main(args)
     end
     format in ("pdf", "png", "svg") || error("--format must be pdf, png, or svg")
     raw = TOML.parsefile(config)
-    root = something(root, get(raw, "results_root", "results/composability"))
-    root = isabspath(root) ? root : joinpath(BENCH_ROOT, root)
+    root = results_path(something(root, get(raw, "results_root", "composability")))
     workloads = map(("krylov", "ode")) do workload
         sources = raw[workload]
         all(k -> k in INPUT_KEYS, keys(sources)) || error("unknown input key in [$workload]")

plotter/plot_grid.jl

@@ -62,9 +62,12 @@ function parse_grid_args(args)
     return (; config, out_dir)
 end
 
+# Relative results paths are under results/paper, where run_all.sh writes.
+results_path(path) = isabspath(path) ? path : joinpath(BENCH_ROOT, "results", "paper", path)
+
 # A run directory holds its CSVs under <T>/; accept either level.
 function csv_dir(path)
-    path = isabspath(path) ? path : joinpath(BENCH_ROOT, path)
+    path = results_path(path)
     isdir(path) || error("results directory not found: $path")
     any(endswith(".csv"), readdir(path)) && return path
     subdirs = filter(d -> any(endswith(".csv"), readdir(d)),

plotter/plot_tunes.jl

@@ -102,8 +102,8 @@ end
 function tunes_main(args=ARGS)
     cfg = parse_tunes_args(args)
     raw = TOML.parsefile(cfg.config)
-    dir = get(raw, "tunes", "results-brev/tunes")
-    speedups = tune_speedups(read_dagger_tunes(isabspath(dir) ? dir : joinpath(BENCH_ROOT, dir)))
+    dir = get(raw, "tunes", "tunes")
+    speedups = tune_speedups(read_dagger_tunes(results_path(dir)))
     isempty(speedups) && error("no tune results with a 1 block/GPU baseline in $dir")
     names = filter(n -> haskey(speedups, n), string.(get(raw, "benchmarks", sort(collect(keys(speedups))))))
     columns = min(get(raw, "columns", 2), length(names))

run_all.sh

@@ -1,64 +1,81 @@
 #!/bin/bash
-# Run every paper benchmark.
+# Run every paper benchmark. Results go to results/paper/<name>, which plot_all.sh reads.
 #   ./run_all.sh [run.jl args...]   e.g. ./run_all.sh --verbose
 # Failed runs are listed at the end; the script keeps going.
 
 set -uo pipefail
 cd "$(dirname "$0")"
 
 JULIA=${JULIA:-${CUNUMERIC_BENCH_JULIA:-julia}}
+OUT=results/paper
 MULTI=configs/multi_gpu
+SINGLE=configs/single_gpu
 EXTRA=("$@")
 FAILED=()
 
+# run <name> <run.jl args...>
 run() {
+    local name=$1; shift
     echo
-    echo "==> $JULIA --project=. run.jl $* ${EXTRA[*]}"
-    "$JULIA" --project=. run.jl "$@" "${EXTRA[@]}" || FAILED+=("$*")
+    echo "==> $name: run.jl $*"
+    "$JULIA" --project=. run.jl "$@" --output=$OUT/$name "${EXTRA[@]}" || FAILED+=("$name")
 }
 
+composability() {
+    echo
+    echo "==> composability: $*"
+    "$JULIA" --project=. run_composability.jl "$@" --output=$OUT/composability \
+        --config=composability/sizes_141GB.toml || FAILED+=("composability $*")
+}
+
+# Dagger tunes first: Dagger runs read their block counts from results/tunes/.
+bash scripts/tune_dagger.sh || FAILED+=("tune_dagger")
+mkdir -p $OUT && cp -r results/tunes $OUT/
+
 # Weak scaling: the seven main benchmarks.
-run --config=$MULTI/gemm.toml
-run --config=$MULTI/montecarlo.toml
-run --config=$MULTI/grayscott.toml
-run --config=$MULTI/cg.toml
-run --config=$MULTI/nas_ep_weak.toml
-run --config=$MULTI/nas_ft_weak.toml
-run --config=$MULTI/nas_mg_weak.toml
+run gemm       --config=$MULTI/gemm.toml
+run montecarlo --config=$MULTI/montecarlo.toml
+run grayscott  --config=$MULTI/grayscott.toml
+run cg         --config=$MULTI/cg.toml
+run nas_ep     --config=$MULTI/nas_ep_weak.toml
+run nas_ft     --config=$MULTI/nas_ft_weak.toml
+run nas_mg     --config=$MULTI/nas_mg_weak.toml
+
+# Variants: Gray-Scott @accelerate forms, Monte Carlo fused vs naive,
+# and unfused Gray-Scott for the IGG comparison.
+run grayscott_forms   --config=$MULTI/grayscott_forms.toml
+run montecarlo_forms  --config=$MULTI/montecarlo_forms.toml
+run grayscott_unfused --config=$MULTI/grayscott.toml --models=cunumeric --fusion=off
 
-# Variants: Gray-Scott @accelerate forms, Monte Carlo fused vs naive.
-run --config=$MULTI/grayscott_forms.toml
-run --config=$MULTI/montecarlo_forms.toml
+# Single-GPU size sweeps.
+run sweep_grayscott  --config=$SINGLE/grayscott_forms.toml
+run sweep_montecarlo --config=$SINGLE/montecarlo_forms.toml
 
 # Strong scaling: NAS (not used in the paper).
-# run --config=$MULTI/nas_ep_strong.toml
-# run --config=$MULTI/nas_ft_strong.toml
-# run --config=$MULTI/nas_mg_strong.toml
+# run nas_ep_strong --config=$MULTI/nas_ep_strong.toml
+# run nas_ft_strong --config=$MULTI/nas_ft_strong.toml
+# run nas_mg_strong --config=$MULTI/nas_mg_strong.toml
 
 # ImplicitGlobalGrid Gray-Scott, same weak-scaling sizes as grayscott.toml.
-echo
-echo "==> bash other/implicitglobalgrid/run_weak_scaling.sh"
-bash other/implicitglobalgrid/run_weak_scaling.sh || FAILED+=("implicitglobalgrid")
-
-composability() {
-    local args=("$@" --config=${COMPOSABILITY_CONFIG:-composability/sizes_141GB.toml})
-    echo
-    echo "==> $JULIA --project=. run_composability.jl ${args[*]}"
-    "$JULIA" --project=. run_composability.jl "${args[@]}" || FAILED+=("composability ${args[*]}")
-}
-STAMP=$(date +%Y%m%d-%H%M%S)
+IGG_OUTPUT=$OUT/implicitglobalgrid bash other/implicitglobalgrid/run_weak_scaling.sh ||
+    FAILED+=("implicitglobalgrid")
 
-# Composability single GPU: OrdinaryDiffEq heat and Krylov CG (with local CG).
-composability --only=ordinarydiffeq,krylov --solvers=cg --local --mode=single \
-    --models=cuda,dagger,cunumeric --output=results/composability-single-$STAMP
+# Gray-Scott comparison figure: CSVs from three of the runs above.
+mkdir -p $OUT/grayscott_compare/Float32
+cp $OUT/grayscott/Float32/grayscott_cunumeric.csv \
+   $OUT/grayscott/Float32/grayscott_cupynumeric.csv \
+   $OUT/grayscott_unfused/Float32/grayscott_cunumeric_nofusion.csv \
+   $OUT/implicitglobalgrid/Float32/grayscott_igg.csv \
+   $OUT/grayscott_compare/Float32/ || FAILED+=("grayscott_compare")
 
-# Composability weak scaling: Krylov CG.
-composability --only=krylov --solvers=cg --mode=multi --gpus=1,2,4,8 \
-    --models=${COMPOSABILITY_MODELS:-dagger,cunumeric} --output=results/composability-multi-$STAMP
+# Composability: single GPU (ODE heat, Krylov CG), then weak scaling.
+composability --only=ordinarydiffeq,krylov --solvers=cg --local --mode=single --models=cuda,dagger,cunumeric
+composability --only=krylov --solvers=cg --mode=multi --gpus=1,2,4,8 --models=dagger,cunumeric
+composability --only=ordinarydiffeq --mode=multi --gpus=1,2,4,8 --models=cunumeric
 
 echo
 if [[ ${#FAILED[@]} -eq 0 ]]; then
-    echo "All runs finished."
+    echo "All runs finished: $OUT"
 else
     echo "Failed runs:"
     printf '  %s\n' "${FAILED[@]}"

src/parse_benchmarks.jl

@@ -65,7 +65,7 @@ end
 function declared_order(path)
     order = String[]
     for line in eachline(path)
-        header = strip(line)
+        header = strip(first(split(line, '#'; limit=2)))  # allow trailing comments
         startswith(header, "[[") && endswith(header, "]]") || continue
         name = strip(header[3:(end - 2)])
         name in order || push!(order, name) # if not in list, push to ordered list

src/runner.jl

@@ -5,6 +5,7 @@ function cli_options(args)
     only = nothing
     fusion = nothing
     models = nothing
+    output = nothing
     dry = false
     verbose = false
     positional = String[]
@@ -18,6 +19,8 @@ function cli_options(args)
             fusion = value == "both" ? [true, false] : [parse_fusion(value)]
         elseif startswith(arg, "--models=")
             models = parse_models(split(split(arg, '='; limit=2)[2], ','))
+        elseif startswith(arg, "--output=")
+            output = abspath(split(arg, '='; limit=2)[2])
         elseif arg == "--dry-run"
             dry = true
         elseif arg in ("-v", "--verbose")
@@ -28,7 +31,7 @@ function cli_options(args)
             push!(positional, arg)
         end
     end
-    return (; config, only, fusion, models, dry, verbose, positional)
+    return (; config, only, fusion, models, output, dry, verbose, positional)
 end
 
 function positional_spec(p, gs)
@@ -93,8 +96,15 @@ function execute_plan(runs, gs, opts, budget, raw; launch=run, prepare=prepare_b
     results_root=normpath(joinpath(@__DIR__, "..", "results")), preflight=preflight_models)
     preflight(runs)
     root = normpath(joinpath(@__DIR__, ".."))
-    mkpath(results_root)
-    dir = mktempdir(results_root; prefix=Dates.format(now(), "yyyymmdd-HHMMSS")*"-", cleanup=false)
+    # --output names the run directory exactly; otherwise a timestamped one.
+    output = get(opts, :output, nothing)
+    dir = if output === nothing
+        mkpath(results_root)
+        mktempdir(results_root; prefix=Dates.format(now(), "yyyymmdd-HHMMSS")*"-", cleanup=false)
+    else
+        isfile(joinpath(output, "manifest.toml")) && error("$output already holds a run")
+        mkpath(output)
+    end
     manifest = plan_manifest(runs, budget, raw)
     manifest_path = joinpath(dir, "manifest.toml")
     save_manifest() = open(io->TOML.print(io, manifest), manifest_path, "w")

test/montecarlo_launch.jl

@@ -13,7 +13,8 @@ const CONFIG = joinpath(ROOT, "configs", "multi_gpu", "montecarlo_forms.toml")
 
 @testset "Existing Monte Carlo config produces cuNumeric worker commands" begin
     gs, specs = parse_config(CONFIG; fusion_override=[true, false])
-    runs = plan_runs(specs, gs, TOML.parsefile(CONFIG), parse_plot_groups(CONFIG), 1_000_000)
+    runs = filter(r -> r.model === :cunumeric,
+        plan_runs(specs, gs, TOML.parsefile(CONFIG), parse_plot_groups(CONFIG), 1_000_000))
     @test length(runs) == 16
     @test Set(r.spec.name for r in runs) == Set(("montecarlo", "montecarlo_naive"))
     @test BENCHMARKS["montecarlo"] === MonteCarloMapReduce