From 2e9a7a024bc040c93585518b5d7364b94ffd2938 Mon Sep 17 00:00:00 2001 From: krasow Date: Sat, 10 Oct 2026 21:49:48 -0500 Subject: [PATCH] run_all now sets a deterministic outputs and copies things where needed --- configs/plots/composability.toml | 24 ++++----- configs/plots/figures.toml | 10 ++-- configs/plots/grid.toml | 14 +++--- configs/plots/grid_strong.toml | 4 +- configs/plots/sweeps.toml | 4 +- configs/plots/tunes.toml | 2 +- plot_all.sh | 8 +-- plotter/plot_composability.jl | 3 +- plotter/plot_grid.jl | 5 +- plotter/plot_tunes.jl | 4 +- run_all.sh | 85 +++++++++++++++++++------------- src/parse_benchmarks.jl | 2 +- src/runner.jl | 16 ++++-- test/montecarlo_launch.jl | 3 +- test/runtests.jl | 14 +++++- test/single_gpu.jl | 5 +- 16 files changed, 120 insertions(+), 83 deletions(-) diff --git a/configs/plots/composability.toml b/configs/plots/composability.toml index c2a9af9..202b26b 100644 --- a/configs/plots/composability.toml +++ b/configs/plots/composability.toml @@ -1,18 +1,18 @@ # CSV selections for plot_composability.jl combined. -# Relative CSV paths are resolved under results_root; override it with -# --results-root=DIR. Relative results_root is relative to the repository. +# CSV paths are relative to results_root (override with --results-root=DIR), +# which is relative to results/paper. # Figure sizing and fonts remain in figures.toml. -results_root = "results-brev/composability" +results_root = "composability" [krylov] -cuda = "KRYLOV_FINAL/single_dagger_cuda/krylov/results.csv" -cunumeric-single = "KRYLOV_FINAL/single_cunumeric/krylov/results.csv" -cunumeric-multi = "KRYLOV_FINAL/multi_cunumeric/results.csv" -dagger-single = "KRYLOV_FINAL/single_dagger_cuda/krylov/results.csv" -dagger-multi = "KRYLOV_FINAL/multi_dagger/results.csv" +cuda = "single/krylov/results.csv" +cunumeric-single = "single/krylov/results.csv" +cunumeric-multi = "multi/krylov/results.csv" +dagger-single = "single/krylov/results.csv" +dagger-multi = "multi/krylov/results.csv" [ode] -cuda = "ODE_FINAL/single/results.csv" -cunumeric-single = "ODE_FINAL/single/results.csv" -cunumeric-multi = "ODE_FINAL/multi/results.csv" -dagger-single = "ODE_FINAL/single/results.csv" +cuda = "single/ordinarydiffeq/results.csv" +cunumeric-single = "single/ordinarydiffeq/results.csv" +cunumeric-multi = "multi/ordinarydiffeq/results.csv" +dagger-single = "single/ordinarydiffeq/results.csv" diff --git a/configs/plots/figures.toml b/configs/plots/figures.toml index 7837b84..4e28428 100644 --- a/configs/plots/figures.toml +++ b/configs/plots/figures.toml @@ -13,7 +13,7 @@ legend_scale = 1.0 [[figure]] # Gray-Scott @accelerate forms config = "configs/multi_gpu/grayscott_forms.toml" -results = "results-brev/20261001-042330-0ADaZK" +results = "grayscott_forms" fusion = ["on", "off"] # Grids left to the GC per step (function 0, let 0, begin 4, expr 6, plain 22) # × ~3.14 GB per grid per GPU (weak scaling keeps it fixed). @@ -24,7 +24,7 @@ log = ["time"] # forms span ~12 ms to ~840 ms [[figure]] # Gray-Scott: cuNumeric (fused/unfused), cuPyNumeric, ImplicitGlobalGrid config = "configs/multi_gpu/grayscott.toml" -results = "results-brev/grayscott-compare" +results = "grayscott_compare" fusion = "both" name = "grayscott_compare" metrics = ["throughput", "time"] @@ -32,7 +32,7 @@ title = "" [[figure]] # Monte Carlo: fused mapreduce vs broadcast + sum (naive), fused and unfused config = "configs/multi_gpu/montecarlo_forms.toml" -results = "results-brev/20261005-224227-evLB1b" +results = "montecarlo_forms" fusion = "both" name = "montecarlo_forms" metrics = ["throughput", "time"] @@ -44,6 +44,6 @@ legend_columns = 2 panel_size = [400, 245] # shorter: less empty space between the line groups ylabel = "Time/step (ms)" panels = [ - { config = "configs/multi_gpu/grayscott.toml", results = "results-brev/grayscott-compare", title = "Gray-Scott", log = true }, - { config = "configs/multi_gpu/montecarlo_forms.toml", results = "results-brev/20261005-224227-evLB1b", title = "Monte Carlo", hide = ["PIE.jl (naive)"], relabel = { "PIE.jl (naive, unfused)" = "PIE.jl (unfused)" } }, + { config = "configs/multi_gpu/grayscott.toml", results = "grayscott_compare", title = "Gray-Scott", log = true }, + { config = "configs/multi_gpu/montecarlo_forms.toml", results = "montecarlo_forms", title = "Monte Carlo", hide = ["PIE.jl (naive)"], relabel = { "PIE.jl (naive, unfused)" = "PIE.jl (unfused)" } }, ] diff --git a/configs/plots/grid.toml b/configs/plots/grid.toml index 183636d..1af9a7c 100644 --- a/configs/plots/grid.toml +++ b/configs/plots/grid.toml @@ -15,31 +15,31 @@ legend_columns = 2 [[panel]] benchmark = "gemm" -results = "results-brev/20261001-023020-EgnbwV" +results = "gemm" note = { text = "JACC omitted:\nno BLAS-3 GEMM", model = "JACC.jl" } [[panel]] benchmark = "montecarlo" -results = "results-brev/20261001-025503-B66XYH" +results = "montecarlo" [[panel]] benchmark = "grayscott" -results = "results-brev/20260930-231831-1QsvHo" +results = "grayscott" [[panel]] benchmark = "cg" -results = "results-brev/20260930-232808-LygubK" +results = "cg" [[panel]] benchmark = "nas_ep" -results = "results-brev/20260930-234926-SAIP3B" +results = "nas_ep" log = true [[panel]] benchmark = "nas_ft" -results = "results-brev/20261001-001153-TBPDeB" +results = "nas_ft" split = { time = 350 } # Dagger at 2+ GPUs [[panel]] benchmark = "nas_mg" -results = "results-brev/20261001-003034-E0lZWa" +results = "nas_mg" diff --git a/configs/plots/grid_strong.toml b/configs/plots/grid_strong.toml index e774966..5ffb941 100644 --- a/configs/plots/grid_strong.toml +++ b/configs/plots/grid_strong.toml @@ -10,9 +10,9 @@ font_size = 6 [[panel]] benchmark = "nas_ep" -results = "results-brev/20261001-073858-GgV9Iq" +results = "nas_ep_strong" log = true [[panel]] benchmark = "nas_ft" -results = "results-brev/20261001-075222-pAw6qj" +results = "nas_ft_strong" diff --git a/configs/plots/sweeps.toml b/configs/plots/sweeps.toml index 6372a23..abc02dc 100644 --- a/configs/plots/sweeps.toml +++ b/configs/plots/sweeps.toml @@ -15,14 +15,14 @@ font_size = 6 [[panel]] benchmark = "grayscott" members = ["grayscott_function_accelerated"] -results = "results-brev/20261005-235912-5SOKy7" +results = "sweep_grayscott" xlabel = "Grid size N (N×N)" labels = { "PIE.jl (function accelerated)" = "PIE.jl", "PIE.jl (function accelerated, unfused)" = "PIE.jl (unfused)" } [[panel]] benchmark = "montecarlo" members = ["montecarlo", "montecarlo_naive"] -results = "results-brev/20261006-001716-XNYg0g" +results = "sweep_montecarlo" xlabel = "Samples" hide = ["PIE.jl (naive)"] labels = { "PIE.jl (naive, unfused)" = "PIE.jl (unfused)" } diff --git a/configs/plots/tunes.toml b/configs/plots/tunes.toml index bc18632..a9355f0 100644 --- a/configs/plots/tunes.toml +++ b/configs/plots/tunes.toml @@ -1,6 +1,6 @@ # Dagger blocks_per_gpu sensitivity from the tune sweep (plot_tunes.jl). -tunes = "results-brev/tunes" +tunes = "tunes" # benchmarks = ["gemm", "montecarlo", "grayscott", "cg", "nas_ep", "nas_ft", "nas_mg"] benchmarks = ["cg", "nas_ep"] diff --git a/plot_all.sh b/plot_all.sh index 4f40c41..7729ef3 100755 --- a/plot_all.sh +++ b/plot_all.sh @@ -1,13 +1,9 @@ #!/bin/bash -# Every paper figure: each configs/plots/grid*.toml -> plots//, then -# figures, complexity, tunes and size sweeps, and last the composability figure -# (its CSVs are under configs/plots/composability.toml's results_root). +# Every paper figure, from results/paper (written by run_all.sh). set -euo pipefail cd "$(dirname "$0")" julia=${JULIA:-${CUNUMERIC_BENCH_JULIA:-julia}} -for config in configs/plots/grid*.toml; do - "$julia" --project=. plotter/plot_grid.jl --config="$config" --out="plots/$(basename "$config" .toml)" -done +"$julia" --project=. plotter/plot_grid.jl --config=configs/plots/grid.toml --out=plots/grid "$julia" --project=. plotter/plot_figures.jl "$julia" --project=. plotter/plot_complexity.jl "$julia" --project=. plotter/plot_tunes.jl diff --git a/plotter/plot_composability.jl b/plotter/plot_composability.jl index 8ef5337..2294f10 100644 --- a/plotter/plot_composability.jl +++ b/plotter/plot_composability.jl @@ -253,8 +253,7 @@ function combined_main(args) end format in ("pdf", "png", "svg") || error("--format must be pdf, png, or svg") raw = TOML.parsefile(config) - root = something(root, get(raw, "results_root", "results/composability")) - root = isabspath(root) ? root : joinpath(BENCH_ROOT, root) + root = results_path(something(root, get(raw, "results_root", "composability"))) workloads = map(("krylov", "ode")) do workload sources = raw[workload] all(k -> k in INPUT_KEYS, keys(sources)) || error("unknown input key in [$workload]") diff --git a/plotter/plot_grid.jl b/plotter/plot_grid.jl index d35a7a7..731da8a 100644 --- a/plotter/plot_grid.jl +++ b/plotter/plot_grid.jl @@ -62,9 +62,12 @@ function parse_grid_args(args) return (; config, out_dir) end +# Relative results paths are under results/paper, where run_all.sh writes. +results_path(path) = isabspath(path) ? path : joinpath(BENCH_ROOT, "results", "paper", path) + # A run directory holds its CSVs under /; accept either level. function csv_dir(path) - path = isabspath(path) ? path : joinpath(BENCH_ROOT, path) + path = results_path(path) isdir(path) || error("results directory not found: $path") any(endswith(".csv"), readdir(path)) && return path subdirs = filter(d -> any(endswith(".csv"), readdir(d)), diff --git a/plotter/plot_tunes.jl b/plotter/plot_tunes.jl index aaa3bc3..673de5b 100644 --- a/plotter/plot_tunes.jl +++ b/plotter/plot_tunes.jl @@ -102,8 +102,8 @@ end function tunes_main(args=ARGS) cfg = parse_tunes_args(args) raw = TOML.parsefile(cfg.config) - dir = get(raw, "tunes", "results-brev/tunes") - speedups = tune_speedups(read_dagger_tunes(isabspath(dir) ? dir : joinpath(BENCH_ROOT, dir))) + dir = get(raw, "tunes", "tunes") + speedups = tune_speedups(read_dagger_tunes(results_path(dir))) isempty(speedups) && error("no tune results with a 1 block/GPU baseline in $dir") names = filter(n -> haskey(speedups, n), string.(get(raw, "benchmarks", sort(collect(keys(speedups)))))) columns = min(get(raw, "columns", 2), length(names)) diff --git a/run_all.sh b/run_all.sh index 5c94e33..708ce09 100755 --- a/run_all.sh +++ b/run_all.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Run every paper benchmark. +# Run every paper benchmark. Results go to results/paper/, which plot_all.sh reads. # ./run_all.sh [run.jl args...] e.g. ./run_all.sh --verbose # Failed runs are listed at the end; the script keeps going. @@ -7,58 +7,75 @@ set -uo pipefail cd "$(dirname "$0")" JULIA=${JULIA:-${CUNUMERIC_BENCH_JULIA:-julia}} +OUT=results/paper MULTI=configs/multi_gpu +SINGLE=configs/single_gpu EXTRA=("$@") FAILED=() +# run run() { + local name=$1; shift echo - echo "==> $JULIA --project=. run.jl $* ${EXTRA[*]}" - "$JULIA" --project=. run.jl "$@" "${EXTRA[@]}" || FAILED+=("$*") + echo "==> $name: run.jl $*" + "$JULIA" --project=. run.jl "$@" --output=$OUT/$name "${EXTRA[@]}" || FAILED+=("$name") } +composability() { + echo + echo "==> composability: $*" + "$JULIA" --project=. run_composability.jl "$@" --output=$OUT/composability \ + --config=composability/sizes_141GB.toml || FAILED+=("composability $*") +} + +# Dagger tunes first: Dagger runs read their block counts from results/tunes/. +bash scripts/tune_dagger.sh || FAILED+=("tune_dagger") +mkdir -p $OUT && cp -r results/tunes $OUT/ + # Weak scaling: the seven main benchmarks. -run --config=$MULTI/gemm.toml -run --config=$MULTI/montecarlo.toml -run --config=$MULTI/grayscott.toml -run --config=$MULTI/cg.toml -run --config=$MULTI/nas_ep_weak.toml -run --config=$MULTI/nas_ft_weak.toml -run --config=$MULTI/nas_mg_weak.toml +run gemm --config=$MULTI/gemm.toml +run montecarlo --config=$MULTI/montecarlo.toml +run grayscott --config=$MULTI/grayscott.toml +run cg --config=$MULTI/cg.toml +run nas_ep --config=$MULTI/nas_ep_weak.toml +run nas_ft --config=$MULTI/nas_ft_weak.toml +run nas_mg --config=$MULTI/nas_mg_weak.toml + +# Variants: Gray-Scott @accelerate forms, Monte Carlo fused vs naive, +# and unfused Gray-Scott for the IGG comparison. +run grayscott_forms --config=$MULTI/grayscott_forms.toml +run montecarlo_forms --config=$MULTI/montecarlo_forms.toml +run grayscott_unfused --config=$MULTI/grayscott.toml --models=cunumeric --fusion=off -# Variants: Gray-Scott @accelerate forms, Monte Carlo fused vs naive. -run --config=$MULTI/grayscott_forms.toml -run --config=$MULTI/montecarlo_forms.toml +# Single-GPU size sweeps. +run sweep_grayscott --config=$SINGLE/grayscott_forms.toml +run sweep_montecarlo --config=$SINGLE/montecarlo_forms.toml # Strong scaling: NAS (not used in the paper). -# run --config=$MULTI/nas_ep_strong.toml -# run --config=$MULTI/nas_ft_strong.toml -# run --config=$MULTI/nas_mg_strong.toml +# run nas_ep_strong --config=$MULTI/nas_ep_strong.toml +# run nas_ft_strong --config=$MULTI/nas_ft_strong.toml +# run nas_mg_strong --config=$MULTI/nas_mg_strong.toml # ImplicitGlobalGrid Gray-Scott, same weak-scaling sizes as grayscott.toml. -echo -echo "==> bash other/implicitglobalgrid/run_weak_scaling.sh" -bash other/implicitglobalgrid/run_weak_scaling.sh || FAILED+=("implicitglobalgrid") - -composability() { - local args=("$@" --config=${COMPOSABILITY_CONFIG:-composability/sizes_141GB.toml}) - echo - echo "==> $JULIA --project=. run_composability.jl ${args[*]}" - "$JULIA" --project=. run_composability.jl "${args[@]}" || FAILED+=("composability ${args[*]}") -} -STAMP=$(date +%Y%m%d-%H%M%S) +IGG_OUTPUT=$OUT/implicitglobalgrid bash other/implicitglobalgrid/run_weak_scaling.sh || + FAILED+=("implicitglobalgrid") -# Composability single GPU: OrdinaryDiffEq heat and Krylov CG (with local CG). -composability --only=ordinarydiffeq,krylov --solvers=cg --local --mode=single \ - --models=cuda,dagger,cunumeric --output=results/composability-single-$STAMP +# Gray-Scott comparison figure: CSVs from three of the runs above. +mkdir -p $OUT/grayscott_compare/Float32 +cp $OUT/grayscott/Float32/grayscott_cunumeric.csv \ + $OUT/grayscott/Float32/grayscott_cupynumeric.csv \ + $OUT/grayscott_unfused/Float32/grayscott_cunumeric_nofusion.csv \ + $OUT/implicitglobalgrid/Float32/grayscott_igg.csv \ + $OUT/grayscott_compare/Float32/ || FAILED+=("grayscott_compare") -# Composability weak scaling: Krylov CG. -composability --only=krylov --solvers=cg --mode=multi --gpus=1,2,4,8 \ - --models=${COMPOSABILITY_MODELS:-dagger,cunumeric} --output=results/composability-multi-$STAMP +# Composability: single GPU (ODE heat, Krylov CG), then weak scaling. +composability --only=ordinarydiffeq,krylov --solvers=cg --local --mode=single --models=cuda,dagger,cunumeric +composability --only=krylov --solvers=cg --mode=multi --gpus=1,2,4,8 --models=dagger,cunumeric +composability --only=ordinarydiffeq --mode=multi --gpus=1,2,4,8 --models=cunumeric echo if [[ ${#FAILED[@]} -eq 0 ]]; then - echo "All runs finished." + echo "All runs finished: $OUT" else echo "Failed runs:" printf ' %s\n' "${FAILED[@]}" diff --git a/src/parse_benchmarks.jl b/src/parse_benchmarks.jl index e0cfea9..bb5f3d2 100644 --- a/src/parse_benchmarks.jl +++ b/src/parse_benchmarks.jl @@ -65,7 +65,7 @@ end function declared_order(path) order = String[] for line in eachline(path) - header = strip(line) + header = strip(first(split(line, '#'; limit=2))) # allow trailing comments startswith(header, "[[") && endswith(header, "]]") || continue name = strip(header[3:(end - 2)]) name in order || push!(order, name) # if not in list, push to ordered list diff --git a/src/runner.jl b/src/runner.jl index 9f17261..63abc6b 100644 --- a/src/runner.jl +++ b/src/runner.jl @@ -5,6 +5,7 @@ function cli_options(args) only = nothing fusion = nothing models = nothing + output = nothing dry = false verbose = false positional = String[] @@ -18,6 +19,8 @@ function cli_options(args) fusion = value == "both" ? [true, false] : [parse_fusion(value)] elseif startswith(arg, "--models=") models = parse_models(split(split(arg, '='; limit=2)[2], ',')) + elseif startswith(arg, "--output=") + output = abspath(split(arg, '='; limit=2)[2]) elseif arg == "--dry-run" dry = true elseif arg in ("-v", "--verbose") @@ -28,7 +31,7 @@ function cli_options(args) push!(positional, arg) end end - return (; config, only, fusion, models, dry, verbose, positional) + return (; config, only, fusion, models, output, dry, verbose, positional) end function positional_spec(p, gs) @@ -93,8 +96,15 @@ function execute_plan(runs, gs, opts, budget, raw; launch=run, prepare=prepare_b results_root=normpath(joinpath(@__DIR__, "..", "results")), preflight=preflight_models) preflight(runs) root = normpath(joinpath(@__DIR__, "..")) - mkpath(results_root) - dir = mktempdir(results_root; prefix=Dates.format(now(), "yyyymmdd-HHMMSS")*"-", cleanup=false) + # --output names the run directory exactly; otherwise a timestamped one. + output = get(opts, :output, nothing) + dir = if output === nothing + mkpath(results_root) + mktempdir(results_root; prefix=Dates.format(now(), "yyyymmdd-HHMMSS")*"-", cleanup=false) + else + isfile(joinpath(output, "manifest.toml")) && error("$output already holds a run") + mkpath(output) + end manifest = plan_manifest(runs, budget, raw) manifest_path = joinpath(dir, "manifest.toml") save_manifest() = open(io->TOML.print(io, manifest), manifest_path, "w") diff --git a/test/montecarlo_launch.jl b/test/montecarlo_launch.jl index 2dd751b..834de7e 100644 --- a/test/montecarlo_launch.jl +++ b/test/montecarlo_launch.jl @@ -13,7 +13,8 @@ const CONFIG = joinpath(ROOT, "configs", "multi_gpu", "montecarlo_forms.toml") @testset "Existing Monte Carlo config produces cuNumeric worker commands" begin gs, specs = parse_config(CONFIG; fusion_override=[true, false]) - runs = plan_runs(specs, gs, TOML.parsefile(CONFIG), parse_plot_groups(CONFIG), 1_000_000) + runs = filter(r -> r.model === :cunumeric, + plan_runs(specs, gs, TOML.parsefile(CONFIG), parse_plot_groups(CONFIG), 1_000_000)) @test length(runs) == 16 @test Set(r.spec.name for r in runs) == Set(("montecarlo", "montecarlo_naive")) @test BENCHMARKS["montecarlo"] === MonteCarloMapReduce diff --git a/test/runtests.jl b/test/runtests.jl index 2f33285..fa3d172 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -146,8 +146,9 @@ end mcspecs, mcgs, TOML.parsefile(MONTECARLO_CONFIG), parse_plot_groups(MONTECARLO_CONFIG), 1_000_000, ) - @test Set(r.spec.name for r in mcruns) == Set(("montecarlo", "montecarlo_naive")) - @test all(r.model == :cunumeric for r in mcruns) + # cuPyNumeric has only the mapreduce variant. + @test Set((r.spec.name, r.model) for r in mcruns) == Set(( + ("montecarlo", :cunumeric), ("montecarlo", :cupynumeric), ("montecarlo_naive", :cunumeric))) @test all( length(unique((r.N, r.M) for r in mcruns if r.spec.gpus == p)) == 1 for p in (1, 2, 4, 8) @@ -483,6 +484,15 @@ end @test manifest["runs"][1]["status"]=="failed" @test manifest["runs"][2]["status"]=="complete" end + mktempdir() do dir + out = joinpath(dir, "paper", "montecarlo") + opts = cli_options(["--only=montecarlo", "--output=$out"]) + go(opts) = execute_plan(runs, gs, opts, 1_000_000, RAW; launch=cmd->nothing, + prepare=(f, v)->nothing, results_root=dir, preflight=runs->nothing) + @test go(opts) == 0 + @test isfile(joinpath(out, "manifest.toml")) + @test_throws ErrorException go(opts) # never reuse a run directory + end end include("cg.jl") diff --git a/test/single_gpu.jl b/test/single_gpu.jl index 0028f94..e44ad98 100644 --- a/test/single_gpu.jl +++ b/test/single_gpu.jl @@ -114,7 +114,7 @@ end @testset "Single-GPU example configs and execution" begin for (file, series_per_size, n_sizes, reference) in ( - ("grayscott_forms.toml", 3, 8, "grayscott"), + ("grayscott_forms.toml", 4, 8, "grayscott"), ("montecarlo_forms.toml", 5, 9, "montecarlo"), ) path = joinpath(@__DIR__, "..", "configs", "single_gpu", file) @@ -147,7 +147,8 @@ end Set(("grayscott_function_accelerated", "grayscott")) @test all(r.model in (:cudajl, :cupynumeric) for r in runs if r.spec.name == "grayscott") @test !any(r.spec.name == "grayscott_plain" for r in runs) - @test all(r.spec.fusion for r in runs if r.spec.name == "grayscott_function_accelerated") + @test Set(r.spec.fusion for r in runs if r.spec.name == "grayscott_function_accelerated") == + Set((true, false)) else @test all(r.spec.fusion for r in runs if r.spec.name == "montecarlo") @test Set(r.spec.fusion for r in runs if r.spec.name == "montecarlo_naive") ==