From 533875f36d8ab49e82c9f684820612674057f13b Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Fri, 4 Sep 2026 13:30:11 +0200 Subject: [PATCH 01/10] Add reproducible VLM evaluation profiles Signed-off-by: Johannes Rausch --- examples/puzzletron/Dockerfile | 5 +- examples/puzzletron/ci_environment.json | 5 +- .../docs/vlm_checkpoint_evaluation.md | 208 +- examples/puzzletron/evaluation/checkpoint.py | 34 +- .../puzzletron/evaluation/vlm/contracts.py | 322 ++- .../puzzletron/evaluation/vlm/evaluator.py | 133 +- examples/puzzletron/evaluation/vlm/model.py | 38 +- .../puzzletron/evaluation/vlm/post_mip.py | 187 +- .../puzzletron/evaluation/vlm/preflight.py | 180 +- .../vlm/preparation/benchmark_data.py | 487 +++- .../vlm/profiles/core3-full-native-v1.json | 76 + .../vlm/profiles/core3-full-vllm-v1.json | 11 + .../evaluation/vlm/profiles/full-v1.json | 2 +- .../vlm/profiles/short-all-native-v1.json | 2 +- .../vlm/profiles/short-all-native-v2.json | 2558 +++++++++++++++++ .../vlm/profiles/short-native-v1.json | 2 +- .../vlm/profiles/short-native-v2.json | 1873 ++++++++++++ .../evaluation/vlm/profiles/short-v1.json | 2 +- .../vlm/profiles/short-vllm-v2.json | 13 + .../vlm/profiles/smoke-native-v1.json | 129 + .../vlm/profiles/smoke-vllm-v1.json | 14 + examples/puzzletron/evaluation/vlm/suites.py | 39 +- examples/puzzletron/evaluation/vlm/tasks.py | 233 +- ....patch => lmms_eval_compat_3e675904.patch} | 6 + modelopt/torch/puzzletron/evaluation/lmms.py | 18 +- .../puzzletron/evaluation/test_checkpoint.py | 29 +- .../vlm/preparation/test_benchmark_data.py | 314 +- .../puzzletron/evaluation/vlm/test_run.py | 967 ++++++- .../puzzletron/test_ci_image_contract.py | 73 + .../torch/puzzletron/test_lmms_evaluation.py | 31 + 30 files changed, 7638 insertions(+), 353 deletions(-) create mode 100644 examples/puzzletron/evaluation/vlm/profiles/core3-full-native-v1.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/core3-full-vllm-v1.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/short-all-native-v2.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/short-native-v2.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/short-vllm-v2.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/smoke-native-v1.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/smoke-vllm-v1.json rename examples/puzzletron/patches/{lmms_eval_wandb_0_29.patch => lmms_eval_compat_3e675904.patch} (88%) diff --git a/examples/puzzletron/Dockerfile b/examples/puzzletron/Dockerfile index 72ffff063d8..132885b6277 100644 --- a/examples/puzzletron/Dockerfile +++ b/examples/puzzletron/Dockerfile @@ -72,8 +72,9 @@ RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \ # Install the native Qwen 3.5 evaluator in the shared worker environment. The # tracked patch removes lmms-eval's unused private WandB printer integration, -# reconciles its WandB range with AutoModel, and replaces the legacy LaTeX -# parser whose ANTLR pin conflicts with Puzzletron's Hydra stack. +# reconciles its WandB range with AutoModel, replaces the legacy LaTeX parser +# whose ANTLR pin conflicts with Puzzletron's Hydra stack, and preserves the +# vLLM adapter's task-specific sampling parameters. RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \ lmms_patch="$(pin lmms_eval.compatibility_patch)" && \ echo "$(pin lmms_eval.compatibility_patch_sha256) /opt/puzzletron/patches/${lmms_patch}" | \ diff --git a/examples/puzzletron/ci_environment.json b/examples/puzzletron/ci_environment.json index 0e0ec1a56e8..4eace49422d 100644 --- a/examples/puzzletron/ci_environment.json +++ b/examples/puzzletron/ci_environment.json @@ -35,16 +35,17 @@ "base_version": "0.7.2", "repository": "https://github.com/EvolvingLMMs-Lab/lmms-eval.git", "commit": "3e675904f8cba6793de12b91979b04d91754bdf3", - "compatibility_patch": "lmms_eval_wandb_0_29.patch", + "compatibility_patch": "lmms_eval_compat_3e675904.patch", "compatibility_patch_context_lines": 0, "compatibility_patch_files": [ "lmms_eval/loggers/wandb_logger.py", + "lmms_eval/models/simple/vllm.py", "lmms_eval/tasks/emma/utils.py", "lmms_eval/tasks/mathvision/eval_utils.py", "lmms_eval/tasks/stare/utils.py", "pyproject.toml" ], - "compatibility_patch_sha256": "960ef2f0d5cea85ccb67e22ab28356ae935f8109876b2f65f88693795549de08", + "compatibility_patch_sha256": "c8cbb9614cd80c2a58c08e669fdd5e94558bf9f0bc78a740af0e086cc69faec2", "required_paths": [ "models/chat/qwen3_5.py", "tasks/gsm8k/gsm8k.yaml", diff --git a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md index 32a9926be88..0933cc578c3 100644 --- a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md +++ b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md @@ -6,10 +6,13 @@ and evaluation settings. Use the same profile for every model being compared. The common choices are: -- `short-all-native-v1` for the recommended short comparison across eight - benchmarks; -- `short-v1` or `short-native-v1` for a faster three-benchmark check; and -- `full-v1` to evaluate every available example in eight benchmarks. +- `short-native-v2` for the maintained three-benchmark campaign screen; +- `short-vllm-v2` and `smoke-vllm-v1` for materialized heterogeneous campaign checkpoints; +- `short-all-native-v2` for a broader eight-benchmark regression screen; +- `core3-full-native-v1` and `core3-full-vllm-v1` for paired full-dataset + teacher references on RealWorldQA, MMMU validation, and MVBench; +- `short-v1`, `short-native-v1`, `short-all-native-v1`, and `full-v1` with an + environment that matches each profile's pinned evaluator revision. For text-only IFEval and GSM8K evaluation, use the separate [text checkpoint evaluator](checkpoint_evaluation.md). @@ -17,31 +20,26 @@ For text-only IFEval and GSM8K evaluation, use the separate ## Prepare the worker environment Run evaluation in the default Puzzletron worker image described in the -[worker environment guide](environment_setup.md). The image includes the exact -upstream `lmms-eval` 0.7.2 source revision recorded in -`examples/puzzletron/ci_environment.json`, including its native Qwen 3.5 image -and video backend. No evaluator overlay or separate VLM requirements install is -needed. - -The image applies one tracked compatibility patch to `lmms-eval`. It removes an -unused private WandB printer integration, aligns the WandB requirement with -AutoModel, and replaces the legacy LaTeX parser whose ANTLR pin conflicts with -Puzzletron's Hydra stack. The maintained replacement provides the same parser -entry point to the three affected math tasks. Image construction verifies the -patch checksum, source revision, resulting checkout diff, required native -backend and task files, and the resolved Python dependency set. Do not replace -or modify that evaluator checkout inside the worker image. +[worker environment guide](environment_setup.md). The image pins the +`lmms-eval` source revision and dependencies recorded in +`examples/puzzletron/ci_environment.json`, includes the native Qwen 3.5 image +and video backend, and preserves each task's output-token budget for vLLM. No +evaluator overlay or separate VLM requirements install is needed. Do not modify +the evaluator checkout inside the image. Preflight rejects evaluator revisions +that do not match the selected profile. ## Understand the two execution paths | Path | Profiles | What it does | | --- | --- | --- | -| Qwen-specific Transformers | `short-native-v1`, `short-all-native-v1` | Loads the checkpoint directly with the Qwen 3.5 model loader. Video prompts include timestamps for sampled frames. | -| General vLLM | `short-v1`, `full-v1` | Runs the checkpoint through vLLM and converts inputs to general image and video messages. Video prompts do not include frame timestamps. | +| Qwen-specific Transformers | `short-native-v1`, `short-native-v2`, `short-all-native-v1`, `short-all-native-v2`, `core3-full-native-v1` | Loads the checkpoint directly with the Qwen 3.5 model loader. Video prompts include timestamps for sampled frames. | +| General vLLM | `short-v1`, `short-vllm-v2`, `smoke-vllm-v1`, `full-v1`, `core3-full-vllm-v1` | Runs the checkpoint through vLLM and converts inputs to general image and video messages. Video prompts do not include frame timestamps. Materialized heterogeneous checkpoints require this path. | -Both paths use the same pinned `lmms-eval` revision but produce different -prompts. Their scores represent the complete paths and do not isolate the -effect of the inference engine. +The native and vLLM paths produce different prompts, so their scores represent +the complete paths and do not isolate the inference engine. +The heterogeneous-checkpoint `short-vllm-v2` and `smoke-vllm-v1` profiles pin +FlashAttention 2 because the runtime's FlashAttention 3 scheduler does +not support their per-layer attention geometry. ## Choose an evaluation @@ -51,15 +49,24 @@ Use a versioned profile when scores will be compared across checkpoints: | --- | --- | --- | | `short-v1` | RealWorldQA, MMMU, and MVBench | 344 predefined examples; MMMU uses four per subject and MVBench uses eight per category | | `short-native-v1` | Same examples as `short-v1`, using the Qwen-specific Transformers adapter | The same 344 predefined examples with timestamps added to video prompts | +| `short-native-v2` | Fixed coverage of RealWorldQA, MMMU, and MVBench | 64 RealWorldQA rows, four rows from each of 30 MMMU subjects, and eight rows from each of 20 MVBench tasks | | `short-all-native-v1` | All eight judge-free benchmarks, using the Qwen-specific Transformers adapter | 690 predefined examples: the same 344 plus 346 from five additional benchmarks | +| `short-all-native-v2` | Fixed coverage of all eight judge-free benchmarks | 690 predefined examples distributed across the recorded source strata | +| `core3-full-native-v1` | Native backend on RealWorldQA, MMMU validation, and MVBench | All 765, 900, and 4,000 rows respectively | +| `core3-full-vllm-v1` | vLLM backend on the same three datasets | All 765, 900, and 4,000 rows respectively | | `full-v1` | Eight judge-free image and video benchmarks | Every available example in each pinned dataset version | -For future short teacher and student comparisons through the Qwen-specific -adapter, prefer `short-all-native-v1` when all eight datasets are available. -It checks more kinds of image and video tasks than the 344-row profiles. Use -the 344-row profiles for faster three-benchmark regression checks or when the -same rows must be compared across both execution paths. Neither short profile -replaces `full-v1` for complete benchmark reporting. +Use `short-native-v2` for campaign comparisons. Use +`short-all-native-v2` when broader image and video regression coverage is more +important than matching the campaign screen. The v1 profiles require their +pinned evaluator revisions. No short profile replaces a full-data profile for +complete benchmark reporting. + +The paired `core3-full-*` profiles provide teacher references. They pin the +Qwen 3.5 0.8B Hub snapshot and +reject other checkpoints or runtime-setting overrides. Keep the two backend +results separate: their prompt construction and MVBench frame annotations +differ by design, so their score delta is not a pure inference-engine effect. Profiles keep the evaluator, datasets, selected examples, video sampling, answer generation, and batch size fixed. They disable Qwen thinking so the @@ -67,15 +74,40 @@ model returns the short answer expected by these benchmarks. Puzzletron stops before evaluation if the installed evaluator or cached data do not match the selected profile. -`short-all-native-v1` balances VideoMMMU across its three categories, -Video-MME across duration and domain, MLVU across task type, and PerceptionTest -across area and reasoning type. LongVideoBench uses deterministic index-spaced -rows. This profile is a reproducible regression screen, not a full-benchmark -quality estimate. +The preflight report and normalized result include an `output_budget_contract` +for every selected task. It records the requested and effective output-token +budget and the adapter-specific resolution rule. In the maintained three-task +profiles, RealWorldQA and MVBench use 16 tokens and MMMU uses 128. The native +adapter applies each task budget directly. The pinned generic vLLM adapter +treats its model-level `max_new_tokens` value as a lower bound, so the report +also records that limitation; backend score differences therefore include +generation-policy and prompt-path differences, not just engine behavior. + +`short-native-v2` selects +64 positions across all 765 RealWorldQA test rows, four positions within each +30-row MMMU subject, and eight positions within each 200-row MVBench task. Its +profile records the generator version, population and stratum counts, selected +index quantiles, recorded source-row identities, and a digest of those identities. +These fixed rows provide a regression screen, not a representative +full-benchmark estimate. Compare a teacher and every candidate with the same +profile. + +`short-all-native-v2` adds midpoint samples from +five video benchmarks: 24 rows from each of three VideoMMMU tasks, four rows +from each of 18 Video-MME duration-and-domain strata, 68 rows across the +LongVideoBench validation split, and 10 rows from each of seven MLVU task +types. Its 64 PerceptionTest rows are balanced across the 13 observed +area-and-reasoning strata: the first 12 in lexical order receive five rows and +the last receives four. Before selecting any rows, the generated adapters +verify the exact source population size and, where the pinned metadata exposes +the selected stratification fields, the complete source stratum counts. They +also verify recorded upstream IDs for Video-MME, MLVU, and PerceptionTest. +LongVideoBench is stratified only by its validation split. This profile is a +fixed regression screen, not a full-benchmark quality estimate. `full-v1` excludes MMVU because complete MMVU evaluation requires an external -judge. Run MMVU separately through `mmvu-smoke` or the judge-enabled legacy -`full` suite. +judge. Run MMVU separately through `mmvu-smoke` or the judge-enabled `full` +suite. These additional options are intended for setup checks and targeted diagnosis: @@ -97,51 +129,30 @@ frames. ## Cache benchmark data -Evaluation is offline and requires every selected dataset revision to already -exist under an explicit Hugging Face cache root. Populate the pinned image -datasets for the selected profile or suite before starting evaluation. The -repository stores only profile metadata and exact-row selectors; it does not -store benchmark records or media. The example uses `short-v1`: +Evaluation reads every selected dataset revision from an explicit Hugging Face +cache root. The campaign `prepare_dataset` stage fills and validates this cache +from the configured evaluation tasks. The repository stores only profile +metadata and exact-row selectors; it does not store benchmark records or media. +To prepare the common image and video cache independently, run: ```bash export HF_HOME=/path/to/huggingface-cache -python - "$HF_HOME" short-v1 <<'PY' -import sys -from pathlib import Path - -from huggingface_hub import snapshot_download - -from examples.puzzletron.evaluation.vlm import profile, suites - -hf_home = Path(sys.argv[1]) -for task in suites.source_tasks(sys.argv[2]): - dataset = profile.VLM_BENCHMARK_DATASETS[task] - if dataset.media_dir is not None: - continue - snapshot_download( - repo_id=dataset.repository, - repo_type="dataset", - revision=dataset.revision, - cache_dir=hf_home / "hub", - ) -PY -``` - -The video preparation command downloads its pinned snapshot and safely extracts -the media: - -```bash python -m examples.puzzletron.evaluation.vlm.preparation.benchmark_data \ --hf-home "$HF_HOME" \ - --tasks mvbench + --tasks realworldqa,mmmu_val,mvbench,video_mmmu,videomme,longvideobench_val_v,mlvu_dev,perceptiontest_val_mc ``` -Pass a comma-separated list to `--tasks` when preparing more than one video -dataset. Use `--download-only` and `--extract-only` to split transfer and -extraction across jobs, or `--range-resume` for a resumable single-writer -download. Run the command with `--help` to list the supported video dataset -names. +The command downloads each listed exact pinned snapshot and safely extracts +media only for tasks that declare a preparation directory. For `short-native-v2`, +prepare `realworldqa`, `mmmu_val`, and `mvbench`. Use `--download-only` and +`--extract-only` to split transfer and extraction across jobs, or +`--range-resume` for a resumable single-writer download. Run the command with +`--help` to list all supported dataset task names. Preparation records exact +snapshot and media inventories. Ordinary resume uses file metadata as a fast +path and checks recorded hashes after metadata changes; incomplete owned media +is rebuilt from the pinned snapshot only when the host supports atomic directory +exchange. Otherwise the existing root is preserved and preparation fails. Video suites also require an installed `decord`-compatible reader. The Puzzletron requirements select the supported reader for the current platform @@ -156,27 +167,37 @@ running inference. ## Run a versioned profile -The checkpoint path is an invocation input, not part of either profile -manifest. Use the same command with a teacher or any materialized Qwen 3.5 -student checkpoint: +For comparison profiles, the checkpoint path is an invocation input. Use the +same command with a teacher or any materialized Qwen 3.5 student checkpoint: ```bash python -m examples.puzzletron.evaluation.vlm.run \ --checkpoint /path/to/checkpoint \ - --output-dir /path/to/results/short-v1 \ + --output-dir /path/to/results/short-all-native-v2 \ + --hf-home "$HF_HOME" \ + --profile short-all-native-v2 +``` + +The `core3-full-*` profiles instead require the exact pinned local Qwen 3.5 +0.8B Hub snapshot. For example: + +```bash +python -m examples.puzzletron.evaluation.vlm.run \ + --checkpoint "$HF_HOME/hub/models--Qwen--Qwen3.5-0.8B/snapshots/2fc06364715b967f1860aea9cf38778875588b17" \ + --output-dir /path/to/results/core3-full-native-v1/realworldqa \ --hf-home "$HF_HOME" \ - --profile short-v1 + --profile core3-full-native-v1 \ + --profile-task realworldqa \ + --preflight-only ``` -Replace `short-v1` with `short-native-v1` after installing the native -requirements. Use `short-all-native-v1` after caching all eight judge-free -benchmarks, or `full-v1` after caching the same datasets for a complete -generic-vLLM evaluation. Use `--preflight-only` before consuming GPU time. +Use `full-v1` only with its pinned evaluator revision. Always run +`--preflight-only` before consuming GPU time. -To run tasks in parallel, run one profile task per job with either -`--profile full-v1 --profile-task TASK` or -`--profile short-all-native-v1 --profile-task TASK`. A complete result needs -one successful job for each of the eight tasks in the selected profile. +To run tasks in parallel, run one profile task per job with +`--profile PROFILE --profile-task TASK`. This works for `full-v1`, the paired +`core3-full-*` profiles, and `short-all-native-v2`. A complete result needs one +successful job for every task in the selected profile. Grouped tasks can use multiple batch-1 workers without changing inference batching. Add `--profile-task-shard INDEX/COUNT` to `mvbench` or `video_mmmu`, @@ -209,12 +230,15 @@ Each execution creates an `attempt_/` directory containing the command, logs, raw evaluator output, and normalized metrics in `summary.json`. Repeating a run creates another attempt directory instead of overwriting earlier output. -`short-v1` and `full-v1` use the generic vLLM adapter. `short-native-v1` and -`short-all-native-v1` use the Qwen-specific Transformers adapter. Both use the -same pinned evaluator revision but construct different model inputs. Keep the -profile name with every score, and do not combine scores from the two paths as -if only the inference engine changed. The recorded preflight report includes -the backend identity, frame policy, generation policy, and evaluator revision. +The execution-path table above identifies each profile's adapter. The native +and generic vLLM paths construct different model inputs, so keep the profile +name with every score and do not combine scores as if only the inference engine +changed. The preflight report records backend identity, frame policy, +generation policy, and exact evaluator revision. + +MMMU results also include `mmmu_parser_audit`. Its per-sample status counts use +`parsed`, `parsed_open`, `invalid_open`, and `fallback_random`, making invalid +open-ended parses and random-fallback scoring visible in each result. If preflight fails, address the reported checkpoint, revision, cache, decoder, or credential mismatch before retrying. Inspect `stderr.txt` in the attempt diff --git a/examples/puzzletron/evaluation/checkpoint.py b/examples/puzzletron/evaluation/checkpoint.py index efe68857b5d..516a78263ee 100644 --- a/examples/puzzletron/evaluation/checkpoint.py +++ b/examples/puzzletron/evaluation/checkpoint.py @@ -37,6 +37,8 @@ __all__ = [ "DEFAULT_PREFLIGHT_TIMEOUT_SECONDS", "HUGGINGFACE_CREDENTIAL_NAMES", + "LMMS_EVAL_LEGACY_REVISION", + "LMMS_EVAL_QWEN35_NATIVE_REVISION", "LMMS_EVAL_REVISION", "credential_free_environment", "lmms_eval_disabled_judge_environment", @@ -54,6 +56,13 @@ ) LMMS_EVAL_SOURCE = CI_ENVIRONMENT["lmms_eval"] LMMS_EVAL_REVISION = LMMS_EVAL_SOURCE["commit"] +LMMS_EVAL_LEGACY_REVISION = "15c32bfec165df13c269ddd3cda03b2ed9137825" +LMMS_EVAL_QWEN35_NATIVE_REVISION = "88b23e2bfa16a1edbc16e9e238ed82130b3a4f56" +_LMMS_EVAL_REPOSITORY = LMMS_EVAL_SOURCE["repository"] +_LMMS_EVAL_HISTORICAL_SOURCES = { + revision: {"repository": _LMMS_EVAL_REPOSITORY, "commit": revision} + for revision in (LMMS_EVAL_LEGACY_REVISION, LMMS_EVAL_QWEN35_NATIVE_REVISION) +} DEFAULT_PREFLIGHT_TIMEOUT_SECONDS = 15 * 60.0 HUGGINGFACE_CREDENTIAL_NAMES = ( "HF_TOKEN", @@ -151,7 +160,8 @@ def positive_float(value: str) -> float: def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> str: """Return the imported evaluator revision after matching its source and patch pin.""" - revision = _imported_lmms_eval_revision() + source = _lmms_eval_source(expected_revision) + revision = _imported_lmms_eval_revision(source) if revision is not None: if revision != expected_revision: raise RuntimeError( @@ -169,10 +179,10 @@ def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> st raise RuntimeError("installed lmms-eval revision provenance is unavailable") from error if isinstance(provenance, dict): try: - ci_environment.verify_installed_vcs_source("lmms-eval", LMMS_EVAL_SOURCE) + ci_environment.verify_installed_vcs_source("lmms-eval", source) except (OSError, subprocess.SubprocessError) as error: raise RuntimeError("installed lmms-eval source provenance is unavailable") from error - revision = LMMS_EVAL_REVISION + revision = expected_revision else: revision = None if revision != expected_revision: @@ -183,7 +193,17 @@ def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> st return revision -def _imported_lmms_eval_revision() -> str | None: +def _lmms_eval_source(expected_revision: str) -> dict[str, object]: + """Resolve a current or immutable historical evaluator source contract.""" + if expected_revision == LMMS_EVAL_REVISION: + return LMMS_EVAL_SOURCE + try: + return _LMMS_EVAL_HISTORICAL_SOURCES[expected_revision] + except KeyError as error: + raise RuntimeError(f"unsupported lmms-eval revision: {expected_revision}") from error + + +def _imported_lmms_eval_revision(source: dict[str, object]) -> str | None: """Verify a source checkout imported directly through ``PYTHONPATH``.""" spec = importlib.util.find_spec("lmms_eval") locations = tuple(spec.submodule_search_locations or ()) if spec is not None else () @@ -192,13 +212,13 @@ def _imported_lmms_eval_revision() -> str | None: checkout = Path(locations[0]).resolve().parent if not (checkout / ".git").exists(): return None - return _verified_checkout_revision(checkout) + return _verified_checkout_revision(checkout, source) -def _verified_checkout_revision(checkout: Path) -> str | None: +def _verified_checkout_revision(checkout: Path, source: dict[str, object]) -> str | None: """Return the revision of one source-and-patch verified Git checkout.""" try: - return ci_environment.verify_vcs_checkout(checkout, "lmms-eval", LMMS_EVAL_SOURCE) + return ci_environment.verify_vcs_checkout(checkout, "lmms-eval", source) except (OSError, subprocess.SubprocessError): return None diff --git a/examples/puzzletron/evaluation/vlm/contracts.py b/examples/puzzletron/evaluation/vlm/contracts.py index 78b257c07f1..9cc831fd1a3 100644 --- a/examples/puzzletron/evaluation/vlm/contracts.py +++ b/examples/puzzletron/evaluation/vlm/contracts.py @@ -35,10 +35,26 @@ _PROFILE_SCHEMA = "modelopt.vlm-evaluation-profile/v1" _PROFILE_ROOT = Path(__file__).with_name("profiles") -PROFILE_NAMES = ("short-v1", "short-native-v1", "short-all-native-v1", "full-v1") +PROFILE_NAMES = ( + "short-v1", + "short-native-v1", + "short-native-v2", + "short-vllm-v2", + "smoke-native-v1", + "smoke-vllm-v1", + "short-all-native-v1", + "short-all-native-v2", + "full-v1", + "core3-full-native-v1", + "core3-full-vllm-v1", +) _PROFILE_TASKS = { "short-v1": ("realworldqa", "mmmu_val", "mvbench"), "short-native-v1": ("realworldqa", "mmmu_val", "mvbench"), + "short-native-v2": ("realworldqa", "mmmu_val", "mvbench"), + "short-vllm-v2": ("realworldqa", "mmmu_val", "mvbench"), + "smoke-native-v1": ("realworldqa", "mmmu_val", "mvbench"), + "smoke-vllm-v1": ("realworldqa", "mmmu_val", "mvbench"), "short-all-native-v1": ( "realworldqa", "mmmu_val", @@ -49,13 +65,32 @@ "mlvu_dev", "perceptiontest_val_mc", ), + "short-all-native-v2": ( + "realworldqa", + "mmmu_val", + "mvbench", + "video_mmmu", + "videomme", + "longvideobench_val_v", + "mlvu_dev", + "perceptiontest_val_mc", + ), "full-v1": tuple(task for task in profile.VLM_BENCHMARK_TASKS if task != "mmvu_val"), + "core3-full-native-v1": ("realworldqa", "mmmu_val", "mvbench"), + "core3-full-vllm-v1": ("realworldqa", "mmmu_val", "mvbench"), } _PROFILE_SELECTIONS = { "short-v1": "exact-rows", "short-native-v1": "exact-rows", + "short-native-v2": "exact-rows", + "short-vllm-v2": "exact-rows", + "smoke-native-v1": "exact-rows", + "smoke-vllm-v1": "exact-rows", "short-all-native-v1": "exact-rows", + "short-all-native-v2": "exact-rows", "full-v1": "all", + "core3-full-native-v1": "all", + "core3-full-vllm-v1": "all", } SHORT_PROFILE_NAMES = tuple( name for name in PROFILE_NAMES if _PROFILE_SELECTIONS[name] == "exact-rows" @@ -71,22 +106,120 @@ "enable_thinking": False, "name": "qwen3_5", }, + "short-native-v2": { + "attention_implementation": "sdpa", + "enable_thinking": False, + "name": "qwen3_5", + }, + "short-vllm-v2": { + "attention_config": {"flash_attn_version": 2}, + "enable_thinking": False, + "name": "vllm", + "reasoning_parser": "qwen3", + }, + "smoke-native-v1": { + "attention_implementation": "sdpa", + "enable_thinking": False, + "name": "qwen3_5", + }, + "smoke-vllm-v1": { + "attention_config": {"flash_attn_version": 2}, + "enable_thinking": False, + "enforce_eager": True, + "name": "vllm", + "reasoning_parser": "qwen3", + }, "short-all-native-v1": { "attention_implementation": "sdpa", "enable_thinking": False, "name": "qwen3_5", }, + "short-all-native-v2": { + "attention_implementation": "sdpa", + "enable_thinking": False, + "name": "qwen3_5", + }, "full-v1": { "enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3", }, + "core3-full-native-v1": { + "attention_implementation": "sdpa", + "enable_thinking": False, + "name": "qwen3_5", + }, + "core3-full-vllm-v1": { + "enable_thinking": False, + "name": "vllm", + "reasoning_parser": "qwen3", + }, } _PROFILE_REVISIONS = { - "short-v1": checkpoint.LMMS_EVAL_REVISION, - "short-native-v1": checkpoint.LMMS_EVAL_REVISION, - "short-all-native-v1": checkpoint.LMMS_EVAL_REVISION, - "full-v1": checkpoint.LMMS_EVAL_REVISION, + "short-v1": checkpoint.LMMS_EVAL_LEGACY_REVISION, + "short-native-v1": checkpoint.LMMS_EVAL_QWEN35_NATIVE_REVISION, + "short-native-v2": checkpoint.LMMS_EVAL_REVISION, + "short-vllm-v2": checkpoint.LMMS_EVAL_REVISION, + "smoke-native-v1": checkpoint.LMMS_EVAL_REVISION, + "smoke-vllm-v1": checkpoint.LMMS_EVAL_REVISION, + "short-all-native-v1": checkpoint.LMMS_EVAL_QWEN35_NATIVE_REVISION, + "short-all-native-v2": checkpoint.LMMS_EVAL_REVISION, + "full-v1": checkpoint.LMMS_EVAL_LEGACY_REVISION, + "core3-full-native-v1": checkpoint.LMMS_EVAL_REVISION, + "core3-full-vllm-v1": checkpoint.LMMS_EVAL_REVISION, +} +_PROFILE_MODELS = { + name: { + "repository": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17", + } + for name in ("core3-full-native-v1", "core3-full-vllm-v1") +} +_PROFILE_POPULATIONS = { + name: {"realworldqa": 765, "mmmu_val": 900, "mvbench": 4000} for name in _PROFILE_MODELS +} +_MVBENCH_LEAF_POPULATIONS = { + "action_sequence": 200, + "moving_count": 200, + "action_prediction": 200, + "episodic_reasoning": 200, + "action_antonym": 200, + "action_count": 200, + "scene_transition": 200, + "object_shuffle": 200, + "object_existence": 200, + "fine_grained_pose": 200, + "unexpected_action": 200, + "moving_direction": 200, + "state_change": 200, + "object_interaction": 200, + "character_order": 200, + "action_localization": 200, + "counterfactual_inference": 200, + "fine_grained_action": 200, + "moving_attribute": 200, + "egocentric_navigation": 200, +} +_AUDITED_SAMPLING_PROFILES = frozenset( + { + "smoke-native-v1", + "smoke-vllm-v1", + "short-native-v2", + "short-vllm-v2", + "short-all-native-v2", + } +) +_SAMPLING_AUDIT_SCHEMA = "modelopt.vlm-sampling-audit/v1" +_SAMPLING_GENERATOR = {"name": "systematic-midpoint", "version": 1} +_SAMPLING_STRATA = { + "realworldqa": "split", + "mmmu_val": "subject", + "mvbench": "leaf_task", + "video_mmmu": "leaf_task", + "videomme": "duration+domain", + "longvideobench_val_v": "split", + "mlvu_dev": "task_type", + "perceptiontest_val_mc": "area+reasoning", } @@ -106,21 +239,25 @@ def source_tasks(self) -> tuple[str, ...]: @property def exact_rows(self) -> dict[str, object] | None: - """Return the legacy exact-row selector payload when the profile uses one.""" + """Return the executable exact-row selector payload when the profile uses one.""" if self.manifest["selection"] != "exact-rows": return None tasks = cast("dict[str, dict[str, object]]", self.manifest["tasks"]) - return { + exact_rows: dict[str, object] = { "schema": "modelopt.vlm-benchmark-quick/v1", "lmms_eval_revision": self.manifest["lmms_eval_revision"], "tasks": { task: { "dataset_revision": entry["dataset_revision"], "rows": entry["rows"], + **({"selection": entry["selection"]} if "selection" in entry else {}), } for task, entry in tasks.items() }, } + if "sampling" in self.manifest: + exact_rows["selection"] = self.manifest["sampling"] + return exact_rows def load_profile(name: str) -> ProfileContract: @@ -143,23 +280,37 @@ def load_profile(name: str) -> ProfileContract: ) -def _resolve_manifest(name: str, manifest: object) -> object: - """Resolve one shallow profile inheritance declaration.""" +def _resolve_manifest(name: str, manifest: object, *, ancestors: tuple[str, ...] = ()) -> object: + """Resolve profile inheritance while rejecting cycles.""" if not isinstance(manifest, dict) or "extends" not in manifest: return manifest base_name = manifest.get("extends") - if not isinstance(base_name, str) or base_name not in PROFILE_NAMES or base_name == name: + if ( + not isinstance(base_name, str) + or base_name not in PROFILE_NAMES + or base_name == name + or base_name in ancestors + ): raise RuntimeError(f"{name} profile extends an unsupported base profile") base_path = _PROFILE_ROOT / f"{base_name}.json" try: base = json.loads(base_path.read_text()) except (OSError, json.JSONDecodeError) as error: raise RuntimeError(f"VLM evaluation base profile is unreadable: {base_path}") from error - if not isinstance(base, dict) or "extends" in base: - raise RuntimeError(f"{name} profile base must be a concrete profile") + base = _resolve_manifest(base_name, base, ancestors=(*ancestors, name)) + if not isinstance(base, dict): + raise RuntimeError(f"{name} profile base must contain an object") overrides = {key: value for key, value in manifest.items() if key != "extends"} if isinstance(base.get("tasks"), dict) and isinstance(overrides.get("tasks"), dict): - overrides["tasks"] = {**base["tasks"], **overrides["tasks"]} + tasks = dict(base["tasks"]) + for task, entry in overrides["tasks"].items(): + base_entry = tasks.get(task) + tasks[task] = ( + {**base_entry, **entry} + if isinstance(base_entry, dict) and isinstance(entry, dict) + else entry + ) + overrides["tasks"] = tasks return {**base, **overrides} @@ -175,6 +326,9 @@ def _validate_manifest(name: str, manifest: dict[str, object]) -> None: "model_type": "qwen3_5", }: raise RuntimeError(f"{name} profile model family is unsupported") + expected_model = _PROFILE_MODELS.get(name) + if expected_model is not None and manifest.get("model") != expected_model: + raise RuntimeError(f"{name} profile model pin differs from the runtime policy") if manifest.get("backend") != _PROFILE_BACKENDS[name]: raise RuntimeError(f"{name} profile backend differs from the runtime policy") if manifest.get("preprocessing") != { @@ -201,6 +355,7 @@ def _validate_manifest(name: str, manifest: dict[str, object]) -> None: raise RuntimeError(f"{name} profile tasks differ from its versioned policy") for task, entry in tasks.items(): _validate_task(name, task, entry, selection=cast("str", selection)) + _validate_sampling(name, manifest, tasks) def _validate_task(name: str, task: object, entry: object, *, selection: str) -> None: @@ -218,8 +373,149 @@ def _validate_task(name: str, task: object, entry: object, *, selection: str) -> observed = {key: entry.get(key) for key in expected} if observed != expected: raise RuntimeError(f"{name} profile task pins differ from the runtime catalog: {task}") + expected_population = _PROFILE_POPULATIONS.get(name, {}).get(task) + if expected_population is not None and entry.get("population_rows") != expected_population: + raise RuntimeError( + f"{name} profile task population differs from its versioned policy: {task}" + ) + if expected_population is not None and task == "mvbench": + if entry.get("leaf_populations") != _MVBENCH_LEAF_POPULATIONS: + raise RuntimeError( + f"{name} profile task leaf populations differ from its versioned policy: {task}" + ) rows = entry.get("rows") if selection == "all" and rows is not None: raise RuntimeError(f"{name} full-data task must not contain exact rows: {task}") if selection == "exact-rows" and (not isinstance(rows, list) or not rows): raise RuntimeError(f"{name} exact-row task must contain rows: {task}") + + +def _validate_sampling( + name: str, + manifest: dict[str, object], + tasks: dict[str, object], +) -> None: + sampling = manifest.get("sampling") + if name in _AUDITED_SAMPLING_PROFILES and not isinstance(sampling, dict): + raise RuntimeError(f"{name} profile must contain a sampling audit") + if sampling is None: + return + if not isinstance(sampling, dict): + raise RuntimeError(f"{name} profile sampling audit must contain an object") + if sampling != { + "schema": _SAMPLING_AUDIT_SCHEMA, + "claim_scope": "deterministic-screening-only", + "generator": _SAMPLING_GENERATOR, + }: + raise RuntimeError(f"{name} profile sampling audit policy is unsupported") + for task, entry in tasks.items(): + if not isinstance(entry, dict): + raise RuntimeError(f"{name} profile task must contain an object: {task}") + _validate_task_sampling(name, task, entry) + + +def _validate_task_sampling(name: str, task: str, entry: dict[str, object]) -> None: + selection = entry.get("selection") + rows = entry.get("rows") + if not isinstance(selection, dict) or not isinstance(rows, list): + raise RuntimeError(f"{name} profile task must contain an audited selection: {task}") + strata = selection.get("strata") + if ( + selection.get("method") != "systematic-midpoint" + or selection.get("stratified_by") != _SAMPLING_STRATA.get(task) + or selection.get("index_space") != "within-stratum" + or not isinstance(strata, list) + or not strata + or selection.get("selected_rows") != len(rows) + or selection.get("population_rows") + != sum(stratum.get("population_rows", 0) for stratum in strata if isinstance(stratum, dict)) + or len(strata) != sum(isinstance(stratum, dict) for stratum in strata) + ): + raise RuntimeError(f"{name} profile task sampling audit is invalid: {task}") + expected_counts: dict[str, tuple[int, int]] = {} + for stratum in strata: + stratum_name = stratum.get("name") + population_rows = stratum.get("population_rows") + selected_rows = stratum.get("selected_rows") + if ( + not isinstance(stratum_name, str) + or not stratum_name + or not isinstance(population_rows, int) + or isinstance(population_rows, bool) + or not isinstance(selected_rows, int) + or isinstance(selected_rows, bool) + or selected_rows < 0 + or population_rows < selected_rows + or stratum_name in expected_counts + ): + raise RuntimeError(f"{name} profile task sampling stratum is invalid: {task}") + expected_counts[stratum_name] = (population_rows, selected_rows) + + actual_indices: dict[str, list[int]] = {stratum: [] for stratum in expected_counts} + for row in rows: + if not isinstance(row, dict): + raise RuntimeError(f"{name} profile task sampling row is invalid: {task}") + stratum, local_index = _sampling_identity(task, row) + if stratum not in actual_indices: + raise RuntimeError(f"{name} profile task sampling stratum differs: {task}") + actual_indices[stratum].append(local_index) + for stratum, (population_rows, selected_rows) in expected_counts.items(): + expected = [ + ((2 * index + 1) * population_rows) // (2 * selected_rows) + for index in range(selected_rows) + ] + if actual_indices[stratum] != expected: + raise RuntimeError(f"{name} profile task rows differ from its sampling audit: {task}") + + identity = hashlib.sha256( + json.dumps(rows, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest() + if selection.get("selected_row_identities_sha256") != identity: + raise RuntimeError(f"{name} profile task row identity fingerprint differs: {task}") + ordered_indices = sorted(index for indices in actual_indices.values() for index in indices) + quantiles = { + "method": "lower-order-statistic", + **{ + f"p{percentile}": ordered_indices[(len(ordered_indices) - 1) * percentile // 100] + for percentile in (0, 25, 50, 75, 100) + }, + } + if selection.get("selected_index_quantiles") != quantiles: + raise RuntimeError(f"{name} profile task index quantiles differ: {task}") + + +def _sampling_identity(task: str, row: dict[str, object]) -> tuple[str, int]: + source_id = row.get("source_sample_id") + source_index = row.get("source_row_index") + if not isinstance(source_id, str) or not isinstance(source_index, int): + raise RuntimeError(f"profile sampling row identity is invalid: {task}") + if task == "realworldqa": + return "test", source_index + if task == "mmmu_val": + prefix, separator, item = source_id.rpartition("_") + if not separator or not item.isdigit() or not prefix.startswith("validation_"): + raise RuntimeError("profile sampling row identity is invalid: mmmu_val") + return prefix.removeprefix("validation_"), int(item) - 1 + if task in {"mvbench", "video_mmmu"}: + leaf = row.get("leaf_task") + if not isinstance(leaf, str) or not leaf.startswith(f"{task}_"): + raise RuntimeError(f"profile sampling row identity is invalid: {task}") + return leaf.removeprefix(f"{task}_"), source_index + if task in { + "videomme", + "longvideobench_val_v", + "mlvu_dev", + "perceptiontest_val_mc", + }: + stratum = row.get("sampling_stratum") + stratum_index = row.get("source_stratum_index") + if ( + not isinstance(stratum, str) + or not stratum + or not isinstance(stratum_index, int) + or isinstance(stratum_index, bool) + or stratum_index < 0 + ): + raise RuntimeError(f"profile sampling row identity is invalid: {task}") + return stratum, stratum_index + raise RuntimeError(f"profile sampling is unsupported for task: {task}") diff --git a/examples/puzzletron/evaluation/vlm/evaluator.py b/examples/puzzletron/evaluation/vlm/evaluator.py index 7c8b26e248f..5de80282a15 100644 --- a/examples/puzzletron/evaluation/vlm/evaluator.py +++ b/examples/puzzletron/evaluation/vlm/evaluator.py @@ -20,22 +20,51 @@ import json import os import tempfile +from collections import Counter from collections.abc import Mapping from hashlib import sha256 from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: import argparse from collections.abc import Callable from examples.puzzletron.evaluation import checkpoint -from examples.puzzletron.evaluation.vlm import preflight, tasks +from examples.puzzletron.evaluation.vlm import preflight, suites, tasks __all__ = ["evaluate"] _COMPLETED_RUN_SCHEMA = "modelopt.vlm-evaluation-completed-run/v1" _COMPLETED_RUN_FILENAME = "completed_run.json" +_MMMU_TASK = suites.task_name("mmmu_val") +_MMMU_PARSER_STATUSES = {"fallback_random", "invalid_open", "parsed", "parsed_open"} + + +def _expected_task_populations( + prepared: preflight.PreparedSuite, + configured_tasks: tuple[str, ...], +) -> dict[str, int] | None: + """Map a full-data profile to the exact generated task objects it loads.""" + contract = prepared.profile_contract + if contract is None or contract.manifest["selection"] != "all": + return None + profile_tasks = cast("dict[str, dict[str, object]]", contract.manifest["tasks"]) + expected: dict[str, int] = {} + for source_task, configured_task in zip(prepared.source_tasks, configured_tasks, strict=True): + entry = profile_tasks[source_task] + leaf_populations = entry.get("leaf_populations") + if isinstance(leaf_populations, dict): + selected_leaves = prepared.profile_task_leaves or tuple(leaf_populations) + expected.update( + { + suites.task_name(source_task, leaf=leaf): cast("int", leaf_populations[leaf]) + for leaf in selected_leaves + } + ) + elif "population_rows" in entry: + expected[configured_task] = cast("int", entry["population_rows"]) + return expected or None def _completion_identity( @@ -203,6 +232,96 @@ def _write_completed_run( temporary_path.unlink(missing_ok=True) +def _attach_mmmu_parser_audit(run_result: Mapping[str, object]) -> None: + """Summarize auditable per-sample MMMU parser outcomes in the normalized result.""" + result_path_value = run_result.get("result_path") + if not isinstance(result_path_value, str): + raise RuntimeError("VLM evaluation result is missing its normalized result path") + result_path = Path(result_path_value) + try: + summary = json.loads(result_path.read_text()) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"invalid VLM evaluation summary: {result_path}") from error + if not isinstance(summary, dict): + raise RuntimeError(f"invalid VLM evaluation summary: {result_path}") + + raw_result_path_value = run_result.get("raw_result_path") + if not isinstance(raw_result_path_value, str): + raise RuntimeError("VLM evaluation result is missing its raw result path") + raw_result_path = Path(raw_result_path_value).resolve() + output_root = result_path.parent.resolve() + try: + raw_result_path.relative_to(output_root) + except ValueError as error: + raise RuntimeError("VLM raw result path escapes its evaluation output") from error + result_suffix = "_results.json" + if not raw_result_path.name.endswith(result_suffix): + raise RuntimeError(f"invalid lmms-eval raw result filename: {raw_result_path.name}") + invocation_prefix = raw_result_path.name[: -len(result_suffix)] + sample_logs = [raw_result_path.with_name(f"{invocation_prefix}_samples_{_MMMU_TASK}.jsonl")] + if not sample_logs[0].is_file(): + raise RuntimeError("MMMU evaluation wrote no auditable sample log for its invocation") + statuses: Counter[str] = Counter() + sample_count = 0 + for sample_log in sample_logs: + try: + lines = sample_log.read_text().splitlines() + except OSError as error: + raise RuntimeError(f"cannot read MMMU sample log: {sample_log}") from error + for line_number, line in enumerate(lines, start=1): + if not line.strip(): + continue + try: + sample = json.loads(line) + except json.JSONDecodeError as error: + raise RuntimeError( + f"invalid MMMU sample log record: {sample_log}:{line_number}" + ) from error + accuracy = sample.get("mmmu_acc") if isinstance(sample, Mapping) else None + parser_status = accuracy.get("parser_status") if isinstance(accuracy, Mapping) else None + if ( + not isinstance(parser_status, list) + or not parser_status + or any(status not in _MMMU_PARSER_STATUSES for status in parser_status) + ): + raise RuntimeError( + f"MMMU sample log has no valid parser status: {sample_log}:{line_number}" + ) + statuses.update(cast("list[str]", parser_status)) + sample_count += 1 + + sample_counts = summary.get("sample_counts") + expected_samples = sample_counts.get(_MMMU_TASK) if isinstance(sample_counts, Mapping) else None + if expected_samples != sample_count: + raise RuntimeError( + f"MMMU parser audit covers {sample_count}/{expected_samples} evaluated samples" + ) + summary["mmmu_parser_audit"] = { + "sample_count": sample_count, + "sample_logs": [ + _file_identity(sample_log, root=result_path.parent) for sample_log in sample_logs + ], + "status_counts": dict(sorted(statuses.items())), + } + content = json.dumps(summary, indent=2, sort_keys=True) + "\n" + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + "w", + dir=result_path.parent, + prefix=f".{result_path.name}.", + delete=False, + ) as temporary: + temporary.write(content) + temporary.flush() + os.fsync(temporary.fileno()) + temporary_path = Path(temporary.name) + os.replace(temporary_path, result_path) + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + def evaluate( args: argparse.Namespace, *, @@ -212,6 +331,12 @@ def evaluate( """Prepare and run one pinned VLM profile invocation.""" prepared = preflight.prepare(args) + if ( + settings_overrides + and prepared.profile_contract is not None + and prepared.profile_contract.manifest.get("model") is not None + ): + raise ValueError("model-pinned evaluation profiles do not allow settings overrides") task_root, configured_tasks = tasks.prepare( args.output_dir, suite=prepared.suite, @@ -233,6 +358,7 @@ def evaluate( hf_home=prepared.hf_home, timeout_seconds=checkpoint.DEFAULT_PREFLIGHT_TIMEOUT_SECONDS, model_name=str(settings["model"]), + expected_populations=_expected_task_populations(prepared, configured_tasks), ) report = dict(prepared.report) report.update( @@ -263,6 +389,7 @@ def evaluate( "profile_fingerprint": report.get("profile_fingerprint"), "profile_name": report.get("profile_name"), "profile_schema": report.get("profile_schema"), + "output_budget_contract": report["output_budget_contract"], } chat_template_sha256 = _chat_template_sha256(settings) if chat_template_sha256 is not None: @@ -286,6 +413,8 @@ def evaluate( output_root=output_root, settings=settings, ) + if "mmmu_val" in prepared.source_tasks: + _attach_mmmu_parser_audit(run_result) _write_completed_run(output_root, identity=identity, result=run_result) runs.append(run_result) return { diff --git a/examples/puzzletron/evaluation/vlm/model.py b/examples/puzzletron/evaluation/vlm/model.py index 49c95762e74..88d11f08e14 100644 --- a/examples/puzzletron/evaluation/vlm/model.py +++ b/examples/puzzletron/evaluation/vlm/model.py @@ -18,6 +18,7 @@ from __future__ import annotations import json +from copy import deepcopy from typing import TYPE_CHECKING from jinja2 import TemplateError @@ -72,7 +73,7 @@ def _render_template(content: str, *, source: Path) -> str: raise ValueError(f"Qwen 3.5 chat template is invalid: {source}") from error -def verify_checkpoint(checkpoint: Path, *, profile: str) -> None: +def verify_checkpoint(checkpoint: Path, *, profile: str, model_backend: str = "qwen3_5") -> None: """Verify a Qwen 3.5 VLM-family checkpoint and its local processor assets.""" config = _checkpoint_config(checkpoint) if config.get("model_type") != _MODEL_TYPE: @@ -90,6 +91,41 @@ def verify_checkpoint(checkpoint: Path, *, profile: str) -> None: text_config = config.get("text_config") if not isinstance(text_config, dict) or text_config.get("model_type") != "qwen3_5_text": raise ValueError(f"{profile} checkpoint text_config.model_type must be qwen3_5_text") + if model_backend == "qwen3_5" and realized_checkpoint: + from modelopt.torch.puzzletron.block_config import maybe_cast_block_configs + from modelopt.torch.puzzletron.utils.vllm_adapter import ( + convert_block_configs_to_per_layer_config, + ) + + block_configs = config.get("block_configs") + homogeneity_error = ( + f"{profile} native qwen3_5 backend cannot prove that an AnyModel checkpoint " + "is homogeneous; use a vLLM evaluation profile" + ) + derived_config = deepcopy(config) + derived_text_config = derived_config["text_config"] + derived_text_config.pop("per_layer_config", None) + if not isinstance(block_configs, list) or len(block_configs) != text_config.get( + "num_hidden_layers" + ): + raise ValueError(homogeneity_error) + try: + typed_block_configs = maybe_cast_block_configs(block_configs) + except (TypeError, ValueError) as error: + raise ValueError(homogeneity_error) from error + if typed_block_configs is None or any( + len(block.subblock_configs) < 2 for block in typed_block_configs + ): + raise ValueError(homogeneity_error) + derived_config["block_configs"] = typed_block_configs + convert_block_configs_to_per_layer_config(derived_config, keep_block_configs=True) + derived_per_layer_config = derived_text_config.get("per_layer_config") + declared_per_layer_config = text_config.get("per_layer_config") + if derived_per_layer_config or declared_per_layer_config not in (None, {}): + raise ValueError( + f"{profile} native qwen3_5 backend cannot load a heterogeneous AnyModel " + "per_layer_config; use a vLLM evaluation profile" + ) invalid_geometry = { key: text_config.get(key) for key in _TEXT_GEOMETRY_FIELDS diff --git a/examples/puzzletron/evaluation/vlm/post_mip.py b/examples/puzzletron/evaluation/vlm/post_mip.py index f77b3d15526..9cbca2be61c 100644 --- a/examples/puzzletron/evaluation/vlm/post_mip.py +++ b/examples/puzzletron/evaluation/vlm/post_mip.py @@ -21,6 +21,7 @@ import json import os import warnings +from collections import Counter from pathlib import Path from typing import TYPE_CHECKING @@ -36,8 +37,12 @@ "TASK_PREFIX100_REPEAT2_PROFILE", "evaluate_e2e_full_eval_checkpoint", "evaluate_frozen_campaign_checkpoint", + "evaluate_frozen_campaign_v2_checkpoint", + "evaluate_frozen_campaign_v3_checkpoint", "evaluate_realworldqa_checkpoint", "evaluate_realworldqa_mmmu_prefix100_checkpoint", + "evaluate_reproducibility_smoke_checkpoint", + "evaluate_reproducibility_smoke_v2_checkpoint", "evaluate_short_v1_checkpoint", "register_profiles", ] @@ -55,7 +60,11 @@ _REALWORLDQA_PROFILE = "qwen35_vlm_realworldqa2_prefix2" _BOUNDED_REPEATED_PROFILE = "qwen35_vlm_realworldqa100_mmmu100_prefix100_repeat2" TASK_PREFIX100_REPEAT2_PROFILE = _BOUNDED_REPEATED_PROFILE -_FROZEN_CAMPAIGN_PROFILE = "qwen35_vlm_realworldqa64_mmmu120_mvbench160_frozen_rows_v1" +_FROZEN_CAMPAIGN_PROFILE_V1 = "qwen35_vlm_realworldqa64_mmmu120_mvbench160_frozen_rows_v1" +_FROZEN_CAMPAIGN_PROFILE_V2 = "qwen35_vlm_realworldqa64_mmmu120_mvbench160_frozen_rows_v2" +_FROZEN_CAMPAIGN_PROFILE_V3 = "qwen35_vlm_realworldqa64_mmmu120_mvbench160_frozen_rows_v3" +_REPRODUCIBILITY_SMOKE_PROFILE = "qwen35_vlm_core3_24row_smoke_v1" +_REPRODUCIBILITY_SMOKE_PROFILE_V2 = "qwen35_vlm_core3_24row_smoke_v2" def _run_profile( @@ -64,6 +73,7 @@ def _run_profile( output_root: str | Path, settings: Mapping[str, Any], suite: str, + evaluation_profile: str | None = None, require_manifest: bool = False, ) -> tuple[argparse.Namespace, dict[str, object], Path]: settings = dict(settings) @@ -76,20 +86,28 @@ def _run_profile( output_dir.mkdir(parents=True, exist_ok=True) row_manifest = settings.pop("row_manifest", None) expected_manifest_sha256 = settings.pop("row_manifest_sha256", None) - if require_manifest and (not row_manifest or not expected_manifest_sha256): + if ( + require_manifest + and evaluation_profile is None + and (not row_manifest or not expected_manifest_sha256) + ): raise ValueError( "frozen 344-row campaign profile requires row_manifest and row_manifest_sha256" ) + if require_manifest and evaluation_profile is not None and not expected_manifest_sha256: + raise ValueError("frozen 344-row campaign evaluation profile requires row_manifest_sha256") + if row_manifest is not None and evaluation_profile is not None: + raise ValueError("an embedded evaluation profile cannot be overridden by row_manifest") + if expected_manifest_sha256 is not None and ( + not isinstance(expected_manifest_sha256, str) + or len(expected_manifest_sha256) != 64 + or any(character not in "0123456789abcdef" for character in expected_manifest_sha256) + ): + raise ValueError( + "frozen 344-row campaign manifest SHA256 must be 64 lowercase hex characters" + ) quick_manifest = Path(row_manifest).expanduser().absolute() if row_manifest else None if quick_manifest is not None: - if ( - not isinstance(expected_manifest_sha256, str) - or len(expected_manifest_sha256) != 64 - or any(character not in "0123456789abcdef" for character in expected_manifest_sha256) - ): - raise ValueError( - "frozen 344-row campaign manifest SHA256 must be 64 lowercase hex characters" - ) actual_manifest_sha256 = suites.manifest_sha256(suites.load_quick_manifest(quick_manifest)) if actual_manifest_sha256 != expected_manifest_sha256: raise ValueError( @@ -99,6 +117,7 @@ def _run_profile( args = argparse.Namespace( checkpoint=Path(checkpoint_path).expanduser().absolute(), output_dir=output_dir, + profile=evaluation_profile, suite=suite, batch_size=int(settings.pop("batch_size", 1)), seed=42, @@ -113,6 +132,14 @@ def _run_profile( profile_path = output_dir / "profile.json" def write_preflight(report: dict[str, object]) -> None: + if evaluation_profile is not None and ( + expected_manifest_sha256 is not None + and report.get("quick_manifest_sha256") != expected_manifest_sha256 + ): + raise ValueError( + "frozen 344-row campaign manifest SHA256 differs from the campaign identity: " + f"{report.get('quick_manifest_sha256')} != {expected_manifest_sha256}" + ) checkpoint.write_generated( profile_path, json.dumps(report, indent=2, sort_keys=True) + "\n", @@ -140,9 +167,25 @@ def register_profiles() -> None: evaluate_realworldqa_mmmu_prefix100_checkpoint, ) register_downstream_evaluation_profile( - _FROZEN_CAMPAIGN_PROFILE, + _FROZEN_CAMPAIGN_PROFILE_V1, evaluate_frozen_campaign_checkpoint, ) + register_downstream_evaluation_profile( + _FROZEN_CAMPAIGN_PROFILE_V2, + evaluate_frozen_campaign_v2_checkpoint, + ) + register_downstream_evaluation_profile( + _FROZEN_CAMPAIGN_PROFILE_V3, + evaluate_frozen_campaign_v3_checkpoint, + ) + register_downstream_evaluation_profile( + _REPRODUCIBILITY_SMOKE_PROFILE, + evaluate_reproducibility_smoke_checkpoint, + ) + register_downstream_evaluation_profile( + _REPRODUCIBILITY_SMOKE_PROFILE_V2, + evaluate_reproducibility_smoke_v2_checkpoint, + ) # Deprecated compatibility aliases. New recipes must use explicit task and # row-selection identities above. register_downstream_evaluation_profile( @@ -184,6 +227,110 @@ def evaluate_frozen_campaign_checkpoint( } +def evaluate_frozen_campaign_v2_checkpoint( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], +) -> dict[str, Any]: + """Evaluate one checkpoint on the current-image frozen campaign profile.""" + + args, result, profile_path = _run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + suite="short", + evaluation_profile="short-native-v2", + require_manifest=True, + ) + runs = result["runs"] + if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): + raise RuntimeError("pinned VLM frozen 344-row profile returned an invalid run count") + return { + **runs[0], + "profile_path": str(profile_path), + "checkpoint": str(args.checkpoint), + } + + +def evaluate_frozen_campaign_v3_checkpoint( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], +) -> dict[str, Any]: + """Evaluate heterogeneous materialized checkpoints with the current vLLM profile.""" + + args, result, profile_path = _run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + suite="short", + evaluation_profile="short-vllm-v2", + require_manifest=True, + ) + runs = result["runs"] + if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): + raise RuntimeError("pinned VLM frozen 344-row profile returned an invalid run count") + return { + **runs[0], + "profile_path": str(profile_path), + "checkpoint": str(args.checkpoint), + } + + +def evaluate_reproducibility_smoke_checkpoint( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], +) -> dict[str, Any]: + """Evaluate one checkpoint on the immutable 24-row lifecycle smoke.""" + + args, result, profile_path = _run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + suite="short", + evaluation_profile="smoke-native-v1", + require_manifest=True, + ) + runs = result["runs"] + if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): + raise RuntimeError("pinned VLM 24-row smoke returned an invalid run count") + return { + **runs[0], + "profile_path": str(profile_path), + "checkpoint": str(args.checkpoint), + } + + +def evaluate_reproducibility_smoke_v2_checkpoint( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], +) -> dict[str, Any]: + """Evaluate a heterogeneous materialized checkpoint on the 24-row smoke.""" + + args, result, profile_path = _run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + suite="short", + evaluation_profile="smoke-vllm-v1", + require_manifest=True, + ) + runs = result["runs"] + if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): + raise RuntimeError("pinned VLM 24-row smoke returned an invalid run count") + return { + **runs[0], + "profile_path": str(profile_path), + "checkpoint": str(args.checkpoint), + } + + def evaluate_short_v1_checkpoint( checkpoint_path: str | Path, *, @@ -193,7 +340,7 @@ def evaluate_short_v1_checkpoint( """Compatibility alias for the explicit frozen-row campaign profile.""" warnings.warn( - f"qwen35_vlm_short_v1 is deprecated; use {_FROZEN_CAMPAIGN_PROFILE}", + f"qwen35_vlm_short_v1 is deprecated; use {_FROZEN_CAMPAIGN_PROFILE_V1}", DeprecationWarning, stacklevel=2, ) @@ -253,6 +400,17 @@ def evaluate_realworldqa_mmmu_prefix100_checkpoint( for name in sorted(metric_names) } result_paths = [str(item["result_path"]) for item in runs] + sample_counts: Counter[str] = Counter() + parser_status_counts: Counter[str] = Counter() + parser_sample_count = 0 + for result_path in result_paths: + payload = json.loads(Path(result_path).read_text()) + sample_counts.update({key: int(value) for key, value in payload["sample_counts"].items()}) + parser_audit = payload.get("mmmu_parser_audit") or {} + parser_sample_count += int(parser_audit.get("sample_count", 0)) + parser_status_counts.update( + {key: int(value) for key, value in (parser_audit.get("status_counts") or {}).items()} + ) summary_path = args.output_dir / "realworldqa_mmmu_prefix100_repeat2_summary.json" atomic_write_json( summary_path, @@ -261,6 +419,11 @@ def evaluate_realworldqa_mmmu_prefix100_checkpoint( "metrics": metrics, "profile": _BOUNDED_REPEATED_PROFILE, "result_paths": result_paths, + "sample_counts": dict(sorted(sample_counts.items())), + "mmmu_parser_audit": { + "sample_count": parser_sample_count, + "status_counts": dict(sorted(parser_status_counts.items())), + }, "suite": args.suite, }, ) diff --git a/examples/puzzletron/evaluation/vlm/preflight.py b/examples/puzzletron/evaluation/vlm/preflight.py index f7b5ca988f4..3f0e516fa85 100644 --- a/examples/puzzletron/evaluation/vlm/preflight.py +++ b/examples/puzzletron/evaluation/vlm/preflight.py @@ -17,7 +17,9 @@ from __future__ import annotations +import hashlib import importlib.util +import json import os from dataclasses import dataclass from pathlib import Path @@ -53,16 +55,51 @@ def _backend_policy( ) -> dict[str, object]: """Return the selected profile's backend contract or the legacy default.""" if profile_contract is None: - return {"name": "vllm"} + return { + "attention_implementation": "sdpa", + "enable_thinking": False, + "name": "qwen3_5", + } return cast("dict[str, object]", profile_contract.manifest["backend"]) +def _output_budget_contract( + source_tasks: tuple[str, ...], *, model_backend: str +) -> dict[str, dict[str, object]]: + """Describe the output-token budget that each adapter will actually apply.""" + if model_backend == "vllm": + resolution = "max(task_max_new_tokens, model_max_new_tokens_floor=1)" + limitation = ( + "the pinned generic vLLM adapter treats its model-level max_new_tokens as a floor" + ) + else: + resolution = "task_max_new_tokens_overrides_adapter_default" + limitation = None + return { + task: { + "adapter": model_backend, + "effective_max_new_tokens": profile.VLM_BENCHMARK_DATASETS[task].max_new_tokens, + "limitation": limitation, + "requested_max_new_tokens": profile.VLM_BENCHMARK_DATASETS[task].max_new_tokens, + "resolution": resolution, + } + for task in source_tasks + } + + def prepare(args: argparse.Namespace) -> PreparedSuite: """Resolve and validate everything needed before model loading starts.""" profile_name = getattr(args, "profile", None) profile_contract = contracts.load_profile(profile_name) if profile_name is not None else None suite, source_tasks, profile_task_leaves = _resolve_task_selection(args, profile_contract) - model.verify_checkpoint(args.checkpoint, profile="VLM benchmark") + backend = _backend_policy(profile_contract) + model.verify_checkpoint( + args.checkpoint, + profile="VLM benchmark", + model_backend=str(backend["name"]), + ) + hf_home = _hf_home(args.hf_home) + _verify_profile_model(args.checkpoint, profile_contract, hf_home=hf_home) execution_policy = suites.execution_policy(suite, timeout_seconds=args.timeout_seconds) revisions = {task: profile.VLM_BENCHMARK_DATASETS[task].revision for task in source_tasks} @@ -83,7 +120,6 @@ def prepare(args: argparse.Namespace) -> PreparedSuite: for task in source_tasks: tasks.task_config(profile.VLM_BENCHMARK_DATASETS[task].task_config) - hf_home = _hf_home(args.hf_home) _verify_media_roots(hf_home, source_tasks) dataset_snapshots = { task: suites.offline_dataset_snapshot(hf_home, task, revisions[task]) @@ -138,8 +174,16 @@ def _resolve_task_selection( if profile_task is not None: if profile_contract is None: raise ValueError("--profile-task requires a versioned evaluation profile") - if profile_contract.name not in {"full-v1", "short-all-native-v1"}: - raise ValueError("--profile-task is supported only for full-v1 and short-all-native-v1") + if profile_contract.name not in { + "full-v1", + "core3-full-native-v1", + "core3-full-vllm-v1", + "short-all-native-v1", + "short-all-native-v2", + }: + raise ValueError( + "--profile-task is supported only for full-data and short-all-native profiles" + ) if profile_task_shard is not None and profile_task is None: raise ValueError("--profile-task-shard requires --profile-task") @@ -151,6 +195,34 @@ def _resolve_task_selection( return suite, source_tasks, _profile_task_leaves(profile_task, profile_task_shard) +def _verify_profile_model( + checkpoint_path: Path, + profile_contract: contracts.ProfileContract | None, + *, + hf_home: Path, +) -> None: + """Require a model-pinned profile to use its exact local Hub snapshot.""" + if profile_contract is None: + return + model_pin = profile_contract.manifest.get("model") + if not isinstance(model_pin, dict): + return + repository = cast("str", model_pin["repository"]) + revision = cast("str", model_pin["revision"]) + configured_hub_cache = os.environ.get("HF_HUB_CACHE") + hub_cache = ( + Path(configured_hub_cache).expanduser().absolute() + if configured_hub_cache + else (hf_home / "hub").absolute() + ) + repository_cache = f"models--{repository.replace('/', '--')}" + expected_snapshot = hub_cache / repository_cache / "snapshots" / revision + if checkpoint_path.resolve() != expected_snapshot.resolve(): + raise ValueError( + f"{profile_contract.name} requires the exact local Hub snapshot {repository}@{revision}" + ) + + def _row_manifest( args: argparse.Namespace, suite: str, @@ -171,12 +243,11 @@ def _row_manifest( manifest_tasks = cast("dict[str, object]", exact_rows["tasks"]) task_entry = cast("dict[str, object]", manifest_tasks[profile_task]) if profile_task_leaves is not None: - selected_leaves = {f"{profile_task}_{leaf}" for leaf in profile_task_leaves} - rows = cast("list[dict[str, object]]", task_entry["rows"]) - task_entry = { - **task_entry, - "rows": [row for row in rows if row.get("leaf_task") in selected_leaves], - } + task_entry = _shard_exact_row_task( + task_entry, + task=profile_task, + leaves=profile_task_leaves, + ) exact_rows = {**exact_rows, "tasks": {profile_task: task_entry}} return suites.validate_exact_rows_manifest( exact_rows, @@ -190,6 +261,45 @@ def _row_manifest( return None +def _shard_exact_row_task( + entry: dict[str, object], *, task: str, leaves: tuple[str, ...] +) -> dict[str, object]: + """Filter grouped rows and derive a self-consistent sampling audit.""" + selected_leaf_tasks = {f"{task}_{leaf}" for leaf in leaves} + rows = [ + row + for row in cast("list[dict[str, object]]", entry["rows"]) + if row.get("leaf_task") in selected_leaf_tasks + ] + selection = entry.get("selection") + if not isinstance(selection, dict): + return {**entry, "rows": rows} + strata = [ + stratum + for stratum in cast("list[dict[str, object]]", selection["strata"]) + if stratum.get("name") in leaves + ] + indices = sorted(cast("int", row["source_row_index"]) for row in rows) + quantiles = { + "method": "lower-order-statistic", + **{ + f"p{percentile}": indices[(len(indices) - 1) * percentile // 100] + for percentile in (0, 25, 50, 75, 100) + }, + } + derived_selection = { + **selection, + "population_rows": sum(cast("int", stratum["population_rows"]) for stratum in strata), + "selected_rows": len(rows), + "strata": strata, + "selected_index_quantiles": quantiles, + "selected_row_identities_sha256": hashlib.sha256( + json.dumps(rows, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest(), + } + return {**entry, "selection": derived_selection, "rows": rows} + + def _profile_task_leaves( profile_task: str | None, shard: tuple[int, int] | None ) -> tuple[str, ...] | None: @@ -294,6 +404,16 @@ def _report( profile_task_shard = getattr(args, "profile_task_shard", None) backend = _backend_policy(profile_contract) model_backend = str(backend["name"]) + output_budget_contract = _output_budget_contract( + source_tasks, + model_backend=model_backend, + ) + profile_population_rows = None + if profile_contract is not None and profile_contract.manifest.get("model") is not None: + profile_tasks = cast("dict[str, dict[str, object]]", profile_contract.manifest["tasks"]) + profile_population_rows = { + task: profile_tasks[task]["population_rows"] for task in source_tasks + } return { "schema": "modelopt.vlm-evaluation-preflight/v1", "profile": suites.EVALUATION_PROFILE, @@ -304,15 +424,23 @@ def _report( "profile_fingerprint": ( profile_contract.fingerprint if profile_contract is not None else None ), + "model_pin": ( + profile_contract.manifest.get("model") if profile_contract is not None else None + ), + "profile_population_rows": profile_population_rows, "suite": suite, "checkpoint": str(args.checkpoint), "lmms_eval_revision": lmms_eval_revision, "model_backend": model_backend, "backend_limitations": ( - ["generic vLLM video messages do not preserve native Qwen 3.5 timestamps"] + [ + "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", + "pinned generic vLLM max_new_tokens is a model-level lower bound", + ] if model_backend == "vllm" else [] ), + "output_budget_contract": output_budget_contract, "source_tasks": list(source_tasks), "profile_task": getattr(args, "profile_task", None), "profile_task_shard": ( @@ -335,6 +463,14 @@ def _report( "quick_selected_rows": ( suites.manifest_selected_rows(quick_manifest) if quick_manifest is not None else None ), + "quick_row_identities": ( + suites.manifest_row_identities(quick_manifest) if quick_manifest is not None else None + ), + "quick_task_denominators": ( + suites.manifest_task_denominators(quick_manifest) + if quick_manifest is not None + else None + ), "judge_free_mmvu_rows": ( [row[0] for row in suites.MMVU_SMOKE_ROWS] if suite == "mmvu-smoke" else None ), @@ -370,7 +506,10 @@ def settings( "batch_size": args.batch_size, "seed": args.seed, "timeout_seconds": execution_policy["timeout_seconds"], - "log_samples": prepared.suite in {"quick", "short", suites.TASK_PREFIX100_REPEAT2_SUITE}, + "log_samples": ( + "mmmu_val" in prepared.source_tasks + or prepared.suite in {"quick", "short", suites.TASK_PREFIX100_REPEAT2_SUITE} + ), "gen_kwargs": { "temperature": generation_policy["temperature"], "do_sample": generation_policy["do_sample"], @@ -393,7 +532,7 @@ def settings( "attn_implementation": backend["attention_implementation"], "device": "cuda", "device_map": "cuda", - "enable_thinking": backend["enable_thinking"], + "enable_thinking": generation_policy["enable_thinking"], "fps": frame_policy["fps"], "max_frames": frame_policy["max_frames"], }, @@ -404,9 +543,18 @@ def settings( "checkpoint_arg": "model", "reasoning_parser": backend.get("reasoning_parser", "qwen3"), "model_args": { + **( + {"attention_config": backend["attention_config"]} + if "attention_config" in backend + else {} + ), "chat_template": str(chat_template), "fps": frame_policy["fps"], + # The pinned adapter takes max(task value, model value). A positive floor of one + # leaves every task-level output budget authoritative. + "max_new_tokens": 1, "max_frame_num": frame_policy["max_frames"], + **({"enforce_eager": backend["enforce_eager"]} if "enforce_eager" in backend else {}), }, } @@ -427,6 +575,6 @@ def _verify_backend_dependencies(model_backend: str) -> None: """Fail before native Qwen evaluation when its vision utilities are unavailable.""" if model_backend == "qwen3_5" and importlib.util.find_spec("qwen_vl_utils") is None: raise RuntimeError( - "native Qwen 3.5 evaluation requires qwen-vl-utils; install the native VLM " - "requirements or use the supported Puzzletron environment" + "native Qwen 3.5 evaluation requires qwen-vl-utils; use the supported " + "Puzzletron worker image" ) diff --git a/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py b/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py index d7e65c51ab9..02275b58a07 100644 --- a/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py +++ b/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py @@ -18,6 +18,9 @@ from __future__ import annotations import argparse +import ctypes +import errno +import fcntl import hashlib import json import os @@ -27,9 +30,9 @@ import tarfile import tempfile import zipfile -from contextlib import suppress +from contextlib import contextmanager, suppress from pathlib import Path, PurePosixPath -from typing import IO, TYPE_CHECKING +from typing import IO, TYPE_CHECKING, cast from urllib.parse import urlparse from huggingface_hub import HfApi, get_hf_file_metadata, hf_hub_url, snapshot_download @@ -44,15 +47,19 @@ from examples.puzzletron.evaluation.vlm import profile # noqa: E402 if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Iterable, Iterator _MARKER_NAME = ".modelopt_vlm_benchmark_preparation.json" _RANGE_MARKER_NAME = ".modelopt_vlm_benchmark_range_download.json" +_SNAPSHOT_MARKER_PREFIX = ".modelopt_vlm_benchmark_snapshot_" _TEMPORARY_DIR_NAME = ".modelopt_vlm_benchmark_temporary" +_LOCK_DIRECTORY_NAME = ".modelopt_vlm_benchmark_locks" _COPY_CHUNK_BYTES = 8 * 1024 * 1024 +_AT_FDCWD = -100 +_RENAME_EXCHANGE = 2 -DATASETS = profile.VLM_BENCHMARK_VIDEO_DATASETS +DATASETS = profile.VLM_BENCHMARK_DATASETS _MVBENCH_MEDIA_ROOTS = ( "FunQA_test", @@ -69,6 +76,20 @@ ) +def benchmark_catalog_contract(tasks: Iterable[str]) -> dict[str, dict[str, object]]: + """Return the authoritative preparation contract for selected benchmark tasks.""" + + return { + task: { + "repository": DATASETS[task].repository, + "revision": DATASETS[task].revision, + "requires_media": DATASETS[task].preparation_dir is not None, + "preparation_dir": DATASETS[task].preparation_dir, + } + for task in tasks + } + + def _task_selection(value: str) -> tuple[str, ...]: tasks = tuple(dict.fromkeys(part.strip() for part in value.split(",") if part.strip())) unknown = sorted(set(tasks) - set(DATASETS)) @@ -79,6 +100,10 @@ def _task_selection(value: str) -> tuple[str, ...]: return tasks +def _path_traverses_symlink(path: Path) -> bool: + return any(candidate.is_symlink() for candidate in (path, *path.parents) if candidate.exists()) + + def _hub_snapshot(hf_home: Path, task: str) -> Path: spec = DATASETS[task] repository_cache = f"datasets--{spec.repository.replace('/', '--')}" @@ -108,6 +133,135 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _inventory_sha256(entries: list[dict[str, object]]) -> str: + payload = json.dumps(entries, separators=(",", ":"), sort_keys=True).encode() + return hashlib.sha256(payload).hexdigest() + + +def _inventory_summary_is_valid(payload: dict[str, object], entries: object) -> bool: + if not isinstance(entries, list) or not entries: + return False + if any( + not isinstance(entry, dict) or not isinstance(entry.get("bytes"), int) for entry in entries + ): + return False + recorded_count = payload.get("file_count") if "file_count" in payload else payload.get("files") + return ( + payload.get("inventory_sha256") == _inventory_sha256(entries) + and recorded_count == len(entries) + and payload.get("bytes") == sum(entry["bytes"] for entry in entries) + ) + + +def _inventory(root: Path, *, repository_cache: Path | None = None) -> list[dict[str, object]]: + entries = [] + for path in sorted(root.rglob("*")): + if path.name == _MARKER_NAME: + continue + if path.is_dir() and not path.is_symlink(): + continue + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + if repository_cache is None: + raise ValueError(f"prepared media inventory contains a symlink: {path}") + resolved = path.resolve(strict=True) + if not resolved.is_relative_to(repository_cache) or not resolved.is_file(): + raise ValueError(f"snapshot inventory path escapes its repository cache: {path}") + stat_result = resolved.stat() + target = resolved.relative_to(repository_cache).as_posix() + blob_name = resolved.name + content_sha256 = _sha256(resolved) + if ( + len(blob_name) == 64 + and all(character in "0123456789abcdef" for character in blob_name.lower()) + and content_sha256 != blob_name.lower() + ): + raise ValueError(f"Hub blob content differs from its SHA-256 identity: {resolved}") + entries.append( + { + "path": relative, + "kind": "hub_blob_symlink", + "target": target, + "bytes": stat_result.st_size, + "mtime_ns": stat_result.st_mtime_ns, + "ctime_ns": stat_result.st_ctime_ns, + "sha256": content_sha256, + } + ) + continue + if not path.is_file(): + raise ValueError(f"inventory path is not a regular file: {path}") + stat_result = path.stat() + entries.append( + { + "path": relative, + "kind": "file", + "bytes": stat_result.st_size, + "mtime_ns": stat_result.st_mtime_ns, + "ctime_ns": stat_result.st_ctime_ns, + "sha256": _sha256(path), + } + ) + if not entries: + raise ValueError(f"inventory root contains no files: {root}") + return entries + + +def _inventory_is_current( + root: Path, + entries: object, + *, + repository_cache: Path | None = None, +) -> bool: + if root.is_symlink() or not root.is_dir() or not isinstance(entries, list) or not entries: + return False + expected_paths = [] + for entry in entries: + if not isinstance(entry, dict): + return False + relative = entry.get("path") + if not isinstance(relative, str): + return False + try: + safe_relative = _safe_relative_path(relative) + except ValueError: + return False + path = root.joinpath(*safe_relative.parts) + expected_paths.append(relative) + if entry.get("kind") == "hub_blob_symlink": + if repository_cache is None or not path.is_symlink(): + return False + try: + resolved = path.resolve(strict=True) + except OSError: + return False + if ( + not resolved.is_relative_to(repository_cache) + or not resolved.is_file() + or resolved.relative_to(repository_cache).as_posix() != entry.get("target") + ): + return False + inspected = resolved + elif entry.get("kind") == "file": + if path.is_symlink() or not path.is_file(): + return False + inspected = path + else: + return False + stat_result = inspected.stat() + if stat_result.st_size != entry.get("bytes"): + return False + expected_sha256 = entry.get("sha256") + if not isinstance(expected_sha256, str) or _sha256(inspected) != expected_sha256: + return False + observed_paths = sorted( + path.relative_to(root).as_posix() + for path in root.rglob("*") + if not (path.is_dir() and not path.is_symlink()) and path.name != _MARKER_NAME + ) + return sorted(expected_paths) == observed_paths + + def _lfs_sha256(entry: object) -> str | None: lfs = getattr(entry, "lfs", None) if lfs is None: @@ -125,13 +279,60 @@ def _range_download_marker(hf_home: Path, task: str) -> Path: return hf_home / "hub" / repository_cache / _RANGE_MARKER_NAME +def _snapshot_inventory_marker(hf_home: Path, task: str) -> Path: + snapshot = _hub_snapshot(hf_home, task) + return snapshot.parent.parent / f"{_SNAPSHOT_MARKER_PREFIX}{DATASETS[task].revision}.json" + + +def _snapshot_inventory_report(hf_home: Path, task: str, snapshot: Path) -> dict[str, object]: + repository_cache = snapshot.parent.parent.resolve() + entries = _inventory(snapshot, repository_cache=repository_cache) + marker = _snapshot_inventory_marker(hf_home, task) + report = { + "schema": "modelopt.vlm-benchmark-snapshot-inventory/v1", + "task": task, + "repository": DATASETS[task].repository, + "revision": DATASETS[task].revision, + "snapshot": str(snapshot), + "files": entries, + "file_count": len(entries), + "bytes": sum(cast("int", entry["bytes"]) for entry in entries), + "inventory_sha256": _inventory_sha256(entries), + } + _write_json_atomic(marker, report) + return {**report, "manifest": str(marker)} + + +def _snapshot_inventory_is_current(report: object) -> bool: + if not isinstance(report, dict): + return False + snapshot_value = report.get("snapshot") + manifest_value = report.get("manifest") + entries = report.get("files") + if not isinstance(snapshot_value, str) or not isinstance(manifest_value, str): + return False + snapshot = Path(snapshot_value) + repository_cache = snapshot.parent.parent.resolve() + marker = Path(manifest_value) + try: + recorded = json.loads(marker.read_text()) + except (OSError, json.JSONDecodeError): + return False + expected = {key: value for key, value in report.items() if key != "manifest"} + return ( + recorded == expected + and _inventory_summary_is_valid(report, entries) + and _inventory_is_current(snapshot, entries, repository_cache=repository_cache) + ) + + def _ensure_directory_path(root: Path, directory: Path) -> None: if not directory.is_relative_to(root): raise ValueError(f"directory escapes its owned root: {directory}") relative = directory.relative_to(root) if ".." in relative.parts: raise ValueError(f"directory escapes its owned root: {directory}") - if root.is_symlink() or not root.is_dir(): + if _path_traverses_symlink(root) or not root.is_dir(): raise ValueError(f"owned root must be a regular directory: {root}") current = root for part in relative.parts: @@ -291,6 +492,8 @@ def _marker_payload(task: str, *, status: str) -> dict[str, object]: "task": task, "repository": spec.repository, "revision": spec.revision, + "requires_media": spec.preparation_dir is not None, + "preparation_dir": spec.preparation_dir, "status": status, } @@ -314,6 +517,28 @@ def _write_marker(target: Path, payload: dict[str, object]) -> None: _write_json_atomic(target / _MARKER_NAME, payload) +@contextmanager +def _task_lock(hf_home: Path, task: str) -> Iterator[None]: + lock_root = hf_home / _LOCK_DIRECTORY_NAME + _ensure_directory_path(hf_home, lock_root) + lock_path = lock_root / f"{task}.lock" + if lock_path.is_symlink(): + raise ValueError(f"benchmark preparation lock must not be a symlink: {lock_path}") + flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(lock_path, flags, 0o600) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ValueError(f"benchmark preparation lock must be a regular file: {lock_path}") + with os.fdopen(descriptor, "r+") as stream: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX) + yield + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + except BaseException: + with suppress(OSError): + os.close(descriptor) + raise + + def _cleanup_temporary_directory(target: Path) -> None: temporary = target / _TEMPORARY_DIR_NAME if not temporary.exists() and not temporary.is_symlink(): @@ -327,7 +552,62 @@ def _cleanup_temporary_directory(target: Path) -> None: temporary.rmdir() -def _prepare_target(hf_home: Path, task: str) -> tuple[Path, dict[str, object] | None]: +def _remove_owned_media_root(hf_home: Path, target: Path) -> None: + if ( + target.is_symlink() + or not target.is_dir() + or not target.is_relative_to(hf_home) + or target == hf_home + ): + raise ValueError(f"prepared media root is unsafe to repair: {target}") + for entry in target.rglob("*"): + if entry.is_symlink(): + raise ValueError(f"prepared media repair refuses a symlink: {entry}") + shutil.rmtree(target) + + +def _atomic_exchange_directories(first: Path, second: Path) -> bool: + """Atomically exchange two directories when the host provides renameat2.""" + + renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None) + if renameat2 is None: + return False + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + _AT_FDCWD, + os.fsencode(first), + _AT_FDCWD, + os.fsencode(second), + _RENAME_EXCHANGE, + ) + if result == 0: + return True + error_number = ctypes.get_errno() + if error_number in {errno.ENOSYS, errno.EINVAL, errno.ENOTSUP}: + return False + raise OSError(error_number, os.strerror(error_number)) + + +def _media_marker_is_current(target: Path, task: str, observed: object) -> bool: + if not isinstance(observed, dict): + return False + expected = _marker_payload(task, status="complete") + if any(observed.get(key) != value for key, value in expected.items()): + return False + inventory = observed.get("inventory") + return _inventory_summary_is_valid(observed, inventory) and _inventory_is_current( + target, inventory + ) + + +def _inspect_prepare_target(hf_home: Path, task: str) -> tuple[Path, dict[str, object] | None]: preparation_dir = DATASETS[task].preparation_dir if preparation_dir is None: raise AssertionError(f"video dataset has no preparation directory: {task}") @@ -342,23 +622,22 @@ def _prepare_target(hf_home: Path, task: str) -> tuple[Path, dict[str, object] | observed = json.loads(marker.read_text()) except (FileNotFoundError, json.JSONDecodeError) as error: raise FileExistsError( - f"refusing a pre-existing media root without a valid marker: {target}" + f"refusing to repair a media root without a readable ownership marker: {target}" ) from error - for key in ("schema", "task", "repository", "revision"): - if observed.get(key) != expected[key]: - raise ValueError(f"preparation marker differs for {task}: {key}") - if observed.get("status") == "complete": + if _media_marker_is_current(target, task, observed): return target, observed - if observed.get("status") != "in_progress": - raise ValueError(f"preparation marker has an invalid status for {task}") - return target, None - staging = Path(tempfile.mkdtemp(prefix=f".{target.name}.modelopt-staging.", dir=target.parent)) - try: - _write_marker(staging, expected) - os.replace(staging, target) - finally: - with suppress(FileNotFoundError, OSError): - staging.rmdir() + if not isinstance(observed, dict): + raise FileExistsError(f"media root ownership marker must be an object: {marker}") + for key, value in expected.items(): + if key == "status": + continue + if observed.get(key) != value: + raise ValueError(f"refusing to repair media with mismatched ownership: {key}") + if observed.get("status") not in {"in_progress", "complete"}: + raise ValueError("refusing to repair media with an invalid ownership-marker status") + for entry in target.rglob("*"): + if entry.is_symlink(): + raise ValueError(f"prepared media repair refuses a symlink: {entry}") return target, None @@ -624,23 +903,42 @@ def _extract(task: str, snapshot: Path, target: Path) -> list[dict[str, object]] def _prepare(hf_home: Path, task: str, snapshot: Path) -> dict[str, object]: - target, complete = _prepare_target(hf_home, task) - if complete is not None: - return complete - _cleanup_temporary_directory(target) - archives = _extract(task, snapshot, target) - _cleanup_temporary_directory(target) - files = [path for path in target.rglob("*") if path.is_file() and path.name != _MARKER_NAME] - report = { - **_marker_payload(task, status="complete"), - "snapshot": str(snapshot), - "media_root": str(target), - "archives": archives, - "files": len(files), - "bytes": sum(path.stat().st_size for path in files), - } - _write_marker(target, report) - return report + with _task_lock(hf_home, task): + target, complete = _inspect_prepare_target(hf_home, task) + if complete is not None: + return complete + staging = Path( + tempfile.mkdtemp(prefix=f".{target.name}.modelopt-staging.", dir=target.parent) + ) + try: + _write_marker(staging, _marker_payload(task, status="in_progress")) + archives = _extract(task, snapshot, staging) + _cleanup_temporary_directory(staging) + inventory = _inventory(staging) + report = { + **_marker_payload(task, status="complete"), + "snapshot": str(snapshot), + "media_root": str(target), + "archives": archives, + "files": len(inventory), + "bytes": sum(cast("int", entry["bytes"]) for entry in inventory), + "inventory": inventory, + "inventory_sha256": _inventory_sha256(inventory), + } + _write_marker(staging, report) + if target.exists(): + if not _atomic_exchange_directories(staging, target): + raise RuntimeError( + "atomic media-directory exchange is unavailable; " + "the existing prepared root was preserved" + ) + _remove_owned_media_root(hf_home, staging) + else: + os.replace(staging, target) + return report + finally: + if staging.exists(): + _remove_owned_media_root(hf_home, staging) def _build_parser() -> argparse.ArgumentParser: @@ -664,35 +962,116 @@ def _build_parser() -> argparse.ArgumentParser: return parser -def main(argv: list[str] | None = None) -> int: - args = _build_parser().parse_args(argv) - hf_home = args.hf_home.expanduser().absolute() - if hf_home.is_symlink(): +def prepare_benchmark_datasets( + hf_home: Path, + tasks: Iterable[str], + *, + max_workers: int = 8, + range_resume: bool = False, + expected_catalog: dict[str, object] | None = None, +) -> list[dict[str, object]]: + """Download pinned benchmark snapshots and prepare media when required.""" + + if max_workers <= 0: + raise ValueError("max_workers must be positive") + if isinstance(tasks, str): + raise TypeError("tasks must be an iterable of task names, not a string") + hf_home = hf_home.expanduser().absolute() + if _path_traverses_symlink(hf_home): raise ValueError(f"HF home must not be a symlink: {hf_home}") hf_home.mkdir(parents=True, exist_ok=True) + raw_tasks = tuple(tasks) + if any(not isinstance(task, str) or not task.strip() for task in raw_tasks): + raise TypeError("tasks must contain non-empty task names") + selected = tuple(task.strip() for task in raw_tasks) + if len(set(selected)) != len(selected): + raise ValueError("VLM benchmark data tasks must be unique") + unknown = sorted(set(selected) - set(DATASETS)) + if not selected: + raise ValueError("at least one VLM benchmark data task is required") + if unknown: + raise ValueError(f"unsupported VLM benchmark data tasks: {unknown}") + catalog = benchmark_catalog_contract(selected) + if expected_catalog is not None and expected_catalog != catalog: + raise ValueError("configured VLM benchmark catalog differs from the authoritative catalog") + reports = [] - for task in args.tasks: + for task in selected: snapshot = ( - _hub_snapshot(hf_home, task) - if args.extract_only - else ( - _range_download(hf_home, task) - if args.range_resume - else _download(hf_home, task, max_workers=args.max_workers) - ) + _range_download(hf_home, task) + if range_resume + else _download(hf_home, task, max_workers=max_workers) ) if not snapshot.is_dir(): raise FileNotFoundError(f"pinned dataset snapshot is missing: {snapshot}") + spec = DATASETS[task] + snapshot_inventory = _snapshot_inventory_report(hf_home, task, snapshot) report: dict[str, object] = { "task": task, - "repository": DATASETS[task].repository, - "revision": DATASETS[task].revision, + "repository": spec.repository, + "revision": spec.revision, "snapshot": str(snapshot), + "requires_media": spec.preparation_dir is not None, + "preparation_dir": spec.preparation_dir, + "snapshot_inventory": snapshot_inventory, "status": "downloaded", } - if not args.download_only: - report = _prepare(hf_home, task, snapshot) + if spec.preparation_dir is not None: + report = {**report, **_prepare(hf_home, task, snapshot)} reports.append(report) + return reports + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + hf_home = args.hf_home.expanduser().absolute() + if _path_traverses_symlink(hf_home): + raise ValueError(f"HF home must not be a symlink: {hf_home}") + hf_home.mkdir(parents=True, exist_ok=True) + if args.extract_only: + reports = [] + for task in args.tasks: + snapshot = _hub_snapshot(hf_home, task) + if not snapshot.is_dir(): + raise FileNotFoundError(f"pinned dataset snapshot is missing: {snapshot}") + spec = DATASETS[task] + report: dict[str, object] = { + "task": task, + "repository": spec.repository, + "revision": spec.revision, + "snapshot": str(snapshot), + "requires_media": spec.preparation_dir is not None, + "status": "downloaded", + } + if spec.preparation_dir is not None: + report = {**report, **_prepare(hf_home, task, snapshot)} + reports.append(report) + elif args.download_only: + reports = [] + for task in args.tasks: + snapshot = ( + _range_download(hf_home, task) + if args.range_resume + else _download(hf_home, task, max_workers=args.max_workers) + ) + spec = DATASETS[task] + reports.append( + { + "task": task, + "repository": spec.repository, + "revision": spec.revision, + "snapshot": str(snapshot), + "requires_media": spec.preparation_dir is not None, + "status": "downloaded", + } + ) + else: + reports = prepare_benchmark_datasets( + hf_home, + args.tasks, + max_workers=args.max_workers, + range_resume=args.range_resume, + ) print(json.dumps({"hf_home": str(hf_home), "tasks": reports}, indent=2, sort_keys=True)) return 0 diff --git a/examples/puzzletron/evaluation/vlm/profiles/core3-full-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/core3-full-native-v1.json new file mode 100644 index 00000000000..0fc3afb009a --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core3-full-native-v1.json @@ -0,0 +1,76 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v1", + "name": "core3-full-native-v1", + "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "model": { + "repository": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17" + }, + "model_family": { + "architecture": "Qwen3_5ForConditionalGeneration", + "model_type": "qwen3_5" + }, + "backend": { + "attention_implementation": "sdpa", + "enable_thinking": false, + "name": "qwen3_5" + }, + "preprocessing": { + "fps": 2, + "max_frames": 32, + "video_reader": "decord" + }, + "generation": { + "do_sample": false, + "temperature": 0 + }, + "seed": 42, + "repetitions": 1, + "batch_size": 1, + "selection": "all", + "tasks": { + "realworldqa": { + "dataset_repository": "lmms-lab/RealWorldQA", + "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", + "max_new_tokens": 16, + "population_rows": 765, + "scoring_task_config": "tasks/realworldqa/realworldqa.yaml" + }, + "mmmu_val": { + "dataset_repository": "lmms-lab/MMMU", + "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", + "max_new_tokens": 128, + "population_rows": 900, + "scoring_task_config": "tasks/mmmu/mmmu_val.yaml" + }, + "mvbench": { + "dataset_repository": "OpenGVLab/MVBench", + "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", + "leaf_populations": { + "action_sequence": 200, + "moving_count": 200, + "action_prediction": 200, + "episodic_reasoning": 200, + "action_antonym": 200, + "action_count": 200, + "scene_transition": 200, + "object_shuffle": 200, + "object_existence": 200, + "fine_grained_pose": 200, + "unexpected_action": 200, + "moving_direction": 200, + "state_change": 200, + "object_interaction": 200, + "character_order": 200, + "action_localization": 200, + "counterfactual_inference": 200, + "fine_grained_action": 200, + "moving_attribute": 200, + "egocentric_navigation": 200 + }, + "max_new_tokens": 16, + "population_rows": 4000, + "scoring_task_config": "tasks/mvbench/mvbench.yaml" + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core3-full-vllm-v1.json b/examples/puzzletron/evaluation/vlm/profiles/core3-full-vllm-v1.json new file mode 100644 index 00000000000..fa414706004 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core3-full-vllm-v1.json @@ -0,0 +1,11 @@ +{ + "extends": "core3-full-native-v1", + "schema": "modelopt.vlm-evaluation-profile/v1", + "name": "core3-full-vllm-v1", + "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "backend": { + "enable_thinking": false, + "name": "vllm", + "reasoning_parser": "qwen3" + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/full-v1.json b/examples/puzzletron/evaluation/vlm/profiles/full-v1.json index cd8315703de..dae15ce25bf 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/full-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/full-v1.json @@ -1,7 +1,7 @@ { "schema": "modelopt.vlm-evaluation-profile/v1", "name": "full-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "lmms_eval_revision": "15c32bfec165df13c269ddd3cda03b2ed9137825", "model_family": { "architecture": "Qwen3_5ForConditionalGeneration", "model_type": "qwen3_5" diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json index cb70fabb241..9149b58cc69 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json @@ -2,7 +2,7 @@ "extends": "short-v1", "schema": "modelopt.vlm-evaluation-profile/v1", "name": "short-all-native-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "lmms_eval_revision": "88b23e2bfa16a1edbc16e9e238ed82130b3a4f56", "backend": { "attention_implementation": "sdpa", "enable_thinking": false, diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v2.json b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v2.json new file mode 100644 index 00000000000..c105aad17cb --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v2.json @@ -0,0 +1,2558 @@ +{ + "extends": "short-native-v2", + "schema": "modelopt.vlm-evaluation-profile/v1", + "name": "short-all-native-v2", + "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "tasks": { + "video_mmmu": { + "dataset_repository": "lmms-lab/VideoMMMU", + "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", + "max_new_tokens": 1024, + "scoring_task_config": "tasks/videommmu/video_mmmu.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "leaf_task", + "index_space": "within-stratum", + "population_rows": 900, + "selected_rows": 72, + "strata": [ + { + "name": "adaptation", + "population_rows": 300, + "selected_rows": 24 + }, + { + "name": "comprehension", + "population_rows": 300, + "selected_rows": 24 + }, + { + "name": "perception", + "population_rows": 300, + "selected_rows": 24 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 6, + "p25": 68, + "p50": 143, + "p75": 218, + "p100": 293 + }, + "selected_row_identities_sha256": "cccb57bd2f0bd914ef88a2f0ce71a8c2eafd4220bd16b94ba5f18fde6c0ab910" + }, + "rows": [ + { + "source_row_index": 6, + "source_sample_id": "adaptation:6", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 18, + "source_sample_id": "adaptation:18", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 31, + "source_sample_id": "adaptation:31", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 43, + "source_sample_id": "adaptation:43", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 56, + "source_sample_id": "adaptation:56", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 68, + "source_sample_id": "adaptation:68", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 81, + "source_sample_id": "adaptation:81", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 93, + "source_sample_id": "adaptation:93", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 106, + "source_sample_id": "adaptation:106", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 118, + "source_sample_id": "adaptation:118", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 131, + "source_sample_id": "adaptation:131", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 143, + "source_sample_id": "adaptation:143", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 156, + "source_sample_id": "adaptation:156", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 168, + "source_sample_id": "adaptation:168", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 181, + "source_sample_id": "adaptation:181", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 193, + "source_sample_id": "adaptation:193", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 206, + "source_sample_id": "adaptation:206", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 218, + "source_sample_id": "adaptation:218", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 231, + "source_sample_id": "adaptation:231", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 243, + "source_sample_id": "adaptation:243", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 256, + "source_sample_id": "adaptation:256", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 268, + "source_sample_id": "adaptation:268", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 281, + "source_sample_id": "adaptation:281", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 293, + "source_sample_id": "adaptation:293", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 6, + "source_sample_id": "comprehension:6", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 18, + "source_sample_id": "comprehension:18", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 31, + "source_sample_id": "comprehension:31", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 43, + "source_sample_id": "comprehension:43", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 56, + "source_sample_id": "comprehension:56", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 68, + "source_sample_id": "comprehension:68", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 81, + "source_sample_id": "comprehension:81", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 93, + "source_sample_id": "comprehension:93", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 106, + "source_sample_id": "comprehension:106", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 118, + "source_sample_id": "comprehension:118", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 131, + "source_sample_id": "comprehension:131", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 143, + "source_sample_id": "comprehension:143", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 156, + "source_sample_id": "comprehension:156", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 168, + "source_sample_id": "comprehension:168", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 181, + "source_sample_id": "comprehension:181", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 193, + "source_sample_id": "comprehension:193", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 206, + "source_sample_id": "comprehension:206", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 218, + "source_sample_id": "comprehension:218", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 231, + "source_sample_id": "comprehension:231", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 243, + "source_sample_id": "comprehension:243", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 256, + "source_sample_id": "comprehension:256", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 268, + "source_sample_id": "comprehension:268", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 281, + "source_sample_id": "comprehension:281", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 293, + "source_sample_id": "comprehension:293", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 6, + "source_sample_id": "perception:6", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 18, + "source_sample_id": "perception:18", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 31, + "source_sample_id": "perception:31", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 43, + "source_sample_id": "perception:43", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 56, + "source_sample_id": "perception:56", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 68, + "source_sample_id": "perception:68", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 81, + "source_sample_id": "perception:81", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 93, + "source_sample_id": "perception:93", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 106, + "source_sample_id": "perception:106", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 118, + "source_sample_id": "perception:118", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 131, + "source_sample_id": "perception:131", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 143, + "source_sample_id": "perception:143", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 156, + "source_sample_id": "perception:156", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 168, + "source_sample_id": "perception:168", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 181, + "source_sample_id": "perception:181", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 193, + "source_sample_id": "perception:193", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 206, + "source_sample_id": "perception:206", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 218, + "source_sample_id": "perception:218", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 231, + "source_sample_id": "perception:231", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 243, + "source_sample_id": "perception:243", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 256, + "source_sample_id": "perception:256", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 268, + "source_sample_id": "perception:268", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 281, + "source_sample_id": "perception:281", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 293, + "source_sample_id": "perception:293", + "leaf_task": "video_mmmu_perception" + } + ] + }, + "videomme": { + "dataset_repository": "lmms-lab/Video-MME", + "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/videomme/videomme.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "duration+domain", + "index_space": "within-stratum", + "population_rows": 2700, + "selected_rows": 72, + "strata": [ + { + "name": "long|Artistic Performance", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "long|Film & Television", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "long|Knowledge", + "population_rows": 270, + "selected_rows": 4 + }, + { + "name": "long|Life Record", + "population_rows": 210, + "selected_rows": 4 + }, + { + "name": "long|Multilingual", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "long|Sports Competition", + "population_rows": 150, + "selected_rows": 4 + }, + { + "name": "medium|Artistic Performance", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "medium|Film & Television", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "medium|Knowledge", + "population_rows": 270, + "selected_rows": 4 + }, + { + "name": "medium|Life Record", + "population_rows": 210, + "selected_rows": 4 + }, + { + "name": "medium|Multilingual", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "medium|Sports Competition", + "population_rows": 150, + "selected_rows": 4 + }, + { + "name": "short|Artistic Performance", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "short|Film & Television", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "short|Knowledge", + "population_rows": 270, + "selected_rows": 4 + }, + { + "name": "short|Life Record", + "population_rows": 210, + "selected_rows": 4 + }, + { + "name": "short|Multilingual", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "short|Sports Competition", + "population_rows": 150, + "selected_rows": 4 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 3, + "p25": 18, + "p50": 56, + "p75": 105, + "p100": 236 + }, + "selected_row_identities_sha256": "fe78a3204ac1d4834fb9acfceaa18a329797cbc2bc840d22212e0de5aa319c81" + }, + "rows": [ + { + "source_row_index": 2355, + "source_sample_id": "videomme:2355", + "sampling_stratum": "long|Artistic Performance", + "source_stratum_index": 15, + "upstream_sample_id": "786-1" + }, + { + "source_row_index": 2385, + "source_sample_id": "videomme:2385", + "sampling_stratum": "long|Artistic Performance", + "source_stratum_index": 45, + "upstream_sample_id": "796-1" + }, + { + "source_row_index": 2415, + "source_sample_id": "videomme:2415", + "sampling_stratum": "long|Artistic Performance", + "source_stratum_index": 75, + "upstream_sample_id": "806-1" + }, + { + "source_row_index": 2445, + "source_sample_id": "videomme:2445", + "sampling_stratum": "long|Artistic Performance", + "source_stratum_index": 105, + "upstream_sample_id": "816-1" + }, + { + "source_row_index": 2085, + "source_sample_id": "videomme:2085", + "sampling_stratum": "long|Film & Television", + "source_stratum_index": 15, + "upstream_sample_id": "696-1" + }, + { + "source_row_index": 2115, + "source_sample_id": "videomme:2115", + "sampling_stratum": "long|Film & Television", + "source_stratum_index": 45, + "upstream_sample_id": "706-1" + }, + { + "source_row_index": 2145, + "source_sample_id": "videomme:2145", + "sampling_stratum": "long|Film & Television", + "source_stratum_index": 75, + "upstream_sample_id": "716-1" + }, + { + "source_row_index": 2175, + "source_sample_id": "videomme:2175", + "sampling_stratum": "long|Film & Television", + "source_stratum_index": 105, + "upstream_sample_id": "726-1" + }, + { + "source_row_index": 1833, + "source_sample_id": "videomme:1833", + "sampling_stratum": "long|Knowledge", + "source_stratum_index": 33, + "upstream_sample_id": "612-1" + }, + { + "source_row_index": 1901, + "source_sample_id": "videomme:1901", + "sampling_stratum": "long|Knowledge", + "source_stratum_index": 101, + "upstream_sample_id": "634-3" + }, + { + "source_row_index": 1968, + "source_sample_id": "videomme:1968", + "sampling_stratum": "long|Knowledge", + "source_stratum_index": 168, + "upstream_sample_id": "657-1" + }, + { + "source_row_index": 2036, + "source_sample_id": "videomme:2036", + "sampling_stratum": "long|Knowledge", + "source_stratum_index": 236, + "upstream_sample_id": "679-3" + }, + { + "source_row_index": 2486, + "source_sample_id": "videomme:2486", + "sampling_stratum": "long|Life Record", + "source_stratum_index": 26, + "upstream_sample_id": "829-3" + }, + { + "source_row_index": 2538, + "source_sample_id": "videomme:2538", + "sampling_stratum": "long|Life Record", + "source_stratum_index": 78, + "upstream_sample_id": "847-1" + }, + { + "source_row_index": 2591, + "source_sample_id": "videomme:2591", + "sampling_stratum": "long|Life Record", + "source_stratum_index": 131, + "upstream_sample_id": "864-3" + }, + { + "source_row_index": 2643, + "source_sample_id": "videomme:2643", + "sampling_stratum": "long|Life Record", + "source_stratum_index": 183, + "upstream_sample_id": "882-1" + }, + { + "source_row_index": 2673, + "source_sample_id": "videomme:2673", + "sampling_stratum": "long|Multilingual", + "source_stratum_index": 3, + "upstream_sample_id": "892-1" + }, + { + "source_row_index": 2681, + "source_sample_id": "videomme:2681", + "sampling_stratum": "long|Multilingual", + "source_stratum_index": 11, + "upstream_sample_id": "894-3" + }, + { + "source_row_index": 2688, + "source_sample_id": "videomme:2688", + "sampling_stratum": "long|Multilingual", + "source_stratum_index": 18, + "upstream_sample_id": "897-1" + }, + { + "source_row_index": 2696, + "source_sample_id": "videomme:2696", + "sampling_stratum": "long|Multilingual", + "source_stratum_index": 26, + "upstream_sample_id": "899-3" + }, + { + "source_row_index": 2208, + "source_sample_id": "videomme:2208", + "sampling_stratum": "long|Sports Competition", + "source_stratum_index": 18, + "upstream_sample_id": "737-1" + }, + { + "source_row_index": 2246, + "source_sample_id": "videomme:2246", + "sampling_stratum": "long|Sports Competition", + "source_stratum_index": 56, + "upstream_sample_id": "749-3" + }, + { + "source_row_index": 2283, + "source_sample_id": "videomme:2283", + "sampling_stratum": "long|Sports Competition", + "source_stratum_index": 93, + "upstream_sample_id": "762-1" + }, + { + "source_row_index": 2321, + "source_sample_id": "videomme:2321", + "sampling_stratum": "long|Sports Competition", + "source_stratum_index": 131, + "upstream_sample_id": "774-3" + }, + { + "source_row_index": 1455, + "source_sample_id": "videomme:1455", + "sampling_stratum": "medium|Artistic Performance", + "source_stratum_index": 15, + "upstream_sample_id": "486-1" + }, + { + "source_row_index": 1485, + "source_sample_id": "videomme:1485", + "sampling_stratum": "medium|Artistic Performance", + "source_stratum_index": 45, + "upstream_sample_id": "496-1" + }, + { + "source_row_index": 1515, + "source_sample_id": "videomme:1515", + "sampling_stratum": "medium|Artistic Performance", + "source_stratum_index": 75, + "upstream_sample_id": "506-1" + }, + { + "source_row_index": 1545, + "source_sample_id": "videomme:1545", + "sampling_stratum": "medium|Artistic Performance", + "source_stratum_index": 105, + "upstream_sample_id": "516-1" + }, + { + "source_row_index": 1185, + "source_sample_id": "videomme:1185", + "sampling_stratum": "medium|Film & Television", + "source_stratum_index": 15, + "upstream_sample_id": "396-1" + }, + { + "source_row_index": 1215, + "source_sample_id": "videomme:1215", + "sampling_stratum": "medium|Film & Television", + "source_stratum_index": 45, + "upstream_sample_id": "406-1" + }, + { + "source_row_index": 1245, + "source_sample_id": "videomme:1245", + "sampling_stratum": "medium|Film & Television", + "source_stratum_index": 75, + "upstream_sample_id": "416-1" + }, + { + "source_row_index": 1275, + "source_sample_id": "videomme:1275", + "sampling_stratum": "medium|Film & Television", + "source_stratum_index": 105, + "upstream_sample_id": "426-1" + }, + { + "source_row_index": 933, + "source_sample_id": "videomme:933", + "sampling_stratum": "medium|Knowledge", + "source_stratum_index": 33, + "upstream_sample_id": "312-1" + }, + { + "source_row_index": 1001, + "source_sample_id": "videomme:1001", + "sampling_stratum": "medium|Knowledge", + "source_stratum_index": 101, + "upstream_sample_id": "334-3" + }, + { + "source_row_index": 1068, + "source_sample_id": "videomme:1068", + "sampling_stratum": "medium|Knowledge", + "source_stratum_index": 168, + "upstream_sample_id": "357-1" + }, + { + "source_row_index": 1136, + "source_sample_id": "videomme:1136", + "sampling_stratum": "medium|Knowledge", + "source_stratum_index": 236, + "upstream_sample_id": "379-3" + }, + { + "source_row_index": 1586, + "source_sample_id": "videomme:1586", + "sampling_stratum": "medium|Life Record", + "source_stratum_index": 26, + "upstream_sample_id": "529-3" + }, + { + "source_row_index": 1638, + "source_sample_id": "videomme:1638", + "sampling_stratum": "medium|Life Record", + "source_stratum_index": 78, + "upstream_sample_id": "547-1" + }, + { + "source_row_index": 1691, + "source_sample_id": "videomme:1691", + "sampling_stratum": "medium|Life Record", + "source_stratum_index": 131, + "upstream_sample_id": "564-3" + }, + { + "source_row_index": 1743, + "source_sample_id": "videomme:1743", + "sampling_stratum": "medium|Life Record", + "source_stratum_index": 183, + "upstream_sample_id": "582-1" + }, + { + "source_row_index": 1773, + "source_sample_id": "videomme:1773", + "sampling_stratum": "medium|Multilingual", + "source_stratum_index": 3, + "upstream_sample_id": "592-1" + }, + { + "source_row_index": 1781, + "source_sample_id": "videomme:1781", + "sampling_stratum": "medium|Multilingual", + "source_stratum_index": 11, + "upstream_sample_id": "594-3" + }, + { + "source_row_index": 1788, + "source_sample_id": "videomme:1788", + "sampling_stratum": "medium|Multilingual", + "source_stratum_index": 18, + "upstream_sample_id": "597-1" + }, + { + "source_row_index": 1796, + "source_sample_id": "videomme:1796", + "sampling_stratum": "medium|Multilingual", + "source_stratum_index": 26, + "upstream_sample_id": "599-3" + }, + { + "source_row_index": 1308, + "source_sample_id": "videomme:1308", + "sampling_stratum": "medium|Sports Competition", + "source_stratum_index": 18, + "upstream_sample_id": "437-1" + }, + { + "source_row_index": 1346, + "source_sample_id": "videomme:1346", + "sampling_stratum": "medium|Sports Competition", + "source_stratum_index": 56, + "upstream_sample_id": "449-3" + }, + { + "source_row_index": 1383, + "source_sample_id": "videomme:1383", + "sampling_stratum": "medium|Sports Competition", + "source_stratum_index": 93, + "upstream_sample_id": "462-1" + }, + { + "source_row_index": 1421, + "source_sample_id": "videomme:1421", + "sampling_stratum": "medium|Sports Competition", + "source_stratum_index": 131, + "upstream_sample_id": "474-3" + }, + { + "source_row_index": 555, + "source_sample_id": "videomme:555", + "sampling_stratum": "short|Artistic Performance", + "source_stratum_index": 15, + "upstream_sample_id": "186-1" + }, + { + "source_row_index": 585, + "source_sample_id": "videomme:585", + "sampling_stratum": "short|Artistic Performance", + "source_stratum_index": 45, + "upstream_sample_id": "196-1" + }, + { + "source_row_index": 615, + "source_sample_id": "videomme:615", + "sampling_stratum": "short|Artistic Performance", + "source_stratum_index": 75, + "upstream_sample_id": "206-1" + }, + { + "source_row_index": 645, + "source_sample_id": "videomme:645", + "sampling_stratum": "short|Artistic Performance", + "source_stratum_index": 105, + "upstream_sample_id": "216-1" + }, + { + "source_row_index": 285, + "source_sample_id": "videomme:285", + "sampling_stratum": "short|Film & Television", + "source_stratum_index": 15, + "upstream_sample_id": "096-1" + }, + { + "source_row_index": 315, + "source_sample_id": "videomme:315", + "sampling_stratum": "short|Film & Television", + "source_stratum_index": 45, + "upstream_sample_id": "106-1" + }, + { + "source_row_index": 345, + "source_sample_id": "videomme:345", + "sampling_stratum": "short|Film & Television", + "source_stratum_index": 75, + "upstream_sample_id": "116-1" + }, + { + "source_row_index": 375, + "source_sample_id": "videomme:375", + "sampling_stratum": "short|Film & Television", + "source_stratum_index": 105, + "upstream_sample_id": "126-1" + }, + { + "source_row_index": 33, + "source_sample_id": "videomme:33", + "sampling_stratum": "short|Knowledge", + "source_stratum_index": 33, + "upstream_sample_id": "012-1" + }, + { + "source_row_index": 101, + "source_sample_id": "videomme:101", + "sampling_stratum": "short|Knowledge", + "source_stratum_index": 101, + "upstream_sample_id": "034-3" + }, + { + "source_row_index": 168, + "source_sample_id": "videomme:168", + "sampling_stratum": "short|Knowledge", + "source_stratum_index": 168, + "upstream_sample_id": "057-1" + }, + { + "source_row_index": 236, + "source_sample_id": "videomme:236", + "sampling_stratum": "short|Knowledge", + "source_stratum_index": 236, + "upstream_sample_id": "079-3" + }, + { + "source_row_index": 686, + "source_sample_id": "videomme:686", + "sampling_stratum": "short|Life Record", + "source_stratum_index": 26, + "upstream_sample_id": "229-3" + }, + { + "source_row_index": 738, + "source_sample_id": "videomme:738", + "sampling_stratum": "short|Life Record", + "source_stratum_index": 78, + "upstream_sample_id": "247-1" + }, + { + "source_row_index": 791, + "source_sample_id": "videomme:791", + "sampling_stratum": "short|Life Record", + "source_stratum_index": 131, + "upstream_sample_id": "264-3" + }, + { + "source_row_index": 843, + "source_sample_id": "videomme:843", + "sampling_stratum": "short|Life Record", + "source_stratum_index": 183, + "upstream_sample_id": "282-1" + }, + { + "source_row_index": 873, + "source_sample_id": "videomme:873", + "sampling_stratum": "short|Multilingual", + "source_stratum_index": 3, + "upstream_sample_id": "292-1" + }, + { + "source_row_index": 881, + "source_sample_id": "videomme:881", + "sampling_stratum": "short|Multilingual", + "source_stratum_index": 11, + "upstream_sample_id": "294-3" + }, + { + "source_row_index": 888, + "source_sample_id": "videomme:888", + "sampling_stratum": "short|Multilingual", + "source_stratum_index": 18, + "upstream_sample_id": "297-1" + }, + { + "source_row_index": 896, + "source_sample_id": "videomme:896", + "sampling_stratum": "short|Multilingual", + "source_stratum_index": 26, + "upstream_sample_id": "299-3" + }, + { + "source_row_index": 408, + "source_sample_id": "videomme:408", + "sampling_stratum": "short|Sports Competition", + "source_stratum_index": 18, + "upstream_sample_id": "137-1" + }, + { + "source_row_index": 446, + "source_sample_id": "videomme:446", + "sampling_stratum": "short|Sports Competition", + "source_stratum_index": 56, + "upstream_sample_id": "149-3" + }, + { + "source_row_index": 483, + "source_sample_id": "videomme:483", + "sampling_stratum": "short|Sports Competition", + "source_stratum_index": 93, + "upstream_sample_id": "162-1" + }, + { + "source_row_index": 521, + "source_sample_id": "videomme:521", + "sampling_stratum": "short|Sports Competition", + "source_stratum_index": 131, + "upstream_sample_id": "174-3" + } + ] + }, + "longvideobench_val_v": { + "dataset_repository": "longvideobench/LongVideoBench", + "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", + "max_new_tokens": 32, + "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "split", + "index_space": "within-stratum", + "population_rows": 1337, + "selected_rows": 68, + "strata": [ + { + "name": "validation", + "population_rows": 1337, + "selected_rows": 68 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 9, + "p25": 324, + "p50": 658, + "p75": 992, + "p100": 1327 + }, + "selected_row_identities_sha256": "124e56d414789cb93e0a8f50d51b2b212972bb6e89f5174663afc55f843c50c2" + }, + "rows": [ + { + "source_row_index": 9, + "source_sample_id": "longvideobench_val_v:9", + "sampling_stratum": "validation", + "source_stratum_index": 9 + }, + { + "source_row_index": 29, + "source_sample_id": "longvideobench_val_v:29", + "sampling_stratum": "validation", + "source_stratum_index": 29 + }, + { + "source_row_index": 49, + "source_sample_id": "longvideobench_val_v:49", + "sampling_stratum": "validation", + "source_stratum_index": 49 + }, + { + "source_row_index": 68, + "source_sample_id": "longvideobench_val_v:68", + "sampling_stratum": "validation", + "source_stratum_index": 68 + }, + { + "source_row_index": 88, + "source_sample_id": "longvideobench_val_v:88", + "sampling_stratum": "validation", + "source_stratum_index": 88 + }, + { + "source_row_index": 108, + "source_sample_id": "longvideobench_val_v:108", + "sampling_stratum": "validation", + "source_stratum_index": 108 + }, + { + "source_row_index": 127, + "source_sample_id": "longvideobench_val_v:127", + "sampling_stratum": "validation", + "source_stratum_index": 127 + }, + { + "source_row_index": 147, + "source_sample_id": "longvideobench_val_v:147", + "sampling_stratum": "validation", + "source_stratum_index": 147 + }, + { + "source_row_index": 167, + "source_sample_id": "longvideobench_val_v:167", + "sampling_stratum": "validation", + "source_stratum_index": 167 + }, + { + "source_row_index": 186, + "source_sample_id": "longvideobench_val_v:186", + "sampling_stratum": "validation", + "source_stratum_index": 186 + }, + { + "source_row_index": 206, + "source_sample_id": "longvideobench_val_v:206", + "sampling_stratum": "validation", + "source_stratum_index": 206 + }, + { + "source_row_index": 226, + "source_sample_id": "longvideobench_val_v:226", + "sampling_stratum": "validation", + "source_stratum_index": 226 + }, + { + "source_row_index": 245, + "source_sample_id": "longvideobench_val_v:245", + "sampling_stratum": "validation", + "source_stratum_index": 245 + }, + { + "source_row_index": 265, + "source_sample_id": "longvideobench_val_v:265", + "sampling_stratum": "validation", + "source_stratum_index": 265 + }, + { + "source_row_index": 285, + "source_sample_id": "longvideobench_val_v:285", + "sampling_stratum": "validation", + "source_stratum_index": 285 + }, + { + "source_row_index": 304, + "source_sample_id": "longvideobench_val_v:304", + "sampling_stratum": "validation", + "source_stratum_index": 304 + }, + { + "source_row_index": 324, + "source_sample_id": "longvideobench_val_v:324", + "sampling_stratum": "validation", + "source_stratum_index": 324 + }, + { + "source_row_index": 344, + "source_sample_id": "longvideobench_val_v:344", + "sampling_stratum": "validation", + "source_stratum_index": 344 + }, + { + "source_row_index": 363, + "source_sample_id": "longvideobench_val_v:363", + "sampling_stratum": "validation", + "source_stratum_index": 363 + }, + { + "source_row_index": 383, + "source_sample_id": "longvideobench_val_v:383", + "sampling_stratum": "validation", + "source_stratum_index": 383 + }, + { + "source_row_index": 403, + "source_sample_id": "longvideobench_val_v:403", + "sampling_stratum": "validation", + "source_stratum_index": 403 + }, + { + "source_row_index": 422, + "source_sample_id": "longvideobench_val_v:422", + "sampling_stratum": "validation", + "source_stratum_index": 422 + }, + { + "source_row_index": 442, + "source_sample_id": "longvideobench_val_v:442", + "sampling_stratum": "validation", + "source_stratum_index": 442 + }, + { + "source_row_index": 462, + "source_sample_id": "longvideobench_val_v:462", + "sampling_stratum": "validation", + "source_stratum_index": 462 + }, + { + "source_row_index": 481, + "source_sample_id": "longvideobench_val_v:481", + "sampling_stratum": "validation", + "source_stratum_index": 481 + }, + { + "source_row_index": 501, + "source_sample_id": "longvideobench_val_v:501", + "sampling_stratum": "validation", + "source_stratum_index": 501 + }, + { + "source_row_index": 521, + "source_sample_id": "longvideobench_val_v:521", + "sampling_stratum": "validation", + "source_stratum_index": 521 + }, + { + "source_row_index": 540, + "source_sample_id": "longvideobench_val_v:540", + "sampling_stratum": "validation", + "source_stratum_index": 540 + }, + { + "source_row_index": 560, + "source_sample_id": "longvideobench_val_v:560", + "sampling_stratum": "validation", + "source_stratum_index": 560 + }, + { + "source_row_index": 580, + "source_sample_id": "longvideobench_val_v:580", + "sampling_stratum": "validation", + "source_stratum_index": 580 + }, + { + "source_row_index": 599, + "source_sample_id": "longvideobench_val_v:599", + "sampling_stratum": "validation", + "source_stratum_index": 599 + }, + { + "source_row_index": 619, + "source_sample_id": "longvideobench_val_v:619", + "sampling_stratum": "validation", + "source_stratum_index": 619 + }, + { + "source_row_index": 639, + "source_sample_id": "longvideobench_val_v:639", + "sampling_stratum": "validation", + "source_stratum_index": 639 + }, + { + "source_row_index": 658, + "source_sample_id": "longvideobench_val_v:658", + "sampling_stratum": "validation", + "source_stratum_index": 658 + }, + { + "source_row_index": 678, + "source_sample_id": "longvideobench_val_v:678", + "sampling_stratum": "validation", + "source_stratum_index": 678 + }, + { + "source_row_index": 697, + "source_sample_id": "longvideobench_val_v:697", + "sampling_stratum": "validation", + "source_stratum_index": 697 + }, + { + "source_row_index": 717, + "source_sample_id": "longvideobench_val_v:717", + "sampling_stratum": "validation", + "source_stratum_index": 717 + }, + { + "source_row_index": 737, + "source_sample_id": "longvideobench_val_v:737", + "sampling_stratum": "validation", + "source_stratum_index": 737 + }, + { + "source_row_index": 756, + "source_sample_id": "longvideobench_val_v:756", + "sampling_stratum": "validation", + "source_stratum_index": 756 + }, + { + "source_row_index": 776, + "source_sample_id": "longvideobench_val_v:776", + "sampling_stratum": "validation", + "source_stratum_index": 776 + }, + { + "source_row_index": 796, + "source_sample_id": "longvideobench_val_v:796", + "sampling_stratum": "validation", + "source_stratum_index": 796 + }, + { + "source_row_index": 815, + "source_sample_id": "longvideobench_val_v:815", + "sampling_stratum": "validation", + "source_stratum_index": 815 + }, + { + "source_row_index": 835, + "source_sample_id": "longvideobench_val_v:835", + "sampling_stratum": "validation", + "source_stratum_index": 835 + }, + { + "source_row_index": 855, + "source_sample_id": "longvideobench_val_v:855", + "sampling_stratum": "validation", + "source_stratum_index": 855 + }, + { + "source_row_index": 874, + "source_sample_id": "longvideobench_val_v:874", + "sampling_stratum": "validation", + "source_stratum_index": 874 + }, + { + "source_row_index": 894, + "source_sample_id": "longvideobench_val_v:894", + "sampling_stratum": "validation", + "source_stratum_index": 894 + }, + { + "source_row_index": 914, + "source_sample_id": "longvideobench_val_v:914", + "sampling_stratum": "validation", + "source_stratum_index": 914 + }, + { + "source_row_index": 933, + "source_sample_id": "longvideobench_val_v:933", + "sampling_stratum": "validation", + "source_stratum_index": 933 + }, + { + "source_row_index": 953, + "source_sample_id": "longvideobench_val_v:953", + "sampling_stratum": "validation", + "source_stratum_index": 953 + }, + { + "source_row_index": 973, + "source_sample_id": "longvideobench_val_v:973", + "sampling_stratum": "validation", + "source_stratum_index": 973 + }, + { + "source_row_index": 992, + "source_sample_id": "longvideobench_val_v:992", + "sampling_stratum": "validation", + "source_stratum_index": 992 + }, + { + "source_row_index": 1012, + "source_sample_id": "longvideobench_val_v:1012", + "sampling_stratum": "validation", + "source_stratum_index": 1012 + }, + { + "source_row_index": 1032, + "source_sample_id": "longvideobench_val_v:1032", + "sampling_stratum": "validation", + "source_stratum_index": 1032 + }, + { + "source_row_index": 1051, + "source_sample_id": "longvideobench_val_v:1051", + "sampling_stratum": "validation", + "source_stratum_index": 1051 + }, + { + "source_row_index": 1071, + "source_sample_id": "longvideobench_val_v:1071", + "sampling_stratum": "validation", + "source_stratum_index": 1071 + }, + { + "source_row_index": 1091, + "source_sample_id": "longvideobench_val_v:1091", + "sampling_stratum": "validation", + "source_stratum_index": 1091 + }, + { + "source_row_index": 1110, + "source_sample_id": "longvideobench_val_v:1110", + "sampling_stratum": "validation", + "source_stratum_index": 1110 + }, + { + "source_row_index": 1130, + "source_sample_id": "longvideobench_val_v:1130", + "sampling_stratum": "validation", + "source_stratum_index": 1130 + }, + { + "source_row_index": 1150, + "source_sample_id": "longvideobench_val_v:1150", + "sampling_stratum": "validation", + "source_stratum_index": 1150 + }, + { + "source_row_index": 1169, + "source_sample_id": "longvideobench_val_v:1169", + "sampling_stratum": "validation", + "source_stratum_index": 1169 + }, + { + "source_row_index": 1189, + "source_sample_id": "longvideobench_val_v:1189", + "sampling_stratum": "validation", + "source_stratum_index": 1189 + }, + { + "source_row_index": 1209, + "source_sample_id": "longvideobench_val_v:1209", + "sampling_stratum": "validation", + "source_stratum_index": 1209 + }, + { + "source_row_index": 1228, + "source_sample_id": "longvideobench_val_v:1228", + "sampling_stratum": "validation", + "source_stratum_index": 1228 + }, + { + "source_row_index": 1248, + "source_sample_id": "longvideobench_val_v:1248", + "sampling_stratum": "validation", + "source_stratum_index": 1248 + }, + { + "source_row_index": 1268, + "source_sample_id": "longvideobench_val_v:1268", + "sampling_stratum": "validation", + "source_stratum_index": 1268 + }, + { + "source_row_index": 1287, + "source_sample_id": "longvideobench_val_v:1287", + "sampling_stratum": "validation", + "source_stratum_index": 1287 + }, + { + "source_row_index": 1307, + "source_sample_id": "longvideobench_val_v:1307", + "sampling_stratum": "validation", + "source_stratum_index": 1307 + }, + { + "source_row_index": 1327, + "source_sample_id": "longvideobench_val_v:1327", + "sampling_stratum": "validation", + "source_stratum_index": 1327 + } + ] + }, + "mlvu_dev": { + "dataset_repository": "sy1998/MLVU_dev", + "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "task_type", + "index_space": "within-stratum", + "population_rows": 2174, + "selected_rows": 70, + "strata": [ + { + "name": "anomaly_reco", + "population_rows": 200, + "selected_rows": 10 + }, + { + "name": "count", + "population_rows": 206, + "selected_rows": 10 + }, + { + "name": "ego", + "population_rows": 352, + "selected_rows": 10 + }, + { + "name": "needle", + "population_rows": 355, + "selected_rows": 10 + }, + { + "name": "order", + "population_rows": 259, + "selected_rows": 10 + }, + { + "name": "plotQA", + "population_rows": 539, + "selected_rows": 10 + }, + { + "name": "topic_reasoning", + "population_rows": 263, + "selected_rows": 10 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 10, + "p25": 70, + "p50": 134, + "p75": 197, + "p100": 512 + }, + "selected_row_identities_sha256": "7951e7c5fd3671e2d29e1634a372958c9de15aa363650a4d72e336ea4b371c80" + }, + "rows": [ + { + "source_row_index": 365, + "source_sample_id": "mlvu_dev:365", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 10, + "upstream_sample_id": "Q365" + }, + { + "source_row_index": 385, + "source_sample_id": "mlvu_dev:385", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 30, + "upstream_sample_id": "Q385" + }, + { + "source_row_index": 405, + "source_sample_id": "mlvu_dev:405", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 50, + "upstream_sample_id": "Q405" + }, + { + "source_row_index": 425, + "source_sample_id": "mlvu_dev:425", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 70, + "upstream_sample_id": "Q425" + }, + { + "source_row_index": 445, + "source_sample_id": "mlvu_dev:445", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 90, + "upstream_sample_id": "Q445" + }, + { + "source_row_index": 465, + "source_sample_id": "mlvu_dev:465", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 110, + "upstream_sample_id": "Q465" + }, + { + "source_row_index": 485, + "source_sample_id": "mlvu_dev:485", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 130, + "upstream_sample_id": "Q485" + }, + { + "source_row_index": 505, + "source_sample_id": "mlvu_dev:505", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 150, + "upstream_sample_id": "Q505" + }, + { + "source_row_index": 525, + "source_sample_id": "mlvu_dev:525", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 170, + "upstream_sample_id": "Q525" + }, + { + "source_row_index": 545, + "source_sample_id": "mlvu_dev:545", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 190, + "upstream_sample_id": "Q545" + }, + { + "source_row_index": 565, + "source_sample_id": "mlvu_dev:565", + "sampling_stratum": "count", + "source_stratum_index": 10, + "upstream_sample_id": "Q565" + }, + { + "source_row_index": 585, + "source_sample_id": "mlvu_dev:585", + "sampling_stratum": "count", + "source_stratum_index": 30, + "upstream_sample_id": "Q585" + }, + { + "source_row_index": 606, + "source_sample_id": "mlvu_dev:606", + "sampling_stratum": "count", + "source_stratum_index": 51, + "upstream_sample_id": "Q606" + }, + { + "source_row_index": 627, + "source_sample_id": "mlvu_dev:627", + "sampling_stratum": "count", + "source_stratum_index": 72, + "upstream_sample_id": "Q627" + }, + { + "source_row_index": 647, + "source_sample_id": "mlvu_dev:647", + "sampling_stratum": "count", + "source_stratum_index": 92, + "upstream_sample_id": "Q647" + }, + { + "source_row_index": 668, + "source_sample_id": "mlvu_dev:668", + "sampling_stratum": "count", + "source_stratum_index": 113, + "upstream_sample_id": "Q668" + }, + { + "source_row_index": 688, + "source_sample_id": "mlvu_dev:688", + "sampling_stratum": "count", + "source_stratum_index": 133, + "upstream_sample_id": "Q688" + }, + { + "source_row_index": 709, + "source_sample_id": "mlvu_dev:709", + "sampling_stratum": "count", + "source_stratum_index": 154, + "upstream_sample_id": "Q709" + }, + { + "source_row_index": 730, + "source_sample_id": "mlvu_dev:730", + "sampling_stratum": "count", + "source_stratum_index": 175, + "upstream_sample_id": "Q730" + }, + { + "source_row_index": 750, + "source_sample_id": "mlvu_dev:750", + "sampling_stratum": "count", + "source_stratum_index": 195, + "upstream_sample_id": "Q750" + }, + { + "source_row_index": 778, + "source_sample_id": "mlvu_dev:778", + "sampling_stratum": "ego", + "source_stratum_index": 17, + "upstream_sample_id": "Q778" + }, + { + "source_row_index": 813, + "source_sample_id": "mlvu_dev:813", + "sampling_stratum": "ego", + "source_stratum_index": 52, + "upstream_sample_id": "Q813" + }, + { + "source_row_index": 849, + "source_sample_id": "mlvu_dev:849", + "sampling_stratum": "ego", + "source_stratum_index": 88, + "upstream_sample_id": "Q849" + }, + { + "source_row_index": 884, + "source_sample_id": "mlvu_dev:884", + "sampling_stratum": "ego", + "source_stratum_index": 123, + "upstream_sample_id": "Q884" + }, + { + "source_row_index": 919, + "source_sample_id": "mlvu_dev:919", + "sampling_stratum": "ego", + "source_stratum_index": 158, + "upstream_sample_id": "Q919" + }, + { + "source_row_index": 954, + "source_sample_id": "mlvu_dev:954", + "sampling_stratum": "ego", + "source_stratum_index": 193, + "upstream_sample_id": "Q954" + }, + { + "source_row_index": 989, + "source_sample_id": "mlvu_dev:989", + "sampling_stratum": "ego", + "source_stratum_index": 228, + "upstream_sample_id": "Q989" + }, + { + "source_row_index": 1025, + "source_sample_id": "mlvu_dev:1025", + "sampling_stratum": "ego", + "source_stratum_index": 264, + "upstream_sample_id": "Q1025" + }, + { + "source_row_index": 1060, + "source_sample_id": "mlvu_dev:1060", + "sampling_stratum": "ego", + "source_stratum_index": 299, + "upstream_sample_id": "Q1060" + }, + { + "source_row_index": 1095, + "source_sample_id": "mlvu_dev:1095", + "sampling_stratum": "ego", + "source_stratum_index": 334, + "upstream_sample_id": "Q1095" + }, + { + "source_row_index": 17, + "source_sample_id": "mlvu_dev:17", + "sampling_stratum": "needle", + "source_stratum_index": 17, + "upstream_sample_id": "Q17" + }, + { + "source_row_index": 53, + "source_sample_id": "mlvu_dev:53", + "sampling_stratum": "needle", + "source_stratum_index": 53, + "upstream_sample_id": "Q53" + }, + { + "source_row_index": 88, + "source_sample_id": "mlvu_dev:88", + "sampling_stratum": "needle", + "source_stratum_index": 88, + "upstream_sample_id": "Q88" + }, + { + "source_row_index": 124, + "source_sample_id": "mlvu_dev:124", + "sampling_stratum": "needle", + "source_stratum_index": 124, + "upstream_sample_id": "Q124" + }, + { + "source_row_index": 159, + "source_sample_id": "mlvu_dev:159", + "sampling_stratum": "needle", + "source_stratum_index": 159, + "upstream_sample_id": "Q159" + }, + { + "source_row_index": 195, + "source_sample_id": "mlvu_dev:195", + "sampling_stratum": "needle", + "source_stratum_index": 195, + "upstream_sample_id": "Q195" + }, + { + "source_row_index": 230, + "source_sample_id": "mlvu_dev:230", + "sampling_stratum": "needle", + "source_stratum_index": 230, + "upstream_sample_id": "Q230" + }, + { + "source_row_index": 266, + "source_sample_id": "mlvu_dev:266", + "sampling_stratum": "needle", + "source_stratum_index": 266, + "upstream_sample_id": "Q266" + }, + { + "source_row_index": 301, + "source_sample_id": "mlvu_dev:301", + "sampling_stratum": "needle", + "source_stratum_index": 301, + "upstream_sample_id": "Q301" + }, + { + "source_row_index": 337, + "source_sample_id": "mlvu_dev:337", + "sampling_stratum": "needle", + "source_stratum_index": 337, + "upstream_sample_id": "Q337" + }, + { + "source_row_index": 1927, + "source_sample_id": "mlvu_dev:1927", + "sampling_stratum": "order", + "source_stratum_index": 12, + "upstream_sample_id": "Q1927" + }, + { + "source_row_index": 1953, + "source_sample_id": "mlvu_dev:1953", + "sampling_stratum": "order", + "source_stratum_index": 38, + "upstream_sample_id": "Q1953" + }, + { + "source_row_index": 1979, + "source_sample_id": "mlvu_dev:1979", + "sampling_stratum": "order", + "source_stratum_index": 64, + "upstream_sample_id": "Q1979" + }, + { + "source_row_index": 2005, + "source_sample_id": "mlvu_dev:2005", + "sampling_stratum": "order", + "source_stratum_index": 90, + "upstream_sample_id": "Q2005" + }, + { + "source_row_index": 2031, + "source_sample_id": "mlvu_dev:2031", + "sampling_stratum": "order", + "source_stratum_index": 116, + "upstream_sample_id": "Q2031" + }, + { + "source_row_index": 2057, + "source_sample_id": "mlvu_dev:2057", + "sampling_stratum": "order", + "source_stratum_index": 142, + "upstream_sample_id": "Q2057" + }, + { + "source_row_index": 2083, + "source_sample_id": "mlvu_dev:2083", + "sampling_stratum": "order", + "source_stratum_index": 168, + "upstream_sample_id": "Q2083" + }, + { + "source_row_index": 2109, + "source_sample_id": "mlvu_dev:2109", + "sampling_stratum": "order", + "source_stratum_index": 194, + "upstream_sample_id": "Q2109" + }, + { + "source_row_index": 2135, + "source_sample_id": "mlvu_dev:2135", + "sampling_stratum": "order", + "source_stratum_index": 220, + "upstream_sample_id": "Q2135" + }, + { + "source_row_index": 2161, + "source_sample_id": "mlvu_dev:2161", + "sampling_stratum": "order", + "source_stratum_index": 246, + "upstream_sample_id": "Q2161" + }, + { + "source_row_index": 1139, + "source_sample_id": "mlvu_dev:1139", + "sampling_stratum": "plotQA", + "source_stratum_index": 26, + "upstream_sample_id": "Q1139" + }, + { + "source_row_index": 1193, + "source_sample_id": "mlvu_dev:1193", + "sampling_stratum": "plotQA", + "source_stratum_index": 80, + "upstream_sample_id": "Q1193" + }, + { + "source_row_index": 1247, + "source_sample_id": "mlvu_dev:1247", + "sampling_stratum": "plotQA", + "source_stratum_index": 134, + "upstream_sample_id": "Q1247" + }, + { + "source_row_index": 1301, + "source_sample_id": "mlvu_dev:1301", + "sampling_stratum": "plotQA", + "source_stratum_index": 188, + "upstream_sample_id": "Q1301" + }, + { + "source_row_index": 1355, + "source_sample_id": "mlvu_dev:1355", + "sampling_stratum": "plotQA", + "source_stratum_index": 242, + "upstream_sample_id": "Q1355" + }, + { + "source_row_index": 1409, + "source_sample_id": "mlvu_dev:1409", + "sampling_stratum": "plotQA", + "source_stratum_index": 296, + "upstream_sample_id": "Q1409" + }, + { + "source_row_index": 1463, + "source_sample_id": "mlvu_dev:1463", + "sampling_stratum": "plotQA", + "source_stratum_index": 350, + "upstream_sample_id": "Q1463" + }, + { + "source_row_index": 1517, + "source_sample_id": "mlvu_dev:1517", + "sampling_stratum": "plotQA", + "source_stratum_index": 404, + "upstream_sample_id": "Q1517" + }, + { + "source_row_index": 1571, + "source_sample_id": "mlvu_dev:1571", + "sampling_stratum": "plotQA", + "source_stratum_index": 458, + "upstream_sample_id": "Q1571" + }, + { + "source_row_index": 1625, + "source_sample_id": "mlvu_dev:1625", + "sampling_stratum": "plotQA", + "source_stratum_index": 512, + "upstream_sample_id": "Q1625" + }, + { + "source_row_index": 1665, + "source_sample_id": "mlvu_dev:1665", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 13, + "upstream_sample_id": "Q1665" + }, + { + "source_row_index": 1691, + "source_sample_id": "mlvu_dev:1691", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 39, + "upstream_sample_id": "Q1691" + }, + { + "source_row_index": 1717, + "source_sample_id": "mlvu_dev:1717", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 65, + "upstream_sample_id": "Q1717" + }, + { + "source_row_index": 1744, + "source_sample_id": "mlvu_dev:1744", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 92, + "upstream_sample_id": "Q1744" + }, + { + "source_row_index": 1770, + "source_sample_id": "mlvu_dev:1770", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 118, + "upstream_sample_id": "Q1770" + }, + { + "source_row_index": 1796, + "source_sample_id": "mlvu_dev:1796", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 144, + "upstream_sample_id": "Q1796" + }, + { + "source_row_index": 1822, + "source_sample_id": "mlvu_dev:1822", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 170, + "upstream_sample_id": "Q1822" + }, + { + "source_row_index": 1849, + "source_sample_id": "mlvu_dev:1849", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 197, + "upstream_sample_id": "Q1849" + }, + { + "source_row_index": 1875, + "source_sample_id": "mlvu_dev:1875", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 223, + "upstream_sample_id": "Q1875" + }, + { + "source_row_index": 1901, + "source_sample_id": "mlvu_dev:1901", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 249, + "upstream_sample_id": "Q1901" + } + ] + }, + "perceptiontest_val_mc": { + "dataset_repository": "lmms-lab/PerceptionTest_Val", + "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "area+reasoning", + "index_space": "within-stratum", + "population_rows": 19140, + "selected_rows": 64, + "strata": [ + { + "name": "abstraction|counterfactual", + "population_rows": 83, + "selected_rows": 5 + }, + { + "name": "abstraction|descriptive", + "population_rows": 5268, + "selected_rows": 5 + }, + { + "name": "abstraction|predictive", + "population_rows": 256, + "selected_rows": 5 + }, + { + "name": "memory|counterfactual", + "population_rows": 36, + "selected_rows": 5 + }, + { + "name": "memory|descriptive", + "population_rows": 575, + "selected_rows": 5 + }, + { + "name": "memory|explanatory", + "population_rows": 135, + "selected_rows": 5 + }, + { + "name": "physics|counterfactual", + "population_rows": 31, + "selected_rows": 5 + }, + { + "name": "physics|descriptive", + "population_rows": 5298, + "selected_rows": 5 + }, + { + "name": "physics|explanatory", + "population_rows": 1226, + "selected_rows": 5 + }, + { + "name": "physics|predictive", + "population_rows": 383, + "selected_rows": 5 + }, + { + "name": "semantics|counterfactual", + "population_rows": 207, + "selected_rows": 5 + }, + { + "name": "semantics|descriptive", + "population_rows": 4717, + "selected_rows": 5 + }, + { + "name": "semantics|explanatory", + "population_rows": 925, + "selected_rows": 4 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 3, + "p25": 38, + "p50": 144, + "p75": 578, + "p100": 4768 + }, + "selected_row_identities_sha256": "7e2d17ac3bff117d51cfc27df79cb7d86e3e3ac7c3d32d94da7b1283befe1594" + }, + "rows": [ + { + "source_row_index": 6448, + "source_sample_id": "perceptiontest_val_mc:6448", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 8, + "upstream_sample_id": "video_4364:3" + }, + { + "source_row_index": 6519, + "source_sample_id": "perceptiontest_val_mc:6519", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 24, + "upstream_sample_id": "video_7896:3" + }, + { + "source_row_index": 6598, + "source_sample_id": "perceptiontest_val_mc:6598", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 41, + "upstream_sample_id": "video_10230:3" + }, + { + "source_row_index": 16262, + "source_sample_id": "perceptiontest_val_mc:16262", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 58, + "upstream_sample_id": "video_9279:3" + }, + { + "source_row_index": 16332, + "source_sample_id": "perceptiontest_val_mc:16332", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 74, + "upstream_sample_id": "video_612:3" + }, + { + "source_row_index": 2460, + "source_sample_id": "perceptiontest_val_mc:2460", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 526, + "upstream_sample_id": "video_3553:2" + }, + { + "source_row_index": 5068, + "source_sample_id": "perceptiontest_val_mc:5068", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 1580, + "upstream_sample_id": "video_11322:2" + }, + { + "source_row_index": 8865, + "source_sample_id": "perceptiontest_val_mc:8865", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 2634, + "upstream_sample_id": "video_1211:2" + }, + { + "source_row_index": 13017, + "source_sample_id": "perceptiontest_val_mc:13017", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 3687, + "upstream_sample_id": "video_7836:1" + }, + { + "source_row_index": 17745, + "source_sample_id": "perceptiontest_val_mc:17745", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 4741, + "upstream_sample_id": "video_4387:1" + }, + { + "source_row_index": 12890, + "source_sample_id": "perceptiontest_val_mc:12890", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 25, + "upstream_sample_id": "video_10565:3" + }, + { + "source_row_index": 13117, + "source_sample_id": "perceptiontest_val_mc:13117", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 76, + "upstream_sample_id": "video_9648:1" + }, + { + "source_row_index": 13244, + "source_sample_id": "perceptiontest_val_mc:13244", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 128, + "upstream_sample_id": "video_8923:1" + }, + { + "source_row_index": 13512, + "source_sample_id": "perceptiontest_val_mc:13512", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 179, + "upstream_sample_id": "video_7597:4" + }, + { + "source_row_index": 18362, + "source_sample_id": "perceptiontest_val_mc:18362", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 230, + "upstream_sample_id": "video_2651:5" + }, + { + "source_row_index": 3874, + "source_sample_id": "perceptiontest_val_mc:3874", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 3, + "upstream_sample_id": "video_9066:0" + }, + { + "source_row_index": 3911, + "source_sample_id": "perceptiontest_val_mc:3911", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 10, + "upstream_sample_id": "video_2719:1" + }, + { + "source_row_index": 3957, + "source_sample_id": "perceptiontest_val_mc:3957", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 18, + "upstream_sample_id": "video_9203:0" + }, + { + "source_row_index": 3998, + "source_sample_id": "perceptiontest_val_mc:3998", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 25, + "upstream_sample_id": "video_5154:1" + }, + { + "source_row_index": 4037, + "source_sample_id": "perceptiontest_val_mc:4037", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 32, + "upstream_sample_id": "video_7255:0" + }, + { + "source_row_index": 994, + "source_sample_id": "perceptiontest_val_mc:994", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 57, + "upstream_sample_id": "video_3941:1" + }, + { + "source_row_index": 9137, + "source_sample_id": "perceptiontest_val_mc:9137", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 172, + "upstream_sample_id": "video_875:5" + }, + { + "source_row_index": 9350, + "source_sample_id": "perceptiontest_val_mc:9350", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 287, + "upstream_sample_id": "video_4505:3" + }, + { + "source_row_index": 9564, + "source_sample_id": "perceptiontest_val_mc:9564", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 402, + "upstream_sample_id": "video_11549:4" + }, + { + "source_row_index": 17386, + "source_sample_id": "perceptiontest_val_mc:17386", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 517, + "upstream_sample_id": "video_8337:3" + }, + { + "source_row_index": 8805, + "source_sample_id": "perceptiontest_val_mc:8805", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 13, + "upstream_sample_id": "video_8238:1" + }, + { + "source_row_index": 8893, + "source_sample_id": "perceptiontest_val_mc:8893", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 40, + "upstream_sample_id": "video_4058:1" + }, + { + "source_row_index": 8981, + "source_sample_id": "perceptiontest_val_mc:8981", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 67, + "upstream_sample_id": "video_4183:1" + }, + { + "source_row_index": 16722, + "source_sample_id": "perceptiontest_val_mc:16722", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 94, + "upstream_sample_id": "video_9149:1" + }, + { + "source_row_index": 18569, + "source_sample_id": "perceptiontest_val_mc:18569", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 121, + "upstream_sample_id": "video_4283:1" + }, + { + "source_row_index": 11717, + "source_sample_id": "perceptiontest_val_mc:11717", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 3, + "upstream_sample_id": "video_9300:3" + }, + { + "source_row_index": 11750, + "source_sample_id": "perceptiontest_val_mc:11750", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 9, + "upstream_sample_id": "video_171:2" + }, + { + "source_row_index": 11798, + "source_sample_id": "perceptiontest_val_mc:11798", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 15, + "upstream_sample_id": "video_4710:3" + }, + { + "source_row_index": 17813, + "source_sample_id": "perceptiontest_val_mc:17813", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 21, + "upstream_sample_id": "video_2589:3" + }, + { + "source_row_index": 17849, + "source_sample_id": "perceptiontest_val_mc:17849", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 27, + "upstream_sample_id": "video_6707:3" + }, + { + "source_row_index": 2162, + "source_sample_id": "perceptiontest_val_mc:2162", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 529, + "upstream_sample_id": "video_10714:0" + }, + { + "source_row_index": 6373, + "source_sample_id": "perceptiontest_val_mc:6373", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 1589, + "upstream_sample_id": "video_3840:0" + }, + { + "source_row_index": 10471, + "source_sample_id": "perceptiontest_val_mc:10471", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 2649, + "upstream_sample_id": "video_1567:3" + }, + { + "source_row_index": 13179, + "source_sample_id": "perceptiontest_val_mc:13179", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 3708, + "upstream_sample_id": "video_1768:0" + }, + { + "source_row_index": 17169, + "source_sample_id": "perceptiontest_val_mc:17169", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 4768, + "upstream_sample_id": "video_2812:0" + }, + { + "source_row_index": 7652, + "source_sample_id": "perceptiontest_val_mc:7652", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 122, + "upstream_sample_id": "video_3751:3" + }, + { + "source_row_index": 8195, + "source_sample_id": "perceptiontest_val_mc:8195", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 367, + "upstream_sample_id": "video_6995:1" + }, + { + "source_row_index": 8619, + "source_sample_id": "perceptiontest_val_mc:8619", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 613, + "upstream_sample_id": "video_1323:1" + }, + { + "source_row_index": 13731, + "source_sample_id": "perceptiontest_val_mc:13731", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 858, + "upstream_sample_id": "video_1625:0" + }, + { + "source_row_index": 16579, + "source_sample_id": "perceptiontest_val_mc:16579", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 1103, + "upstream_sample_id": "video_617:1" + }, + { + "source_row_index": 4535, + "source_sample_id": "perceptiontest_val_mc:4535", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 38, + "upstream_sample_id": "video_5408:5" + }, + { + "source_row_index": 5180, + "source_sample_id": "perceptiontest_val_mc:5180", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 114, + "upstream_sample_id": "video_2732:4" + }, + { + "source_row_index": 5898, + "source_sample_id": "perceptiontest_val_mc:5898", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 191, + "upstream_sample_id": "video_7998:4" + }, + { + "source_row_index": 13907, + "source_sample_id": "perceptiontest_val_mc:13907", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 268, + "upstream_sample_id": "video_1539:0" + }, + { + "source_row_index": 15729, + "source_sample_id": "perceptiontest_val_mc:15729", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 344, + "upstream_sample_id": "video_356:4" + }, + { + "source_row_index": 81, + "source_sample_id": "perceptiontest_val_mc:81", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 20, + "upstream_sample_id": "video_1482:1" + }, + { + "source_row_index": 1794, + "source_sample_id": "perceptiontest_val_mc:1794", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 62, + "upstream_sample_id": "video_1906:9" + }, + { + "source_row_index": 2310, + "source_sample_id": "perceptiontest_val_mc:2310", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 103, + "upstream_sample_id": "video_10287:11" + }, + { + "source_row_index": 2784, + "source_sample_id": "perceptiontest_val_mc:2784", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 144, + "upstream_sample_id": "video_10446:10" + }, + { + "source_row_index": 14956, + "source_sample_id": "perceptiontest_val_mc:14956", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 186, + "upstream_sample_id": "video_5971:11" + }, + { + "source_row_index": 1137, + "source_sample_id": "perceptiontest_val_mc:1137", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 471, + "upstream_sample_id": "video_6936:0" + }, + { + "source_row_index": 3530, + "source_sample_id": "perceptiontest_val_mc:3530", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 1415, + "upstream_sample_id": "video_5178:4" + }, + { + "source_row_index": 7494, + "source_sample_id": "perceptiontest_val_mc:7494", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 2358, + "upstream_sample_id": "video_2103:0" + }, + { + "source_row_index": 13219, + "source_sample_id": "perceptiontest_val_mc:13219", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 3301, + "upstream_sample_id": "video_1425:2" + }, + { + "source_row_index": 16942, + "source_sample_id": "perceptiontest_val_mc:16942", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 4245, + "upstream_sample_id": "video_2446:0" + }, + { + "source_row_index": 1615, + "source_sample_id": "perceptiontest_val_mc:1615", + "sampling_stratum": "semantics|explanatory", + "source_stratum_index": 115, + "upstream_sample_id": "video_3837:1" + }, + { + "source_row_index": 9793, + "source_sample_id": "perceptiontest_val_mc:9793", + "sampling_stratum": "semantics|explanatory", + "source_stratum_index": 346, + "upstream_sample_id": "video_10838:1" + }, + { + "source_row_index": 14460, + "source_sample_id": "perceptiontest_val_mc:14460", + "sampling_stratum": "semantics|explanatory", + "source_stratum_index": 578, + "upstream_sample_id": "video_9722:2" + }, + { + "source_row_index": 17057, + "source_sample_id": "perceptiontest_val_mc:17057", + "sampling_stratum": "semantics|explanatory", + "source_stratum_index": 809, + "upstream_sample_id": "video_665:1" + } + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json index 604e5fcc853..7c08a8a5b26 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json @@ -2,7 +2,7 @@ "extends": "short-v1", "schema": "modelopt.vlm-evaluation-profile/v1", "name": "short-native-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "lmms_eval_revision": "88b23e2bfa16a1edbc16e9e238ed82130b3a4f56", "backend": { "attention_implementation": "sdpa", "enable_thinking": false, diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-native-v2.json b/examples/puzzletron/evaluation/vlm/profiles/short-native-v2.json new file mode 100644 index 00000000000..689f63b6672 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/short-native-v2.json @@ -0,0 +1,1873 @@ +{ + "extends": "short-native-v1", + "schema": "modelopt.vlm-evaluation-profile/v1", + "name": "short-native-v2", + "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "sampling": { + "schema": "modelopt.vlm-sampling-audit/v1", + "claim_scope": "deterministic-screening-only", + "generator": { + "name": "systematic-midpoint", + "version": 1 + } + }, + "tasks": { + "realworldqa": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "split", + "index_space": "within-stratum", + "population_rows": 765, + "selected_rows": 64, + "strata": [ + { + "name": "test", + "population_rows": 765, + "selected_rows": 64 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 5, + "p25": 185, + "p50": 376, + "p75": 567, + "p100": 759 + }, + "selected_row_identities_sha256": "c8a03d3811daa83ecd8d82f60d0f6a04736594c1ca3bae2b665b6f6076273b0b" + }, + "rows": [ + { + "source_row_index": 5, + "source_sample_id": "test:5" + }, + { + "source_row_index": 17, + "source_sample_id": "test:17" + }, + { + "source_row_index": 29, + "source_sample_id": "test:29" + }, + { + "source_row_index": 41, + "source_sample_id": "test:41" + }, + { + "source_row_index": 53, + "source_sample_id": "test:53" + }, + { + "source_row_index": 65, + "source_sample_id": "test:65" + }, + { + "source_row_index": 77, + "source_sample_id": "test:77" + }, + { + "source_row_index": 89, + "source_sample_id": "test:89" + }, + { + "source_row_index": 101, + "source_sample_id": "test:101" + }, + { + "source_row_index": 113, + "source_sample_id": "test:113" + }, + { + "source_row_index": 125, + "source_sample_id": "test:125" + }, + { + "source_row_index": 137, + "source_sample_id": "test:137" + }, + { + "source_row_index": 149, + "source_sample_id": "test:149" + }, + { + "source_row_index": 161, + "source_sample_id": "test:161" + }, + { + "source_row_index": 173, + "source_sample_id": "test:173" + }, + { + "source_row_index": 185, + "source_sample_id": "test:185" + }, + { + "source_row_index": 197, + "source_sample_id": "test:197" + }, + { + "source_row_index": 209, + "source_sample_id": "test:209" + }, + { + "source_row_index": 221, + "source_sample_id": "test:221" + }, + { + "source_row_index": 233, + "source_sample_id": "test:233" + }, + { + "source_row_index": 245, + "source_sample_id": "test:245" + }, + { + "source_row_index": 256, + "source_sample_id": "test:256" + }, + { + "source_row_index": 268, + "source_sample_id": "test:268" + }, + { + "source_row_index": 280, + "source_sample_id": "test:280" + }, + { + "source_row_index": 292, + "source_sample_id": "test:292" + }, + { + "source_row_index": 304, + "source_sample_id": "test:304" + }, + { + "source_row_index": 316, + "source_sample_id": "test:316" + }, + { + "source_row_index": 328, + "source_sample_id": "test:328" + }, + { + "source_row_index": 340, + "source_sample_id": "test:340" + }, + { + "source_row_index": 352, + "source_sample_id": "test:352" + }, + { + "source_row_index": 364, + "source_sample_id": "test:364" + }, + { + "source_row_index": 376, + "source_sample_id": "test:376" + }, + { + "source_row_index": 388, + "source_sample_id": "test:388" + }, + { + "source_row_index": 400, + "source_sample_id": "test:400" + }, + { + "source_row_index": 412, + "source_sample_id": "test:412" + }, + { + "source_row_index": 424, + "source_sample_id": "test:424" + }, + { + "source_row_index": 436, + "source_sample_id": "test:436" + }, + { + "source_row_index": 448, + "source_sample_id": "test:448" + }, + { + "source_row_index": 460, + "source_sample_id": "test:460" + }, + { + "source_row_index": 472, + "source_sample_id": "test:472" + }, + { + "source_row_index": 484, + "source_sample_id": "test:484" + }, + { + "source_row_index": 496, + "source_sample_id": "test:496" + }, + { + "source_row_index": 508, + "source_sample_id": "test:508" + }, + { + "source_row_index": 519, + "source_sample_id": "test:519" + }, + { + "source_row_index": 531, + "source_sample_id": "test:531" + }, + { + "source_row_index": 543, + "source_sample_id": "test:543" + }, + { + "source_row_index": 555, + "source_sample_id": "test:555" + }, + { + "source_row_index": 567, + "source_sample_id": "test:567" + }, + { + "source_row_index": 579, + "source_sample_id": "test:579" + }, + { + "source_row_index": 591, + "source_sample_id": "test:591" + }, + { + "source_row_index": 603, + "source_sample_id": "test:603" + }, + { + "source_row_index": 615, + "source_sample_id": "test:615" + }, + { + "source_row_index": 627, + "source_sample_id": "test:627" + }, + { + "source_row_index": 639, + "source_sample_id": "test:639" + }, + { + "source_row_index": 651, + "source_sample_id": "test:651" + }, + { + "source_row_index": 663, + "source_sample_id": "test:663" + }, + { + "source_row_index": 675, + "source_sample_id": "test:675" + }, + { + "source_row_index": 687, + "source_sample_id": "test:687" + }, + { + "source_row_index": 699, + "source_sample_id": "test:699" + }, + { + "source_row_index": 711, + "source_sample_id": "test:711" + }, + { + "source_row_index": 723, + "source_sample_id": "test:723" + }, + { + "source_row_index": 735, + "source_sample_id": "test:735" + }, + { + "source_row_index": 747, + "source_sample_id": "test:747" + }, + { + "source_row_index": 759, + "source_sample_id": "test:759" + } + ] + }, + "mmmu_val": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "subject", + "index_space": "within-stratum", + "population_rows": 900, + "selected_rows": 120, + "strata": [ + { + "name": "Accounting", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Agriculture", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Architecture_and_Engineering", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Art", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Art_Theory", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Basic_Medical_Science", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Biology", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Chemistry", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Clinical_Medicine", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Computer_Science", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Design", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Diagnostics_and_Laboratory_Medicine", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Economics", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Electronics", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Energy_and_Power", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Finance", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Geography", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "History", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Literature", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Manage", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Marketing", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Materials", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Math", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Mechanical_Engineering", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Music", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Pharmacy", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Physics", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Psychology", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Public_Health", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Sociology", + "population_rows": 30, + "selected_rows": 4 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 3, + "p25": 3, + "p50": 11, + "p75": 18, + "p100": 26 + }, + "selected_row_identities_sha256": "b09e9454c2d03cddf09f32526b390099408a8dabbe24318fb221e5c5fc60556d" + }, + "rows": [ + { + "source_row_index": 3, + "source_sample_id": "validation_Accounting_4" + }, + { + "source_row_index": 11, + "source_sample_id": "validation_Accounting_12" + }, + { + "source_row_index": 18, + "source_sample_id": "validation_Accounting_19" + }, + { + "source_row_index": 26, + "source_sample_id": "validation_Accounting_27" + }, + { + "source_row_index": 33, + "source_sample_id": "validation_Agriculture_4" + }, + { + "source_row_index": 41, + "source_sample_id": "validation_Agriculture_12" + }, + { + "source_row_index": 48, + "source_sample_id": "validation_Agriculture_19" + }, + { + "source_row_index": 56, + "source_sample_id": "validation_Agriculture_27" + }, + { + "source_row_index": 63, + "source_sample_id": "validation_Architecture_and_Engineering_4" + }, + { + "source_row_index": 71, + "source_sample_id": "validation_Architecture_and_Engineering_12" + }, + { + "source_row_index": 78, + "source_sample_id": "validation_Architecture_and_Engineering_19" + }, + { + "source_row_index": 86, + "source_sample_id": "validation_Architecture_and_Engineering_27" + }, + { + "source_row_index": 93, + "source_sample_id": "validation_Art_4" + }, + { + "source_row_index": 101, + "source_sample_id": "validation_Art_12" + }, + { + "source_row_index": 108, + "source_sample_id": "validation_Art_19" + }, + { + "source_row_index": 116, + "source_sample_id": "validation_Art_27" + }, + { + "source_row_index": 123, + "source_sample_id": "validation_Art_Theory_4" + }, + { + "source_row_index": 131, + "source_sample_id": "validation_Art_Theory_12" + }, + { + "source_row_index": 138, + "source_sample_id": "validation_Art_Theory_19" + }, + { + "source_row_index": 146, + "source_sample_id": "validation_Art_Theory_27" + }, + { + "source_row_index": 153, + "source_sample_id": "validation_Basic_Medical_Science_4" + }, + { + "source_row_index": 161, + "source_sample_id": "validation_Basic_Medical_Science_12" + }, + { + "source_row_index": 168, + "source_sample_id": "validation_Basic_Medical_Science_19" + }, + { + "source_row_index": 176, + "source_sample_id": "validation_Basic_Medical_Science_27" + }, + { + "source_row_index": 183, + "source_sample_id": "validation_Biology_4" + }, + { + "source_row_index": 191, + "source_sample_id": "validation_Biology_12" + }, + { + "source_row_index": 198, + "source_sample_id": "validation_Biology_19" + }, + { + "source_row_index": 206, + "source_sample_id": "validation_Biology_27" + }, + { + "source_row_index": 213, + "source_sample_id": "validation_Chemistry_4" + }, + { + "source_row_index": 221, + "source_sample_id": "validation_Chemistry_12" + }, + { + "source_row_index": 228, + "source_sample_id": "validation_Chemistry_19" + }, + { + "source_row_index": 236, + "source_sample_id": "validation_Chemistry_27" + }, + { + "source_row_index": 243, + "source_sample_id": "validation_Clinical_Medicine_4" + }, + { + "source_row_index": 251, + "source_sample_id": "validation_Clinical_Medicine_12" + }, + { + "source_row_index": 258, + "source_sample_id": "validation_Clinical_Medicine_19" + }, + { + "source_row_index": 266, + "source_sample_id": "validation_Clinical_Medicine_27" + }, + { + "source_row_index": 273, + "source_sample_id": "validation_Computer_Science_4" + }, + { + "source_row_index": 281, + "source_sample_id": "validation_Computer_Science_12" + }, + { + "source_row_index": 288, + "source_sample_id": "validation_Computer_Science_19" + }, + { + "source_row_index": 296, + "source_sample_id": "validation_Computer_Science_27" + }, + { + "source_row_index": 303, + "source_sample_id": "validation_Design_4" + }, + { + "source_row_index": 311, + "source_sample_id": "validation_Design_12" + }, + { + "source_row_index": 318, + "source_sample_id": "validation_Design_19" + }, + { + "source_row_index": 326, + "source_sample_id": "validation_Design_27" + }, + { + "source_row_index": 333, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_4" + }, + { + "source_row_index": 341, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_12" + }, + { + "source_row_index": 348, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_19" + }, + { + "source_row_index": 356, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_27" + }, + { + "source_row_index": 363, + "source_sample_id": "validation_Economics_4" + }, + { + "source_row_index": 371, + "source_sample_id": "validation_Economics_12" + }, + { + "source_row_index": 378, + "source_sample_id": "validation_Economics_19" + }, + { + "source_row_index": 386, + "source_sample_id": "validation_Economics_27" + }, + { + "source_row_index": 393, + "source_sample_id": "validation_Electronics_4" + }, + { + "source_row_index": 401, + "source_sample_id": "validation_Electronics_12" + }, + { + "source_row_index": 408, + "source_sample_id": "validation_Electronics_19" + }, + { + "source_row_index": 416, + "source_sample_id": "validation_Electronics_27" + }, + { + "source_row_index": 423, + "source_sample_id": "validation_Energy_and_Power_4" + }, + { + "source_row_index": 431, + "source_sample_id": "validation_Energy_and_Power_12" + }, + { + "source_row_index": 438, + "source_sample_id": "validation_Energy_and_Power_19" + }, + { + "source_row_index": 446, + "source_sample_id": "validation_Energy_and_Power_27" + }, + { + "source_row_index": 453, + "source_sample_id": "validation_Finance_4" + }, + { + "source_row_index": 461, + "source_sample_id": "validation_Finance_12" + }, + { + "source_row_index": 468, + "source_sample_id": "validation_Finance_19" + }, + { + "source_row_index": 476, + "source_sample_id": "validation_Finance_27" + }, + { + "source_row_index": 483, + "source_sample_id": "validation_Geography_4" + }, + { + "source_row_index": 491, + "source_sample_id": "validation_Geography_12" + }, + { + "source_row_index": 498, + "source_sample_id": "validation_Geography_19" + }, + { + "source_row_index": 506, + "source_sample_id": "validation_Geography_27" + }, + { + "source_row_index": 513, + "source_sample_id": "validation_History_4" + }, + { + "source_row_index": 521, + "source_sample_id": "validation_History_12" + }, + { + "source_row_index": 528, + "source_sample_id": "validation_History_19" + }, + { + "source_row_index": 536, + "source_sample_id": "validation_History_27" + }, + { + "source_row_index": 543, + "source_sample_id": "validation_Literature_4" + }, + { + "source_row_index": 551, + "source_sample_id": "validation_Literature_12" + }, + { + "source_row_index": 558, + "source_sample_id": "validation_Literature_19" + }, + { + "source_row_index": 566, + "source_sample_id": "validation_Literature_27" + }, + { + "source_row_index": 573, + "source_sample_id": "validation_Manage_4" + }, + { + "source_row_index": 581, + "source_sample_id": "validation_Manage_12" + }, + { + "source_row_index": 588, + "source_sample_id": "validation_Manage_19" + }, + { + "source_row_index": 596, + "source_sample_id": "validation_Manage_27" + }, + { + "source_row_index": 603, + "source_sample_id": "validation_Marketing_4" + }, + { + "source_row_index": 611, + "source_sample_id": "validation_Marketing_12" + }, + { + "source_row_index": 618, + "source_sample_id": "validation_Marketing_19" + }, + { + "source_row_index": 626, + "source_sample_id": "validation_Marketing_27" + }, + { + "source_row_index": 633, + "source_sample_id": "validation_Materials_4" + }, + { + "source_row_index": 641, + "source_sample_id": "validation_Materials_12" + }, + { + "source_row_index": 648, + "source_sample_id": "validation_Materials_19" + }, + { + "source_row_index": 656, + "source_sample_id": "validation_Materials_27" + }, + { + "source_row_index": 663, + "source_sample_id": "validation_Math_4" + }, + { + "source_row_index": 671, + "source_sample_id": "validation_Math_12" + }, + { + "source_row_index": 678, + "source_sample_id": "validation_Math_19" + }, + { + "source_row_index": 686, + "source_sample_id": "validation_Math_27" + }, + { + "source_row_index": 693, + "source_sample_id": "validation_Mechanical_Engineering_4" + }, + { + "source_row_index": 701, + "source_sample_id": "validation_Mechanical_Engineering_12" + }, + { + "source_row_index": 708, + "source_sample_id": "validation_Mechanical_Engineering_19" + }, + { + "source_row_index": 716, + "source_sample_id": "validation_Mechanical_Engineering_27" + }, + { + "source_row_index": 723, + "source_sample_id": "validation_Music_4" + }, + { + "source_row_index": 731, + "source_sample_id": "validation_Music_12" + }, + { + "source_row_index": 738, + "source_sample_id": "validation_Music_19" + }, + { + "source_row_index": 746, + "source_sample_id": "validation_Music_27" + }, + { + "source_row_index": 753, + "source_sample_id": "validation_Pharmacy_4" + }, + { + "source_row_index": 761, + "source_sample_id": "validation_Pharmacy_12" + }, + { + "source_row_index": 768, + "source_sample_id": "validation_Pharmacy_19" + }, + { + "source_row_index": 776, + "source_sample_id": "validation_Pharmacy_27" + }, + { + "source_row_index": 783, + "source_sample_id": "validation_Physics_4" + }, + { + "source_row_index": 791, + "source_sample_id": "validation_Physics_12" + }, + { + "source_row_index": 798, + "source_sample_id": "validation_Physics_19" + }, + { + "source_row_index": 806, + "source_sample_id": "validation_Physics_27" + }, + { + "source_row_index": 813, + "source_sample_id": "validation_Psychology_4" + }, + { + "source_row_index": 821, + "source_sample_id": "validation_Psychology_12" + }, + { + "source_row_index": 828, + "source_sample_id": "validation_Psychology_19" + }, + { + "source_row_index": 836, + "source_sample_id": "validation_Psychology_27" + }, + { + "source_row_index": 843, + "source_sample_id": "validation_Public_Health_4" + }, + { + "source_row_index": 851, + "source_sample_id": "validation_Public_Health_12" + }, + { + "source_row_index": 858, + "source_sample_id": "validation_Public_Health_19" + }, + { + "source_row_index": 866, + "source_sample_id": "validation_Public_Health_27" + }, + { + "source_row_index": 873, + "source_sample_id": "validation_Sociology_4" + }, + { + "source_row_index": 881, + "source_sample_id": "validation_Sociology_12" + }, + { + "source_row_index": 888, + "source_sample_id": "validation_Sociology_19" + }, + { + "source_row_index": 896, + "source_sample_id": "validation_Sociology_27" + } + ] + }, + "mvbench": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "leaf_task", + "index_space": "within-stratum", + "population_rows": 4000, + "selected_rows": 160, + "strata": [ + { + "name": "action_sequence", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "moving_count", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "action_prediction", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "episodic_reasoning", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "action_antonym", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "action_count", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "scene_transition", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "object_shuffle", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "object_existence", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "fine_grained_pose", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "unexpected_action", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "moving_direction", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "state_change", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "object_interaction", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "character_order", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "action_localization", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "counterfactual_inference", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "fine_grained_action", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "moving_attribute", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "egocentric_navigation", + "population_rows": 200, + "selected_rows": 8 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 12, + "p25": 37, + "p50": 87, + "p75": 137, + "p100": 187 + }, + "selected_row_identities_sha256": "e1a4c0665e284828758de4a4f4cb847f98b37df314eb2cb5a9f58631f065b82c" + }, + "rows": [ + { + "source_row_index": 12, + "source_sample_id": "action_sequence:12", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 37, + "source_sample_id": "action_sequence:37", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 62, + "source_sample_id": "action_sequence:62", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 87, + "source_sample_id": "action_sequence:87", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 112, + "source_sample_id": "action_sequence:112", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 137, + "source_sample_id": "action_sequence:137", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 162, + "source_sample_id": "action_sequence:162", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 187, + "source_sample_id": "action_sequence:187", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 12, + "source_sample_id": "moving_count:12", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 37, + "source_sample_id": "moving_count:37", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 62, + "source_sample_id": "moving_count:62", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 87, + "source_sample_id": "moving_count:87", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 112, + "source_sample_id": "moving_count:112", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 137, + "source_sample_id": "moving_count:137", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 162, + "source_sample_id": "moving_count:162", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 187, + "source_sample_id": "moving_count:187", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 12, + "source_sample_id": "action_prediction:12", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 37, + "source_sample_id": "action_prediction:37", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 62, + "source_sample_id": "action_prediction:62", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 87, + "source_sample_id": "action_prediction:87", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 112, + "source_sample_id": "action_prediction:112", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 137, + "source_sample_id": "action_prediction:137", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 162, + "source_sample_id": "action_prediction:162", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 187, + "source_sample_id": "action_prediction:187", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 12, + "source_sample_id": "episodic_reasoning:12", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 37, + "source_sample_id": "episodic_reasoning:37", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 62, + "source_sample_id": "episodic_reasoning:62", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 87, + "source_sample_id": "episodic_reasoning:87", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 112, + "source_sample_id": "episodic_reasoning:112", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 137, + "source_sample_id": "episodic_reasoning:137", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 162, + "source_sample_id": "episodic_reasoning:162", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 187, + "source_sample_id": "episodic_reasoning:187", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 12, + "source_sample_id": "action_antonym:12", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 37, + "source_sample_id": "action_antonym:37", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 62, + "source_sample_id": "action_antonym:62", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 87, + "source_sample_id": "action_antonym:87", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 112, + "source_sample_id": "action_antonym:112", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 137, + "source_sample_id": "action_antonym:137", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 162, + "source_sample_id": "action_antonym:162", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 187, + "source_sample_id": "action_antonym:187", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 12, + "source_sample_id": "action_count:12", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 37, + "source_sample_id": "action_count:37", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 62, + "source_sample_id": "action_count:62", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 87, + "source_sample_id": "action_count:87", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 112, + "source_sample_id": "action_count:112", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 137, + "source_sample_id": "action_count:137", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 162, + "source_sample_id": "action_count:162", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 187, + "source_sample_id": "action_count:187", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 12, + "source_sample_id": "scene_transition:12", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 37, + "source_sample_id": "scene_transition:37", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 62, + "source_sample_id": "scene_transition:62", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 87, + "source_sample_id": "scene_transition:87", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 112, + "source_sample_id": "scene_transition:112", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 137, + "source_sample_id": "scene_transition:137", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 162, + "source_sample_id": "scene_transition:162", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 187, + "source_sample_id": "scene_transition:187", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 12, + "source_sample_id": "object_shuffle:12", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 37, + "source_sample_id": "object_shuffle:37", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 62, + "source_sample_id": "object_shuffle:62", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 87, + "source_sample_id": "object_shuffle:87", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 112, + "source_sample_id": "object_shuffle:112", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 137, + "source_sample_id": "object_shuffle:137", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 162, + "source_sample_id": "object_shuffle:162", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 187, + "source_sample_id": "object_shuffle:187", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 12, + "source_sample_id": "object_existence:12", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 37, + "source_sample_id": "object_existence:37", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 62, + "source_sample_id": "object_existence:62", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 87, + "source_sample_id": "object_existence:87", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 112, + "source_sample_id": "object_existence:112", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 137, + "source_sample_id": "object_existence:137", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 162, + "source_sample_id": "object_existence:162", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 187, + "source_sample_id": "object_existence:187", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 12, + "source_sample_id": "fine_grained_pose:12", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 37, + "source_sample_id": "fine_grained_pose:37", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 62, + "source_sample_id": "fine_grained_pose:62", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 87, + "source_sample_id": "fine_grained_pose:87", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 112, + "source_sample_id": "fine_grained_pose:112", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 137, + "source_sample_id": "fine_grained_pose:137", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 162, + "source_sample_id": "fine_grained_pose:162", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 187, + "source_sample_id": "fine_grained_pose:187", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 12, + "source_sample_id": "unexpected_action:12", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 37, + "source_sample_id": "unexpected_action:37", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 62, + "source_sample_id": "unexpected_action:62", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 87, + "source_sample_id": "unexpected_action:87", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 112, + "source_sample_id": "unexpected_action:112", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 137, + "source_sample_id": "unexpected_action:137", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 162, + "source_sample_id": "unexpected_action:162", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 187, + "source_sample_id": "unexpected_action:187", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 12, + "source_sample_id": "moving_direction:12", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 37, + "source_sample_id": "moving_direction:37", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 62, + "source_sample_id": "moving_direction:62", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 87, + "source_sample_id": "moving_direction:87", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 112, + "source_sample_id": "moving_direction:112", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 137, + "source_sample_id": "moving_direction:137", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 162, + "source_sample_id": "moving_direction:162", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 187, + "source_sample_id": "moving_direction:187", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 12, + "source_sample_id": "state_change:12", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 37, + "source_sample_id": "state_change:37", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 62, + "source_sample_id": "state_change:62", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 87, + "source_sample_id": "state_change:87", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 112, + "source_sample_id": "state_change:112", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 137, + "source_sample_id": "state_change:137", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 162, + "source_sample_id": "state_change:162", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 187, + "source_sample_id": "state_change:187", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 12, + "source_sample_id": "object_interaction:12", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 37, + "source_sample_id": "object_interaction:37", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 62, + "source_sample_id": "object_interaction:62", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 87, + "source_sample_id": "object_interaction:87", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 112, + "source_sample_id": "object_interaction:112", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 137, + "source_sample_id": "object_interaction:137", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 162, + "source_sample_id": "object_interaction:162", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 187, + "source_sample_id": "object_interaction:187", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 12, + "source_sample_id": "character_order:12", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 37, + "source_sample_id": "character_order:37", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 62, + "source_sample_id": "character_order:62", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 87, + "source_sample_id": "character_order:87", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 112, + "source_sample_id": "character_order:112", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 137, + "source_sample_id": "character_order:137", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 162, + "source_sample_id": "character_order:162", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 187, + "source_sample_id": "character_order:187", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 12, + "source_sample_id": "action_localization:12", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 37, + "source_sample_id": "action_localization:37", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 62, + "source_sample_id": "action_localization:62", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 87, + "source_sample_id": "action_localization:87", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 112, + "source_sample_id": "action_localization:112", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 137, + "source_sample_id": "action_localization:137", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 162, + "source_sample_id": "action_localization:162", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 187, + "source_sample_id": "action_localization:187", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 12, + "source_sample_id": "counterfactual_inference:12", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 37, + "source_sample_id": "counterfactual_inference:37", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 62, + "source_sample_id": "counterfactual_inference:62", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 87, + "source_sample_id": "counterfactual_inference:87", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 112, + "source_sample_id": "counterfactual_inference:112", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 137, + "source_sample_id": "counterfactual_inference:137", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 162, + "source_sample_id": "counterfactual_inference:162", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 187, + "source_sample_id": "counterfactual_inference:187", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 12, + "source_sample_id": "fine_grained_action:12", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 37, + "source_sample_id": "fine_grained_action:37", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 62, + "source_sample_id": "fine_grained_action:62", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 87, + "source_sample_id": "fine_grained_action:87", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 112, + "source_sample_id": "fine_grained_action:112", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 137, + "source_sample_id": "fine_grained_action:137", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 162, + "source_sample_id": "fine_grained_action:162", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 187, + "source_sample_id": "fine_grained_action:187", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 12, + "source_sample_id": "moving_attribute:12", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 37, + "source_sample_id": "moving_attribute:37", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 62, + "source_sample_id": "moving_attribute:62", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 87, + "source_sample_id": "moving_attribute:87", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 112, + "source_sample_id": "moving_attribute:112", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 137, + "source_sample_id": "moving_attribute:137", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 162, + "source_sample_id": "moving_attribute:162", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 187, + "source_sample_id": "moving_attribute:187", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 12, + "source_sample_id": "egocentric_navigation:12", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 37, + "source_sample_id": "egocentric_navigation:37", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 62, + "source_sample_id": "egocentric_navigation:62", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 87, + "source_sample_id": "egocentric_navigation:87", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 112, + "source_sample_id": "egocentric_navigation:112", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 137, + "source_sample_id": "egocentric_navigation:137", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 162, + "source_sample_id": "egocentric_navigation:162", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 187, + "source_sample_id": "egocentric_navigation:187", + "leaf_task": "mvbench_egocentric_navigation" + } + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-v1.json index 68d0e500aca..ddae52d9f5d 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-v1.json @@ -1,7 +1,7 @@ { "schema": "modelopt.vlm-evaluation-profile/v1", "name": "short-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "lmms_eval_revision": "15c32bfec165df13c269ddd3cda03b2ed9137825", "model_family": { "architecture": "Qwen3_5ForConditionalGeneration", "model_type": "qwen3_5" diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-vllm-v2.json b/examples/puzzletron/evaluation/vlm/profiles/short-vllm-v2.json new file mode 100644 index 00000000000..ec5258ea832 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/short-vllm-v2.json @@ -0,0 +1,13 @@ +{ + "extends": "short-native-v2", + "schema": "modelopt.vlm-evaluation-profile/v1", + "name": "short-vllm-v2", + "backend": { + "attention_config": { + "flash_attn_version": 2 + }, + "enable_thinking": false, + "name": "vllm", + "reasoning_parser": "qwen3" + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/smoke-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/smoke-native-v1.json new file mode 100644 index 00000000000..ee780042956 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/smoke-native-v1.json @@ -0,0 +1,129 @@ +{ + "extends": "short-native-v2", + "schema": "modelopt.vlm-evaluation-profile/v1", + "name": "smoke-native-v1", + "tasks": { + "realworldqa": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "split", + "index_space": "within-stratum", + "population_rows": 765, + "selected_rows": 8, + "strata": [ + {"name": "test", "population_rows": 765, "selected_rows": 8} + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 47, + "p25": 143, + "p50": 334, + "p75": 525, + "p100": 717 + }, + "selected_row_identities_sha256": "18cfff65ce81ca45537a6de3f17184ed51d23c24247e364b5a565a8ef368f70c" + }, + "rows": [ + {"source_row_index": 47, "source_sample_id": "test:47"}, + {"source_row_index": 143, "source_sample_id": "test:143"}, + {"source_row_index": 239, "source_sample_id": "test:239"}, + {"source_row_index": 334, "source_sample_id": "test:334"}, + {"source_row_index": 430, "source_sample_id": "test:430"}, + {"source_row_index": 525, "source_sample_id": "test:525"}, + {"source_row_index": 621, "source_sample_id": "test:621"}, + {"source_row_index": 717, "source_sample_id": "test:717"} + ] + }, + "mmmu_val": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "subject", + "index_space": "within-stratum", + "population_rows": 900, + "selected_rows": 8, + "strata": [ + {"name": "Accounting", "population_rows": 30, "selected_rows": 0}, + {"name": "Agriculture", "population_rows": 30, "selected_rows": 1}, + {"name": "Architecture_and_Engineering", "population_rows": 30, "selected_rows": 0}, + {"name": "Art", "population_rows": 30, "selected_rows": 0}, + {"name": "Art_Theory", "population_rows": 30, "selected_rows": 0}, + {"name": "Basic_Medical_Science", "population_rows": 30, "selected_rows": 1}, + {"name": "Biology", "population_rows": 30, "selected_rows": 0}, + {"name": "Chemistry", "population_rows": 30, "selected_rows": 0}, + {"name": "Clinical_Medicine", "population_rows": 30, "selected_rows": 0}, + {"name": "Computer_Science", "population_rows": 30, "selected_rows": 1}, + {"name": "Design", "population_rows": 30, "selected_rows": 0}, + {"name": "Diagnostics_and_Laboratory_Medicine", "population_rows": 30, "selected_rows": 0}, + {"name": "Economics", "population_rows": 30, "selected_rows": 0}, + {"name": "Electronics", "population_rows": 30, "selected_rows": 1}, + {"name": "Energy_and_Power", "population_rows": 30, "selected_rows": 0}, + {"name": "Finance", "population_rows": 30, "selected_rows": 0}, + {"name": "Geography", "population_rows": 30, "selected_rows": 1}, + {"name": "History", "population_rows": 30, "selected_rows": 0}, + {"name": "Literature", "population_rows": 30, "selected_rows": 0}, + {"name": "Manage", "population_rows": 30, "selected_rows": 0}, + {"name": "Marketing", "population_rows": 30, "selected_rows": 1}, + {"name": "Materials", "population_rows": 30, "selected_rows": 0}, + {"name": "Math", "population_rows": 30, "selected_rows": 0}, + {"name": "Mechanical_Engineering", "population_rows": 30, "selected_rows": 0}, + {"name": "Music", "population_rows": 30, "selected_rows": 1}, + {"name": "Pharmacy", "population_rows": 30, "selected_rows": 0}, + {"name": "Physics", "population_rows": 30, "selected_rows": 0}, + {"name": "Psychology", "population_rows": 30, "selected_rows": 0}, + {"name": "Public_Health", "population_rows": 30, "selected_rows": 1}, + {"name": "Sociology", "population_rows": 30, "selected_rows": 0} + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 15, + "p25": 15, + "p50": 15, + "p75": 15, + "p100": 15 + }, + "selected_row_identities_sha256": "9b2488fac1079e6c920455c9dda9eee5755229868bf710bd1b75902434fdb702" + }, + "rows": [ + {"source_row_index": 45, "source_sample_id": "validation_Agriculture_16"}, + {"source_row_index": 165, "source_sample_id": "validation_Basic_Medical_Science_16"}, + {"source_row_index": 285, "source_sample_id": "validation_Computer_Science_16"}, + {"source_row_index": 405, "source_sample_id": "validation_Electronics_16"}, + {"source_row_index": 495, "source_sample_id": "validation_Geography_16"}, + {"source_row_index": 615, "source_sample_id": "validation_Marketing_16"}, + {"source_row_index": 735, "source_sample_id": "validation_Music_16"}, + {"source_row_index": 855, "source_sample_id": "validation_Public_Health_16"} + ] + }, + "mvbench": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "leaf_task", + "index_space": "within-stratum", + "population_rows": 200, + "selected_rows": 8, + "strata": [ + {"name": "action_sequence", "population_rows": 200, "selected_rows": 8} + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 12, + "p25": 37, + "p50": 87, + "p75": 137, + "p100": 187 + }, + "selected_row_identities_sha256": "45b0dfeef2216c9741aecd10cbd68b7f1c83f0dbbc71c5b3d01752d9191be0b6" + }, + "rows": [ + {"source_row_index": 12, "source_sample_id": "action_sequence:12", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 37, "source_sample_id": "action_sequence:37", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 62, "source_sample_id": "action_sequence:62", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 87, "source_sample_id": "action_sequence:87", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 112, "source_sample_id": "action_sequence:112", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 137, "source_sample_id": "action_sequence:137", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 162, "source_sample_id": "action_sequence:162", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 187, "source_sample_id": "action_sequence:187", "leaf_task": "mvbench_action_sequence"} + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/smoke-vllm-v1.json b/examples/puzzletron/evaluation/vlm/profiles/smoke-vllm-v1.json new file mode 100644 index 00000000000..3af583918be --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/smoke-vllm-v1.json @@ -0,0 +1,14 @@ +{ + "extends": "smoke-native-v1", + "schema": "modelopt.vlm-evaluation-profile/v1", + "name": "smoke-vllm-v1", + "backend": { + "attention_config": { + "flash_attn_version": 2 + }, + "enable_thinking": false, + "enforce_eager": true, + "name": "vllm", + "reasoning_parser": "qwen3" + } +} diff --git a/examples/puzzletron/evaluation/vlm/suites.py b/examples/puzzletron/evaluation/vlm/suites.py index e8f2fbc7d0a..8f3c0928a22 100644 --- a/examples/puzzletron/evaluation/vlm/suites.py +++ b/examples/puzzletron/evaluation/vlm/suites.py @@ -52,8 +52,10 @@ "execution_policy", "generation_kwargs", "load_quick_manifest", + "manifest_row_identities", "manifest_selected_rows", "manifest_sha256", + "manifest_task_denominators", "offline_dataset_snapshot", "source_tasks", "task_name", @@ -131,6 +133,7 @@ class FramePolicy(TypedDict): class GenerationPolicy(TypedDict): """Generation fields shared by provenance and execution.""" + enable_thinking: bool temperature: int do_sample: bool @@ -195,7 +198,7 @@ def execution_policy(suite: str, *, timeout_seconds: float | None) -> ExecutionP limit = None return { "frame": {"reader": "decord", "fps": 2, "max_frames": 32}, - "generation": {"temperature": 0, "do_sample": False}, + "generation": {"enable_thinking": False, "temperature": 0, "do_sample": False}, "limit": limit, "repetitions": 2 if suite in {"short", TASK_PREFIX100_REPEAT2_SUITE} else 1, "timeout_seconds": ( @@ -319,6 +322,40 @@ def manifest_selected_rows(manifest: dict[str, object]) -> int: return sum(len(cast("list[object]", entry["rows"])) for entry in manifest_tasks.values()) +def manifest_row_identities(manifest: dict[str, object]) -> dict[str, list[dict[str, object]]]: + """Project the exact selected row identities in task and row order.""" + + manifest_tasks = cast("dict[str, dict[str, object]]", manifest["tasks"]) + return { + task: [ + { + key: row[key] + for key in ("source_row_index", "source_sample_id", "leaf_task") + if key in row + } + for row in cast("list[dict[str, object]]", entry["rows"]) + ] + for task, entry in manifest_tasks.items() + } + + +def manifest_task_denominators( + manifest: dict[str, object], +) -> dict[str, dict[str, int | None]]: + """Project selected and population row denominators for every exact task.""" + + manifest_tasks = cast("dict[str, dict[str, object]]", manifest["tasks"]) + denominators = {} + for task, entry in manifest_tasks.items(): + selection = cast("dict[str, object]", entry.get("selection", {})) + population_rows = selection.get("population_rows", entry.get("population_rows")) + denominators[task] = { + "selected_rows": len(cast("list[object]", entry["rows"])), + "population_rows": cast("int | None", population_rows), + } + return denominators + + def offline_dataset_snapshot(hf_home: Path, task: str, revision: str) -> Path: """Resolve an exact local Hub dataset snapshot without dereferencing mount aliases.""" configured_hub_cache = os.environ.get("HF_HUB_CACHE") diff --git a/examples/puzzletron/evaluation/vlm/tasks.py b/examples/puzzletron/evaluation/vlm/tasks.py index 8b12a1b9e81..2bf1d63bcaa 100644 --- a/examples/puzzletron/evaluation/vlm/tasks.py +++ b/examples/puzzletron/evaluation/vlm/tasks.py @@ -62,6 +62,7 @@ def _write_task_config( process_docs: str | None = None, process_docs_module: str = "modelopt_quick_selection", process_results: str | None = None, + process_results_module: str = "modelopt_mmvu_guard", ) -> None: values = { "include": str(include), @@ -80,29 +81,58 @@ def _write_task_config( if doc_to_visual is not None: lines.append(f"doc_to_visual: !function modelopt_video_paths.{doc_to_visual}") if process_results is not None: - lines.append(f"process_results: !function modelopt_mmvu_guard.{process_results}") + lines.append(f"process_results: !function {process_results_module}.{process_results}") checkpoint.write_generated(path, "\n".join(lines) + "\n") +def _manifest_group_leaves( + manifest: dict[str, object], task: str, leaves: tuple[str, ...] +) -> tuple[str, ...]: + manifest_tasks = cast("dict[str, dict[str, object]]", manifest["tasks"]) + rows = cast("list[dict[str, object]]", manifest_tasks[task]["rows"]) + selected_leaf_tasks = {cast("str", row["leaf_task"]) for row in rows} + selected = tuple(leaf for leaf in leaves if f"{task}_{leaf}" in selected_leaf_tasks) + if not selected: + raise ValueError(f"exact-row manifest selects no leaves for grouped task {task}") + return selected + + def _write_quick_selection_module(tasks_root: Path, manifest: dict[str, object]) -> None: entries: dict[str, dict[str, object]] = {} functions: list[str] = [] manifest_tasks = cast("dict[str, dict[str, object]]", manifest["tasks"]) for task in manifest_tasks: - rows = cast("list[dict[str, object]]", manifest_tasks[task]["rows"]) + task_entry = manifest_tasks[task] + rows = cast("list[dict[str, object]]", task_entry["rows"]) + sampling = task_entry.get("selection") + audited = cast("dict[str, object]", sampling) if isinstance(sampling, dict) else None if task in {"mvbench", "video_mmmu"}: - leaves = ( + all_leaves = ( suites.MVBENCH_LEAF_TASKS if task == "mvbench" else suites.VIDEO_MMMU_LEAF_TASKS ) + leaves = _manifest_group_leaves(manifest, task, all_leaves) for leaf in leaves: leaf_task = f"{task}_{leaf}" selected = [row for row in rows if row["leaf_task"] == leaf_task] key = suites.task_name(task, leaf=leaf) + stratum = ( + next( + item + for item in cast("list[dict[str, object]]", audited["strata"]) + if item["name"] == leaf + ) + if audited is not None + else None + ) entries[key] = { "kind": task, "config": leaf, "indices": [row["source_row_index"] for row in selected], "source_ids": [row["source_sample_id"] for row in selected], + "population_rows": stratum["population_rows"] if stratum else None, + "strata": {leaf: stratum["population_rows"]} if stratum else None, + "upstream_ids": [row.get("upstream_sample_id") for row in selected], + "sampling_positions": None, } functions.append( f"def select_{key}(documents):\n return _select(documents, {key!r})\n" @@ -113,20 +143,101 @@ def _write_quick_selection_module(tasks_root: Path, manifest: dict[str, object]) "kind": task, "indices": [row["source_row_index"] for row in rows], "source_ids": [row["source_sample_id"] for row in rows], + "population_rows": audited.get("population_rows") if audited else None, + "strata": ( + { + item["name"]: item["population_rows"] + for item in cast("list[dict[str, object]]", audited["strata"]) + } + if audited + else None + ), + "upstream_ids": [row.get("upstream_sample_id") for row in rows], + "sampling_positions": ( + [(row["sampling_stratum"], row["source_stratum_index"]) for row in rows] + if all( + "sampling_stratum" in row and "source_stratum_index" in row for row in rows + ) + else None + ), } functions.append( f"def select_{key}(documents):\n return _select(documents, {key!r})\n" ) source = f'''"""Generated exact-row selectors for a VLM benchmark profile.""" +from collections import Counter + _SELECTIONS = {entries!r} +def _column(documents, name): + values = documents[name] + if len(values) != len(documents): + raise ValueError(f"exact-row manifest source column {{name}} has the wrong length") + return values + + +def _source_strata(documents, kind): + if kind == "mmmu_val": + return [value.removeprefix("validation_").rsplit("_", 1)[0] for value in _column(documents, "id")] + if kind == "videomme": + return [f"{{duration}}|{{domain}}" for duration, domain in zip( + _column(documents, "duration"), _column(documents, "domain"), strict=True + )] + if kind == "mlvu_dev": + return list(_column(documents, "task_type")) + if kind == "perceptiontest_val_mc": + return [f"{{area}}|{{reasoning}}" for area, reasoning in zip( + _column(documents, "area"), _column(documents, "reasoning"), strict=True + )] + return None + + +def _verify_population(documents, name, selection): + expected_rows = selection["population_rows"] + if expected_rows is None: + return None + if len(documents) != expected_rows: + raise ValueError( + f"exact-row manifest source population drifted for {{name}}: " + f"{{len(documents)}} != {{expected_rows}}" + ) + observed_strata = _source_strata(documents, selection["kind"]) + if observed_strata is not None and Counter(observed_strata) != selection["strata"]: + raise ValueError(f"exact-row manifest source strata drifted for {{name}}") + return observed_strata + + +def _stratum_ranks(strata): + counts = Counter() + ranks = [] + for stratum in strata: + ranks.append(counts[stratum]) + counts[stratum] += 1 + return ranks + + +def _upstream_id(document, kind): + if kind in {{"videomme", "mlvu_dev"}}: + return str(document["question_id"]) + if kind == "perceptiontest_val_mc": + return f"{{document['video_name']}}:{{document['question_id']}}" + return None + + def _select(documents, name): selection = _SELECTIONS[name] + observed_strata = _verify_population(documents, name, selection) indices = selection["indices"] + expected_positions = selection["sampling_positions"] + if observed_strata is not None and expected_positions is not None: + ranks = _stratum_ranks(observed_strata) + observed_positions = [(observed_strata[index], ranks[index]) for index in indices] + if observed_positions != expected_positions: + raise ValueError(f"exact-row manifest source sampling positions drifted for {{name}}") observed = [] - for index in indices: + for position, index in enumerate(indices): if index >= len(documents): raise ValueError(f"exact-row manifest row {{index}} is outside {{name}}") document = documents[index] @@ -138,6 +249,9 @@ def _select(documents, name): observed.append(f"{{selection['config']}}:{{index}}") else: observed.append(f"{{selection['kind']}}:{{index}}") + expected_upstream_id = selection["upstream_ids"][position] + if expected_upstream_id is not None and _upstream_id(document, selection["kind"]) != expected_upstream_id: + raise ValueError(f"exact-row manifest upstream identity drifted for {{name}}") if observed != selection["source_ids"]: raise ValueError(f"exact-row manifest source identities drifted for {{name}}") return documents.select(indices) @@ -218,6 +332,64 @@ def process_results(document, results): ) +def _write_mmmu_audit(tasks_root: Path) -> None: + checkpoint.write_generated( + tasks_root / "modelopt_mmmu_audit.py", + '''"""Expose whether the pinned MMMU parser used its random fallback.""" + +import ast +import random + +from lmms_eval.tasks.mmmu import utils as _upstream + + +class _TrackedChoices(list): + def __init__(self, choices): + super().__init__(choices) + self.used_random_fallback = False + + def __getitem__(self, index): + self.used_random_fallback = True + return super().__getitem__(index) + + +def _multiple_choice_status(document, response): + index_to_answer, choices = _upstream.get_multi_choice_info( + ast.literal_eval(document["options"]) + ) + tracked_choices = _TrackedChoices(choices) + random_state = random.getstate() + try: + _upstream.parse_multi_choice_response(response, tracked_choices, index_to_answer) + finally: + random.setstate(random_state) + return "fallback_random" if tracked_choices.used_random_fallback else "parsed" + + +def process_results(document, results): + processed = _upstream.mmmu_process_results(document, results) + accuracy = processed.get("mmmu_acc") + if not isinstance(accuracy, dict): + raise RuntimeError("MMMU result is missing its per-sample accuracy leaf") + parsed_predictions = accuracy.get("parsed_pred") + if not isinstance(parsed_predictions, list) or len(parsed_predictions) != len(results): + raise RuntimeError("MMMU result has invalid parsed predictions") + if document.get("question_type") == "multiple-choice": + statuses = [ + _multiple_choice_status(document, response) + for response in results + ] + else: + statuses = [ + "parsed_open" if str(prediction).strip() else "invalid_open" + for prediction in parsed_predictions + ] + accuracy["parser_status"] = statuses + return processed +''', + ) + + def _write_video_path_adapter(tasks_root: Path) -> None: alias_root = tasks_root / "video_path_aliases" checkpoint.write_generated( @@ -275,6 +447,7 @@ def verify_offline( hf_home: Path, timeout_seconds: float, model_name: str = "vllm", + expected_populations: dict[str, int] | None = None, ) -> dict[str, object]: """Instantiate every generated task with network access disabled.""" script = """ @@ -285,7 +458,8 @@ def verify_offline( from lmms_eval.tasks import TaskManager -model_name, root, *tasks = sys.argv[1:] +model_name, root, expected_json, *tasks = sys.argv[1:] +expected_populations = json.loads(expected_json) credential_names = ("HF_TOKEN", "HUGGINGFACEHUB_API_TOKEN", "HUGGING_FACE_HUB_TOKEN") if inherited := [name for name in credential_names if name in os.environ]: raise RuntimeError(f"offline task preflight inherited Hub credentials: {inherited}") @@ -311,10 +485,16 @@ def task_objects(value): "modelopt_vlm_benchmark_mmmu_val", } media_documents = 0 +document_counts = {} +seen_task_objects = set() for task in task_objects(loaded): - task_name = task.config.task - if task_name in image_tasks: + task_object_id = id(task) + if task_object_id in seen_task_objects: continue + seen_task_objects.add(task_object_id) + task_name = task.config.task + if task_name in document_counts: + raise RuntimeError(f"distinct task objects share configured task name: {task_name}") if task.has_test_docs(): documents = task.test_docs() elif task.has_validation_docs(): @@ -322,7 +502,11 @@ def task_objects(value): elif task.has_training_docs(): documents = task.training_docs() else: - raise RuntimeError(f"configured media task has no evaluation split: {task_name}") + raise RuntimeError(f"configured task has no evaluation split: {task_name}") + document_counts[task_name] = len(documents) + if task_name in image_tasks: + _ = len(documents) + continue for document in documents: visuals = task.doc_to_visual(document) if not visuals: @@ -337,11 +521,21 @@ def task_objects(value): ) media_documents += 1 +observed_populations = { + task_name: document_counts.get(task_name, 0) for task_name in expected_populations +} +if observed_populations != expected_populations: + raise RuntimeError( + f"configured task population mismatch: {observed_populations} != {expected_populations}" + ) + print( json.dumps( { "configured_tasks": tasks, + "document_counts": document_counts, "media_documents": media_documents, + "observed_populations": observed_populations, "status": "passed", }, sort_keys=True, @@ -361,7 +555,15 @@ def task_objects(value): # A child interpreter is required to import lmms-eval in a clean offline environment. The # fixed interpreter/script and argument-vector invocation avoid shell parsing or interpolation. completed = subprocess.run( - [sys.executable, "-c", script, model_name, str(tasks_root), *configured_tasks], + [ + sys.executable, + "-c", + script, + model_name, + str(tasks_root), + json.dumps(expected_populations or {}, sort_keys=True), + *configured_tasks, + ], check=False, capture_output=True, env=env, @@ -399,6 +601,8 @@ def prepare( _write_mmvu_smoke_selection_module(tasks_root) if suite == "full": _write_mmvu_guard(tasks_root) + if "mmmu_val" in source_tasks: + _write_mmmu_audit(tasks_root) if set(source_tasks) & {"videomme", "perceptiontest_val_mc"}: _write_video_path_adapter(tasks_root) @@ -449,6 +653,8 @@ def _write_task_group( dataset_path: Path, quick_manifest: dict[str, object] | None, ) -> str: + if quick_manifest is not None: + leaves = _manifest_group_leaves(quick_manifest, task, leaves) generated_leaves = [] for leaf in leaves: leaf_task = suites.task_name(task, leaf=leaf) @@ -498,6 +704,13 @@ def _write_single_task( doc_to_visual=doc_to_visual, process_docs=process_docs, process_docs_module=process_docs_module, - process_results="process_results" if suite == "full" and task == "mmvu_val" else None, + process_results=( + "process_results" + if task == "mmmu_val" or (suite == "full" and task == "mmvu_val") + else None + ), + process_results_module=( + "modelopt_mmmu_audit" if task == "mmmu_val" else "modelopt_mmvu_guard" + ), ) return configured_task diff --git a/examples/puzzletron/patches/lmms_eval_wandb_0_29.patch b/examples/puzzletron/patches/lmms_eval_compat_3e675904.patch similarity index 88% rename from examples/puzzletron/patches/lmms_eval_wandb_0_29.patch rename to examples/puzzletron/patches/lmms_eval_compat_3e675904.patch index 6ddc4f76ca7..98022f7e2da 100644 --- a/examples/puzzletron/patches/lmms_eval_wandb_0_29.patch +++ b/examples/puzzletron/patches/lmms_eval_compat_3e675904.patch @@ -23,6 +23,12 @@ index 89978777..57a31642 100644 @@ -55,2 +40,0 @@ class WandbLogger: - self.printer = get_wandb_printer() - +diff --git a/lmms_eval/models/simple/vllm.py b/lmms_eval/models/simple/vllm.py +index 22946997..e0af27ef 100644 +--- a/lmms_eval/models/simple/vllm.py ++++ b/lmms_eval/models/simple/vllm.py +@@ -521 +520,0 @@ class VLLM(lmms): +- sampling_params = SamplingParams(**params) diff --git a/lmms_eval/tasks/emma/utils.py b/lmms_eval/tasks/emma/utils.py index 001d70c8..d90a88fe 100644 --- a/lmms_eval/tasks/emma/utils.py diff --git a/modelopt/torch/puzzletron/evaluation/lmms.py b/modelopt/torch/puzzletron/evaluation/lmms.py index 72efcc2fd6b..30d94bf8a53 100644 --- a/modelopt/torch/puzzletron/evaluation/lmms.py +++ b/modelopt/torch/puzzletron/evaluation/lmms.py @@ -43,6 +43,7 @@ { "dtype", "gpu_memory_utilization", + "attention_config", "chat_template", "max_model_len", "trust_remote_code", @@ -609,6 +610,7 @@ def _result_payload(output_path: Path) -> tuple[dict[str, Any], Path]: def _write_streams(output_path: Path, result: _ProcessResult) -> dict[str, str]: + output_path.mkdir(parents=True, exist_ok=True) stream_paths = {} for stream_name, text in (("stdout", result.stdout), ("stderr", result.stderr)): stream_path = output_path / f"{stream_name}.txt" @@ -769,17 +771,17 @@ def run_lmms_eval_checkpoint( checkpoint=str(checkpoint_path), output_path=output, ) - command_path = _atomic_json( - output / "command.json", - { - "argv": argv, - "env_overrides": sorted(str(key) for key in dict(settings.get("env") or {})), - "timeout": timeout, - }, - ) + command_payload = { + "argv": argv, + "env_overrides": sorted(str(key) for key in dict(settings.get("env") or {})), + "timeout": timeout, + } + command_path = _atomic_json(output / "command.json", command_payload) try: result = _run_process(argv, cwd=str(output), env=env, timeout=timeout) except LmmsEvalTimeoutError as error: + if not command_path.is_file(): + command_path = _atomic_json(output / "command.json", command_payload) captured = _ProcessResult(argv, -1, error.output, error.stderr) stream_paths = _write_streams(output, captured) _annotate_error(error, command_path=command_path, stream_paths=stream_paths) diff --git a/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py b/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py index da4ae62ff4e..728cadf8486 100644 --- a/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py +++ b/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py @@ -45,12 +45,9 @@ def test_load_runner_restores_import_state(monkeypatch): assert "modelopt.torch.puzzletron.evaluation.lmms" not in sys.modules -def test_verify_lmms_eval_revision_rejects_unpatched_vcs_install(monkeypatch): - provenance = { - "url": checkpoint.LMMS_EVAL_SOURCE["repository"], - "vcs_info": {"commit_id": checkpoint.LMMS_EVAL_REVISION}, - } - monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda: None) +def test_verify_lmms_eval_revision_rejects_unpatched_current_vcs_install(monkeypatch): + provenance = {"vcs_info": {"commit_id": checkpoint.LMMS_EVAL_REVISION}} + monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) monkeypatch.setattr( checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) ) @@ -59,9 +56,23 @@ def test_verify_lmms_eval_revision_rejects_unpatched_vcs_install(monkeypatch): checkpoint.verify_lmms_eval_revision() +@pytest.mark.parametrize( + "expected", + [checkpoint.LMMS_EVAL_LEGACY_REVISION, checkpoint.LMMS_EVAL_QWEN35_NATIVE_REVISION], +) +def test_verify_lmms_eval_revision_accepts_clean_historical_vcs_install(monkeypatch, expected): + provenance = {"url": checkpoint._LMMS_EVAL_REPOSITORY, "vcs_info": {"commit_id": expected}} + monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) + monkeypatch.setattr( + checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) + ) + + assert checkpoint.verify_lmms_eval_revision(expected) == expected + + def test_verify_lmms_eval_revision_accepts_clean_pinned_editable_checkout(monkeypatch, tmp_path): provenance = {"dir_info": {"editable": True}, "url": tmp_path.as_uri()} - monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda: None) + monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) monkeypatch.setattr( checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) ) @@ -102,7 +113,7 @@ def distribution(_name): def test_verify_lmms_eval_revision_rejects_dirty_editable_checkout(monkeypatch, tmp_path): provenance = {"dir_info": {"editable": True}, "url": tmp_path.as_uri()} - monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda: None) + monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) monkeypatch.setattr( checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) ) @@ -126,7 +137,7 @@ def test_verify_lmms_eval_revision_rejects_dirty_editable_checkout(monkeypatch, ], ) def test_verify_lmms_eval_revision_rejects_unverifiable_editable_install(monkeypatch, provenance): - monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda: None) + monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) monkeypatch.setattr( checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) ) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py index 1f1ef0f57b1..c4591a96467 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py @@ -19,6 +19,7 @@ import io import json import tarfile +import threading import zipfile from pathlib import Path from types import SimpleNamespace @@ -28,6 +29,16 @@ from examples.puzzletron.evaluation.vlm.preparation import benchmark_data as preparation _EXPECTED_DATASETS = { + "realworldqa": ( + "lmms-lab/RealWorldQA", + "907c4e5228fd1703c710ed937601cb5f89ab8d5c", + None, + ), + "mmmu_val": ( + "lmms-lab/MMMU", + "364f2e2eb107b36e07ff4c5a15f5947a759cef47", + None, + ), "video_mmmu": ( "lmms-lab/VideoMMMU", "d1c35ac933123d79e877b7f1b9506afb0309cf1b", @@ -64,6 +75,14 @@ def _write_zip(path: Path, members: dict[str, bytes]) -> None: archive.writestr(name, payload) +def _emulate_atomic_exchange(first: Path, second: Path) -> bool: + displaced = second.with_name(f".{second.name}.test-exchange") + second.rename(displaced) + first.rename(second) + displaced.rename(first) + return True + + @pytest.mark.parametrize( ("task", "repository", "revision", "directory"), [(task, *values) for task, values in _EXPECTED_DATASETS.items()], @@ -76,6 +95,60 @@ def test_every_preparation_contract_is_explicitly_pinned(task, repository, revis revision, directory, ) + assert preparation.benchmark_catalog_contract((task,))[task] == { + "repository": repository, + "revision": revision, + "requires_media": directory is not None, + "preparation_dir": directory, + } + + +def test_prepare_benchmark_datasets_dispatches_media_only_for_media_tasks(tmp_path, monkeypatch): + hf_home = tmp_path / "hf-home" + prepared = [] + + def download(root, task, *, max_workers): + del max_workers + snapshot = preparation._hub_snapshot(root, task) + snapshot.mkdir(parents=True) + (snapshot / "dataset-info.json").write_text("{}") + return snapshot + + def prepare(root, task, snapshot): + prepared.append(task) + media_root = root / preparation.DATASETS[task].preparation_dir + media_root.mkdir(parents=True) + (media_root / "sample.mp4").write_bytes(b"video") + payload = { + **preparation._marker_payload(task, status="complete"), + "snapshot": str(snapshot), + "media_root": str(media_root), + "files": 1, + "bytes": 5, + } + preparation._write_marker(media_root, payload) + return payload + + monkeypatch.setattr(preparation, "_download", download) + monkeypatch.setattr(preparation, "_prepare", prepare) + + reports = preparation.prepare_benchmark_datasets( + hf_home, ["realworldqa", "mmmu_val", "mvbench"], max_workers=3 + ) + + assert [report["task"] for report in reports] == ["realworldqa", "mmmu_val", "mvbench"] + assert [report["requires_media"] for report in reports] == [False, False, True] + assert prepared == ["mvbench"] + + +def test_prepare_benchmark_datasets_rejects_symlinked_hf_home(tmp_path): + target = tmp_path / "target" + target.mkdir() + alias = tmp_path / "hf-home" + alias.symlink_to(target, target_is_directory=True) + + with pytest.raises(ValueError, match="must not be a symlink"): + preparation.prepare_benchmark_datasets(alias, ("realworldqa",)) def test_zip_preparation_is_revision_bound_idempotent_and_byte_verified(tmp_path): @@ -104,6 +177,171 @@ def test_zip_preparation_is_revision_bound_idempotent_and_byte_verified(tmp_path preparation._extract_zip(archive, target) +@pytest.mark.parametrize("damage", ["missing", "corrupt", "unexpected"]) +def test_complete_media_marker_repairs_owned_root_from_pinned_snapshot( + tmp_path, monkeypatch, damage +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + preparation._prepare(hf_home, "mmvu_val", snapshot) + monkeypatch.setattr(preparation, "_atomic_exchange_directories", _emulate_atomic_exchange) + target = hf_home / "mmvu" + media = target / "videos/sample.mp4" + + if damage == "missing": + media.unlink() + elif damage == "corrupt": + media.write_bytes(b"wrong") + elif damage == "unexpected": + (target / "unexpected.bin").write_bytes(b"stale") + report = preparation._prepare(hf_home, "mmvu_val", snapshot) + + assert report["status"] == "complete" + assert media.read_bytes() == b"video" + assert not (target / "unexpected.bin").exists() + assert preparation._media_marker_is_current( + target, + "mmvu_val", + json.loads((target / preparation._MARKER_NAME).read_text()), + ) + + +def test_repair_without_atomic_exchange_preserves_live_root(monkeypatch, tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + preparation._prepare(hf_home, "mmvu_val", snapshot) + target = hf_home / "mmvu" + media = target / "videos/sample.mp4" + media.write_bytes(b"wrong") + monkeypatch.setattr(preparation, "_atomic_exchange_directories", lambda *_args: False) + + with pytest.raises(RuntimeError, match="atomic media-directory exchange is unavailable"): + preparation._prepare(hf_home, "mmvu_val", snapshot) + + assert target.is_dir() + assert media.read_bytes() == b"wrong" + assert not tuple(target.parent.glob(f".{target.name}.modelopt-staging.*")) + assert not tuple(target.parent.glob(f".{target.name}.modelopt-replaced.*")) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + (None, None, "readable ownership marker"), + ("corrupt", "{", "readable ownership marker"), + ("schema", "other/v1", "mismatched ownership: schema"), + ("task", "mvbench", "mismatched ownership: task"), + ("repository", "other/repository", "mismatched ownership: repository"), + ("revision", "other-revision", "mismatched ownership: revision"), + ("requires_media", False, "mismatched ownership: requires_media"), + ("preparation_dir", "other-root", "mismatched ownership: preparation_dir"), + ], +) +def test_missing_or_mismatched_media_marker_preserves_unproven_root( + tmp_path, field, value, message +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + preparation._prepare(hf_home, "mmvu_val", snapshot) + target = hf_home / "mmvu" + media = target / "videos/sample.mp4" + marker = target / preparation._MARKER_NAME + if field is None: + marker.unlink() + elif field == "corrupt": + marker.write_text(value) + else: + payload = json.loads(marker.read_text()) + payload[field] = value + marker.write_text(json.dumps(payload)) + + with pytest.raises((FileExistsError, ValueError), match=message): + preparation._prepare(hf_home, "mmvu_val", snapshot) + + assert target.is_dir() + assert media.read_bytes() == b"video" + + +def test_media_repair_rejects_symlinks_in_owned_root(tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + preparation._prepare(hf_home, "mmvu_val", snapshot) + target = hf_home / "mmvu" + (target / "videos/sample.mp4").unlink() + (target / "videos/sample.mp4").symlink_to(tmp_path / "outside") + + with pytest.raises(ValueError, match="repair refuses a symlink"): + preparation._prepare(hf_home, "mmvu_val", snapshot) + + +def test_snapshot_inventory_rejects_partial_and_same_size_corruption(tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "realworldqa") + snapshot.mkdir(parents=True) + first = snapshot / "first.json" + second = snapshot / "second.json" + first.write_bytes(b"one") + second.write_bytes(b"two") + report = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + + assert preparation._snapshot_inventory_is_current(report) + second.unlink() + assert not preparation._snapshot_inventory_is_current(report) + second.write_bytes(b"two") + assert preparation._snapshot_inventory_is_current(report) + first.write_bytes(b"bad") + assert not preparation._snapshot_inventory_is_current(report) + + +def test_snapshot_inventory_seals_and_validates_hub_blob_symlink(tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "realworldqa") + snapshot.mkdir(parents=True) + payload = b"pinned blob" + blob_sha256 = hashlib.sha256(payload).hexdigest() + blob = snapshot.parent.parent / "blobs" / blob_sha256 + blob.parent.mkdir() + blob.write_bytes(payload) + (snapshot / "dataset.parquet").symlink_to(Path("../../blobs") / blob_sha256) + + report = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + + assert preparation._snapshot_inventory_is_current(report) + blob.write_bytes(b"broken blob") + assert not preparation._snapshot_inventory_is_current(report) + + +def test_snapshot_inventory_rejects_blob_whose_content_differs_from_sha_name(tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "realworldqa") + snapshot.mkdir(parents=True) + blob = snapshot.parent.parent / "blobs" / ("0" * 64) + blob.parent.mkdir() + blob.write_bytes(b"not the named content") + (snapshot / "dataset.parquet").symlink_to(Path("../../blobs") / blob.name) + + with pytest.raises(ValueError, match="differs from its SHA-256 identity"): + preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + + +def test_prepare_benchmark_datasets_rejects_catalog_pin_drift(tmp_path): + catalog = preparation.benchmark_catalog_contract(("realworldqa",)) + catalog["realworldqa"]["revision"] = "stale" + + with pytest.raises(ValueError, match="differs from the authoritative catalog"): + preparation.prepare_benchmark_datasets( + tmp_path / "hf-home", ("realworldqa",), expected_catalog=catalog + ) + + @pytest.mark.parametrize("member", ["../escape.mp4", "/absolute.mp4"]) def test_archive_extraction_rejects_paths_outside_owned_root(tmp_path, member): archive = tmp_path / "unsafe.zip" @@ -152,24 +390,78 @@ def test_archive_extraction_rejects_links_and_streams_multipart_tar(tmp_path): def test_interrupted_initialization_leaves_target_retryable(monkeypatch, tmp_path): hf_home = tmp_path / "hf-home" - hf_home.mkdir() + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) target = hf_home / preparation.DATASETS["mmvu_val"].preparation_dir - write_marker = preparation._write_marker + extract = preparation._extract - def interrupt(staging, payload): - write_marker(staging, payload) + def interrupt(*_args): raise RuntimeError("interrupted") - monkeypatch.setattr(preparation, "_write_marker", interrupt) + monkeypatch.setattr(preparation, "_extract", interrupt) with pytest.raises(RuntimeError, match="interrupted"): - preparation._prepare_target(hf_home, "mmvu_val") + preparation._prepare(hf_home, "mmvu_val", snapshot) assert not target.exists() + assert not tuple(target.parent.glob(f".{target.name}.modelopt-staging.*")) + + monkeypatch.setattr(preparation, "_extract", extract) + report = preparation._prepare(hf_home, "mmvu_val", snapshot) + assert report["status"] == "complete" + assert (target / "videos/sample.mp4").read_bytes() == b"video" + - monkeypatch.setattr(preparation, "_write_marker", write_marker) - prepared, complete = preparation._prepare_target(hf_home, "mmvu_val") - assert prepared == target - assert complete is None - assert json.loads((target / preparation._MARKER_NAME).read_text())["status"] == "in_progress" +def test_concurrent_media_preparation_is_task_locked_and_publishes_only_complete_root( + monkeypatch, tmp_path +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + target = hf_home / "mmvu" + entered = threading.Event() + release = threading.Event() + second_done = threading.Event() + extraction_count = 0 + results = [] + errors = [] + + def extract(_task, _snapshot, staging): + nonlocal extraction_count + extraction_count += 1 + (staging / "videos").mkdir() + (staging / "videos/sample.mp4").write_bytes(b"video") + entered.set() + assert release.wait(timeout=5) + return [] + + def run(*, second=False): + try: + results.append(preparation._prepare(hf_home, "mmvu_val", snapshot)) + except BaseException as error: # pragma: no cover - asserted below + errors.append(error) + finally: + if second: + second_done.set() + + monkeypatch.setattr(preparation, "_extract", extract) + first = threading.Thread(target=run) + first.start() + assert entered.wait(timeout=5) + second = threading.Thread(target=run, kwargs={"second": True}) + second.start() + + assert not second_done.wait(timeout=0.1) + assert not target.exists() + release.set() + first.join(timeout=5) + second.join(timeout=5) + + assert not errors + assert not first.is_alive() and not second.is_alive() + assert extraction_count == 1 + assert len(results) == 2 + assert all(result["status"] == "complete" for result in results) + assert (target / "videos/sample.mp4").read_bytes() == b"video" def test_range_download_resumes_without_forwarding_credentials_and_verifies_hash( diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py index ed6171c6114..7e446581749 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py @@ -19,9 +19,9 @@ import importlib.util import json import os +import random import subprocess import sys -from collections import Counter from hashlib import sha256 from pathlib import Path from types import ModuleType @@ -40,6 +40,7 @@ ) from examples.puzzletron.evaluation.vlm import model as vlm_model from examples.puzzletron.evaluation.vlm import run as evaluation +from modelopt.torch.puzzletron.evaluation import lmms _QWEN_CONFIG = { "architectures": ["Qwen3_5ForConditionalGeneration"], @@ -57,6 +58,27 @@ _TASK_CONFIGS = {name: item.task_config for name, item in profile.VLM_BENCHMARK_DATASETS.items()} +def _homogeneous_qwen_block_configs() -> list[dict[str, object]]: + block = { + "subblock_configs": [ + { + "kind": "attention", + "name": "attention", + "no_op": False, + "num_kv_heads": 2, + "num_query_heads": 8, + }, + { + "kind": "ffn", + "name": "ffn", + "no_op": False, + "intermediate_size": 3584, + }, + ] + } + return [json.loads(json.dumps(block)) for _ in range(24)] + + def test_direct_launcher_does_not_shadow_standard_library_profile(): script = Path(evaluation.__file__).absolute() subprocess.run( @@ -77,8 +99,7 @@ def test_direct_launcher_does_not_shadow_standard_library_profile(): ) -def _write_checkpoint(root: Path) -> Path: - model = root / "model" +def _write_checkpoint_at(model: Path) -> Path: model.mkdir() (model / "config.json").write_text(json.dumps(_QWEN_CONFIG) + "\n") (model / "preprocessor_config.json").write_text("{}\n") @@ -89,6 +110,21 @@ def _write_checkpoint(root: Path) -> Path: return model +def _write_checkpoint(root: Path) -> Path: + return _write_checkpoint_at(root / "model") + + +def _write_core3_teacher_snapshot(root: Path) -> tuple[Path, Path]: + hf_home = root / "hf-home" + snapshot = ( + hf_home + / "hub/models--Qwen--Qwen3.5-0.8B/snapshots" + / "2fc06364715b967f1860aea9cf38778875588b17" + ) + snapshot.parent.mkdir(parents=True) + return _write_checkpoint_at(snapshot), hf_home + + def test_no_think_template_is_local_and_requires_checkpoint_switch(tmp_path): checkpoint_path = _write_checkpoint(tmp_path) tasks_root = tmp_path / "tasks" @@ -132,6 +168,59 @@ def test_chat_template_fingerprint_accepts_file_and_inline_content(tmp_path): assert evaluator._chat_template_sha256({"model_args": {"chat_template": content}}) == expected +def test_mmmu_parser_audit_is_attached_to_normalized_result(tmp_path): + attempt = tmp_path / "attempt" + attempt.mkdir() + task_name = suites.task_name("mmmu_val") + result_path = attempt / "summary.json" + result_path.write_text(json.dumps({"sample_counts": {task_name: 2}})) + raw_result_path = attempt / "20260903_120000_results.json" + raw_result_path.write_text("{}\n") + sample_path = attempt / f"20260903_120000_samples_{task_name}.jsonl" + sample_path.write_text( + "\n".join( + json.dumps({"mmmu_acc": {"parser_status": [status]}}) + for status in ("parsed", "fallback_random") + ) + + "\n" + ) + (attempt / f"20260903_110000_samples_{task_name}.jsonl").write_text("{not-json}\n") + + evaluator._attach_mmmu_parser_audit( + {"raw_result_path": str(raw_result_path), "result_path": str(result_path)} + ) + + audit = json.loads(result_path.read_text())["mmmu_parser_audit"] + assert audit["sample_count"] == 2 + assert audit["status_counts"] == {"fallback_random": 1, "parsed": 1} + assert audit["sample_logs"] == [ + { + "path": sample_path.name, + "sha256": hashlib.sha256(sample_path.read_bytes()).hexdigest(), + "size": sample_path.stat().st_size, + } + ] + + +def test_mmmu_parser_audit_rejects_unlabeled_sample(tmp_path): + task_name = suites.task_name("mmmu_val") + result_path = tmp_path / "summary.json" + result_path.write_text(json.dumps({"sample_counts": {task_name: 1}})) + raw_result_path = tmp_path / "new_results.json" + raw_result_path.write_text("{}\n") + (tmp_path / f"old_samples_{task_name}.jsonl").write_text( + json.dumps({"mmmu_acc": {"parser_status": ["parsed"]}}) + "\n" + ) + (tmp_path / f"new_samples_{task_name}.jsonl").write_text( + json.dumps({"mmmu_acc": {"parsed_pred": ["A"]}}) + "\n" + ) + + with pytest.raises(RuntimeError, match="no valid parser status"): + evaluator._attach_mmmu_parser_audit( + {"raw_result_path": str(raw_result_path), "result_path": str(result_path)} + ) + + def test_checkpoint_contract_accepts_only_matching_realized_anymodel(tmp_path): checkpoint_path = _write_checkpoint(tmp_path) config_path = checkpoint_path / "config.json" @@ -139,11 +228,18 @@ def test_checkpoint_contract_accepts_only_matching_realized_anymodel(tmp_path): config.update( architectures=["AnyModel"], base_architecture="Qwen3_5ForConditionalGeneration", + block_configs=_homogeneous_qwen_block_configs(), ) config_path.write_text(json.dumps(config) + "\n") vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + config.pop("block_configs") + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="cannot prove.*homogeneous"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config["block_configs"] = _homogeneous_qwen_block_configs() config["base_architecture"] = "OtherForConditionalGeneration" config_path.write_text(json.dumps(config) + "\n") with pytest.raises(ValueError, match="AnyModel base_architecture"): @@ -158,6 +254,35 @@ def test_checkpoint_contract_accepts_only_matching_realized_anymodel(tmp_path): vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") +def test_checkpoint_contract_routes_heterogeneous_anymodel_to_vllm(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + config_path = checkpoint_path / "config.json" + config = json.loads(config_path.read_text()) + config.update( + architectures=["AnyModel"], + base_architecture="Qwen3_5ForConditionalGeneration", + block_configs=_homogeneous_qwen_block_configs(), + ) + config["block_configs"][19]["subblock_configs"][0]["num_query_heads"] = 6 + config["text_config"]["per_layer_config"] = { + "19": {"num_attention_heads": 6, "num_key_value_heads": 2} + } + config_path.write_text(json.dumps(config) + "\n") + + with pytest.raises(ValueError, match="native qwen3_5 backend cannot load"): + vlm_model.verify_checkpoint( + checkpoint_path, + profile="VLM benchmark", + model_backend="qwen3_5", + ) + + vlm_model.verify_checkpoint( + checkpoint_path, + profile="VLM benchmark", + model_backend="vllm", + ) + + def test_checkpoint_contract_accepts_other_positive_qwen35_geometry(tmp_path): checkpoint_path = _write_checkpoint(tmp_path) config_path = checkpoint_path / "config.json" @@ -222,7 +347,9 @@ def _write_lmms_tasks(root: Path, tasks: tuple[str, ...]) -> Path: def _use_offline_fakes(monkeypatch, lmms_root: Path) -> None: + monkeypatch.delenv("HF_HUB_CACHE", raising=False) monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: object()) monkeypatch.setattr( checkpoint, "verify_lmms_eval_revision", @@ -243,6 +370,15 @@ def _use_offline_fakes(monkeypatch, lmms_root: Path) -> None: ) +def _write_fake_mmmu_artifacts(result_path: Path) -> Path: + result_path.write_text(json.dumps({"sample_counts": {suites.task_name("mmmu_val"): 1}}) + "\n") + raw_result_path = result_path.parent / "run_results.json" + raw_result_path.write_text("{}\n") + sample_path = result_path.parent / f"run_samples_{suites.task_name('mmmu_val')}.jsonl" + sample_path.write_text(json.dumps({"mmmu_acc": {"parser_status": ["parsed"]}}) + "\n") + return raw_result_path + + def _full_inputs(monkeypatch, tmp_path): model = _write_checkpoint(tmp_path) lmms_root = _write_lmms_tasks(tmp_path, profile.VLM_BENCHMARK_TASKS) @@ -298,7 +434,9 @@ def _quick_manifest(path: Path) -> Path: return path -def test_short_profile_materializes_pinned_tasks_and_vllm_backend(monkeypatch, tmp_path, capsys): +def test_short_profile_materializes_pinned_tasks_and_native_qwen_backend( + monkeypatch, tmp_path, capsys +): model = _write_checkpoint(tmp_path) source_tasks = ("realworldqa", "mmmu_val") lmms_root = _write_lmms_tasks(tmp_path, source_tasks) @@ -306,28 +444,24 @@ def test_short_profile_materializes_pinned_tasks_and_vllm_backend(monkeypatch, t hf_home = tmp_path / "hf-home" hf_home.mkdir() output = tmp_path / "results" - for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES: - monkeypatch.setenv(name, f"inherited-{name.lower()}") calls = [] def fake_runner(checkpoint_path, *, output_root, settings): calls.append( { "checkpoint": checkpoint_path, - "credentials": { - name: os.environ.get(name) for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES - }, "output_root": output_root, "settings": settings, } ) result_path = output_root / "result.json" result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text("{}\n") + raw_result_path = _write_fake_mmmu_artifacts(result_path) return { "attempt": len(calls), "metrics": {"accuracy": len(calls) / 10}, "output_root": str(output_root), + "raw_result_path": str(raw_result_path), "result_path": str(result_path), } @@ -346,17 +480,16 @@ def fake_runner(checkpoint_path, *, output_root, settings): assert evaluation.main(argv) == 0 result = json.loads(capsys.readouterr().out) - assert result["schema"] == "modelopt.vlm-evaluation-result/v1" report = result["preflight"] - assert report["source_tasks"] == list(source_tasks) - assert report["short_repetitions"] == 2 - assert report["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION generated = json.loads( (output / "task_configs/modelopt_vlm_benchmark_realworldqa.yaml").read_text() ) assert generated["dataset_path"].endswith( profile.VLM_BENCHMARK_DATASETS["realworldqa"].revision ) + assert generated["generation_kwargs"]["max_new_tokens"] == 16 + mmmu_text = (output / "task_configs/modelopt_vlm_benchmark_mmmu_val.yaml").read_text() + assert '"max_new_tokens": 128' in mmmu_text expected_tasks = ( "modelopt_vlm_benchmark_realworldqa", "modelopt_vlm_benchmark_mmmu_val", @@ -366,36 +499,27 @@ def fake_runner(checkpoint_path, *, output_root, settings): output / "short-repetition-1", output / "short-repetition-2", ] - assert all(not any(call["credentials"].values()) for call in calls) assert all(call["settings"]["tasks"] == ",".join(expected_tasks) for call in calls) assert [run["attempt"] for run in result["runs"]] == [1, 2] - for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES: - assert os.environ[name] == f"inherited-{name.lower()}" settings = calls[0]["settings"] - assert settings["model"] == "vllm" - assert settings["log_samples"] is True - assert settings["checkpoint_arg"] == "model" - assert settings["reasoning_parser"] == "qwen3" - chat_template = Path(settings["model_args"]["chat_template"]) - assert chat_template.read_text().startswith("{%- set enable_thinking = false %}\n") - assert "topology" not in settings - assert settings["env"]["HF_HUB_OFFLINE"] == "1" - assert settings["env"]["API_TYPE"] == "openai" - assert settings["env"]["MODEL_VERSION"] == "modelopt-disabled-lmms-eval-judge" - assert settings["env"]["OPENAI_API_KEY"] == "modelopt-disabled-lmms-eval-judge" - assert settings["env"]["OPENAI_API_URL"] == "http://127.0.0.1:9" - assert report["model_backend"] == settings["model"] - assert report["backend_limitations"] == [ - "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", - ] - assert report["sample_limit"] == settings["limit"] - assert report["timeout_seconds"] == settings["timeout_seconds"] - assert report["frame_policy"] == { - "reader": settings["env"]["FORCE_QWENVL_VIDEO_READER"], - "fps": settings["model_args"]["fps"], - "max_frames": settings["model_args"]["max_frame_num"], + assert settings["model"] == "qwen3_5" + assert report["backend_limitations"] == [] + assert report["output_budget_contract"] == { + "mmmu_val": { + "adapter": "qwen3_5", + "effective_max_new_tokens": 128, + "limitation": None, + "requested_max_new_tokens": 128, + "resolution": "task_max_new_tokens_overrides_adapter_default", + }, + "realworldqa": { + "adapter": "qwen3_5", + "effective_max_new_tokens": 16, + "limitation": None, + "requested_max_new_tokens": 16, + "resolution": "task_max_new_tokens_overrides_adapter_default", + }, } - assert report["generation_policy"] == settings["gen_kwargs"] @pytest.mark.parametrize("suite", ["short", suites.TASK_PREFIX100_REPEAT2_SUITE]) @@ -412,10 +536,8 @@ def test_repeated_profile_resumes_completed_repetitions(monkeypatch, tmp_path, s def fake_runner(checkpoint_path, *, output_root, settings): calls.append(output_root) result_path = output_root / "attempt" / "summary.json" - raw_result_path = output_root / "attempt" / "samples.json" result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text("{}\n") - raw_result_path.write_text("{}\n") + raw_result_path = _write_fake_mmmu_artifacts(result_path) return { "metrics": {"accuracy": len(calls) / 10}, "raw_result_path": str(raw_result_path), @@ -472,10 +594,8 @@ def test_short_profile_reruns_stale_completed_repetitions( def fake_runner(checkpoint_path, *, output_root, settings): calls.append(output_root) result_path = output_root / "attempt" / "summary.json" - raw_result_path = output_root / "attempt" / "samples.json" result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text("{}\n") - raw_result_path.write_text("{}\n") + raw_result_path = _write_fake_mmmu_artifacts(result_path) return { "metrics": {"accuracy": 0.5}, "raw_result_path": str(raw_result_path), @@ -500,17 +620,19 @@ def fake_runner(checkpoint_path, *, output_root, settings): if corruption == "checkpoint": (model / "preprocessor_config.json").write_text('{"changed": true}\n') elif corruption == "artifact": - (output / "short-repetition-1" / "attempt" / "samples.json").unlink() + (output / "short-repetition-1" / "attempt" / "run_results.json").unlink() elif corruption == "result": (output / "short-repetition-1" / "attempt" / "summary.json").unlink() else: + original_backend_policy = preflight._backend_policy - def changed_chat_template(_checkpoint_path, output_directory): - target = output_directory / "modelopt_qwen35_no_think.jinja" - target.write_text("changed template\n") - return target + def changed_backend_policy(profile_contract): + return { + **original_backend_policy(profile_contract), + "attention_implementation": "eager", + } - monkeypatch.setattr(vlm_model, "no_think_chat_template", changed_chat_template) + monkeypatch.setattr(preflight, "_backend_policy", changed_backend_policy) evaluation.evaluate(args) assert len(calls) == expected_calls @@ -544,7 +666,11 @@ def test_realworldqa_mmmu_prefix100_policy_is_explicit_and_repeated(): policy = suites.execution_policy(suite, timeout_seconds=14400) assert policy["limit"] == 100 assert policy["repetitions"] == 2 - assert policy["generation"] == {"temperature": 0, "do_sample": False} + assert policy["generation"] == { + "enable_thinking": False, + "temperature": 0, + "do_sample": False, + } assert suites.execution_policy("full", timeout_seconds=None)["limit"] is None assert suites.execution_policy("full-v1", timeout_seconds=None)["limit"] is None @@ -576,46 +702,137 @@ def test_deprecated_suite_alias_records_the_canonical_identity(monkeypatch, tmp_ assert prepared.report["suite"] == suites.TASK_PREFIX100_REPEAT2_SUITE -def test_versioned_profile_contracts_pin_selection_and_fingerprints(tmp_path): +def test_versioned_profile_contracts_pin_backends_and_fingerprints(): profiles = {name: contracts.load_profile(name) for name in contracts.PROFILE_NAMES} assert {name: contract.fingerprint for name, contract in profiles.items()} == { - "short-v1": "3b0803c0deff0873d2c8e0963f1167dfbbf2ef7309b5eb33f089aa2048f6cf91", - "short-native-v1": "aca78320b188c4c6f41a7e5ec0017a0c738b98ec797e9f2f6f9f1aabce34dfa7", - "short-all-native-v1": "88074cec92cd6aa972cd2e48ddc6adbaa84ab26b24e36be2e3842b71317cdd7e", - "full-v1": "5b0849975f65e4bbbdc93d52ff2866d54968f5cc3de6d0928077e3d6ab320e6a", + "short-v1": "984c23ef0e7c05248895ece69c12327b3cdbb45051189ec540f7fc1ada763177", + "short-native-v1": "217b8ba8fd1df0002407e75f6e7d5588e3a871a6df2ad24117b66377894b2f35", + "short-native-v2": "d89134cdf4dfaaafe86b2fe9512bb183fa6528953fb5909e865bb961e95d4ee7", + "short-vllm-v2": "fc8a0a874fa2220610c50c33ee07fee4ce9e28c4032c0becdbc0059e1e1a58e8", + "smoke-native-v1": "734aea43cefa14776e016693a6c5ad2e32efed33fada2b3cdb856ce0cafa3271", + "smoke-vllm-v1": "abc0c2de8576717b50695e4f91ac239459a0f2789f45e01cddbe6d4976ead0e7", + "short-all-native-v1": "06b17ea010ee0cd789e49c581bcb3be4a7624c8471b4b3102bfa2922e0929e68", + "short-all-native-v2": "5db871b4cdd2f713161237f762619172ec7459318f1017de359faf96ff42fd36", + "full-v1": "29b1db6123ea3e16a9c5693e81e0f31607ff8a08e436681c66c32bf5dcc7e67a", + "core3-full-native-v1": ( + "82b053bcf74d7cfe5eab2bbbb94083b79c4ac16c0fff6f5a824a5d8b1079e06c" + ), + "core3-full-vllm-v1": ("6423b6fea1d988a4f9c79572bf17e47ad1d851760a90ee775033deeb68b700ec"), } - short = profiles["short-v1"] - short_all_native = profiles["short-all-native-v1"] + current_short = profiles["short-native-v2"] + smoke = profiles["smoke-native-v1"] + materialized_short = profiles["short-vllm-v2"] + materialized_smoke = profiles["smoke-vllm-v1"] + assert current_short.manifest["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION + assert current_short.manifest["backend"]["name"] == "qwen3_5" + assert materialized_short.manifest["backend"]["name"] == "vllm" + assert materialized_short.exact_rows == current_short.exact_rows + assert materialized_smoke.manifest["backend"]["name"] == "vllm" + assert materialized_smoke.manifest["backend"]["enforce_eager"] is True + assert materialized_smoke.exact_rows == smoke.exact_rows + assert profiles["full-v1"].exact_rows is None + + +@pytest.mark.parametrize( + ("name", "backend"), + [ + ( + "core3-full-native-v1", + { + "attention_implementation": "sdpa", + "enable_thinking": False, + "name": "qwen3_5", + }, + ), + ( + "core3-full-vllm-v1", + {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, + ), + ], +) +def test_core3_full_teacher_profiles_pin_paired_population_and_runtime(name, backend): + contract = contracts.load_profile(name) + + assert contract.manifest["model"] == { + "repository": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17", + } + assert contract.manifest["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION + assert contract.manifest["backend"] == backend + assert contract.manifest["generation"] == {"do_sample": False, "temperature": 0} + assert contract.manifest["seed"] == 42 + assert contract.manifest["repetitions"] == 1 + assert contract.manifest["batch_size"] == 1 + assert contract.manifest["selection"] == "all" + assert contract.exact_rows is None + assert { + task: entry["population_rows"] for task, entry in contract.manifest["tasks"].items() + } == {"realworldqa": 765, "mmmu_val": 900, "mvbench": 4000} + assert contract.manifest["tasks"]["mvbench"]["leaf_populations"] == dict.fromkeys( + suites.MVBENCH_LEAF_TASKS, 200 + ) assert { - task: len(entry["rows"]) for task, entry in short_all_native.manifest["tasks"].items() + task: entry["dataset_revision"] for task, entry in contract.manifest["tasks"].items() } == { - "realworldqa": 64, - "mmmu_val": 120, - "mvbench": 160, - "video_mmmu": 72, - "videomme": 72, - "longvideobench_val_v": 68, - "mlvu_dev": 70, - "perceptiontest_val_mc": 64, + task: profile.VLM_BENCHMARK_DATASETS[task].revision + for task in ("realworldqa", "mmmu_val", "mvbench") } - assert short.exact_rows is not None - assert profiles["full-v1"].exact_rows is None - mmmu_rows = short.manifest["tasks"]["mmmu_val"]["rows"] - assert Counter(row["source_row_index"] // 30 for row in mmmu_rows) == Counter( - dict.fromkeys(range(30), 4) - ) - exact_rows = short.exact_rows - assert exact_rows is not None - path = tmp_path / "short-v1-rows.json" - path.write_text(json.dumps(exact_rows)) - validated = suites.load_quick_manifest(path) - assert suites.manifest_sha256(validated) + +@pytest.mark.parametrize( + ("name", "field", "value", "message"), + [ + ( + "core3-full-native-v1", + "backend", + {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, + "backend differs", + ), + ( + "core3-full-vllm-v1", + "model", + {"repository": "Qwen/Qwen3.5-0.8B", "revision": "different"}, + "model pin differs", + ), + ("core3-full-native-v1", "population", 764, "population differs"), + ], +) +def test_core3_full_teacher_profiles_reject_contract_overrides( + monkeypatch, tmp_path, name, field, value, message +): + for profile_name in ("core3-full-native-v1", "core3-full-vllm-v1"): + source = contracts._PROFILE_ROOT / f"{profile_name}.json" + (tmp_path / source.name).write_text(source.read_text()) + manifest_path = tmp_path / f"{name}.json" + manifest = json.loads(manifest_path.read_text()) + if field == "population": + manifest["tasks"]["realworldqa"]["population_rows"] = value + else: + manifest[field] = value + manifest_path.write_text(json.dumps(manifest)) + monkeypatch.setattr(contracts, "_PROFILE_ROOT", tmp_path) + + with pytest.raises(RuntimeError, match=message): + contracts.load_profile(name) + + +def test_audited_profile_rejects_rows_that_drift_from_systematic_selection(monkeypatch, tmp_path): + for name in ("short-v1", "short-native-v1", "short-native-v2"): + source = contracts._PROFILE_ROOT / f"{name}.json" + (tmp_path / source.name).write_text(source.read_text()) + manifest_path = tmp_path / "short-native-v2.json" + manifest = json.loads(manifest_path.read_text()) + manifest["tasks"]["realworldqa"]["rows"][0]["source_row_index"] = 6 + manifest_path.write_text(json.dumps(manifest)) + monkeypatch.setattr(contracts, "_PROFILE_ROOT", tmp_path) + + with pytest.raises(RuntimeError, match="rows differ from its sampling audit"): + contracts.load_profile("short-native-v2") def test_short_all_native_profile_builds_grouped_and_single_selectors(tmp_path): - contract = contracts.load_profile("short-all-native-v1") + contract = contracts.load_profile("short-all-native-v2") exact_rows = contract.exact_rows assert exact_rows is not None validated = suites.validate_exact_rows_manifest( @@ -633,14 +850,18 @@ def test_short_all_native_profile_builds_grouped_and_single_selectors(tmp_path): spec.loader.exec_module(selectors) class Documents: - def __init__(self, size): + def __init__(self, size, *, columns=None, rows=None): self.size = size + self.columns = columns or {} + self.rows = rows or {} def __len__(self): return self.size - def __getitem__(self, _index): - return {} + def __getitem__(self, index): + if isinstance(index, str): + return self.columns[index] + return self.rows.get(index, {}) def select(self, indices): return list(indices) @@ -654,8 +875,101 @@ def select(self, indices): assert ( selectors.select_modelopt_vlm_benchmark_video_mmmu_adaptation(Documents(300)) == adaptation ) - videomme = [row["source_row_index"] for row in tasks_manifest["videomme"]["rows"]] - assert selectors.select_modelopt_vlm_benchmark_videomme(Documents(2700)) == videomme + with pytest.raises(ValueError, match="source population drifted"): + selectors.select_modelopt_vlm_benchmark_video_mmmu_adaptation(Documents(299)) + with pytest.raises(ValueError, match="source population drifted"): + selectors.select_modelopt_vlm_benchmark_realworldqa(Documents(764)) + with pytest.raises(ValueError, match="source population drifted"): + selectors.select_modelopt_vlm_benchmark_mvbench_action_sequence(Documents(199)) + + mmmu_task = tasks_manifest["mmmu_val"] + mmmu_ids = [ + f"validation_{stratum['name']}_{index + 1}" + for stratum in mmmu_task["selection"]["strata"] + for index in range(stratum["population_rows"]) + ] + mmmu_rows = { + row["source_row_index"]: {"id": row["source_sample_id"]} for row in mmmu_task["rows"] + } + mmmu_documents = Documents(900, columns={"id": mmmu_ids}, rows=mmmu_rows) + assert selectors.select_modelopt_vlm_benchmark_mmmu_val(mmmu_documents) == [ + row["source_row_index"] for row in mmmu_task["rows"] + ] + mmmu_ids[0] = mmmu_ids[30] + with pytest.raises(ValueError, match="source strata drifted"): + selectors.select_modelopt_vlm_benchmark_mmmu_val(mmmu_documents) + + assert callable(selectors.select_modelopt_vlm_benchmark_videomme) + + +@pytest.mark.parametrize("task", ["videomme", "mlvu_dev", "perceptiontest_val_mc"]) +@pytest.mark.parametrize( + ("drift", "expected_rank", "observed_strata"), + [ + (None, 1, ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"]), + ("selected stratum", 1, ["alpha|kind", "beta|kind", "beta|kind", "alpha|kind"]), + ("local rank", 0, ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"]), + ], +) +def test_audited_selector_checks_selected_stratum_and_local_rank( + tmp_path, task, drift, expected_rank, observed_strata +): + expected_stratum = "alpha" if task == "mlvu_dev" else "alpha|kind" + other_stratum = "beta" if task == "mlvu_dev" else "beta|kind" + upstream_id = "video:q" if task == "perceptiontest_val_mc" else "q" + manifest = { + "tasks": { + task: { + "rows": [ + { + "source_row_index": 2, + "source_sample_id": f"{task}:2", + "sampling_stratum": expected_stratum, + "source_stratum_index": expected_rank, + "upstream_sample_id": upstream_id, + } + ], + "selection": { + "population_rows": 4, + "strata": [ + {"name": expected_stratum, "population_rows": 2}, + {"name": other_stratum, "population_rows": 2}, + ], + }, + } + } + } + tasks._write_quick_selection_module(tmp_path, manifest) + spec = importlib.util.spec_from_file_location( + f"sampling_position_{task}_{drift}", tmp_path / "modelopt_quick_selection.py" + ) + assert spec is not None and spec.loader is not None + selectors = importlib.util.module_from_spec(spec) + spec.loader.exec_module(selectors) + + class Documents: + def __len__(self): + return 4 + + def __getitem__(self, index): + if isinstance(index, str): + if task == "mlvu_dev": + return [value.split("|", 1)[0] for value in observed_strata] + column = 0 if index in {"duration", "area"} else 1 + return [value.split("|", 1)[column] for value in observed_strata] + if task == "perceptiontest_val_mc": + return {"video_name": "video", "question_id": "q"} + return {"question_id": "q"} + + def select(self, indices): + return list(indices) + + selector = getattr(selectors, f"select_{suites.task_name(task)}") + if drift is None: + assert selector(Documents()) == [2] + else: + with pytest.raises(ValueError, match="source sampling positions drifted"): + selector(Documents()) def test_versioned_profile_preflight_reports_immutable_contract(monkeypatch, tmp_path): @@ -681,6 +995,12 @@ def test_versioned_profile_preflight_reports_immutable_contract(monkeypatch, tmp assert prepared.report["profile_fingerprint"] == contract.fingerprint assert prepared.report["source_tasks"] == list(contract.source_tasks) assert prepared.report["quick_selected_rows"] == 344 + assert prepared.report["quick_row_identities"] == suites.manifest_row_identities( + prepared.quick_manifest + ) + assert prepared.report["quick_task_denominators"] == suites.manifest_task_denominators( + prepared.quick_manifest + ) def test_native_profile_builds_qwen35_backend_settings(monkeypatch, tmp_path): @@ -693,13 +1013,14 @@ def test_native_profile_builds_qwen35_backend_settings(monkeypatch, tmp_path): "--output-dir", str(tmp_path / "results"), "--profile", - "short-native-v1", + "short-all-native-v2", "--hf-home", str(hf_home), ] ) prepared = preflight.prepare(args) + (tmp_path / "tasks").mkdir() settings = preflight.settings( args, tasks_root=tmp_path / "tasks", @@ -722,6 +1043,252 @@ def test_native_profile_builds_qwen35_backend_settings(monkeypatch, tmp_path): assert not (tmp_path / "tasks/modelopt_qwen35_no_think.jinja").exists() +@pytest.mark.parametrize( + ("profile_name", "expected_eager"), + [("smoke-vllm-v1", True), ("short-vllm-v2", None)], +) +def test_vllm_profile_forwards_runtime_settings( + monkeypatch, tmp_path, profile_name, expected_eager +): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + profile_name, + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + tasks_root = tmp_path / "tasks" + tasks_root.mkdir() + settings = preflight.settings( + args, + tasks_root=tasks_root, + configured_tasks=("modelopt_vlm_benchmark_realworldqa",), + prepared=prepared, + ) + argv, _, _ = lmms._build_command( + settings, + checkpoint=str(model), + output_path=tmp_path / "lmms-results", + ) + model_args = argv[argv.index("--model_args") + 1] + + assert settings["model_args"].get("enforce_eager") is expected_eager + assert ("enforce_eager=True" in model_args) is (expected_eager is True) + assert settings["model_args"]["attention_config"] == {"flash_attn_version": 2} + assert 'attention_config={"flash_attn_version":2}' in model_args + + +@pytest.mark.parametrize("name", ["core3-full-native-v1", "core3-full-vllm-v1"]) +def test_core3_full_teacher_profiles_preserve_backend_prompt_policy(monkeypatch, tmp_path, name): + model, hf_home = _write_core3_teacher_snapshot(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("mmmu_val",)) + _use_offline_fakes(monkeypatch, lmms_root) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + name, + "--profile-task", + "mmmu_val", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + tasks_root = tmp_path / "tasks" + tasks_root.mkdir() + settings = preflight.settings( + args, + tasks_root=tasks_root, + configured_tasks=("modelopt_vlm_benchmark_mmmu_val",), + prepared=prepared, + ) + + assert prepared.report["model_pin"] == { + "repository": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17", + } + assert prepared.report["profile_population_rows"] == {"mmmu_val": 900} + assert prepared.report["output_budget_contract"]["mmmu_val"]["effective_max_new_tokens"] == 128 + if name == "core3-full-native-v1": + assert settings["model"] == "qwen3_5" + assert prepared.report["backend_limitations"] == [] + else: + assert settings["model"] == "vllm" + assert prepared.report["output_budget_contract"]["mmmu_val"] == { + "adapter": "vllm", + "effective_max_new_tokens": 128, + "limitation": ( + "the pinned generic vLLM adapter treats its model-level max_new_tokens as a floor" + ), + "requested_max_new_tokens": 128, + "resolution": "max(task_max_new_tokens, model_max_new_tokens_floor=1)", + } + assert prepared.report["backend_limitations"] == [ + "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", + "pinned generic vLLM max_new_tokens is a model-level lower bound", + ] + + +def test_core3_full_teacher_profile_population_expectations_follow_group_shard( + monkeypatch, tmp_path +): + model, hf_home = _write_core3_teacher_snapshot(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("mvbench",)) + _use_offline_fakes(monkeypatch, lmms_root) + media = hf_home / profile.VLM_BENCHMARK_DATASETS["mvbench"].media_dir + media.mkdir(parents=True) + (media / "sample").write_bytes(b"media") + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core3-full-native-v1", + "--profile-task", + "mvbench", + "--profile-task-shard", + "3/8", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + expected_leaves = ("episodic_reasoning", "moving_direction", "egocentric_navigation") + + assert prepared.profile_task_leaves == expected_leaves + assert evaluator._expected_task_populations(prepared, ("modelopt_vlm_benchmark_mvbench",)) == { + suites.task_name("mvbench", leaf=leaf): 200 for leaf in expected_leaves + } + + +def test_core3_full_teacher_profile_accepts_snapshot_symlink(monkeypatch, tmp_path): + snapshot, hf_home = _write_core3_teacher_snapshot(tmp_path) + checkpoint_alias = tmp_path / "teacher" + checkpoint_alias.symlink_to(snapshot, target_is_directory=True) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) + _use_offline_fakes(monkeypatch, lmms_root) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(checkpoint_alias), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core3-full-native-v1", + "--profile-task", + "realworldqa", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + + assert prepared.source_tasks == ("realworldqa",) + assert prepared.report["model_pin"]["revision"] == snapshot.name + + +@pytest.mark.parametrize("kind", ["untracked-copy", "wrong-revision"]) +def test_core3_full_teacher_profile_rejects_unpinned_checkpoint(monkeypatch, tmp_path, kind): + monkeypatch.delenv("HF_HUB_CACHE", raising=False) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + if kind == "untracked-copy": + model = _write_checkpoint(tmp_path) + else: + parent = hf_home / "hub/models--Qwen--Qwen3.5-0.8B/snapshots" + parent.mkdir(parents=True) + model = _write_checkpoint_at(parent / "different") + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core3-full-native-v1", + "--profile-task", + "realworldqa", + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match="requires the exact local Hub snapshot"): + preflight.prepare(args) + + +def test_core3_full_teacher_profile_rejects_settings_override(monkeypatch, tmp_path): + model, hf_home = _write_core3_teacher_snapshot(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) + _use_offline_fakes(monkeypatch, lmms_root) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core3-full-native-v1", + "--profile-task", + "realworldqa", + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match="do not allow settings overrides"): + evaluator.evaluate(args, settings_overrides={"model": "vllm"}) + + +def test_historical_short_profile_preserves_vllm_backend(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "short-v1", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + (tmp_path / "tasks").mkdir() + settings = preflight.settings( + args, + tasks_root=tmp_path / "tasks", + configured_tasks=("modelopt_vlm_benchmark_realworldqa",), + prepared=prepared, + ) + + assert prepared.report["lmms_eval_revision"] == checkpoint.LMMS_EVAL_LEGACY_REVISION + assert settings["model"] == "vllm" + assert settings["checkpoint_arg"] == "model" + assert settings["reasoning_parser"] == "qwen3" + assert settings["model_args"]["max_frame_num"] == 32 + assert Path(settings["model_args"]["chat_template"]).exists() + + def test_versioned_profile_rejects_seed_override(monkeypatch, tmp_path): model, hf_home = _full_inputs(monkeypatch, tmp_path) args = evaluation._build_parser().parse_args( @@ -787,6 +1354,8 @@ def test_all_row_profile_task_preserves_contract_identity(monkeypatch, tmp_path) assert prepared.source_tasks == (task,) assert prepared.report["quick_selected_rows"] is None + assert prepared.report["quick_row_identities"] is None + assert prepared.report["quick_task_denominators"] is None assert ( prepared.report["profile_fingerprint"] == contracts.load_profile(profile_name).fingerprint ) @@ -801,7 +1370,7 @@ def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, t "--output-dir", str(tmp_path / "results"), "--profile", - "short-all-native-v1", + "short-all-native-v2", "--profile-task", "mvbench", "--profile-task-shard", @@ -821,13 +1390,40 @@ def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, t assert {row["leaf_task"] for row in manifest_rows} == { f"mvbench_{leaf}" for leaf in expected_leaves } + manifest_selection = prepared.quick_manifest["tasks"]["mvbench"]["selection"] + assert manifest_selection["population_rows"] == 600 + assert manifest_selection["selected_rows"] == 24 + assert [stratum["name"] for stratum in manifest_selection["strata"]] == list(expected_leaves) + assert manifest_selection["selected_index_quantiles"] == { + "method": "lower-order-statistic", + "p0": 12, + "p25": 37, + "p50": 87, + "p75": 137, + "p100": 187, + } + assert ( + manifest_selection["selected_row_identities_sha256"] + == hashlib.sha256( + json.dumps(manifest_rows, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest() + ) assert prepared.report["quick_selected_rows"] == 24 + assert prepared.report["quick_row_identities"] == suites.manifest_row_identities( + prepared.quick_manifest + ) + assert prepared.report["quick_task_denominators"] == { + "mvbench": {"population_rows": 600, "selected_rows": 24} + } + assert ( + evaluator._expected_task_populations(prepared, ("modelopt_vlm_benchmark_mvbench",)) is None + ) assert prepared.report["quick_manifest_sha256"] == suites.manifest_sha256( prepared.quick_manifest ) assert ( prepared.report["profile_fingerprint"] - == contracts.load_profile("short-all-native-v1").fingerprint + == contracts.load_profile("short-all-native-v2").fingerprint ) tasks_root, _ = tasks.prepare( tmp_path / "results", @@ -841,6 +1437,51 @@ def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, t assert group["task"] == [f"modelopt_vlm_benchmark_mvbench_{leaf}" for leaf in expected_leaves] +def test_smoke_profile_generates_only_manifest_backed_mvbench_leaves(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "smoke-native-v1", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + tasks_root, configured_tasks = tasks.prepare( + args.output_dir, + suite=prepared.suite, + source_tasks=prepared.source_tasks, + profile_task_leaves=prepared.profile_task_leaves, + dataset_snapshots=prepared.dataset_snapshots, + quick_manifest=prepared.quick_manifest, + ) + + assert configured_tasks == ( + "modelopt_vlm_benchmark_realworldqa", + "modelopt_vlm_benchmark_mmmu_val", + "modelopt_vlm_benchmark_mvbench", + ) + group = json.loads((tasks_root / "modelopt_vlm_benchmark_mvbench.yaml").read_text()) + assert group["task"] == ["modelopt_vlm_benchmark_mvbench_action_sequence"] + assert (tasks_root / "modelopt_vlm_benchmark_mvbench_action_sequence.yaml").is_file() + assert not (tasks_root / "modelopt_vlm_benchmark_mvbench_egocentric_navigation.yaml").exists() + + spec = importlib.util.spec_from_file_location( + "smoke_selectors", tasks_root / "modelopt_quick_selection.py" + ) + assert spec is not None and spec.loader is not None + selectors = importlib.util.module_from_spec(spec) + spec.loader.exec_module(selectors) + assert hasattr(selectors, "select_modelopt_vlm_benchmark_mvbench_action_sequence") + assert not hasattr(selectors, "select_modelopt_vlm_benchmark_mvbench_egocentric_navigation") + + @pytest.mark.parametrize( ("selection", "message"), [ @@ -947,7 +1588,17 @@ def fake_evaluate(args, *, settings_overrides, preflight_callback): (0.4 + score_offset, 0.6 + score_offset), start=1 ): result_path = tmp_path / f"run-{index}.json" - result_path.write_text("{}") + result_path.write_text( + json.dumps( + { + "sample_counts": {"realworldqa": 100, "mmmu_val": 100}, + "mmmu_parser_audit": { + "sample_count": 100, + "status_counts": {"parsed": 90, "fallback_random": 10}, + }, + } + ) + ) runs.append( { "metrics": { @@ -988,6 +1639,11 @@ def fake_evaluate(args, *, settings_overrides, preflight_callback): assert summary["profile"] == post_mip.TASK_PREFIX100_REPEAT2_PROFILE assert summary["metrics"] == result["metrics"] assert summary["result_paths"] == result["run_result_paths"] + assert summary["sample_counts"] == {"mmmu_val": 200, "realworldqa": 200} + assert summary["mmmu_parser_audit"] == { + "sample_count": 200, + "status_counts": {"fallback_random": 20, "parsed": 180}, + } refreshed = post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( model, @@ -1080,9 +1736,89 @@ def test_mmvu_guard_is_limited_to_full_suite(monkeypatch, tmp_path): assert "\nprocess_results: !function modelopt_mmvu_guard.process_results\n" in full_generated +def test_mmmu_adapter_labels_parser_fallback_without_changing_prediction(monkeypatch, tmp_path): + snapshot = tmp_path / "snapshot" + snapshot.mkdir() + lmms_root = _write_lmms_tasks(tmp_path, ("mmmu_val",)) + monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) + tasks_root, _ = tasks.prepare( + tmp_path / "results", + suite="short", + source_tasks=("mmmu_val",), + dataset_snapshots={"mmmu_val": snapshot}, + quick_manifest=None, + ) + + upstream = ModuleType("lmms_eval.tasks.mmmu.utils") + + def get_multi_choice_info(options): + choices = [chr(ord("A") + index) for index in range(len(options))] + return dict(zip(choices, options, strict=True)), choices + + def parse_multi_choice_response(response, all_choices, _index_to_answer): + return random.choice(all_choices) if response == "unparseable" else "A" + + def mmmu_process_results(document, results): + if document["question_type"] == "multiple-choice": + index_to_answer, choices = get_multi_choice_info(json.loads(document["options"])) + parsed = [ + parse_multi_choice_response(response, choices, index_to_answer) + for response in results + ] + else: + parsed = [""] * len(results) + accuracy = {"parsed_pred": parsed} + return {"mmmu_acc": accuracy, "mmmu_acc_pass_at_k": accuracy} + + upstream.get_multi_choice_info = get_multi_choice_info + upstream.parse_multi_choice_response = parse_multi_choice_response + upstream.mmmu_process_results = mmmu_process_results + package_modules = { + "lmms_eval": ModuleType("lmms_eval"), + "lmms_eval.tasks": ModuleType("lmms_eval.tasks"), + "lmms_eval.tasks.mmmu": ModuleType("lmms_eval.tasks.mmmu"), + "lmms_eval.tasks.mmmu.utils": upstream, + } + package_modules["lmms_eval.tasks.mmmu"].utils = upstream + for name, module in package_modules.items(): + monkeypatch.setitem(sys.modules, name, module) + spec = importlib.util.spec_from_file_location( + "modelopt_mmmu_audit", tasks_root / "modelopt_mmmu_audit.py" + ) + assert spec is not None and spec.loader is not None + audit_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(audit_module) + document = { + "options": json.dumps(["first", "second"]), + "question_type": "multiple-choice", + } + + parsed = audit_module.process_results(document, ["(A)"]) + random.seed(123) + parse_multi_choice_response("unparseable", ["A", "B"], {}) + expected_random_state = random.getstate() + random.seed(123) + fallback = audit_module.process_results(document, ["unparseable"]) + invalid_open = audit_module.process_results( + {"question_type": "open"}, + ["unparseable"], + ) + + assert parsed["mmmu_acc"] == { + "parsed_pred": ["A"], + "parser_status": ["parsed"], + } + assert fallback["mmmu_acc"]["parsed_pred"][0] in {"A", "B"} + assert fallback["mmmu_acc"]["parser_status"] == ["fallback_random"] + assert random.getstate() == expected_random_state + assert invalid_open["mmmu_acc"]["parser_status"] == ["invalid_open"] + generated = (tasks_root / f"{suites.task_name('mmmu_val')}.yaml").read_text() + assert "process_results: !function modelopt_mmmu_audit.process_results\n" in generated + + def test_quick_manifest_requires_exact_pins_counts_and_leaf_balance(tmp_path): path = _quick_manifest(tmp_path / "quick.json") - assert suites.manifest_sha256(suites.load_quick_manifest(path)) + suites.load_quick_manifest(path) manifest = json.loads(path.read_text()) manifest["tasks"]["mmmu_val"]["rows"].pop() @@ -1146,7 +1882,7 @@ def test_offline_preflight_scrubs_credentials_and_traverses_media(monkeypatch, t package.mkdir(parents=True) (tasks_root / "lmms_eval/__init__.py").write_text("") (package / "__init__.py").write_text("""import os -class Config: task = "modelopt_vlm_benchmark_mvbench" +class Config: task = "modelopt_vlm_benchmark_mvbench_action_sequence" class Task: config = Config() def has_test_docs(self): return True @@ -1170,7 +1906,11 @@ def __init__(self, include_path, model_name): "HUGGING_FACE_HUB_TOKEN", ) assert all(name not in os.environ for name in credential_names) - def load_task_or_group(self, tasks): return {Group(): {"leaf": Task()}} + def load_task_or_group(self, tasks): + first = Task() + if os.environ.get("FAKE_DISTINCT_DUPLICATE"): + return {Group(): {"first": first, "second": Task()}} + return {Group(): {"first": first, "repeat": first}} """) hf_home = tmp_path / "hf-home" hf_home.mkdir() @@ -1184,11 +1924,35 @@ def load_task_or_group(self, tasks): return {Group(): {"leaf": Task()}} hf_home=hf_home, timeout_seconds=123, model_name="qwen3_5", + expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 1}, ) + assert report["document_counts"] == {"modelopt_vlm_benchmark_mvbench_action_sequence": 1} assert report["media_documents"] == 1 + assert report["observed_populations"] == {"modelopt_vlm_benchmark_mvbench_action_sequence": 1} assert report["status"] == "passed" + with pytest.raises(RuntimeError, match="configured task population mismatch"): + tasks.verify_offline( + tasks_root, + ("modelopt_vlm_benchmark_mvbench",), + hf_home=hf_home, + timeout_seconds=123, + model_name="qwen3_5", + expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 2}, + ) + + monkeypatch.setenv("FAKE_DISTINCT_DUPLICATE", "1") + with pytest.raises(RuntimeError, match="distinct task objects share configured task name"): + tasks.verify_offline( + tasks_root, + ("modelopt_vlm_benchmark_mvbench",), + hf_home=hf_home, + timeout_seconds=123, + model_name="qwen3_5", + expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 1}, + ) + def test_video_adapter_normalizes_supported_suffixes_and_rejects_unknown(monkeypatch, tmp_path): uppercase = tmp_path / "sample.MP4" @@ -1233,21 +1997,6 @@ def test_video_adapter_normalizes_supported_suffixes_and_rejects_unknown(monkeyp adapter._normalize([str(unknown)]) -def test_profile_contract_pins_every_task_and_revision(): - assert profile.VLM_BENCHMARK_TASKS == ( - "realworldqa", - "mmmu_val", - "video_mmmu", - "mvbench", - "mmvu_val", - "videomme", - "longvideobench_val_v", - "mlvu_dev", - "perceptiontest_val_mc", - ) - assert all(len(item.revision) == 40 for item in profile.VLM_BENCHMARK_DATASETS.values()) - - def test_video_reader_validation_is_limited_to_video_suites(monkeypatch): monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: None) diff --git a/tests/unit/torch/puzzletron/test_ci_image_contract.py b/tests/unit/torch/puzzletron/test_ci_image_contract.py index 61c7331ad70..ad571cbfb3a 100644 --- a/tests/unit/torch/puzzletron/test_ci_image_contract.py +++ b/tests/unit/torch/puzzletron/test_ci_image_contract.py @@ -18,6 +18,7 @@ import hashlib import json import re +import subprocess import yaml @@ -119,6 +120,7 @@ def test_lmms_eval_compatibility_patch_reconciles_worker_dependencies(project_ro "- printer = get_printer(Settings()._jupyter)", "- return printer", "- self.printer = get_wandb_printer()", + "- sampling_params = SamplingParams(**params)", "-from latex2sympy2 import latex2sympy", "+from latex2sympy2_extended import latex2sympy", "-from latex2sympy2 import latex2sympy", @@ -132,6 +134,7 @@ def test_lmms_eval_compatibility_patch_reconciles_worker_dependencies(project_ro ] assert lmms_source["compatibility_patch_files"] == [ "lmms_eval/loggers/wandb_logger.py", + "lmms_eval/models/simple/vllm.py", "lmms_eval/tasks/emma/utils.py", "lmms_eval/tasks/mathvision/eval_utils.py", "lmms_eval/tasks/stare/utils.py", @@ -145,6 +148,76 @@ def test_lmms_eval_compatibility_patch_reconciles_worker_dependencies(project_ro assert 'python -m pip install -e "${LMMS_EVAL_ROOT}[qwen]"' in dockerfile +def test_lmms_eval_vllm_patch_preserves_task_sampling_and_rejects_drift( + project_root_path, tmp_path +): + puzzletron_root = project_root_path / "examples/puzzletron" + environment = json.loads((puzzletron_root / "ci_environment.json").read_text()) + patch_text = ( + puzzletron_root / "patches" / environment["lmms_eval"]["compatibility_patch"] + ).read_text() + marker = "diff --git a/lmms_eval/models/simple/vllm.py b/lmms_eval/models/simple/vllm.py" + start = patch_text.index(marker) + end = patch_text.index("\ndiff --git ", start + len(marker)) + 1 + vllm_patch = tmp_path / "vllm.patch" + vllm_patch.write_text(patch_text[start:end]) + + correct_sampling = ( + " sampling_params = " + "SamplingParams(**self._build_sampling_params_dict(gen_kwargs))" + ) + undefined_overwrite = " sampling_params = SamplingParams(**params)" + + def write_fixture(root, *, overwrite): + source = root / "lmms_eval/models/simple/vllm.py" + source.parent.mkdir(parents=True) + lines = ["# pinned upstream fixture"] * 522 + lines[474] = ( + ' gen_kwargs["max_new_tokens"] = ' + 'self._select_max_new_tokens(gen_kwargs.get("max_new_tokens"))' + ) + lines[477] = correct_sampling + lines[520] = overwrite + lines[521] = ( + ' self._write_watchdog_heartbeat("chat_start", ' + "batch_idx=batch_idx, batch_requests=batch_requests)" + ) + source.write_text("\n".join(lines) + "\n") + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + return source + + exact_checkout = tmp_path / "exact" + exact_checkout.mkdir() + exact_source = write_fixture(exact_checkout, overwrite=undefined_overwrite) + subprocess.run( + ["git", "apply", "--unidiff-zero", "--check", str(vllm_patch)], + cwd=exact_checkout, + check=True, + ) + subprocess.run( + ["git", "apply", "--unidiff-zero", str(vllm_patch)], + cwd=exact_checkout, + check=True, + ) + patched_source = exact_source.read_text() + assert correct_sampling in patched_source + assert undefined_overwrite not in patched_source + + drifted_checkout = tmp_path / "drifted" + drifted_checkout.mkdir() + write_fixture( + drifted_checkout, + overwrite=" sampling_params = SamplingParams(**other_params)", + ) + rejected = subprocess.run( + ["git", "apply", "--unidiff-zero", "--check", str(vllm_patch)], + cwd=drifted_checkout, + capture_output=True, + text=True, + ) + assert rejected.returncode != 0 + + def test_image_checks_native_lmms_eval_contract(project_root_path): puzzletron_root = project_root_path / "examples/puzzletron" environment = json.loads((puzzletron_root / "ci_environment.json").read_text()) diff --git a/tests/unit/torch/puzzletron/test_lmms_evaluation.py b/tests/unit/torch/puzzletron/test_lmms_evaluation.py index 1ba168edd9c..bd86a120693 100644 --- a/tests/unit/torch/puzzletron/test_lmms_evaluation.py +++ b/tests/unit/torch/puzzletron/test_lmms_evaluation.py @@ -17,6 +17,7 @@ import json import os +import shutil import subprocess import sys from pathlib import Path @@ -500,6 +501,36 @@ def time_out(argv, **_kwargs): assert Path(error.stderr_path).read_text() == "evaluation timed out\n" +def test_run_checkpoint_preserves_timeout_artifacts_if_attempt_directory_disappears( + monkeypatch, tmp_path +): + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + + def time_out(argv, *, cwd, **_kwargs): + shutil.rmtree(cwd) + raise lmms.LmmsEvalTimeoutError( + argv, + 7, + output="partial evaluator output\n", + stderr="evaluation timed out\n", + ) + + monkeypatch.setattr(lmms, "_run_process", time_out) + + with pytest.raises(lmms.LmmsEvalTimeoutError) as exc_info: + lmms.run_lmms_eval_checkpoint( + checkpoint, + output_root=tmp_path / "results", + settings=_settings("ifeval"), + ) + + error = exc_info.value + assert Path(error.command_path).is_file() + assert Path(error.stdout_path).read_text() == "partial evaluator output\n" + assert Path(error.stderr_path).read_text() == "evaluation timed out\n" + + def test_completion_validates_resolved_task_expansion(): sample_counts = lmms._validate_completion( { From 0e6cf467ff907ef7e56ccce85a2fa0d90140b343 Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Fri, 4 Sep 2026 15:12:39 +0200 Subject: [PATCH 02/10] Compose VLM evaluation profiles Signed-off-by: Johannes Rausch --- .../docs/vlm_checkpoint_evaluation.md | 99 +- .../puzzletron/evaluation/vlm/contracts.py | 368 ++-- .../puzzletron/evaluation/vlm/post_mip.py | 8 +- .../puzzletron/evaluation/vlm/preflight.py | 13 +- .../anymodel-vllm-eager_r1.json} | 7 +- .../anymodel-vllm_r1.json} | 7 +- .../profiles/backends/qwen-3.5-native_r1.json | 9 + .../profiles/backends/qwen-3.5-vllm_r1.json | 9 + .../core-3_24-examples_r1-native.json | 7 + .../profiles/core-3_24-examples_r1-vllm.json | 7 + .../core-3_344-examples_r1-native.json | 7 + .../profiles/core-3_344-examples_r1-vllm.json | 7 + .../vlm/profiles/core-3_full_r1-native.json | 7 + .../vlm/profiles/core-3_full_r1-vllm.json | 7 + .../vlm/profiles/core3-full-vllm-v1.json | 11 - .../evaluators/lmms-eval-legacy_r1.json | 5 + .../evaluators/lmms-eval-modelopt_r1.json | 5 + .../lmms-eval-qwen-3.5-native_r1.json | 5 + .../evaluation/vlm/profiles/full-v1.json | 78 +- .../judge-free-8_690-examples_r1-native.json | 7 + .../core-3_24-examples_r1.json} | 6 +- .../core-3_344-examples_legacy-r1.json | 1567 ++++++++++++++++ .../core-3_344-examples_r1.json} | 7 +- .../core-3_full_r1.json} | 26 +- .../judge-free-8_690-examples_legacy-r1.json | 1503 ++++++++++++++++ .../judge-free-8_690-examples_r1.json} | 7 +- .../judge-free-8_full_legacy-r1.json | 55 + .../vlm/profiles/short-all-native-v1.json | 1510 +--------------- .../vlm/profiles/short-native-v1.json | 12 +- .../evaluation/vlm/profiles/short-v1.json | 1590 +---------------- .../puzzletron/evaluation/vlm/test_run.py | 109 +- 31 files changed, 3601 insertions(+), 3464 deletions(-) rename examples/puzzletron/evaluation/vlm/profiles/{smoke-vllm-v1.json => backends/anymodel-vllm-eager_r1.json} (59%) rename examples/puzzletron/evaluation/vlm/profiles/{short-vllm-v2.json => backends/anymodel-vllm_r1.json} (55%) create mode 100644 examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-native_r1.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-vllm_r1.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-native.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-vllm.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-native.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-vllm.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-native.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-vllm.json delete mode 100644 examples/puzzletron/evaluation/vlm/profiles/core3-full-vllm-v1.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-legacy_r1.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-modelopt_r1.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-qwen-3.5-native_r1.json create mode 100644 examples/puzzletron/evaluation/vlm/profiles/judge-free-8_690-examples_r1-native.json rename examples/puzzletron/evaluation/vlm/profiles/{smoke-native-v1.json => sample_sets/core-3_24-examples_r1.json} (98%) create mode 100644 examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_legacy-r1.json rename examples/puzzletron/evaluation/vlm/profiles/{short-native-v2.json => sample_sets/core-3_344-examples_r1.json} (99%) rename examples/puzzletron/evaluation/vlm/profiles/{core3-full-native-v1.json => sample_sets/core-3_full_r1.json} (73%) create mode 100644 examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_legacy-r1.json rename examples/puzzletron/evaluation/vlm/profiles/{short-all-native-v2.json => sample_sets/judge-free-8_690-examples_r1.json} (99%) create mode 100644 examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_full_legacy-r1.json diff --git a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md index 0933cc578c3..b07586467f0 100644 --- a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md +++ b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md @@ -1,15 +1,23 @@ # VLM checkpoint evaluation Use this evaluator to test a local Qwen 3.5 checkpoint on image and video -benchmarks. A profile chooses the benchmarks, examples, model-loading path, -and evaluation settings. Use the same profile for every model being compared. +benchmarks. A runnable profile composes three independently named contracts: + +- a sample set, which fixes the benchmark scope and selected examples; +- a backend profile, which fixes model loading and prompt construction; and +- an evaluator profile, which pins the `lmms-eval` revision. + +Callers still select one profile. Keeping the components separate makes it +clear that native and vLLM runs can use identical examples while changing only +the backend. Use the same runnable profile for every model being compared. The common choices are: -- `short-native-v2` for the maintained three-benchmark campaign screen; -- `short-vllm-v2` and `smoke-vllm-v1` for materialized heterogeneous campaign checkpoints; -- `short-all-native-v2` for a broader eight-benchmark regression screen; -- `core3-full-native-v1` and `core3-full-vllm-v1` for paired full-dataset +- `core-3_344-examples_r1-native` for the maintained three-benchmark campaign screen; +- `core-3_344-examples_r1-vllm` for materialized heterogeneous campaign checkpoints; +- `core-3_24-examples_r1-native` and `core-3_24-examples_r1-vllm` for lifecycle smoke checks; +- `judge-free-8_690-examples_r1-native` for a broader eight-benchmark regression screen; +- `core-3_full_r1-native` and `core-3_full_r1-vllm` for paired full-dataset teacher references on RealWorldQA, MMMU validation, and MVBench; - `short-v1`, `short-native-v1`, `short-all-native-v1`, and `full-v1` with an environment that matches each profile's pinned evaluator revision. @@ -32,12 +40,12 @@ that do not match the selected profile. | Path | Profiles | What it does | | --- | --- | --- | -| Qwen-specific Transformers | `short-native-v1`, `short-native-v2`, `short-all-native-v1`, `short-all-native-v2`, `core3-full-native-v1` | Loads the checkpoint directly with the Qwen 3.5 model loader. Video prompts include timestamps for sampled frames. | -| General vLLM | `short-v1`, `short-vllm-v2`, `smoke-vllm-v1`, `full-v1`, `core3-full-vllm-v1` | Runs the checkpoint through vLLM and converts inputs to general image and video messages. Video prompts do not include frame timestamps. Materialized heterogeneous checkpoints require this path. | +| Qwen-specific Transformers | `short-native-v1`, `core-3_24-examples_r1-native`, `core-3_344-examples_r1-native`, `short-all-native-v1`, `judge-free-8_690-examples_r1-native`, `core-3_full_r1-native` | Loads the checkpoint directly with the Qwen 3.5 model loader. Video prompts include timestamps for sampled frames. | +| General vLLM | `short-v1`, `core-3_344-examples_r1-vllm`, `core-3_24-examples_r1-vllm`, `full-v1`, `core-3_full_r1-vllm` | Runs the checkpoint through vLLM and converts inputs to general image and video messages. Video prompts do not include frame timestamps. Materialized heterogeneous checkpoints require this path. | The native and vLLM paths produce different prompts, so their scores represent the complete paths and do not isolate the inference engine. -The heterogeneous-checkpoint `short-vllm-v2` and `smoke-vllm-v1` profiles pin +The heterogeneous-checkpoint `core-3_344-examples_r1-vllm` and `core-3_24-examples_r1-vllm` profiles pin FlashAttention 2 because the runtime's FlashAttention 3 scheduler does not support their per-layer attention geometry. @@ -45,24 +53,39 @@ not support their per-layer attention geometry. Use a versioned profile when scores will be compared across checkpoints: -| Profile | Coverage | Examples evaluated | -| --- | --- | --- | -| `short-v1` | RealWorldQA, MMMU, and MVBench | 344 predefined examples; MMMU uses four per subject and MVBench uses eight per category | -| `short-native-v1` | Same examples as `short-v1`, using the Qwen-specific Transformers adapter | The same 344 predefined examples with timestamps added to video prompts | -| `short-native-v2` | Fixed coverage of RealWorldQA, MMMU, and MVBench | 64 RealWorldQA rows, four rows from each of 30 MMMU subjects, and eight rows from each of 20 MVBench tasks | -| `short-all-native-v1` | All eight judge-free benchmarks, using the Qwen-specific Transformers adapter | 690 predefined examples: the same 344 plus 346 from five additional benchmarks | -| `short-all-native-v2` | Fixed coverage of all eight judge-free benchmarks | 690 predefined examples distributed across the recorded source strata | -| `core3-full-native-v1` | Native backend on RealWorldQA, MMMU validation, and MVBench | All 765, 900, and 4,000 rows respectively | -| `core3-full-vllm-v1` | vLLM backend on the same three datasets | All 765, 900, and 4,000 rows respectively | -| `full-v1` | Eight judge-free image and video benchmarks | Every available example in each pinned dataset version | - -Use `short-native-v2` for campaign comparisons. Use -`short-all-native-v2` when broader image and video regression coverage is more +The sample-set portion of a current profile name describes the data: + +| Sample set | Benchmarks | Selected examples | Pinned population | +| --- | --- | ---: | ---: | +| `core-3_24-examples_r1` | RealWorldQA, MMMU validation, one MVBench task | 24 | 1,865 | +| `core-3_344-examples_r1` | RealWorldQA, MMMU validation, MVBench | 344 | 5,665 | +| `judge-free-8_690-examples_r1` | All eight judge-free benchmarks | 690 | 31,916 | +| `core-3_full_r1` | RealWorldQA, MMMU validation, MVBench | all | 5,665 | + +Here, `core-3` means the three campaign benchmarks and `judge-free-8` means +the current eight-benchmark set that needs no external judge. The number before +`examples` is the number actually evaluated, not the full dataset population. +`r1` versions the sample-set definition. It does not version the backend or +evaluator. + +Append `-native` or `-vllm` to select a composed runnable profile. For example, +`core-3_344-examples_r1-native` and `core-3_344-examples_r1-vllm` share the +same 344 examples and evaluator, while their backend profiles differ. The 344 +examples are 64 of 765 RealWorldQA examples, 120 of 900 MMMU examples, and 160 +of 4,000 MVBench examples. The 690-example set contains those 344 plus 346 +examples selected from VideoMMMU, Video-MME, LongVideoBench, MLVU, and +PerceptionTest. + +The historical `short-v1`, `short-native-v1`, `short-all-native-v1`, and +`full-v1` names remain available because they pin older evaluator contracts. + +Use `core-3_344-examples_r1-native` for campaign comparisons. Use +`judge-free-8_690-examples_r1-native` when broader image and video regression coverage is more important than matching the campaign screen. The v1 profiles require their pinned evaluator revisions. No short profile replaces a full-data profile for complete benchmark reporting. -The paired `core3-full-*` profiles provide teacher references. They pin the +The paired `core-3_full_r1-*` profiles provide teacher references. They pin the Qwen 3.5 0.8B Hub snapshot and reject other checkpoints or runtime-setting overrides. Keep the two backend results separate: their prompt construction and MVBench frame annotations @@ -83,7 +106,7 @@ treats its model-level `max_new_tokens` value as a lower bound, so the report also records that limitation; backend score differences therefore include generation-policy and prompt-path differences, not just engine behavior. -`short-native-v2` selects +`core-3_344-examples_r1-native` selects 64 positions across all 765 RealWorldQA test rows, four positions within each 30-row MMMU subject, and eight positions within each 200-row MVBench task. Its profile records the generator version, population and stratum counts, selected @@ -92,7 +115,7 @@ These fixed rows provide a regression screen, not a representative full-benchmark estimate. Compare a teacher and every candidate with the same profile. -`short-all-native-v2` adds midpoint samples from +`judge-free-8_690-examples_r1-native` adds fixed, evenly spaced samples from five video benchmarks: 24 rows from each of three VideoMMMU tasks, four rows from each of 18 Video-MME duration-and-domain strata, 68 rows across the LongVideoBench validation split, and 10 rows from each of seven MLVU task @@ -130,8 +153,7 @@ frames. ## Cache benchmark data Evaluation reads every selected dataset revision from an explicit Hugging Face -cache root. The campaign `prepare_dataset` stage fills and validates this cache -from the configured evaluation tasks. The repository stores only profile +cache root. The repository stores only profile metadata and exact-row selectors; it does not store benchmark records or media. To prepare the common image and video cache independently, run: @@ -144,15 +166,16 @@ python -m examples.puzzletron.evaluation.vlm.preparation.benchmark_data \ ``` The command downloads each listed exact pinned snapshot and safely extracts -media only for tasks that declare a preparation directory. For `short-native-v2`, +media only for tasks that declare a preparation directory. For `core-3_344-examples_r1-native`, prepare `realworldqa`, `mmmu_val`, and `mvbench`. Use `--download-only` and `--extract-only` to split transfer and extraction across jobs, or `--range-resume` for a resumable single-writer download. Run the command with `--help` to list all supported dataset task names. Preparation records exact -snapshot and media inventories. Ordinary resume uses file metadata as a fast -path and checks recorded hashes after metadata changes; incomplete owned media -is rebuilt from the pinned snapshot only when the host supports atomic directory -exchange. Otherwise the existing root is preserved and preparation fails. +snapshot and media inventories. Preparation verifies the recorded snapshot and +media hashes on reuse, which can read the complete cached content. Incomplete +owned media is rebuilt from the pinned snapshot only when the host supports +atomic directory exchange. Otherwise the existing root is preserved and +preparation fails. Video suites also require an installed `decord`-compatible reader. The Puzzletron requirements select the supported reader for the current platform @@ -173,20 +196,20 @@ same command with a teacher or any materialized Qwen 3.5 student checkpoint: ```bash python -m examples.puzzletron.evaluation.vlm.run \ --checkpoint /path/to/checkpoint \ - --output-dir /path/to/results/short-all-native-v2 \ + --output-dir /path/to/results/judge-free-8_690-examples_r1-native \ --hf-home "$HF_HOME" \ - --profile short-all-native-v2 + --profile judge-free-8_690-examples_r1-native ``` -The `core3-full-*` profiles instead require the exact pinned local Qwen 3.5 +The `core-3_full_r1-*` profiles instead require the exact pinned local Qwen 3.5 0.8B Hub snapshot. For example: ```bash python -m examples.puzzletron.evaluation.vlm.run \ --checkpoint "$HF_HOME/hub/models--Qwen--Qwen3.5-0.8B/snapshots/2fc06364715b967f1860aea9cf38778875588b17" \ - --output-dir /path/to/results/core3-full-native-v1/realworldqa \ + --output-dir /path/to/results/core-3_full_r1-native/realworldqa \ --hf-home "$HF_HOME" \ - --profile core3-full-native-v1 \ + --profile core-3_full_r1-native \ --profile-task realworldqa \ --preflight-only ``` @@ -196,7 +219,7 @@ Use `full-v1` only with its pinned evaluator revision. Always run To run tasks in parallel, run one profile task per job with `--profile PROFILE --profile-task TASK`. This works for `full-v1`, the paired -`core3-full-*` profiles, and `short-all-native-v2`. A complete result needs one +`core-3_full_r1-*` profiles, and `judge-free-8_690-examples_r1-native`. A complete result needs one successful job for every task in the selected profile. Grouped tasks can use multiple batch-1 workers without changing inference diff --git a/examples/puzzletron/evaluation/vlm/contracts.py b/examples/puzzletron/evaluation/vlm/contracts.py index 9cc831fd1a3..5e410ce93b1 100644 --- a/examples/puzzletron/evaluation/vlm/contracts.py +++ b/examples/puzzletron/evaluation/vlm/contracts.py @@ -33,151 +33,154 @@ "load_profile", ] -_PROFILE_SCHEMA = "modelopt.vlm-evaluation-profile/v1" +_PROFILE_SCHEMA = "modelopt.vlm-evaluation-profile/v2" +_SAMPLE_SET_SCHEMA = "modelopt.vlm-sample-set/v1" +_BACKEND_PROFILE_SCHEMA = "modelopt.vlm-backend-profile/v1" +_EVALUATOR_PROFILE_SCHEMA = "modelopt.vlm-evaluator-profile/v1" _PROFILE_ROOT = Path(__file__).with_name("profiles") PROFILE_NAMES = ( "short-v1", "short-native-v1", - "short-native-v2", - "short-vllm-v2", - "smoke-native-v1", - "smoke-vllm-v1", + "core-3_344-examples_r1-native", + "core-3_344-examples_r1-vllm", + "core-3_24-examples_r1-native", + "core-3_24-examples_r1-vllm", "short-all-native-v1", - "short-all-native-v2", + "judge-free-8_690-examples_r1-native", "full-v1", - "core3-full-native-v1", - "core3-full-vllm-v1", + "core-3_full_r1-native", + "core-3_full_r1-vllm", ) -_PROFILE_TASKS = { - "short-v1": ("realworldqa", "mmmu_val", "mvbench"), - "short-native-v1": ("realworldqa", "mmmu_val", "mvbench"), - "short-native-v2": ("realworldqa", "mmmu_val", "mvbench"), - "short-vllm-v2": ("realworldqa", "mmmu_val", "mvbench"), - "smoke-native-v1": ("realworldqa", "mmmu_val", "mvbench"), - "smoke-vllm-v1": ("realworldqa", "mmmu_val", "mvbench"), +_PROFILE_COMPONENTS = { + "short-v1": ( + "core-3_344-examples_legacy-r1", + "qwen-3.5-vllm_r1", + "lmms-eval-legacy_r1", + ), + "short-native-v1": ( + "core-3_344-examples_legacy-r1", + "qwen-3.5-native_r1", + "lmms-eval-qwen-3.5-native_r1", + ), + "core-3_344-examples_r1-native": ( + "core-3_344-examples_r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", + ), + "core-3_344-examples_r1-vllm": ( + "core-3_344-examples_r1", + "anymodel-vllm_r1", + "lmms-eval-modelopt_r1", + ), + "core-3_24-examples_r1-native": ( + "core-3_24-examples_r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", + ), + "core-3_24-examples_r1-vllm": ( + "core-3_24-examples_r1", + "anymodel-vllm-eager_r1", + "lmms-eval-modelopt_r1", + ), "short-all-native-v1": ( - "realworldqa", - "mmmu_val", - "mvbench", - "video_mmmu", - "videomme", - "longvideobench_val_v", - "mlvu_dev", - "perceptiontest_val_mc", + "judge-free-8_690-examples_legacy-r1", + "qwen-3.5-native_r1", + "lmms-eval-qwen-3.5-native_r1", ), - "short-all-native-v2": ( - "realworldqa", - "mmmu_val", - "mvbench", - "video_mmmu", - "videomme", - "longvideobench_val_v", - "mlvu_dev", - "perceptiontest_val_mc", + "judge-free-8_690-examples_r1-native": ( + "judge-free-8_690-examples_r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", + ), + "full-v1": ( + "judge-free-8_full_legacy-r1", + "qwen-3.5-vllm_r1", + "lmms-eval-legacy_r1", + ), + "core-3_full_r1-native": ( + "core-3_full_r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", + ), + "core-3_full_r1-vllm": ( + "core-3_full_r1", + "qwen-3.5-vllm_r1", + "lmms-eval-modelopt_r1", ), - "full-v1": tuple(task for task in profile.VLM_BENCHMARK_TASKS if task != "mmvu_val"), - "core3-full-native-v1": ("realworldqa", "mmmu_val", "mvbench"), - "core3-full-vllm-v1": ("realworldqa", "mmmu_val", "mvbench"), } -_PROFILE_SELECTIONS = { - "short-v1": "exact-rows", - "short-native-v1": "exact-rows", - "short-native-v2": "exact-rows", - "short-vllm-v2": "exact-rows", - "smoke-native-v1": "exact-rows", - "smoke-vllm-v1": "exact-rows", - "short-all-native-v1": "exact-rows", - "short-all-native-v2": "exact-rows", - "full-v1": "all", - "core3-full-native-v1": "all", - "core3-full-vllm-v1": "all", +_CORE_3_TASKS = ("realworldqa", "mmmu_val", "mvbench") +_JUDGE_FREE_8_TASKS = ( + "realworldqa", + "mmmu_val", + "mvbench", + "video_mmmu", + "videomme", + "longvideobench_val_v", + "mlvu_dev", + "perceptiontest_val_mc", +) +_SAMPLE_SET_TASKS = { + "core-3_344-examples_legacy-r1": _CORE_3_TASKS, + "core-3_344-examples_r1": _CORE_3_TASKS, + "core-3_24-examples_r1": _CORE_3_TASKS, + "judge-free-8_690-examples_legacy-r1": _JUDGE_FREE_8_TASKS, + "judge-free-8_690-examples_r1": _JUDGE_FREE_8_TASKS, + "judge-free-8_full_legacy-r1": tuple( + task for task in profile.VLM_BENCHMARK_TASKS if task != "mmvu_val" + ), + "core-3_full_r1": _CORE_3_TASKS, +} +_SAMPLE_SET_SELECTIONS = { + "core-3_344-examples_legacy-r1": "exact-rows", + "core-3_344-examples_r1": "exact-rows", + "core-3_24-examples_r1": "exact-rows", + "judge-free-8_690-examples_legacy-r1": "exact-rows", + "judge-free-8_690-examples_r1": "exact-rows", + "judge-free-8_full_legacy-r1": "all", + "core-3_full_r1": "all", } SHORT_PROFILE_NAMES = tuple( - name for name in PROFILE_NAMES if _PROFILE_SELECTIONS[name] == "exact-rows" + name + for name in PROFILE_NAMES + if _SAMPLE_SET_SELECTIONS[_PROFILE_COMPONENTS[name][0]] == "exact-rows" ) -_PROFILE_BACKENDS = { - "short-v1": { +_BACKEND_SETTINGS = { + "qwen-3.5-vllm_r1": { "enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3", }, - "short-native-v1": { - "attention_implementation": "sdpa", - "enable_thinking": False, - "name": "qwen3_5", - }, - "short-native-v2": { + "qwen-3.5-native_r1": { "attention_implementation": "sdpa", "enable_thinking": False, "name": "qwen3_5", }, - "short-vllm-v2": { + "anymodel-vllm_r1": { "attention_config": {"flash_attn_version": 2}, "enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3", }, - "smoke-native-v1": { - "attention_implementation": "sdpa", - "enable_thinking": False, - "name": "qwen3_5", - }, - "smoke-vllm-v1": { + "anymodel-vllm-eager_r1": { "attention_config": {"flash_attn_version": 2}, "enable_thinking": False, "enforce_eager": True, "name": "vllm", "reasoning_parser": "qwen3", }, - "short-all-native-v1": { - "attention_implementation": "sdpa", - "enable_thinking": False, - "name": "qwen3_5", - }, - "short-all-native-v2": { - "attention_implementation": "sdpa", - "enable_thinking": False, - "name": "qwen3_5", - }, - "full-v1": { - "enable_thinking": False, - "name": "vllm", - "reasoning_parser": "qwen3", - }, - "core3-full-native-v1": { - "attention_implementation": "sdpa", - "enable_thinking": False, - "name": "qwen3_5", - }, - "core3-full-vllm-v1": { - "enable_thinking": False, - "name": "vllm", - "reasoning_parser": "qwen3", - }, } -_PROFILE_REVISIONS = { - "short-v1": checkpoint.LMMS_EVAL_LEGACY_REVISION, - "short-native-v1": checkpoint.LMMS_EVAL_QWEN35_NATIVE_REVISION, - "short-native-v2": checkpoint.LMMS_EVAL_REVISION, - "short-vllm-v2": checkpoint.LMMS_EVAL_REVISION, - "smoke-native-v1": checkpoint.LMMS_EVAL_REVISION, - "smoke-vllm-v1": checkpoint.LMMS_EVAL_REVISION, - "short-all-native-v1": checkpoint.LMMS_EVAL_QWEN35_NATIVE_REVISION, - "short-all-native-v2": checkpoint.LMMS_EVAL_REVISION, - "full-v1": checkpoint.LMMS_EVAL_LEGACY_REVISION, - "core3-full-native-v1": checkpoint.LMMS_EVAL_REVISION, - "core3-full-vllm-v1": checkpoint.LMMS_EVAL_REVISION, +_EVALUATOR_REVISIONS = { + "lmms-eval-legacy_r1": checkpoint.LMMS_EVAL_LEGACY_REVISION, + "lmms-eval-qwen-3.5-native_r1": checkpoint.LMMS_EVAL_QWEN35_NATIVE_REVISION, + "lmms-eval-modelopt_r1": checkpoint.LMMS_EVAL_REVISION, } -_PROFILE_MODELS = { - name: { +_SAMPLE_SET_MODELS = { + "core-3_full_r1": { "repository": "Qwen/Qwen3.5-0.8B", "revision": "2fc06364715b967f1860aea9cf38778875588b17", } - for name in ("core3-full-native-v1", "core3-full-vllm-v1") -} -_PROFILE_POPULATIONS = { - name: {"realworldqa": 765, "mmmu_val": 900, "mvbench": 4000} for name in _PROFILE_MODELS } +_SAMPLE_SET_POPULATIONS = {"core-3_full_r1": {"realworldqa": 765, "mmmu_val": 900, "mvbench": 4000}} _MVBENCH_LEAF_POPULATIONS = { "action_sequence": 200, "moving_count": 200, @@ -200,13 +203,11 @@ "moving_attribute": 200, "egocentric_navigation": 200, } -_AUDITED_SAMPLING_PROFILES = frozenset( +_AUDITED_SAMPLE_SETS = frozenset( { - "smoke-native-v1", - "smoke-vllm-v1", - "short-native-v2", - "short-vllm-v2", - "short-all-native-v2", + "core-3_24-examples_r1", + "core-3_344-examples_r1", + "judge-free-8_690-examples_r1", } ) _SAMPLING_AUDIT_SCHEMA = "modelopt.vlm-sampling-audit/v1" @@ -231,6 +232,21 @@ class ProfileContract: manifest: dict[str, object] fingerprint: str + @property + def sample_set(self) -> str: + """Return the selected example-set contract name.""" + return cast("str", self.manifest["sample_set"]) + + @property + def backend_profile(self) -> str: + """Return the selected model-backend contract name.""" + return cast("str", self.manifest["backend_profile"]) + + @property + def evaluator_profile(self) -> str: + """Return the selected evaluator contract name.""" + return cast("str", self.manifest["evaluator_profile"]) + @property def source_tasks(self) -> tuple[str, ...]: """Return benchmark tasks in their declared evaluation order.""" @@ -264,13 +280,49 @@ def load_profile(name: str) -> ProfileContract: """Load a named profile after validating every executable pin.""" if name not in PROFILE_NAMES: raise ValueError(f"unsupported VLM evaluation profile: {name}") - path = _PROFILE_ROOT / f"{name}.json" - try: - manifest = _resolve_manifest(name, json.loads(path.read_text())) - except (OSError, json.JSONDecodeError) as error: - raise RuntimeError(f"VLM evaluation profile is unreadable: {path}") from error - if not isinstance(manifest, dict): - raise RuntimeError(f"VLM evaluation profile must contain an object: {path}") + composition = _load_component(_PROFILE_ROOT, name, _PROFILE_SCHEMA, "profile") + expected_components = _PROFILE_COMPONENTS[name] + observed_components = tuple( + composition.get(key) for key in ("sample_set", "backend_profile", "evaluator_profile") + ) + if observed_components != expected_components: + raise RuntimeError(f"{name} profile composition differs from the runtime policy") + + sample_set_name, backend_name, evaluator_name = expected_components + sample_set = _resolve_sample_set(sample_set_name) + backend = _load_component( + _PROFILE_ROOT / "backends", backend_name, _BACKEND_PROFILE_SCHEMA, "backend profile" + ) + evaluator = _load_component( + _PROFILE_ROOT / "evaluators", + evaluator_name, + _EVALUATOR_PROFILE_SCHEMA, + "evaluator profile", + ) + if backend.get("settings") != _BACKEND_SETTINGS[backend_name]: + raise RuntimeError(f"{backend_name} backend profile differs from the runtime policy") + if evaluator.get("lmms_eval_revision") != _EVALUATOR_REVISIONS[evaluator_name]: + raise RuntimeError(f"{evaluator_name} evaluator profile differs from the runtime pin") + + manifest = { + **composition, + "lmms_eval_revision": evaluator["lmms_eval_revision"], + "model_family": { + "architecture": "Qwen3_5ForConditionalGeneration", + "model_type": "qwen3_5", + }, + "backend": backend["settings"], + "preprocessing": {"fps": 2, "max_frames": 32, "video_reader": "decord"}, + "generation": {"do_sample": False, "temperature": 0}, + "seed": 42, + "repetitions": 1, + "batch_size": 1, + **{ + key: value + for key, value in sample_set.items() + if key not in {"schema", "name", "extends"} + }, + } _validate_manifest(name, manifest) canonical = json.dumps(manifest, separators=(",", ":"), sort_keys=True).encode() return ProfileContract( @@ -280,30 +332,45 @@ def load_profile(name: str) -> ProfileContract: ) -def _resolve_manifest(name: str, manifest: object, *, ancestors: tuple[str, ...] = ()) -> object: - """Resolve profile inheritance while rejecting cycles.""" - if not isinstance(manifest, dict) or "extends" not in manifest: - return manifest - base_name = manifest.get("extends") +def _load_component(root: Path, name: str, schema: str, label: str) -> dict[str, object]: + """Load one named component and validate its identity envelope.""" + path = root / f"{name}.json" + try: + component = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"VLM evaluation {label} is unreadable: {path}") from error + if not isinstance(component, dict): + raise RuntimeError(f"VLM evaluation {label} must contain an object: {path}") + if component.get("schema") != schema: + raise RuntimeError(f"{name} {label} schema must be {schema}") + if component.get("name") != name: + raise RuntimeError(f"{name} {label} name does not match its filename") + return component + + +def _resolve_sample_set(name: str, *, ancestors: tuple[str, ...] = ()) -> dict[str, object]: + """Resolve sample-set inheritance while rejecting cycles.""" + if name not in _SAMPLE_SET_TASKS: + raise RuntimeError(f"unsupported VLM evaluation sample set: {name}") + sample_set = _load_component( + _PROFILE_ROOT / "sample_sets", name, _SAMPLE_SET_SCHEMA, "sample set" + ) + if "extends" not in sample_set: + return sample_set + base_name = sample_set.get("extends") if ( not isinstance(base_name, str) - or base_name not in PROFILE_NAMES + or base_name not in _SAMPLE_SET_TASKS or base_name == name or base_name in ancestors ): - raise RuntimeError(f"{name} profile extends an unsupported base profile") - base_path = _PROFILE_ROOT / f"{base_name}.json" - try: - base = json.loads(base_path.read_text()) - except (OSError, json.JSONDecodeError) as error: - raise RuntimeError(f"VLM evaluation base profile is unreadable: {base_path}") from error - base = _resolve_manifest(base_name, base, ancestors=(*ancestors, name)) - if not isinstance(base, dict): - raise RuntimeError(f"{name} profile base must contain an object") - overrides = {key: value for key, value in manifest.items() if key != "extends"} + raise RuntimeError(f"{name} sample set extends an unsupported base sample set") + base = _resolve_sample_set(base_name, ancestors=(*ancestors, name)) + overrides = {key: value for key, value in sample_set.items() if key != "extends"} if isinstance(base.get("tasks"), dict) and isinstance(overrides.get("tasks"), dict): - tasks = dict(base["tasks"]) - for task, entry in overrides["tasks"].items(): + tasks = dict(cast("dict[str, object]", base["tasks"])) + override_tasks = cast("dict[str, object]", overrides["tasks"]) + for task, entry in override_tasks.items(): base_entry = tasks.get(task) tasks[task] = ( {**base_entry, **entry} @@ -319,17 +386,24 @@ def _validate_manifest(name: str, manifest: dict[str, object]) -> None: raise RuntimeError(f"{name} profile schema must be {_PROFILE_SCHEMA}") if manifest.get("name") != name: raise RuntimeError(f"{name} profile name does not match its filename") - if manifest.get("lmms_eval_revision") != _PROFILE_REVISIONS[name]: - raise RuntimeError(f"{name} profile lmms_eval_revision differs from the runtime pin") + sample_set_name, backend_name, evaluator_name = _PROFILE_COMPONENTS[name] + if manifest.get("sample_set") != sample_set_name: + raise RuntimeError(f"{name} profile sample set differs from its composition") + if manifest.get("backend_profile") != backend_name: + raise RuntimeError(f"{name} profile backend differs from its composition") + if manifest.get("evaluator_profile") != evaluator_name: + raise RuntimeError(f"{name} profile evaluator differs from its composition") + if manifest.get("lmms_eval_revision") != _EVALUATOR_REVISIONS[evaluator_name]: + raise RuntimeError(f"{name} profile lmms_eval_revision differs from the evaluator pin") if manifest.get("model_family") != { "architecture": "Qwen3_5ForConditionalGeneration", "model_type": "qwen3_5", }: raise RuntimeError(f"{name} profile model family is unsupported") - expected_model = _PROFILE_MODELS.get(name) + expected_model = _SAMPLE_SET_MODELS.get(sample_set_name) if expected_model is not None and manifest.get("model") != expected_model: raise RuntimeError(f"{name} profile model pin differs from the runtime policy") - if manifest.get("backend") != _PROFILE_BACKENDS[name]: + if manifest.get("backend") != _BACKEND_SETTINGS[backend_name]: raise RuntimeError(f"{name} profile backend differs from the runtime policy") if manifest.get("preprocessing") != { "fps": 2, @@ -346,16 +420,16 @@ def _validate_manifest(name: str, manifest: dict[str, object]) -> None: ): raise RuntimeError(f"{name} profile execution identity differs from the runtime policy") selection = manifest.get("selection") - if selection != _PROFILE_SELECTIONS[name]: + if selection != _SAMPLE_SET_SELECTIONS[sample_set_name]: raise RuntimeError(f"{name} profile selection differs from its versioned policy") tasks = manifest.get("tasks") if not isinstance(tasks, dict) or not tasks: raise RuntimeError(f"{name} profile tasks must contain an object") - if tuple(tasks) != _PROFILE_TASKS[name]: + if tuple(tasks) != _SAMPLE_SET_TASKS[sample_set_name]: raise RuntimeError(f"{name} profile tasks differ from its versioned policy") for task, entry in tasks.items(): - _validate_task(name, task, entry, selection=cast("str", selection)) - _validate_sampling(name, manifest, tasks) + _validate_task(sample_set_name, task, entry, selection=cast("str", selection)) + _validate_sampling(sample_set_name, manifest, tasks) def _validate_task(name: str, task: object, entry: object, *, selection: str) -> None: @@ -373,7 +447,7 @@ def _validate_task(name: str, task: object, entry: object, *, selection: str) -> observed = {key: entry.get(key) for key in expected} if observed != expected: raise RuntimeError(f"{name} profile task pins differ from the runtime catalog: {task}") - expected_population = _PROFILE_POPULATIONS.get(name, {}).get(task) + expected_population = _SAMPLE_SET_POPULATIONS.get(name, {}).get(task) if expected_population is not None and entry.get("population_rows") != expected_population: raise RuntimeError( f"{name} profile task population differs from its versioned policy: {task}" @@ -396,7 +470,7 @@ def _validate_sampling( tasks: dict[str, object], ) -> None: sampling = manifest.get("sampling") - if name in _AUDITED_SAMPLING_PROFILES and not isinstance(sampling, dict): + if name in _AUDITED_SAMPLE_SETS and not isinstance(sampling, dict): raise RuntimeError(f"{name} profile must contain a sampling audit") if sampling is None: return diff --git a/examples/puzzletron/evaluation/vlm/post_mip.py b/examples/puzzletron/evaluation/vlm/post_mip.py index 9cbca2be61c..e06b18ebd21 100644 --- a/examples/puzzletron/evaluation/vlm/post_mip.py +++ b/examples/puzzletron/evaluation/vlm/post_mip.py @@ -240,7 +240,7 @@ def evaluate_frozen_campaign_v2_checkpoint( output_root=output_root, settings=settings, suite="short", - evaluation_profile="short-native-v2", + evaluation_profile="core-3_344-examples_r1-native", require_manifest=True, ) runs = result["runs"] @@ -266,7 +266,7 @@ def evaluate_frozen_campaign_v3_checkpoint( output_root=output_root, settings=settings, suite="short", - evaluation_profile="short-vllm-v2", + evaluation_profile="core-3_344-examples_r1-vllm", require_manifest=True, ) runs = result["runs"] @@ -292,7 +292,7 @@ def evaluate_reproducibility_smoke_checkpoint( output_root=output_root, settings=settings, suite="short", - evaluation_profile="smoke-native-v1", + evaluation_profile="core-3_24-examples_r1-native", require_manifest=True, ) runs = result["runs"] @@ -318,7 +318,7 @@ def evaluate_reproducibility_smoke_v2_checkpoint( output_root=output_root, settings=settings, suite="short", - evaluation_profile="smoke-vllm-v1", + evaluation_profile="core-3_24-examples_r1-vllm", require_manifest=True, ) runs = result["runs"] diff --git a/examples/puzzletron/evaluation/vlm/preflight.py b/examples/puzzletron/evaluation/vlm/preflight.py index 3f0e516fa85..6542fc4f6e3 100644 --- a/examples/puzzletron/evaluation/vlm/preflight.py +++ b/examples/puzzletron/evaluation/vlm/preflight.py @@ -176,10 +176,10 @@ def _resolve_task_selection( raise ValueError("--profile-task requires a versioned evaluation profile") if profile_contract.name not in { "full-v1", - "core3-full-native-v1", - "core3-full-vllm-v1", + "core-3_full_r1-native", + "core-3_full_r1-vllm", "short-all-native-v1", - "short-all-native-v2", + "judge-free-8_690-examples_r1-native", }: raise ValueError( "--profile-task is supported only for full-data and short-all-native profiles" @@ -424,6 +424,13 @@ def _report( "profile_fingerprint": ( profile_contract.fingerprint if profile_contract is not None else None ), + "sample_set": profile_contract.sample_set if profile_contract is not None else None, + "backend_profile": ( + profile_contract.backend_profile if profile_contract is not None else None + ), + "evaluator_profile": ( + profile_contract.evaluator_profile if profile_contract is not None else None + ), "model_pin": ( profile_contract.manifest.get("model") if profile_contract is not None else None ), diff --git a/examples/puzzletron/evaluation/vlm/profiles/smoke-vllm-v1.json b/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm-eager_r1.json similarity index 59% rename from examples/puzzletron/evaluation/vlm/profiles/smoke-vllm-v1.json rename to examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm-eager_r1.json index 3af583918be..05eab6d9138 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/smoke-vllm-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm-eager_r1.json @@ -1,8 +1,7 @@ { - "extends": "smoke-native-v1", - "schema": "modelopt.vlm-evaluation-profile/v1", - "name": "smoke-vllm-v1", - "backend": { + "schema": "modelopt.vlm-backend-profile/v1", + "name": "anymodel-vllm-eager_r1", + "settings": { "attention_config": { "flash_attn_version": 2 }, diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-vllm-v2.json b/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm_r1.json similarity index 55% rename from examples/puzzletron/evaluation/vlm/profiles/short-vllm-v2.json rename to examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm_r1.json index ec5258ea832..c7d24f57363 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-vllm-v2.json +++ b/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm_r1.json @@ -1,8 +1,7 @@ { - "extends": "short-native-v2", - "schema": "modelopt.vlm-evaluation-profile/v1", - "name": "short-vllm-v2", - "backend": { + "schema": "modelopt.vlm-backend-profile/v1", + "name": "anymodel-vllm_r1", + "settings": { "attention_config": { "flash_attn_version": 2 }, diff --git a/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-native_r1.json b/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-native_r1.json new file mode 100644 index 00000000000..c1b9d493473 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-native_r1.json @@ -0,0 +1,9 @@ +{ + "schema": "modelopt.vlm-backend-profile/v1", + "name": "qwen-3.5-native_r1", + "settings": { + "attention_implementation": "sdpa", + "enable_thinking": false, + "name": "qwen3_5" + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-vllm_r1.json b/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-vllm_r1.json new file mode 100644 index 00000000000..d18c494ab7e --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-vllm_r1.json @@ -0,0 +1,9 @@ +{ + "schema": "modelopt.vlm-backend-profile/v1", + "name": "qwen-3.5-vllm_r1", + "settings": { + "enable_thinking": false, + "name": "vllm", + "reasoning_parser": "qwen3" + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-native.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-native.json new file mode 100644 index 00000000000..5803071aa53 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-native.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_24-examples_r1-native", + "sample_set": "core-3_24-examples_r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-vllm.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-vllm.json new file mode 100644 index 00000000000..5f56da0e5dd --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-vllm.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_24-examples_r1-vllm", + "sample_set": "core-3_24-examples_r1", + "backend_profile": "anymodel-vllm-eager_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-native.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-native.json new file mode 100644 index 00000000000..6df08f73487 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-native.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_344-examples_r1-native", + "sample_set": "core-3_344-examples_r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-vllm.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-vllm.json new file mode 100644 index 00000000000..eceb3eaccdd --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-vllm.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_344-examples_r1-vllm", + "sample_set": "core-3_344-examples_r1", + "backend_profile": "anymodel-vllm_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-native.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-native.json new file mode 100644 index 00000000000..bfe5219df26 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-native.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_full_r1-native", + "sample_set": "core-3_full_r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-vllm.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-vllm.json new file mode 100644 index 00000000000..785e6512df2 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-vllm.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_full_r1-vllm", + "sample_set": "core-3_full_r1", + "backend_profile": "qwen-3.5-vllm_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core3-full-vllm-v1.json b/examples/puzzletron/evaluation/vlm/profiles/core3-full-vllm-v1.json deleted file mode 100644 index fa414706004..00000000000 --- a/examples/puzzletron/evaluation/vlm/profiles/core3-full-vllm-v1.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "core3-full-native-v1", - "schema": "modelopt.vlm-evaluation-profile/v1", - "name": "core3-full-vllm-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", - "backend": { - "enable_thinking": false, - "name": "vllm", - "reasoning_parser": "qwen3" - } -} diff --git a/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-legacy_r1.json b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-legacy_r1.json new file mode 100644 index 00000000000..19890381d7b --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-legacy_r1.json @@ -0,0 +1,5 @@ +{ + "schema": "modelopt.vlm-evaluator-profile/v1", + "name": "lmms-eval-legacy_r1", + "lmms_eval_revision": "15c32bfec165df13c269ddd3cda03b2ed9137825" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-modelopt_r1.json b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-modelopt_r1.json new file mode 100644 index 00000000000..a563853acda --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-modelopt_r1.json @@ -0,0 +1,5 @@ +{ + "schema": "modelopt.vlm-evaluator-profile/v1", + "name": "lmms-eval-modelopt_r1", + "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-qwen-3.5-native_r1.json b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-qwen-3.5-native_r1.json new file mode 100644 index 00000000000..3b59b917cdb --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-qwen-3.5-native_r1.json @@ -0,0 +1,5 @@ +{ + "schema": "modelopt.vlm-evaluator-profile/v1", + "name": "lmms-eval-qwen-3.5-native_r1", + "lmms_eval_revision": "88b23e2bfa16a1edbc16e9e238ed82130b3a4f56" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/full-v1.json b/examples/puzzletron/evaluation/vlm/profiles/full-v1.json index dae15ce25bf..443debb0fa5 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/full-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/full-v1.json @@ -1,77 +1,7 @@ { - "schema": "modelopt.vlm-evaluation-profile/v1", + "schema": "modelopt.vlm-evaluation-profile/v2", "name": "full-v1", - "lmms_eval_revision": "15c32bfec165df13c269ddd3cda03b2ed9137825", - "model_family": { - "architecture": "Qwen3_5ForConditionalGeneration", - "model_type": "qwen3_5" - }, - "backend": { - "enable_thinking": false, - "name": "vllm", - "reasoning_parser": "qwen3" - }, - "preprocessing": { - "fps": 2, - "max_frames": 32, - "video_reader": "decord" - }, - "generation": { - "do_sample": false, - "temperature": 0 - }, - "seed": 42, - "repetitions": 1, - "batch_size": 1, - "selection": "all", - "tasks": { - "realworldqa": { - "dataset_repository": "lmms-lab/RealWorldQA", - "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", - "max_new_tokens": 16, - "scoring_task_config": "tasks/realworldqa/realworldqa.yaml" - }, - "mmmu_val": { - "dataset_repository": "lmms-lab/MMMU", - "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", - "max_new_tokens": 128, - "scoring_task_config": "tasks/mmmu/mmmu_val.yaml" - }, - "video_mmmu": { - "dataset_repository": "lmms-lab/VideoMMMU", - "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", - "max_new_tokens": 1024, - "scoring_task_config": "tasks/videommmu/video_mmmu.yaml" - }, - "mvbench": { - "dataset_repository": "OpenGVLab/MVBench", - "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", - "max_new_tokens": 16, - "scoring_task_config": "tasks/mvbench/mvbench.yaml" - }, - "videomme": { - "dataset_repository": "lmms-lab/Video-MME", - "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", - "max_new_tokens": 16, - "scoring_task_config": "tasks/videomme/videomme.yaml" - }, - "longvideobench_val_v": { - "dataset_repository": "longvideobench/LongVideoBench", - "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", - "max_new_tokens": 32, - "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml" - }, - "mlvu_dev": { - "dataset_repository": "sy1998/MLVU_dev", - "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", - "max_new_tokens": 16, - "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml" - }, - "perceptiontest_val_mc": { - "dataset_repository": "lmms-lab/PerceptionTest_Val", - "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", - "max_new_tokens": 16, - "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml" - } - } + "sample_set": "judge-free-8_full_legacy-r1", + "backend_profile": "qwen-3.5-vllm_r1", + "evaluator_profile": "lmms-eval-legacy_r1" } diff --git a/examples/puzzletron/evaluation/vlm/profiles/judge-free-8_690-examples_r1-native.json b/examples/puzzletron/evaluation/vlm/profiles/judge-free-8_690-examples_r1-native.json new file mode 100644 index 00000000000..81a03c06544 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/judge-free-8_690-examples_r1-native.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "judge-free-8_690-examples_r1-native", + "sample_set": "judge-free-8_690-examples_r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/smoke-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_24-examples_r1.json similarity index 98% rename from examples/puzzletron/evaluation/vlm/profiles/smoke-native-v1.json rename to examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_24-examples_r1.json index ee780042956..b234f3b6449 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/smoke-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_24-examples_r1.json @@ -1,7 +1,7 @@ { - "extends": "short-native-v2", - "schema": "modelopt.vlm-evaluation-profile/v1", - "name": "smoke-native-v1", + "extends": "core-3_344-examples_r1", + "schema": "modelopt.vlm-sample-set/v1", + "name": "core-3_24-examples_r1", "tasks": { "realworldqa": { "selection": { diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_legacy-r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_legacy-r1.json new file mode 100644 index 00000000000..5a940cf34a7 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_legacy-r1.json @@ -0,0 +1,1567 @@ +{ + "schema": "modelopt.vlm-sample-set/v1", + "name": "core-3_344-examples_legacy-r1", + "selection": "exact-rows", + "tasks": { + "realworldqa": { + "dataset_repository": "lmms-lab/RealWorldQA", + "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", + "max_new_tokens": 16, + "scoring_task_config": "tasks/realworldqa/realworldqa.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "test:0" + }, + { + "source_row_index": 1, + "source_sample_id": "test:1" + }, + { + "source_row_index": 2, + "source_sample_id": "test:2" + }, + { + "source_row_index": 3, + "source_sample_id": "test:3" + }, + { + "source_row_index": 4, + "source_sample_id": "test:4" + }, + { + "source_row_index": 5, + "source_sample_id": "test:5" + }, + { + "source_row_index": 6, + "source_sample_id": "test:6" + }, + { + "source_row_index": 7, + "source_sample_id": "test:7" + }, + { + "source_row_index": 8, + "source_sample_id": "test:8" + }, + { + "source_row_index": 9, + "source_sample_id": "test:9" + }, + { + "source_row_index": 10, + "source_sample_id": "test:10" + }, + { + "source_row_index": 11, + "source_sample_id": "test:11" + }, + { + "source_row_index": 12, + "source_sample_id": "test:12" + }, + { + "source_row_index": 13, + "source_sample_id": "test:13" + }, + { + "source_row_index": 14, + "source_sample_id": "test:14" + }, + { + "source_row_index": 15, + "source_sample_id": "test:15" + }, + { + "source_row_index": 16, + "source_sample_id": "test:16" + }, + { + "source_row_index": 17, + "source_sample_id": "test:17" + }, + { + "source_row_index": 18, + "source_sample_id": "test:18" + }, + { + "source_row_index": 19, + "source_sample_id": "test:19" + }, + { + "source_row_index": 20, + "source_sample_id": "test:20" + }, + { + "source_row_index": 21, + "source_sample_id": "test:21" + }, + { + "source_row_index": 22, + "source_sample_id": "test:22" + }, + { + "source_row_index": 23, + "source_sample_id": "test:23" + }, + { + "source_row_index": 24, + "source_sample_id": "test:24" + }, + { + "source_row_index": 25, + "source_sample_id": "test:25" + }, + { + "source_row_index": 26, + "source_sample_id": "test:26" + }, + { + "source_row_index": 27, + "source_sample_id": "test:27" + }, + { + "source_row_index": 28, + "source_sample_id": "test:28" + }, + { + "source_row_index": 29, + "source_sample_id": "test:29" + }, + { + "source_row_index": 30, + "source_sample_id": "test:30" + }, + { + "source_row_index": 31, + "source_sample_id": "test:31" + }, + { + "source_row_index": 32, + "source_sample_id": "test:32" + }, + { + "source_row_index": 33, + "source_sample_id": "test:33" + }, + { + "source_row_index": 34, + "source_sample_id": "test:34" + }, + { + "source_row_index": 35, + "source_sample_id": "test:35" + }, + { + "source_row_index": 36, + "source_sample_id": "test:36" + }, + { + "source_row_index": 37, + "source_sample_id": "test:37" + }, + { + "source_row_index": 38, + "source_sample_id": "test:38" + }, + { + "source_row_index": 39, + "source_sample_id": "test:39" + }, + { + "source_row_index": 40, + "source_sample_id": "test:40" + }, + { + "source_row_index": 41, + "source_sample_id": "test:41" + }, + { + "source_row_index": 42, + "source_sample_id": "test:42" + }, + { + "source_row_index": 43, + "source_sample_id": "test:43" + }, + { + "source_row_index": 44, + "source_sample_id": "test:44" + }, + { + "source_row_index": 45, + "source_sample_id": "test:45" + }, + { + "source_row_index": 46, + "source_sample_id": "test:46" + }, + { + "source_row_index": 47, + "source_sample_id": "test:47" + }, + { + "source_row_index": 48, + "source_sample_id": "test:48" + }, + { + "source_row_index": 49, + "source_sample_id": "test:49" + }, + { + "source_row_index": 50, + "source_sample_id": "test:50" + }, + { + "source_row_index": 51, + "source_sample_id": "test:51" + }, + { + "source_row_index": 52, + "source_sample_id": "test:52" + }, + { + "source_row_index": 53, + "source_sample_id": "test:53" + }, + { + "source_row_index": 54, + "source_sample_id": "test:54" + }, + { + "source_row_index": 55, + "source_sample_id": "test:55" + }, + { + "source_row_index": 56, + "source_sample_id": "test:56" + }, + { + "source_row_index": 57, + "source_sample_id": "test:57" + }, + { + "source_row_index": 58, + "source_sample_id": "test:58" + }, + { + "source_row_index": 59, + "source_sample_id": "test:59" + }, + { + "source_row_index": 60, + "source_sample_id": "test:60" + }, + { + "source_row_index": 61, + "source_sample_id": "test:61" + }, + { + "source_row_index": 62, + "source_sample_id": "test:62" + }, + { + "source_row_index": 63, + "source_sample_id": "test:63" + } + ] + }, + "mmmu_val": { + "dataset_repository": "lmms-lab/MMMU", + "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", + "max_new_tokens": 128, + "scoring_task_config": "tasks/mmmu/mmmu_val.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "validation_Accounting_1" + }, + { + "source_row_index": 7, + "source_sample_id": "validation_Accounting_8" + }, + { + "source_row_index": 15, + "source_sample_id": "validation_Accounting_16" + }, + { + "source_row_index": 22, + "source_sample_id": "validation_Accounting_23" + }, + { + "source_row_index": 30, + "source_sample_id": "validation_Agriculture_1" + }, + { + "source_row_index": 37, + "source_sample_id": "validation_Agriculture_8" + }, + { + "source_row_index": 45, + "source_sample_id": "validation_Agriculture_16" + }, + { + "source_row_index": 52, + "source_sample_id": "validation_Agriculture_23" + }, + { + "source_row_index": 60, + "source_sample_id": "validation_Architecture_and_Engineering_1" + }, + { + "source_row_index": 67, + "source_sample_id": "validation_Architecture_and_Engineering_8" + }, + { + "source_row_index": 75, + "source_sample_id": "validation_Architecture_and_Engineering_16" + }, + { + "source_row_index": 82, + "source_sample_id": "validation_Architecture_and_Engineering_23" + }, + { + "source_row_index": 90, + "source_sample_id": "validation_Art_1" + }, + { + "source_row_index": 97, + "source_sample_id": "validation_Art_8" + }, + { + "source_row_index": 105, + "source_sample_id": "validation_Art_16" + }, + { + "source_row_index": 112, + "source_sample_id": "validation_Art_23" + }, + { + "source_row_index": 120, + "source_sample_id": "validation_Art_Theory_1" + }, + { + "source_row_index": 127, + "source_sample_id": "validation_Art_Theory_8" + }, + { + "source_row_index": 135, + "source_sample_id": "validation_Art_Theory_16" + }, + { + "source_row_index": 142, + "source_sample_id": "validation_Art_Theory_23" + }, + { + "source_row_index": 150, + "source_sample_id": "validation_Basic_Medical_Science_1" + }, + { + "source_row_index": 157, + "source_sample_id": "validation_Basic_Medical_Science_8" + }, + { + "source_row_index": 165, + "source_sample_id": "validation_Basic_Medical_Science_16" + }, + { + "source_row_index": 172, + "source_sample_id": "validation_Basic_Medical_Science_23" + }, + { + "source_row_index": 180, + "source_sample_id": "validation_Biology_1" + }, + { + "source_row_index": 187, + "source_sample_id": "validation_Biology_8" + }, + { + "source_row_index": 195, + "source_sample_id": "validation_Biology_16" + }, + { + "source_row_index": 202, + "source_sample_id": "validation_Biology_23" + }, + { + "source_row_index": 210, + "source_sample_id": "validation_Chemistry_1" + }, + { + "source_row_index": 217, + "source_sample_id": "validation_Chemistry_8" + }, + { + "source_row_index": 225, + "source_sample_id": "validation_Chemistry_16" + }, + { + "source_row_index": 232, + "source_sample_id": "validation_Chemistry_23" + }, + { + "source_row_index": 240, + "source_sample_id": "validation_Clinical_Medicine_1" + }, + { + "source_row_index": 247, + "source_sample_id": "validation_Clinical_Medicine_8" + }, + { + "source_row_index": 255, + "source_sample_id": "validation_Clinical_Medicine_16" + }, + { + "source_row_index": 262, + "source_sample_id": "validation_Clinical_Medicine_23" + }, + { + "source_row_index": 270, + "source_sample_id": "validation_Computer_Science_1" + }, + { + "source_row_index": 277, + "source_sample_id": "validation_Computer_Science_8" + }, + { + "source_row_index": 285, + "source_sample_id": "validation_Computer_Science_16" + }, + { + "source_row_index": 292, + "source_sample_id": "validation_Computer_Science_23" + }, + { + "source_row_index": 300, + "source_sample_id": "validation_Design_1" + }, + { + "source_row_index": 307, + "source_sample_id": "validation_Design_8" + }, + { + "source_row_index": 315, + "source_sample_id": "validation_Design_16" + }, + { + "source_row_index": 322, + "source_sample_id": "validation_Design_23" + }, + { + "source_row_index": 330, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_1" + }, + { + "source_row_index": 337, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_8" + }, + { + "source_row_index": 345, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_16" + }, + { + "source_row_index": 352, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_23" + }, + { + "source_row_index": 360, + "source_sample_id": "validation_Economics_1" + }, + { + "source_row_index": 367, + "source_sample_id": "validation_Economics_8" + }, + { + "source_row_index": 375, + "source_sample_id": "validation_Economics_16" + }, + { + "source_row_index": 382, + "source_sample_id": "validation_Economics_23" + }, + { + "source_row_index": 390, + "source_sample_id": "validation_Electronics_1" + }, + { + "source_row_index": 397, + "source_sample_id": "validation_Electronics_8" + }, + { + "source_row_index": 405, + "source_sample_id": "validation_Electronics_16" + }, + { + "source_row_index": 412, + "source_sample_id": "validation_Electronics_23" + }, + { + "source_row_index": 420, + "source_sample_id": "validation_Energy_and_Power_1" + }, + { + "source_row_index": 427, + "source_sample_id": "validation_Energy_and_Power_8" + }, + { + "source_row_index": 435, + "source_sample_id": "validation_Energy_and_Power_16" + }, + { + "source_row_index": 442, + "source_sample_id": "validation_Energy_and_Power_23" + }, + { + "source_row_index": 450, + "source_sample_id": "validation_Finance_1" + }, + { + "source_row_index": 457, + "source_sample_id": "validation_Finance_8" + }, + { + "source_row_index": 465, + "source_sample_id": "validation_Finance_16" + }, + { + "source_row_index": 472, + "source_sample_id": "validation_Finance_23" + }, + { + "source_row_index": 480, + "source_sample_id": "validation_Geography_1" + }, + { + "source_row_index": 487, + "source_sample_id": "validation_Geography_8" + }, + { + "source_row_index": 495, + "source_sample_id": "validation_Geography_16" + }, + { + "source_row_index": 502, + "source_sample_id": "validation_Geography_23" + }, + { + "source_row_index": 510, + "source_sample_id": "validation_History_1" + }, + { + "source_row_index": 517, + "source_sample_id": "validation_History_8" + }, + { + "source_row_index": 525, + "source_sample_id": "validation_History_16" + }, + { + "source_row_index": 532, + "source_sample_id": "validation_History_23" + }, + { + "source_row_index": 540, + "source_sample_id": "validation_Literature_1" + }, + { + "source_row_index": 547, + "source_sample_id": "validation_Literature_8" + }, + { + "source_row_index": 555, + "source_sample_id": "validation_Literature_16" + }, + { + "source_row_index": 562, + "source_sample_id": "validation_Literature_23" + }, + { + "source_row_index": 570, + "source_sample_id": "validation_Manage_1" + }, + { + "source_row_index": 577, + "source_sample_id": "validation_Manage_8" + }, + { + "source_row_index": 585, + "source_sample_id": "validation_Manage_16" + }, + { + "source_row_index": 592, + "source_sample_id": "validation_Manage_23" + }, + { + "source_row_index": 600, + "source_sample_id": "validation_Marketing_1" + }, + { + "source_row_index": 607, + "source_sample_id": "validation_Marketing_8" + }, + { + "source_row_index": 615, + "source_sample_id": "validation_Marketing_16" + }, + { + "source_row_index": 622, + "source_sample_id": "validation_Marketing_23" + }, + { + "source_row_index": 630, + "source_sample_id": "validation_Materials_1" + }, + { + "source_row_index": 637, + "source_sample_id": "validation_Materials_8" + }, + { + "source_row_index": 645, + "source_sample_id": "validation_Materials_16" + }, + { + "source_row_index": 652, + "source_sample_id": "validation_Materials_23" + }, + { + "source_row_index": 660, + "source_sample_id": "validation_Math_1" + }, + { + "source_row_index": 667, + "source_sample_id": "validation_Math_8" + }, + { + "source_row_index": 675, + "source_sample_id": "validation_Math_16" + }, + { + "source_row_index": 682, + "source_sample_id": "validation_Math_23" + }, + { + "source_row_index": 690, + "source_sample_id": "validation_Mechanical_Engineering_1" + }, + { + "source_row_index": 697, + "source_sample_id": "validation_Mechanical_Engineering_8" + }, + { + "source_row_index": 705, + "source_sample_id": "validation_Mechanical_Engineering_16" + }, + { + "source_row_index": 712, + "source_sample_id": "validation_Mechanical_Engineering_23" + }, + { + "source_row_index": 720, + "source_sample_id": "validation_Music_1" + }, + { + "source_row_index": 727, + "source_sample_id": "validation_Music_8" + }, + { + "source_row_index": 735, + "source_sample_id": "validation_Music_16" + }, + { + "source_row_index": 742, + "source_sample_id": "validation_Music_23" + }, + { + "source_row_index": 750, + "source_sample_id": "validation_Pharmacy_1" + }, + { + "source_row_index": 757, + "source_sample_id": "validation_Pharmacy_8" + }, + { + "source_row_index": 765, + "source_sample_id": "validation_Pharmacy_16" + }, + { + "source_row_index": 772, + "source_sample_id": "validation_Pharmacy_23" + }, + { + "source_row_index": 780, + "source_sample_id": "validation_Physics_1" + }, + { + "source_row_index": 787, + "source_sample_id": "validation_Physics_8" + }, + { + "source_row_index": 795, + "source_sample_id": "validation_Physics_16" + }, + { + "source_row_index": 802, + "source_sample_id": "validation_Physics_23" + }, + { + "source_row_index": 810, + "source_sample_id": "validation_Psychology_1" + }, + { + "source_row_index": 817, + "source_sample_id": "validation_Psychology_8" + }, + { + "source_row_index": 825, + "source_sample_id": "validation_Psychology_16" + }, + { + "source_row_index": 832, + "source_sample_id": "validation_Psychology_23" + }, + { + "source_row_index": 840, + "source_sample_id": "validation_Public_Health_1" + }, + { + "source_row_index": 847, + "source_sample_id": "validation_Public_Health_8" + }, + { + "source_row_index": 855, + "source_sample_id": "validation_Public_Health_16" + }, + { + "source_row_index": 862, + "source_sample_id": "validation_Public_Health_23" + }, + { + "source_row_index": 870, + "source_sample_id": "validation_Sociology_1" + }, + { + "source_row_index": 877, + "source_sample_id": "validation_Sociology_8" + }, + { + "source_row_index": 885, + "source_sample_id": "validation_Sociology_16" + }, + { + "source_row_index": 892, + "source_sample_id": "validation_Sociology_23" + } + ] + }, + "mvbench": { + "dataset_repository": "OpenGVLab/MVBench", + "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mvbench/mvbench.yaml", + "rows": [ + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 0, + "source_sample_id": "action_sequence:0" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 1, + "source_sample_id": "action_sequence:1" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 2, + "source_sample_id": "action_sequence:2" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 3, + "source_sample_id": "action_sequence:3" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 4, + "source_sample_id": "action_sequence:4" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 5, + "source_sample_id": "action_sequence:5" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 6, + "source_sample_id": "action_sequence:6" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 7, + "source_sample_id": "action_sequence:7" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 0, + "source_sample_id": "moving_count:0" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 1, + "source_sample_id": "moving_count:1" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 2, + "source_sample_id": "moving_count:2" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 3, + "source_sample_id": "moving_count:3" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 4, + "source_sample_id": "moving_count:4" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 5, + "source_sample_id": "moving_count:5" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 6, + "source_sample_id": "moving_count:6" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 7, + "source_sample_id": "moving_count:7" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 0, + "source_sample_id": "action_prediction:0" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 1, + "source_sample_id": "action_prediction:1" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 2, + "source_sample_id": "action_prediction:2" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 3, + "source_sample_id": "action_prediction:3" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 4, + "source_sample_id": "action_prediction:4" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 5, + "source_sample_id": "action_prediction:5" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 6, + "source_sample_id": "action_prediction:6" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 7, + "source_sample_id": "action_prediction:7" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 0, + "source_sample_id": "episodic_reasoning:0" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 1, + "source_sample_id": "episodic_reasoning:1" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 2, + "source_sample_id": "episodic_reasoning:2" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 3, + "source_sample_id": "episodic_reasoning:3" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 4, + "source_sample_id": "episodic_reasoning:4" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 5, + "source_sample_id": "episodic_reasoning:5" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 6, + "source_sample_id": "episodic_reasoning:6" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 7, + "source_sample_id": "episodic_reasoning:7" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 0, + "source_sample_id": "action_antonym:0" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 1, + "source_sample_id": "action_antonym:1" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 2, + "source_sample_id": "action_antonym:2" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 3, + "source_sample_id": "action_antonym:3" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 4, + "source_sample_id": "action_antonym:4" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 5, + "source_sample_id": "action_antonym:5" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 6, + "source_sample_id": "action_antonym:6" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 7, + "source_sample_id": "action_antonym:7" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 0, + "source_sample_id": "action_count:0" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 1, + "source_sample_id": "action_count:1" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 2, + "source_sample_id": "action_count:2" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 3, + "source_sample_id": "action_count:3" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 4, + "source_sample_id": "action_count:4" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 5, + "source_sample_id": "action_count:5" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 6, + "source_sample_id": "action_count:6" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 7, + "source_sample_id": "action_count:7" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 0, + "source_sample_id": "scene_transition:0" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 1, + "source_sample_id": "scene_transition:1" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 2, + "source_sample_id": "scene_transition:2" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 3, + "source_sample_id": "scene_transition:3" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 4, + "source_sample_id": "scene_transition:4" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 5, + "source_sample_id": "scene_transition:5" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 6, + "source_sample_id": "scene_transition:6" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 7, + "source_sample_id": "scene_transition:7" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 0, + "source_sample_id": "object_shuffle:0" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 1, + "source_sample_id": "object_shuffle:1" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 2, + "source_sample_id": "object_shuffle:2" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 3, + "source_sample_id": "object_shuffle:3" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 4, + "source_sample_id": "object_shuffle:4" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 5, + "source_sample_id": "object_shuffle:5" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 6, + "source_sample_id": "object_shuffle:6" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 7, + "source_sample_id": "object_shuffle:7" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 0, + "source_sample_id": "object_existence:0" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 1, + "source_sample_id": "object_existence:1" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 2, + "source_sample_id": "object_existence:2" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 3, + "source_sample_id": "object_existence:3" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 4, + "source_sample_id": "object_existence:4" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 5, + "source_sample_id": "object_existence:5" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 6, + "source_sample_id": "object_existence:6" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 7, + "source_sample_id": "object_existence:7" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 0, + "source_sample_id": "fine_grained_pose:0" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 1, + "source_sample_id": "fine_grained_pose:1" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 2, + "source_sample_id": "fine_grained_pose:2" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 3, + "source_sample_id": "fine_grained_pose:3" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 4, + "source_sample_id": "fine_grained_pose:4" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 5, + "source_sample_id": "fine_grained_pose:5" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 6, + "source_sample_id": "fine_grained_pose:6" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 7, + "source_sample_id": "fine_grained_pose:7" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 0, + "source_sample_id": "unexpected_action:0" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 1, + "source_sample_id": "unexpected_action:1" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 2, + "source_sample_id": "unexpected_action:2" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 3, + "source_sample_id": "unexpected_action:3" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 4, + "source_sample_id": "unexpected_action:4" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 5, + "source_sample_id": "unexpected_action:5" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 6, + "source_sample_id": "unexpected_action:6" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 7, + "source_sample_id": "unexpected_action:7" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 0, + "source_sample_id": "moving_direction:0" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 1, + "source_sample_id": "moving_direction:1" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 2, + "source_sample_id": "moving_direction:2" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 3, + "source_sample_id": "moving_direction:3" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 4, + "source_sample_id": "moving_direction:4" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 5, + "source_sample_id": "moving_direction:5" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 6, + "source_sample_id": "moving_direction:6" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 7, + "source_sample_id": "moving_direction:7" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 0, + "source_sample_id": "state_change:0" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 1, + "source_sample_id": "state_change:1" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 2, + "source_sample_id": "state_change:2" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 3, + "source_sample_id": "state_change:3" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 4, + "source_sample_id": "state_change:4" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 5, + "source_sample_id": "state_change:5" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 6, + "source_sample_id": "state_change:6" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 7, + "source_sample_id": "state_change:7" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 0, + "source_sample_id": "object_interaction:0" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 1, + "source_sample_id": "object_interaction:1" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 2, + "source_sample_id": "object_interaction:2" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 3, + "source_sample_id": "object_interaction:3" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 4, + "source_sample_id": "object_interaction:4" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 5, + "source_sample_id": "object_interaction:5" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 6, + "source_sample_id": "object_interaction:6" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 7, + "source_sample_id": "object_interaction:7" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 0, + "source_sample_id": "character_order:0" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 1, + "source_sample_id": "character_order:1" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 2, + "source_sample_id": "character_order:2" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 3, + "source_sample_id": "character_order:3" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 4, + "source_sample_id": "character_order:4" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 5, + "source_sample_id": "character_order:5" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 6, + "source_sample_id": "character_order:6" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 7, + "source_sample_id": "character_order:7" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 0, + "source_sample_id": "action_localization:0" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 1, + "source_sample_id": "action_localization:1" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 2, + "source_sample_id": "action_localization:2" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 3, + "source_sample_id": "action_localization:3" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 4, + "source_sample_id": "action_localization:4" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 5, + "source_sample_id": "action_localization:5" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 6, + "source_sample_id": "action_localization:6" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 7, + "source_sample_id": "action_localization:7" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 0, + "source_sample_id": "counterfactual_inference:0" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 1, + "source_sample_id": "counterfactual_inference:1" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 2, + "source_sample_id": "counterfactual_inference:2" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 3, + "source_sample_id": "counterfactual_inference:3" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 4, + "source_sample_id": "counterfactual_inference:4" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 5, + "source_sample_id": "counterfactual_inference:5" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 6, + "source_sample_id": "counterfactual_inference:6" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 7, + "source_sample_id": "counterfactual_inference:7" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 0, + "source_sample_id": "fine_grained_action:0" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 1, + "source_sample_id": "fine_grained_action:1" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 2, + "source_sample_id": "fine_grained_action:2" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 3, + "source_sample_id": "fine_grained_action:3" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 4, + "source_sample_id": "fine_grained_action:4" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 5, + "source_sample_id": "fine_grained_action:5" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 6, + "source_sample_id": "fine_grained_action:6" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 7, + "source_sample_id": "fine_grained_action:7" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 0, + "source_sample_id": "moving_attribute:0" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 1, + "source_sample_id": "moving_attribute:1" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 2, + "source_sample_id": "moving_attribute:2" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 3, + "source_sample_id": "moving_attribute:3" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 4, + "source_sample_id": "moving_attribute:4" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 5, + "source_sample_id": "moving_attribute:5" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 6, + "source_sample_id": "moving_attribute:6" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 7, + "source_sample_id": "moving_attribute:7" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 0, + "source_sample_id": "egocentric_navigation:0" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 1, + "source_sample_id": "egocentric_navigation:1" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 2, + "source_sample_id": "egocentric_navigation:2" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 3, + "source_sample_id": "egocentric_navigation:3" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 4, + "source_sample_id": "egocentric_navigation:4" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 5, + "source_sample_id": "egocentric_navigation:5" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 6, + "source_sample_id": "egocentric_navigation:6" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 7, + "source_sample_id": "egocentric_navigation:7" + } + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-native-v2.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_r1.json similarity index 99% rename from examples/puzzletron/evaluation/vlm/profiles/short-native-v2.json rename to examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_r1.json index 689f63b6672..c6d4aa32a82 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-native-v2.json +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_r1.json @@ -1,8 +1,7 @@ { - "extends": "short-native-v1", - "schema": "modelopt.vlm-evaluation-profile/v1", - "name": "short-native-v2", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "extends": "core-3_344-examples_legacy-r1", + "schema": "modelopt.vlm-sample-set/v1", + "name": "core-3_344-examples_r1", "sampling": { "schema": "modelopt.vlm-sampling-audit/v1", "claim_scope": "deterministic-screening-only", diff --git a/examples/puzzletron/evaluation/vlm/profiles/core3-full-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_full_r1.json similarity index 73% rename from examples/puzzletron/evaluation/vlm/profiles/core3-full-native-v1.json rename to examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_full_r1.json index 0fc3afb009a..5363b77afc9 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/core3-full-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_full_r1.json @@ -1,32 +1,10 @@ { - "schema": "modelopt.vlm-evaluation-profile/v1", - "name": "core3-full-native-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "schema": "modelopt.vlm-sample-set/v1", + "name": "core-3_full_r1", "model": { "repository": "Qwen/Qwen3.5-0.8B", "revision": "2fc06364715b967f1860aea9cf38778875588b17" }, - "model_family": { - "architecture": "Qwen3_5ForConditionalGeneration", - "model_type": "qwen3_5" - }, - "backend": { - "attention_implementation": "sdpa", - "enable_thinking": false, - "name": "qwen3_5" - }, - "preprocessing": { - "fps": 2, - "max_frames": 32, - "video_reader": "decord" - }, - "generation": { - "do_sample": false, - "temperature": 0 - }, - "seed": 42, - "repetitions": 1, - "batch_size": 1, "selection": "all", "tasks": { "realworldqa": { diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_legacy-r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_legacy-r1.json new file mode 100644 index 00000000000..f25f49a97d2 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_legacy-r1.json @@ -0,0 +1,1503 @@ +{ + "extends": "core-3_344-examples_legacy-r1", + "schema": "modelopt.vlm-sample-set/v1", + "name": "judge-free-8_690-examples_legacy-r1", + "tasks": { + "video_mmmu": { + "dataset_repository": "lmms-lab/VideoMMMU", + "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", + "max_new_tokens": 1024, + "scoring_task_config": "tasks/videommmu/video_mmmu.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "adaptation:0", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 13, + "source_sample_id": "adaptation:13", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 26, + "source_sample_id": "adaptation:26", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 39, + "source_sample_id": "adaptation:39", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 52, + "source_sample_id": "adaptation:52", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 65, + "source_sample_id": "adaptation:65", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 78, + "source_sample_id": "adaptation:78", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 91, + "source_sample_id": "adaptation:91", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 104, + "source_sample_id": "adaptation:104", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 117, + "source_sample_id": "adaptation:117", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 130, + "source_sample_id": "adaptation:130", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 143, + "source_sample_id": "adaptation:143", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 156, + "source_sample_id": "adaptation:156", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 169, + "source_sample_id": "adaptation:169", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 182, + "source_sample_id": "adaptation:182", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 195, + "source_sample_id": "adaptation:195", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 208, + "source_sample_id": "adaptation:208", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 221, + "source_sample_id": "adaptation:221", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 234, + "source_sample_id": "adaptation:234", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 247, + "source_sample_id": "adaptation:247", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 260, + "source_sample_id": "adaptation:260", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 273, + "source_sample_id": "adaptation:273", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 286, + "source_sample_id": "adaptation:286", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 299, + "source_sample_id": "adaptation:299", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 0, + "source_sample_id": "comprehension:0", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 13, + "source_sample_id": "comprehension:13", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 26, + "source_sample_id": "comprehension:26", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 39, + "source_sample_id": "comprehension:39", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 52, + "source_sample_id": "comprehension:52", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 65, + "source_sample_id": "comprehension:65", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 78, + "source_sample_id": "comprehension:78", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 91, + "source_sample_id": "comprehension:91", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 104, + "source_sample_id": "comprehension:104", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 117, + "source_sample_id": "comprehension:117", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 130, + "source_sample_id": "comprehension:130", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 143, + "source_sample_id": "comprehension:143", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 156, + "source_sample_id": "comprehension:156", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 169, + "source_sample_id": "comprehension:169", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 182, + "source_sample_id": "comprehension:182", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 195, + "source_sample_id": "comprehension:195", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 208, + "source_sample_id": "comprehension:208", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 221, + "source_sample_id": "comprehension:221", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 234, + "source_sample_id": "comprehension:234", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 247, + "source_sample_id": "comprehension:247", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 260, + "source_sample_id": "comprehension:260", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 273, + "source_sample_id": "comprehension:273", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 286, + "source_sample_id": "comprehension:286", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 299, + "source_sample_id": "comprehension:299", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 0, + "source_sample_id": "perception:0", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 13, + "source_sample_id": "perception:13", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 26, + "source_sample_id": "perception:26", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 39, + "source_sample_id": "perception:39", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 52, + "source_sample_id": "perception:52", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 65, + "source_sample_id": "perception:65", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 78, + "source_sample_id": "perception:78", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 91, + "source_sample_id": "perception:91", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 104, + "source_sample_id": "perception:104", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 117, + "source_sample_id": "perception:117", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 130, + "source_sample_id": "perception:130", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 143, + "source_sample_id": "perception:143", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 156, + "source_sample_id": "perception:156", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 169, + "source_sample_id": "perception:169", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 182, + "source_sample_id": "perception:182", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 195, + "source_sample_id": "perception:195", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 208, + "source_sample_id": "perception:208", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 221, + "source_sample_id": "perception:221", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 234, + "source_sample_id": "perception:234", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 247, + "source_sample_id": "perception:247", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 260, + "source_sample_id": "perception:260", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 273, + "source_sample_id": "perception:273", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 286, + "source_sample_id": "perception:286", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 299, + "source_sample_id": "perception:299", + "leaf_task": "video_mmmu_perception" + } + ] + }, + "videomme": { + "dataset_repository": "lmms-lab/Video-MME", + "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/videomme/videomme.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "videomme:0" + }, + { + "source_row_index": 90, + "source_sample_id": "videomme:90" + }, + { + "source_row_index": 179, + "source_sample_id": "videomme:179" + }, + { + "source_row_index": 269, + "source_sample_id": "videomme:269" + }, + { + "source_row_index": 270, + "source_sample_id": "videomme:270" + }, + { + "source_row_index": 310, + "source_sample_id": "videomme:310" + }, + { + "source_row_index": 349, + "source_sample_id": "videomme:349" + }, + { + "source_row_index": 389, + "source_sample_id": "videomme:389" + }, + { + "source_row_index": 390, + "source_sample_id": "videomme:390" + }, + { + "source_row_index": 440, + "source_sample_id": "videomme:440" + }, + { + "source_row_index": 489, + "source_sample_id": "videomme:489" + }, + { + "source_row_index": 539, + "source_sample_id": "videomme:539" + }, + { + "source_row_index": 540, + "source_sample_id": "videomme:540" + }, + { + "source_row_index": 580, + "source_sample_id": "videomme:580" + }, + { + "source_row_index": 619, + "source_sample_id": "videomme:619" + }, + { + "source_row_index": 659, + "source_sample_id": "videomme:659" + }, + { + "source_row_index": 660, + "source_sample_id": "videomme:660" + }, + { + "source_row_index": 730, + "source_sample_id": "videomme:730" + }, + { + "source_row_index": 799, + "source_sample_id": "videomme:799" + }, + { + "source_row_index": 869, + "source_sample_id": "videomme:869" + }, + { + "source_row_index": 870, + "source_sample_id": "videomme:870" + }, + { + "source_row_index": 880, + "source_sample_id": "videomme:880" + }, + { + "source_row_index": 889, + "source_sample_id": "videomme:889" + }, + { + "source_row_index": 899, + "source_sample_id": "videomme:899" + }, + { + "source_row_index": 900, + "source_sample_id": "videomme:900" + }, + { + "source_row_index": 990, + "source_sample_id": "videomme:990" + }, + { + "source_row_index": 1079, + "source_sample_id": "videomme:1079" + }, + { + "source_row_index": 1169, + "source_sample_id": "videomme:1169" + }, + { + "source_row_index": 1170, + "source_sample_id": "videomme:1170" + }, + { + "source_row_index": 1210, + "source_sample_id": "videomme:1210" + }, + { + "source_row_index": 1249, + "source_sample_id": "videomme:1249" + }, + { + "source_row_index": 1289, + "source_sample_id": "videomme:1289" + }, + { + "source_row_index": 1290, + "source_sample_id": "videomme:1290" + }, + { + "source_row_index": 1340, + "source_sample_id": "videomme:1340" + }, + { + "source_row_index": 1389, + "source_sample_id": "videomme:1389" + }, + { + "source_row_index": 1439, + "source_sample_id": "videomme:1439" + }, + { + "source_row_index": 1440, + "source_sample_id": "videomme:1440" + }, + { + "source_row_index": 1480, + "source_sample_id": "videomme:1480" + }, + { + "source_row_index": 1519, + "source_sample_id": "videomme:1519" + }, + { + "source_row_index": 1559, + "source_sample_id": "videomme:1559" + }, + { + "source_row_index": 1560, + "source_sample_id": "videomme:1560" + }, + { + "source_row_index": 1630, + "source_sample_id": "videomme:1630" + }, + { + "source_row_index": 1699, + "source_sample_id": "videomme:1699" + }, + { + "source_row_index": 1769, + "source_sample_id": "videomme:1769" + }, + { + "source_row_index": 1770, + "source_sample_id": "videomme:1770" + }, + { + "source_row_index": 1780, + "source_sample_id": "videomme:1780" + }, + { + "source_row_index": 1789, + "source_sample_id": "videomme:1789" + }, + { + "source_row_index": 1799, + "source_sample_id": "videomme:1799" + }, + { + "source_row_index": 1800, + "source_sample_id": "videomme:1800" + }, + { + "source_row_index": 1890, + "source_sample_id": "videomme:1890" + }, + { + "source_row_index": 1979, + "source_sample_id": "videomme:1979" + }, + { + "source_row_index": 2069, + "source_sample_id": "videomme:2069" + }, + { + "source_row_index": 2070, + "source_sample_id": "videomme:2070" + }, + { + "source_row_index": 2110, + "source_sample_id": "videomme:2110" + }, + { + "source_row_index": 2149, + "source_sample_id": "videomme:2149" + }, + { + "source_row_index": 2189, + "source_sample_id": "videomme:2189" + }, + { + "source_row_index": 2190, + "source_sample_id": "videomme:2190" + }, + { + "source_row_index": 2240, + "source_sample_id": "videomme:2240" + }, + { + "source_row_index": 2289, + "source_sample_id": "videomme:2289" + }, + { + "source_row_index": 2339, + "source_sample_id": "videomme:2339" + }, + { + "source_row_index": 2340, + "source_sample_id": "videomme:2340" + }, + { + "source_row_index": 2380, + "source_sample_id": "videomme:2380" + }, + { + "source_row_index": 2419, + "source_sample_id": "videomme:2419" + }, + { + "source_row_index": 2459, + "source_sample_id": "videomme:2459" + }, + { + "source_row_index": 2460, + "source_sample_id": "videomme:2460" + }, + { + "source_row_index": 2530, + "source_sample_id": "videomme:2530" + }, + { + "source_row_index": 2599, + "source_sample_id": "videomme:2599" + }, + { + "source_row_index": 2669, + "source_sample_id": "videomme:2669" + }, + { + "source_row_index": 2670, + "source_sample_id": "videomme:2670" + }, + { + "source_row_index": 2680, + "source_sample_id": "videomme:2680" + }, + { + "source_row_index": 2689, + "source_sample_id": "videomme:2689" + }, + { + "source_row_index": 2699, + "source_sample_id": "videomme:2699" + } + ] + }, + "longvideobench_val_v": { + "dataset_repository": "longvideobench/LongVideoBench", + "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", + "max_new_tokens": 32, + "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "longvideobench_val_v:0" + }, + { + "source_row_index": 20, + "source_sample_id": "longvideobench_val_v:20" + }, + { + "source_row_index": 40, + "source_sample_id": "longvideobench_val_v:40" + }, + { + "source_row_index": 60, + "source_sample_id": "longvideobench_val_v:60" + }, + { + "source_row_index": 80, + "source_sample_id": "longvideobench_val_v:80" + }, + { + "source_row_index": 100, + "source_sample_id": "longvideobench_val_v:100" + }, + { + "source_row_index": 120, + "source_sample_id": "longvideobench_val_v:120" + }, + { + "source_row_index": 140, + "source_sample_id": "longvideobench_val_v:140" + }, + { + "source_row_index": 160, + "source_sample_id": "longvideobench_val_v:160" + }, + { + "source_row_index": 179, + "source_sample_id": "longvideobench_val_v:179" + }, + { + "source_row_index": 199, + "source_sample_id": "longvideobench_val_v:199" + }, + { + "source_row_index": 219, + "source_sample_id": "longvideobench_val_v:219" + }, + { + "source_row_index": 239, + "source_sample_id": "longvideobench_val_v:239" + }, + { + "source_row_index": 259, + "source_sample_id": "longvideobench_val_v:259" + }, + { + "source_row_index": 279, + "source_sample_id": "longvideobench_val_v:279" + }, + { + "source_row_index": 299, + "source_sample_id": "longvideobench_val_v:299" + }, + { + "source_row_index": 319, + "source_sample_id": "longvideobench_val_v:319" + }, + { + "source_row_index": 339, + "source_sample_id": "longvideobench_val_v:339" + }, + { + "source_row_index": 359, + "source_sample_id": "longvideobench_val_v:359" + }, + { + "source_row_index": 379, + "source_sample_id": "longvideobench_val_v:379" + }, + { + "source_row_index": 399, + "source_sample_id": "longvideobench_val_v:399" + }, + { + "source_row_index": 419, + "source_sample_id": "longvideobench_val_v:419" + }, + { + "source_row_index": 439, + "source_sample_id": "longvideobench_val_v:439" + }, + { + "source_row_index": 459, + "source_sample_id": "longvideobench_val_v:459" + }, + { + "source_row_index": 479, + "source_sample_id": "longvideobench_val_v:479" + }, + { + "source_row_index": 499, + "source_sample_id": "longvideobench_val_v:499" + }, + { + "source_row_index": 518, + "source_sample_id": "longvideobench_val_v:518" + }, + { + "source_row_index": 538, + "source_sample_id": "longvideobench_val_v:538" + }, + { + "source_row_index": 558, + "source_sample_id": "longvideobench_val_v:558" + }, + { + "source_row_index": 578, + "source_sample_id": "longvideobench_val_v:578" + }, + { + "source_row_index": 598, + "source_sample_id": "longvideobench_val_v:598" + }, + { + "source_row_index": 618, + "source_sample_id": "longvideobench_val_v:618" + }, + { + "source_row_index": 638, + "source_sample_id": "longvideobench_val_v:638" + }, + { + "source_row_index": 658, + "source_sample_id": "longvideobench_val_v:658" + }, + { + "source_row_index": 678, + "source_sample_id": "longvideobench_val_v:678" + }, + { + "source_row_index": 698, + "source_sample_id": "longvideobench_val_v:698" + }, + { + "source_row_index": 718, + "source_sample_id": "longvideobench_val_v:718" + }, + { + "source_row_index": 738, + "source_sample_id": "longvideobench_val_v:738" + }, + { + "source_row_index": 758, + "source_sample_id": "longvideobench_val_v:758" + }, + { + "source_row_index": 778, + "source_sample_id": "longvideobench_val_v:778" + }, + { + "source_row_index": 798, + "source_sample_id": "longvideobench_val_v:798" + }, + { + "source_row_index": 818, + "source_sample_id": "longvideobench_val_v:818" + }, + { + "source_row_index": 837, + "source_sample_id": "longvideobench_val_v:837" + }, + { + "source_row_index": 857, + "source_sample_id": "longvideobench_val_v:857" + }, + { + "source_row_index": 877, + "source_sample_id": "longvideobench_val_v:877" + }, + { + "source_row_index": 897, + "source_sample_id": "longvideobench_val_v:897" + }, + { + "source_row_index": 917, + "source_sample_id": "longvideobench_val_v:917" + }, + { + "source_row_index": 937, + "source_sample_id": "longvideobench_val_v:937" + }, + { + "source_row_index": 957, + "source_sample_id": "longvideobench_val_v:957" + }, + { + "source_row_index": 977, + "source_sample_id": "longvideobench_val_v:977" + }, + { + "source_row_index": 997, + "source_sample_id": "longvideobench_val_v:997" + }, + { + "source_row_index": 1017, + "source_sample_id": "longvideobench_val_v:1017" + }, + { + "source_row_index": 1037, + "source_sample_id": "longvideobench_val_v:1037" + }, + { + "source_row_index": 1057, + "source_sample_id": "longvideobench_val_v:1057" + }, + { + "source_row_index": 1077, + "source_sample_id": "longvideobench_val_v:1077" + }, + { + "source_row_index": 1097, + "source_sample_id": "longvideobench_val_v:1097" + }, + { + "source_row_index": 1117, + "source_sample_id": "longvideobench_val_v:1117" + }, + { + "source_row_index": 1137, + "source_sample_id": "longvideobench_val_v:1137" + }, + { + "source_row_index": 1157, + "source_sample_id": "longvideobench_val_v:1157" + }, + { + "source_row_index": 1176, + "source_sample_id": "longvideobench_val_v:1176" + }, + { + "source_row_index": 1196, + "source_sample_id": "longvideobench_val_v:1196" + }, + { + "source_row_index": 1216, + "source_sample_id": "longvideobench_val_v:1216" + }, + { + "source_row_index": 1236, + "source_sample_id": "longvideobench_val_v:1236" + }, + { + "source_row_index": 1256, + "source_sample_id": "longvideobench_val_v:1256" + }, + { + "source_row_index": 1276, + "source_sample_id": "longvideobench_val_v:1276" + }, + { + "source_row_index": 1296, + "source_sample_id": "longvideobench_val_v:1296" + }, + { + "source_row_index": 1316, + "source_sample_id": "longvideobench_val_v:1316" + }, + { + "source_row_index": 1336, + "source_sample_id": "longvideobench_val_v:1336" + } + ] + }, + "mlvu_dev": { + "dataset_repository": "sy1998/MLVU_dev", + "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "mlvu_dev:0" + }, + { + "source_row_index": 39, + "source_sample_id": "mlvu_dev:39" + }, + { + "source_row_index": 79, + "source_sample_id": "mlvu_dev:79" + }, + { + "source_row_index": 118, + "source_sample_id": "mlvu_dev:118" + }, + { + "source_row_index": 157, + "source_sample_id": "mlvu_dev:157" + }, + { + "source_row_index": 197, + "source_sample_id": "mlvu_dev:197" + }, + { + "source_row_index": 236, + "source_sample_id": "mlvu_dev:236" + }, + { + "source_row_index": 275, + "source_sample_id": "mlvu_dev:275" + }, + { + "source_row_index": 315, + "source_sample_id": "mlvu_dev:315" + }, + { + "source_row_index": 354, + "source_sample_id": "mlvu_dev:354" + }, + { + "source_row_index": 355, + "source_sample_id": "mlvu_dev:355" + }, + { + "source_row_index": 377, + "source_sample_id": "mlvu_dev:377" + }, + { + "source_row_index": 399, + "source_sample_id": "mlvu_dev:399" + }, + { + "source_row_index": 421, + "source_sample_id": "mlvu_dev:421" + }, + { + "source_row_index": 443, + "source_sample_id": "mlvu_dev:443" + }, + { + "source_row_index": 466, + "source_sample_id": "mlvu_dev:466" + }, + { + "source_row_index": 488, + "source_sample_id": "mlvu_dev:488" + }, + { + "source_row_index": 510, + "source_sample_id": "mlvu_dev:510" + }, + { + "source_row_index": 532, + "source_sample_id": "mlvu_dev:532" + }, + { + "source_row_index": 554, + "source_sample_id": "mlvu_dev:554" + }, + { + "source_row_index": 555, + "source_sample_id": "mlvu_dev:555" + }, + { + "source_row_index": 578, + "source_sample_id": "mlvu_dev:578" + }, + { + "source_row_index": 601, + "source_sample_id": "mlvu_dev:601" + }, + { + "source_row_index": 623, + "source_sample_id": "mlvu_dev:623" + }, + { + "source_row_index": 646, + "source_sample_id": "mlvu_dev:646" + }, + { + "source_row_index": 669, + "source_sample_id": "mlvu_dev:669" + }, + { + "source_row_index": 692, + "source_sample_id": "mlvu_dev:692" + }, + { + "source_row_index": 714, + "source_sample_id": "mlvu_dev:714" + }, + { + "source_row_index": 737, + "source_sample_id": "mlvu_dev:737" + }, + { + "source_row_index": 760, + "source_sample_id": "mlvu_dev:760" + }, + { + "source_row_index": 761, + "source_sample_id": "mlvu_dev:761" + }, + { + "source_row_index": 800, + "source_sample_id": "mlvu_dev:800" + }, + { + "source_row_index": 839, + "source_sample_id": "mlvu_dev:839" + }, + { + "source_row_index": 878, + "source_sample_id": "mlvu_dev:878" + }, + { + "source_row_index": 917, + "source_sample_id": "mlvu_dev:917" + }, + { + "source_row_index": 956, + "source_sample_id": "mlvu_dev:956" + }, + { + "source_row_index": 995, + "source_sample_id": "mlvu_dev:995" + }, + { + "source_row_index": 1034, + "source_sample_id": "mlvu_dev:1034" + }, + { + "source_row_index": 1073, + "source_sample_id": "mlvu_dev:1073" + }, + { + "source_row_index": 1112, + "source_sample_id": "mlvu_dev:1112" + }, + { + "source_row_index": 1113, + "source_sample_id": "mlvu_dev:1113" + }, + { + "source_row_index": 1173, + "source_sample_id": "mlvu_dev:1173" + }, + { + "source_row_index": 1233, + "source_sample_id": "mlvu_dev:1233" + }, + { + "source_row_index": 1292, + "source_sample_id": "mlvu_dev:1292" + }, + { + "source_row_index": 1352, + "source_sample_id": "mlvu_dev:1352" + }, + { + "source_row_index": 1412, + "source_sample_id": "mlvu_dev:1412" + }, + { + "source_row_index": 1472, + "source_sample_id": "mlvu_dev:1472" + }, + { + "source_row_index": 1531, + "source_sample_id": "mlvu_dev:1531" + }, + { + "source_row_index": 1591, + "source_sample_id": "mlvu_dev:1591" + }, + { + "source_row_index": 1651, + "source_sample_id": "mlvu_dev:1651" + }, + { + "source_row_index": 1652, + "source_sample_id": "mlvu_dev:1652" + }, + { + "source_row_index": 1681, + "source_sample_id": "mlvu_dev:1681" + }, + { + "source_row_index": 1710, + "source_sample_id": "mlvu_dev:1710" + }, + { + "source_row_index": 1739, + "source_sample_id": "mlvu_dev:1739" + }, + { + "source_row_index": 1768, + "source_sample_id": "mlvu_dev:1768" + }, + { + "source_row_index": 1798, + "source_sample_id": "mlvu_dev:1798" + }, + { + "source_row_index": 1827, + "source_sample_id": "mlvu_dev:1827" + }, + { + "source_row_index": 1856, + "source_sample_id": "mlvu_dev:1856" + }, + { + "source_row_index": 1885, + "source_sample_id": "mlvu_dev:1885" + }, + { + "source_row_index": 1914, + "source_sample_id": "mlvu_dev:1914" + }, + { + "source_row_index": 1915, + "source_sample_id": "mlvu_dev:1915" + }, + { + "source_row_index": 1944, + "source_sample_id": "mlvu_dev:1944" + }, + { + "source_row_index": 1972, + "source_sample_id": "mlvu_dev:1972" + }, + { + "source_row_index": 2001, + "source_sample_id": "mlvu_dev:2001" + }, + { + "source_row_index": 2030, + "source_sample_id": "mlvu_dev:2030" + }, + { + "source_row_index": 2058, + "source_sample_id": "mlvu_dev:2058" + }, + { + "source_row_index": 2087, + "source_sample_id": "mlvu_dev:2087" + }, + { + "source_row_index": 2116, + "source_sample_id": "mlvu_dev:2116" + }, + { + "source_row_index": 2144, + "source_sample_id": "mlvu_dev:2144" + }, + { + "source_row_index": 2173, + "source_sample_id": "mlvu_dev:2173" + } + ] + }, + "perceptiontest_val_mc": { + "dataset_repository": "lmms-lab/PerceptionTest_Val", + "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "perceptiontest_val_mc:0" + }, + { + "source_row_index": 1, + "source_sample_id": "perceptiontest_val_mc:1" + }, + { + "source_row_index": 3, + "source_sample_id": "perceptiontest_val_mc:3" + }, + { + "source_row_index": 20, + "source_sample_id": "perceptiontest_val_mc:20" + }, + { + "source_row_index": 211, + "source_sample_id": "perceptiontest_val_mc:211" + }, + { + "source_row_index": 544, + "source_sample_id": "perceptiontest_val_mc:544" + }, + { + "source_row_index": 753, + "source_sample_id": "perceptiontest_val_mc:753" + }, + { + "source_row_index": 1672, + "source_sample_id": "perceptiontest_val_mc:1672" + }, + { + "source_row_index": 2310, + "source_sample_id": "perceptiontest_val_mc:2310" + }, + { + "source_row_index": 2386, + "source_sample_id": "perceptiontest_val_mc:2386" + }, + { + "source_row_index": 2846, + "source_sample_id": "perceptiontest_val_mc:2846" + }, + { + "source_row_index": 2894, + "source_sample_id": "perceptiontest_val_mc:2894" + }, + { + "source_row_index": 3857, + "source_sample_id": "perceptiontest_val_mc:3857" + }, + { + "source_row_index": 3905, + "source_sample_id": "perceptiontest_val_mc:3905" + }, + { + "source_row_index": 3957, + "source_sample_id": "perceptiontest_val_mc:3957" + }, + { + "source_row_index": 4004, + "source_sample_id": "perceptiontest_val_mc:4004" + }, + { + "source_row_index": 4031, + "source_sample_id": "perceptiontest_val_mc:4031" + }, + { + "source_row_index": 4053, + "source_sample_id": "perceptiontest_val_mc:4053" + }, + { + "source_row_index": 4167, + "source_sample_id": "perceptiontest_val_mc:4167" + }, + { + "source_row_index": 4228, + "source_sample_id": "perceptiontest_val_mc:4228" + }, + { + "source_row_index": 4994, + "source_sample_id": "perceptiontest_val_mc:4994" + }, + { + "source_row_index": 5673, + "source_sample_id": "perceptiontest_val_mc:5673" + }, + { + "source_row_index": 5898, + "source_sample_id": "perceptiontest_val_mc:5898" + }, + { + "source_row_index": 6414, + "source_sample_id": "perceptiontest_val_mc:6414" + }, + { + "source_row_index": 6502, + "source_sample_id": "perceptiontest_val_mc:6502" + }, + { + "source_row_index": 6598, + "source_sample_id": "perceptiontest_val_mc:6598" + }, + { + "source_row_index": 7494, + "source_sample_id": "perceptiontest_val_mc:7494" + }, + { + "source_row_index": 8074, + "source_sample_id": "perceptiontest_val_mc:8074" + }, + { + "source_row_index": 8617, + "source_sample_id": "perceptiontest_val_mc:8617" + }, + { + "source_row_index": 8763, + "source_sample_id": "perceptiontest_val_mc:8763" + }, + { + "source_row_index": 8865, + "source_sample_id": "perceptiontest_val_mc:8865" + }, + { + "source_row_index": 8873, + "source_sample_id": "perceptiontest_val_mc:8873" + }, + { + "source_row_index": 8981, + "source_sample_id": "perceptiontest_val_mc:8981" + }, + { + "source_row_index": 9350, + "source_sample_id": "perceptiontest_val_mc:9350" + }, + { + "source_row_index": 9617, + "source_sample_id": "perceptiontest_val_mc:9617" + }, + { + "source_row_index": 9749, + "source_sample_id": "perceptiontest_val_mc:9749" + }, + { + "source_row_index": 10470, + "source_sample_id": "perceptiontest_val_mc:10470" + }, + { + "source_row_index": 11697, + "source_sample_id": "perceptiontest_val_mc:11697" + }, + { + "source_row_index": 11745, + "source_sample_id": "perceptiontest_val_mc:11745" + }, + { + "source_row_index": 11798, + "source_sample_id": "perceptiontest_val_mc:11798" + }, + { + "source_row_index": 13051, + "source_sample_id": "perceptiontest_val_mc:13051" + }, + { + "source_row_index": 13244, + "source_sample_id": "perceptiontest_val_mc:13244" + }, + { + "source_row_index": 13581, + "source_sample_id": "perceptiontest_val_mc:13581" + }, + { + "source_row_index": 13929, + "source_sample_id": "perceptiontest_val_mc:13929" + }, + { + "source_row_index": 13999, + "source_sample_id": "perceptiontest_val_mc:13999" + }, + { + "source_row_index": 14073, + "source_sample_id": "perceptiontest_val_mc:14073" + }, + { + "source_row_index": 14200, + "source_sample_id": "perceptiontest_val_mc:14200" + }, + { + "source_row_index": 14884, + "source_sample_id": "perceptiontest_val_mc:14884" + }, + { + "source_row_index": 15201, + "source_sample_id": "perceptiontest_val_mc:15201" + }, + { + "source_row_index": 15432, + "source_sample_id": "perceptiontest_val_mc:15432" + }, + { + "source_row_index": 16191, + "source_sample_id": "perceptiontest_val_mc:16191" + }, + { + "source_row_index": 16280, + "source_sample_id": "perceptiontest_val_mc:16280" + }, + { + "source_row_index": 16741, + "source_sample_id": "perceptiontest_val_mc:16741" + }, + { + "source_row_index": 17109, + "source_sample_id": "perceptiontest_val_mc:17109" + }, + { + "source_row_index": 17820, + "source_sample_id": "perceptiontest_val_mc:17820" + }, + { + "source_row_index": 17866, + "source_sample_id": "perceptiontest_val_mc:17866" + }, + { + "source_row_index": 18478, + "source_sample_id": "perceptiontest_val_mc:18478" + }, + { + "source_row_index": 18489, + "source_sample_id": "perceptiontest_val_mc:18489" + }, + { + "source_row_index": 18867, + "source_sample_id": "perceptiontest_val_mc:18867" + }, + { + "source_row_index": 19021, + "source_sample_id": "perceptiontest_val_mc:19021" + }, + { + "source_row_index": 19102, + "source_sample_id": "perceptiontest_val_mc:19102" + }, + { + "source_row_index": 19135, + "source_sample_id": "perceptiontest_val_mc:19135" + }, + { + "source_row_index": 19137, + "source_sample_id": "perceptiontest_val_mc:19137" + }, + { + "source_row_index": 19139, + "source_sample_id": "perceptiontest_val_mc:19139" + } + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v2.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_r1.json similarity index 99% rename from examples/puzzletron/evaluation/vlm/profiles/short-all-native-v2.json rename to examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_r1.json index c105aad17cb..32e9e77aa5f 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v2.json +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_r1.json @@ -1,8 +1,7 @@ { - "extends": "short-native-v2", - "schema": "modelopt.vlm-evaluation-profile/v1", - "name": "short-all-native-v2", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", + "extends": "core-3_344-examples_r1", + "schema": "modelopt.vlm-sample-set/v1", + "name": "judge-free-8_690-examples_r1", "tasks": { "video_mmmu": { "dataset_repository": "lmms-lab/VideoMMMU", diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_full_legacy-r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_full_legacy-r1.json new file mode 100644 index 00000000000..112b8dac3c9 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_full_legacy-r1.json @@ -0,0 +1,55 @@ +{ + "schema": "modelopt.vlm-sample-set/v1", + "name": "judge-free-8_full_legacy-r1", + "selection": "all", + "tasks": { + "realworldqa": { + "dataset_repository": "lmms-lab/RealWorldQA", + "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", + "max_new_tokens": 16, + "scoring_task_config": "tasks/realworldqa/realworldqa.yaml" + }, + "mmmu_val": { + "dataset_repository": "lmms-lab/MMMU", + "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", + "max_new_tokens": 128, + "scoring_task_config": "tasks/mmmu/mmmu_val.yaml" + }, + "video_mmmu": { + "dataset_repository": "lmms-lab/VideoMMMU", + "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", + "max_new_tokens": 1024, + "scoring_task_config": "tasks/videommmu/video_mmmu.yaml" + }, + "mvbench": { + "dataset_repository": "OpenGVLab/MVBench", + "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mvbench/mvbench.yaml" + }, + "videomme": { + "dataset_repository": "lmms-lab/Video-MME", + "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/videomme/videomme.yaml" + }, + "longvideobench_val_v": { + "dataset_repository": "longvideobench/LongVideoBench", + "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", + "max_new_tokens": 32, + "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml" + }, + "mlvu_dev": { + "dataset_repository": "sy1998/MLVU_dev", + "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml" + }, + "perceptiontest_val_mc": { + "dataset_repository": "lmms-lab/PerceptionTest_Val", + "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml" + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json index 9149b58cc69..c31d91fed43 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json @@ -1,1509 +1,7 @@ { - "extends": "short-v1", - "schema": "modelopt.vlm-evaluation-profile/v1", + "schema": "modelopt.vlm-evaluation-profile/v2", "name": "short-all-native-v1", - "lmms_eval_revision": "88b23e2bfa16a1edbc16e9e238ed82130b3a4f56", - "backend": { - "attention_implementation": "sdpa", - "enable_thinking": false, - "name": "qwen3_5" - }, - "tasks": { - "video_mmmu": { - "dataset_repository": "lmms-lab/VideoMMMU", - "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", - "max_new_tokens": 1024, - "scoring_task_config": "tasks/videommmu/video_mmmu.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "adaptation:0", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 13, - "source_sample_id": "adaptation:13", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 26, - "source_sample_id": "adaptation:26", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 39, - "source_sample_id": "adaptation:39", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 52, - "source_sample_id": "adaptation:52", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 65, - "source_sample_id": "adaptation:65", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 78, - "source_sample_id": "adaptation:78", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 91, - "source_sample_id": "adaptation:91", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 104, - "source_sample_id": "adaptation:104", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 117, - "source_sample_id": "adaptation:117", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 130, - "source_sample_id": "adaptation:130", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 143, - "source_sample_id": "adaptation:143", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 156, - "source_sample_id": "adaptation:156", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 169, - "source_sample_id": "adaptation:169", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 182, - "source_sample_id": "adaptation:182", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 195, - "source_sample_id": "adaptation:195", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 208, - "source_sample_id": "adaptation:208", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 221, - "source_sample_id": "adaptation:221", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 234, - "source_sample_id": "adaptation:234", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 247, - "source_sample_id": "adaptation:247", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 260, - "source_sample_id": "adaptation:260", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 273, - "source_sample_id": "adaptation:273", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 286, - "source_sample_id": "adaptation:286", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 299, - "source_sample_id": "adaptation:299", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 0, - "source_sample_id": "comprehension:0", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 13, - "source_sample_id": "comprehension:13", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 26, - "source_sample_id": "comprehension:26", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 39, - "source_sample_id": "comprehension:39", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 52, - "source_sample_id": "comprehension:52", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 65, - "source_sample_id": "comprehension:65", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 78, - "source_sample_id": "comprehension:78", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 91, - "source_sample_id": "comprehension:91", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 104, - "source_sample_id": "comprehension:104", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 117, - "source_sample_id": "comprehension:117", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 130, - "source_sample_id": "comprehension:130", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 143, - "source_sample_id": "comprehension:143", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 156, - "source_sample_id": "comprehension:156", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 169, - "source_sample_id": "comprehension:169", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 182, - "source_sample_id": "comprehension:182", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 195, - "source_sample_id": "comprehension:195", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 208, - "source_sample_id": "comprehension:208", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 221, - "source_sample_id": "comprehension:221", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 234, - "source_sample_id": "comprehension:234", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 247, - "source_sample_id": "comprehension:247", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 260, - "source_sample_id": "comprehension:260", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 273, - "source_sample_id": "comprehension:273", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 286, - "source_sample_id": "comprehension:286", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 299, - "source_sample_id": "comprehension:299", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 0, - "source_sample_id": "perception:0", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 13, - "source_sample_id": "perception:13", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 26, - "source_sample_id": "perception:26", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 39, - "source_sample_id": "perception:39", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 52, - "source_sample_id": "perception:52", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 65, - "source_sample_id": "perception:65", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 78, - "source_sample_id": "perception:78", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 91, - "source_sample_id": "perception:91", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 104, - "source_sample_id": "perception:104", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 117, - "source_sample_id": "perception:117", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 130, - "source_sample_id": "perception:130", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 143, - "source_sample_id": "perception:143", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 156, - "source_sample_id": "perception:156", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 169, - "source_sample_id": "perception:169", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 182, - "source_sample_id": "perception:182", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 195, - "source_sample_id": "perception:195", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 208, - "source_sample_id": "perception:208", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 221, - "source_sample_id": "perception:221", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 234, - "source_sample_id": "perception:234", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 247, - "source_sample_id": "perception:247", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 260, - "source_sample_id": "perception:260", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 273, - "source_sample_id": "perception:273", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 286, - "source_sample_id": "perception:286", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 299, - "source_sample_id": "perception:299", - "leaf_task": "video_mmmu_perception" - } - ] - }, - "videomme": { - "dataset_repository": "lmms-lab/Video-MME", - "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", - "max_new_tokens": 16, - "scoring_task_config": "tasks/videomme/videomme.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "videomme:0" - }, - { - "source_row_index": 90, - "source_sample_id": "videomme:90" - }, - { - "source_row_index": 179, - "source_sample_id": "videomme:179" - }, - { - "source_row_index": 269, - "source_sample_id": "videomme:269" - }, - { - "source_row_index": 270, - "source_sample_id": "videomme:270" - }, - { - "source_row_index": 310, - "source_sample_id": "videomme:310" - }, - { - "source_row_index": 349, - "source_sample_id": "videomme:349" - }, - { - "source_row_index": 389, - "source_sample_id": "videomme:389" - }, - { - "source_row_index": 390, - "source_sample_id": "videomme:390" - }, - { - "source_row_index": 440, - "source_sample_id": "videomme:440" - }, - { - "source_row_index": 489, - "source_sample_id": "videomme:489" - }, - { - "source_row_index": 539, - "source_sample_id": "videomme:539" - }, - { - "source_row_index": 540, - "source_sample_id": "videomme:540" - }, - { - "source_row_index": 580, - "source_sample_id": "videomme:580" - }, - { - "source_row_index": 619, - "source_sample_id": "videomme:619" - }, - { - "source_row_index": 659, - "source_sample_id": "videomme:659" - }, - { - "source_row_index": 660, - "source_sample_id": "videomme:660" - }, - { - "source_row_index": 730, - "source_sample_id": "videomme:730" - }, - { - "source_row_index": 799, - "source_sample_id": "videomme:799" - }, - { - "source_row_index": 869, - "source_sample_id": "videomme:869" - }, - { - "source_row_index": 870, - "source_sample_id": "videomme:870" - }, - { - "source_row_index": 880, - "source_sample_id": "videomme:880" - }, - { - "source_row_index": 889, - "source_sample_id": "videomme:889" - }, - { - "source_row_index": 899, - "source_sample_id": "videomme:899" - }, - { - "source_row_index": 900, - "source_sample_id": "videomme:900" - }, - { - "source_row_index": 990, - "source_sample_id": "videomme:990" - }, - { - "source_row_index": 1079, - "source_sample_id": "videomme:1079" - }, - { - "source_row_index": 1169, - "source_sample_id": "videomme:1169" - }, - { - "source_row_index": 1170, - "source_sample_id": "videomme:1170" - }, - { - "source_row_index": 1210, - "source_sample_id": "videomme:1210" - }, - { - "source_row_index": 1249, - "source_sample_id": "videomme:1249" - }, - { - "source_row_index": 1289, - "source_sample_id": "videomme:1289" - }, - { - "source_row_index": 1290, - "source_sample_id": "videomme:1290" - }, - { - "source_row_index": 1340, - "source_sample_id": "videomme:1340" - }, - { - "source_row_index": 1389, - "source_sample_id": "videomme:1389" - }, - { - "source_row_index": 1439, - "source_sample_id": "videomme:1439" - }, - { - "source_row_index": 1440, - "source_sample_id": "videomme:1440" - }, - { - "source_row_index": 1480, - "source_sample_id": "videomme:1480" - }, - { - "source_row_index": 1519, - "source_sample_id": "videomme:1519" - }, - { - "source_row_index": 1559, - "source_sample_id": "videomme:1559" - }, - { - "source_row_index": 1560, - "source_sample_id": "videomme:1560" - }, - { - "source_row_index": 1630, - "source_sample_id": "videomme:1630" - }, - { - "source_row_index": 1699, - "source_sample_id": "videomme:1699" - }, - { - "source_row_index": 1769, - "source_sample_id": "videomme:1769" - }, - { - "source_row_index": 1770, - "source_sample_id": "videomme:1770" - }, - { - "source_row_index": 1780, - "source_sample_id": "videomme:1780" - }, - { - "source_row_index": 1789, - "source_sample_id": "videomme:1789" - }, - { - "source_row_index": 1799, - "source_sample_id": "videomme:1799" - }, - { - "source_row_index": 1800, - "source_sample_id": "videomme:1800" - }, - { - "source_row_index": 1890, - "source_sample_id": "videomme:1890" - }, - { - "source_row_index": 1979, - "source_sample_id": "videomme:1979" - }, - { - "source_row_index": 2069, - "source_sample_id": "videomme:2069" - }, - { - "source_row_index": 2070, - "source_sample_id": "videomme:2070" - }, - { - "source_row_index": 2110, - "source_sample_id": "videomme:2110" - }, - { - "source_row_index": 2149, - "source_sample_id": "videomme:2149" - }, - { - "source_row_index": 2189, - "source_sample_id": "videomme:2189" - }, - { - "source_row_index": 2190, - "source_sample_id": "videomme:2190" - }, - { - "source_row_index": 2240, - "source_sample_id": "videomme:2240" - }, - { - "source_row_index": 2289, - "source_sample_id": "videomme:2289" - }, - { - "source_row_index": 2339, - "source_sample_id": "videomme:2339" - }, - { - "source_row_index": 2340, - "source_sample_id": "videomme:2340" - }, - { - "source_row_index": 2380, - "source_sample_id": "videomme:2380" - }, - { - "source_row_index": 2419, - "source_sample_id": "videomme:2419" - }, - { - "source_row_index": 2459, - "source_sample_id": "videomme:2459" - }, - { - "source_row_index": 2460, - "source_sample_id": "videomme:2460" - }, - { - "source_row_index": 2530, - "source_sample_id": "videomme:2530" - }, - { - "source_row_index": 2599, - "source_sample_id": "videomme:2599" - }, - { - "source_row_index": 2669, - "source_sample_id": "videomme:2669" - }, - { - "source_row_index": 2670, - "source_sample_id": "videomme:2670" - }, - { - "source_row_index": 2680, - "source_sample_id": "videomme:2680" - }, - { - "source_row_index": 2689, - "source_sample_id": "videomme:2689" - }, - { - "source_row_index": 2699, - "source_sample_id": "videomme:2699" - } - ] - }, - "longvideobench_val_v": { - "dataset_repository": "longvideobench/LongVideoBench", - "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", - "max_new_tokens": 32, - "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "longvideobench_val_v:0" - }, - { - "source_row_index": 20, - "source_sample_id": "longvideobench_val_v:20" - }, - { - "source_row_index": 40, - "source_sample_id": "longvideobench_val_v:40" - }, - { - "source_row_index": 60, - "source_sample_id": "longvideobench_val_v:60" - }, - { - "source_row_index": 80, - "source_sample_id": "longvideobench_val_v:80" - }, - { - "source_row_index": 100, - "source_sample_id": "longvideobench_val_v:100" - }, - { - "source_row_index": 120, - "source_sample_id": "longvideobench_val_v:120" - }, - { - "source_row_index": 140, - "source_sample_id": "longvideobench_val_v:140" - }, - { - "source_row_index": 160, - "source_sample_id": "longvideobench_val_v:160" - }, - { - "source_row_index": 179, - "source_sample_id": "longvideobench_val_v:179" - }, - { - "source_row_index": 199, - "source_sample_id": "longvideobench_val_v:199" - }, - { - "source_row_index": 219, - "source_sample_id": "longvideobench_val_v:219" - }, - { - "source_row_index": 239, - "source_sample_id": "longvideobench_val_v:239" - }, - { - "source_row_index": 259, - "source_sample_id": "longvideobench_val_v:259" - }, - { - "source_row_index": 279, - "source_sample_id": "longvideobench_val_v:279" - }, - { - "source_row_index": 299, - "source_sample_id": "longvideobench_val_v:299" - }, - { - "source_row_index": 319, - "source_sample_id": "longvideobench_val_v:319" - }, - { - "source_row_index": 339, - "source_sample_id": "longvideobench_val_v:339" - }, - { - "source_row_index": 359, - "source_sample_id": "longvideobench_val_v:359" - }, - { - "source_row_index": 379, - "source_sample_id": "longvideobench_val_v:379" - }, - { - "source_row_index": 399, - "source_sample_id": "longvideobench_val_v:399" - }, - { - "source_row_index": 419, - "source_sample_id": "longvideobench_val_v:419" - }, - { - "source_row_index": 439, - "source_sample_id": "longvideobench_val_v:439" - }, - { - "source_row_index": 459, - "source_sample_id": "longvideobench_val_v:459" - }, - { - "source_row_index": 479, - "source_sample_id": "longvideobench_val_v:479" - }, - { - "source_row_index": 499, - "source_sample_id": "longvideobench_val_v:499" - }, - { - "source_row_index": 518, - "source_sample_id": "longvideobench_val_v:518" - }, - { - "source_row_index": 538, - "source_sample_id": "longvideobench_val_v:538" - }, - { - "source_row_index": 558, - "source_sample_id": "longvideobench_val_v:558" - }, - { - "source_row_index": 578, - "source_sample_id": "longvideobench_val_v:578" - }, - { - "source_row_index": 598, - "source_sample_id": "longvideobench_val_v:598" - }, - { - "source_row_index": 618, - "source_sample_id": "longvideobench_val_v:618" - }, - { - "source_row_index": 638, - "source_sample_id": "longvideobench_val_v:638" - }, - { - "source_row_index": 658, - "source_sample_id": "longvideobench_val_v:658" - }, - { - "source_row_index": 678, - "source_sample_id": "longvideobench_val_v:678" - }, - { - "source_row_index": 698, - "source_sample_id": "longvideobench_val_v:698" - }, - { - "source_row_index": 718, - "source_sample_id": "longvideobench_val_v:718" - }, - { - "source_row_index": 738, - "source_sample_id": "longvideobench_val_v:738" - }, - { - "source_row_index": 758, - "source_sample_id": "longvideobench_val_v:758" - }, - { - "source_row_index": 778, - "source_sample_id": "longvideobench_val_v:778" - }, - { - "source_row_index": 798, - "source_sample_id": "longvideobench_val_v:798" - }, - { - "source_row_index": 818, - "source_sample_id": "longvideobench_val_v:818" - }, - { - "source_row_index": 837, - "source_sample_id": "longvideobench_val_v:837" - }, - { - "source_row_index": 857, - "source_sample_id": "longvideobench_val_v:857" - }, - { - "source_row_index": 877, - "source_sample_id": "longvideobench_val_v:877" - }, - { - "source_row_index": 897, - "source_sample_id": "longvideobench_val_v:897" - }, - { - "source_row_index": 917, - "source_sample_id": "longvideobench_val_v:917" - }, - { - "source_row_index": 937, - "source_sample_id": "longvideobench_val_v:937" - }, - { - "source_row_index": 957, - "source_sample_id": "longvideobench_val_v:957" - }, - { - "source_row_index": 977, - "source_sample_id": "longvideobench_val_v:977" - }, - { - "source_row_index": 997, - "source_sample_id": "longvideobench_val_v:997" - }, - { - "source_row_index": 1017, - "source_sample_id": "longvideobench_val_v:1017" - }, - { - "source_row_index": 1037, - "source_sample_id": "longvideobench_val_v:1037" - }, - { - "source_row_index": 1057, - "source_sample_id": "longvideobench_val_v:1057" - }, - { - "source_row_index": 1077, - "source_sample_id": "longvideobench_val_v:1077" - }, - { - "source_row_index": 1097, - "source_sample_id": "longvideobench_val_v:1097" - }, - { - "source_row_index": 1117, - "source_sample_id": "longvideobench_val_v:1117" - }, - { - "source_row_index": 1137, - "source_sample_id": "longvideobench_val_v:1137" - }, - { - "source_row_index": 1157, - "source_sample_id": "longvideobench_val_v:1157" - }, - { - "source_row_index": 1176, - "source_sample_id": "longvideobench_val_v:1176" - }, - { - "source_row_index": 1196, - "source_sample_id": "longvideobench_val_v:1196" - }, - { - "source_row_index": 1216, - "source_sample_id": "longvideobench_val_v:1216" - }, - { - "source_row_index": 1236, - "source_sample_id": "longvideobench_val_v:1236" - }, - { - "source_row_index": 1256, - "source_sample_id": "longvideobench_val_v:1256" - }, - { - "source_row_index": 1276, - "source_sample_id": "longvideobench_val_v:1276" - }, - { - "source_row_index": 1296, - "source_sample_id": "longvideobench_val_v:1296" - }, - { - "source_row_index": 1316, - "source_sample_id": "longvideobench_val_v:1316" - }, - { - "source_row_index": 1336, - "source_sample_id": "longvideobench_val_v:1336" - } - ] - }, - "mlvu_dev": { - "dataset_repository": "sy1998/MLVU_dev", - "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", - "max_new_tokens": 16, - "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "mlvu_dev:0" - }, - { - "source_row_index": 39, - "source_sample_id": "mlvu_dev:39" - }, - { - "source_row_index": 79, - "source_sample_id": "mlvu_dev:79" - }, - { - "source_row_index": 118, - "source_sample_id": "mlvu_dev:118" - }, - { - "source_row_index": 157, - "source_sample_id": "mlvu_dev:157" - }, - { - "source_row_index": 197, - "source_sample_id": "mlvu_dev:197" - }, - { - "source_row_index": 236, - "source_sample_id": "mlvu_dev:236" - }, - { - "source_row_index": 275, - "source_sample_id": "mlvu_dev:275" - }, - { - "source_row_index": 315, - "source_sample_id": "mlvu_dev:315" - }, - { - "source_row_index": 354, - "source_sample_id": "mlvu_dev:354" - }, - { - "source_row_index": 355, - "source_sample_id": "mlvu_dev:355" - }, - { - "source_row_index": 377, - "source_sample_id": "mlvu_dev:377" - }, - { - "source_row_index": 399, - "source_sample_id": "mlvu_dev:399" - }, - { - "source_row_index": 421, - "source_sample_id": "mlvu_dev:421" - }, - { - "source_row_index": 443, - "source_sample_id": "mlvu_dev:443" - }, - { - "source_row_index": 466, - "source_sample_id": "mlvu_dev:466" - }, - { - "source_row_index": 488, - "source_sample_id": "mlvu_dev:488" - }, - { - "source_row_index": 510, - "source_sample_id": "mlvu_dev:510" - }, - { - "source_row_index": 532, - "source_sample_id": "mlvu_dev:532" - }, - { - "source_row_index": 554, - "source_sample_id": "mlvu_dev:554" - }, - { - "source_row_index": 555, - "source_sample_id": "mlvu_dev:555" - }, - { - "source_row_index": 578, - "source_sample_id": "mlvu_dev:578" - }, - { - "source_row_index": 601, - "source_sample_id": "mlvu_dev:601" - }, - { - "source_row_index": 623, - "source_sample_id": "mlvu_dev:623" - }, - { - "source_row_index": 646, - "source_sample_id": "mlvu_dev:646" - }, - { - "source_row_index": 669, - "source_sample_id": "mlvu_dev:669" - }, - { - "source_row_index": 692, - "source_sample_id": "mlvu_dev:692" - }, - { - "source_row_index": 714, - "source_sample_id": "mlvu_dev:714" - }, - { - "source_row_index": 737, - "source_sample_id": "mlvu_dev:737" - }, - { - "source_row_index": 760, - "source_sample_id": "mlvu_dev:760" - }, - { - "source_row_index": 761, - "source_sample_id": "mlvu_dev:761" - }, - { - "source_row_index": 800, - "source_sample_id": "mlvu_dev:800" - }, - { - "source_row_index": 839, - "source_sample_id": "mlvu_dev:839" - }, - { - "source_row_index": 878, - "source_sample_id": "mlvu_dev:878" - }, - { - "source_row_index": 917, - "source_sample_id": "mlvu_dev:917" - }, - { - "source_row_index": 956, - "source_sample_id": "mlvu_dev:956" - }, - { - "source_row_index": 995, - "source_sample_id": "mlvu_dev:995" - }, - { - "source_row_index": 1034, - "source_sample_id": "mlvu_dev:1034" - }, - { - "source_row_index": 1073, - "source_sample_id": "mlvu_dev:1073" - }, - { - "source_row_index": 1112, - "source_sample_id": "mlvu_dev:1112" - }, - { - "source_row_index": 1113, - "source_sample_id": "mlvu_dev:1113" - }, - { - "source_row_index": 1173, - "source_sample_id": "mlvu_dev:1173" - }, - { - "source_row_index": 1233, - "source_sample_id": "mlvu_dev:1233" - }, - { - "source_row_index": 1292, - "source_sample_id": "mlvu_dev:1292" - }, - { - "source_row_index": 1352, - "source_sample_id": "mlvu_dev:1352" - }, - { - "source_row_index": 1412, - "source_sample_id": "mlvu_dev:1412" - }, - { - "source_row_index": 1472, - "source_sample_id": "mlvu_dev:1472" - }, - { - "source_row_index": 1531, - "source_sample_id": "mlvu_dev:1531" - }, - { - "source_row_index": 1591, - "source_sample_id": "mlvu_dev:1591" - }, - { - "source_row_index": 1651, - "source_sample_id": "mlvu_dev:1651" - }, - { - "source_row_index": 1652, - "source_sample_id": "mlvu_dev:1652" - }, - { - "source_row_index": 1681, - "source_sample_id": "mlvu_dev:1681" - }, - { - "source_row_index": 1710, - "source_sample_id": "mlvu_dev:1710" - }, - { - "source_row_index": 1739, - "source_sample_id": "mlvu_dev:1739" - }, - { - "source_row_index": 1768, - "source_sample_id": "mlvu_dev:1768" - }, - { - "source_row_index": 1798, - "source_sample_id": "mlvu_dev:1798" - }, - { - "source_row_index": 1827, - "source_sample_id": "mlvu_dev:1827" - }, - { - "source_row_index": 1856, - "source_sample_id": "mlvu_dev:1856" - }, - { - "source_row_index": 1885, - "source_sample_id": "mlvu_dev:1885" - }, - { - "source_row_index": 1914, - "source_sample_id": "mlvu_dev:1914" - }, - { - "source_row_index": 1915, - "source_sample_id": "mlvu_dev:1915" - }, - { - "source_row_index": 1944, - "source_sample_id": "mlvu_dev:1944" - }, - { - "source_row_index": 1972, - "source_sample_id": "mlvu_dev:1972" - }, - { - "source_row_index": 2001, - "source_sample_id": "mlvu_dev:2001" - }, - { - "source_row_index": 2030, - "source_sample_id": "mlvu_dev:2030" - }, - { - "source_row_index": 2058, - "source_sample_id": "mlvu_dev:2058" - }, - { - "source_row_index": 2087, - "source_sample_id": "mlvu_dev:2087" - }, - { - "source_row_index": 2116, - "source_sample_id": "mlvu_dev:2116" - }, - { - "source_row_index": 2144, - "source_sample_id": "mlvu_dev:2144" - }, - { - "source_row_index": 2173, - "source_sample_id": "mlvu_dev:2173" - } - ] - }, - "perceptiontest_val_mc": { - "dataset_repository": "lmms-lab/PerceptionTest_Val", - "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", - "max_new_tokens": 16, - "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "perceptiontest_val_mc:0" - }, - { - "source_row_index": 1, - "source_sample_id": "perceptiontest_val_mc:1" - }, - { - "source_row_index": 3, - "source_sample_id": "perceptiontest_val_mc:3" - }, - { - "source_row_index": 20, - "source_sample_id": "perceptiontest_val_mc:20" - }, - { - "source_row_index": 211, - "source_sample_id": "perceptiontest_val_mc:211" - }, - { - "source_row_index": 544, - "source_sample_id": "perceptiontest_val_mc:544" - }, - { - "source_row_index": 753, - "source_sample_id": "perceptiontest_val_mc:753" - }, - { - "source_row_index": 1672, - "source_sample_id": "perceptiontest_val_mc:1672" - }, - { - "source_row_index": 2310, - "source_sample_id": "perceptiontest_val_mc:2310" - }, - { - "source_row_index": 2386, - "source_sample_id": "perceptiontest_val_mc:2386" - }, - { - "source_row_index": 2846, - "source_sample_id": "perceptiontest_val_mc:2846" - }, - { - "source_row_index": 2894, - "source_sample_id": "perceptiontest_val_mc:2894" - }, - { - "source_row_index": 3857, - "source_sample_id": "perceptiontest_val_mc:3857" - }, - { - "source_row_index": 3905, - "source_sample_id": "perceptiontest_val_mc:3905" - }, - { - "source_row_index": 3957, - "source_sample_id": "perceptiontest_val_mc:3957" - }, - { - "source_row_index": 4004, - "source_sample_id": "perceptiontest_val_mc:4004" - }, - { - "source_row_index": 4031, - "source_sample_id": "perceptiontest_val_mc:4031" - }, - { - "source_row_index": 4053, - "source_sample_id": "perceptiontest_val_mc:4053" - }, - { - "source_row_index": 4167, - "source_sample_id": "perceptiontest_val_mc:4167" - }, - { - "source_row_index": 4228, - "source_sample_id": "perceptiontest_val_mc:4228" - }, - { - "source_row_index": 4994, - "source_sample_id": "perceptiontest_val_mc:4994" - }, - { - "source_row_index": 5673, - "source_sample_id": "perceptiontest_val_mc:5673" - }, - { - "source_row_index": 5898, - "source_sample_id": "perceptiontest_val_mc:5898" - }, - { - "source_row_index": 6414, - "source_sample_id": "perceptiontest_val_mc:6414" - }, - { - "source_row_index": 6502, - "source_sample_id": "perceptiontest_val_mc:6502" - }, - { - "source_row_index": 6598, - "source_sample_id": "perceptiontest_val_mc:6598" - }, - { - "source_row_index": 7494, - "source_sample_id": "perceptiontest_val_mc:7494" - }, - { - "source_row_index": 8074, - "source_sample_id": "perceptiontest_val_mc:8074" - }, - { - "source_row_index": 8617, - "source_sample_id": "perceptiontest_val_mc:8617" - }, - { - "source_row_index": 8763, - "source_sample_id": "perceptiontest_val_mc:8763" - }, - { - "source_row_index": 8865, - "source_sample_id": "perceptiontest_val_mc:8865" - }, - { - "source_row_index": 8873, - "source_sample_id": "perceptiontest_val_mc:8873" - }, - { - "source_row_index": 8981, - "source_sample_id": "perceptiontest_val_mc:8981" - }, - { - "source_row_index": 9350, - "source_sample_id": "perceptiontest_val_mc:9350" - }, - { - "source_row_index": 9617, - "source_sample_id": "perceptiontest_val_mc:9617" - }, - { - "source_row_index": 9749, - "source_sample_id": "perceptiontest_val_mc:9749" - }, - { - "source_row_index": 10470, - "source_sample_id": "perceptiontest_val_mc:10470" - }, - { - "source_row_index": 11697, - "source_sample_id": "perceptiontest_val_mc:11697" - }, - { - "source_row_index": 11745, - "source_sample_id": "perceptiontest_val_mc:11745" - }, - { - "source_row_index": 11798, - "source_sample_id": "perceptiontest_val_mc:11798" - }, - { - "source_row_index": 13051, - "source_sample_id": "perceptiontest_val_mc:13051" - }, - { - "source_row_index": 13244, - "source_sample_id": "perceptiontest_val_mc:13244" - }, - { - "source_row_index": 13581, - "source_sample_id": "perceptiontest_val_mc:13581" - }, - { - "source_row_index": 13929, - "source_sample_id": "perceptiontest_val_mc:13929" - }, - { - "source_row_index": 13999, - "source_sample_id": "perceptiontest_val_mc:13999" - }, - { - "source_row_index": 14073, - "source_sample_id": "perceptiontest_val_mc:14073" - }, - { - "source_row_index": 14200, - "source_sample_id": "perceptiontest_val_mc:14200" - }, - { - "source_row_index": 14884, - "source_sample_id": "perceptiontest_val_mc:14884" - }, - { - "source_row_index": 15201, - "source_sample_id": "perceptiontest_val_mc:15201" - }, - { - "source_row_index": 15432, - "source_sample_id": "perceptiontest_val_mc:15432" - }, - { - "source_row_index": 16191, - "source_sample_id": "perceptiontest_val_mc:16191" - }, - { - "source_row_index": 16280, - "source_sample_id": "perceptiontest_val_mc:16280" - }, - { - "source_row_index": 16741, - "source_sample_id": "perceptiontest_val_mc:16741" - }, - { - "source_row_index": 17109, - "source_sample_id": "perceptiontest_val_mc:17109" - }, - { - "source_row_index": 17820, - "source_sample_id": "perceptiontest_val_mc:17820" - }, - { - "source_row_index": 17866, - "source_sample_id": "perceptiontest_val_mc:17866" - }, - { - "source_row_index": 18478, - "source_sample_id": "perceptiontest_val_mc:18478" - }, - { - "source_row_index": 18489, - "source_sample_id": "perceptiontest_val_mc:18489" - }, - { - "source_row_index": 18867, - "source_sample_id": "perceptiontest_val_mc:18867" - }, - { - "source_row_index": 19021, - "source_sample_id": "perceptiontest_val_mc:19021" - }, - { - "source_row_index": 19102, - "source_sample_id": "perceptiontest_val_mc:19102" - }, - { - "source_row_index": 19135, - "source_sample_id": "perceptiontest_val_mc:19135" - }, - { - "source_row_index": 19137, - "source_sample_id": "perceptiontest_val_mc:19137" - }, - { - "source_row_index": 19139, - "source_sample_id": "perceptiontest_val_mc:19139" - } - ] - } - } + "sample_set": "judge-free-8_690-examples_legacy-r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-qwen-3.5-native_r1" } diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json index 7c08a8a5b26..330a1369698 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json @@ -1,11 +1,7 @@ { - "extends": "short-v1", - "schema": "modelopt.vlm-evaluation-profile/v1", + "schema": "modelopt.vlm-evaluation-profile/v2", "name": "short-native-v1", - "lmms_eval_revision": "88b23e2bfa16a1edbc16e9e238ed82130b3a4f56", - "backend": { - "attention_implementation": "sdpa", - "enable_thinking": false, - "name": "qwen3_5" - } + "sample_set": "core-3_344-examples_legacy-r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-qwen-3.5-native_r1" } diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-v1.json index ddae52d9f5d..9b7570fb5fa 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-v1.json @@ -1,1589 +1,7 @@ { - "schema": "modelopt.vlm-evaluation-profile/v1", + "schema": "modelopt.vlm-evaluation-profile/v2", "name": "short-v1", - "lmms_eval_revision": "15c32bfec165df13c269ddd3cda03b2ed9137825", - "model_family": { - "architecture": "Qwen3_5ForConditionalGeneration", - "model_type": "qwen3_5" - }, - "backend": { - "enable_thinking": false, - "name": "vllm", - "reasoning_parser": "qwen3" - }, - "preprocessing": { - "fps": 2, - "max_frames": 32, - "video_reader": "decord" - }, - "generation": { - "do_sample": false, - "temperature": 0 - }, - "seed": 42, - "repetitions": 1, - "batch_size": 1, - "selection": "exact-rows", - "tasks": { - "realworldqa": { - "dataset_repository": "lmms-lab/RealWorldQA", - "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", - "max_new_tokens": 16, - "scoring_task_config": "tasks/realworldqa/realworldqa.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "test:0" - }, - { - "source_row_index": 1, - "source_sample_id": "test:1" - }, - { - "source_row_index": 2, - "source_sample_id": "test:2" - }, - { - "source_row_index": 3, - "source_sample_id": "test:3" - }, - { - "source_row_index": 4, - "source_sample_id": "test:4" - }, - { - "source_row_index": 5, - "source_sample_id": "test:5" - }, - { - "source_row_index": 6, - "source_sample_id": "test:6" - }, - { - "source_row_index": 7, - "source_sample_id": "test:7" - }, - { - "source_row_index": 8, - "source_sample_id": "test:8" - }, - { - "source_row_index": 9, - "source_sample_id": "test:9" - }, - { - "source_row_index": 10, - "source_sample_id": "test:10" - }, - { - "source_row_index": 11, - "source_sample_id": "test:11" - }, - { - "source_row_index": 12, - "source_sample_id": "test:12" - }, - { - "source_row_index": 13, - "source_sample_id": "test:13" - }, - { - "source_row_index": 14, - "source_sample_id": "test:14" - }, - { - "source_row_index": 15, - "source_sample_id": "test:15" - }, - { - "source_row_index": 16, - "source_sample_id": "test:16" - }, - { - "source_row_index": 17, - "source_sample_id": "test:17" - }, - { - "source_row_index": 18, - "source_sample_id": "test:18" - }, - { - "source_row_index": 19, - "source_sample_id": "test:19" - }, - { - "source_row_index": 20, - "source_sample_id": "test:20" - }, - { - "source_row_index": 21, - "source_sample_id": "test:21" - }, - { - "source_row_index": 22, - "source_sample_id": "test:22" - }, - { - "source_row_index": 23, - "source_sample_id": "test:23" - }, - { - "source_row_index": 24, - "source_sample_id": "test:24" - }, - { - "source_row_index": 25, - "source_sample_id": "test:25" - }, - { - "source_row_index": 26, - "source_sample_id": "test:26" - }, - { - "source_row_index": 27, - "source_sample_id": "test:27" - }, - { - "source_row_index": 28, - "source_sample_id": "test:28" - }, - { - "source_row_index": 29, - "source_sample_id": "test:29" - }, - { - "source_row_index": 30, - "source_sample_id": "test:30" - }, - { - "source_row_index": 31, - "source_sample_id": "test:31" - }, - { - "source_row_index": 32, - "source_sample_id": "test:32" - }, - { - "source_row_index": 33, - "source_sample_id": "test:33" - }, - { - "source_row_index": 34, - "source_sample_id": "test:34" - }, - { - "source_row_index": 35, - "source_sample_id": "test:35" - }, - { - "source_row_index": 36, - "source_sample_id": "test:36" - }, - { - "source_row_index": 37, - "source_sample_id": "test:37" - }, - { - "source_row_index": 38, - "source_sample_id": "test:38" - }, - { - "source_row_index": 39, - "source_sample_id": "test:39" - }, - { - "source_row_index": 40, - "source_sample_id": "test:40" - }, - { - "source_row_index": 41, - "source_sample_id": "test:41" - }, - { - "source_row_index": 42, - "source_sample_id": "test:42" - }, - { - "source_row_index": 43, - "source_sample_id": "test:43" - }, - { - "source_row_index": 44, - "source_sample_id": "test:44" - }, - { - "source_row_index": 45, - "source_sample_id": "test:45" - }, - { - "source_row_index": 46, - "source_sample_id": "test:46" - }, - { - "source_row_index": 47, - "source_sample_id": "test:47" - }, - { - "source_row_index": 48, - "source_sample_id": "test:48" - }, - { - "source_row_index": 49, - "source_sample_id": "test:49" - }, - { - "source_row_index": 50, - "source_sample_id": "test:50" - }, - { - "source_row_index": 51, - "source_sample_id": "test:51" - }, - { - "source_row_index": 52, - "source_sample_id": "test:52" - }, - { - "source_row_index": 53, - "source_sample_id": "test:53" - }, - { - "source_row_index": 54, - "source_sample_id": "test:54" - }, - { - "source_row_index": 55, - "source_sample_id": "test:55" - }, - { - "source_row_index": 56, - "source_sample_id": "test:56" - }, - { - "source_row_index": 57, - "source_sample_id": "test:57" - }, - { - "source_row_index": 58, - "source_sample_id": "test:58" - }, - { - "source_row_index": 59, - "source_sample_id": "test:59" - }, - { - "source_row_index": 60, - "source_sample_id": "test:60" - }, - { - "source_row_index": 61, - "source_sample_id": "test:61" - }, - { - "source_row_index": 62, - "source_sample_id": "test:62" - }, - { - "source_row_index": 63, - "source_sample_id": "test:63" - } - ] - }, - "mmmu_val": { - "dataset_repository": "lmms-lab/MMMU", - "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", - "max_new_tokens": 128, - "scoring_task_config": "tasks/mmmu/mmmu_val.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "validation_Accounting_1" - }, - { - "source_row_index": 7, - "source_sample_id": "validation_Accounting_8" - }, - { - "source_row_index": 15, - "source_sample_id": "validation_Accounting_16" - }, - { - "source_row_index": 22, - "source_sample_id": "validation_Accounting_23" - }, - { - "source_row_index": 30, - "source_sample_id": "validation_Agriculture_1" - }, - { - "source_row_index": 37, - "source_sample_id": "validation_Agriculture_8" - }, - { - "source_row_index": 45, - "source_sample_id": "validation_Agriculture_16" - }, - { - "source_row_index": 52, - "source_sample_id": "validation_Agriculture_23" - }, - { - "source_row_index": 60, - "source_sample_id": "validation_Architecture_and_Engineering_1" - }, - { - "source_row_index": 67, - "source_sample_id": "validation_Architecture_and_Engineering_8" - }, - { - "source_row_index": 75, - "source_sample_id": "validation_Architecture_and_Engineering_16" - }, - { - "source_row_index": 82, - "source_sample_id": "validation_Architecture_and_Engineering_23" - }, - { - "source_row_index": 90, - "source_sample_id": "validation_Art_1" - }, - { - "source_row_index": 97, - "source_sample_id": "validation_Art_8" - }, - { - "source_row_index": 105, - "source_sample_id": "validation_Art_16" - }, - { - "source_row_index": 112, - "source_sample_id": "validation_Art_23" - }, - { - "source_row_index": 120, - "source_sample_id": "validation_Art_Theory_1" - }, - { - "source_row_index": 127, - "source_sample_id": "validation_Art_Theory_8" - }, - { - "source_row_index": 135, - "source_sample_id": "validation_Art_Theory_16" - }, - { - "source_row_index": 142, - "source_sample_id": "validation_Art_Theory_23" - }, - { - "source_row_index": 150, - "source_sample_id": "validation_Basic_Medical_Science_1" - }, - { - "source_row_index": 157, - "source_sample_id": "validation_Basic_Medical_Science_8" - }, - { - "source_row_index": 165, - "source_sample_id": "validation_Basic_Medical_Science_16" - }, - { - "source_row_index": 172, - "source_sample_id": "validation_Basic_Medical_Science_23" - }, - { - "source_row_index": 180, - "source_sample_id": "validation_Biology_1" - }, - { - "source_row_index": 187, - "source_sample_id": "validation_Biology_8" - }, - { - "source_row_index": 195, - "source_sample_id": "validation_Biology_16" - }, - { - "source_row_index": 202, - "source_sample_id": "validation_Biology_23" - }, - { - "source_row_index": 210, - "source_sample_id": "validation_Chemistry_1" - }, - { - "source_row_index": 217, - "source_sample_id": "validation_Chemistry_8" - }, - { - "source_row_index": 225, - "source_sample_id": "validation_Chemistry_16" - }, - { - "source_row_index": 232, - "source_sample_id": "validation_Chemistry_23" - }, - { - "source_row_index": 240, - "source_sample_id": "validation_Clinical_Medicine_1" - }, - { - "source_row_index": 247, - "source_sample_id": "validation_Clinical_Medicine_8" - }, - { - "source_row_index": 255, - "source_sample_id": "validation_Clinical_Medicine_16" - }, - { - "source_row_index": 262, - "source_sample_id": "validation_Clinical_Medicine_23" - }, - { - "source_row_index": 270, - "source_sample_id": "validation_Computer_Science_1" - }, - { - "source_row_index": 277, - "source_sample_id": "validation_Computer_Science_8" - }, - { - "source_row_index": 285, - "source_sample_id": "validation_Computer_Science_16" - }, - { - "source_row_index": 292, - "source_sample_id": "validation_Computer_Science_23" - }, - { - "source_row_index": 300, - "source_sample_id": "validation_Design_1" - }, - { - "source_row_index": 307, - "source_sample_id": "validation_Design_8" - }, - { - "source_row_index": 315, - "source_sample_id": "validation_Design_16" - }, - { - "source_row_index": 322, - "source_sample_id": "validation_Design_23" - }, - { - "source_row_index": 330, - "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_1" - }, - { - "source_row_index": 337, - "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_8" - }, - { - "source_row_index": 345, - "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_16" - }, - { - "source_row_index": 352, - "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_23" - }, - { - "source_row_index": 360, - "source_sample_id": "validation_Economics_1" - }, - { - "source_row_index": 367, - "source_sample_id": "validation_Economics_8" - }, - { - "source_row_index": 375, - "source_sample_id": "validation_Economics_16" - }, - { - "source_row_index": 382, - "source_sample_id": "validation_Economics_23" - }, - { - "source_row_index": 390, - "source_sample_id": "validation_Electronics_1" - }, - { - "source_row_index": 397, - "source_sample_id": "validation_Electronics_8" - }, - { - "source_row_index": 405, - "source_sample_id": "validation_Electronics_16" - }, - { - "source_row_index": 412, - "source_sample_id": "validation_Electronics_23" - }, - { - "source_row_index": 420, - "source_sample_id": "validation_Energy_and_Power_1" - }, - { - "source_row_index": 427, - "source_sample_id": "validation_Energy_and_Power_8" - }, - { - "source_row_index": 435, - "source_sample_id": "validation_Energy_and_Power_16" - }, - { - "source_row_index": 442, - "source_sample_id": "validation_Energy_and_Power_23" - }, - { - "source_row_index": 450, - "source_sample_id": "validation_Finance_1" - }, - { - "source_row_index": 457, - "source_sample_id": "validation_Finance_8" - }, - { - "source_row_index": 465, - "source_sample_id": "validation_Finance_16" - }, - { - "source_row_index": 472, - "source_sample_id": "validation_Finance_23" - }, - { - "source_row_index": 480, - "source_sample_id": "validation_Geography_1" - }, - { - "source_row_index": 487, - "source_sample_id": "validation_Geography_8" - }, - { - "source_row_index": 495, - "source_sample_id": "validation_Geography_16" - }, - { - "source_row_index": 502, - "source_sample_id": "validation_Geography_23" - }, - { - "source_row_index": 510, - "source_sample_id": "validation_History_1" - }, - { - "source_row_index": 517, - "source_sample_id": "validation_History_8" - }, - { - "source_row_index": 525, - "source_sample_id": "validation_History_16" - }, - { - "source_row_index": 532, - "source_sample_id": "validation_History_23" - }, - { - "source_row_index": 540, - "source_sample_id": "validation_Literature_1" - }, - { - "source_row_index": 547, - "source_sample_id": "validation_Literature_8" - }, - { - "source_row_index": 555, - "source_sample_id": "validation_Literature_16" - }, - { - "source_row_index": 562, - "source_sample_id": "validation_Literature_23" - }, - { - "source_row_index": 570, - "source_sample_id": "validation_Manage_1" - }, - { - "source_row_index": 577, - "source_sample_id": "validation_Manage_8" - }, - { - "source_row_index": 585, - "source_sample_id": "validation_Manage_16" - }, - { - "source_row_index": 592, - "source_sample_id": "validation_Manage_23" - }, - { - "source_row_index": 600, - "source_sample_id": "validation_Marketing_1" - }, - { - "source_row_index": 607, - "source_sample_id": "validation_Marketing_8" - }, - { - "source_row_index": 615, - "source_sample_id": "validation_Marketing_16" - }, - { - "source_row_index": 622, - "source_sample_id": "validation_Marketing_23" - }, - { - "source_row_index": 630, - "source_sample_id": "validation_Materials_1" - }, - { - "source_row_index": 637, - "source_sample_id": "validation_Materials_8" - }, - { - "source_row_index": 645, - "source_sample_id": "validation_Materials_16" - }, - { - "source_row_index": 652, - "source_sample_id": "validation_Materials_23" - }, - { - "source_row_index": 660, - "source_sample_id": "validation_Math_1" - }, - { - "source_row_index": 667, - "source_sample_id": "validation_Math_8" - }, - { - "source_row_index": 675, - "source_sample_id": "validation_Math_16" - }, - { - "source_row_index": 682, - "source_sample_id": "validation_Math_23" - }, - { - "source_row_index": 690, - "source_sample_id": "validation_Mechanical_Engineering_1" - }, - { - "source_row_index": 697, - "source_sample_id": "validation_Mechanical_Engineering_8" - }, - { - "source_row_index": 705, - "source_sample_id": "validation_Mechanical_Engineering_16" - }, - { - "source_row_index": 712, - "source_sample_id": "validation_Mechanical_Engineering_23" - }, - { - "source_row_index": 720, - "source_sample_id": "validation_Music_1" - }, - { - "source_row_index": 727, - "source_sample_id": "validation_Music_8" - }, - { - "source_row_index": 735, - "source_sample_id": "validation_Music_16" - }, - { - "source_row_index": 742, - "source_sample_id": "validation_Music_23" - }, - { - "source_row_index": 750, - "source_sample_id": "validation_Pharmacy_1" - }, - { - "source_row_index": 757, - "source_sample_id": "validation_Pharmacy_8" - }, - { - "source_row_index": 765, - "source_sample_id": "validation_Pharmacy_16" - }, - { - "source_row_index": 772, - "source_sample_id": "validation_Pharmacy_23" - }, - { - "source_row_index": 780, - "source_sample_id": "validation_Physics_1" - }, - { - "source_row_index": 787, - "source_sample_id": "validation_Physics_8" - }, - { - "source_row_index": 795, - "source_sample_id": "validation_Physics_16" - }, - { - "source_row_index": 802, - "source_sample_id": "validation_Physics_23" - }, - { - "source_row_index": 810, - "source_sample_id": "validation_Psychology_1" - }, - { - "source_row_index": 817, - "source_sample_id": "validation_Psychology_8" - }, - { - "source_row_index": 825, - "source_sample_id": "validation_Psychology_16" - }, - { - "source_row_index": 832, - "source_sample_id": "validation_Psychology_23" - }, - { - "source_row_index": 840, - "source_sample_id": "validation_Public_Health_1" - }, - { - "source_row_index": 847, - "source_sample_id": "validation_Public_Health_8" - }, - { - "source_row_index": 855, - "source_sample_id": "validation_Public_Health_16" - }, - { - "source_row_index": 862, - "source_sample_id": "validation_Public_Health_23" - }, - { - "source_row_index": 870, - "source_sample_id": "validation_Sociology_1" - }, - { - "source_row_index": 877, - "source_sample_id": "validation_Sociology_8" - }, - { - "source_row_index": 885, - "source_sample_id": "validation_Sociology_16" - }, - { - "source_row_index": 892, - "source_sample_id": "validation_Sociology_23" - } - ] - }, - "mvbench": { - "dataset_repository": "OpenGVLab/MVBench", - "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", - "max_new_tokens": 16, - "scoring_task_config": "tasks/mvbench/mvbench.yaml", - "rows": [ - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 0, - "source_sample_id": "action_sequence:0" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 1, - "source_sample_id": "action_sequence:1" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 2, - "source_sample_id": "action_sequence:2" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 3, - "source_sample_id": "action_sequence:3" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 4, - "source_sample_id": "action_sequence:4" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 5, - "source_sample_id": "action_sequence:5" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 6, - "source_sample_id": "action_sequence:6" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 7, - "source_sample_id": "action_sequence:7" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 0, - "source_sample_id": "moving_count:0" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 1, - "source_sample_id": "moving_count:1" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 2, - "source_sample_id": "moving_count:2" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 3, - "source_sample_id": "moving_count:3" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 4, - "source_sample_id": "moving_count:4" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 5, - "source_sample_id": "moving_count:5" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 6, - "source_sample_id": "moving_count:6" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 7, - "source_sample_id": "moving_count:7" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 0, - "source_sample_id": "action_prediction:0" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 1, - "source_sample_id": "action_prediction:1" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 2, - "source_sample_id": "action_prediction:2" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 3, - "source_sample_id": "action_prediction:3" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 4, - "source_sample_id": "action_prediction:4" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 5, - "source_sample_id": "action_prediction:5" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 6, - "source_sample_id": "action_prediction:6" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 7, - "source_sample_id": "action_prediction:7" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 0, - "source_sample_id": "episodic_reasoning:0" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 1, - "source_sample_id": "episodic_reasoning:1" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 2, - "source_sample_id": "episodic_reasoning:2" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 3, - "source_sample_id": "episodic_reasoning:3" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 4, - "source_sample_id": "episodic_reasoning:4" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 5, - "source_sample_id": "episodic_reasoning:5" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 6, - "source_sample_id": "episodic_reasoning:6" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 7, - "source_sample_id": "episodic_reasoning:7" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 0, - "source_sample_id": "action_antonym:0" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 1, - "source_sample_id": "action_antonym:1" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 2, - "source_sample_id": "action_antonym:2" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 3, - "source_sample_id": "action_antonym:3" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 4, - "source_sample_id": "action_antonym:4" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 5, - "source_sample_id": "action_antonym:5" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 6, - "source_sample_id": "action_antonym:6" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 7, - "source_sample_id": "action_antonym:7" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 0, - "source_sample_id": "action_count:0" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 1, - "source_sample_id": "action_count:1" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 2, - "source_sample_id": "action_count:2" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 3, - "source_sample_id": "action_count:3" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 4, - "source_sample_id": "action_count:4" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 5, - "source_sample_id": "action_count:5" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 6, - "source_sample_id": "action_count:6" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 7, - "source_sample_id": "action_count:7" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 0, - "source_sample_id": "scene_transition:0" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 1, - "source_sample_id": "scene_transition:1" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 2, - "source_sample_id": "scene_transition:2" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 3, - "source_sample_id": "scene_transition:3" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 4, - "source_sample_id": "scene_transition:4" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 5, - "source_sample_id": "scene_transition:5" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 6, - "source_sample_id": "scene_transition:6" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 7, - "source_sample_id": "scene_transition:7" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 0, - "source_sample_id": "object_shuffle:0" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 1, - "source_sample_id": "object_shuffle:1" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 2, - "source_sample_id": "object_shuffle:2" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 3, - "source_sample_id": "object_shuffle:3" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 4, - "source_sample_id": "object_shuffle:4" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 5, - "source_sample_id": "object_shuffle:5" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 6, - "source_sample_id": "object_shuffle:6" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 7, - "source_sample_id": "object_shuffle:7" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 0, - "source_sample_id": "object_existence:0" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 1, - "source_sample_id": "object_existence:1" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 2, - "source_sample_id": "object_existence:2" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 3, - "source_sample_id": "object_existence:3" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 4, - "source_sample_id": "object_existence:4" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 5, - "source_sample_id": "object_existence:5" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 6, - "source_sample_id": "object_existence:6" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 7, - "source_sample_id": "object_existence:7" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 0, - "source_sample_id": "fine_grained_pose:0" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 1, - "source_sample_id": "fine_grained_pose:1" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 2, - "source_sample_id": "fine_grained_pose:2" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 3, - "source_sample_id": "fine_grained_pose:3" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 4, - "source_sample_id": "fine_grained_pose:4" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 5, - "source_sample_id": "fine_grained_pose:5" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 6, - "source_sample_id": "fine_grained_pose:6" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 7, - "source_sample_id": "fine_grained_pose:7" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 0, - "source_sample_id": "unexpected_action:0" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 1, - "source_sample_id": "unexpected_action:1" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 2, - "source_sample_id": "unexpected_action:2" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 3, - "source_sample_id": "unexpected_action:3" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 4, - "source_sample_id": "unexpected_action:4" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 5, - "source_sample_id": "unexpected_action:5" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 6, - "source_sample_id": "unexpected_action:6" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 7, - "source_sample_id": "unexpected_action:7" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 0, - "source_sample_id": "moving_direction:0" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 1, - "source_sample_id": "moving_direction:1" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 2, - "source_sample_id": "moving_direction:2" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 3, - "source_sample_id": "moving_direction:3" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 4, - "source_sample_id": "moving_direction:4" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 5, - "source_sample_id": "moving_direction:5" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 6, - "source_sample_id": "moving_direction:6" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 7, - "source_sample_id": "moving_direction:7" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 0, - "source_sample_id": "state_change:0" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 1, - "source_sample_id": "state_change:1" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 2, - "source_sample_id": "state_change:2" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 3, - "source_sample_id": "state_change:3" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 4, - "source_sample_id": "state_change:4" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 5, - "source_sample_id": "state_change:5" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 6, - "source_sample_id": "state_change:6" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 7, - "source_sample_id": "state_change:7" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 0, - "source_sample_id": "object_interaction:0" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 1, - "source_sample_id": "object_interaction:1" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 2, - "source_sample_id": "object_interaction:2" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 3, - "source_sample_id": "object_interaction:3" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 4, - "source_sample_id": "object_interaction:4" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 5, - "source_sample_id": "object_interaction:5" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 6, - "source_sample_id": "object_interaction:6" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 7, - "source_sample_id": "object_interaction:7" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 0, - "source_sample_id": "character_order:0" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 1, - "source_sample_id": "character_order:1" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 2, - "source_sample_id": "character_order:2" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 3, - "source_sample_id": "character_order:3" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 4, - "source_sample_id": "character_order:4" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 5, - "source_sample_id": "character_order:5" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 6, - "source_sample_id": "character_order:6" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 7, - "source_sample_id": "character_order:7" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 0, - "source_sample_id": "action_localization:0" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 1, - "source_sample_id": "action_localization:1" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 2, - "source_sample_id": "action_localization:2" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 3, - "source_sample_id": "action_localization:3" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 4, - "source_sample_id": "action_localization:4" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 5, - "source_sample_id": "action_localization:5" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 6, - "source_sample_id": "action_localization:6" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 7, - "source_sample_id": "action_localization:7" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 0, - "source_sample_id": "counterfactual_inference:0" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 1, - "source_sample_id": "counterfactual_inference:1" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 2, - "source_sample_id": "counterfactual_inference:2" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 3, - "source_sample_id": "counterfactual_inference:3" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 4, - "source_sample_id": "counterfactual_inference:4" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 5, - "source_sample_id": "counterfactual_inference:5" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 6, - "source_sample_id": "counterfactual_inference:6" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 7, - "source_sample_id": "counterfactual_inference:7" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 0, - "source_sample_id": "fine_grained_action:0" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 1, - "source_sample_id": "fine_grained_action:1" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 2, - "source_sample_id": "fine_grained_action:2" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 3, - "source_sample_id": "fine_grained_action:3" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 4, - "source_sample_id": "fine_grained_action:4" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 5, - "source_sample_id": "fine_grained_action:5" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 6, - "source_sample_id": "fine_grained_action:6" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 7, - "source_sample_id": "fine_grained_action:7" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 0, - "source_sample_id": "moving_attribute:0" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 1, - "source_sample_id": "moving_attribute:1" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 2, - "source_sample_id": "moving_attribute:2" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 3, - "source_sample_id": "moving_attribute:3" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 4, - "source_sample_id": "moving_attribute:4" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 5, - "source_sample_id": "moving_attribute:5" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 6, - "source_sample_id": "moving_attribute:6" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 7, - "source_sample_id": "moving_attribute:7" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 0, - "source_sample_id": "egocentric_navigation:0" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 1, - "source_sample_id": "egocentric_navigation:1" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 2, - "source_sample_id": "egocentric_navigation:2" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 3, - "source_sample_id": "egocentric_navigation:3" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 4, - "source_sample_id": "egocentric_navigation:4" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 5, - "source_sample_id": "egocentric_navigation:5" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 6, - "source_sample_id": "egocentric_navigation:6" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 7, - "source_sample_id": "egocentric_navigation:7" - } - ] - } - } + "sample_set": "core-3_344-examples_legacy-r1", + "backend_profile": "qwen-3.5-vllm_r1", + "evaluator_profile": "lmms-eval-legacy_r1" } diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py index 7e446581749..bcea2313a03 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py @@ -20,6 +20,7 @@ import json import os import random +import shutil import subprocess import sys from hashlib import sha256 @@ -704,33 +705,49 @@ def test_deprecated_suite_alias_records_the_canonical_identity(monkeypatch, tmp_ def test_versioned_profile_contracts_pin_backends_and_fingerprints(): profiles = {name: contracts.load_profile(name) for name in contracts.PROFILE_NAMES} + for name in profiles: + composition = json.loads((contracts._PROFILE_ROOT / f"{name}.json").read_text()) + assert set(composition) == { + "schema", + "name", + "sample_set", + "backend_profile", + "evaluator_profile", + } assert {name: contract.fingerprint for name, contract in profiles.items()} == { - "short-v1": "984c23ef0e7c05248895ece69c12327b3cdbb45051189ec540f7fc1ada763177", - "short-native-v1": "217b8ba8fd1df0002407e75f6e7d5588e3a871a6df2ad24117b66377894b2f35", - "short-native-v2": "d89134cdf4dfaaafe86b2fe9512bb183fa6528953fb5909e865bb961e95d4ee7", - "short-vllm-v2": "fc8a0a874fa2220610c50c33ee07fee4ce9e28c4032c0becdbc0059e1e1a58e8", - "smoke-native-v1": "734aea43cefa14776e016693a6c5ad2e32efed33fada2b3cdb856ce0cafa3271", - "smoke-vllm-v1": "abc0c2de8576717b50695e4f91ac239459a0f2789f45e01cddbe6d4976ead0e7", - "short-all-native-v1": "06b17ea010ee0cd789e49c581bcb3be4a7624c8471b4b3102bfa2922e0929e68", - "short-all-native-v2": "5db871b4cdd2f713161237f762619172ec7459318f1017de359faf96ff42fd36", - "full-v1": "29b1db6123ea3e16a9c5693e81e0f31607ff8a08e436681c66c32bf5dcc7e67a", - "core3-full-native-v1": ( - "82b053bcf74d7cfe5eab2bbbb94083b79c4ac16c0fff6f5a824a5d8b1079e06c" - ), - "core3-full-vllm-v1": ("6423b6fea1d988a4f9c79572bf17e47ad1d851760a90ee775033deeb68b700ec"), + "short-v1": "8286a094c3cfb5c2608a6e1469d6525bb1c4be11a7ab8789ed97dd249d7bea71", + "short-native-v1": "b15054c251af54a5298233b1c01a3babf76c4281d055469fd26b76129c34f258", + "core-3_344-examples_r1-native": "2017656d093de7d95d25c7e34241b1d708150157f0c4e6a0bf6bd48649c2191a", + "core-3_344-examples_r1-vllm": "859908fdb32b6bcaddb5400cd4430f4c9026264db38c7a8b56a98f42109c1f78", + "core-3_24-examples_r1-native": "0e51e27d57e27f0c5e4943d077308766387fa739b2d1c413b7b951327358cefc", + "core-3_24-examples_r1-vllm": "9c68168f05003e695258dc119351610b4e98bafb3e4f3e773c4e64ce5d17835a", + "short-all-native-v1": "9d7334371316a2a7774ee7e520ce0fc57e3c42ecfd7749ccd217e02ab59b6ee3", + "judge-free-8_690-examples_r1-native": "78457702288ba2d9d7b903366f7030302936377690b0ec37a0704e3eda8fd851", + "full-v1": "680483a7e2eceeab82a5e0b2767cc751f1951f58ffedc0aa190481d6ec978307", + "core-3_full_r1-native": "976efbd056fecb686e64b912ed50251b1c16e5efe1d3ae3d179f208cc587c0a7", + "core-3_full_r1-vllm": "40fd44fbb4812bd927d3f82e33d6eddec5b4641736c65c48250b8ea77acaec81", } - current_short = profiles["short-native-v2"] - smoke = profiles["smoke-native-v1"] - materialized_short = profiles["short-vllm-v2"] - materialized_smoke = profiles["smoke-vllm-v1"] + current_short = profiles["core-3_344-examples_r1-native"] + smoke = profiles["core-3_24-examples_r1-native"] + materialized_short = profiles["core-3_344-examples_r1-vllm"] + materialized_smoke = profiles["core-3_24-examples_r1-vllm"] assert current_short.manifest["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION assert current_short.manifest["backend"]["name"] == "qwen3_5" assert materialized_short.manifest["backend"]["name"] == "vllm" + assert materialized_short.sample_set == current_short.sample_set + assert materialized_short.backend_profile != current_short.backend_profile + assert materialized_short.evaluator_profile == current_short.evaluator_profile assert materialized_short.exact_rows == current_short.exact_rows assert materialized_smoke.manifest["backend"]["name"] == "vllm" assert materialized_smoke.manifest["backend"]["enforce_eager"] is True assert materialized_smoke.exact_rows == smoke.exact_rows + assert suites.manifest_selected_rows(current_short.exact_rows) == 344 + assert suites.manifest_selected_rows(smoke.exact_rows) == 24 + assert ( + suites.manifest_selected_rows(profiles["judge-free-8_690-examples_r1-native"].exact_rows) + == 690 + ) assert profiles["full-v1"].exact_rows is None @@ -738,7 +755,7 @@ def test_versioned_profile_contracts_pin_backends_and_fingerprints(): ("name", "backend"), [ ( - "core3-full-native-v1", + "core-3_full_r1-native", { "attention_implementation": "sdpa", "enable_thinking": False, @@ -746,7 +763,7 @@ def test_versioned_profile_contracts_pin_backends_and_fingerprints(): }, ), ( - "core3-full-vllm-v1", + "core-3_full_r1-vllm", {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, ), ], @@ -784,30 +801,33 @@ def test_core3_full_teacher_profiles_pin_paired_population_and_runtime(name, bac ("name", "field", "value", "message"), [ ( - "core3-full-native-v1", + "core-3_full_r1-native", "backend", {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, - "backend differs", + "backend profile differs", ), ( - "core3-full-vllm-v1", + "core-3_full_r1-vllm", "model", {"repository": "Qwen/Qwen3.5-0.8B", "revision": "different"}, "model pin differs", ), - ("core3-full-native-v1", "population", 764, "population differs"), + ("core-3_full_r1-native", "population", 764, "population differs"), ], ) def test_core3_full_teacher_profiles_reject_contract_overrides( monkeypatch, tmp_path, name, field, value, message ): - for profile_name in ("core3-full-native-v1", "core3-full-vllm-v1"): - source = contracts._PROFILE_ROOT / f"{profile_name}.json" - (tmp_path / source.name).write_text(source.read_text()) - manifest_path = tmp_path / f"{name}.json" + shutil.copytree(contracts._PROFILE_ROOT, tmp_path, dirs_exist_ok=True) + if field == "backend": + manifest_path = tmp_path / "backends" / "qwen-3.5-native_r1.json" + else: + manifest_path = tmp_path / "sample_sets" / "core-3_full_r1.json" manifest = json.loads(manifest_path.read_text()) if field == "population": manifest["tasks"]["realworldqa"]["population_rows"] = value + elif field == "backend": + manifest["settings"] = value else: manifest[field] = value manifest_path.write_text(json.dumps(manifest)) @@ -818,21 +838,19 @@ def test_core3_full_teacher_profiles_reject_contract_overrides( def test_audited_profile_rejects_rows_that_drift_from_systematic_selection(monkeypatch, tmp_path): - for name in ("short-v1", "short-native-v1", "short-native-v2"): - source = contracts._PROFILE_ROOT / f"{name}.json" - (tmp_path / source.name).write_text(source.read_text()) - manifest_path = tmp_path / "short-native-v2.json" + shutil.copytree(contracts._PROFILE_ROOT, tmp_path, dirs_exist_ok=True) + manifest_path = tmp_path / "sample_sets" / "core-3_344-examples_r1.json" manifest = json.loads(manifest_path.read_text()) manifest["tasks"]["realworldqa"]["rows"][0]["source_row_index"] = 6 manifest_path.write_text(json.dumps(manifest)) monkeypatch.setattr(contracts, "_PROFILE_ROOT", tmp_path) with pytest.raises(RuntimeError, match="rows differ from its sampling audit"): - contracts.load_profile("short-native-v2") + contracts.load_profile("core-3_344-examples_r1-native") def test_short_all_native_profile_builds_grouped_and_single_selectors(tmp_path): - contract = contracts.load_profile("short-all-native-v2") + contract = contracts.load_profile("judge-free-8_690-examples_r1-native") exact_rows = contract.exact_rows assert exact_rows is not None validated = suites.validate_exact_rows_manifest( @@ -993,6 +1011,9 @@ def test_versioned_profile_preflight_reports_immutable_contract(monkeypatch, tmp contract = contracts.load_profile(name) assert prepared.report["profile_name"] == name assert prepared.report["profile_fingerprint"] == contract.fingerprint + assert prepared.report["sample_set"] == contract.sample_set + assert prepared.report["backend_profile"] == contract.backend_profile + assert prepared.report["evaluator_profile"] == contract.evaluator_profile assert prepared.report["source_tasks"] == list(contract.source_tasks) assert prepared.report["quick_selected_rows"] == 344 assert prepared.report["quick_row_identities"] == suites.manifest_row_identities( @@ -1013,7 +1034,7 @@ def test_native_profile_builds_qwen35_backend_settings(monkeypatch, tmp_path): "--output-dir", str(tmp_path / "results"), "--profile", - "short-all-native-v2", + "judge-free-8_690-examples_r1-native", "--hf-home", str(hf_home), ] @@ -1045,7 +1066,7 @@ def test_native_profile_builds_qwen35_backend_settings(monkeypatch, tmp_path): @pytest.mark.parametrize( ("profile_name", "expected_eager"), - [("smoke-vllm-v1", True), ("short-vllm-v2", None)], + [("core-3_24-examples_r1-vllm", True), ("core-3_344-examples_r1-vllm", None)], ) def test_vllm_profile_forwards_runtime_settings( monkeypatch, tmp_path, profile_name, expected_eager @@ -1086,7 +1107,7 @@ def test_vllm_profile_forwards_runtime_settings( assert 'attention_config={"flash_attn_version":2}' in model_args -@pytest.mark.parametrize("name", ["core3-full-native-v1", "core3-full-vllm-v1"]) +@pytest.mark.parametrize("name", ["core-3_full_r1-native", "core-3_full_r1-vllm"]) def test_core3_full_teacher_profiles_preserve_backend_prompt_policy(monkeypatch, tmp_path, name): model, hf_home = _write_core3_teacher_snapshot(tmp_path) lmms_root = _write_lmms_tasks(tmp_path, ("mmmu_val",)) @@ -1122,7 +1143,7 @@ def test_core3_full_teacher_profiles_preserve_backend_prompt_policy(monkeypatch, } assert prepared.report["profile_population_rows"] == {"mmmu_val": 900} assert prepared.report["output_budget_contract"]["mmmu_val"]["effective_max_new_tokens"] == 128 - if name == "core3-full-native-v1": + if name == "core-3_full_r1-native": assert settings["model"] == "qwen3_5" assert prepared.report["backend_limitations"] == [] else: @@ -1158,7 +1179,7 @@ def test_core3_full_teacher_profile_population_expectations_follow_group_shard( "--output-dir", str(tmp_path / "results"), "--profile", - "core3-full-native-v1", + "core-3_full_r1-native", "--profile-task", "mvbench", "--profile-task-shard", @@ -1190,7 +1211,7 @@ def test_core3_full_teacher_profile_accepts_snapshot_symlink(monkeypatch, tmp_pa "--output-dir", str(tmp_path / "results"), "--profile", - "core3-full-native-v1", + "core-3_full_r1-native", "--profile-task", "realworldqa", "--hf-home", @@ -1222,7 +1243,7 @@ def test_core3_full_teacher_profile_rejects_unpinned_checkpoint(monkeypatch, tmp "--output-dir", str(tmp_path / "results"), "--profile", - "core3-full-native-v1", + "core-3_full_r1-native", "--profile-task", "realworldqa", "--hf-home", @@ -1245,7 +1266,7 @@ def test_core3_full_teacher_profile_rejects_settings_override(monkeypatch, tmp_p "--output-dir", str(tmp_path / "results"), "--profile", - "core3-full-native-v1", + "core-3_full_r1-native", "--profile-task", "realworldqa", "--hf-home", @@ -1370,7 +1391,7 @@ def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, t "--output-dir", str(tmp_path / "results"), "--profile", - "short-all-native-v2", + "judge-free-8_690-examples_r1-native", "--profile-task", "mvbench", "--profile-task-shard", @@ -1423,7 +1444,7 @@ def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, t ) assert ( prepared.report["profile_fingerprint"] - == contracts.load_profile("short-all-native-v2").fingerprint + == contracts.load_profile("judge-free-8_690-examples_r1-native").fingerprint ) tasks_root, _ = tasks.prepare( tmp_path / "results", @@ -1446,7 +1467,7 @@ def test_smoke_profile_generates_only_manifest_backed_mvbench_leaves(monkeypatch "--output-dir", str(tmp_path / "results"), "--profile", - "smoke-native-v1", + "core-3_24-examples_r1-native", "--hf-home", str(hf_home), ] From cd23101bc40913676c5634dba45b2cec46885d4b Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Fri, 4 Sep 2026 15:28:27 +0200 Subject: [PATCH 03/10] Preserve default VLM backend Signed-off-by: Johannes Rausch --- CHANGELOG.rst | 2 +- .../puzzletron/evaluation/vlm/preflight.py | 4 +-- .../puzzletron/evaluation/vlm/test_run.py | 36 ++++++++----------- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8360f03ab21..a00d3a89177 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,7 @@ Changelog **New Features** -- Add Puzzletron dynamic post-MIP downstream evaluation through ``lmms-eval`` with vLLM-backed checkpoint evaluation, setup-wizard topology/resource prompts, non-interactive setup automation, and an opt-in Nemotron-3 Nano 30B A3B BF16 example flow. +- Add Puzzletron dynamic post-MIP downstream evaluation through ``lmms-eval`` with vLLM-backed checkpoint evaluation, composable VLM sample-set/backend/evaluator profiles, offline benchmark preparation, setup-wizard topology/resource prompts, non-interactive setup automation, and an opt-in Nemotron-3 Nano 30B A3B BF16 example flow. - Add the ``day0-release`` agent skill (``.agents/skills/day0-release/``), a deterministic end-to-end driver that chains the PTQ → evaluation → comparison skills (the evaluation stage deploys the checkpoint itself) with an enforced gate after each stage and returns a publish decision (ACCEPT / REGRESSION / ANOMALOUS / INFEASIBLE). Ships three GPU-free, unit-tested gate scripts (``gate_ptq.py``, ``gate_run.py``, ``gate_compare.py``) that validate checkpoint coverage, evaluation-run completeness, and baseline-vs-candidate accuracy threshold. v1 reports and stops on regression; the recipe-search loop is deferred. - Add **streaming** speculative-decoding training (EAGLE3 / DFlash): the draft trains on base-model hidden states produced on the fly by a co-located ``vllm serve`` (no disk dump), moved trainer-side over NIXL RDMA, scaling to multi-node (dedicated serve replicas + DDP trainers). New launcher examples for NVFP4 Kimi-K2.5 / K2.6 on GB200/aarch64 under ``tools/launcher/examples/moonshotai/``. diff --git a/examples/puzzletron/evaluation/vlm/preflight.py b/examples/puzzletron/evaluation/vlm/preflight.py index 6542fc4f6e3..e30411fe7fa 100644 --- a/examples/puzzletron/evaluation/vlm/preflight.py +++ b/examples/puzzletron/evaluation/vlm/preflight.py @@ -56,9 +56,9 @@ def _backend_policy( """Return the selected profile's backend contract or the legacy default.""" if profile_contract is None: return { - "attention_implementation": "sdpa", "enable_thinking": False, - "name": "qwen3_5", + "name": "vllm", + "reasoning_parser": "qwen3", } return cast("dict[str, object]", profile_contract.manifest["backend"]) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py index bcea2313a03..f69fb7d05bd 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py @@ -435,9 +435,7 @@ def _quick_manifest(path: Path) -> Path: return path -def test_short_profile_materializes_pinned_tasks_and_native_qwen_backend( - monkeypatch, tmp_path, capsys -): +def test_short_profile_preserves_default_vllm_backend(monkeypatch, tmp_path, capsys): model = _write_checkpoint(tmp_path) source_tasks = ("realworldqa", "mmmu_val") lmms_root = _write_lmms_tasks(tmp_path, source_tasks) @@ -503,24 +501,18 @@ def fake_runner(checkpoint_path, *, output_root, settings): assert all(call["settings"]["tasks"] == ",".join(expected_tasks) for call in calls) assert [run["attempt"] for run in result["runs"]] == [1, 2] settings = calls[0]["settings"] - assert settings["model"] == "qwen3_5" - assert report["backend_limitations"] == [] - assert report["output_budget_contract"] == { - "mmmu_val": { - "adapter": "qwen3_5", - "effective_max_new_tokens": 128, - "limitation": None, - "requested_max_new_tokens": 128, - "resolution": "task_max_new_tokens_overrides_adapter_default", - }, - "realworldqa": { - "adapter": "qwen3_5", - "effective_max_new_tokens": 16, - "limitation": None, - "requested_max_new_tokens": 16, - "resolution": "task_max_new_tokens_overrides_adapter_default", - }, - } + assert settings["model"] == "vllm" + assert report["backend_limitations"] == [ + "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", + "pinned generic vLLM max_new_tokens is a model-level lower bound", + ] + for task, expected_tokens in {"mmmu_val": 128, "realworldqa": 16}.items(): + budget = report["output_budget_contract"][task] + assert budget["adapter"] == "vllm" + assert budget["requested_max_new_tokens"] == expected_tokens + assert budget["effective_max_new_tokens"] == expected_tokens + assert budget["limitation"] is not None + assert budget["resolution"] == "max(task_max_new_tokens, model_max_new_tokens_floor=1)" @pytest.mark.parametrize("suite", ["short", suites.TASK_PREFIX100_REPEAT2_SUITE]) @@ -630,7 +622,7 @@ def fake_runner(checkpoint_path, *, output_root, settings): def changed_backend_policy(profile_contract): return { **original_backend_policy(profile_contract), - "attention_implementation": "eager", + "enforce_eager": True, } monkeypatch.setattr(preflight, "_backend_policy", changed_backend_policy) From b35a482c5cd33df6fc7613ba1102b82e2a9ce91a Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Fri, 4 Sep 2026 17:52:24 +0200 Subject: [PATCH 04/10] Split and prune VLM evaluation tests Signed-off-by: Johannes Rausch --- .../puzzletron/evaluation/vlm/_test_utils.py | 170 ++ .../vlm/preparation/test_benchmark_data.py | 6 +- .../evaluation/vlm/test_contracts.py | 184 ++ .../evaluation/vlm/test_evaluator.py | 357 +++ .../puzzletron/evaluation/vlm/test_model.py | 180 ++ .../evaluation/vlm/test_post_mip.py | 211 ++ .../evaluation/vlm/test_preflight.py | 535 +++++ .../puzzletron/evaluation/vlm/test_run.py | 1991 +---------------- .../puzzletron/evaluation/vlm/test_tasks.py | 470 ++++ .../puzzletron/test_ci_image_contract.py | 24 +- 10 files changed, 2113 insertions(+), 2015 deletions(-) create mode 100644 tests/unit/torch/puzzletron/evaluation/vlm/_test_utils.py create mode 100644 tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py create mode 100644 tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py create mode 100644 tests/unit/torch/puzzletron/evaluation/vlm/test_model.py create mode 100644 tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py create mode 100644 tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py create mode 100644 tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/_test_utils.py b/tests/unit/torch/puzzletron/evaluation/vlm/_test_utils.py new file mode 100644 index 00000000000..18a1715ab59 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/_test_utils.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared local fixtures for VLM evaluation unit tests.""" + +import json +from pathlib import Path + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import preflight, profile, suites, tasks + +_QWEN_CONFIG = { + "architectures": ["Qwen3_5ForConditionalGeneration"], + "model_type": "qwen3_5", + "text_config": { + "hidden_size": 1024, + "intermediate_size": 3584, + "model_type": "qwen3_5_text", + "num_attention_heads": 8, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "vocab_size": 248320, + }, +} +_TASK_CONFIGS = {name: item.task_config for name, item in profile.VLM_BENCHMARK_DATASETS.items()} + + +def _write_checkpoint_at(model: Path) -> Path: + model.mkdir() + (model / "config.json").write_text(json.dumps(_QWEN_CONFIG) + "\n") + (model / "preprocessor_config.json").write_text("{}\n") + (model / "chat_template.jinja").write_text( + "{% if enable_thinking is defined and enable_thinking is false %}" + "\n\n\n\n{% else %}\n{% endif %}" + ) + return model + + +def _write_checkpoint(root: Path) -> Path: + return _write_checkpoint_at(root / "model") + + +def _write_core3_teacher_snapshot(root: Path) -> tuple[Path, Path]: + hf_home = root / "hf-home" + snapshot = ( + hf_home + / "hub/models--Qwen--Qwen3.5-0.8B/snapshots" + / "2fc06364715b967f1860aea9cf38778875588b17" + ) + snapshot.parent.mkdir(parents=True) + return _write_checkpoint_at(snapshot), hf_home + + +def _write_lmms_tasks(root: Path, tasks: tuple[str, ...]) -> Path: + lmms_root = root / "lmms_eval" + for task in tasks: + task_configs = [_TASK_CONFIGS[task]] + if task == "video_mmmu": + task_configs.extend( + f"tasks/videommmu/{leaf}.yaml" for leaf in suites.VIDEO_MMMU_LEAF_TASKS + ) + elif task == "mvbench": + task_configs.extend( + f"tasks/mvbench/mvbench_{leaf}.yaml" for leaf in suites.MVBENCH_LEAF_TASKS + ) + for relative_path in task_configs: + config = lmms_root / relative_path + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text(f"task: {task}\n") + return lmms_root + + +def _use_offline_fakes(monkeypatch, lmms_root: Path) -> None: + monkeypatch.delenv("HF_HUB_CACHE", raising=False) + monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: object()) + monkeypatch.setattr( + checkpoint, + "verify_lmms_eval_revision", + lambda expected=checkpoint.LMMS_EVAL_REVISION: expected, + ) + monkeypatch.setattr( + suites, + "offline_dataset_snapshot", + lambda hf_home, task, revision: hf_home / ".snapshots" / task / revision, + ) + monkeypatch.setattr( + tasks, + "verify_offline", + lambda _root, tasks, **_kwargs: { + "configured_tasks": list(tasks), + "status": "passed", + }, + ) + + +def _write_fake_mmmu_artifacts(result_path: Path) -> Path: + result_path.write_text(json.dumps({"sample_counts": {suites.task_name("mmmu_val"): 1}}) + "\n") + raw_result_path = result_path.parent / "run_results.json" + raw_result_path.write_text("{}\n") + sample_path = result_path.parent / f"run_samples_{suites.task_name('mmmu_val')}.jsonl" + sample_path.write_text(json.dumps({"mmmu_acc": {"parser_status": ["parsed"]}}) + "\n") + return raw_result_path + + +def _full_inputs(monkeypatch, tmp_path): + model = _write_checkpoint(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, profile.VLM_BENCHMARK_TASKS) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + for dataset in profile.VLM_BENCHMARK_VIDEO_DATASETS.values(): + media = hf_home / dataset.media_dir + media.mkdir(parents=True, exist_ok=True) + (media / "sample").write_bytes(b"media") + return model, hf_home + + +def _quick_manifest(path: Path) -> Path: + counts = {"realworldqa": 64, "mmmu_val": 120} + tasks = { + task: { + "dataset_revision": profile.VLM_BENCHMARK_DATASETS[task].revision, + "rows": [ + { + "source_row_index": index, + "source_sample_id": ( + f"test:{index}" if task == "realworldqa" else f"question-{index}" + ), + } + for index in range(count) + ], + } + for task, count in counts.items() + } + tasks["mvbench"] = { + "dataset_revision": profile.VLM_BENCHMARK_DATASETS["mvbench"].revision, + "rows": [ + { + "leaf_task": f"mvbench_{leaf}", + "source_row_index": index, + "source_sample_id": f"{leaf}:{index}", + } + for leaf in suites.MVBENCH_LEAF_TASKS + for index in range(8) + ], + } + path.write_text( + json.dumps( + { + "schema": "modelopt.vlm-benchmark-quick/v1", + "lmms_eval_revision": checkpoint.LMMS_EVAL_REVISION, + "tasks": tasks, + } + ) + + "\n" + ) + return path diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py index c4591a96467..7a8135068b4 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py @@ -233,12 +233,8 @@ def test_repair_without_atomic_exchange_preserves_live_root(monkeypatch, tmp_pat [ (None, None, "readable ownership marker"), ("corrupt", "{", "readable ownership marker"), - ("schema", "other/v1", "mismatched ownership: schema"), - ("task", "mvbench", "mismatched ownership: task"), - ("repository", "other/repository", "mismatched ownership: repository"), ("revision", "other-revision", "mismatched ownership: revision"), - ("requires_media", False, "mismatched ownership: requires_media"), - ("preparation_dir", "other-root", "mismatched ownership: preparation_dir"), + ("status", "unknown", "invalid ownership-marker status"), ], ) def test_missing_or_mismatched_media_marker_preserves_unproven_root( diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py new file mode 100644 index 00000000000..18f56588a76 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for composed VLM profiles and sample-set contracts.""" + +import json +import shutil + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import contracts, profile, suites +from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import _quick_manifest + + +def test_versioned_profile_contracts_pin_backends_and_fingerprints(): + profiles = {name: contracts.load_profile(name) for name in contracts.PROFILE_NAMES} + assert {name: contract.fingerprint for name, contract in profiles.items()} == { + "short-v1": "8286a094c3cfb5c2608a6e1469d6525bb1c4be11a7ab8789ed97dd249d7bea71", + "short-native-v1": "b15054c251af54a5298233b1c01a3babf76c4281d055469fd26b76129c34f258", + "core-3_344-examples_r1-native": "2017656d093de7d95d25c7e34241b1d708150157f0c4e6a0bf6bd48649c2191a", + "core-3_344-examples_r1-vllm": "859908fdb32b6bcaddb5400cd4430f4c9026264db38c7a8b56a98f42109c1f78", + "core-3_24-examples_r1-native": "0e51e27d57e27f0c5e4943d077308766387fa739b2d1c413b7b951327358cefc", + "core-3_24-examples_r1-vllm": "9c68168f05003e695258dc119351610b4e98bafb3e4f3e773c4e64ce5d17835a", + "short-all-native-v1": "9d7334371316a2a7774ee7e520ce0fc57e3c42ecfd7749ccd217e02ab59b6ee3", + "judge-free-8_690-examples_r1-native": "78457702288ba2d9d7b903366f7030302936377690b0ec37a0704e3eda8fd851", + "full-v1": "680483a7e2eceeab82a5e0b2767cc751f1951f58ffedc0aa190481d6ec978307", + "core-3_full_r1-native": "976efbd056fecb686e64b912ed50251b1c16e5efe1d3ae3d179f208cc587c0a7", + "core-3_full_r1-vllm": "40fd44fbb4812bd927d3f82e33d6eddec5b4641736c65c48250b8ea77acaec81", + } + + current_short = profiles["core-3_344-examples_r1-native"] + smoke = profiles["core-3_24-examples_r1-native"] + materialized_short = profiles["core-3_344-examples_r1-vllm"] + materialized_smoke = profiles["core-3_24-examples_r1-vllm"] + assert current_short.manifest["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION + assert current_short.manifest["backend"]["name"] == "qwen3_5" + assert materialized_short.manifest["backend"]["name"] == "vllm" + assert materialized_short.sample_set == current_short.sample_set + assert materialized_short.backend_profile != current_short.backend_profile + assert materialized_short.evaluator_profile == current_short.evaluator_profile + assert materialized_short.exact_rows == current_short.exact_rows + assert materialized_smoke.manifest["backend"]["name"] == "vllm" + assert materialized_smoke.manifest["backend"]["enforce_eager"] is True + assert materialized_smoke.exact_rows == smoke.exact_rows + assert suites.manifest_selected_rows(current_short.exact_rows) == 344 + assert suites.manifest_selected_rows(smoke.exact_rows) == 24 + assert ( + suites.manifest_selected_rows(profiles["judge-free-8_690-examples_r1-native"].exact_rows) + == 690 + ) + assert profiles["full-v1"].exact_rows is None + + +@pytest.mark.parametrize( + ("name", "backend"), + [ + ( + "core-3_full_r1-native", + { + "attention_implementation": "sdpa", + "enable_thinking": False, + "name": "qwen3_5", + }, + ), + ( + "core-3_full_r1-vllm", + {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, + ), + ], +) +def test_core3_full_teacher_profiles_pin_paired_population_and_runtime(name, backend): + contract = contracts.load_profile(name) + + assert contract.manifest["model"] == { + "repository": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17", + } + assert contract.manifest["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION + assert contract.manifest["backend"] == backend + assert contract.manifest["generation"] == {"do_sample": False, "temperature": 0} + assert contract.manifest["seed"] == 42 + assert contract.manifest["repetitions"] == 1 + assert contract.manifest["batch_size"] == 1 + assert contract.manifest["selection"] == "all" + assert contract.exact_rows is None + assert { + task: entry["population_rows"] for task, entry in contract.manifest["tasks"].items() + } == {"realworldqa": 765, "mmmu_val": 900, "mvbench": 4000} + assert contract.manifest["tasks"]["mvbench"]["leaf_populations"] == dict.fromkeys( + suites.MVBENCH_LEAF_TASKS, 200 + ) + assert { + task: entry["dataset_revision"] for task, entry in contract.manifest["tasks"].items() + } == { + task: profile.VLM_BENCHMARK_DATASETS[task].revision + for task in ("realworldqa", "mmmu_val", "mvbench") + } + + +@pytest.mark.parametrize( + ("name", "field", "value", "message"), + [ + ( + "core-3_full_r1-native", + "backend", + {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, + "backend profile differs", + ), + ( + "core-3_full_r1-vllm", + "model", + {"repository": "Qwen/Qwen3.5-0.8B", "revision": "different"}, + "model pin differs", + ), + ("core-3_full_r1-native", "population", 764, "population differs"), + ], +) +def test_core3_full_teacher_profiles_reject_contract_overrides( + monkeypatch, tmp_path, name, field, value, message +): + shutil.copytree(contracts._PROFILE_ROOT, tmp_path, dirs_exist_ok=True) + if field == "backend": + manifest_path = tmp_path / "backends" / "qwen-3.5-native_r1.json" + else: + manifest_path = tmp_path / "sample_sets" / "core-3_full_r1.json" + manifest = json.loads(manifest_path.read_text()) + if field == "population": + manifest["tasks"]["realworldqa"]["population_rows"] = value + elif field == "backend": + manifest["settings"] = value + else: + manifest[field] = value + manifest_path.write_text(json.dumps(manifest)) + monkeypatch.setattr(contracts, "_PROFILE_ROOT", tmp_path) + + with pytest.raises(RuntimeError, match=message): + contracts.load_profile(name) + + +def test_audited_profile_rejects_rows_that_drift_from_systematic_selection(monkeypatch, tmp_path): + shutil.copytree(contracts._PROFILE_ROOT, tmp_path, dirs_exist_ok=True) + manifest_path = tmp_path / "sample_sets" / "core-3_344-examples_r1.json" + manifest = json.loads(manifest_path.read_text()) + manifest["tasks"]["realworldqa"]["rows"][0]["source_row_index"] = 6 + manifest_path.write_text(json.dumps(manifest)) + monkeypatch.setattr(contracts, "_PROFILE_ROOT", tmp_path) + + with pytest.raises(RuntimeError, match="rows differ from its sampling audit"): + contracts.load_profile("core-3_344-examples_r1-native") + + +def test_quick_manifest_requires_exact_pins_counts_and_leaf_balance(tmp_path): + path = _quick_manifest(tmp_path / "quick.json") + suites.load_quick_manifest(path) + + manifest = json.loads(path.read_text()) + manifest["tasks"]["mmmu_val"]["rows"].pop() + path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="exactly 120 rows"): + suites.load_quick_manifest(path) + + path = _quick_manifest(path) + manifest = json.loads(path.read_text()) + manifest["tasks"]["mvbench"]["rows"][-1] = { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 8, + "source_sample_id": "action_sequence:8", + } + path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="exactly 8 rows per leaf task"): + suites.load_quick_manifest(path) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py new file mode 100644 index 00000000000..d84a32cfb14 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py @@ -0,0 +1,357 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for VLM evaluation execution and result recovery.""" + +import hashlib +import json + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import evaluator, preflight, profile, suites +from examples.puzzletron.evaluation.vlm import run as evaluation +from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import ( + _use_offline_fakes, + _write_checkpoint, + _write_fake_mmmu_artifacts, + _write_lmms_tasks, +) + + +def test_chat_template_fingerprint_accepts_file_and_inline_content(tmp_path): + content = "{% if messages %}{{ messages[0]['content'] }}{% endif %}" + template_path = tmp_path / "chat_template.jinja" + template_path.write_text(content) + expected = hashlib.sha256(content.encode()).hexdigest() + + assert ( + evaluator._chat_template_sha256({"model_args": {"chat_template": str(template_path)}}) + == expected + ) + assert evaluator._chat_template_sha256({"model_args": {"chat_template": content}}) == expected + + +def test_mmmu_parser_audit_is_attached_to_normalized_result(tmp_path): + attempt = tmp_path / "attempt" + attempt.mkdir() + task_name = suites.task_name("mmmu_val") + result_path = attempt / "summary.json" + result_path.write_text(json.dumps({"sample_counts": {task_name: 2}})) + raw_result_path = attempt / "20260903_120000_results.json" + raw_result_path.write_text("{}\n") + sample_path = attempt / f"20260903_120000_samples_{task_name}.jsonl" + sample_path.write_text( + "\n".join( + json.dumps({"mmmu_acc": {"parser_status": [status]}}) + for status in ("parsed", "fallback_random") + ) + + "\n" + ) + (attempt / f"20260903_110000_samples_{task_name}.jsonl").write_text("{not-json}\n") + + evaluator._attach_mmmu_parser_audit( + {"raw_result_path": str(raw_result_path), "result_path": str(result_path)} + ) + + audit = json.loads(result_path.read_text())["mmmu_parser_audit"] + assert audit["sample_count"] == 2 + assert audit["status_counts"] == {"fallback_random": 1, "parsed": 1} + assert audit["sample_logs"] == [ + { + "path": sample_path.name, + "sha256": hashlib.sha256(sample_path.read_bytes()).hexdigest(), + "size": sample_path.stat().st_size, + } + ] + + +def test_mmmu_parser_audit_rejects_unlabeled_sample(tmp_path): + task_name = suites.task_name("mmmu_val") + result_path = tmp_path / "summary.json" + result_path.write_text(json.dumps({"sample_counts": {task_name: 1}})) + raw_result_path = tmp_path / "new_results.json" + raw_result_path.write_text("{}\n") + (tmp_path / f"old_samples_{task_name}.jsonl").write_text( + json.dumps({"mmmu_acc": {"parser_status": ["parsed"]}}) + "\n" + ) + (tmp_path / f"new_samples_{task_name}.jsonl").write_text( + json.dumps({"mmmu_acc": {"parsed_pred": ["A"]}}) + "\n" + ) + + with pytest.raises(RuntimeError, match="no valid parser status"): + evaluator._attach_mmmu_parser_audit( + {"raw_result_path": str(raw_result_path), "result_path": str(result_path)} + ) + + +def test_short_profile_preserves_default_vllm_backend(monkeypatch, tmp_path, capsys): + model = _write_checkpoint(tmp_path) + source_tasks = ("realworldqa", "mmmu_val") + lmms_root = _write_lmms_tasks(tmp_path, source_tasks) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + output = tmp_path / "results" + calls = [] + + def fake_runner(checkpoint_path, *, output_root, settings): + calls.append( + { + "checkpoint": checkpoint_path, + "output_root": output_root, + "settings": settings, + } + ) + result_path = output_root / "result.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + raw_result_path = _write_fake_mmmu_artifacts(result_path) + return { + "attempt": len(calls), + "metrics": {"accuracy": len(calls) / 10}, + "output_root": str(output_root), + "raw_result_path": str(raw_result_path), + "result_path": str(result_path), + } + + monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) + argv = [ + "--checkpoint", + str(model), + "--output-dir", + str(output), + "--suite", + "short", + "--hf-home", + str(hf_home), + ] + + assert evaluation.main(argv) == 0 + + result = json.loads(capsys.readouterr().out) + report = result["preflight"] + generated = json.loads( + (output / "task_configs/modelopt_vlm_benchmark_realworldqa.yaml").read_text() + ) + assert generated["dataset_path"].endswith( + profile.VLM_BENCHMARK_DATASETS["realworldqa"].revision + ) + assert generated["generation_kwargs"]["max_new_tokens"] == 16 + mmmu_text = (output / "task_configs/modelopt_vlm_benchmark_mmmu_val.yaml").read_text() + assert '"max_new_tokens": 128' in mmmu_text + expected_tasks = ( + "modelopt_vlm_benchmark_realworldqa", + "modelopt_vlm_benchmark_mmmu_val", + ) + assert [call["checkpoint"] for call in calls] == [model, model] + assert [call["output_root"] for call in calls] == [ + output / "short-repetition-1", + output / "short-repetition-2", + ] + assert all(call["settings"]["tasks"] == ",".join(expected_tasks) for call in calls) + assert [run["attempt"] for run in result["runs"]] == [1, 2] + settings = calls[0]["settings"] + assert settings["model"] == "vllm" + assert report["backend_limitations"] == [ + "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", + "pinned generic vLLM max_new_tokens is a model-level lower bound", + ] + for task, expected_tokens in {"mmmu_val": 128, "realworldqa": 16}.items(): + budget = report["output_budget_contract"][task] + assert budget["adapter"] == "vllm" + assert budget["requested_max_new_tokens"] == expected_tokens + assert budget["effective_max_new_tokens"] == expected_tokens + assert budget["limitation"] is not None + assert budget["resolution"] == "max(task_max_new_tokens, model_max_new_tokens_floor=1)" + + +@pytest.mark.parametrize("suite", ["short", suites.TASK_PREFIX100_REPEAT2_SUITE]) +def test_repeated_profile_resumes_completed_repetitions(monkeypatch, tmp_path, suite): + model = _write_checkpoint(tmp_path) + source_tasks = ("realworldqa", "mmmu_val") + lmms_root = _write_lmms_tasks(tmp_path, source_tasks) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + output = tmp_path / "results" + calls = [] + + def fake_runner(checkpoint_path, *, output_root, settings): + calls.append(output_root) + result_path = output_root / "attempt" / "summary.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + raw_result_path = _write_fake_mmmu_artifacts(result_path) + return { + "metrics": {"accuracy": len(calls) / 10}, + "raw_result_path": str(raw_result_path), + "result_path": str(result_path), + } + + monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(output), + "--suite", + suite, + "--hf-home", + str(hf_home), + ] + ) + + first = evaluation.evaluate(args) + second = evaluation.evaluate(args) + + assert len(calls) == 2 + assert second["runs"] == first["runs"] + for repetition in (1, 2): + completed = json.loads( + (output / f"{suite}-repetition-{repetition}" / "completed_run.json").read_text() + ) + assert completed["schema"] == "modelopt.vlm-evaluation-completed-run/v1" + assert completed["identity"]["repetition"] == repetition + assert completed["identity"]["checkpoint"]["fingerprint"] + assert completed["identity"]["profile"]["suite"] == suite + + +@pytest.mark.parametrize( + ("corruption", "expected_calls"), + [("checkpoint", 4), ("artifact", 3), ("result", 3), ("profile", 4)], +) +def test_short_profile_reruns_stale_completed_repetitions( + monkeypatch, + tmp_path, + corruption, + expected_calls, +): + model = _write_checkpoint(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa", "mmmu_val")) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + output = tmp_path / "results" + calls = [] + + def fake_runner(checkpoint_path, *, output_root, settings): + calls.append(output_root) + result_path = output_root / "attempt" / "summary.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + raw_result_path = _write_fake_mmmu_artifacts(result_path) + return { + "metrics": {"accuracy": 0.5}, + "raw_result_path": str(raw_result_path), + "result_path": str(result_path), + } + + monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(output), + "--suite", + "short", + "--hf-home", + str(hf_home), + ] + ) + evaluation.evaluate(args) + + if corruption == "checkpoint": + (model / "preprocessor_config.json").write_text('{"changed": true}\n') + elif corruption == "artifact": + (output / "short-repetition-1" / "attempt" / "run_results.json").unlink() + elif corruption == "result": + (output / "short-repetition-1" / "attempt" / "summary.json").unlink() + else: + original_backend_policy = preflight._backend_policy + + def changed_backend_policy(profile_contract): + return { + **original_backend_policy(profile_contract), + "enforce_eager": True, + } + + monkeypatch.setattr(preflight, "_backend_policy", changed_backend_policy) + + evaluation.evaluate(args) + assert len(calls) == expected_calls + + +@pytest.mark.parametrize( + "record", + [ + "{\n", + json.dumps( + { + "identity": {}, + "result": {"metrics": []}, + "schema": "modelopt.vlm-evaluation-completed-run/v1", + } + ), + ], +) +def test_completed_repetition_records_fail_closed_when_malformed(tmp_path, record): + output = tmp_path / "results" + output.mkdir() + (output / "completed_run.json").write_text(record) + + with pytest.raises(RuntimeError, match="invalid completed VLM evaluation"): + evaluator._load_completed_run(output, identity={}) + + +def test_realworldqa_mmmu_prefix100_policy_is_explicit_and_repeated(): + suite = suites.TASK_PREFIX100_REPEAT2_SUITE + assert suites.source_tasks(suite) == ("realworldqa", "mmmu_val") + policy = suites.execution_policy(suite, timeout_seconds=14400) + assert policy["limit"] == 100 + assert policy["repetitions"] == 2 + assert policy["generation"] == { + "enable_thinking": False, + "temperature": 0, + "do_sample": False, + } + assert suites.execution_policy("full", timeout_seconds=None)["limit"] is None + assert suites.execution_policy("full-v1", timeout_seconds=None)["limit"] is None + + +@pytest.mark.parametrize("alias", suites.DEPRECATED_SUITE_ALIASES) +def test_deprecated_suite_alias_records_the_canonical_identity(monkeypatch, tmp_path, alias): + model = _write_checkpoint(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa", "mmmu_val")) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--suite", + alias, + "--hf-home", + str(hf_home), + ] + ) + + with pytest.warns(FutureWarning, match="is deprecated"): + prepared = preflight.prepare(args) + + assert prepared.suite == suites.TASK_PREFIX100_REPEAT2_SUITE + assert prepared.report["suite"] == suites.TASK_PREFIX100_REPEAT2_SUITE diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py new file mode 100644 index 00000000000..890f50176fe --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for VLM checkpoint and prompt-template validation.""" + +import json + +import pytest + +from examples.puzzletron.evaluation.vlm import model as vlm_model +from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import _write_checkpoint + + +def _homogeneous_qwen_block_configs() -> list[dict[str, object]]: + block = { + "subblock_configs": [ + { + "kind": "attention", + "name": "attention", + "no_op": False, + "num_kv_heads": 2, + "num_query_heads": 8, + }, + { + "kind": "ffn", + "name": "ffn", + "no_op": False, + "intermediate_size": 3584, + }, + ] + } + return [json.loads(json.dumps(block)) for _ in range(24)] + + +def test_no_think_template_is_local_and_requires_checkpoint_switch(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + tasks_root = tmp_path / "tasks" + tasks_root.mkdir() + + (checkpoint_path / "chat_template.jinja").write_text( + "{% if enable_thinking is defined and enable_thinking is true %}" + "\n{% else %}\n\n\n\n{% endif %}" + ) + generated = vlm_model.no_think_chat_template(checkpoint_path, tasks_root) + assert generated.parent == tasks_root + (checkpoint_path / "chat_template.jinja").write_text("unsupported\n") + with pytest.raises(ValueError, match="cannot disable thinking"): + vlm_model.no_think_chat_template(checkpoint_path, tasks_root) + + +def test_no_think_template_rejects_unsafe_checkpoint_expression(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + (checkpoint_path / "chat_template.jinja").write_text( + "{{ ''.__class__.__mro__ }}" + "{% if enable_thinking is defined and enable_thinking is false %}" + "\n\n\n\n{% endif %}" + ) + tasks_root = tmp_path / "tasks" + tasks_root.mkdir() + + with pytest.raises(ValueError, match="chat template is invalid"): + vlm_model.no_think_chat_template(checkpoint_path, tasks_root) + + +def test_checkpoint_contract_accepts_only_matching_realized_anymodel(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + config_path = checkpoint_path / "config.json" + config = json.loads(config_path.read_text()) + config.update( + architectures=["AnyModel"], + base_architecture="Qwen3_5ForConditionalGeneration", + block_configs=_homogeneous_qwen_block_configs(), + ) + config_path.write_text(json.dumps(config) + "\n") + + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config.pop("block_configs") + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="cannot prove.*homogeneous"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config["block_configs"] = _homogeneous_qwen_block_configs() + config["base_architecture"] = "OtherForConditionalGeneration" + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="AnyModel base_architecture"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config.update( + architectures=["AnyModel", "Qwen3_5ForConditionalGeneration"], + base_architecture="Qwen3_5ForConditionalGeneration", + ) + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="AnyModel base_architecture"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + +def test_checkpoint_contract_routes_heterogeneous_anymodel_to_vllm(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + config_path = checkpoint_path / "config.json" + config = json.loads(config_path.read_text()) + config.update( + architectures=["AnyModel"], + base_architecture="Qwen3_5ForConditionalGeneration", + block_configs=_homogeneous_qwen_block_configs(), + ) + config["block_configs"][19]["subblock_configs"][0]["num_query_heads"] = 6 + config["text_config"]["per_layer_config"] = { + "19": {"num_attention_heads": 6, "num_key_value_heads": 2} + } + config_path.write_text(json.dumps(config) + "\n") + + with pytest.raises(ValueError, match="native qwen3_5 backend cannot load"): + vlm_model.verify_checkpoint( + checkpoint_path, + profile="VLM benchmark", + model_backend="qwen3_5", + ) + + vlm_model.verify_checkpoint( + checkpoint_path, + profile="VLM benchmark", + model_backend="vllm", + ) + + +def test_checkpoint_contract_accepts_other_positive_qwen35_geometry(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + config_path = checkpoint_path / "config.json" + config = json.loads(config_path.read_text()) + config["text_config"].update( + hidden_size=2560, + intermediate_size=9728, + num_attention_heads=20, + num_hidden_layers=40, + num_key_value_heads=4, + ) + config_path.write_text(json.dumps(config) + "\n") + + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config["text_config"]["hidden_size"] = 0 + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="invalid Qwen 3.5 geometry"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + +@pytest.mark.parametrize("processor_content", [None, "", "[]\n", "{\n", b"\xff"]) +def test_checkpoint_contract_requires_valid_local_processor_assets(tmp_path, processor_content): + checkpoint_path = _write_checkpoint(tmp_path) + processor_path = checkpoint_path / "preprocessor_config.json" + if processor_content is None: + processor_path.unlink() + elif isinstance(processor_content, bytes): + processor_path.write_bytes(processor_content) + else: + processor_path.write_text(processor_content) + + with pytest.raises(ValueError, match="processor asset"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + +def test_checkpoint_contract_rejects_malformed_companion_processor_asset(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + (checkpoint_path / "video_preprocessor_config.json").write_text("{\n") + + with pytest.raises(ValueError, match="video_preprocessor_config.json"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py new file mode 100644 index 00000000000..4e62a4a95c6 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for VLM post-MIP evaluation adapters.""" + +import json +from pathlib import Path + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import post_mip, suites +from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import ( + _use_offline_fakes, + _write_checkpoint, + _write_lmms_tasks, +) + + +def test_post_mip_realworldqa_adapter_runs_pinned_profile(monkeypatch, tmp_path): + model = _write_checkpoint(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + monkeypatch.setenv("HF_HOME", str(hf_home)) + output = tmp_path / "output" + captured = {} + + def fake_runner(checkpoint_path, *, output_root, settings): + report = json.loads((output / "profile.json").read_text()) + assert report["configured_tasks"] == ["modelopt_vlm_benchmark_realworldqa"] + assert report["sample_limit"] == 2 + captured.update( + checkpoint=checkpoint_path, + output_root=output_root, + settings=settings, + ) + result_path = output_root / "result.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text("{}\n") + return { + "metrics": {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5}, + "result_path": str(result_path), + } + + monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) + result = post_mip.evaluate_realworldqa_checkpoint( + model, + output_root=output, + settings={ + "batch_size": 1, + "timeout_seconds": 900, + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + }, + ) + + assert captured["checkpoint"] == model + assert captured["output_root"] == output + assert captured["settings"]["tasks"] == "modelopt_vlm_benchmark_realworldqa" + assert captured["settings"]["limit"] == 2 + assert captured["settings"]["timeout_seconds"] == 900 + assert captured["settings"]["dtype"] == "bfloat16" + assert captured["settings"]["topology"] == {"tensor_parallel_size": 1} + assert result["metrics"] == {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5} + assert result["profile_path"] == str(output / "profile.json") + + +def test_post_mip_prefix100_adapter_averages_repeated_bounded_tasks( + monkeypatch, + tmp_path, +): + model = tmp_path / "model" + model.mkdir() + output = tmp_path / "output" + captured = {"invocations": 0} + + def fake_evaluate(args, *, settings_overrides, preflight_callback): + captured["invocations"] += 1 + captured.update(args=args, settings_overrides=settings_overrides) + preflight_callback({"profile": suites.EVALUATION_PROFILE, "sample_limit": None}) + runs = [] + score_offset = (captured["invocations"] - 1) * 0.2 + for index, realworldqa_score in enumerate( + (0.4 + score_offset, 0.6 + score_offset), start=1 + ): + result_path = tmp_path / f"run-{index}.json" + result_path.write_text( + json.dumps( + { + "sample_counts": {"realworldqa": 100, "mmmu_val": 100}, + "mmmu_parser_audit": { + "sample_count": 100, + "status_counts": {"parsed": 90, "fallback_random": 10}, + }, + } + ) + ) + runs.append( + { + "metrics": { + "modelopt_vlm_benchmark_realworldqa.exact_match_none": (realworldqa_score), + "modelopt_vlm_benchmark_mmmu_val.mmmu_acc_none": 0.3, + }, + "result_path": str(result_path), + } + ) + return {"runs": runs} + + monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) + result = post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( + model, + output_root=output, + settings={ + "batch_size": 1, + "timeout_seconds": 14400, + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + }, + ) + + assert captured["args"].suite == suites.TASK_PREFIX100_REPEAT2_SUITE + assert captured["args"].batch_size == 1 + assert captured["args"].seed == 42 + assert captured["settings_overrides"] == { + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + } + assert result["metrics"] == { + "modelopt_vlm_benchmark_mmmu_val.mmmu_acc_none": 0.3, + "modelopt_vlm_benchmark_realworldqa.exact_match_none": 0.5, + } + assert result["profile"] == post_mip.TASK_PREFIX100_REPEAT2_PROFILE + summary = json.loads(Path(result["result_path"]).read_text()) + assert summary["suite"] == suites.TASK_PREFIX100_REPEAT2_SUITE + assert summary["profile"] == post_mip.TASK_PREFIX100_REPEAT2_PROFILE + assert summary["metrics"] == result["metrics"] + assert summary["result_paths"] == result["run_result_paths"] + assert summary["sample_counts"] == {"mmmu_val": 200, "realworldqa": 200} + assert summary["mmmu_parser_audit"] == { + "sample_count": 200, + "status_counts": {"fallback_random": 20, "parsed": 180}, + } + + refreshed = post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( + model, + output_root=output, + settings={ + "batch_size": 1, + "timeout_seconds": 14400, + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + }, + ) + assert refreshed["metrics"][ + "modelopt_vlm_benchmark_realworldqa.exact_match_none" + ] == pytest.approx(0.7) + assert json.loads(Path(refreshed["result_path"]).read_text())["metrics"] == refreshed["metrics"] + + +def test_post_mip_prefix100_rejects_different_repetition_metrics( + monkeypatch, + tmp_path, +): + def fake_evaluate(args, *, settings_overrides, preflight_callback): + return { + "runs": [ + {"metrics": {"realworldqa.accuracy": 0.5}, "result_path": "first.json"}, + {"metrics": {"mmmu.accuracy": 0.5}, "result_path": "second.json"}, + ] + } + + monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) + + with pytest.raises(RuntimeError, match="produced different metrics"): + post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( + tmp_path / "model", + output_root=tmp_path / "output", + settings={}, + ) + + +def test_deprecated_post_mip_profile_alias_forwards_to_canonical(monkeypatch, tmp_path): + expected = {"metrics": {"accuracy": 0.5}} + monkeypatch.setattr( + post_mip, + "evaluate_realworldqa_mmmu_prefix100_checkpoint", + lambda *_args, **_kwargs: expected, + ) + + with pytest.warns(FutureWarning, match="is deprecated"): + result = post_mip.evaluate_e2e_full_eval_checkpoint( + tmp_path / "model", + output_root=tmp_path / "output", + settings={}, + ) + + assert result is expected diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py new file mode 100644 index 00000000000..9676a6eb791 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py @@ -0,0 +1,535 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for VLM profile preflight and runtime settings.""" + +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import ( + contracts, + evaluator, + preflight, + profile, + suites, + tasks, +) +from examples.puzzletron.evaluation.vlm import run as evaluation +from modelopt.torch.puzzletron.evaluation import lmms +from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import ( + _full_inputs, + _use_offline_fakes, + _write_checkpoint, + _write_checkpoint_at, + _write_core3_teacher_snapshot, + _write_lmms_tasks, +) + + +def test_versioned_profile_preflight_reports_immutable_contract(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + name = "short-v1" + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + name, + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + + contract = contracts.load_profile(name) + assert prepared.report["profile_name"] == name + assert prepared.report["profile_fingerprint"] == contract.fingerprint + assert prepared.report["sample_set"] == contract.sample_set + assert prepared.report["backend_profile"] == contract.backend_profile + assert prepared.report["evaluator_profile"] == contract.evaluator_profile + assert prepared.report["source_tasks"] == list(contract.source_tasks) + assert prepared.report["quick_selected_rows"] == 344 + assert prepared.report["quick_row_identities"] == suites.manifest_row_identities( + prepared.quick_manifest + ) + assert prepared.report["quick_task_denominators"] == suites.manifest_task_denominators( + prepared.quick_manifest + ) + + +def test_native_profile_builds_qwen35_backend_settings(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: object()) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "judge-free-8_690-examples_r1-native", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + (tmp_path / "tasks").mkdir() + settings = preflight.settings( + args, + tasks_root=tmp_path / "tasks", + configured_tasks=("modelopt_vlm_benchmark_mvbench",), + prepared=prepared, + ) + + assert prepared.report["model_backend"] == "qwen3_5" + assert settings["model"] == "qwen3_5" + assert settings["checkpoint_arg"] == "pretrained" + assert settings["model_args"] == { + "attn_implementation": "sdpa", + "device": "cuda", + "device_map": "cuda", + "enable_thinking": False, + "fps": 2, + "max_frames": 32, + } + assert "reasoning_parser" not in settings + assert not (tmp_path / "tasks/modelopt_qwen35_no_think.jinja").exists() + + +@pytest.mark.parametrize( + ("profile_name", "expected_eager"), + [("core-3_24-examples_r1-vllm", True), ("core-3_344-examples_r1-vllm", None)], +) +def test_vllm_profile_forwards_runtime_settings( + monkeypatch, tmp_path, profile_name, expected_eager +): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + profile_name, + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + tasks_root = tmp_path / "tasks" + tasks_root.mkdir() + settings = preflight.settings( + args, + tasks_root=tasks_root, + configured_tasks=("modelopt_vlm_benchmark_realworldqa",), + prepared=prepared, + ) + argv, _, _ = lmms._build_command( + settings, + checkpoint=str(model), + output_path=tmp_path / "lmms-results", + ) + model_args = argv[argv.index("--model_args") + 1] + + assert settings["model_args"].get("enforce_eager") is expected_eager + assert ("enforce_eager=True" in model_args) is (expected_eager is True) + assert settings["model_args"]["attention_config"] == {"flash_attn_version": 2} + assert 'attention_config={"flash_attn_version":2}' in model_args + + +def test_core3_full_teacher_profile_population_expectations_follow_group_shard( + monkeypatch, tmp_path +): + model, hf_home = _write_core3_teacher_snapshot(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("mvbench",)) + _use_offline_fakes(monkeypatch, lmms_root) + media = hf_home / profile.VLM_BENCHMARK_DATASETS["mvbench"].media_dir + media.mkdir(parents=True) + (media / "sample").write_bytes(b"media") + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_full_r1-native", + "--profile-task", + "mvbench", + "--profile-task-shard", + "3/8", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + expected_leaves = ("episodic_reasoning", "moving_direction", "egocentric_navigation") + + assert prepared.profile_task_leaves == expected_leaves + assert evaluator._expected_task_populations(prepared, ("modelopt_vlm_benchmark_mvbench",)) == { + suites.task_name("mvbench", leaf=leaf): 200 for leaf in expected_leaves + } + + +def test_core3_full_teacher_profile_accepts_snapshot_symlink(monkeypatch, tmp_path): + snapshot, hf_home = _write_core3_teacher_snapshot(tmp_path) + checkpoint_alias = tmp_path / "teacher" + checkpoint_alias.symlink_to(snapshot, target_is_directory=True) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) + _use_offline_fakes(monkeypatch, lmms_root) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(checkpoint_alias), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_full_r1-native", + "--profile-task", + "realworldqa", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + + assert prepared.source_tasks == ("realworldqa",) + assert prepared.report["model_pin"]["revision"] == snapshot.name + + +@pytest.mark.parametrize("kind", ["untracked-copy", "wrong-revision"]) +def test_core3_full_teacher_profile_rejects_unpinned_checkpoint(monkeypatch, tmp_path, kind): + monkeypatch.delenv("HF_HUB_CACHE", raising=False) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + if kind == "untracked-copy": + model = _write_checkpoint(tmp_path) + else: + parent = hf_home / "hub/models--Qwen--Qwen3.5-0.8B/snapshots" + parent.mkdir(parents=True) + model = _write_checkpoint_at(parent / "different") + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_full_r1-native", + "--profile-task", + "realworldqa", + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match="requires the exact local Hub snapshot"): + preflight.prepare(args) + + +def test_core3_full_teacher_profile_rejects_settings_override(monkeypatch, tmp_path): + model, hf_home = _write_core3_teacher_snapshot(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) + _use_offline_fakes(monkeypatch, lmms_root) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_full_r1-native", + "--profile-task", + "realworldqa", + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match="do not allow settings overrides"): + evaluator.evaluate(args, settings_overrides={"model": "vllm"}) + + +def test_historical_short_profile_preserves_vllm_backend(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "short-v1", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + (tmp_path / "tasks").mkdir() + settings = preflight.settings( + args, + tasks_root=tmp_path / "tasks", + configured_tasks=("modelopt_vlm_benchmark_realworldqa",), + prepared=prepared, + ) + + assert prepared.report["lmms_eval_revision"] == checkpoint.LMMS_EVAL_LEGACY_REVISION + assert settings["model"] == "vllm" + assert settings["checkpoint_arg"] == "model" + assert settings["reasoning_parser"] == "qwen3" + assert settings["model_args"]["max_frame_num"] == 32 + assert Path(settings["model_args"]["chat_template"]).exists() + + +@pytest.mark.parametrize( + ("option", "value"), + [("--seed", "7"), ("--batch-size", "8")], +) +def test_versioned_profile_rejects_runtime_override(monkeypatch, tmp_path, option, value): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "short-v1", + option, + value, + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match=f"{option} cannot override"): + preflight.prepare(args) + + +def test_all_row_profile_task_preserves_contract_identity(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + profile_name = "full-v1" + task = "realworldqa" + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + profile_name, + "--profile-task", + task, + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + + assert prepared.source_tasks == (task,) + assert prepared.report["quick_selected_rows"] is None + assert prepared.report["quick_row_identities"] is None + assert prepared.report["quick_task_denominators"] is None + assert ( + prepared.report["profile_fingerprint"] == contracts.load_profile(profile_name).fingerprint + ) + + +def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "judge-free-8_690-examples_r1-native", + "--profile-task", + "mvbench", + "--profile-task-shard", + "3/8", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + + expected_leaves = ("episodic_reasoning", "moving_direction", "egocentric_navigation") + assert prepared.profile_task_leaves == expected_leaves + assert prepared.quick_manifest is not None + manifest_rows = prepared.quick_manifest["tasks"]["mvbench"]["rows"] + assert len(manifest_rows) == 24 + assert {row["leaf_task"] for row in manifest_rows} == { + f"mvbench_{leaf}" for leaf in expected_leaves + } + manifest_selection = prepared.quick_manifest["tasks"]["mvbench"]["selection"] + assert manifest_selection["population_rows"] == 600 + assert manifest_selection["selected_rows"] == 24 + assert [stratum["name"] for stratum in manifest_selection["strata"]] == list(expected_leaves) + assert manifest_selection["selected_index_quantiles"] == { + "method": "lower-order-statistic", + "p0": 12, + "p25": 37, + "p50": 87, + "p75": 137, + "p100": 187, + } + assert ( + manifest_selection["selected_row_identities_sha256"] + == hashlib.sha256( + json.dumps(manifest_rows, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest() + ) + assert prepared.report["quick_selected_rows"] == 24 + assert prepared.report["quick_row_identities"] == suites.manifest_row_identities( + prepared.quick_manifest + ) + assert prepared.report["quick_task_denominators"] == { + "mvbench": {"population_rows": 600, "selected_rows": 24} + } + assert ( + evaluator._expected_task_populations(prepared, ("modelopt_vlm_benchmark_mvbench",)) is None + ) + assert prepared.report["quick_manifest_sha256"] == suites.manifest_sha256( + prepared.quick_manifest + ) + assert ( + prepared.report["profile_fingerprint"] + == contracts.load_profile("judge-free-8_690-examples_r1-native").fingerprint + ) + tasks_root, _ = tasks.prepare( + tmp_path / "results", + suite=prepared.suite, + source_tasks=prepared.source_tasks, + profile_task_leaves=prepared.profile_task_leaves, + dataset_snapshots=prepared.dataset_snapshots, + quick_manifest=prepared.quick_manifest, + ) + group = json.loads((tasks_root / "modelopt_vlm_benchmark_mvbench.yaml").read_text()) + assert group["task"] == [f"modelopt_vlm_benchmark_mvbench_{leaf}" for leaf in expected_leaves] + + +@pytest.mark.parametrize( + ("selection", "message"), + [ + (("--suite", "short", "--profile-task", "realworldqa"), "requires"), + (("--profile", "short-v1", "--profile-task", "realworldqa"), "supported only"), + (("--profile", "full-v1", "--profile-task-shard", "0/8"), "requires"), + ( + ( + "--profile", + "full-v1", + "--profile-task", + "realworldqa", + "--profile-task-shard", + "0/8", + ), + "supports only", + ), + ], +) +def test_profile_task_rejects_invalid_parent(monkeypatch, tmp_path, selection, message): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + *selection, + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match=message): + preflight.prepare(args) + + +@pytest.mark.parametrize( + ("extra", "environment", "message"), + [ + ([], {"OPENAI_API_KEY": "token"}, "explicit --allow-judge-calls"), + (["--allow-judge-calls"], {}, "judge credentials are missing"), + ], +) +def test_full_profile_fails_closed_without_judge_authorization_or_credentials( + monkeypatch, tmp_path, extra, environment, message +): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + for name in ("OPENAI_API_KEY", "AZURE_API_KEY", "AZURE_ENDPOINT"): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--suite", + "full", + "--hf-home", + str(hf_home), + "--mmvu-judge-api-type", + "openai", + "--mmvu-judge-model", + "judge", + *extra, + ] + ) + with pytest.raises(ValueError, match=message): + preflight.prepare(args) + + +def test_video_reader_validation_is_limited_to_video_suites(monkeypatch): + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: None) + + preflight._verify_video_reader(("realworldqa", "mmmu_val")) + with pytest.raises(RuntimeError, match="decord-compatible reader"): + preflight._verify_video_reader(("mvbench",)) + + +def test_native_backend_validation_requires_qwen_vision_utilities(monkeypatch): + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: None) + + preflight._verify_backend_dependencies("vllm") + with pytest.raises(RuntimeError, match="qwen-vl-utils"): + preflight._verify_backend_dependencies("qwen3_5") + + +def test_credential_scope_restores_inherited_values(monkeypatch): + for index, name in enumerate(checkpoint.HUGGINGFACE_CREDENTIAL_NAMES): + monkeypatch.setenv(name, f"secret-{index}") + with checkpoint.without_huggingface_credentials(): + assert all(name not in os.environ for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) + assert all(name in os.environ for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py index f69fb7d05bd..0165acb5210 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py @@ -13,71 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""High-value behavior tests for local VLM evaluation workflows.""" +"""Tests for the VLM evaluation command-line entry point.""" import hashlib -import importlib.util import json -import os -import random -import shutil import subprocess import sys -from hashlib import sha256 from pathlib import Path -from types import ModuleType - -import pytest from examples.puzzletron.evaluation import checkpoint -from examples.puzzletron.evaluation.vlm import ( - contracts, - evaluator, - post_mip, - preflight, - profile, - suites, - tasks, -) -from examples.puzzletron.evaluation.vlm import model as vlm_model from examples.puzzletron.evaluation.vlm import run as evaluation -from modelopt.torch.puzzletron.evaluation import lmms - -_QWEN_CONFIG = { - "architectures": ["Qwen3_5ForConditionalGeneration"], - "model_type": "qwen3_5", - "text_config": { - "hidden_size": 1024, - "intermediate_size": 3584, - "model_type": "qwen3_5_text", - "num_attention_heads": 8, - "num_hidden_layers": 24, - "num_key_value_heads": 2, - "vocab_size": 248320, - }, -} -_TASK_CONFIGS = {name: item.task_config for name, item in profile.VLM_BENCHMARK_DATASETS.items()} - - -def _homogeneous_qwen_block_configs() -> list[dict[str, object]]: - block = { - "subblock_configs": [ - { - "kind": "attention", - "name": "attention", - "no_op": False, - "num_kv_heads": 2, - "num_query_heads": 8, - }, - { - "kind": "ffn", - "name": "ffn", - "no_op": False, - "intermediate_size": 3584, - }, - ] - } - return [json.loads(json.dumps(block)) for _ in range(24)] def test_direct_launcher_does_not_shadow_standard_library_profile(): @@ -100,1932 +45,6 @@ def test_direct_launcher_does_not_shadow_standard_library_profile(): ) -def _write_checkpoint_at(model: Path) -> Path: - model.mkdir() - (model / "config.json").write_text(json.dumps(_QWEN_CONFIG) + "\n") - (model / "preprocessor_config.json").write_text("{}\n") - (model / "chat_template.jinja").write_text( - "{% if enable_thinking is defined and enable_thinking is false %}" - "\n\n\n\n{% else %}\n{% endif %}" - ) - return model - - -def _write_checkpoint(root: Path) -> Path: - return _write_checkpoint_at(root / "model") - - -def _write_core3_teacher_snapshot(root: Path) -> tuple[Path, Path]: - hf_home = root / "hf-home" - snapshot = ( - hf_home - / "hub/models--Qwen--Qwen3.5-0.8B/snapshots" - / "2fc06364715b967f1860aea9cf38778875588b17" - ) - snapshot.parent.mkdir(parents=True) - return _write_checkpoint_at(snapshot), hf_home - - -def test_no_think_template_is_local_and_requires_checkpoint_switch(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - tasks_root = tmp_path / "tasks" - tasks_root.mkdir() - - (checkpoint_path / "chat_template.jinja").write_text( - "{% if enable_thinking is defined and enable_thinking is true %}" - "\n{% else %}\n\n\n\n{% endif %}" - ) - generated = vlm_model.no_think_chat_template(checkpoint_path, tasks_root) - assert generated.parent == tasks_root - (checkpoint_path / "chat_template.jinja").write_text("unsupported\n") - with pytest.raises(ValueError, match="cannot disable thinking"): - vlm_model.no_think_chat_template(checkpoint_path, tasks_root) - - -def test_no_think_template_rejects_unsafe_checkpoint_expression(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - (checkpoint_path / "chat_template.jinja").write_text( - "{{ ''.__class__.__mro__ }}" - "{% if enable_thinking is defined and enable_thinking is false %}" - "\n\n\n\n{% endif %}" - ) - tasks_root = tmp_path / "tasks" - tasks_root.mkdir() - - with pytest.raises(ValueError, match="chat template is invalid"): - vlm_model.no_think_chat_template(checkpoint_path, tasks_root) - - -def test_chat_template_fingerprint_accepts_file_and_inline_content(tmp_path): - content = "{% if messages %}{{ messages[0]['content'] }}{% endif %}" - template_path = tmp_path / "chat_template.jinja" - template_path.write_text(content) - expected = sha256(content.encode()).hexdigest() - - assert ( - evaluator._chat_template_sha256({"model_args": {"chat_template": str(template_path)}}) - == expected - ) - assert evaluator._chat_template_sha256({"model_args": {"chat_template": content}}) == expected - - -def test_mmmu_parser_audit_is_attached_to_normalized_result(tmp_path): - attempt = tmp_path / "attempt" - attempt.mkdir() - task_name = suites.task_name("mmmu_val") - result_path = attempt / "summary.json" - result_path.write_text(json.dumps({"sample_counts": {task_name: 2}})) - raw_result_path = attempt / "20260903_120000_results.json" - raw_result_path.write_text("{}\n") - sample_path = attempt / f"20260903_120000_samples_{task_name}.jsonl" - sample_path.write_text( - "\n".join( - json.dumps({"mmmu_acc": {"parser_status": [status]}}) - for status in ("parsed", "fallback_random") - ) - + "\n" - ) - (attempt / f"20260903_110000_samples_{task_name}.jsonl").write_text("{not-json}\n") - - evaluator._attach_mmmu_parser_audit( - {"raw_result_path": str(raw_result_path), "result_path": str(result_path)} - ) - - audit = json.loads(result_path.read_text())["mmmu_parser_audit"] - assert audit["sample_count"] == 2 - assert audit["status_counts"] == {"fallback_random": 1, "parsed": 1} - assert audit["sample_logs"] == [ - { - "path": sample_path.name, - "sha256": hashlib.sha256(sample_path.read_bytes()).hexdigest(), - "size": sample_path.stat().st_size, - } - ] - - -def test_mmmu_parser_audit_rejects_unlabeled_sample(tmp_path): - task_name = suites.task_name("mmmu_val") - result_path = tmp_path / "summary.json" - result_path.write_text(json.dumps({"sample_counts": {task_name: 1}})) - raw_result_path = tmp_path / "new_results.json" - raw_result_path.write_text("{}\n") - (tmp_path / f"old_samples_{task_name}.jsonl").write_text( - json.dumps({"mmmu_acc": {"parser_status": ["parsed"]}}) + "\n" - ) - (tmp_path / f"new_samples_{task_name}.jsonl").write_text( - json.dumps({"mmmu_acc": {"parsed_pred": ["A"]}}) + "\n" - ) - - with pytest.raises(RuntimeError, match="no valid parser status"): - evaluator._attach_mmmu_parser_audit( - {"raw_result_path": str(raw_result_path), "result_path": str(result_path)} - ) - - -def test_checkpoint_contract_accepts_only_matching_realized_anymodel(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - config_path = checkpoint_path / "config.json" - config = json.loads(config_path.read_text()) - config.update( - architectures=["AnyModel"], - base_architecture="Qwen3_5ForConditionalGeneration", - block_configs=_homogeneous_qwen_block_configs(), - ) - config_path.write_text(json.dumps(config) + "\n") - - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - config.pop("block_configs") - config_path.write_text(json.dumps(config) + "\n") - with pytest.raises(ValueError, match="cannot prove.*homogeneous"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - config["block_configs"] = _homogeneous_qwen_block_configs() - config["base_architecture"] = "OtherForConditionalGeneration" - config_path.write_text(json.dumps(config) + "\n") - with pytest.raises(ValueError, match="AnyModel base_architecture"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - config.update( - architectures=["AnyModel", "Qwen3_5ForConditionalGeneration"], - base_architecture="Qwen3_5ForConditionalGeneration", - ) - config_path.write_text(json.dumps(config) + "\n") - with pytest.raises(ValueError, match="AnyModel base_architecture"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - -def test_checkpoint_contract_routes_heterogeneous_anymodel_to_vllm(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - config_path = checkpoint_path / "config.json" - config = json.loads(config_path.read_text()) - config.update( - architectures=["AnyModel"], - base_architecture="Qwen3_5ForConditionalGeneration", - block_configs=_homogeneous_qwen_block_configs(), - ) - config["block_configs"][19]["subblock_configs"][0]["num_query_heads"] = 6 - config["text_config"]["per_layer_config"] = { - "19": {"num_attention_heads": 6, "num_key_value_heads": 2} - } - config_path.write_text(json.dumps(config) + "\n") - - with pytest.raises(ValueError, match="native qwen3_5 backend cannot load"): - vlm_model.verify_checkpoint( - checkpoint_path, - profile="VLM benchmark", - model_backend="qwen3_5", - ) - - vlm_model.verify_checkpoint( - checkpoint_path, - profile="VLM benchmark", - model_backend="vllm", - ) - - -def test_checkpoint_contract_accepts_other_positive_qwen35_geometry(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - config_path = checkpoint_path / "config.json" - config = json.loads(config_path.read_text()) - config["text_config"].update( - hidden_size=2560, - intermediate_size=9728, - num_attention_heads=20, - num_hidden_layers=40, - num_key_value_heads=4, - ) - config_path.write_text(json.dumps(config) + "\n") - - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - config["text_config"]["hidden_size"] = 0 - config_path.write_text(json.dumps(config) + "\n") - with pytest.raises(ValueError, match="invalid Qwen 3.5 geometry"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - -@pytest.mark.parametrize("processor_content", [None, "", "[]\n", "{\n", b"\xff"]) -def test_checkpoint_contract_requires_valid_local_processor_assets(tmp_path, processor_content): - checkpoint_path = _write_checkpoint(tmp_path) - processor_path = checkpoint_path / "preprocessor_config.json" - if processor_content is None: - processor_path.unlink() - elif isinstance(processor_content, bytes): - processor_path.write_bytes(processor_content) - else: - processor_path.write_text(processor_content) - - with pytest.raises(ValueError, match="processor asset"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - -def test_checkpoint_contract_rejects_malformed_companion_processor_asset(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - (checkpoint_path / "video_preprocessor_config.json").write_text("{\n") - - with pytest.raises(ValueError, match="video_preprocessor_config.json"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - -def _write_lmms_tasks(root: Path, tasks: tuple[str, ...]) -> Path: - lmms_root = root / "lmms_eval" - for task in tasks: - task_configs = [_TASK_CONFIGS[task]] - if task == "video_mmmu": - task_configs.extend( - f"tasks/videommmu/{leaf}.yaml" for leaf in suites.VIDEO_MMMU_LEAF_TASKS - ) - elif task == "mvbench": - task_configs.extend( - f"tasks/mvbench/mvbench_{leaf}.yaml" for leaf in suites.MVBENCH_LEAF_TASKS - ) - for relative_path in task_configs: - config = lmms_root / relative_path - config.parent.mkdir(parents=True, exist_ok=True) - config.write_text(f"task: {task}\n") - return lmms_root - - -def _use_offline_fakes(monkeypatch, lmms_root: Path) -> None: - monkeypatch.delenv("HF_HUB_CACHE", raising=False) - monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) - monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: object()) - monkeypatch.setattr( - checkpoint, - "verify_lmms_eval_revision", - lambda expected=checkpoint.LMMS_EVAL_REVISION: expected, - ) - monkeypatch.setattr( - suites, - "offline_dataset_snapshot", - lambda hf_home, task, revision: hf_home / ".snapshots" / task / revision, - ) - monkeypatch.setattr( - tasks, - "verify_offline", - lambda _root, tasks, **_kwargs: { - "configured_tasks": list(tasks), - "status": "passed", - }, - ) - - -def _write_fake_mmmu_artifacts(result_path: Path) -> Path: - result_path.write_text(json.dumps({"sample_counts": {suites.task_name("mmmu_val"): 1}}) + "\n") - raw_result_path = result_path.parent / "run_results.json" - raw_result_path.write_text("{}\n") - sample_path = result_path.parent / f"run_samples_{suites.task_name('mmmu_val')}.jsonl" - sample_path.write_text(json.dumps({"mmmu_acc": {"parser_status": ["parsed"]}}) + "\n") - return raw_result_path - - -def _full_inputs(monkeypatch, tmp_path): - model = _write_checkpoint(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, profile.VLM_BENCHMARK_TASKS) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - for dataset in profile.VLM_BENCHMARK_VIDEO_DATASETS.values(): - media = hf_home / dataset.media_dir - media.mkdir(parents=True, exist_ok=True) - (media / "sample").write_bytes(b"media") - return model, hf_home - - -def _quick_manifest(path: Path) -> Path: - counts = {"realworldqa": 64, "mmmu_val": 120} - tasks = { - task: { - "dataset_revision": profile.VLM_BENCHMARK_DATASETS[task].revision, - "rows": [ - { - "source_row_index": index, - "source_sample_id": ( - f"test:{index}" if task == "realworldqa" else f"question-{index}" - ), - } - for index in range(count) - ], - } - for task, count in counts.items() - } - tasks["mvbench"] = { - "dataset_revision": profile.VLM_BENCHMARK_DATASETS["mvbench"].revision, - "rows": [ - { - "leaf_task": f"mvbench_{leaf}", - "source_row_index": index, - "source_sample_id": f"{leaf}:{index}", - } - for leaf in suites.MVBENCH_LEAF_TASKS - for index in range(8) - ], - } - path.write_text( - json.dumps( - { - "schema": "modelopt.vlm-benchmark-quick/v1", - "lmms_eval_revision": checkpoint.LMMS_EVAL_REVISION, - "tasks": tasks, - } - ) - + "\n" - ) - return path - - -def test_short_profile_preserves_default_vllm_backend(monkeypatch, tmp_path, capsys): - model = _write_checkpoint(tmp_path) - source_tasks = ("realworldqa", "mmmu_val") - lmms_root = _write_lmms_tasks(tmp_path, source_tasks) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - output = tmp_path / "results" - calls = [] - - def fake_runner(checkpoint_path, *, output_root, settings): - calls.append( - { - "checkpoint": checkpoint_path, - "output_root": output_root, - "settings": settings, - } - ) - result_path = output_root / "result.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - raw_result_path = _write_fake_mmmu_artifacts(result_path) - return { - "attempt": len(calls), - "metrics": {"accuracy": len(calls) / 10}, - "output_root": str(output_root), - "raw_result_path": str(raw_result_path), - "result_path": str(result_path), - } - - monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) - argv = [ - "--checkpoint", - str(model), - "--output-dir", - str(output), - "--suite", - "short", - "--hf-home", - str(hf_home), - ] - - assert evaluation.main(argv) == 0 - - result = json.loads(capsys.readouterr().out) - report = result["preflight"] - generated = json.loads( - (output / "task_configs/modelopt_vlm_benchmark_realworldqa.yaml").read_text() - ) - assert generated["dataset_path"].endswith( - profile.VLM_BENCHMARK_DATASETS["realworldqa"].revision - ) - assert generated["generation_kwargs"]["max_new_tokens"] == 16 - mmmu_text = (output / "task_configs/modelopt_vlm_benchmark_mmmu_val.yaml").read_text() - assert '"max_new_tokens": 128' in mmmu_text - expected_tasks = ( - "modelopt_vlm_benchmark_realworldqa", - "modelopt_vlm_benchmark_mmmu_val", - ) - assert [call["checkpoint"] for call in calls] == [model, model] - assert [call["output_root"] for call in calls] == [ - output / "short-repetition-1", - output / "short-repetition-2", - ] - assert all(call["settings"]["tasks"] == ",".join(expected_tasks) for call in calls) - assert [run["attempt"] for run in result["runs"]] == [1, 2] - settings = calls[0]["settings"] - assert settings["model"] == "vllm" - assert report["backend_limitations"] == [ - "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", - "pinned generic vLLM max_new_tokens is a model-level lower bound", - ] - for task, expected_tokens in {"mmmu_val": 128, "realworldqa": 16}.items(): - budget = report["output_budget_contract"][task] - assert budget["adapter"] == "vllm" - assert budget["requested_max_new_tokens"] == expected_tokens - assert budget["effective_max_new_tokens"] == expected_tokens - assert budget["limitation"] is not None - assert budget["resolution"] == "max(task_max_new_tokens, model_max_new_tokens_floor=1)" - - -@pytest.mark.parametrize("suite", ["short", suites.TASK_PREFIX100_REPEAT2_SUITE]) -def test_repeated_profile_resumes_completed_repetitions(monkeypatch, tmp_path, suite): - model = _write_checkpoint(tmp_path) - source_tasks = ("realworldqa", "mmmu_val") - lmms_root = _write_lmms_tasks(tmp_path, source_tasks) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - output = tmp_path / "results" - calls = [] - - def fake_runner(checkpoint_path, *, output_root, settings): - calls.append(output_root) - result_path = output_root / "attempt" / "summary.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - raw_result_path = _write_fake_mmmu_artifacts(result_path) - return { - "metrics": {"accuracy": len(calls) / 10}, - "raw_result_path": str(raw_result_path), - "result_path": str(result_path), - } - - monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(output), - "--suite", - suite, - "--hf-home", - str(hf_home), - ] - ) - - first = evaluation.evaluate(args) - second = evaluation.evaluate(args) - - assert len(calls) == 2 - assert second["runs"] == first["runs"] - for repetition in (1, 2): - completed = json.loads( - (output / f"{suite}-repetition-{repetition}" / "completed_run.json").read_text() - ) - assert completed["schema"] == "modelopt.vlm-evaluation-completed-run/v1" - assert completed["identity"]["repetition"] == repetition - assert completed["identity"]["checkpoint"]["fingerprint"] - assert completed["identity"]["profile"]["suite"] == suite - - -@pytest.mark.parametrize( - ("corruption", "expected_calls"), - [("checkpoint", 4), ("artifact", 3), ("result", 3), ("profile", 4)], -) -def test_short_profile_reruns_stale_completed_repetitions( - monkeypatch, - tmp_path, - corruption, - expected_calls, -): - model = _write_checkpoint(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa", "mmmu_val")) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - output = tmp_path / "results" - calls = [] - - def fake_runner(checkpoint_path, *, output_root, settings): - calls.append(output_root) - result_path = output_root / "attempt" / "summary.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - raw_result_path = _write_fake_mmmu_artifacts(result_path) - return { - "metrics": {"accuracy": 0.5}, - "raw_result_path": str(raw_result_path), - "result_path": str(result_path), - } - - monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(output), - "--suite", - "short", - "--hf-home", - str(hf_home), - ] - ) - evaluation.evaluate(args) - - if corruption == "checkpoint": - (model / "preprocessor_config.json").write_text('{"changed": true}\n') - elif corruption == "artifact": - (output / "short-repetition-1" / "attempt" / "run_results.json").unlink() - elif corruption == "result": - (output / "short-repetition-1" / "attempt" / "summary.json").unlink() - else: - original_backend_policy = preflight._backend_policy - - def changed_backend_policy(profile_contract): - return { - **original_backend_policy(profile_contract), - "enforce_eager": True, - } - - monkeypatch.setattr(preflight, "_backend_policy", changed_backend_policy) - - evaluation.evaluate(args) - assert len(calls) == expected_calls - - -@pytest.mark.parametrize( - "record", - [ - "{\n", - json.dumps( - { - "identity": {}, - "result": {"metrics": []}, - "schema": "modelopt.vlm-evaluation-completed-run/v1", - } - ), - ], -) -def test_completed_repetition_records_fail_closed_when_malformed(tmp_path, record): - output = tmp_path / "results" - output.mkdir() - (output / "completed_run.json").write_text(record) - - with pytest.raises(RuntimeError, match="invalid completed VLM evaluation"): - evaluator._load_completed_run(output, identity={}) - - -def test_realworldqa_mmmu_prefix100_policy_is_explicit_and_repeated(): - suite = suites.TASK_PREFIX100_REPEAT2_SUITE - assert suites.source_tasks(suite) == ("realworldqa", "mmmu_val") - policy = suites.execution_policy(suite, timeout_seconds=14400) - assert policy["limit"] == 100 - assert policy["repetitions"] == 2 - assert policy["generation"] == { - "enable_thinking": False, - "temperature": 0, - "do_sample": False, - } - assert suites.execution_policy("full", timeout_seconds=None)["limit"] is None - assert suites.execution_policy("full-v1", timeout_seconds=None)["limit"] is None - - -@pytest.mark.parametrize("alias", suites.DEPRECATED_SUITE_ALIASES) -def test_deprecated_suite_alias_records_the_canonical_identity(monkeypatch, tmp_path, alias): - model = _write_checkpoint(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa", "mmmu_val")) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--suite", - alias, - "--hf-home", - str(hf_home), - ] - ) - - with pytest.warns(FutureWarning, match="is deprecated"): - prepared = preflight.prepare(args) - - assert prepared.suite == suites.TASK_PREFIX100_REPEAT2_SUITE - assert prepared.report["suite"] == suites.TASK_PREFIX100_REPEAT2_SUITE - - -def test_versioned_profile_contracts_pin_backends_and_fingerprints(): - profiles = {name: contracts.load_profile(name) for name in contracts.PROFILE_NAMES} - for name in profiles: - composition = json.loads((contracts._PROFILE_ROOT / f"{name}.json").read_text()) - assert set(composition) == { - "schema", - "name", - "sample_set", - "backend_profile", - "evaluator_profile", - } - assert {name: contract.fingerprint for name, contract in profiles.items()} == { - "short-v1": "8286a094c3cfb5c2608a6e1469d6525bb1c4be11a7ab8789ed97dd249d7bea71", - "short-native-v1": "b15054c251af54a5298233b1c01a3babf76c4281d055469fd26b76129c34f258", - "core-3_344-examples_r1-native": "2017656d093de7d95d25c7e34241b1d708150157f0c4e6a0bf6bd48649c2191a", - "core-3_344-examples_r1-vllm": "859908fdb32b6bcaddb5400cd4430f4c9026264db38c7a8b56a98f42109c1f78", - "core-3_24-examples_r1-native": "0e51e27d57e27f0c5e4943d077308766387fa739b2d1c413b7b951327358cefc", - "core-3_24-examples_r1-vllm": "9c68168f05003e695258dc119351610b4e98bafb3e4f3e773c4e64ce5d17835a", - "short-all-native-v1": "9d7334371316a2a7774ee7e520ce0fc57e3c42ecfd7749ccd217e02ab59b6ee3", - "judge-free-8_690-examples_r1-native": "78457702288ba2d9d7b903366f7030302936377690b0ec37a0704e3eda8fd851", - "full-v1": "680483a7e2eceeab82a5e0b2767cc751f1951f58ffedc0aa190481d6ec978307", - "core-3_full_r1-native": "976efbd056fecb686e64b912ed50251b1c16e5efe1d3ae3d179f208cc587c0a7", - "core-3_full_r1-vllm": "40fd44fbb4812bd927d3f82e33d6eddec5b4641736c65c48250b8ea77acaec81", - } - - current_short = profiles["core-3_344-examples_r1-native"] - smoke = profiles["core-3_24-examples_r1-native"] - materialized_short = profiles["core-3_344-examples_r1-vllm"] - materialized_smoke = profiles["core-3_24-examples_r1-vllm"] - assert current_short.manifest["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION - assert current_short.manifest["backend"]["name"] == "qwen3_5" - assert materialized_short.manifest["backend"]["name"] == "vllm" - assert materialized_short.sample_set == current_short.sample_set - assert materialized_short.backend_profile != current_short.backend_profile - assert materialized_short.evaluator_profile == current_short.evaluator_profile - assert materialized_short.exact_rows == current_short.exact_rows - assert materialized_smoke.manifest["backend"]["name"] == "vllm" - assert materialized_smoke.manifest["backend"]["enforce_eager"] is True - assert materialized_smoke.exact_rows == smoke.exact_rows - assert suites.manifest_selected_rows(current_short.exact_rows) == 344 - assert suites.manifest_selected_rows(smoke.exact_rows) == 24 - assert ( - suites.manifest_selected_rows(profiles["judge-free-8_690-examples_r1-native"].exact_rows) - == 690 - ) - assert profiles["full-v1"].exact_rows is None - - -@pytest.mark.parametrize( - ("name", "backend"), - [ - ( - "core-3_full_r1-native", - { - "attention_implementation": "sdpa", - "enable_thinking": False, - "name": "qwen3_5", - }, - ), - ( - "core-3_full_r1-vllm", - {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, - ), - ], -) -def test_core3_full_teacher_profiles_pin_paired_population_and_runtime(name, backend): - contract = contracts.load_profile(name) - - assert contract.manifest["model"] == { - "repository": "Qwen/Qwen3.5-0.8B", - "revision": "2fc06364715b967f1860aea9cf38778875588b17", - } - assert contract.manifest["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION - assert contract.manifest["backend"] == backend - assert contract.manifest["generation"] == {"do_sample": False, "temperature": 0} - assert contract.manifest["seed"] == 42 - assert contract.manifest["repetitions"] == 1 - assert contract.manifest["batch_size"] == 1 - assert contract.manifest["selection"] == "all" - assert contract.exact_rows is None - assert { - task: entry["population_rows"] for task, entry in contract.manifest["tasks"].items() - } == {"realworldqa": 765, "mmmu_val": 900, "mvbench": 4000} - assert contract.manifest["tasks"]["mvbench"]["leaf_populations"] == dict.fromkeys( - suites.MVBENCH_LEAF_TASKS, 200 - ) - assert { - task: entry["dataset_revision"] for task, entry in contract.manifest["tasks"].items() - } == { - task: profile.VLM_BENCHMARK_DATASETS[task].revision - for task in ("realworldqa", "mmmu_val", "mvbench") - } - - -@pytest.mark.parametrize( - ("name", "field", "value", "message"), - [ - ( - "core-3_full_r1-native", - "backend", - {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, - "backend profile differs", - ), - ( - "core-3_full_r1-vllm", - "model", - {"repository": "Qwen/Qwen3.5-0.8B", "revision": "different"}, - "model pin differs", - ), - ("core-3_full_r1-native", "population", 764, "population differs"), - ], -) -def test_core3_full_teacher_profiles_reject_contract_overrides( - monkeypatch, tmp_path, name, field, value, message -): - shutil.copytree(contracts._PROFILE_ROOT, tmp_path, dirs_exist_ok=True) - if field == "backend": - manifest_path = tmp_path / "backends" / "qwen-3.5-native_r1.json" - else: - manifest_path = tmp_path / "sample_sets" / "core-3_full_r1.json" - manifest = json.loads(manifest_path.read_text()) - if field == "population": - manifest["tasks"]["realworldqa"]["population_rows"] = value - elif field == "backend": - manifest["settings"] = value - else: - manifest[field] = value - manifest_path.write_text(json.dumps(manifest)) - monkeypatch.setattr(contracts, "_PROFILE_ROOT", tmp_path) - - with pytest.raises(RuntimeError, match=message): - contracts.load_profile(name) - - -def test_audited_profile_rejects_rows_that_drift_from_systematic_selection(monkeypatch, tmp_path): - shutil.copytree(contracts._PROFILE_ROOT, tmp_path, dirs_exist_ok=True) - manifest_path = tmp_path / "sample_sets" / "core-3_344-examples_r1.json" - manifest = json.loads(manifest_path.read_text()) - manifest["tasks"]["realworldqa"]["rows"][0]["source_row_index"] = 6 - manifest_path.write_text(json.dumps(manifest)) - monkeypatch.setattr(contracts, "_PROFILE_ROOT", tmp_path) - - with pytest.raises(RuntimeError, match="rows differ from its sampling audit"): - contracts.load_profile("core-3_344-examples_r1-native") - - -def test_short_all_native_profile_builds_grouped_and_single_selectors(tmp_path): - contract = contracts.load_profile("judge-free-8_690-examples_r1-native") - exact_rows = contract.exact_rows - assert exact_rows is not None - validated = suites.validate_exact_rows_manifest( - exact_rows, - expected_revision=checkpoint.LMMS_EVAL_REVISION, - expected_tasks=contract.source_tasks, - ) - - tasks._write_quick_selection_module(tmp_path, validated) - spec = importlib.util.spec_from_file_location( - "short_all_selectors", tmp_path / "modelopt_quick_selection.py" - ) - assert spec is not None and spec.loader is not None - selectors = importlib.util.module_from_spec(spec) - spec.loader.exec_module(selectors) - - class Documents: - def __init__(self, size, *, columns=None, rows=None): - self.size = size - self.columns = columns or {} - self.rows = rows or {} - - def __len__(self): - return self.size - - def __getitem__(self, index): - if isinstance(index, str): - return self.columns[index] - return self.rows.get(index, {}) - - def select(self, indices): - return list(indices) - - tasks_manifest = validated["tasks"] - adaptation = [ - row["source_row_index"] - for row in tasks_manifest["video_mmmu"]["rows"] - if row["leaf_task"] == "video_mmmu_adaptation" - ] - assert ( - selectors.select_modelopt_vlm_benchmark_video_mmmu_adaptation(Documents(300)) == adaptation - ) - with pytest.raises(ValueError, match="source population drifted"): - selectors.select_modelopt_vlm_benchmark_video_mmmu_adaptation(Documents(299)) - with pytest.raises(ValueError, match="source population drifted"): - selectors.select_modelopt_vlm_benchmark_realworldqa(Documents(764)) - with pytest.raises(ValueError, match="source population drifted"): - selectors.select_modelopt_vlm_benchmark_mvbench_action_sequence(Documents(199)) - - mmmu_task = tasks_manifest["mmmu_val"] - mmmu_ids = [ - f"validation_{stratum['name']}_{index + 1}" - for stratum in mmmu_task["selection"]["strata"] - for index in range(stratum["population_rows"]) - ] - mmmu_rows = { - row["source_row_index"]: {"id": row["source_sample_id"]} for row in mmmu_task["rows"] - } - mmmu_documents = Documents(900, columns={"id": mmmu_ids}, rows=mmmu_rows) - assert selectors.select_modelopt_vlm_benchmark_mmmu_val(mmmu_documents) == [ - row["source_row_index"] for row in mmmu_task["rows"] - ] - mmmu_ids[0] = mmmu_ids[30] - with pytest.raises(ValueError, match="source strata drifted"): - selectors.select_modelopt_vlm_benchmark_mmmu_val(mmmu_documents) - - assert callable(selectors.select_modelopt_vlm_benchmark_videomme) - - -@pytest.mark.parametrize("task", ["videomme", "mlvu_dev", "perceptiontest_val_mc"]) -@pytest.mark.parametrize( - ("drift", "expected_rank", "observed_strata"), - [ - (None, 1, ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"]), - ("selected stratum", 1, ["alpha|kind", "beta|kind", "beta|kind", "alpha|kind"]), - ("local rank", 0, ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"]), - ], -) -def test_audited_selector_checks_selected_stratum_and_local_rank( - tmp_path, task, drift, expected_rank, observed_strata -): - expected_stratum = "alpha" if task == "mlvu_dev" else "alpha|kind" - other_stratum = "beta" if task == "mlvu_dev" else "beta|kind" - upstream_id = "video:q" if task == "perceptiontest_val_mc" else "q" - manifest = { - "tasks": { - task: { - "rows": [ - { - "source_row_index": 2, - "source_sample_id": f"{task}:2", - "sampling_stratum": expected_stratum, - "source_stratum_index": expected_rank, - "upstream_sample_id": upstream_id, - } - ], - "selection": { - "population_rows": 4, - "strata": [ - {"name": expected_stratum, "population_rows": 2}, - {"name": other_stratum, "population_rows": 2}, - ], - }, - } - } - } - tasks._write_quick_selection_module(tmp_path, manifest) - spec = importlib.util.spec_from_file_location( - f"sampling_position_{task}_{drift}", tmp_path / "modelopt_quick_selection.py" - ) - assert spec is not None and spec.loader is not None - selectors = importlib.util.module_from_spec(spec) - spec.loader.exec_module(selectors) - - class Documents: - def __len__(self): - return 4 - - def __getitem__(self, index): - if isinstance(index, str): - if task == "mlvu_dev": - return [value.split("|", 1)[0] for value in observed_strata] - column = 0 if index in {"duration", "area"} else 1 - return [value.split("|", 1)[column] for value in observed_strata] - if task == "perceptiontest_val_mc": - return {"video_name": "video", "question_id": "q"} - return {"question_id": "q"} - - def select(self, indices): - return list(indices) - - selector = getattr(selectors, f"select_{suites.task_name(task)}") - if drift is None: - assert selector(Documents()) == [2] - else: - with pytest.raises(ValueError, match="source sampling positions drifted"): - selector(Documents()) - - -def test_versioned_profile_preflight_reports_immutable_contract(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - name = "short-v1" - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - name, - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - - contract = contracts.load_profile(name) - assert prepared.report["profile_name"] == name - assert prepared.report["profile_fingerprint"] == contract.fingerprint - assert prepared.report["sample_set"] == contract.sample_set - assert prepared.report["backend_profile"] == contract.backend_profile - assert prepared.report["evaluator_profile"] == contract.evaluator_profile - assert prepared.report["source_tasks"] == list(contract.source_tasks) - assert prepared.report["quick_selected_rows"] == 344 - assert prepared.report["quick_row_identities"] == suites.manifest_row_identities( - prepared.quick_manifest - ) - assert prepared.report["quick_task_denominators"] == suites.manifest_task_denominators( - prepared.quick_manifest - ) - - -def test_native_profile_builds_qwen35_backend_settings(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: object()) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "judge-free-8_690-examples_r1-native", - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - (tmp_path / "tasks").mkdir() - settings = preflight.settings( - args, - tasks_root=tmp_path / "tasks", - configured_tasks=("modelopt_vlm_benchmark_mvbench",), - prepared=prepared, - ) - - assert prepared.report["model_backend"] == "qwen3_5" - assert settings["model"] == "qwen3_5" - assert settings["checkpoint_arg"] == "pretrained" - assert settings["model_args"] == { - "attn_implementation": "sdpa", - "device": "cuda", - "device_map": "cuda", - "enable_thinking": False, - "fps": 2, - "max_frames": 32, - } - assert "reasoning_parser" not in settings - assert not (tmp_path / "tasks/modelopt_qwen35_no_think.jinja").exists() - - -@pytest.mark.parametrize( - ("profile_name", "expected_eager"), - [("core-3_24-examples_r1-vllm", True), ("core-3_344-examples_r1-vllm", None)], -) -def test_vllm_profile_forwards_runtime_settings( - monkeypatch, tmp_path, profile_name, expected_eager -): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - profile_name, - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - tasks_root = tmp_path / "tasks" - tasks_root.mkdir() - settings = preflight.settings( - args, - tasks_root=tasks_root, - configured_tasks=("modelopt_vlm_benchmark_realworldqa",), - prepared=prepared, - ) - argv, _, _ = lmms._build_command( - settings, - checkpoint=str(model), - output_path=tmp_path / "lmms-results", - ) - model_args = argv[argv.index("--model_args") + 1] - - assert settings["model_args"].get("enforce_eager") is expected_eager - assert ("enforce_eager=True" in model_args) is (expected_eager is True) - assert settings["model_args"]["attention_config"] == {"flash_attn_version": 2} - assert 'attention_config={"flash_attn_version":2}' in model_args - - -@pytest.mark.parametrize("name", ["core-3_full_r1-native", "core-3_full_r1-vllm"]) -def test_core3_full_teacher_profiles_preserve_backend_prompt_policy(monkeypatch, tmp_path, name): - model, hf_home = _write_core3_teacher_snapshot(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, ("mmmu_val",)) - _use_offline_fakes(monkeypatch, lmms_root) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - name, - "--profile-task", - "mmmu_val", - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - tasks_root = tmp_path / "tasks" - tasks_root.mkdir() - settings = preflight.settings( - args, - tasks_root=tasks_root, - configured_tasks=("modelopt_vlm_benchmark_mmmu_val",), - prepared=prepared, - ) - - assert prepared.report["model_pin"] == { - "repository": "Qwen/Qwen3.5-0.8B", - "revision": "2fc06364715b967f1860aea9cf38778875588b17", - } - assert prepared.report["profile_population_rows"] == {"mmmu_val": 900} - assert prepared.report["output_budget_contract"]["mmmu_val"]["effective_max_new_tokens"] == 128 - if name == "core-3_full_r1-native": - assert settings["model"] == "qwen3_5" - assert prepared.report["backend_limitations"] == [] - else: - assert settings["model"] == "vllm" - assert prepared.report["output_budget_contract"]["mmmu_val"] == { - "adapter": "vllm", - "effective_max_new_tokens": 128, - "limitation": ( - "the pinned generic vLLM adapter treats its model-level max_new_tokens as a floor" - ), - "requested_max_new_tokens": 128, - "resolution": "max(task_max_new_tokens, model_max_new_tokens_floor=1)", - } - assert prepared.report["backend_limitations"] == [ - "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", - "pinned generic vLLM max_new_tokens is a model-level lower bound", - ] - - -def test_core3_full_teacher_profile_population_expectations_follow_group_shard( - monkeypatch, tmp_path -): - model, hf_home = _write_core3_teacher_snapshot(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, ("mvbench",)) - _use_offline_fakes(monkeypatch, lmms_root) - media = hf_home / profile.VLM_BENCHMARK_DATASETS["mvbench"].media_dir - media.mkdir(parents=True) - (media / "sample").write_bytes(b"media") - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "core-3_full_r1-native", - "--profile-task", - "mvbench", - "--profile-task-shard", - "3/8", - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - expected_leaves = ("episodic_reasoning", "moving_direction", "egocentric_navigation") - - assert prepared.profile_task_leaves == expected_leaves - assert evaluator._expected_task_populations(prepared, ("modelopt_vlm_benchmark_mvbench",)) == { - suites.task_name("mvbench", leaf=leaf): 200 for leaf in expected_leaves - } - - -def test_core3_full_teacher_profile_accepts_snapshot_symlink(monkeypatch, tmp_path): - snapshot, hf_home = _write_core3_teacher_snapshot(tmp_path) - checkpoint_alias = tmp_path / "teacher" - checkpoint_alias.symlink_to(snapshot, target_is_directory=True) - lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) - _use_offline_fakes(monkeypatch, lmms_root) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(checkpoint_alias), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "core-3_full_r1-native", - "--profile-task", - "realworldqa", - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - - assert prepared.source_tasks == ("realworldqa",) - assert prepared.report["model_pin"]["revision"] == snapshot.name - - -@pytest.mark.parametrize("kind", ["untracked-copy", "wrong-revision"]) -def test_core3_full_teacher_profile_rejects_unpinned_checkpoint(monkeypatch, tmp_path, kind): - monkeypatch.delenv("HF_HUB_CACHE", raising=False) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - if kind == "untracked-copy": - model = _write_checkpoint(tmp_path) - else: - parent = hf_home / "hub/models--Qwen--Qwen3.5-0.8B/snapshots" - parent.mkdir(parents=True) - model = _write_checkpoint_at(parent / "different") - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "core-3_full_r1-native", - "--profile-task", - "realworldqa", - "--hf-home", - str(hf_home), - ] - ) - - with pytest.raises(ValueError, match="requires the exact local Hub snapshot"): - preflight.prepare(args) - - -def test_core3_full_teacher_profile_rejects_settings_override(monkeypatch, tmp_path): - model, hf_home = _write_core3_teacher_snapshot(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) - _use_offline_fakes(monkeypatch, lmms_root) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "core-3_full_r1-native", - "--profile-task", - "realworldqa", - "--hf-home", - str(hf_home), - ] - ) - - with pytest.raises(ValueError, match="do not allow settings overrides"): - evaluator.evaluate(args, settings_overrides={"model": "vllm"}) - - -def test_historical_short_profile_preserves_vllm_backend(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "short-v1", - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - (tmp_path / "tasks").mkdir() - settings = preflight.settings( - args, - tasks_root=tmp_path / "tasks", - configured_tasks=("modelopt_vlm_benchmark_realworldqa",), - prepared=prepared, - ) - - assert prepared.report["lmms_eval_revision"] == checkpoint.LMMS_EVAL_LEGACY_REVISION - assert settings["model"] == "vllm" - assert settings["checkpoint_arg"] == "model" - assert settings["reasoning_parser"] == "qwen3" - assert settings["model_args"]["max_frame_num"] == 32 - assert Path(settings["model_args"]["chat_template"]).exists() - - -def test_versioned_profile_rejects_seed_override(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "short-v1", - "--seed", - "7", - "--hf-home", - str(hf_home), - ] - ) - - with pytest.raises(ValueError, match="--seed cannot override"): - preflight.prepare(args) - - -def test_versioned_profile_rejects_batch_size_override(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "short-v1", - "--batch-size", - "8", - "--hf-home", - str(hf_home), - ] - ) - - with pytest.raises(ValueError, match="--batch-size cannot override"): - preflight.prepare(args) - - -def test_all_row_profile_task_preserves_contract_identity(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - profile_name = "full-v1" - task = "realworldqa" - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - profile_name, - "--profile-task", - task, - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - - assert prepared.source_tasks == (task,) - assert prepared.report["quick_selected_rows"] is None - assert prepared.report["quick_row_identities"] is None - assert prepared.report["quick_task_denominators"] is None - assert ( - prepared.report["profile_fingerprint"] == contracts.load_profile(profile_name).fingerprint - ) - - -def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "judge-free-8_690-examples_r1-native", - "--profile-task", - "mvbench", - "--profile-task-shard", - "3/8", - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - - expected_leaves = ("episodic_reasoning", "moving_direction", "egocentric_navigation") - assert prepared.profile_task_leaves == expected_leaves - assert prepared.quick_manifest is not None - manifest_rows = prepared.quick_manifest["tasks"]["mvbench"]["rows"] - assert len(manifest_rows) == 24 - assert {row["leaf_task"] for row in manifest_rows} == { - f"mvbench_{leaf}" for leaf in expected_leaves - } - manifest_selection = prepared.quick_manifest["tasks"]["mvbench"]["selection"] - assert manifest_selection["population_rows"] == 600 - assert manifest_selection["selected_rows"] == 24 - assert [stratum["name"] for stratum in manifest_selection["strata"]] == list(expected_leaves) - assert manifest_selection["selected_index_quantiles"] == { - "method": "lower-order-statistic", - "p0": 12, - "p25": 37, - "p50": 87, - "p75": 137, - "p100": 187, - } - assert ( - manifest_selection["selected_row_identities_sha256"] - == hashlib.sha256( - json.dumps(manifest_rows, separators=(",", ":"), sort_keys=True).encode() - ).hexdigest() - ) - assert prepared.report["quick_selected_rows"] == 24 - assert prepared.report["quick_row_identities"] == suites.manifest_row_identities( - prepared.quick_manifest - ) - assert prepared.report["quick_task_denominators"] == { - "mvbench": {"population_rows": 600, "selected_rows": 24} - } - assert ( - evaluator._expected_task_populations(prepared, ("modelopt_vlm_benchmark_mvbench",)) is None - ) - assert prepared.report["quick_manifest_sha256"] == suites.manifest_sha256( - prepared.quick_manifest - ) - assert ( - prepared.report["profile_fingerprint"] - == contracts.load_profile("judge-free-8_690-examples_r1-native").fingerprint - ) - tasks_root, _ = tasks.prepare( - tmp_path / "results", - suite=prepared.suite, - source_tasks=prepared.source_tasks, - profile_task_leaves=prepared.profile_task_leaves, - dataset_snapshots=prepared.dataset_snapshots, - quick_manifest=prepared.quick_manifest, - ) - group = json.loads((tasks_root / "modelopt_vlm_benchmark_mvbench.yaml").read_text()) - assert group["task"] == [f"modelopt_vlm_benchmark_mvbench_{leaf}" for leaf in expected_leaves] - - -def test_smoke_profile_generates_only_manifest_backed_mvbench_leaves(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "core-3_24-examples_r1-native", - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - tasks_root, configured_tasks = tasks.prepare( - args.output_dir, - suite=prepared.suite, - source_tasks=prepared.source_tasks, - profile_task_leaves=prepared.profile_task_leaves, - dataset_snapshots=prepared.dataset_snapshots, - quick_manifest=prepared.quick_manifest, - ) - - assert configured_tasks == ( - "modelopt_vlm_benchmark_realworldqa", - "modelopt_vlm_benchmark_mmmu_val", - "modelopt_vlm_benchmark_mvbench", - ) - group = json.loads((tasks_root / "modelopt_vlm_benchmark_mvbench.yaml").read_text()) - assert group["task"] == ["modelopt_vlm_benchmark_mvbench_action_sequence"] - assert (tasks_root / "modelopt_vlm_benchmark_mvbench_action_sequence.yaml").is_file() - assert not (tasks_root / "modelopt_vlm_benchmark_mvbench_egocentric_navigation.yaml").exists() - - spec = importlib.util.spec_from_file_location( - "smoke_selectors", tasks_root / "modelopt_quick_selection.py" - ) - assert spec is not None and spec.loader is not None - selectors = importlib.util.module_from_spec(spec) - spec.loader.exec_module(selectors) - assert hasattr(selectors, "select_modelopt_vlm_benchmark_mvbench_action_sequence") - assert not hasattr(selectors, "select_modelopt_vlm_benchmark_mvbench_egocentric_navigation") - - -@pytest.mark.parametrize( - ("selection", "message"), - [ - (("--suite", "short", "--profile-task", "realworldqa"), "requires"), - (("--profile", "short-v1", "--profile-task", "realworldqa"), "supported only"), - (("--profile", "full-v1", "--profile-task-shard", "0/8"), "requires"), - ( - ( - "--profile", - "full-v1", - "--profile-task", - "realworldqa", - "--profile-task-shard", - "0/8", - ), - "supports only", - ), - ], -) -def test_profile_task_rejects_invalid_parent(monkeypatch, tmp_path, selection, message): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - *selection, - "--hf-home", - str(hf_home), - ] - ) - - with pytest.raises(ValueError, match=message): - preflight.prepare(args) - - -def test_post_mip_realworldqa_adapter_runs_pinned_profile(monkeypatch, tmp_path): - model = _write_checkpoint(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - monkeypatch.setenv("HF_HOME", str(hf_home)) - output = tmp_path / "output" - captured = {} - - def fake_runner(checkpoint_path, *, output_root, settings): - report = json.loads((output / "profile.json").read_text()) - assert report["configured_tasks"] == ["modelopt_vlm_benchmark_realworldqa"] - assert report["sample_limit"] == 2 - captured.update( - checkpoint=checkpoint_path, - output_root=output_root, - settings=settings, - ) - result_path = output_root / "result.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text("{}\n") - return { - "metrics": {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5}, - "result_path": str(result_path), - } - - monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) - result = post_mip.evaluate_realworldqa_checkpoint( - model, - output_root=output, - settings={ - "batch_size": 1, - "timeout_seconds": 900, - "dtype": "bfloat16", - "topology": {"tensor_parallel_size": 1}, - }, - ) - - assert captured["checkpoint"] == model - assert captured["output_root"] == output - assert captured["settings"]["tasks"] == "modelopt_vlm_benchmark_realworldqa" - assert captured["settings"]["limit"] == 2 - assert captured["settings"]["timeout_seconds"] == 900 - assert captured["settings"]["dtype"] == "bfloat16" - assert captured["settings"]["topology"] == {"tensor_parallel_size": 1} - assert result["metrics"] == {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5} - assert result["profile_path"] == str(output / "profile.json") - - -def test_post_mip_prefix100_adapter_averages_repeated_bounded_tasks( - monkeypatch, - tmp_path, -): - model = tmp_path / "model" - model.mkdir() - output = tmp_path / "output" - captured = {"invocations": 0} - - def fake_evaluate(args, *, settings_overrides, preflight_callback): - captured["invocations"] += 1 - captured.update(args=args, settings_overrides=settings_overrides) - preflight_callback({"profile": suites.EVALUATION_PROFILE, "sample_limit": None}) - runs = [] - score_offset = (captured["invocations"] - 1) * 0.2 - for index, realworldqa_score in enumerate( - (0.4 + score_offset, 0.6 + score_offset), start=1 - ): - result_path = tmp_path / f"run-{index}.json" - result_path.write_text( - json.dumps( - { - "sample_counts": {"realworldqa": 100, "mmmu_val": 100}, - "mmmu_parser_audit": { - "sample_count": 100, - "status_counts": {"parsed": 90, "fallback_random": 10}, - }, - } - ) - ) - runs.append( - { - "metrics": { - "modelopt_vlm_benchmark_realworldqa.exact_match_none": (realworldqa_score), - "modelopt_vlm_benchmark_mmmu_val.mmmu_acc_none": 0.3, - }, - "result_path": str(result_path), - } - ) - return {"runs": runs} - - monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) - result = post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( - model, - output_root=output, - settings={ - "batch_size": 1, - "timeout_seconds": 14400, - "dtype": "bfloat16", - "topology": {"tensor_parallel_size": 1}, - }, - ) - - assert captured["args"].suite == suites.TASK_PREFIX100_REPEAT2_SUITE - assert captured["args"].batch_size == 1 - assert captured["args"].seed == 42 - assert captured["settings_overrides"] == { - "dtype": "bfloat16", - "topology": {"tensor_parallel_size": 1}, - } - assert result["metrics"] == { - "modelopt_vlm_benchmark_mmmu_val.mmmu_acc_none": 0.3, - "modelopt_vlm_benchmark_realworldqa.exact_match_none": 0.5, - } - assert result["profile"] == post_mip.TASK_PREFIX100_REPEAT2_PROFILE - summary = json.loads(Path(result["result_path"]).read_text()) - assert summary["suite"] == suites.TASK_PREFIX100_REPEAT2_SUITE - assert summary["profile"] == post_mip.TASK_PREFIX100_REPEAT2_PROFILE - assert summary["metrics"] == result["metrics"] - assert summary["result_paths"] == result["run_result_paths"] - assert summary["sample_counts"] == {"mmmu_val": 200, "realworldqa": 200} - assert summary["mmmu_parser_audit"] == { - "sample_count": 200, - "status_counts": {"fallback_random": 20, "parsed": 180}, - } - - refreshed = post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( - model, - output_root=output, - settings={ - "batch_size": 1, - "timeout_seconds": 14400, - "dtype": "bfloat16", - "topology": {"tensor_parallel_size": 1}, - }, - ) - assert refreshed["metrics"][ - "modelopt_vlm_benchmark_realworldqa.exact_match_none" - ] == pytest.approx(0.7) - assert json.loads(Path(refreshed["result_path"]).read_text())["metrics"] == refreshed["metrics"] - - -def test_post_mip_prefix100_rejects_different_repetition_metrics( - monkeypatch, - tmp_path, -): - def fake_evaluate(args, *, settings_overrides, preflight_callback): - return { - "runs": [ - {"metrics": {"realworldqa.accuracy": 0.5}, "result_path": "first.json"}, - {"metrics": {"mmmu.accuracy": 0.5}, "result_path": "second.json"}, - ] - } - - monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) - - with pytest.raises(RuntimeError, match="produced different metrics"): - post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( - tmp_path / "model", - output_root=tmp_path / "output", - settings={}, - ) - - -def test_deprecated_post_mip_profile_alias_forwards_to_canonical(monkeypatch, tmp_path): - expected = {"metrics": {"accuracy": 0.5}} - monkeypatch.setattr( - post_mip, - "evaluate_realworldqa_mmmu_prefix100_checkpoint", - lambda *_args, **_kwargs: expected, - ) - - with pytest.warns(FutureWarning, match="is deprecated"): - result = post_mip.evaluate_e2e_full_eval_checkpoint( - tmp_path / "model", - output_root=tmp_path / "output", - settings={}, - ) - - assert result is expected - - -def test_mmvu_guard_is_limited_to_full_suite(monkeypatch, tmp_path): - snapshot = tmp_path / "snapshot" - snapshot.mkdir() - lmms_root = _write_lmms_tasks(tmp_path, profile.VLM_BENCHMARK_TASKS) - monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) - - tasks_root, configured_tasks = tasks.prepare( - tmp_path / "results", - suite="mmvu-smoke", - dataset_snapshots={"mmvu_val": snapshot}, - quick_manifest=None, - ) - - assert configured_tasks == ("modelopt_vlm_benchmark_mmvu_val",) - assert not (tasks_root / "modelopt_mmvu_guard.py").exists() - generated = (tasks_root / "modelopt_vlm_benchmark_mmvu_val.yaml").read_text() - assert "\nprocess_results:" not in generated - assert ( - "process_docs: !function " - "modelopt_mmvu_smoke_selection.select_modelopt_vlm_benchmark_mmvu_val\n" in generated - ) - assert (tasks_root / "modelopt_mmvu_smoke_selection.py").is_file() - assert not (tasks_root / "modelopt_quick_selection.py").exists() - - full_root, _ = tasks.prepare( - tmp_path / "full-results", - suite="full", - dataset_snapshots=dict.fromkeys(profile.VLM_BENCHMARK_TASKS, snapshot), - quick_manifest=None, - ) - assert (full_root / "modelopt_mmvu_guard.py").is_file() - full_generated = (full_root / "modelopt_vlm_benchmark_mmvu_val.yaml").read_text() - assert "\nprocess_results: !function modelopt_mmvu_guard.process_results\n" in full_generated - - -def test_mmmu_adapter_labels_parser_fallback_without_changing_prediction(monkeypatch, tmp_path): - snapshot = tmp_path / "snapshot" - snapshot.mkdir() - lmms_root = _write_lmms_tasks(tmp_path, ("mmmu_val",)) - monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) - tasks_root, _ = tasks.prepare( - tmp_path / "results", - suite="short", - source_tasks=("mmmu_val",), - dataset_snapshots={"mmmu_val": snapshot}, - quick_manifest=None, - ) - - upstream = ModuleType("lmms_eval.tasks.mmmu.utils") - - def get_multi_choice_info(options): - choices = [chr(ord("A") + index) for index in range(len(options))] - return dict(zip(choices, options, strict=True)), choices - - def parse_multi_choice_response(response, all_choices, _index_to_answer): - return random.choice(all_choices) if response == "unparseable" else "A" - - def mmmu_process_results(document, results): - if document["question_type"] == "multiple-choice": - index_to_answer, choices = get_multi_choice_info(json.loads(document["options"])) - parsed = [ - parse_multi_choice_response(response, choices, index_to_answer) - for response in results - ] - else: - parsed = [""] * len(results) - accuracy = {"parsed_pred": parsed} - return {"mmmu_acc": accuracy, "mmmu_acc_pass_at_k": accuracy} - - upstream.get_multi_choice_info = get_multi_choice_info - upstream.parse_multi_choice_response = parse_multi_choice_response - upstream.mmmu_process_results = mmmu_process_results - package_modules = { - "lmms_eval": ModuleType("lmms_eval"), - "lmms_eval.tasks": ModuleType("lmms_eval.tasks"), - "lmms_eval.tasks.mmmu": ModuleType("lmms_eval.tasks.mmmu"), - "lmms_eval.tasks.mmmu.utils": upstream, - } - package_modules["lmms_eval.tasks.mmmu"].utils = upstream - for name, module in package_modules.items(): - monkeypatch.setitem(sys.modules, name, module) - spec = importlib.util.spec_from_file_location( - "modelopt_mmmu_audit", tasks_root / "modelopt_mmmu_audit.py" - ) - assert spec is not None and spec.loader is not None - audit_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(audit_module) - document = { - "options": json.dumps(["first", "second"]), - "question_type": "multiple-choice", - } - - parsed = audit_module.process_results(document, ["(A)"]) - random.seed(123) - parse_multi_choice_response("unparseable", ["A", "B"], {}) - expected_random_state = random.getstate() - random.seed(123) - fallback = audit_module.process_results(document, ["unparseable"]) - invalid_open = audit_module.process_results( - {"question_type": "open"}, - ["unparseable"], - ) - - assert parsed["mmmu_acc"] == { - "parsed_pred": ["A"], - "parser_status": ["parsed"], - } - assert fallback["mmmu_acc"]["parsed_pred"][0] in {"A", "B"} - assert fallback["mmmu_acc"]["parser_status"] == ["fallback_random"] - assert random.getstate() == expected_random_state - assert invalid_open["mmmu_acc"]["parser_status"] == ["invalid_open"] - generated = (tasks_root / f"{suites.task_name('mmmu_val')}.yaml").read_text() - assert "process_results: !function modelopt_mmmu_audit.process_results\n" in generated - - -def test_quick_manifest_requires_exact_pins_counts_and_leaf_balance(tmp_path): - path = _quick_manifest(tmp_path / "quick.json") - suites.load_quick_manifest(path) - - manifest = json.loads(path.read_text()) - manifest["tasks"]["mmmu_val"]["rows"].pop() - path.write_text(json.dumps(manifest)) - with pytest.raises(ValueError, match="exactly 120 rows"): - suites.load_quick_manifest(path) - - path = _quick_manifest(path) - manifest = json.loads(path.read_text()) - manifest["tasks"]["mvbench"]["rows"][-1] = { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 8, - "source_sample_id": "action_sequence:8", - } - path.write_text(json.dumps(manifest)) - with pytest.raises(ValueError, match="exactly 8 rows per leaf task"): - suites.load_quick_manifest(path) - - -@pytest.mark.parametrize( - ("extra", "environment", "message"), - [ - ([], {"OPENAI_API_KEY": "token"}, "explicit --allow-judge-calls"), - (["--allow-judge-calls"], {}, "judge credentials are missing"), - ], -) -def test_full_profile_fails_closed_without_judge_authorization_or_credentials( - monkeypatch, tmp_path, extra, environment, message -): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - for name in ("OPENAI_API_KEY", "AZURE_API_KEY", "AZURE_ENDPOINT"): - monkeypatch.delenv(name, raising=False) - for name, value in environment.items(): - monkeypatch.setenv(name, value) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--suite", - "full", - "--hf-home", - str(hf_home), - "--mmvu-judge-api-type", - "openai", - "--mmvu-judge-model", - "judge", - *extra, - ] - ) - with pytest.raises(ValueError, match=message): - preflight.prepare(args) - - -def test_offline_preflight_scrubs_credentials_and_traverses_media(monkeypatch, tmp_path): - for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES: - monkeypatch.setenv(name, "must-not-reach-child") - tasks_root = tmp_path / "tasks" - package = tasks_root / "lmms_eval/tasks" - package.mkdir(parents=True) - (tasks_root / "lmms_eval/__init__.py").write_text("") - (package / "__init__.py").write_text("""import os -class Config: task = "modelopt_vlm_benchmark_mvbench_action_sequence" -class Task: - config = Config() - def has_test_docs(self): return True - def has_validation_docs(self): return False - def has_training_docs(self): return False - def test_docs(self): return [{"video": os.environ["FAKE_MEDIA_PATH"]}] - def doc_to_visual(self, document): return [document["video"]] -class Group: group_name = "modelopt_vlm_benchmark_mvbench" -class TaskManager: - def __init__(self, include_path, model_name): - assert model_name == "qwen3_5" - assert os.environ["HF_DATASETS_OFFLINE"] == "1" - assert os.environ["HF_HUB_OFFLINE"] == "1" - assert os.environ["API_TYPE"] == "openai" - assert os.environ["MODEL_VERSION"] == "modelopt-disabled-lmms-eval-judge" - assert os.environ["OPENAI_API_KEY"] == "modelopt-disabled-lmms-eval-judge" - assert os.environ["OPENAI_API_URL"] == "http://127.0.0.1:9" - credential_names = ( - "HF_TOKEN", - "HUGGINGFACEHUB_API_TOKEN", - "HUGGING_FACE_HUB_TOKEN", - ) - assert all(name not in os.environ for name in credential_names) - def load_task_or_group(self, tasks): - first = Task() - if os.environ.get("FAKE_DISTINCT_DUPLICATE"): - return {Group(): {"first": first, "second": Task()}} - return {Group(): {"first": first, "repeat": first}} -""") - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - media = tmp_path / "sample.mp4" - media.write_bytes(b"video") - monkeypatch.setenv("FAKE_MEDIA_PATH", str(media)) - - report = tasks.verify_offline( - tasks_root, - ("modelopt_vlm_benchmark_mvbench",), - hf_home=hf_home, - timeout_seconds=123, - model_name="qwen3_5", - expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 1}, - ) - - assert report["document_counts"] == {"modelopt_vlm_benchmark_mvbench_action_sequence": 1} - assert report["media_documents"] == 1 - assert report["observed_populations"] == {"modelopt_vlm_benchmark_mvbench_action_sequence": 1} - assert report["status"] == "passed" - - with pytest.raises(RuntimeError, match="configured task population mismatch"): - tasks.verify_offline( - tasks_root, - ("modelopt_vlm_benchmark_mvbench",), - hf_home=hf_home, - timeout_seconds=123, - model_name="qwen3_5", - expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 2}, - ) - - monkeypatch.setenv("FAKE_DISTINCT_DUPLICATE", "1") - with pytest.raises(RuntimeError, match="distinct task objects share configured task name"): - tasks.verify_offline( - tasks_root, - ("modelopt_vlm_benchmark_mvbench",), - hf_home=hf_home, - timeout_seconds=123, - model_name="qwen3_5", - expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 1}, - ) - - -def test_video_adapter_normalizes_supported_suffixes_and_rejects_unknown(monkeypatch, tmp_path): - uppercase = tmp_path / "sample.MP4" - matroska = tmp_path / "sample.mkv" - unknown = tmp_path / "sample.webm" - for path in (uppercase, matroska, unknown): - path.write_bytes(b"video") - videomme = ModuleType("lmms_eval.tasks.videomme.utils") - videomme.videomme_doc_to_visual = lambda _document: [str(matroska)] - perception = ModuleType("lmms_eval.tasks.perceptiontest.val.utils") - perception.perceptiontest_val_doc_to_visual = lambda _document: [str(uppercase)] - packages = { - "lmms_eval": ModuleType("lmms_eval"), - "lmms_eval.tasks": ModuleType("lmms_eval.tasks"), - "lmms_eval.tasks.videomme": ModuleType("lmms_eval.tasks.videomme"), - "lmms_eval.tasks.videomme.utils": videomme, - "lmms_eval.tasks.perceptiontest": ModuleType("lmms_eval.tasks.perceptiontest"), - "lmms_eval.tasks.perceptiontest.val": ModuleType("lmms_eval.tasks.perceptiontest.val"), - "lmms_eval.tasks.perceptiontest.val.utils": perception, - } - packages["lmms_eval.tasks.videomme"].utils = videomme - packages["lmms_eval.tasks.perceptiontest.val"].utils = perception - for name, module in packages.items(): - monkeypatch.setitem(sys.modules, name, module) - - tasks._write_video_path_adapter(tmp_path) - spec = importlib.util.spec_from_file_location( - "adapter_test", tmp_path / "modelopt_video_paths.py" - ) - assert spec is not None and spec.loader is not None - adapter = importlib.util.module_from_spec(spec) - spec.loader.exec_module(adapter) - - for generated, source in ( - (adapter.videomme_doc_to_visual({})[0], matroska), - (adapter.perceptiontest_doc_to_visual({})[0], uppercase), - ): - alias = Path(generated) - assert alias.suffix == ".mp4" - assert alias.resolve() == source - with pytest.raises(ValueError, match="unsupported Qwen 3.5 video suffix"): - adapter._normalize([str(unknown)]) - - -def test_video_reader_validation_is_limited_to_video_suites(monkeypatch): - monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: None) - - preflight._verify_video_reader(("realworldqa", "mmmu_val")) - with pytest.raises(RuntimeError, match="decord-compatible reader"): - preflight._verify_video_reader(("mvbench",)) - - -def test_native_backend_validation_requires_qwen_vision_utilities(monkeypatch): - monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: None) - - preflight._verify_backend_dependencies("vllm") - with pytest.raises(RuntimeError, match="qwen-vl-utils"): - preflight._verify_backend_dependencies("qwen3_5") - - def test_requirements_pin_matches_runtime_lmms_eval_revision(): requirements = (checkpoint.REPOSITORY_ROOT / "examples/puzzletron/requirements.txt").read_text() assert "lmms-eval.git" not in requirements @@ -2061,11 +80,3 @@ def test_vlm_parser_defaults_to_short_suite(): def test_huggingface_dependency_supports_range_metadata_api(): pyproject = (checkpoint.REPOSITORY_ROOT / "pyproject.toml").read_text() assert '"huggingface_hub>=0.30.0",' in pyproject - - -def test_credential_scope_restores_inherited_values(monkeypatch): - for index, name in enumerate(checkpoint.HUGGINGFACE_CREDENTIAL_NAMES): - monkeypatch.setenv(name, f"secret-{index}") - with checkpoint.without_huggingface_credentials(): - assert all(name not in os.environ for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) - assert all(name in os.environ for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py new file mode 100644 index 00000000000..0a107f30b96 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py @@ -0,0 +1,470 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for generated lmms-eval task adapters and selectors.""" + +import importlib.util +import json +import random +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import contracts, preflight, profile, suites, tasks +from examples.puzzletron.evaluation.vlm import run as evaluation +from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import _full_inputs, _write_lmms_tasks + + +def test_short_all_native_profile_builds_grouped_and_single_selectors(tmp_path): + contract = contracts.load_profile("judge-free-8_690-examples_r1-native") + exact_rows = contract.exact_rows + assert exact_rows is not None + validated = suites.validate_exact_rows_manifest( + exact_rows, + expected_revision=checkpoint.LMMS_EVAL_REVISION, + expected_tasks=contract.source_tasks, + ) + + tasks._write_quick_selection_module(tmp_path, validated) + spec = importlib.util.spec_from_file_location( + "short_all_selectors", tmp_path / "modelopt_quick_selection.py" + ) + assert spec is not None and spec.loader is not None + selectors = importlib.util.module_from_spec(spec) + spec.loader.exec_module(selectors) + + class Documents: + def __init__(self, size, *, columns=None, rows=None): + self.size = size + self.columns = columns or {} + self.rows = rows or {} + + def __len__(self): + return self.size + + def __getitem__(self, index): + if isinstance(index, str): + return self.columns[index] + return self.rows.get(index, {}) + + def select(self, indices): + return list(indices) + + tasks_manifest = validated["tasks"] + adaptation = [ + row["source_row_index"] + for row in tasks_manifest["video_mmmu"]["rows"] + if row["leaf_task"] == "video_mmmu_adaptation" + ] + assert ( + selectors.select_modelopt_vlm_benchmark_video_mmmu_adaptation(Documents(300)) == adaptation + ) + with pytest.raises(ValueError, match="source population drifted"): + selectors.select_modelopt_vlm_benchmark_video_mmmu_adaptation(Documents(299)) + with pytest.raises(ValueError, match="source population drifted"): + selectors.select_modelopt_vlm_benchmark_realworldqa(Documents(764)) + with pytest.raises(ValueError, match="source population drifted"): + selectors.select_modelopt_vlm_benchmark_mvbench_action_sequence(Documents(199)) + + mmmu_task = tasks_manifest["mmmu_val"] + mmmu_ids = [ + f"validation_{stratum['name']}_{index + 1}" + for stratum in mmmu_task["selection"]["strata"] + for index in range(stratum["population_rows"]) + ] + mmmu_rows = { + row["source_row_index"]: {"id": row["source_sample_id"]} for row in mmmu_task["rows"] + } + mmmu_documents = Documents(900, columns={"id": mmmu_ids}, rows=mmmu_rows) + assert selectors.select_modelopt_vlm_benchmark_mmmu_val(mmmu_documents) == [ + row["source_row_index"] for row in mmmu_task["rows"] + ] + mmmu_ids[0] = mmmu_ids[30] + with pytest.raises(ValueError, match="source strata drifted"): + selectors.select_modelopt_vlm_benchmark_mmmu_val(mmmu_documents) + + assert callable(selectors.select_modelopt_vlm_benchmark_videomme) + + +@pytest.mark.parametrize( + ("task", "drift", "expected_rank", "observed_strata"), + [ + ("videomme", None, 1, ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"]), + ("mlvu_dev", None, 1, ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"]), + ( + "perceptiontest_val_mc", + None, + 1, + ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"], + ), + ( + "videomme", + "selected stratum", + 1, + ["alpha|kind", "beta|kind", "beta|kind", "alpha|kind"], + ), + ( + "videomme", + "local rank", + 0, + ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"], + ), + ], +) +def test_audited_selector_checks_selected_stratum_and_local_rank( + tmp_path, task, drift, expected_rank, observed_strata +): + expected_stratum = "alpha" if task == "mlvu_dev" else "alpha|kind" + other_stratum = "beta" if task == "mlvu_dev" else "beta|kind" + upstream_id = "video:q" if task == "perceptiontest_val_mc" else "q" + manifest = { + "tasks": { + task: { + "rows": [ + { + "source_row_index": 2, + "source_sample_id": f"{task}:2", + "sampling_stratum": expected_stratum, + "source_stratum_index": expected_rank, + "upstream_sample_id": upstream_id, + } + ], + "selection": { + "population_rows": 4, + "strata": [ + {"name": expected_stratum, "population_rows": 2}, + {"name": other_stratum, "population_rows": 2}, + ], + }, + } + } + } + tasks._write_quick_selection_module(tmp_path, manifest) + spec = importlib.util.spec_from_file_location( + f"sampling_position_{task}_{drift}", tmp_path / "modelopt_quick_selection.py" + ) + assert spec is not None and spec.loader is not None + selectors = importlib.util.module_from_spec(spec) + spec.loader.exec_module(selectors) + + class Documents: + def __len__(self): + return 4 + + def __getitem__(self, index): + if isinstance(index, str): + if task == "mlvu_dev": + return [value.split("|", 1)[0] for value in observed_strata] + column = 0 if index in {"duration", "area"} else 1 + return [value.split("|", 1)[column] for value in observed_strata] + if task == "perceptiontest_val_mc": + return {"video_name": "video", "question_id": "q"} + return {"question_id": "q"} + + def select(self, indices): + return list(indices) + + selector = getattr(selectors, f"select_{suites.task_name(task)}") + if drift is None: + assert selector(Documents()) == [2] + else: + with pytest.raises(ValueError, match="source sampling positions drifted"): + selector(Documents()) + + +def test_smoke_profile_generates_only_manifest_backed_mvbench_leaves(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_24-examples_r1-native", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + tasks_root, configured_tasks = tasks.prepare( + args.output_dir, + suite=prepared.suite, + source_tasks=prepared.source_tasks, + profile_task_leaves=prepared.profile_task_leaves, + dataset_snapshots=prepared.dataset_snapshots, + quick_manifest=prepared.quick_manifest, + ) + + assert configured_tasks == ( + "modelopt_vlm_benchmark_realworldqa", + "modelopt_vlm_benchmark_mmmu_val", + "modelopt_vlm_benchmark_mvbench", + ) + group = json.loads((tasks_root / "modelopt_vlm_benchmark_mvbench.yaml").read_text()) + assert group["task"] == ["modelopt_vlm_benchmark_mvbench_action_sequence"] + assert (tasks_root / "modelopt_vlm_benchmark_mvbench_action_sequence.yaml").is_file() + assert not (tasks_root / "modelopt_vlm_benchmark_mvbench_egocentric_navigation.yaml").exists() + + spec = importlib.util.spec_from_file_location( + "smoke_selectors", tasks_root / "modelopt_quick_selection.py" + ) + assert spec is not None and spec.loader is not None + selectors = importlib.util.module_from_spec(spec) + spec.loader.exec_module(selectors) + assert hasattr(selectors, "select_modelopt_vlm_benchmark_mvbench_action_sequence") + assert not hasattr(selectors, "select_modelopt_vlm_benchmark_mvbench_egocentric_navigation") + + +def test_mmvu_guard_is_limited_to_full_suite(monkeypatch, tmp_path): + snapshot = tmp_path / "snapshot" + snapshot.mkdir() + lmms_root = _write_lmms_tasks(tmp_path, profile.VLM_BENCHMARK_TASKS) + monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) + + tasks_root, configured_tasks = tasks.prepare( + tmp_path / "results", + suite="mmvu-smoke", + dataset_snapshots={"mmvu_val": snapshot}, + quick_manifest=None, + ) + + assert configured_tasks == ("modelopt_vlm_benchmark_mmvu_val",) + assert not (tasks_root / "modelopt_mmvu_guard.py").exists() + generated = (tasks_root / "modelopt_vlm_benchmark_mmvu_val.yaml").read_text() + assert "\nprocess_results:" not in generated + assert ( + "process_docs: !function " + "modelopt_mmvu_smoke_selection.select_modelopt_vlm_benchmark_mmvu_val\n" in generated + ) + assert (tasks_root / "modelopt_mmvu_smoke_selection.py").is_file() + assert not (tasks_root / "modelopt_quick_selection.py").exists() + + full_root, _ = tasks.prepare( + tmp_path / "full-results", + suite="full", + dataset_snapshots=dict.fromkeys(profile.VLM_BENCHMARK_TASKS, snapshot), + quick_manifest=None, + ) + assert (full_root / "modelopt_mmvu_guard.py").is_file() + full_generated = (full_root / "modelopt_vlm_benchmark_mmvu_val.yaml").read_text() + assert "\nprocess_results: !function modelopt_mmvu_guard.process_results\n" in full_generated + + +def test_mmmu_adapter_labels_parser_fallback_without_changing_prediction(monkeypatch, tmp_path): + snapshot = tmp_path / "snapshot" + snapshot.mkdir() + lmms_root = _write_lmms_tasks(tmp_path, ("mmmu_val",)) + monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) + tasks_root, _ = tasks.prepare( + tmp_path / "results", + suite="short", + source_tasks=("mmmu_val",), + dataset_snapshots={"mmmu_val": snapshot}, + quick_manifest=None, + ) + + upstream = ModuleType("lmms_eval.tasks.mmmu.utils") + + def get_multi_choice_info(options): + choices = [chr(ord("A") + index) for index in range(len(options))] + return dict(zip(choices, options, strict=True)), choices + + def parse_multi_choice_response(response, all_choices, _index_to_answer): + return random.choice(all_choices) if response == "unparseable" else "A" + + def mmmu_process_results(document, results): + if document["question_type"] == "multiple-choice": + index_to_answer, choices = get_multi_choice_info(json.loads(document["options"])) + parsed = [ + parse_multi_choice_response(response, choices, index_to_answer) + for response in results + ] + else: + parsed = [""] * len(results) + accuracy = {"parsed_pred": parsed} + return {"mmmu_acc": accuracy, "mmmu_acc_pass_at_k": accuracy} + + upstream.get_multi_choice_info = get_multi_choice_info + upstream.parse_multi_choice_response = parse_multi_choice_response + upstream.mmmu_process_results = mmmu_process_results + package_modules = { + "lmms_eval": ModuleType("lmms_eval"), + "lmms_eval.tasks": ModuleType("lmms_eval.tasks"), + "lmms_eval.tasks.mmmu": ModuleType("lmms_eval.tasks.mmmu"), + "lmms_eval.tasks.mmmu.utils": upstream, + } + package_modules["lmms_eval.tasks.mmmu"].utils = upstream + for name, module in package_modules.items(): + monkeypatch.setitem(sys.modules, name, module) + spec = importlib.util.spec_from_file_location( + "modelopt_mmmu_audit", tasks_root / "modelopt_mmmu_audit.py" + ) + assert spec is not None and spec.loader is not None + audit_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(audit_module) + document = { + "options": json.dumps(["first", "second"]), + "question_type": "multiple-choice", + } + + parsed = audit_module.process_results(document, ["(A)"]) + random.seed(123) + parse_multi_choice_response("unparseable", ["A", "B"], {}) + expected_random_state = random.getstate() + random.seed(123) + fallback = audit_module.process_results(document, ["unparseable"]) + invalid_open = audit_module.process_results( + {"question_type": "open"}, + ["unparseable"], + ) + + assert parsed["mmmu_acc"] == { + "parsed_pred": ["A"], + "parser_status": ["parsed"], + } + assert fallback["mmmu_acc"]["parsed_pred"][0] in {"A", "B"} + assert fallback["mmmu_acc"]["parser_status"] == ["fallback_random"] + assert random.getstate() == expected_random_state + assert invalid_open["mmmu_acc"]["parser_status"] == ["invalid_open"] + generated = (tasks_root / f"{suites.task_name('mmmu_val')}.yaml").read_text() + assert "process_results: !function modelopt_mmmu_audit.process_results\n" in generated + + +def test_offline_preflight_scrubs_credentials_and_traverses_media(monkeypatch, tmp_path): + for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES: + monkeypatch.setenv(name, "must-not-reach-child") + tasks_root = tmp_path / "tasks" + package = tasks_root / "lmms_eval/tasks" + package.mkdir(parents=True) + (tasks_root / "lmms_eval/__init__.py").write_text("") + (package / "__init__.py").write_text("""import os +class Config: task = "modelopt_vlm_benchmark_mvbench_action_sequence" +class Task: + config = Config() + def has_test_docs(self): return True + def has_validation_docs(self): return False + def has_training_docs(self): return False + def test_docs(self): return [{"video": os.environ["FAKE_MEDIA_PATH"]}] + def doc_to_visual(self, document): return [document["video"]] +class Group: group_name = "modelopt_vlm_benchmark_mvbench" +class TaskManager: + def __init__(self, include_path, model_name): + assert model_name == "qwen3_5" + assert os.environ["HF_DATASETS_OFFLINE"] == "1" + assert os.environ["HF_HUB_OFFLINE"] == "1" + assert os.environ["API_TYPE"] == "openai" + assert os.environ["MODEL_VERSION"] == "modelopt-disabled-lmms-eval-judge" + assert os.environ["OPENAI_API_KEY"] == "modelopt-disabled-lmms-eval-judge" + assert os.environ["OPENAI_API_URL"] == "http://127.0.0.1:9" + credential_names = ( + "HF_TOKEN", + "HUGGINGFACEHUB_API_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + ) + assert all(name not in os.environ for name in credential_names) + def load_task_or_group(self, tasks): + first = Task() + if os.environ.get("FAKE_DISTINCT_DUPLICATE"): + return {Group(): {"first": first, "second": Task()}} + return {Group(): {"first": first, "repeat": first}} +""") + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + media = tmp_path / "sample.mp4" + media.write_bytes(b"video") + monkeypatch.setenv("FAKE_MEDIA_PATH", str(media)) + + report = tasks.verify_offline( + tasks_root, + ("modelopt_vlm_benchmark_mvbench",), + hf_home=hf_home, + timeout_seconds=123, + model_name="qwen3_5", + expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 1}, + ) + + assert report["document_counts"] == {"modelopt_vlm_benchmark_mvbench_action_sequence": 1} + assert report["media_documents"] == 1 + assert report["observed_populations"] == {"modelopt_vlm_benchmark_mvbench_action_sequence": 1} + assert report["status"] == "passed" + + with pytest.raises(RuntimeError, match="configured task population mismatch"): + tasks.verify_offline( + tasks_root, + ("modelopt_vlm_benchmark_mvbench",), + hf_home=hf_home, + timeout_seconds=123, + model_name="qwen3_5", + expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 2}, + ) + + monkeypatch.setenv("FAKE_DISTINCT_DUPLICATE", "1") + with pytest.raises(RuntimeError, match="distinct task objects share configured task name"): + tasks.verify_offline( + tasks_root, + ("modelopt_vlm_benchmark_mvbench",), + hf_home=hf_home, + timeout_seconds=123, + model_name="qwen3_5", + expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 1}, + ) + + +def test_video_adapter_normalizes_supported_suffixes_and_rejects_unknown(monkeypatch, tmp_path): + uppercase = tmp_path / "sample.MP4" + matroska = tmp_path / "sample.mkv" + unknown = tmp_path / "sample.webm" + for path in (uppercase, matroska, unknown): + path.write_bytes(b"video") + videomme = ModuleType("lmms_eval.tasks.videomme.utils") + videomme.videomme_doc_to_visual = lambda _document: [str(matroska)] + perception = ModuleType("lmms_eval.tasks.perceptiontest.val.utils") + perception.perceptiontest_val_doc_to_visual = lambda _document: [str(uppercase)] + packages = { + "lmms_eval": ModuleType("lmms_eval"), + "lmms_eval.tasks": ModuleType("lmms_eval.tasks"), + "lmms_eval.tasks.videomme": ModuleType("lmms_eval.tasks.videomme"), + "lmms_eval.tasks.videomme.utils": videomme, + "lmms_eval.tasks.perceptiontest": ModuleType("lmms_eval.tasks.perceptiontest"), + "lmms_eval.tasks.perceptiontest.val": ModuleType("lmms_eval.tasks.perceptiontest.val"), + "lmms_eval.tasks.perceptiontest.val.utils": perception, + } + packages["lmms_eval.tasks.videomme"].utils = videomme + packages["lmms_eval.tasks.perceptiontest.val"].utils = perception + for name, module in packages.items(): + monkeypatch.setitem(sys.modules, name, module) + + tasks._write_video_path_adapter(tmp_path) + spec = importlib.util.spec_from_file_location( + "adapter_test", tmp_path / "modelopt_video_paths.py" + ) + assert spec is not None and spec.loader is not None + adapter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(adapter) + + for generated, source in ( + (adapter.videomme_doc_to_visual({})[0], matroska), + (adapter.perceptiontest_doc_to_visual({})[0], uppercase), + ): + alias = Path(generated) + assert alias.suffix == ".mp4" + assert alias.resolve() == source + with pytest.raises(ValueError, match="unsupported Qwen 3.5 video suffix"): + adapter._normalize([str(unknown)]) diff --git a/tests/unit/torch/puzzletron/test_ci_image_contract.py b/tests/unit/torch/puzzletron/test_ci_image_contract.py index ad571cbfb3a..715a6fc8fef 100644 --- a/tests/unit/torch/puzzletron/test_ci_image_contract.py +++ b/tests/unit/torch/puzzletron/test_ci_image_contract.py @@ -148,9 +148,7 @@ def test_lmms_eval_compatibility_patch_reconciles_worker_dependencies(project_ro assert 'python -m pip install -e "${LMMS_EVAL_ROOT}[qwen]"' in dockerfile -def test_lmms_eval_vllm_patch_preserves_task_sampling_and_rejects_drift( - project_root_path, tmp_path -): +def test_lmms_eval_vllm_patch_preserves_task_sampling(project_root_path, tmp_path): puzzletron_root = project_root_path / "examples/puzzletron" environment = json.loads((puzzletron_root / "ci_environment.json").read_text()) patch_text = ( @@ -168,7 +166,7 @@ def test_lmms_eval_vllm_patch_preserves_task_sampling_and_rejects_drift( ) undefined_overwrite = " sampling_params = SamplingParams(**params)" - def write_fixture(root, *, overwrite): + def write_fixture(root): source = root / "lmms_eval/models/simple/vllm.py" source.parent.mkdir(parents=True) lines = ["# pinned upstream fixture"] * 522 @@ -177,7 +175,7 @@ def write_fixture(root, *, overwrite): 'self._select_max_new_tokens(gen_kwargs.get("max_new_tokens"))' ) lines[477] = correct_sampling - lines[520] = overwrite + lines[520] = undefined_overwrite lines[521] = ( ' self._write_watchdog_heartbeat("chat_start", ' "batch_idx=batch_idx, batch_requests=batch_requests)" @@ -188,7 +186,7 @@ def write_fixture(root, *, overwrite): exact_checkout = tmp_path / "exact" exact_checkout.mkdir() - exact_source = write_fixture(exact_checkout, overwrite=undefined_overwrite) + exact_source = write_fixture(exact_checkout) subprocess.run( ["git", "apply", "--unidiff-zero", "--check", str(vllm_patch)], cwd=exact_checkout, @@ -203,20 +201,6 @@ def write_fixture(root, *, overwrite): assert correct_sampling in patched_source assert undefined_overwrite not in patched_source - drifted_checkout = tmp_path / "drifted" - drifted_checkout.mkdir() - write_fixture( - drifted_checkout, - overwrite=" sampling_params = SamplingParams(**other_params)", - ) - rejected = subprocess.run( - ["git", "apply", "--unidiff-zero", "--check", str(vllm_patch)], - cwd=drifted_checkout, - capture_output=True, - text=True, - ) - assert rejected.returncode != 0 - def test_image_checks_native_lmms_eval_contract(project_root_path): puzzletron_root = project_root_path / "examples/puzzletron" From 94436995deac9f28aa4e4c990ea63cda10a2637f Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Sun, 6 Sep 2026 16:45:23 +0200 Subject: [PATCH 05/10] Clarify VLM evaluation profiles Signed-off-by: Johannes Rausch --- CHANGELOG.rst | 2 +- .../docs/vlm_checkpoint_evaluation.md | 43 +++++++++++++------ examples/puzzletron/docs/worker_image.md | 3 ++ .../puzzletron/evaluation/vlm/contracts.py | 5 +++ 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a00d3a89177..8360f03ab21 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,7 @@ Changelog **New Features** -- Add Puzzletron dynamic post-MIP downstream evaluation through ``lmms-eval`` with vLLM-backed checkpoint evaluation, composable VLM sample-set/backend/evaluator profiles, offline benchmark preparation, setup-wizard topology/resource prompts, non-interactive setup automation, and an opt-in Nemotron-3 Nano 30B A3B BF16 example flow. +- Add Puzzletron dynamic post-MIP downstream evaluation through ``lmms-eval`` with vLLM-backed checkpoint evaluation, setup-wizard topology/resource prompts, non-interactive setup automation, and an opt-in Nemotron-3 Nano 30B A3B BF16 example flow. - Add the ``day0-release`` agent skill (``.agents/skills/day0-release/``), a deterministic end-to-end driver that chains the PTQ → evaluation → comparison skills (the evaluation stage deploys the checkpoint itself) with an enforced gate after each stage and returns a publish decision (ACCEPT / REGRESSION / ANOMALOUS / INFEASIBLE). Ships three GPU-free, unit-tested gate scripts (``gate_ptq.py``, ``gate_run.py``, ``gate_compare.py``) that validate checkpoint coverage, evaluation-run completeness, and baseline-vs-candidate accuracy threshold. v1 reports and stops on regression; the recipe-search loop is deferred. - Add **streaming** speculative-decoding training (EAGLE3 / DFlash): the draft trains on base-model hidden states produced on the fly by a co-located ``vllm serve`` (no disk dump), moved trainer-side over NIXL RDMA, scaling to multi-node (dedicated serve replicas + DDP trainers). New launcher examples for NVFP4 Kimi-K2.5 / K2.6 on GB200/aarch64 under ``tools/launcher/examples/moonshotai/``. diff --git a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md index b07586467f0..79e37377126 100644 --- a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md +++ b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md @@ -11,16 +11,24 @@ Callers still select one profile. Keeping the components separate makes it clear that native and vLLM runs can use identical examples while changing only the backend. Use the same runnable profile for every model being compared. -The common choices are: +Choose a profile by the checkpoint and the amount of coverage needed: -- `core-3_344-examples_r1-native` for the maintained three-benchmark campaign screen; -- `core-3_344-examples_r1-vllm` for materialized heterogeneous campaign checkpoints; -- `core-3_24-examples_r1-native` and `core-3_24-examples_r1-vllm` for lifecycle smoke checks; -- `judge-free-8_690-examples_r1-native` for a broader eight-benchmark regression screen; -- `core-3_full_r1-native` and `core-3_full_r1-vllm` for paired full-dataset - teacher references on RealWorldQA, MMMU validation, and MVBench; -- `short-v1`, `short-native-v1`, `short-all-native-v1`, and `full-v1` with an - environment that matches each profile's pinned evaluator revision. +| Profile | What it evaluates | Use it for | +| --- | --- | --- | +| `core-3_24-examples_r1-native` | The same eight examples from each of RealWorldQA, MMMU validation, and one MVBench task (24 total), loaded with the native Qwen backend | A fast loading, prompt, and scoring smoke check for a Qwen checkpoint | +| `core-3_24-examples_r1-vllm` | The same 24 examples, served through vLLM | A fast smoke check for a materialized heterogeneous checkpoint | +| `core-3_344-examples_r1-native` | 64 RealWorldQA, 120 MMMU validation, and 160 MVBench examples (344 total), loaded with the native Qwen backend | Routine teacher-versus-candidate campaign comparisons | +| `core-3_344-examples_r1-vllm` | The same 344 examples, served through vLLM | Routine comparisons of materialized heterogeneous checkpoints | +| `judge-free-8_690-examples_r1-native` | 690 fixed examples across the eight benchmarks that require no external judge | A broader image-and-video regression screen | +| `core-3_full_r1-native` | All 5,665 RealWorldQA, MMMU validation, and MVBench examples, loaded with the native Qwen backend | A full-dataset reference for the pinned Qwen 3.5 0.8B teacher | +| `core-3_full_r1-vllm` | The same 5,665 examples, served through vLLM | A separate full-dataset vLLM reference for the pinned teacher | + +The four older names `short-v1`, `short-native-v1`, `short-all-native-v1`, and +`full-v1` are temporary compatibility profiles. They preserve the exact rows, +backend settings, and evaluator revisions of commands that existed before +component profiles. Keep them only for reproducing those earlier runs; use the +descriptive profiles above for new work. Removing the compatibility names will +be an explicit breaking cleanup after downstream users have migrated. For text-only IFEval and GSM8K evaluation, use the separate [text checkpoint evaluator](checkpoint_evaluation.md). @@ -33,8 +41,18 @@ Run evaluation in the default Puzzletron worker image described in the `examples/puzzletron/ci_environment.json`, includes the native Qwen 3.5 image and video backend, and preserves each task's output-token budget for vLLM. No evaluator overlay or separate VLM requirements install is needed. Do not modify -the evaluator checkout inside the image. Preflight rejects evaluator revisions -that do not match the selected profile. +the evaluator checkout inside the image. + +The evaluator revision is part of a profile's result contract: task definitions, +prompt adapters, parsers, and scoring can change between `lmms-eval` commits. +Preflight therefore rejects any revision other than the one selected by the +profile instead of producing a score that only appears comparable. + +The pinned `lmms_eval_compat_3e675904.patch` also prevents the vLLM adapter from +overwriting each task's output-token budget. The patch and its checksum are +build inputs in `ci_environment.json`; rebuild the worker image from this +repository revision before using these profiles. An image built from an older +revision does not contain the fix. ## Understand the two execution paths @@ -76,9 +94,6 @@ of 4,000 MVBench examples. The 690-example set contains those 344 plus 346 examples selected from VideoMMMU, Video-MME, LongVideoBench, MLVU, and PerceptionTest. -The historical `short-v1`, `short-native-v1`, `short-all-native-v1`, and -`full-v1` names remain available because they pin older evaluator contracts. - Use `core-3_344-examples_r1-native` for campaign comparisons. Use `judge-free-8_690-examples_r1-native` when broader image and video regression coverage is more important than matching the campaign screen. The v1 profiles require their diff --git a/examples/puzzletron/docs/worker_image.md b/examples/puzzletron/docs/worker_image.md index 1876f734933..c9cd0cfaa08 100644 --- a/examples/puzzletron/docs/worker_image.md +++ b/examples/puzzletron/docs/worker_image.md @@ -4,6 +4,9 @@ The [`Dockerfile`](../Dockerfile) contains the worker installation steps. [`ci_environment.json`](../ci_environment.json) stores the versions, source revisions, CUDA targets, and downloaded-file checksums used by those steps. The Dockerfile reads the same file for installation and its build-time checks. +Rebuild the image whenever the Dockerfile, `ci_environment.json`, or a patch +referenced by that manifest changes. Patches are applied while the image is +built, so an existing image does not acquire a later patch automatically. ## Build diff --git a/examples/puzzletron/evaluation/vlm/contracts.py b/examples/puzzletron/evaluation/vlm/contracts.py index 5e410ce93b1..a792fc5176c 100644 --- a/examples/puzzletron/evaluation/vlm/contracts.py +++ b/examples/puzzletron/evaluation/vlm/contracts.py @@ -52,6 +52,11 @@ "core-3_full_r1-vllm", ) _PROFILE_COMPONENTS = { + # short-v1, short-native-v1, short-all-native-v1, and full-v1 are temporary + # compatibility compositions. They preserve the exact rows, backend, and + # evaluator revision of names that predate component profiles. New runs + # should use descriptive profiles; remove these aliases only as an explicit + # breaking change after downstream users have migrated. "short-v1": ( "core-3_344-examples_legacy-r1", "qwen-3.5-vllm_r1", From 84832f611ef7443a5f4cfb3ddbf1d63441b0d543 Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Sun, 6 Sep 2026 17:19:00 +0200 Subject: [PATCH 06/10] Address VLM evaluation review findings Signed-off-by: Johannes Rausch --- examples/puzzletron/evaluation/vlm/model.py | 1 + .../puzzletron/evaluation/vlm/post_mip.py | 65 +++++++------------ .../puzzletron/evaluation/vlm/preflight.py | 6 +- .../vlm/preparation/benchmark_data.py | 17 +++-- examples/puzzletron/evaluation/vlm/suites.py | 4 +- .../puzzletron/evaluation/test_checkpoint.py | 5 +- .../vlm/preparation/test_benchmark_data.py | 39 +++++++++++ .../evaluation/vlm/test_contracts.py | 9 +++ .../puzzletron/evaluation/vlm/test_model.py | 9 +++ .../evaluation/vlm/test_preflight.py | 31 ++++++++- 10 files changed, 133 insertions(+), 53 deletions(-) diff --git a/examples/puzzletron/evaluation/vlm/model.py b/examples/puzzletron/evaluation/vlm/model.py index 88d11f08e14..a09167bd9ee 100644 --- a/examples/puzzletron/evaluation/vlm/model.py +++ b/examples/puzzletron/evaluation/vlm/model.py @@ -92,6 +92,7 @@ def verify_checkpoint(checkpoint: Path, *, profile: str, model_backend: str = "q if not isinstance(text_config, dict) or text_config.get("model_type") != "qwen3_5_text": raise ValueError(f"{profile} checkpoint text_config.model_type must be qwen3_5_text") if model_backend == "qwen3_5" and realized_checkpoint: + # Keep torch-dependent Puzzletron imports out of the lightweight preflight import path. from modelopt.torch.puzzletron.block_config import maybe_cast_block_configs from modelopt.torch.puzzletron.utils.vllm_adapter import ( convert_block_configs_to_per_layer_config, diff --git a/examples/puzzletron/evaluation/vlm/post_mip.py b/examples/puzzletron/evaluation/vlm/post_mip.py index e06b18ebd21..d83884cc02e 100644 --- a/examples/puzzletron/evaluation/vlm/post_mip.py +++ b/examples/puzzletron/evaluation/vlm/post_mip.py @@ -95,7 +95,7 @@ def _run_profile( "frozen 344-row campaign profile requires row_manifest and row_manifest_sha256" ) if require_manifest and evaluation_profile is not None and not expected_manifest_sha256: - raise ValueError("frozen 344-row campaign evaluation profile requires row_manifest_sha256") + raise ValueError("pinned VLM evaluation profile requires row_manifest_sha256") if row_manifest is not None and evaluation_profile is not None: raise ValueError("an embedded evaluation profile cannot be overridden by row_manifest") if expected_manifest_sha256 is not None and ( @@ -103,15 +103,13 @@ def _run_profile( or len(expected_manifest_sha256) != 64 or any(character not in "0123456789abcdef" for character in expected_manifest_sha256) ): - raise ValueError( - "frozen 344-row campaign manifest SHA256 must be 64 lowercase hex characters" - ) + raise ValueError("pinned VLM manifest SHA256 must be 64 lowercase hex characters") quick_manifest = Path(row_manifest).expanduser().absolute() if row_manifest else None if quick_manifest is not None: actual_manifest_sha256 = suites.manifest_sha256(suites.load_quick_manifest(quick_manifest)) if actual_manifest_sha256 != expected_manifest_sha256: raise ValueError( - "frozen 344-row campaign manifest SHA256 differs from the campaign identity: " + "pinned VLM manifest SHA256 differs from the profile identity: " f"{actual_manifest_sha256} != {expected_manifest_sha256}" ) args = argparse.Namespace( @@ -235,17 +233,34 @@ def evaluate_frozen_campaign_v2_checkpoint( ) -> dict[str, Any]: """Evaluate one checkpoint on the current-image frozen campaign profile.""" + return _evaluate_single_run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + evaluation_profile="core-3_344-examples_r1-native", + ) + + +def _evaluate_single_run_profile( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], + evaluation_profile: str, +) -> dict[str, Any]: + """Evaluate one checkpoint with an embedded single-run profile.""" + args, result, profile_path = _run_profile( checkpoint_path, output_root=output_root, settings=settings, suite="short", - evaluation_profile="core-3_344-examples_r1-native", + evaluation_profile=evaluation_profile, require_manifest=True, ) runs = result["runs"] if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): - raise RuntimeError("pinned VLM frozen 344-row profile returned an invalid run count") + raise RuntimeError(f"pinned VLM profile {evaluation_profile} returned an invalid run count") return { **runs[0], "profile_path": str(profile_path), @@ -261,22 +276,12 @@ def evaluate_frozen_campaign_v3_checkpoint( ) -> dict[str, Any]: """Evaluate heterogeneous materialized checkpoints with the current vLLM profile.""" - args, result, profile_path = _run_profile( + return _evaluate_single_run_profile( checkpoint_path, output_root=output_root, settings=settings, - suite="short", evaluation_profile="core-3_344-examples_r1-vllm", - require_manifest=True, ) - runs = result["runs"] - if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): - raise RuntimeError("pinned VLM frozen 344-row profile returned an invalid run count") - return { - **runs[0], - "profile_path": str(profile_path), - "checkpoint": str(args.checkpoint), - } def evaluate_reproducibility_smoke_checkpoint( @@ -287,22 +292,12 @@ def evaluate_reproducibility_smoke_checkpoint( ) -> dict[str, Any]: """Evaluate one checkpoint on the immutable 24-row lifecycle smoke.""" - args, result, profile_path = _run_profile( + return _evaluate_single_run_profile( checkpoint_path, output_root=output_root, settings=settings, - suite="short", evaluation_profile="core-3_24-examples_r1-native", - require_manifest=True, ) - runs = result["runs"] - if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): - raise RuntimeError("pinned VLM 24-row smoke returned an invalid run count") - return { - **runs[0], - "profile_path": str(profile_path), - "checkpoint": str(args.checkpoint), - } def evaluate_reproducibility_smoke_v2_checkpoint( @@ -313,22 +308,12 @@ def evaluate_reproducibility_smoke_v2_checkpoint( ) -> dict[str, Any]: """Evaluate a heterogeneous materialized checkpoint on the 24-row smoke.""" - args, result, profile_path = _run_profile( + return _evaluate_single_run_profile( checkpoint_path, output_root=output_root, settings=settings, - suite="short", evaluation_profile="core-3_24-examples_r1-vllm", - require_manifest=True, ) - runs = result["runs"] - if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): - raise RuntimeError("pinned VLM 24-row smoke returned an invalid run count") - return { - **runs[0], - "profile_path": str(profile_path), - "checkpoint": str(args.checkpoint), - } def evaluate_short_v1_checkpoint( diff --git a/examples/puzzletron/evaluation/vlm/preflight.py b/examples/puzzletron/evaluation/vlm/preflight.py index e30411fe7fa..139011dc41c 100644 --- a/examples/puzzletron/evaluation/vlm/preflight.py +++ b/examples/puzzletron/evaluation/vlm/preflight.py @@ -182,7 +182,9 @@ def _resolve_task_selection( "judge-free-8_690-examples_r1-native", }: raise ValueError( - "--profile-task is supported only for full-data and short-all-native profiles" + "--profile-task is supported only for full-v1, core-3_full_r1-native, " + "core-3_full_r1-vllm, short-all-native-v1, and " + "judge-free-8_690-examples_r1-native" ) if profile_task_shard is not None and profile_task is None: raise ValueError("--profile-task-shard requires --profile-task") @@ -280,6 +282,8 @@ def _shard_exact_row_task( if stratum.get("name") in leaves ] indices = sorted(cast("int", row["source_row_index"]) for row in rows) + if not indices: + return {**entry, "rows": rows} quantiles = { "method": "lower-order-statistic", **{ diff --git a/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py b/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py index 02275b58a07..2721f6227e2 100644 --- a/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py +++ b/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py @@ -101,7 +101,7 @@ def _task_selection(value: str) -> tuple[str, ...]: def _path_traverses_symlink(path: Path) -> bool: - return any(candidate.is_symlink() for candidate in (path, *path.parents) if candidate.exists()) + return any(candidate.is_symlink() for candidate in (path, *path.parents)) def _hub_snapshot(hf_home: Path, task: str) -> Path: @@ -529,14 +529,16 @@ def _task_lock(hf_home: Path, task: str) -> Iterator[None]: try: if not stat.S_ISREG(os.fstat(descriptor).st_mode): raise ValueError(f"benchmark preparation lock must be a regular file: {lock_path}") - with os.fdopen(descriptor, "r+") as stream: - fcntl.flock(stream.fileno(), fcntl.LOCK_EX) - yield - fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + stream = os.fdopen(descriptor, "r+") except BaseException: - with suppress(OSError): - os.close(descriptor) + os.close(descriptor) raise + with stream: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) def _cleanup_temporary_directory(target: Path) -> None: @@ -910,6 +912,7 @@ def _prepare(hf_home: Path, task: str, snapshot: Path) -> dict[str, object]: staging = Path( tempfile.mkdtemp(prefix=f".{target.name}.modelopt-staging.", dir=target.parent) ) + staging.chmod(0o755) try: _write_marker(staging, _marker_payload(task, status="in_progress")) archives = _extract(task, snapshot, staging) diff --git a/examples/puzzletron/evaluation/vlm/suites.py b/examples/puzzletron/evaluation/vlm/suites.py index 8f3c0928a22..32a887b6fa5 100644 --- a/examples/puzzletron/evaluation/vlm/suites.py +++ b/examples/puzzletron/evaluation/vlm/suites.py @@ -347,7 +347,9 @@ def manifest_task_denominators( manifest_tasks = cast("dict[str, dict[str, object]]", manifest["tasks"]) denominators = {} for task, entry in manifest_tasks.items(): - selection = cast("dict[str, object]", entry.get("selection", {})) + selection = entry.get("selection", {}) + if not isinstance(selection, dict): + raise ValueError(f"exact-row manifest task selection must be an object: {task}") population_rows = selection.get("population_rows", entry.get("population_rows")) denominators[task] = { "selected_rows": len(cast("list[object]", entry["rows"])), diff --git a/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py b/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py index 728cadf8486..43d2c553092 100644 --- a/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py +++ b/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py @@ -46,7 +46,10 @@ def test_load_runner_restores_import_state(monkeypatch): def test_verify_lmms_eval_revision_rejects_unpatched_current_vcs_install(monkeypatch): - provenance = {"vcs_info": {"commit_id": checkpoint.LMMS_EVAL_REVISION}} + provenance = { + "url": checkpoint._LMMS_EVAL_REPOSITORY, + "vcs_info": {"commit_id": checkpoint.LMMS_EVAL_REVISION}, + } monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) monkeypatch.setattr( checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py index 7a8135068b4..855b5a80301 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py @@ -18,6 +18,7 @@ import hashlib import io import json +import stat import tarfile import threading import zipfile @@ -83,6 +84,21 @@ def _emulate_atomic_exchange(first: Path, second: Path) -> bool: return True +def test_atomic_exchange_directories_when_supported(tmp_path): + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + (first / "identity").write_text("first") + (second / "identity").write_text("second") + + if not preparation._atomic_exchange_directories(first, second): + pytest.skip("atomic directory exchange is unavailable on this host") + + assert (first / "identity").read_text() == "second" + assert (second / "identity").read_text() == "first" + + @pytest.mark.parametrize( ("task", "repository", "revision", "directory"), [(task, *values) for task, values in _EXPECTED_DATASETS.items()], @@ -151,6 +167,14 @@ def test_prepare_benchmark_datasets_rejects_symlinked_hf_home(tmp_path): preparation.prepare_benchmark_datasets(alias, ("realworldqa",)) +def test_prepare_benchmark_datasets_rejects_dangling_symlinked_hf_home(tmp_path): + alias = tmp_path / "hf-home" + alias.symlink_to(tmp_path / "missing", target_is_directory=True) + + with pytest.raises(ValueError, match="must not be a symlink"): + preparation.prepare_benchmark_datasets(alias, ("realworldqa",)) + + def test_zip_preparation_is_revision_bound_idempotent_and_byte_verified(tmp_path): hf_home = tmp_path / "hf-home" snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") @@ -163,6 +187,7 @@ def test_zip_preparation_is_revision_bound_idempotent_and_byte_verified(tmp_path assert second == first assert first["status"] == "complete" assert first["files"] == 1 + assert stat.S_IMODE((hf_home / "mmvu").stat().st_mode) == 0o755 marker = json.loads((hf_home / "mmvu" / preparation._MARKER_NAME).read_text()) assert marker["revision"] == _EXPECTED_DATASETS["mmvu_val"][1] @@ -407,6 +432,20 @@ def interrupt(*_args): assert (target / "videos/sample.mp4").read_bytes() == b"video" +def test_task_lock_can_be_reacquired_after_body_failure(tmp_path): + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + + with ( + pytest.raises(RuntimeError, match="failed while locked"), + preparation._task_lock(hf_home, "mmvu_val"), + ): + raise RuntimeError("failed while locked") + + with preparation._task_lock(hf_home, "mmvu_val"): + pass + + def test_concurrent_media_preparation_is_task_locked_and_publishes_only_complete_root( monkeypatch, tmp_path ): diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py index 18f56588a76..6d66adff782 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py @@ -25,6 +25,15 @@ from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import _quick_manifest +def test_manifest_task_denominators_rejects_non_mapping_selection(): + manifest = contracts.load_profile("core-3_344-examples_r1-native").exact_rows + assert manifest is not None + manifest["tasks"]["realworldqa"]["selection"] = "invalid" + + with pytest.raises(ValueError, match="task selection must be an object"): + suites.manifest_task_denominators(manifest) + + def test_versioned_profile_contracts_pin_backends_and_fingerprints(): profiles = {name: contracts.load_profile(name) for name in contracts.PROFILE_NAMES} assert {name: contract.fingerprint for name, contract in profiles.items()} == { diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py index 890f50176fe..2e54ad40ccc 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py @@ -122,6 +122,15 @@ def test_checkpoint_contract_routes_heterogeneous_anymodel_to_vllm(tmp_path): } config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="native qwen3_5 backend cannot load"): + vlm_model.verify_checkpoint( + checkpoint_path, + profile="VLM benchmark", + model_backend="qwen3_5", + ) + + config["text_config"].pop("per_layer_config") + config_path.write_text(json.dumps(config) + "\n") with pytest.raises(ValueError, match="native qwen3_5 backend cannot load"): vlm_model.verify_checkpoint( checkpoint_path, diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py index 9676a6eb791..7f0dc9292b9 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py @@ -438,6 +438,27 @@ def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, t assert group["task"] == [f"modelopt_vlm_benchmark_mvbench_{leaf}" for leaf in expected_leaves] +def test_empty_exact_row_leaf_filter_reaches_manifest_validation(): + contract = contracts.load_profile("core-3_344-examples_r1-native") + manifest = contract.exact_rows + assert manifest is not None + entry = manifest["tasks"]["mvbench"] + manifest["tasks"] = { + "mvbench": preflight._shard_exact_row_task( + entry, + task="mvbench", + leaves=("not-selected",), + ) + } + + with pytest.raises(ValueError, match="must select at least one row"): + suites.validate_exact_rows_manifest( + manifest, + expected_revision=str(contract.manifest["lmms_eval_revision"]), + expected_tasks=("mvbench",), + ) + + @pytest.mark.parametrize( ("selection", "message"), [ @@ -528,8 +549,12 @@ def test_native_backend_validation_requires_qwen_vision_utilities(monkeypatch): def test_credential_scope_restores_inherited_values(monkeypatch): - for index, name in enumerate(checkpoint.HUGGINGFACE_CREDENTIAL_NAMES): - monkeypatch.setenv(name, f"secret-{index}") + expected = { + name: f"secret-{index}" + for index, name in enumerate(checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) + } + for name, value in expected.items(): + monkeypatch.setenv(name, value) with checkpoint.without_huggingface_credentials(): assert all(name not in os.environ for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) - assert all(name in os.environ for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) + assert {name: os.environ.get(name) for name in expected} == expected From e8e361f8ecca1c35df0142292160e1045532c1a2 Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Sun, 6 Sep 2026 22:27:43 +0200 Subject: [PATCH 07/10] Avoid rehashing unchanged VLM benchmark data Reuse recorded inventories when paths and metadata match, while retaining an explicit full-content verification path. Rename the VLM test helper so it cannot shadow the shared test utility package in spawned workers. Signed-off-by: Johannes Rausch --- .../docs/vlm_checkpoint_evaluation.md | 11 +- .../vlm/preparation/benchmark_data.py | 102 +++++++++++++++--- .../vlm/preparation/test_benchmark_data.py | 62 ++++++++++- .../evaluation/vlm/test_contracts.py | 2 +- .../evaluation/vlm/test_evaluator.py | 2 +- .../puzzletron/evaluation/vlm/test_model.py | 2 +- .../evaluation/vlm/test_post_mip.py | 2 +- .../evaluation/vlm/test_preflight.py | 2 +- .../puzzletron/evaluation/vlm/test_tasks.py | 5 +- .../vlm/{_test_utils.py => vlm_test_utils.py} | 0 .../torch/puzzletron/test_post_mip_runner.py | 2 +- 11 files changed, 163 insertions(+), 29 deletions(-) rename tests/unit/torch/puzzletron/evaluation/vlm/{_test_utils.py => vlm_test_utils.py} (100%) diff --git a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md index 79e37377126..a3fc4609fa4 100644 --- a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md +++ b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md @@ -186,11 +186,12 @@ prepare `realworldqa`, `mmmu_val`, and `mvbench`. Use `--download-only` and `--extract-only` to split transfer and extraction across jobs, or `--range-resume` for a resumable single-writer download. Run the command with `--help` to list all supported dataset task names. Preparation records exact -snapshot and media inventories. Preparation verifies the recorded snapshot and -media hashes on reuse, which can read the complete cached content. Incomplete -owned media is rebuilt from the pinned snapshot only when the host supports -atomic directory exchange. Otherwise the existing root is preserved and -preparation fails. +snapshot and media inventories. Routine reuse checks the recorded paths, sizes, +and timestamps without rereading all cached content. Add `--verify-content` to +recompute and compare every recorded SHA-256 digest. Changed or incomplete owned +media is rebuilt from the pinned snapshot only when the host supports atomic +directory exchange. Otherwise the existing root is preserved and preparation +fails. Video suites also require an installed `decord`-compatible reader. The Puzzletron requirements select the supported reader for the current platform diff --git a/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py b/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py index 2721f6227e2..4c93678d3df 100644 --- a/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py +++ b/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py @@ -212,6 +212,7 @@ def _inventory_is_current( entries: object, *, repository_cache: Path | None = None, + verify_content: bool = False, ) -> bool: if root.is_symlink() or not root.is_dir() or not isinstance(entries, list) or not entries: return False @@ -249,10 +250,16 @@ def _inventory_is_current( else: return False stat_result = inspected.stat() - if stat_result.st_size != entry.get("bytes"): + if ( + stat_result.st_size != entry.get("bytes") + or stat_result.st_mtime_ns != entry.get("mtime_ns") + or stat_result.st_ctime_ns != entry.get("ctime_ns") + ): return False expected_sha256 = entry.get("sha256") - if not isinstance(expected_sha256, str) or _sha256(inspected) != expected_sha256: + if not isinstance(expected_sha256, str) or ( + verify_content and _sha256(inspected) != expected_sha256 + ): return False observed_paths = sorted( path.relative_to(root).as_posix() @@ -284,16 +291,42 @@ def _snapshot_inventory_marker(hf_home: Path, task: str) -> Path: return snapshot.parent.parent / f"{_SNAPSHOT_MARKER_PREFIX}{DATASETS[task].revision}.json" -def _snapshot_inventory_report(hf_home: Path, task: str, snapshot: Path) -> dict[str, object]: +def _snapshot_inventory_report( + hf_home: Path, + task: str, + snapshot: Path, + *, + verify_content: bool = False, +) -> dict[str, object]: repository_cache = snapshot.parent.parent.resolve() - entries = _inventory(snapshot, repository_cache=repository_cache) marker = _snapshot_inventory_marker(hf_home, task) - report = { + try: + cached = json.loads(marker.read_text()) + except (OSError, json.JSONDecodeError): + cached = None + expected_identity = { "schema": "modelopt.vlm-benchmark-snapshot-inventory/v1", "task": task, "repository": DATASETS[task].repository, "revision": DATASETS[task].revision, "snapshot": str(snapshot), + } + if ( + isinstance(cached, dict) + and all(cached.get(key) == value for key, value in expected_identity.items()) + and _inventory_summary_is_valid(cached, cached.get("files")) + and _inventory_is_current( + snapshot, + cached.get("files"), + repository_cache=repository_cache, + verify_content=verify_content, + ) + ): + return {**cached, "manifest": str(marker)} + + entries = _inventory(snapshot, repository_cache=repository_cache) + report = { + **expected_identity, "files": entries, "file_count": len(entries), "bytes": sum(cast("int", entry["bytes"]) for entry in entries), @@ -303,7 +336,7 @@ def _snapshot_inventory_report(hf_home: Path, task: str, snapshot: Path) -> dict return {**report, "manifest": str(marker)} -def _snapshot_inventory_is_current(report: object) -> bool: +def _snapshot_inventory_is_current(report: object, *, verify_content: bool = True) -> bool: if not isinstance(report, dict): return False snapshot_value = report.get("snapshot") @@ -322,7 +355,12 @@ def _snapshot_inventory_is_current(report: object) -> bool: return ( recorded == expected and _inventory_summary_is_valid(report, entries) - and _inventory_is_current(snapshot, entries, repository_cache=repository_cache) + and _inventory_is_current( + snapshot, + entries, + repository_cache=repository_cache, + verify_content=verify_content, + ) ) @@ -597,7 +635,13 @@ def _atomic_exchange_directories(first: Path, second: Path) -> bool: raise OSError(error_number, os.strerror(error_number)) -def _media_marker_is_current(target: Path, task: str, observed: object) -> bool: +def _media_marker_is_current( + target: Path, + task: str, + observed: object, + *, + verify_content: bool = False, +) -> bool: if not isinstance(observed, dict): return False expected = _marker_payload(task, status="complete") @@ -605,11 +649,16 @@ def _media_marker_is_current(target: Path, task: str, observed: object) -> bool: return False inventory = observed.get("inventory") return _inventory_summary_is_valid(observed, inventory) and _inventory_is_current( - target, inventory + target, inventory, verify_content=verify_content ) -def _inspect_prepare_target(hf_home: Path, task: str) -> tuple[Path, dict[str, object] | None]: +def _inspect_prepare_target( + hf_home: Path, + task: str, + *, + verify_content: bool = False, +) -> tuple[Path, dict[str, object] | None]: preparation_dir = DATASETS[task].preparation_dir if preparation_dir is None: raise AssertionError(f"video dataset has no preparation directory: {task}") @@ -626,7 +675,7 @@ def _inspect_prepare_target(hf_home: Path, task: str) -> tuple[Path, dict[str, o raise FileExistsError( f"refusing to repair a media root without a readable ownership marker: {target}" ) from error - if _media_marker_is_current(target, task, observed): + if _media_marker_is_current(target, task, observed, verify_content=verify_content): return target, observed if not isinstance(observed, dict): raise FileExistsError(f"media root ownership marker must be an object: {marker}") @@ -904,9 +953,15 @@ def _extract(task: str, snapshot: Path, target: Path) -> list[dict[str, object]] raise ValueError(f"unsupported VLM benchmark data task: {task}") -def _prepare(hf_home: Path, task: str, snapshot: Path) -> dict[str, object]: +def _prepare( + hf_home: Path, + task: str, + snapshot: Path, + *, + verify_content: bool = False, +) -> dict[str, object]: with _task_lock(hf_home, task): - target, complete = _inspect_prepare_target(hf_home, task) + target, complete = _inspect_prepare_target(hf_home, task, verify_content=verify_content) if complete is not None: return complete staging = Path( @@ -962,6 +1017,11 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Use a deterministic single-writer HTTP range download that resumes across jobs", ) + parser.add_argument( + "--verify-content", + action="store_true", + help="Re-hash cached snapshot and prepared-media files instead of trusting unchanged metadata", + ) return parser @@ -971,6 +1031,7 @@ def prepare_benchmark_datasets( *, max_workers: int = 8, range_resume: bool = False, + verify_content: bool = False, expected_catalog: dict[str, object] | None = None, ) -> list[dict[str, object]]: """Download pinned benchmark snapshots and prepare media when required.""" @@ -1008,7 +1069,9 @@ def prepare_benchmark_datasets( if not snapshot.is_dir(): raise FileNotFoundError(f"pinned dataset snapshot is missing: {snapshot}") spec = DATASETS[task] - snapshot_inventory = _snapshot_inventory_report(hf_home, task, snapshot) + snapshot_inventory = _snapshot_inventory_report( + hf_home, task, snapshot, verify_content=verify_content + ) report: dict[str, object] = { "task": task, "repository": spec.repository, @@ -1020,7 +1083,10 @@ def prepare_benchmark_datasets( "status": "downloaded", } if spec.preparation_dir is not None: - report = {**report, **_prepare(hf_home, task, snapshot)} + report = { + **report, + **_prepare(hf_home, task, snapshot, verify_content=verify_content), + } reports.append(report) return reports @@ -1047,7 +1113,10 @@ def main(argv: list[str] | None = None) -> int: "status": "downloaded", } if spec.preparation_dir is not None: - report = {**report, **_prepare(hf_home, task, snapshot)} + report = { + **report, + **_prepare(hf_home, task, snapshot, verify_content=args.verify_content), + } reports.append(report) elif args.download_only: reports = [] @@ -1074,6 +1143,7 @@ def main(argv: list[str] | None = None) -> int: args.tasks, max_workers=args.max_workers, range_resume=args.range_resume, + verify_content=args.verify_content, ) print(json.dumps({"hf_home": str(hf_home), "tasks": reports}, indent=2, sort_keys=True)) return 0 diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py index 855b5a80301..6a83bab18b9 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py @@ -130,7 +130,8 @@ def download(root, task, *, max_workers): (snapshot / "dataset-info.json").write_text("{}") return snapshot - def prepare(root, task, snapshot): + def prepare(root, task, snapshot, *, verify_content=False): + assert not verify_content prepared.append(task) media_root = root / preparation.DATASETS[task].preparation_dir media_root.mkdir(parents=True) @@ -202,6 +203,29 @@ def test_zip_preparation_is_revision_bound_idempotent_and_byte_verified(tmp_path preparation._extract_zip(archive, target) +def test_prepared_media_reuses_metadata_unless_content_verification_is_requested( + tmp_path, monkeypatch +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + expected = preparation._prepare(hf_home, "mmvu_val", snapshot) + original_sha256 = preparation._sha256 + hashed = [] + + def record_hash(path): + hashed.append(path) + return original_sha256(path) + + monkeypatch.setattr(preparation, "_sha256", record_hash) + assert preparation._prepare(hf_home, "mmvu_val", snapshot) == expected + assert not hashed + + assert preparation._prepare(hf_home, "mmvu_val", snapshot, verify_content=True) == expected + assert hashed == [hf_home / "mmvu/videos/sample.mp4"] + + @pytest.mark.parametrize("damage", ["missing", "corrupt", "unexpected"]) def test_complete_media_marker_repairs_owned_root_from_pinned_snapshot( tmp_path, monkeypatch, damage @@ -317,11 +341,47 @@ def test_snapshot_inventory_rejects_partial_and_same_size_corruption(tmp_path): second.unlink() assert not preparation._snapshot_inventory_is_current(report) second.write_bytes(b"two") + report = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) assert preparation._snapshot_inventory_is_current(report) first.write_bytes(b"bad") assert not preparation._snapshot_inventory_is_current(report) +def test_snapshot_inventory_reuses_metadata_unless_content_verification_is_requested( + tmp_path, monkeypatch +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "realworldqa") + snapshot.mkdir(parents=True) + sample = snapshot / "sample.json" + sample.write_bytes(b"one") + expected = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + original_sha256 = preparation._sha256 + hashed = [] + + def record_hash(path): + hashed.append(path) + return original_sha256(path) + + monkeypatch.setattr(preparation, "_sha256", record_hash) + assert preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) == expected + assert not hashed + + assert ( + preparation._snapshot_inventory_report( + hf_home, "realworldqa", snapshot, verify_content=True + ) + == expected + ) + assert hashed == [sample] + + hashed.clear() + sample.write_bytes(b"two") + refreshed = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + assert hashed == [sample] + assert refreshed["files"][0]["sha256"] == hashlib.sha256(b"two").hexdigest() + + def test_snapshot_inventory_seals_and_validates_hub_blob_symlink(tmp_path): hf_home = tmp_path / "hf-home" snapshot = preparation._hub_snapshot(hf_home, "realworldqa") diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py index 6d66adff782..bb7d5ab7087 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py @@ -22,7 +22,7 @@ from examples.puzzletron.evaluation import checkpoint from examples.puzzletron.evaluation.vlm import contracts, profile, suites -from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import _quick_manifest +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import _quick_manifest def test_manifest_task_denominators_rejects_non_mapping_selection(): diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py index d84a32cfb14..cbc96d0eed9 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py @@ -23,7 +23,7 @@ from examples.puzzletron.evaluation import checkpoint from examples.puzzletron.evaluation.vlm import evaluator, preflight, profile, suites from examples.puzzletron.evaluation.vlm import run as evaluation -from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import ( +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import ( _use_offline_fakes, _write_checkpoint, _write_fake_mmmu_artifacts, diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py index 2e54ad40ccc..df6a6dfa80b 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py @@ -20,7 +20,7 @@ import pytest from examples.puzzletron.evaluation.vlm import model as vlm_model -from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import _write_checkpoint +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import _write_checkpoint def _homogeneous_qwen_block_configs() -> list[dict[str, object]]: diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py index 4e62a4a95c6..4de3cf79497 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py @@ -22,7 +22,7 @@ from examples.puzzletron.evaluation import checkpoint from examples.puzzletron.evaluation.vlm import post_mip, suites -from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import ( +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import ( _use_offline_fakes, _write_checkpoint, _write_lmms_tasks, diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py index 7f0dc9292b9..c6e81f74d2b 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py @@ -33,7 +33,7 @@ ) from examples.puzzletron.evaluation.vlm import run as evaluation from modelopt.torch.puzzletron.evaluation import lmms -from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import ( +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import ( _full_inputs, _use_offline_fakes, _write_checkpoint, diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py index 0a107f30b96..219e1c6204c 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py @@ -27,7 +27,10 @@ from examples.puzzletron.evaluation import checkpoint from examples.puzzletron.evaluation.vlm import contracts, preflight, profile, suites, tasks from examples.puzzletron.evaluation.vlm import run as evaluation -from tests.unit.torch.puzzletron.evaluation.vlm._test_utils import _full_inputs, _write_lmms_tasks +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import ( + _full_inputs, + _write_lmms_tasks, +) def test_short_all_native_profile_builds_grouped_and_single_selectors(tmp_path): diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/_test_utils.py b/tests/unit/torch/puzzletron/evaluation/vlm/vlm_test_utils.py similarity index 100% rename from tests/unit/torch/puzzletron/evaluation/vlm/_test_utils.py rename to tests/unit/torch/puzzletron/evaluation/vlm/vlm_test_utils.py diff --git a/tests/unit/torch/puzzletron/test_post_mip_runner.py b/tests/unit/torch/puzzletron/test_post_mip_runner.py index 4ccb6cfde4a..13c5ea39d4a 100644 --- a/tests/unit/torch/puzzletron/test_post_mip_runner.py +++ b/tests/unit/torch/puzzletron/test_post_mip_runner.py @@ -527,7 +527,7 @@ def fake_evaluate(args, *, settings_overrides, preflight_callback): assert captured["settings"] == {} assert result["checkpoint"] == str(checkpoint) - with pytest.raises(ValueError, match="differs from the campaign identity"): + with pytest.raises(ValueError, match="differs from the profile identity"): post_mip.evaluate_short_v1_checkpoint( checkpoint, output_root=tmp_path / "mismatch", From da23db3c190c8bdfe78cb9be1af6269e123a368f Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Sun, 6 Sep 2026 22:53:52 +0200 Subject: [PATCH 08/10] Fix post-MIP topology test configuration Signed-off-by: Johannes Rausch --- .../unit/torch/puzzletron/test_orchestration_compiler.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit/torch/puzzletron/test_orchestration_compiler.py b/tests/unit/torch/puzzletron/test_orchestration_compiler.py index 5adab0934da..3008a023cff 100644 --- a/tests/unit/torch/puzzletron/test_orchestration_compiler.py +++ b/tests/unit/torch/puzzletron/test_orchestration_compiler.py @@ -571,10 +571,17 @@ def test_post_mip_compiler_topologically_orders_serialized_nodes() -> None: "best": { "type": "filter", "input": "final_eval", + "mode": "top_k", "metric": "final_eval.kl_div", + "top_k": 1, }, "final_eval": {"type": "evaluation", "input": "initial"}, - "initial": {"type": "filter", "metric": "mip.score"}, + "initial": { + "type": "filter", + "mode": "top_k", + "metric": "mip.score", + "top_k": 1, + }, }, } } From bd9b5aa86f255f06e353014a3fe99365a507e30d Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Mon, 7 Sep 2026 01:02:03 +0200 Subject: [PATCH 09/10] Deprecate legacy VLM profile names Signed-off-by: Johannes Rausch --- .../docs/vlm_checkpoint_evaluation.md | 8 ++-- examples/puzzletron/evaluation/checkpoint.py | 34 ++++------------ .../puzzletron/evaluation/vlm/contracts.py | 39 +++++++++++++++---- .../puzzletron/evaluation/vlm/preflight.py | 2 + .../evaluators/lmms-eval-legacy_r1.json | 5 --- .../lmms-eval-qwen-3.5-native_r1.json | 5 --- .../evaluation/vlm/profiles/full-v1.json | 2 +- .../vlm/profiles/short-all-native-v1.json | 2 +- .../vlm/profiles/short-native-v1.json | 2 +- .../evaluation/vlm/profiles/short-v1.json | 2 +- .../puzzletron/evaluation/test_checkpoint.py | 25 ++++-------- .../evaluation/vlm/test_contracts.py | 17 ++++++-- .../evaluation/vlm/test_preflight.py | 7 ++-- 13 files changed, 74 insertions(+), 76 deletions(-) delete mode 100644 examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-legacy_r1.json delete mode 100644 examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-qwen-3.5-native_r1.json diff --git a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md index a3fc4609fa4..be777348ae7 100644 --- a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md +++ b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md @@ -25,10 +25,10 @@ Choose a profile by the checkpoint and the amount of coverage needed: The four older names `short-v1`, `short-native-v1`, `short-all-native-v1`, and `full-v1` are temporary compatibility profiles. They preserve the exact rows, -backend settings, and evaluator revisions of commands that existed before -component profiles. Keep them only for reproducing those earlier runs; use the -descriptive profiles above for new work. Removing the compatibility names will -be an explicit breaking cleanup after downstream users have migrated. +backend settings, and maintained evaluator revision of commands that existed +before component profiles. They emit a deprecation warning and will be removed +after downstream callers migrate. Keep them only for reproducing earlier runs; +use the descriptive profiles above for new work. For text-only IFEval and GSM8K evaluation, use the separate [text checkpoint evaluator](checkpoint_evaluation.md). diff --git a/examples/puzzletron/evaluation/checkpoint.py b/examples/puzzletron/evaluation/checkpoint.py index 516a78263ee..9a561a4b38c 100644 --- a/examples/puzzletron/evaluation/checkpoint.py +++ b/examples/puzzletron/evaluation/checkpoint.py @@ -37,8 +37,6 @@ __all__ = [ "DEFAULT_PREFLIGHT_TIMEOUT_SECONDS", "HUGGINGFACE_CREDENTIAL_NAMES", - "LMMS_EVAL_LEGACY_REVISION", - "LMMS_EVAL_QWEN35_NATIVE_REVISION", "LMMS_EVAL_REVISION", "credential_free_environment", "lmms_eval_disabled_judge_environment", @@ -56,13 +54,6 @@ ) LMMS_EVAL_SOURCE = CI_ENVIRONMENT["lmms_eval"] LMMS_EVAL_REVISION = LMMS_EVAL_SOURCE["commit"] -LMMS_EVAL_LEGACY_REVISION = "15c32bfec165df13c269ddd3cda03b2ed9137825" -LMMS_EVAL_QWEN35_NATIVE_REVISION = "88b23e2bfa16a1edbc16e9e238ed82130b3a4f56" -_LMMS_EVAL_REPOSITORY = LMMS_EVAL_SOURCE["repository"] -_LMMS_EVAL_HISTORICAL_SOURCES = { - revision: {"repository": _LMMS_EVAL_REPOSITORY, "commit": revision} - for revision in (LMMS_EVAL_LEGACY_REVISION, LMMS_EVAL_QWEN35_NATIVE_REVISION) -} DEFAULT_PREFLIGHT_TIMEOUT_SECONDS = 15 * 60.0 HUGGINGFACE_CREDENTIAL_NAMES = ( "HF_TOKEN", @@ -160,8 +151,9 @@ def positive_float(value: str) -> float: def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> str: """Return the imported evaluator revision after matching its source and patch pin.""" - source = _lmms_eval_source(expected_revision) - revision = _imported_lmms_eval_revision(source) + if expected_revision != LMMS_EVAL_REVISION: + raise RuntimeError(f"unsupported lmms-eval revision: {expected_revision}") + revision = _imported_lmms_eval_revision() if revision is not None: if revision != expected_revision: raise RuntimeError( @@ -179,7 +171,7 @@ def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> st raise RuntimeError("installed lmms-eval revision provenance is unavailable") from error if isinstance(provenance, dict): try: - ci_environment.verify_installed_vcs_source("lmms-eval", source) + ci_environment.verify_installed_vcs_source("lmms-eval", LMMS_EVAL_SOURCE) except (OSError, subprocess.SubprocessError) as error: raise RuntimeError("installed lmms-eval source provenance is unavailable") from error revision = expected_revision @@ -193,17 +185,7 @@ def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> st return revision -def _lmms_eval_source(expected_revision: str) -> dict[str, object]: - """Resolve a current or immutable historical evaluator source contract.""" - if expected_revision == LMMS_EVAL_REVISION: - return LMMS_EVAL_SOURCE - try: - return _LMMS_EVAL_HISTORICAL_SOURCES[expected_revision] - except KeyError as error: - raise RuntimeError(f"unsupported lmms-eval revision: {expected_revision}") from error - - -def _imported_lmms_eval_revision(source: dict[str, object]) -> str | None: +def _imported_lmms_eval_revision() -> str | None: """Verify a source checkout imported directly through ``PYTHONPATH``.""" spec = importlib.util.find_spec("lmms_eval") locations = tuple(spec.submodule_search_locations or ()) if spec is not None else () @@ -212,13 +194,13 @@ def _imported_lmms_eval_revision(source: dict[str, object]) -> str | None: checkout = Path(locations[0]).resolve().parent if not (checkout / ".git").exists(): return None - return _verified_checkout_revision(checkout, source) + return _verified_checkout_revision(checkout) -def _verified_checkout_revision(checkout: Path, source: dict[str, object]) -> str | None: +def _verified_checkout_revision(checkout: Path) -> str | None: """Return the revision of one source-and-patch verified Git checkout.""" try: - return ci_environment.verify_vcs_checkout(checkout, "lmms-eval", source) + return ci_environment.verify_vcs_checkout(checkout, "lmms-eval", LMMS_EVAL_SOURCE) except (OSError, subprocess.SubprocessError): return None diff --git a/examples/puzzletron/evaluation/vlm/contracts.py b/examples/puzzletron/evaluation/vlm/contracts.py index a792fc5176c..03ebedcd369 100644 --- a/examples/puzzletron/evaluation/vlm/contracts.py +++ b/examples/puzzletron/evaluation/vlm/contracts.py @@ -19,6 +19,7 @@ import hashlib import json +import warnings from dataclasses import dataclass from pathlib import Path from typing import cast @@ -31,6 +32,7 @@ "SHORT_PROFILE_NAMES", "ProfileContract", "load_profile", + "warn_deprecated_profile", ] _PROFILE_SCHEMA = "modelopt.vlm-evaluation-profile/v2" @@ -55,17 +57,17 @@ # short-v1, short-native-v1, short-all-native-v1, and full-v1 are temporary # compatibility compositions. They preserve the exact rows, backend, and # evaluator revision of names that predate component profiles. New runs - # should use descriptive profiles; remove these aliases only as an explicit - # breaking change after downstream users have migrated. + # should use descriptive profiles. Remove these entries and their legacy + # sample sets after downstream callers have migrated. "short-v1": ( "core-3_344-examples_legacy-r1", "qwen-3.5-vllm_r1", - "lmms-eval-legacy_r1", + "lmms-eval-modelopt_r1", ), "short-native-v1": ( "core-3_344-examples_legacy-r1", "qwen-3.5-native_r1", - "lmms-eval-qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", ), "core-3_344-examples_r1-native": ( "core-3_344-examples_r1", @@ -90,7 +92,7 @@ "short-all-native-v1": ( "judge-free-8_690-examples_legacy-r1", "qwen-3.5-native_r1", - "lmms-eval-qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", ), "judge-free-8_690-examples_r1-native": ( "judge-free-8_690-examples_r1", @@ -100,7 +102,7 @@ "full-v1": ( "judge-free-8_full_legacy-r1", "qwen-3.5-vllm_r1", - "lmms-eval-legacy_r1", + "lmms-eval-modelopt_r1", ), "core-3_full_r1-native": ( "core-3_full_r1", @@ -113,6 +115,12 @@ "lmms-eval-modelopt_r1", ), } +_DEPRECATED_PROFILE_REPLACEMENTS = { + "short-v1": "core-3_344-examples_r1-vllm", + "short-native-v1": "core-3_344-examples_r1-native", + "short-all-native-v1": "judge-free-8_690-examples_r1-native", + "full-v1": None, +} _CORE_3_TASKS = ("realworldqa", "mmmu_val", "mvbench") _JUDGE_FREE_8_TASKS = ( "realworldqa", @@ -175,8 +183,6 @@ }, } _EVALUATOR_REVISIONS = { - "lmms-eval-legacy_r1": checkpoint.LMMS_EVAL_LEGACY_REVISION, - "lmms-eval-qwen-3.5-native_r1": checkpoint.LMMS_EVAL_QWEN35_NATIVE_REVISION, "lmms-eval-modelopt_r1": checkpoint.LMMS_EVAL_REVISION, } _SAMPLE_SET_MODELS = { @@ -281,6 +287,23 @@ def exact_rows(self) -> dict[str, object] | None: return exact_rows +def warn_deprecated_profile(name: str) -> None: + """Warn when a temporary compatibility profile is selected for execution.""" + if name in _DEPRECATED_PROFILE_REPLACEMENTS: + replacement = _DEPRECATED_PROFILE_REPLACEMENTS[name] + guidance = ( + f"use {replacement} for new runs" + if replacement is not None + else "choose a descriptive profile for new runs" + ) + warnings.warn( + f"{name} is a deprecated compatibility profile; {guidance}. " + "It will be removed after downstream callers migrate.", + FutureWarning, + stacklevel=2, + ) + + def load_profile(name: str) -> ProfileContract: """Load a named profile after validating every executable pin.""" if name not in PROFILE_NAMES: diff --git a/examples/puzzletron/evaluation/vlm/preflight.py b/examples/puzzletron/evaluation/vlm/preflight.py index 139011dc41c..affa7271896 100644 --- a/examples/puzzletron/evaluation/vlm/preflight.py +++ b/examples/puzzletron/evaluation/vlm/preflight.py @@ -90,6 +90,8 @@ def _output_budget_contract( def prepare(args: argparse.Namespace) -> PreparedSuite: """Resolve and validate everything needed before model loading starts.""" profile_name = getattr(args, "profile", None) + if profile_name is not None: + contracts.warn_deprecated_profile(profile_name) profile_contract = contracts.load_profile(profile_name) if profile_name is not None else None suite, source_tasks, profile_task_leaves = _resolve_task_selection(args, profile_contract) backend = _backend_policy(profile_contract) diff --git a/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-legacy_r1.json b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-legacy_r1.json deleted file mode 100644 index 19890381d7b..00000000000 --- a/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-legacy_r1.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "schema": "modelopt.vlm-evaluator-profile/v1", - "name": "lmms-eval-legacy_r1", - "lmms_eval_revision": "15c32bfec165df13c269ddd3cda03b2ed9137825" -} diff --git a/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-qwen-3.5-native_r1.json b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-qwen-3.5-native_r1.json deleted file mode 100644 index 3b59b917cdb..00000000000 --- a/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-qwen-3.5-native_r1.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "schema": "modelopt.vlm-evaluator-profile/v1", - "name": "lmms-eval-qwen-3.5-native_r1", - "lmms_eval_revision": "88b23e2bfa16a1edbc16e9e238ed82130b3a4f56" -} diff --git a/examples/puzzletron/evaluation/vlm/profiles/full-v1.json b/examples/puzzletron/evaluation/vlm/profiles/full-v1.json index 443debb0fa5..0497f432f83 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/full-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/full-v1.json @@ -3,5 +3,5 @@ "name": "full-v1", "sample_set": "judge-free-8_full_legacy-r1", "backend_profile": "qwen-3.5-vllm_r1", - "evaluator_profile": "lmms-eval-legacy_r1" + "evaluator_profile": "lmms-eval-modelopt_r1" } diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json index c31d91fed43..dc9467f21ae 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json @@ -3,5 +3,5 @@ "name": "short-all-native-v1", "sample_set": "judge-free-8_690-examples_legacy-r1", "backend_profile": "qwen-3.5-native_r1", - "evaluator_profile": "lmms-eval-qwen-3.5-native_r1" + "evaluator_profile": "lmms-eval-modelopt_r1" } diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json index 330a1369698..41f6b548ff3 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json @@ -3,5 +3,5 @@ "name": "short-native-v1", "sample_set": "core-3_344-examples_legacy-r1", "backend_profile": "qwen-3.5-native_r1", - "evaluator_profile": "lmms-eval-qwen-3.5-native_r1" + "evaluator_profile": "lmms-eval-modelopt_r1" } diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-v1.json index 9b7570fb5fa..ca10add87ca 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-v1.json @@ -3,5 +3,5 @@ "name": "short-v1", "sample_set": "core-3_344-examples_legacy-r1", "backend_profile": "qwen-3.5-vllm_r1", - "evaluator_profile": "lmms-eval-legacy_r1" + "evaluator_profile": "lmms-eval-modelopt_r1" } diff --git a/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py b/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py index 43d2c553092..af129c0d18f 100644 --- a/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py +++ b/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py @@ -47,10 +47,10 @@ def test_load_runner_restores_import_state(monkeypatch): def test_verify_lmms_eval_revision_rejects_unpatched_current_vcs_install(monkeypatch): provenance = { - "url": checkpoint._LMMS_EVAL_REPOSITORY, + "url": checkpoint.LMMS_EVAL_SOURCE["repository"], "vcs_info": {"commit_id": checkpoint.LMMS_EVAL_REVISION}, } - monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) + monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda: None) monkeypatch.setattr( checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) ) @@ -59,23 +59,14 @@ def test_verify_lmms_eval_revision_rejects_unpatched_current_vcs_install(monkeyp checkpoint.verify_lmms_eval_revision() -@pytest.mark.parametrize( - "expected", - [checkpoint.LMMS_EVAL_LEGACY_REVISION, checkpoint.LMMS_EVAL_QWEN35_NATIVE_REVISION], -) -def test_verify_lmms_eval_revision_accepts_clean_historical_vcs_install(monkeypatch, expected): - provenance = {"url": checkpoint._LMMS_EVAL_REPOSITORY, "vcs_info": {"commit_id": expected}} - monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) - monkeypatch.setattr( - checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) - ) - - assert checkpoint.verify_lmms_eval_revision(expected) == expected +def test_verify_lmms_eval_revision_rejects_unsupported_revision(): + with pytest.raises(RuntimeError, match="unsupported lmms-eval revision"): + checkpoint.verify_lmms_eval_revision("different") def test_verify_lmms_eval_revision_accepts_clean_pinned_editable_checkout(monkeypatch, tmp_path): provenance = {"dir_info": {"editable": True}, "url": tmp_path.as_uri()} - monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) + monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda: None) monkeypatch.setattr( checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) ) @@ -116,7 +107,7 @@ def distribution(_name): def test_verify_lmms_eval_revision_rejects_dirty_editable_checkout(monkeypatch, tmp_path): provenance = {"dir_info": {"editable": True}, "url": tmp_path.as_uri()} - monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) + monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda: None) monkeypatch.setattr( checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) ) @@ -140,7 +131,7 @@ def test_verify_lmms_eval_revision_rejects_dirty_editable_checkout(monkeypatch, ], ) def test_verify_lmms_eval_revision_rejects_unverifiable_editable_install(monkeypatch, provenance): - monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda _source: None) + monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda: None) monkeypatch.setattr( checkpoint.importlib.metadata, "distribution", lambda _name: _distribution(provenance) ) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py index bb7d5ab7087..3294dc38cf2 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py @@ -37,15 +37,15 @@ def test_manifest_task_denominators_rejects_non_mapping_selection(): def test_versioned_profile_contracts_pin_backends_and_fingerprints(): profiles = {name: contracts.load_profile(name) for name in contracts.PROFILE_NAMES} assert {name: contract.fingerprint for name, contract in profiles.items()} == { - "short-v1": "8286a094c3cfb5c2608a6e1469d6525bb1c4be11a7ab8789ed97dd249d7bea71", - "short-native-v1": "b15054c251af54a5298233b1c01a3babf76c4281d055469fd26b76129c34f258", + "short-v1": "f111b49238fd5a1843a22afb6ea7da02c2562e198e6401217b7d41d69b3df49f", + "short-native-v1": "578358d3644fe5d71ee26f8c5a2e8a03f2fad662e40ae8e4eefe19e316185aee", "core-3_344-examples_r1-native": "2017656d093de7d95d25c7e34241b1d708150157f0c4e6a0bf6bd48649c2191a", "core-3_344-examples_r1-vllm": "859908fdb32b6bcaddb5400cd4430f4c9026264db38c7a8b56a98f42109c1f78", "core-3_24-examples_r1-native": "0e51e27d57e27f0c5e4943d077308766387fa739b2d1c413b7b951327358cefc", "core-3_24-examples_r1-vllm": "9c68168f05003e695258dc119351610b4e98bafb3e4f3e773c4e64ce5d17835a", - "short-all-native-v1": "9d7334371316a2a7774ee7e520ce0fc57e3c42ecfd7749ccd217e02ab59b6ee3", + "short-all-native-v1": "91a0ec543e9ddf055502ff42fbe98e15125d1174d9133f8e60317fdb1b7b77e0", "judge-free-8_690-examples_r1-native": "78457702288ba2d9d7b903366f7030302936377690b0ec37a0704e3eda8fd851", - "full-v1": "680483a7e2eceeab82a5e0b2767cc751f1951f58ffedc0aa190481d6ec978307", + "full-v1": "544a5c5cd5d91248ccf2d2fbe92df5f99f7c4e93a8741a2de8e6bba91aaea5a4", "core-3_full_r1-native": "976efbd056fecb686e64b912ed50251b1c16e5efe1d3ae3d179f208cc587c0a7", "core-3_full_r1-vllm": "40fd44fbb4812bd927d3f82e33d6eddec5b4641736c65c48250b8ea77acaec81", } @@ -73,6 +73,15 @@ def test_versioned_profile_contracts_pin_backends_and_fingerprints(): assert profiles["full-v1"].exact_rows is None +def test_compatibility_profiles_warn_until_downstream_callers_migrate(): + for name in ("short-v1", "short-native-v1", "short-all-native-v1", "full-v1"): + with pytest.warns(FutureWarning, match=rf"{name} is a deprecated compatibility profile"): + contracts.warn_deprecated_profile(name) + assert contracts.load_profile(name).manifest["lmms_eval_revision"] == ( + checkpoint.LMMS_EVAL_REVISION + ) + + @pytest.mark.parametrize( ("name", "backend"), [ diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py index c6e81f74d2b..6e2d37e91a0 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py @@ -275,7 +275,7 @@ def test_core3_full_teacher_profile_rejects_settings_override(monkeypatch, tmp_p evaluator.evaluate(args, settings_overrides={"model": "vllm"}) -def test_historical_short_profile_preserves_vllm_backend(monkeypatch, tmp_path): +def test_deprecated_short_profile_preserves_vllm_backend(monkeypatch, tmp_path): model, hf_home = _full_inputs(monkeypatch, tmp_path) args = evaluation._build_parser().parse_args( [ @@ -290,7 +290,8 @@ def test_historical_short_profile_preserves_vllm_backend(monkeypatch, tmp_path): ] ) - prepared = preflight.prepare(args) + with pytest.warns(FutureWarning, match="short-v1 is a deprecated compatibility profile"): + prepared = preflight.prepare(args) (tmp_path / "tasks").mkdir() settings = preflight.settings( args, @@ -299,7 +300,7 @@ def test_historical_short_profile_preserves_vllm_backend(monkeypatch, tmp_path): prepared=prepared, ) - assert prepared.report["lmms_eval_revision"] == checkpoint.LMMS_EVAL_LEGACY_REVISION + assert prepared.report["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION assert settings["model"] == "vllm" assert settings["checkpoint_arg"] == "model" assert settings["reasoning_parser"] == "qwen3" From 1974f2bf90cb5e3c3aa1afa8f3c73160a26f20cf Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Tue, 8 Sep 2026 00:39:55 +0200 Subject: [PATCH 10/10] Prune redundant VLM test coverage Signed-off-by: Johannes Rausch --- .../evaluation/vlm/test_evaluator.py | 5 +- .../puzzletron/evaluation/vlm/test_model.py | 2 +- .../evaluation/vlm/test_preflight.py | 78 ++++--------------- .../puzzletron/evaluation/vlm/test_run.py | 28 ------- .../puzzletron/evaluation/vlm/test_tasks.py | 2 +- .../torch/puzzletron/test_post_mip_runner.py | 24 +++--- 6 files changed, 31 insertions(+), 108 deletions(-) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py index cbc96d0eed9..bb989efe5db 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py @@ -177,8 +177,8 @@ def fake_runner(checkpoint_path, *, output_root, settings): assert budget["resolution"] == "max(task_max_new_tokens, model_max_new_tokens_floor=1)" -@pytest.mark.parametrize("suite", ["short", suites.TASK_PREFIX100_REPEAT2_SUITE]) -def test_repeated_profile_resumes_completed_repetitions(monkeypatch, tmp_path, suite): +def test_repeated_profile_resumes_completed_repetitions(monkeypatch, tmp_path): + suite = suites.TASK_PREFIX100_REPEAT2_SUITE model = _write_checkpoint(tmp_path) source_tasks = ("realworldqa", "mmmu_val") lmms_root = _write_lmms_tasks(tmp_path, source_tasks) @@ -327,7 +327,6 @@ def test_realworldqa_mmmu_prefix100_policy_is_explicit_and_repeated(): "do_sample": False, } assert suites.execution_policy("full", timeout_seconds=None)["limit"] is None - assert suites.execution_policy("full-v1", timeout_seconds=None)["limit"] is None @pytest.mark.parametrize("alias", suites.DEPRECATED_SUITE_ALIASES) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py index df6a6dfa80b..404c2f06a68 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py @@ -166,7 +166,7 @@ def test_checkpoint_contract_accepts_other_positive_qwen35_geometry(tmp_path): vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") -@pytest.mark.parametrize("processor_content", [None, "", "[]\n", "{\n", b"\xff"]) +@pytest.mark.parametrize("processor_content", [None, "[]\n", b"\xff"]) def test_checkpoint_contract_requires_valid_local_processor_assets(tmp_path, processor_content): checkpoint_path = _write_checkpoint(tmp_path) processor_path = checkpoint_path / "preprocessor_config.json" diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py index 6e2d37e91a0..e44d4dba73f 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py @@ -15,7 +15,6 @@ """Tests for VLM profile preflight and runtime settings.""" -import hashlib import json import os from pathlib import Path @@ -45,7 +44,7 @@ def test_versioned_profile_preflight_reports_immutable_contract(monkeypatch, tmp_path): model, hf_home = _full_inputs(monkeypatch, tmp_path) - name = "short-v1" + name = "core-3_344-examples_r1-native" args = evaluation._build_parser().parse_args( [ "--checkpoint", @@ -321,7 +320,7 @@ def test_versioned_profile_rejects_runtime_override(monkeypatch, tmp_path, optio "--output-dir", str(tmp_path / "results"), "--profile", - "short-v1", + "core-3_344-examples_r1-native", option, value, "--hf-home", @@ -333,36 +332,6 @@ def test_versioned_profile_rejects_runtime_override(monkeypatch, tmp_path, optio preflight.prepare(args) -def test_all_row_profile_task_preserves_contract_identity(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - profile_name = "full-v1" - task = "realworldqa" - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - profile_name, - "--profile-task", - task, - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - - assert prepared.source_tasks == (task,) - assert prepared.report["quick_selected_rows"] is None - assert prepared.report["quick_row_identities"] is None - assert prepared.report["quick_task_denominators"] is None - assert ( - prepared.report["profile_fingerprint"] == contracts.load_profile(profile_name).fingerprint - ) - - def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, tmp_path): model, hf_home = _full_inputs(monkeypatch, tmp_path) args = evaluation._build_parser().parse_args( @@ -392,41 +361,12 @@ def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, t assert {row["leaf_task"] for row in manifest_rows} == { f"mvbench_{leaf}" for leaf in expected_leaves } - manifest_selection = prepared.quick_manifest["tasks"]["mvbench"]["selection"] - assert manifest_selection["population_rows"] == 600 - assert manifest_selection["selected_rows"] == 24 - assert [stratum["name"] for stratum in manifest_selection["strata"]] == list(expected_leaves) - assert manifest_selection["selected_index_quantiles"] == { - "method": "lower-order-statistic", - "p0": 12, - "p25": 37, - "p50": 87, - "p75": 137, - "p100": 187, - } - assert ( - manifest_selection["selected_row_identities_sha256"] - == hashlib.sha256( - json.dumps(manifest_rows, separators=(",", ":"), sort_keys=True).encode() - ).hexdigest() - ) - assert prepared.report["quick_selected_rows"] == 24 - assert prepared.report["quick_row_identities"] == suites.manifest_row_identities( - prepared.quick_manifest - ) assert prepared.report["quick_task_denominators"] == { "mvbench": {"population_rows": 600, "selected_rows": 24} } assert ( evaluator._expected_task_populations(prepared, ("modelopt_vlm_benchmark_mvbench",)) is None ) - assert prepared.report["quick_manifest_sha256"] == suites.manifest_sha256( - prepared.quick_manifest - ) - assert ( - prepared.report["profile_fingerprint"] - == contracts.load_profile("judge-free-8_690-examples_r1-native").fingerprint - ) tasks_root, _ = tasks.prepare( tmp_path / "results", suite=prepared.suite, @@ -464,12 +404,20 @@ def test_empty_exact_row_leaf_filter_reaches_manifest_validation(): ("selection", "message"), [ (("--suite", "short", "--profile-task", "realworldqa"), "requires"), - (("--profile", "short-v1", "--profile-task", "realworldqa"), "supported only"), - (("--profile", "full-v1", "--profile-task-shard", "0/8"), "requires"), ( ( "--profile", - "full-v1", + "core-3_344-examples_r1-native", + "--profile-task", + "realworldqa", + ), + "supported only", + ), + (("--profile", "core-3_full_r1-native", "--profile-task-shard", "0/8"), "requires"), + ( + ( + "--profile", + "core-3_full_r1-native", "--profile-task", "realworldqa", "--profile-task-shard", diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py index 0165acb5210..ba43a7c1e2f 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py @@ -15,13 +15,10 @@ """Tests for the VLM evaluation command-line entry point.""" -import hashlib -import json import subprocess import sys from pathlib import Path -from examples.puzzletron.evaluation import checkpoint from examples.puzzletron.evaluation.vlm import run as evaluation @@ -45,26 +42,6 @@ def test_direct_launcher_does_not_shadow_standard_library_profile(): ) -def test_requirements_pin_matches_runtime_lmms_eval_revision(): - requirements = (checkpoint.REPOSITORY_ROOT / "examples/puzzletron/requirements.txt").read_text() - assert "lmms-eval.git" not in requirements - assert 'eva-decord==0.6.1; platform_system == "Linux"' in requirements.splitlines() - assert "wandb==0.29.0" in requirements.splitlines() - environment = json.loads( - (checkpoint.REPOSITORY_ROOT / "examples/puzzletron/ci_environment.json").read_text() - ) - assert environment["lmms_eval"]["commit"] == checkpoint.LMMS_EVAL_REVISION - patch = ( - checkpoint.REPOSITORY_ROOT - / "examples/puzzletron/patches" - / environment["lmms_eval"]["compatibility_patch"] - ) - assert ( - hashlib.sha256(patch.read_bytes()).hexdigest() - == environment["lmms_eval"]["compatibility_patch_sha256"] - ) - - def test_vlm_parser_exposes_only_suite_owned_sample_limits(): help_text = evaluation._build_parser().format_help() assert "--limit" not in help_text @@ -75,8 +52,3 @@ def test_vlm_parser_exposes_only_suite_owned_sample_limits(): def test_vlm_parser_defaults_to_short_suite(): assert evaluation._build_parser().get_default("suite") == "short" - - -def test_huggingface_dependency_supports_range_metadata_api(): - pyproject = (checkpoint.REPOSITORY_ROOT / "pyproject.toml").read_text() - assert '"huggingface_hub>=0.30.0",' in pyproject diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py index 219e1c6204c..871dac1f4a2 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py @@ -33,7 +33,7 @@ ) -def test_short_all_native_profile_builds_grouped_and_single_selectors(tmp_path): +def test_judge_free_profile_builds_grouped_and_single_selectors(tmp_path): contract = contracts.load_profile("judge-free-8_690-examples_r1-native") exact_rows = contract.exact_rows assert exact_rows is not None diff --git a/tests/unit/torch/puzzletron/test_post_mip_runner.py b/tests/unit/torch/puzzletron/test_post_mip_runner.py index 13c5ea39d4a..9dac6795131 100644 --- a/tests/unit/torch/puzzletron/test_post_mip_runner.py +++ b/tests/unit/torch/puzzletron/test_post_mip_runner.py @@ -512,22 +512,26 @@ def fake_evaluate(args, *, settings_overrides, preflight_callback): return {"runs": [{"metrics": {"accuracy": 0.5}, "result_path": "result.json"}]} monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) - result = post_mip.evaluate_short_v1_checkpoint( - checkpoint, - output_root=tmp_path / "output", - settings={ - "row_manifest": str(manifest), - "row_manifest_sha256": "a" * 64, - "batch_size": 1, - }, - ) + with pytest.warns(DeprecationWarning, match="qwen35_vlm_short_v1 is deprecated"): + result = post_mip.evaluate_short_v1_checkpoint( + checkpoint, + output_root=tmp_path / "output", + settings={ + "row_manifest": str(manifest), + "row_manifest_sha256": "a" * 64, + "batch_size": 1, + }, + ) assert captured["args"].suite == "quick" assert captured["args"].quick_manifest == manifest assert captured["settings"] == {} assert result["checkpoint"] == str(checkpoint) - with pytest.raises(ValueError, match="differs from the profile identity"): + with ( + pytest.warns(DeprecationWarning, match="qwen35_vlm_short_v1 is deprecated"), + pytest.raises(ValueError, match="differs from the profile identity"), + ): post_mip.evaluate_short_v1_checkpoint( checkpoint, output_root=tmp_path / "mismatch",