diff --git a/examples/puzzletron/Dockerfile b/examples/puzzletron/Dockerfile index 72ffff063d8..132885b6277 100644 --- a/examples/puzzletron/Dockerfile +++ b/examples/puzzletron/Dockerfile @@ -72,8 +72,9 @@ RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \ # Install the native Qwen 3.5 evaluator in the shared worker environment. The # tracked patch removes lmms-eval's unused private WandB printer integration, -# reconciles its WandB range with AutoModel, and replaces the legacy LaTeX -# parser whose ANTLR pin conflicts with Puzzletron's Hydra stack. +# reconciles its WandB range with AutoModel, replaces the legacy LaTeX parser +# whose ANTLR pin conflicts with Puzzletron's Hydra stack, and preserves the +# vLLM adapter's task-specific sampling parameters. RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \ lmms_patch="$(pin lmms_eval.compatibility_patch)" && \ echo "$(pin lmms_eval.compatibility_patch_sha256) /opt/puzzletron/patches/${lmms_patch}" | \ diff --git a/examples/puzzletron/ci_environment.json b/examples/puzzletron/ci_environment.json index 0e0ec1a56e8..4eace49422d 100644 --- a/examples/puzzletron/ci_environment.json +++ b/examples/puzzletron/ci_environment.json @@ -35,16 +35,17 @@ "base_version": "0.7.2", "repository": "https://github.com/EvolvingLMMs-Lab/lmms-eval.git", "commit": "3e675904f8cba6793de12b91979b04d91754bdf3", - "compatibility_patch": "lmms_eval_wandb_0_29.patch", + "compatibility_patch": "lmms_eval_compat_3e675904.patch", "compatibility_patch_context_lines": 0, "compatibility_patch_files": [ "lmms_eval/loggers/wandb_logger.py", + "lmms_eval/models/simple/vllm.py", "lmms_eval/tasks/emma/utils.py", "lmms_eval/tasks/mathvision/eval_utils.py", "lmms_eval/tasks/stare/utils.py", "pyproject.toml" ], - "compatibility_patch_sha256": "960ef2f0d5cea85ccb67e22ab28356ae935f8109876b2f65f88693795549de08", + "compatibility_patch_sha256": "c8cbb9614cd80c2a58c08e669fdd5e94558bf9f0bc78a740af0e086cc69faec2", "required_paths": [ "models/chat/qwen3_5.py", "tasks/gsm8k/gsm8k.yaml", diff --git a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md index 32a9926be88..be777348ae7 100644 --- a/examples/puzzletron/docs/vlm_checkpoint_evaluation.md +++ b/examples/puzzletron/docs/vlm_checkpoint_evaluation.md @@ -1,15 +1,34 @@ # VLM checkpoint evaluation Use this evaluator to test a local Qwen 3.5 checkpoint on image and video -benchmarks. A profile chooses the benchmarks, examples, model-loading path, -and evaluation settings. Use the same profile for every model being compared. +benchmarks. A runnable profile composes three independently named contracts: -The common choices are: +- a sample set, which fixes the benchmark scope and selected examples; +- a backend profile, which fixes model loading and prompt construction; and +- an evaluator profile, which pins the `lmms-eval` revision. -- `short-all-native-v1` for the recommended short comparison across eight - benchmarks; -- `short-v1` or `short-native-v1` for a faster three-benchmark check; and -- `full-v1` to evaluate every available example in eight benchmarks. +Callers still select one profile. Keeping the components separate makes it +clear that native and vLLM runs can use identical examples while changing only +the backend. Use the same runnable profile for every model being compared. + +Choose a profile by the checkpoint and the amount of coverage needed: + +| Profile | What it evaluates | Use it for | +| --- | --- | --- | +| `core-3_24-examples_r1-native` | The same eight examples from each of RealWorldQA, MMMU validation, and one MVBench task (24 total), loaded with the native Qwen backend | A fast loading, prompt, and scoring smoke check for a Qwen checkpoint | +| `core-3_24-examples_r1-vllm` | The same 24 examples, served through vLLM | A fast smoke check for a materialized heterogeneous checkpoint | +| `core-3_344-examples_r1-native` | 64 RealWorldQA, 120 MMMU validation, and 160 MVBench examples (344 total), loaded with the native Qwen backend | Routine teacher-versus-candidate campaign comparisons | +| `core-3_344-examples_r1-vllm` | The same 344 examples, served through vLLM | Routine comparisons of materialized heterogeneous checkpoints | +| `judge-free-8_690-examples_r1-native` | 690 fixed examples across the eight benchmarks that require no external judge | A broader image-and-video regression screen | +| `core-3_full_r1-native` | All 5,665 RealWorldQA, MMMU validation, and MVBench examples, loaded with the native Qwen backend | A full-dataset reference for the pinned Qwen 3.5 0.8B teacher | +| `core-3_full_r1-vllm` | The same 5,665 examples, served through vLLM | A separate full-dataset vLLM reference for the pinned teacher | + +The four older names `short-v1`, `short-native-v1`, `short-all-native-v1`, and +`full-v1` are temporary compatibility profiles. They preserve the exact rows, +backend settings, and maintained evaluator revision of commands that existed +before component profiles. They emit a deprecation warning and will be removed +after downstream callers migrate. Keep them only for reproducing earlier runs; +use the descriptive profiles above for new work. For text-only IFEval and GSM8K evaluation, use the separate [text checkpoint evaluator](checkpoint_evaluation.md). @@ -17,49 +36,75 @@ For text-only IFEval and GSM8K evaluation, use the separate ## Prepare the worker environment Run evaluation in the default Puzzletron worker image described in the -[worker environment guide](environment_setup.md). The image includes the exact -upstream `lmms-eval` 0.7.2 source revision recorded in -`examples/puzzletron/ci_environment.json`, including its native Qwen 3.5 image -and video backend. No evaluator overlay or separate VLM requirements install is -needed. - -The image applies one tracked compatibility patch to `lmms-eval`. It removes an -unused private WandB printer integration, aligns the WandB requirement with -AutoModel, and replaces the legacy LaTeX parser whose ANTLR pin conflicts with -Puzzletron's Hydra stack. The maintained replacement provides the same parser -entry point to the three affected math tasks. Image construction verifies the -patch checksum, source revision, resulting checkout diff, required native -backend and task files, and the resolved Python dependency set. Do not replace -or modify that evaluator checkout inside the worker image. +[worker environment guide](environment_setup.md). The image pins the +`lmms-eval` source revision and dependencies recorded in +`examples/puzzletron/ci_environment.json`, includes the native Qwen 3.5 image +and video backend, and preserves each task's output-token budget for vLLM. No +evaluator overlay or separate VLM requirements install is needed. Do not modify +the evaluator checkout inside the image. + +The evaluator revision is part of a profile's result contract: task definitions, +prompt adapters, parsers, and scoring can change between `lmms-eval` commits. +Preflight therefore rejects any revision other than the one selected by the +profile instead of producing a score that only appears comparable. + +The pinned `lmms_eval_compat_3e675904.patch` also prevents the vLLM adapter from +overwriting each task's output-token budget. The patch and its checksum are +build inputs in `ci_environment.json`; rebuild the worker image from this +repository revision before using these profiles. An image built from an older +revision does not contain the fix. ## Understand the two execution paths | Path | Profiles | What it does | | --- | --- | --- | -| Qwen-specific Transformers | `short-native-v1`, `short-all-native-v1` | Loads the checkpoint directly with the Qwen 3.5 model loader. Video prompts include timestamps for sampled frames. | -| General vLLM | `short-v1`, `full-v1` | Runs the checkpoint through vLLM and converts inputs to general image and video messages. Video prompts do not include frame timestamps. | +| Qwen-specific Transformers | `short-native-v1`, `core-3_24-examples_r1-native`, `core-3_344-examples_r1-native`, `short-all-native-v1`, `judge-free-8_690-examples_r1-native`, `core-3_full_r1-native` | Loads the checkpoint directly with the Qwen 3.5 model loader. Video prompts include timestamps for sampled frames. | +| General vLLM | `short-v1`, `core-3_344-examples_r1-vllm`, `core-3_24-examples_r1-vllm`, `full-v1`, `core-3_full_r1-vllm` | Runs the checkpoint through vLLM and converts inputs to general image and video messages. Video prompts do not include frame timestamps. Materialized heterogeneous checkpoints require this path. | -Both paths use the same pinned `lmms-eval` revision but produce different -prompts. Their scores represent the complete paths and do not isolate the -effect of the inference engine. +The native and vLLM paths produce different prompts, so their scores represent +the complete paths and do not isolate the inference engine. +The heterogeneous-checkpoint `core-3_344-examples_r1-vllm` and `core-3_24-examples_r1-vllm` profiles pin +FlashAttention 2 because the runtime's FlashAttention 3 scheduler does +not support their per-layer attention geometry. ## Choose an evaluation Use a versioned profile when scores will be compared across checkpoints: -| Profile | Coverage | Examples evaluated | -| --- | --- | --- | -| `short-v1` | RealWorldQA, MMMU, and MVBench | 344 predefined examples; MMMU uses four per subject and MVBench uses eight per category | -| `short-native-v1` | Same examples as `short-v1`, using the Qwen-specific Transformers adapter | The same 344 predefined examples with timestamps added to video prompts | -| `short-all-native-v1` | All eight judge-free benchmarks, using the Qwen-specific Transformers adapter | 690 predefined examples: the same 344 plus 346 from five additional benchmarks | -| `full-v1` | Eight judge-free image and video benchmarks | Every available example in each pinned dataset version | - -For future short teacher and student comparisons through the Qwen-specific -adapter, prefer `short-all-native-v1` when all eight datasets are available. -It checks more kinds of image and video tasks than the 344-row profiles. Use -the 344-row profiles for faster three-benchmark regression checks or when the -same rows must be compared across both execution paths. Neither short profile -replaces `full-v1` for complete benchmark reporting. +The sample-set portion of a current profile name describes the data: + +| Sample set | Benchmarks | Selected examples | Pinned population | +| --- | --- | ---: | ---: | +| `core-3_24-examples_r1` | RealWorldQA, MMMU validation, one MVBench task | 24 | 1,865 | +| `core-3_344-examples_r1` | RealWorldQA, MMMU validation, MVBench | 344 | 5,665 | +| `judge-free-8_690-examples_r1` | All eight judge-free benchmarks | 690 | 31,916 | +| `core-3_full_r1` | RealWorldQA, MMMU validation, MVBench | all | 5,665 | + +Here, `core-3` means the three campaign benchmarks and `judge-free-8` means +the current eight-benchmark set that needs no external judge. The number before +`examples` is the number actually evaluated, not the full dataset population. +`r1` versions the sample-set definition. It does not version the backend or +evaluator. + +Append `-native` or `-vllm` to select a composed runnable profile. For example, +`core-3_344-examples_r1-native` and `core-3_344-examples_r1-vllm` share the +same 344 examples and evaluator, while their backend profiles differ. The 344 +examples are 64 of 765 RealWorldQA examples, 120 of 900 MMMU examples, and 160 +of 4,000 MVBench examples. The 690-example set contains those 344 plus 346 +examples selected from VideoMMMU, Video-MME, LongVideoBench, MLVU, and +PerceptionTest. + +Use `core-3_344-examples_r1-native` for campaign comparisons. Use +`judge-free-8_690-examples_r1-native` when broader image and video regression coverage is more +important than matching the campaign screen. The v1 profiles require their +pinned evaluator revisions. No short profile replaces a full-data profile for +complete benchmark reporting. + +The paired `core-3_full_r1-*` profiles provide teacher references. They pin the +Qwen 3.5 0.8B Hub snapshot and +reject other checkpoints or runtime-setting overrides. Keep the two backend +results separate: their prompt construction and MVBench frame annotations +differ by design, so their score delta is not a pure inference-engine effect. Profiles keep the evaluator, datasets, selected examples, video sampling, answer generation, and batch size fixed. They disable Qwen thinking so the @@ -67,15 +112,40 @@ model returns the short answer expected by these benchmarks. Puzzletron stops before evaluation if the installed evaluator or cached data do not match the selected profile. -`short-all-native-v1` balances VideoMMMU across its three categories, -Video-MME across duration and domain, MLVU across task type, and PerceptionTest -across area and reasoning type. LongVideoBench uses deterministic index-spaced -rows. This profile is a reproducible regression screen, not a full-benchmark -quality estimate. +The preflight report and normalized result include an `output_budget_contract` +for every selected task. It records the requested and effective output-token +budget and the adapter-specific resolution rule. In the maintained three-task +profiles, RealWorldQA and MVBench use 16 tokens and MMMU uses 128. The native +adapter applies each task budget directly. The pinned generic vLLM adapter +treats its model-level `max_new_tokens` value as a lower bound, so the report +also records that limitation; backend score differences therefore include +generation-policy and prompt-path differences, not just engine behavior. + +`core-3_344-examples_r1-native` selects +64 positions across all 765 RealWorldQA test rows, four positions within each +30-row MMMU subject, and eight positions within each 200-row MVBench task. Its +profile records the generator version, population and stratum counts, selected +index quantiles, recorded source-row identities, and a digest of those identities. +These fixed rows provide a regression screen, not a representative +full-benchmark estimate. Compare a teacher and every candidate with the same +profile. + +`judge-free-8_690-examples_r1-native` adds fixed, evenly spaced samples from +five video benchmarks: 24 rows from each of three VideoMMMU tasks, four rows +from each of 18 Video-MME duration-and-domain strata, 68 rows across the +LongVideoBench validation split, and 10 rows from each of seven MLVU task +types. Its 64 PerceptionTest rows are balanced across the 13 observed +area-and-reasoning strata: the first 12 in lexical order receive five rows and +the last receives four. Before selecting any rows, the generated adapters +verify the exact source population size and, where the pinned metadata exposes +the selected stratification fields, the complete source stratum counts. They +also verify recorded upstream IDs for Video-MME, MLVU, and PerceptionTest. +LongVideoBench is stratified only by its validation split. This profile is a +fixed regression screen, not a full-benchmark quality estimate. `full-v1` excludes MMVU because complete MMVU evaluation requires an external -judge. Run MMVU separately through `mmvu-smoke` or the judge-enabled legacy -`full` suite. +judge. Run MMVU separately through `mmvu-smoke` or the judge-enabled `full` +suite. These additional options are intended for setup checks and targeted diagnosis: @@ -97,51 +167,31 @@ frames. ## Cache benchmark data -Evaluation is offline and requires every selected dataset revision to already -exist under an explicit Hugging Face cache root. Populate the pinned image -datasets for the selected profile or suite before starting evaluation. The -repository stores only profile metadata and exact-row selectors; it does not -store benchmark records or media. The example uses `short-v1`: +Evaluation reads every selected dataset revision from an explicit Hugging Face +cache root. The repository stores only profile +metadata and exact-row selectors; it does not store benchmark records or media. +To prepare the common image and video cache independently, run: ```bash export HF_HOME=/path/to/huggingface-cache -python - "$HF_HOME" short-v1 <<'PY' -import sys -from pathlib import Path - -from huggingface_hub import snapshot_download - -from examples.puzzletron.evaluation.vlm import profile, suites - -hf_home = Path(sys.argv[1]) -for task in suites.source_tasks(sys.argv[2]): - dataset = profile.VLM_BENCHMARK_DATASETS[task] - if dataset.media_dir is not None: - continue - snapshot_download( - repo_id=dataset.repository, - repo_type="dataset", - revision=dataset.revision, - cache_dir=hf_home / "hub", - ) -PY -``` - -The video preparation command downloads its pinned snapshot and safely extracts -the media: - -```bash python -m examples.puzzletron.evaluation.vlm.preparation.benchmark_data \ --hf-home "$HF_HOME" \ - --tasks mvbench + --tasks realworldqa,mmmu_val,mvbench,video_mmmu,videomme,longvideobench_val_v,mlvu_dev,perceptiontest_val_mc ``` -Pass a comma-separated list to `--tasks` when preparing more than one video -dataset. Use `--download-only` and `--extract-only` to split transfer and -extraction across jobs, or `--range-resume` for a resumable single-writer -download. Run the command with `--help` to list the supported video dataset -names. +The command downloads each listed exact pinned snapshot and safely extracts +media only for tasks that declare a preparation directory. For `core-3_344-examples_r1-native`, +prepare `realworldqa`, `mmmu_val`, and `mvbench`. Use `--download-only` and +`--extract-only` to split transfer and extraction across jobs, or +`--range-resume` for a resumable single-writer download. Run the command with +`--help` to list all supported dataset task names. Preparation records exact +snapshot and media inventories. Routine reuse checks the recorded paths, sizes, +and timestamps without rereading all cached content. Add `--verify-content` to +recompute and compare every recorded SHA-256 digest. Changed or incomplete owned +media is rebuilt from the pinned snapshot only when the host supports atomic +directory exchange. Otherwise the existing root is preserved and preparation +fails. Video suites also require an installed `decord`-compatible reader. The Puzzletron requirements select the supported reader for the current platform @@ -156,27 +206,37 @@ running inference. ## Run a versioned profile -The checkpoint path is an invocation input, not part of either profile -manifest. Use the same command with a teacher or any materialized Qwen 3.5 -student checkpoint: +For comparison profiles, the checkpoint path is an invocation input. Use the +same command with a teacher or any materialized Qwen 3.5 student checkpoint: ```bash python -m examples.puzzletron.evaluation.vlm.run \ --checkpoint /path/to/checkpoint \ - --output-dir /path/to/results/short-v1 \ + --output-dir /path/to/results/judge-free-8_690-examples_r1-native \ --hf-home "$HF_HOME" \ - --profile short-v1 + --profile judge-free-8_690-examples_r1-native ``` -Replace `short-v1` with `short-native-v1` after installing the native -requirements. Use `short-all-native-v1` after caching all eight judge-free -benchmarks, or `full-v1` after caching the same datasets for a complete -generic-vLLM evaluation. Use `--preflight-only` before consuming GPU time. +The `core-3_full_r1-*` profiles instead require the exact pinned local Qwen 3.5 +0.8B Hub snapshot. For example: -To run tasks in parallel, run one profile task per job with either -`--profile full-v1 --profile-task TASK` or -`--profile short-all-native-v1 --profile-task TASK`. A complete result needs -one successful job for each of the eight tasks in the selected profile. +```bash +python -m examples.puzzletron.evaluation.vlm.run \ + --checkpoint "$HF_HOME/hub/models--Qwen--Qwen3.5-0.8B/snapshots/2fc06364715b967f1860aea9cf38778875588b17" \ + --output-dir /path/to/results/core-3_full_r1-native/realworldqa \ + --hf-home "$HF_HOME" \ + --profile core-3_full_r1-native \ + --profile-task realworldqa \ + --preflight-only +``` + +Use `full-v1` only with its pinned evaluator revision. Always run +`--preflight-only` before consuming GPU time. + +To run tasks in parallel, run one profile task per job with +`--profile PROFILE --profile-task TASK`. This works for `full-v1`, the paired +`core-3_full_r1-*` profiles, and `judge-free-8_690-examples_r1-native`. A complete result needs one +successful job for every task in the selected profile. Grouped tasks can use multiple batch-1 workers without changing inference batching. Add `--profile-task-shard INDEX/COUNT` to `mvbench` or `video_mmmu`, @@ -209,12 +269,15 @@ Each execution creates an `attempt_/` directory containing the command, logs, raw evaluator output, and normalized metrics in `summary.json`. Repeating a run creates another attempt directory instead of overwriting earlier output. -`short-v1` and `full-v1` use the generic vLLM adapter. `short-native-v1` and -`short-all-native-v1` use the Qwen-specific Transformers adapter. Both use the -same pinned evaluator revision but construct different model inputs. Keep the -profile name with every score, and do not combine scores from the two paths as -if only the inference engine changed. The recorded preflight report includes -the backend identity, frame policy, generation policy, and evaluator revision. +The execution-path table above identifies each profile's adapter. The native +and generic vLLM paths construct different model inputs, so keep the profile +name with every score and do not combine scores as if only the inference engine +changed. The preflight report records backend identity, frame policy, +generation policy, and exact evaluator revision. + +MMMU results also include `mmmu_parser_audit`. Its per-sample status counts use +`parsed`, `parsed_open`, `invalid_open`, and `fallback_random`, making invalid +open-ended parses and random-fallback scoring visible in each result. If preflight fails, address the reported checkpoint, revision, cache, decoder, or credential mismatch before retrying. Inspect `stderr.txt` in the attempt diff --git a/examples/puzzletron/docs/worker_image.md b/examples/puzzletron/docs/worker_image.md index 1876f734933..c9cd0cfaa08 100644 --- a/examples/puzzletron/docs/worker_image.md +++ b/examples/puzzletron/docs/worker_image.md @@ -4,6 +4,9 @@ The [`Dockerfile`](../Dockerfile) contains the worker installation steps. [`ci_environment.json`](../ci_environment.json) stores the versions, source revisions, CUDA targets, and downloaded-file checksums used by those steps. The Dockerfile reads the same file for installation and its build-time checks. +Rebuild the image whenever the Dockerfile, `ci_environment.json`, or a patch +referenced by that manifest changes. Patches are applied while the image is +built, so an existing image does not acquire a later patch automatically. ## Build diff --git a/examples/puzzletron/evaluation/checkpoint.py b/examples/puzzletron/evaluation/checkpoint.py index efe68857b5d..9a561a4b38c 100644 --- a/examples/puzzletron/evaluation/checkpoint.py +++ b/examples/puzzletron/evaluation/checkpoint.py @@ -151,6 +151,8 @@ def positive_float(value: str) -> float: def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> str: """Return the imported evaluator revision after matching its source and patch pin.""" + if expected_revision != LMMS_EVAL_REVISION: + raise RuntimeError(f"unsupported lmms-eval revision: {expected_revision}") revision = _imported_lmms_eval_revision() if revision is not None: if revision != expected_revision: @@ -172,7 +174,7 @@ def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> st ci_environment.verify_installed_vcs_source("lmms-eval", LMMS_EVAL_SOURCE) except (OSError, subprocess.SubprocessError) as error: raise RuntimeError("installed lmms-eval source provenance is unavailable") from error - revision = LMMS_EVAL_REVISION + revision = expected_revision else: revision = None if revision != expected_revision: diff --git a/examples/puzzletron/evaluation/vlm/contracts.py b/examples/puzzletron/evaluation/vlm/contracts.py index 78b257c07f1..03ebedcd369 100644 --- a/examples/puzzletron/evaluation/vlm/contracts.py +++ b/examples/puzzletron/evaluation/vlm/contracts.py @@ -19,6 +19,7 @@ import hashlib import json +import warnings from dataclasses import dataclass from pathlib import Path from typing import cast @@ -31,62 +32,206 @@ "SHORT_PROFILE_NAMES", "ProfileContract", "load_profile", + "warn_deprecated_profile", ] -_PROFILE_SCHEMA = "modelopt.vlm-evaluation-profile/v1" +_PROFILE_SCHEMA = "modelopt.vlm-evaluation-profile/v2" +_SAMPLE_SET_SCHEMA = "modelopt.vlm-sample-set/v1" +_BACKEND_PROFILE_SCHEMA = "modelopt.vlm-backend-profile/v1" +_EVALUATOR_PROFILE_SCHEMA = "modelopt.vlm-evaluator-profile/v1" _PROFILE_ROOT = Path(__file__).with_name("profiles") -PROFILE_NAMES = ("short-v1", "short-native-v1", "short-all-native-v1", "full-v1") -_PROFILE_TASKS = { - "short-v1": ("realworldqa", "mmmu_val", "mvbench"), - "short-native-v1": ("realworldqa", "mmmu_val", "mvbench"), +PROFILE_NAMES = ( + "short-v1", + "short-native-v1", + "core-3_344-examples_r1-native", + "core-3_344-examples_r1-vllm", + "core-3_24-examples_r1-native", + "core-3_24-examples_r1-vllm", + "short-all-native-v1", + "judge-free-8_690-examples_r1-native", + "full-v1", + "core-3_full_r1-native", + "core-3_full_r1-vllm", +) +_PROFILE_COMPONENTS = { + # short-v1, short-native-v1, short-all-native-v1, and full-v1 are temporary + # compatibility compositions. They preserve the exact rows, backend, and + # evaluator revision of names that predate component profiles. New runs + # should use descriptive profiles. Remove these entries and their legacy + # sample sets after downstream callers have migrated. + "short-v1": ( + "core-3_344-examples_legacy-r1", + "qwen-3.5-vllm_r1", + "lmms-eval-modelopt_r1", + ), + "short-native-v1": ( + "core-3_344-examples_legacy-r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", + ), + "core-3_344-examples_r1-native": ( + "core-3_344-examples_r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", + ), + "core-3_344-examples_r1-vllm": ( + "core-3_344-examples_r1", + "anymodel-vllm_r1", + "lmms-eval-modelopt_r1", + ), + "core-3_24-examples_r1-native": ( + "core-3_24-examples_r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", + ), + "core-3_24-examples_r1-vllm": ( + "core-3_24-examples_r1", + "anymodel-vllm-eager_r1", + "lmms-eval-modelopt_r1", + ), "short-all-native-v1": ( - "realworldqa", - "mmmu_val", - "mvbench", - "video_mmmu", - "videomme", - "longvideobench_val_v", - "mlvu_dev", - "perceptiontest_val_mc", + "judge-free-8_690-examples_legacy-r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", + ), + "judge-free-8_690-examples_r1-native": ( + "judge-free-8_690-examples_r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", + ), + "full-v1": ( + "judge-free-8_full_legacy-r1", + "qwen-3.5-vllm_r1", + "lmms-eval-modelopt_r1", + ), + "core-3_full_r1-native": ( + "core-3_full_r1", + "qwen-3.5-native_r1", + "lmms-eval-modelopt_r1", ), - "full-v1": tuple(task for task in profile.VLM_BENCHMARK_TASKS if task != "mmvu_val"), + "core-3_full_r1-vllm": ( + "core-3_full_r1", + "qwen-3.5-vllm_r1", + "lmms-eval-modelopt_r1", + ), +} +_DEPRECATED_PROFILE_REPLACEMENTS = { + "short-v1": "core-3_344-examples_r1-vllm", + "short-native-v1": "core-3_344-examples_r1-native", + "short-all-native-v1": "judge-free-8_690-examples_r1-native", + "full-v1": None, } -_PROFILE_SELECTIONS = { - "short-v1": "exact-rows", - "short-native-v1": "exact-rows", - "short-all-native-v1": "exact-rows", - "full-v1": "all", +_CORE_3_TASKS = ("realworldqa", "mmmu_val", "mvbench") +_JUDGE_FREE_8_TASKS = ( + "realworldqa", + "mmmu_val", + "mvbench", + "video_mmmu", + "videomme", + "longvideobench_val_v", + "mlvu_dev", + "perceptiontest_val_mc", +) +_SAMPLE_SET_TASKS = { + "core-3_344-examples_legacy-r1": _CORE_3_TASKS, + "core-3_344-examples_r1": _CORE_3_TASKS, + "core-3_24-examples_r1": _CORE_3_TASKS, + "judge-free-8_690-examples_legacy-r1": _JUDGE_FREE_8_TASKS, + "judge-free-8_690-examples_r1": _JUDGE_FREE_8_TASKS, + "judge-free-8_full_legacy-r1": tuple( + task for task in profile.VLM_BENCHMARK_TASKS if task != "mmvu_val" + ), + "core-3_full_r1": _CORE_3_TASKS, +} +_SAMPLE_SET_SELECTIONS = { + "core-3_344-examples_legacy-r1": "exact-rows", + "core-3_344-examples_r1": "exact-rows", + "core-3_24-examples_r1": "exact-rows", + "judge-free-8_690-examples_legacy-r1": "exact-rows", + "judge-free-8_690-examples_r1": "exact-rows", + "judge-free-8_full_legacy-r1": "all", + "core-3_full_r1": "all", } SHORT_PROFILE_NAMES = tuple( - name for name in PROFILE_NAMES if _PROFILE_SELECTIONS[name] == "exact-rows" + name + for name in PROFILE_NAMES + if _SAMPLE_SET_SELECTIONS[_PROFILE_COMPONENTS[name][0]] == "exact-rows" ) -_PROFILE_BACKENDS = { - "short-v1": { +_BACKEND_SETTINGS = { + "qwen-3.5-vllm_r1": { "enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3", }, - "short-native-v1": { + "qwen-3.5-native_r1": { "attention_implementation": "sdpa", "enable_thinking": False, "name": "qwen3_5", }, - "short-all-native-v1": { - "attention_implementation": "sdpa", + "anymodel-vllm_r1": { + "attention_config": {"flash_attn_version": 2}, "enable_thinking": False, - "name": "qwen3_5", + "name": "vllm", + "reasoning_parser": "qwen3", }, - "full-v1": { + "anymodel-vllm-eager_r1": { + "attention_config": {"flash_attn_version": 2}, "enable_thinking": False, + "enforce_eager": True, "name": "vllm", "reasoning_parser": "qwen3", }, } -_PROFILE_REVISIONS = { - "short-v1": checkpoint.LMMS_EVAL_REVISION, - "short-native-v1": checkpoint.LMMS_EVAL_REVISION, - "short-all-native-v1": checkpoint.LMMS_EVAL_REVISION, - "full-v1": checkpoint.LMMS_EVAL_REVISION, +_EVALUATOR_REVISIONS = { + "lmms-eval-modelopt_r1": checkpoint.LMMS_EVAL_REVISION, +} +_SAMPLE_SET_MODELS = { + "core-3_full_r1": { + "repository": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17", + } +} +_SAMPLE_SET_POPULATIONS = {"core-3_full_r1": {"realworldqa": 765, "mmmu_val": 900, "mvbench": 4000}} +_MVBENCH_LEAF_POPULATIONS = { + "action_sequence": 200, + "moving_count": 200, + "action_prediction": 200, + "episodic_reasoning": 200, + "action_antonym": 200, + "action_count": 200, + "scene_transition": 200, + "object_shuffle": 200, + "object_existence": 200, + "fine_grained_pose": 200, + "unexpected_action": 200, + "moving_direction": 200, + "state_change": 200, + "object_interaction": 200, + "character_order": 200, + "action_localization": 200, + "counterfactual_inference": 200, + "fine_grained_action": 200, + "moving_attribute": 200, + "egocentric_navigation": 200, +} +_AUDITED_SAMPLE_SETS = frozenset( + { + "core-3_24-examples_r1", + "core-3_344-examples_r1", + "judge-free-8_690-examples_r1", + } +) +_SAMPLING_AUDIT_SCHEMA = "modelopt.vlm-sampling-audit/v1" +_SAMPLING_GENERATOR = {"name": "systematic-midpoint", "version": 1} +_SAMPLING_STRATA = { + "realworldqa": "split", + "mmmu_val": "subject", + "mvbench": "leaf_task", + "video_mmmu": "leaf_task", + "videomme": "duration+domain", + "longvideobench_val_v": "split", + "mlvu_dev": "task_type", + "perceptiontest_val_mc": "area+reasoning", } @@ -98,6 +243,21 @@ class ProfileContract: manifest: dict[str, object] fingerprint: str + @property + def sample_set(self) -> str: + """Return the selected example-set contract name.""" + return cast("str", self.manifest["sample_set"]) + + @property + def backend_profile(self) -> str: + """Return the selected model-backend contract name.""" + return cast("str", self.manifest["backend_profile"]) + + @property + def evaluator_profile(self) -> str: + """Return the selected evaluator contract name.""" + return cast("str", self.manifest["evaluator_profile"]) + @property def source_tasks(self) -> tuple[str, ...]: """Return benchmark tasks in their declared evaluation order.""" @@ -106,34 +266,91 @@ def source_tasks(self) -> tuple[str, ...]: @property def exact_rows(self) -> dict[str, object] | None: - """Return the legacy exact-row selector payload when the profile uses one.""" + """Return the executable exact-row selector payload when the profile uses one.""" if self.manifest["selection"] != "exact-rows": return None tasks = cast("dict[str, dict[str, object]]", self.manifest["tasks"]) - return { + exact_rows: dict[str, object] = { "schema": "modelopt.vlm-benchmark-quick/v1", "lmms_eval_revision": self.manifest["lmms_eval_revision"], "tasks": { task: { "dataset_revision": entry["dataset_revision"], "rows": entry["rows"], + **({"selection": entry["selection"]} if "selection" in entry else {}), } for task, entry in tasks.items() }, } + if "sampling" in self.manifest: + exact_rows["selection"] = self.manifest["sampling"] + return exact_rows + + +def warn_deprecated_profile(name: str) -> None: + """Warn when a temporary compatibility profile is selected for execution.""" + if name in _DEPRECATED_PROFILE_REPLACEMENTS: + replacement = _DEPRECATED_PROFILE_REPLACEMENTS[name] + guidance = ( + f"use {replacement} for new runs" + if replacement is not None + else "choose a descriptive profile for new runs" + ) + warnings.warn( + f"{name} is a deprecated compatibility profile; {guidance}. " + "It will be removed after downstream callers migrate.", + FutureWarning, + stacklevel=2, + ) def load_profile(name: str) -> ProfileContract: """Load a named profile after validating every executable pin.""" if name not in PROFILE_NAMES: raise ValueError(f"unsupported VLM evaluation profile: {name}") - path = _PROFILE_ROOT / f"{name}.json" - try: - manifest = _resolve_manifest(name, json.loads(path.read_text())) - except (OSError, json.JSONDecodeError) as error: - raise RuntimeError(f"VLM evaluation profile is unreadable: {path}") from error - if not isinstance(manifest, dict): - raise RuntimeError(f"VLM evaluation profile must contain an object: {path}") + composition = _load_component(_PROFILE_ROOT, name, _PROFILE_SCHEMA, "profile") + expected_components = _PROFILE_COMPONENTS[name] + observed_components = tuple( + composition.get(key) for key in ("sample_set", "backend_profile", "evaluator_profile") + ) + if observed_components != expected_components: + raise RuntimeError(f"{name} profile composition differs from the runtime policy") + + sample_set_name, backend_name, evaluator_name = expected_components + sample_set = _resolve_sample_set(sample_set_name) + backend = _load_component( + _PROFILE_ROOT / "backends", backend_name, _BACKEND_PROFILE_SCHEMA, "backend profile" + ) + evaluator = _load_component( + _PROFILE_ROOT / "evaluators", + evaluator_name, + _EVALUATOR_PROFILE_SCHEMA, + "evaluator profile", + ) + if backend.get("settings") != _BACKEND_SETTINGS[backend_name]: + raise RuntimeError(f"{backend_name} backend profile differs from the runtime policy") + if evaluator.get("lmms_eval_revision") != _EVALUATOR_REVISIONS[evaluator_name]: + raise RuntimeError(f"{evaluator_name} evaluator profile differs from the runtime pin") + + manifest = { + **composition, + "lmms_eval_revision": evaluator["lmms_eval_revision"], + "model_family": { + "architecture": "Qwen3_5ForConditionalGeneration", + "model_type": "qwen3_5", + }, + "backend": backend["settings"], + "preprocessing": {"fps": 2, "max_frames": 32, "video_reader": "decord"}, + "generation": {"do_sample": False, "temperature": 0}, + "seed": 42, + "repetitions": 1, + "batch_size": 1, + **{ + key: value + for key, value in sample_set.items() + if key not in {"schema", "name", "extends"} + }, + } _validate_manifest(name, manifest) canonical = json.dumps(manifest, separators=(",", ":"), sort_keys=True).encode() return ProfileContract( @@ -143,23 +360,52 @@ def load_profile(name: str) -> ProfileContract: ) -def _resolve_manifest(name: str, manifest: object) -> object: - """Resolve one shallow profile inheritance declaration.""" - if not isinstance(manifest, dict) or "extends" not in manifest: - return manifest - base_name = manifest.get("extends") - if not isinstance(base_name, str) or base_name not in PROFILE_NAMES or base_name == name: - raise RuntimeError(f"{name} profile extends an unsupported base profile") - base_path = _PROFILE_ROOT / f"{base_name}.json" +def _load_component(root: Path, name: str, schema: str, label: str) -> dict[str, object]: + """Load one named component and validate its identity envelope.""" + path = root / f"{name}.json" try: - base = json.loads(base_path.read_text()) + component = json.loads(path.read_text()) except (OSError, json.JSONDecodeError) as error: - raise RuntimeError(f"VLM evaluation base profile is unreadable: {base_path}") from error - if not isinstance(base, dict) or "extends" in base: - raise RuntimeError(f"{name} profile base must be a concrete profile") - overrides = {key: value for key, value in manifest.items() if key != "extends"} + raise RuntimeError(f"VLM evaluation {label} is unreadable: {path}") from error + if not isinstance(component, dict): + raise RuntimeError(f"VLM evaluation {label} must contain an object: {path}") + if component.get("schema") != schema: + raise RuntimeError(f"{name} {label} schema must be {schema}") + if component.get("name") != name: + raise RuntimeError(f"{name} {label} name does not match its filename") + return component + + +def _resolve_sample_set(name: str, *, ancestors: tuple[str, ...] = ()) -> dict[str, object]: + """Resolve sample-set inheritance while rejecting cycles.""" + if name not in _SAMPLE_SET_TASKS: + raise RuntimeError(f"unsupported VLM evaluation sample set: {name}") + sample_set = _load_component( + _PROFILE_ROOT / "sample_sets", name, _SAMPLE_SET_SCHEMA, "sample set" + ) + if "extends" not in sample_set: + return sample_set + base_name = sample_set.get("extends") + if ( + not isinstance(base_name, str) + or base_name not in _SAMPLE_SET_TASKS + or base_name == name + or base_name in ancestors + ): + raise RuntimeError(f"{name} sample set extends an unsupported base sample set") + base = _resolve_sample_set(base_name, ancestors=(*ancestors, name)) + overrides = {key: value for key, value in sample_set.items() if key != "extends"} if isinstance(base.get("tasks"), dict) and isinstance(overrides.get("tasks"), dict): - overrides["tasks"] = {**base["tasks"], **overrides["tasks"]} + tasks = dict(cast("dict[str, object]", base["tasks"])) + override_tasks = cast("dict[str, object]", overrides["tasks"]) + for task, entry in override_tasks.items(): + base_entry = tasks.get(task) + tasks[task] = ( + {**base_entry, **entry} + if isinstance(base_entry, dict) and isinstance(entry, dict) + else entry + ) + overrides["tasks"] = tasks return {**base, **overrides} @@ -168,14 +414,24 @@ def _validate_manifest(name: str, manifest: dict[str, object]) -> None: raise RuntimeError(f"{name} profile schema must be {_PROFILE_SCHEMA}") if manifest.get("name") != name: raise RuntimeError(f"{name} profile name does not match its filename") - if manifest.get("lmms_eval_revision") != _PROFILE_REVISIONS[name]: - raise RuntimeError(f"{name} profile lmms_eval_revision differs from the runtime pin") + sample_set_name, backend_name, evaluator_name = _PROFILE_COMPONENTS[name] + if manifest.get("sample_set") != sample_set_name: + raise RuntimeError(f"{name} profile sample set differs from its composition") + if manifest.get("backend_profile") != backend_name: + raise RuntimeError(f"{name} profile backend differs from its composition") + if manifest.get("evaluator_profile") != evaluator_name: + raise RuntimeError(f"{name} profile evaluator differs from its composition") + if manifest.get("lmms_eval_revision") != _EVALUATOR_REVISIONS[evaluator_name]: + raise RuntimeError(f"{name} profile lmms_eval_revision differs from the evaluator pin") if manifest.get("model_family") != { "architecture": "Qwen3_5ForConditionalGeneration", "model_type": "qwen3_5", }: raise RuntimeError(f"{name} profile model family is unsupported") - if manifest.get("backend") != _PROFILE_BACKENDS[name]: + expected_model = _SAMPLE_SET_MODELS.get(sample_set_name) + if expected_model is not None and manifest.get("model") != expected_model: + raise RuntimeError(f"{name} profile model pin differs from the runtime policy") + if manifest.get("backend") != _BACKEND_SETTINGS[backend_name]: raise RuntimeError(f"{name} profile backend differs from the runtime policy") if manifest.get("preprocessing") != { "fps": 2, @@ -192,15 +448,16 @@ def _validate_manifest(name: str, manifest: dict[str, object]) -> None: ): raise RuntimeError(f"{name} profile execution identity differs from the runtime policy") selection = manifest.get("selection") - if selection != _PROFILE_SELECTIONS[name]: + if selection != _SAMPLE_SET_SELECTIONS[sample_set_name]: raise RuntimeError(f"{name} profile selection differs from its versioned policy") tasks = manifest.get("tasks") if not isinstance(tasks, dict) or not tasks: raise RuntimeError(f"{name} profile tasks must contain an object") - if tuple(tasks) != _PROFILE_TASKS[name]: + if tuple(tasks) != _SAMPLE_SET_TASKS[sample_set_name]: raise RuntimeError(f"{name} profile tasks differ from its versioned policy") for task, entry in tasks.items(): - _validate_task(name, task, entry, selection=cast("str", selection)) + _validate_task(sample_set_name, task, entry, selection=cast("str", selection)) + _validate_sampling(sample_set_name, manifest, tasks) def _validate_task(name: str, task: object, entry: object, *, selection: str) -> None: @@ -218,8 +475,149 @@ def _validate_task(name: str, task: object, entry: object, *, selection: str) -> observed = {key: entry.get(key) for key in expected} if observed != expected: raise RuntimeError(f"{name} profile task pins differ from the runtime catalog: {task}") + expected_population = _SAMPLE_SET_POPULATIONS.get(name, {}).get(task) + if expected_population is not None and entry.get("population_rows") != expected_population: + raise RuntimeError( + f"{name} profile task population differs from its versioned policy: {task}" + ) + if expected_population is not None and task == "mvbench": + if entry.get("leaf_populations") != _MVBENCH_LEAF_POPULATIONS: + raise RuntimeError( + f"{name} profile task leaf populations differ from its versioned policy: {task}" + ) rows = entry.get("rows") if selection == "all" and rows is not None: raise RuntimeError(f"{name} full-data task must not contain exact rows: {task}") if selection == "exact-rows" and (not isinstance(rows, list) or not rows): raise RuntimeError(f"{name} exact-row task must contain rows: {task}") + + +def _validate_sampling( + name: str, + manifest: dict[str, object], + tasks: dict[str, object], +) -> None: + sampling = manifest.get("sampling") + if name in _AUDITED_SAMPLE_SETS and not isinstance(sampling, dict): + raise RuntimeError(f"{name} profile must contain a sampling audit") + if sampling is None: + return + if not isinstance(sampling, dict): + raise RuntimeError(f"{name} profile sampling audit must contain an object") + if sampling != { + "schema": _SAMPLING_AUDIT_SCHEMA, + "claim_scope": "deterministic-screening-only", + "generator": _SAMPLING_GENERATOR, + }: + raise RuntimeError(f"{name} profile sampling audit policy is unsupported") + for task, entry in tasks.items(): + if not isinstance(entry, dict): + raise RuntimeError(f"{name} profile task must contain an object: {task}") + _validate_task_sampling(name, task, entry) + + +def _validate_task_sampling(name: str, task: str, entry: dict[str, object]) -> None: + selection = entry.get("selection") + rows = entry.get("rows") + if not isinstance(selection, dict) or not isinstance(rows, list): + raise RuntimeError(f"{name} profile task must contain an audited selection: {task}") + strata = selection.get("strata") + if ( + selection.get("method") != "systematic-midpoint" + or selection.get("stratified_by") != _SAMPLING_STRATA.get(task) + or selection.get("index_space") != "within-stratum" + or not isinstance(strata, list) + or not strata + or selection.get("selected_rows") != len(rows) + or selection.get("population_rows") + != sum(stratum.get("population_rows", 0) for stratum in strata if isinstance(stratum, dict)) + or len(strata) != sum(isinstance(stratum, dict) for stratum in strata) + ): + raise RuntimeError(f"{name} profile task sampling audit is invalid: {task}") + expected_counts: dict[str, tuple[int, int]] = {} + for stratum in strata: + stratum_name = stratum.get("name") + population_rows = stratum.get("population_rows") + selected_rows = stratum.get("selected_rows") + if ( + not isinstance(stratum_name, str) + or not stratum_name + or not isinstance(population_rows, int) + or isinstance(population_rows, bool) + or not isinstance(selected_rows, int) + or isinstance(selected_rows, bool) + or selected_rows < 0 + or population_rows < selected_rows + or stratum_name in expected_counts + ): + raise RuntimeError(f"{name} profile task sampling stratum is invalid: {task}") + expected_counts[stratum_name] = (population_rows, selected_rows) + + actual_indices: dict[str, list[int]] = {stratum: [] for stratum in expected_counts} + for row in rows: + if not isinstance(row, dict): + raise RuntimeError(f"{name} profile task sampling row is invalid: {task}") + stratum, local_index = _sampling_identity(task, row) + if stratum not in actual_indices: + raise RuntimeError(f"{name} profile task sampling stratum differs: {task}") + actual_indices[stratum].append(local_index) + for stratum, (population_rows, selected_rows) in expected_counts.items(): + expected = [ + ((2 * index + 1) * population_rows) // (2 * selected_rows) + for index in range(selected_rows) + ] + if actual_indices[stratum] != expected: + raise RuntimeError(f"{name} profile task rows differ from its sampling audit: {task}") + + identity = hashlib.sha256( + json.dumps(rows, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest() + if selection.get("selected_row_identities_sha256") != identity: + raise RuntimeError(f"{name} profile task row identity fingerprint differs: {task}") + ordered_indices = sorted(index for indices in actual_indices.values() for index in indices) + quantiles = { + "method": "lower-order-statistic", + **{ + f"p{percentile}": ordered_indices[(len(ordered_indices) - 1) * percentile // 100] + for percentile in (0, 25, 50, 75, 100) + }, + } + if selection.get("selected_index_quantiles") != quantiles: + raise RuntimeError(f"{name} profile task index quantiles differ: {task}") + + +def _sampling_identity(task: str, row: dict[str, object]) -> tuple[str, int]: + source_id = row.get("source_sample_id") + source_index = row.get("source_row_index") + if not isinstance(source_id, str) or not isinstance(source_index, int): + raise RuntimeError(f"profile sampling row identity is invalid: {task}") + if task == "realworldqa": + return "test", source_index + if task == "mmmu_val": + prefix, separator, item = source_id.rpartition("_") + if not separator or not item.isdigit() or not prefix.startswith("validation_"): + raise RuntimeError("profile sampling row identity is invalid: mmmu_val") + return prefix.removeprefix("validation_"), int(item) - 1 + if task in {"mvbench", "video_mmmu"}: + leaf = row.get("leaf_task") + if not isinstance(leaf, str) or not leaf.startswith(f"{task}_"): + raise RuntimeError(f"profile sampling row identity is invalid: {task}") + return leaf.removeprefix(f"{task}_"), source_index + if task in { + "videomme", + "longvideobench_val_v", + "mlvu_dev", + "perceptiontest_val_mc", + }: + stratum = row.get("sampling_stratum") + stratum_index = row.get("source_stratum_index") + if ( + not isinstance(stratum, str) + or not stratum + or not isinstance(stratum_index, int) + or isinstance(stratum_index, bool) + or stratum_index < 0 + ): + raise RuntimeError(f"profile sampling row identity is invalid: {task}") + return stratum, stratum_index + raise RuntimeError(f"profile sampling is unsupported for task: {task}") diff --git a/examples/puzzletron/evaluation/vlm/evaluator.py b/examples/puzzletron/evaluation/vlm/evaluator.py index 7c8b26e248f..5de80282a15 100644 --- a/examples/puzzletron/evaluation/vlm/evaluator.py +++ b/examples/puzzletron/evaluation/vlm/evaluator.py @@ -20,22 +20,51 @@ import json import os import tempfile +from collections import Counter from collections.abc import Mapping from hashlib import sha256 from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: import argparse from collections.abc import Callable from examples.puzzletron.evaluation import checkpoint -from examples.puzzletron.evaluation.vlm import preflight, tasks +from examples.puzzletron.evaluation.vlm import preflight, suites, tasks __all__ = ["evaluate"] _COMPLETED_RUN_SCHEMA = "modelopt.vlm-evaluation-completed-run/v1" _COMPLETED_RUN_FILENAME = "completed_run.json" +_MMMU_TASK = suites.task_name("mmmu_val") +_MMMU_PARSER_STATUSES = {"fallback_random", "invalid_open", "parsed", "parsed_open"} + + +def _expected_task_populations( + prepared: preflight.PreparedSuite, + configured_tasks: tuple[str, ...], +) -> dict[str, int] | None: + """Map a full-data profile to the exact generated task objects it loads.""" + contract = prepared.profile_contract + if contract is None or contract.manifest["selection"] != "all": + return None + profile_tasks = cast("dict[str, dict[str, object]]", contract.manifest["tasks"]) + expected: dict[str, int] = {} + for source_task, configured_task in zip(prepared.source_tasks, configured_tasks, strict=True): + entry = profile_tasks[source_task] + leaf_populations = entry.get("leaf_populations") + if isinstance(leaf_populations, dict): + selected_leaves = prepared.profile_task_leaves or tuple(leaf_populations) + expected.update( + { + suites.task_name(source_task, leaf=leaf): cast("int", leaf_populations[leaf]) + for leaf in selected_leaves + } + ) + elif "population_rows" in entry: + expected[configured_task] = cast("int", entry["population_rows"]) + return expected or None def _completion_identity( @@ -203,6 +232,96 @@ def _write_completed_run( temporary_path.unlink(missing_ok=True) +def _attach_mmmu_parser_audit(run_result: Mapping[str, object]) -> None: + """Summarize auditable per-sample MMMU parser outcomes in the normalized result.""" + result_path_value = run_result.get("result_path") + if not isinstance(result_path_value, str): + raise RuntimeError("VLM evaluation result is missing its normalized result path") + result_path = Path(result_path_value) + try: + summary = json.loads(result_path.read_text()) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"invalid VLM evaluation summary: {result_path}") from error + if not isinstance(summary, dict): + raise RuntimeError(f"invalid VLM evaluation summary: {result_path}") + + raw_result_path_value = run_result.get("raw_result_path") + if not isinstance(raw_result_path_value, str): + raise RuntimeError("VLM evaluation result is missing its raw result path") + raw_result_path = Path(raw_result_path_value).resolve() + output_root = result_path.parent.resolve() + try: + raw_result_path.relative_to(output_root) + except ValueError as error: + raise RuntimeError("VLM raw result path escapes its evaluation output") from error + result_suffix = "_results.json" + if not raw_result_path.name.endswith(result_suffix): + raise RuntimeError(f"invalid lmms-eval raw result filename: {raw_result_path.name}") + invocation_prefix = raw_result_path.name[: -len(result_suffix)] + sample_logs = [raw_result_path.with_name(f"{invocation_prefix}_samples_{_MMMU_TASK}.jsonl")] + if not sample_logs[0].is_file(): + raise RuntimeError("MMMU evaluation wrote no auditable sample log for its invocation") + statuses: Counter[str] = Counter() + sample_count = 0 + for sample_log in sample_logs: + try: + lines = sample_log.read_text().splitlines() + except OSError as error: + raise RuntimeError(f"cannot read MMMU sample log: {sample_log}") from error + for line_number, line in enumerate(lines, start=1): + if not line.strip(): + continue + try: + sample = json.loads(line) + except json.JSONDecodeError as error: + raise RuntimeError( + f"invalid MMMU sample log record: {sample_log}:{line_number}" + ) from error + accuracy = sample.get("mmmu_acc") if isinstance(sample, Mapping) else None + parser_status = accuracy.get("parser_status") if isinstance(accuracy, Mapping) else None + if ( + not isinstance(parser_status, list) + or not parser_status + or any(status not in _MMMU_PARSER_STATUSES for status in parser_status) + ): + raise RuntimeError( + f"MMMU sample log has no valid parser status: {sample_log}:{line_number}" + ) + statuses.update(cast("list[str]", parser_status)) + sample_count += 1 + + sample_counts = summary.get("sample_counts") + expected_samples = sample_counts.get(_MMMU_TASK) if isinstance(sample_counts, Mapping) else None + if expected_samples != sample_count: + raise RuntimeError( + f"MMMU parser audit covers {sample_count}/{expected_samples} evaluated samples" + ) + summary["mmmu_parser_audit"] = { + "sample_count": sample_count, + "sample_logs": [ + _file_identity(sample_log, root=result_path.parent) for sample_log in sample_logs + ], + "status_counts": dict(sorted(statuses.items())), + } + content = json.dumps(summary, indent=2, sort_keys=True) + "\n" + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + "w", + dir=result_path.parent, + prefix=f".{result_path.name}.", + delete=False, + ) as temporary: + temporary.write(content) + temporary.flush() + os.fsync(temporary.fileno()) + temporary_path = Path(temporary.name) + os.replace(temporary_path, result_path) + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + def evaluate( args: argparse.Namespace, *, @@ -212,6 +331,12 @@ def evaluate( """Prepare and run one pinned VLM profile invocation.""" prepared = preflight.prepare(args) + if ( + settings_overrides + and prepared.profile_contract is not None + and prepared.profile_contract.manifest.get("model") is not None + ): + raise ValueError("model-pinned evaluation profiles do not allow settings overrides") task_root, configured_tasks = tasks.prepare( args.output_dir, suite=prepared.suite, @@ -233,6 +358,7 @@ def evaluate( hf_home=prepared.hf_home, timeout_seconds=checkpoint.DEFAULT_PREFLIGHT_TIMEOUT_SECONDS, model_name=str(settings["model"]), + expected_populations=_expected_task_populations(prepared, configured_tasks), ) report = dict(prepared.report) report.update( @@ -263,6 +389,7 @@ def evaluate( "profile_fingerprint": report.get("profile_fingerprint"), "profile_name": report.get("profile_name"), "profile_schema": report.get("profile_schema"), + "output_budget_contract": report["output_budget_contract"], } chat_template_sha256 = _chat_template_sha256(settings) if chat_template_sha256 is not None: @@ -286,6 +413,8 @@ def evaluate( output_root=output_root, settings=settings, ) + if "mmmu_val" in prepared.source_tasks: + _attach_mmmu_parser_audit(run_result) _write_completed_run(output_root, identity=identity, result=run_result) runs.append(run_result) return { diff --git a/examples/puzzletron/evaluation/vlm/model.py b/examples/puzzletron/evaluation/vlm/model.py index 49c95762e74..a09167bd9ee 100644 --- a/examples/puzzletron/evaluation/vlm/model.py +++ b/examples/puzzletron/evaluation/vlm/model.py @@ -18,6 +18,7 @@ from __future__ import annotations import json +from copy import deepcopy from typing import TYPE_CHECKING from jinja2 import TemplateError @@ -72,7 +73,7 @@ def _render_template(content: str, *, source: Path) -> str: raise ValueError(f"Qwen 3.5 chat template is invalid: {source}") from error -def verify_checkpoint(checkpoint: Path, *, profile: str) -> None: +def verify_checkpoint(checkpoint: Path, *, profile: str, model_backend: str = "qwen3_5") -> None: """Verify a Qwen 3.5 VLM-family checkpoint and its local processor assets.""" config = _checkpoint_config(checkpoint) if config.get("model_type") != _MODEL_TYPE: @@ -90,6 +91,42 @@ def verify_checkpoint(checkpoint: Path, *, profile: str) -> None: text_config = config.get("text_config") if not isinstance(text_config, dict) or text_config.get("model_type") != "qwen3_5_text": raise ValueError(f"{profile} checkpoint text_config.model_type must be qwen3_5_text") + if model_backend == "qwen3_5" and realized_checkpoint: + # Keep torch-dependent Puzzletron imports out of the lightweight preflight import path. + from modelopt.torch.puzzletron.block_config import maybe_cast_block_configs + from modelopt.torch.puzzletron.utils.vllm_adapter import ( + convert_block_configs_to_per_layer_config, + ) + + block_configs = config.get("block_configs") + homogeneity_error = ( + f"{profile} native qwen3_5 backend cannot prove that an AnyModel checkpoint " + "is homogeneous; use a vLLM evaluation profile" + ) + derived_config = deepcopy(config) + derived_text_config = derived_config["text_config"] + derived_text_config.pop("per_layer_config", None) + if not isinstance(block_configs, list) or len(block_configs) != text_config.get( + "num_hidden_layers" + ): + raise ValueError(homogeneity_error) + try: + typed_block_configs = maybe_cast_block_configs(block_configs) + except (TypeError, ValueError) as error: + raise ValueError(homogeneity_error) from error + if typed_block_configs is None or any( + len(block.subblock_configs) < 2 for block in typed_block_configs + ): + raise ValueError(homogeneity_error) + derived_config["block_configs"] = typed_block_configs + convert_block_configs_to_per_layer_config(derived_config, keep_block_configs=True) + derived_per_layer_config = derived_text_config.get("per_layer_config") + declared_per_layer_config = text_config.get("per_layer_config") + if derived_per_layer_config or declared_per_layer_config not in (None, {}): + raise ValueError( + f"{profile} native qwen3_5 backend cannot load a heterogeneous AnyModel " + "per_layer_config; use a vLLM evaluation profile" + ) invalid_geometry = { key: text_config.get(key) for key in _TEXT_GEOMETRY_FIELDS diff --git a/examples/puzzletron/evaluation/vlm/post_mip.py b/examples/puzzletron/evaluation/vlm/post_mip.py index f77b3d15526..d83884cc02e 100644 --- a/examples/puzzletron/evaluation/vlm/post_mip.py +++ b/examples/puzzletron/evaluation/vlm/post_mip.py @@ -21,6 +21,7 @@ import json import os import warnings +from collections import Counter from pathlib import Path from typing import TYPE_CHECKING @@ -36,8 +37,12 @@ "TASK_PREFIX100_REPEAT2_PROFILE", "evaluate_e2e_full_eval_checkpoint", "evaluate_frozen_campaign_checkpoint", + "evaluate_frozen_campaign_v2_checkpoint", + "evaluate_frozen_campaign_v3_checkpoint", "evaluate_realworldqa_checkpoint", "evaluate_realworldqa_mmmu_prefix100_checkpoint", + "evaluate_reproducibility_smoke_checkpoint", + "evaluate_reproducibility_smoke_v2_checkpoint", "evaluate_short_v1_checkpoint", "register_profiles", ] @@ -55,7 +60,11 @@ _REALWORLDQA_PROFILE = "qwen35_vlm_realworldqa2_prefix2" _BOUNDED_REPEATED_PROFILE = "qwen35_vlm_realworldqa100_mmmu100_prefix100_repeat2" TASK_PREFIX100_REPEAT2_PROFILE = _BOUNDED_REPEATED_PROFILE -_FROZEN_CAMPAIGN_PROFILE = "qwen35_vlm_realworldqa64_mmmu120_mvbench160_frozen_rows_v1" +_FROZEN_CAMPAIGN_PROFILE_V1 = "qwen35_vlm_realworldqa64_mmmu120_mvbench160_frozen_rows_v1" +_FROZEN_CAMPAIGN_PROFILE_V2 = "qwen35_vlm_realworldqa64_mmmu120_mvbench160_frozen_rows_v2" +_FROZEN_CAMPAIGN_PROFILE_V3 = "qwen35_vlm_realworldqa64_mmmu120_mvbench160_frozen_rows_v3" +_REPRODUCIBILITY_SMOKE_PROFILE = "qwen35_vlm_core3_24row_smoke_v1" +_REPRODUCIBILITY_SMOKE_PROFILE_V2 = "qwen35_vlm_core3_24row_smoke_v2" def _run_profile( @@ -64,6 +73,7 @@ def _run_profile( output_root: str | Path, settings: Mapping[str, Any], suite: str, + evaluation_profile: str | None = None, require_manifest: bool = False, ) -> tuple[argparse.Namespace, dict[str, object], Path]: settings = dict(settings) @@ -76,29 +86,36 @@ def _run_profile( output_dir.mkdir(parents=True, exist_ok=True) row_manifest = settings.pop("row_manifest", None) expected_manifest_sha256 = settings.pop("row_manifest_sha256", None) - if require_manifest and (not row_manifest or not expected_manifest_sha256): + if ( + require_manifest + and evaluation_profile is None + and (not row_manifest or not expected_manifest_sha256) + ): raise ValueError( "frozen 344-row campaign profile requires row_manifest and row_manifest_sha256" ) + if require_manifest and evaluation_profile is not None and not expected_manifest_sha256: + raise ValueError("pinned VLM evaluation profile requires row_manifest_sha256") + if row_manifest is not None and evaluation_profile is not None: + raise ValueError("an embedded evaluation profile cannot be overridden by row_manifest") + if expected_manifest_sha256 is not None and ( + not isinstance(expected_manifest_sha256, str) + or len(expected_manifest_sha256) != 64 + or any(character not in "0123456789abcdef" for character in expected_manifest_sha256) + ): + raise ValueError("pinned VLM manifest SHA256 must be 64 lowercase hex characters") quick_manifest = Path(row_manifest).expanduser().absolute() if row_manifest else None if quick_manifest is not None: - if ( - not isinstance(expected_manifest_sha256, str) - or len(expected_manifest_sha256) != 64 - or any(character not in "0123456789abcdef" for character in expected_manifest_sha256) - ): - raise ValueError( - "frozen 344-row campaign manifest SHA256 must be 64 lowercase hex characters" - ) actual_manifest_sha256 = suites.manifest_sha256(suites.load_quick_manifest(quick_manifest)) if actual_manifest_sha256 != expected_manifest_sha256: raise ValueError( - "frozen 344-row campaign manifest SHA256 differs from the campaign identity: " + "pinned VLM manifest SHA256 differs from the profile identity: " f"{actual_manifest_sha256} != {expected_manifest_sha256}" ) args = argparse.Namespace( checkpoint=Path(checkpoint_path).expanduser().absolute(), output_dir=output_dir, + profile=evaluation_profile, suite=suite, batch_size=int(settings.pop("batch_size", 1)), seed=42, @@ -113,6 +130,14 @@ def _run_profile( profile_path = output_dir / "profile.json" def write_preflight(report: dict[str, object]) -> None: + if evaluation_profile is not None and ( + expected_manifest_sha256 is not None + and report.get("quick_manifest_sha256") != expected_manifest_sha256 + ): + raise ValueError( + "frozen 344-row campaign manifest SHA256 differs from the campaign identity: " + f"{report.get('quick_manifest_sha256')} != {expected_manifest_sha256}" + ) checkpoint.write_generated( profile_path, json.dumps(report, indent=2, sort_keys=True) + "\n", @@ -140,9 +165,25 @@ def register_profiles() -> None: evaluate_realworldqa_mmmu_prefix100_checkpoint, ) register_downstream_evaluation_profile( - _FROZEN_CAMPAIGN_PROFILE, + _FROZEN_CAMPAIGN_PROFILE_V1, evaluate_frozen_campaign_checkpoint, ) + register_downstream_evaluation_profile( + _FROZEN_CAMPAIGN_PROFILE_V2, + evaluate_frozen_campaign_v2_checkpoint, + ) + register_downstream_evaluation_profile( + _FROZEN_CAMPAIGN_PROFILE_V3, + evaluate_frozen_campaign_v3_checkpoint, + ) + register_downstream_evaluation_profile( + _REPRODUCIBILITY_SMOKE_PROFILE, + evaluate_reproducibility_smoke_checkpoint, + ) + register_downstream_evaluation_profile( + _REPRODUCIBILITY_SMOKE_PROFILE_V2, + evaluate_reproducibility_smoke_v2_checkpoint, + ) # Deprecated compatibility aliases. New recipes must use explicit task and # row-selection identities above. register_downstream_evaluation_profile( @@ -184,6 +225,97 @@ def evaluate_frozen_campaign_checkpoint( } +def evaluate_frozen_campaign_v2_checkpoint( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], +) -> dict[str, Any]: + """Evaluate one checkpoint on the current-image frozen campaign profile.""" + + return _evaluate_single_run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + evaluation_profile="core-3_344-examples_r1-native", + ) + + +def _evaluate_single_run_profile( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], + evaluation_profile: str, +) -> dict[str, Any]: + """Evaluate one checkpoint with an embedded single-run profile.""" + + args, result, profile_path = _run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + suite="short", + evaluation_profile=evaluation_profile, + require_manifest=True, + ) + runs = result["runs"] + if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): + raise RuntimeError(f"pinned VLM profile {evaluation_profile} returned an invalid run count") + return { + **runs[0], + "profile_path": str(profile_path), + "checkpoint": str(args.checkpoint), + } + + +def evaluate_frozen_campaign_v3_checkpoint( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], +) -> dict[str, Any]: + """Evaluate heterogeneous materialized checkpoints with the current vLLM profile.""" + + return _evaluate_single_run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + evaluation_profile="core-3_344-examples_r1-vllm", + ) + + +def evaluate_reproducibility_smoke_checkpoint( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], +) -> dict[str, Any]: + """Evaluate one checkpoint on the immutable 24-row lifecycle smoke.""" + + return _evaluate_single_run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + evaluation_profile="core-3_24-examples_r1-native", + ) + + +def evaluate_reproducibility_smoke_v2_checkpoint( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], +) -> dict[str, Any]: + """Evaluate a heterogeneous materialized checkpoint on the 24-row smoke.""" + + return _evaluate_single_run_profile( + checkpoint_path, + output_root=output_root, + settings=settings, + evaluation_profile="core-3_24-examples_r1-vllm", + ) + + def evaluate_short_v1_checkpoint( checkpoint_path: str | Path, *, @@ -193,7 +325,7 @@ def evaluate_short_v1_checkpoint( """Compatibility alias for the explicit frozen-row campaign profile.""" warnings.warn( - f"qwen35_vlm_short_v1 is deprecated; use {_FROZEN_CAMPAIGN_PROFILE}", + f"qwen35_vlm_short_v1 is deprecated; use {_FROZEN_CAMPAIGN_PROFILE_V1}", DeprecationWarning, stacklevel=2, ) @@ -253,6 +385,17 @@ def evaluate_realworldqa_mmmu_prefix100_checkpoint( for name in sorted(metric_names) } result_paths = [str(item["result_path"]) for item in runs] + sample_counts: Counter[str] = Counter() + parser_status_counts: Counter[str] = Counter() + parser_sample_count = 0 + for result_path in result_paths: + payload = json.loads(Path(result_path).read_text()) + sample_counts.update({key: int(value) for key, value in payload["sample_counts"].items()}) + parser_audit = payload.get("mmmu_parser_audit") or {} + parser_sample_count += int(parser_audit.get("sample_count", 0)) + parser_status_counts.update( + {key: int(value) for key, value in (parser_audit.get("status_counts") or {}).items()} + ) summary_path = args.output_dir / "realworldqa_mmmu_prefix100_repeat2_summary.json" atomic_write_json( summary_path, @@ -261,6 +404,11 @@ def evaluate_realworldqa_mmmu_prefix100_checkpoint( "metrics": metrics, "profile": _BOUNDED_REPEATED_PROFILE, "result_paths": result_paths, + "sample_counts": dict(sorted(sample_counts.items())), + "mmmu_parser_audit": { + "sample_count": parser_sample_count, + "status_counts": dict(sorted(parser_status_counts.items())), + }, "suite": args.suite, }, ) diff --git a/examples/puzzletron/evaluation/vlm/preflight.py b/examples/puzzletron/evaluation/vlm/preflight.py index f7b5ca988f4..affa7271896 100644 --- a/examples/puzzletron/evaluation/vlm/preflight.py +++ b/examples/puzzletron/evaluation/vlm/preflight.py @@ -17,7 +17,9 @@ from __future__ import annotations +import hashlib import importlib.util +import json import os from dataclasses import dataclass from pathlib import Path @@ -53,16 +55,53 @@ def _backend_policy( ) -> dict[str, object]: """Return the selected profile's backend contract or the legacy default.""" if profile_contract is None: - return {"name": "vllm"} + return { + "enable_thinking": False, + "name": "vllm", + "reasoning_parser": "qwen3", + } return cast("dict[str, object]", profile_contract.manifest["backend"]) +def _output_budget_contract( + source_tasks: tuple[str, ...], *, model_backend: str +) -> dict[str, dict[str, object]]: + """Describe the output-token budget that each adapter will actually apply.""" + if model_backend == "vllm": + resolution = "max(task_max_new_tokens, model_max_new_tokens_floor=1)" + limitation = ( + "the pinned generic vLLM adapter treats its model-level max_new_tokens as a floor" + ) + else: + resolution = "task_max_new_tokens_overrides_adapter_default" + limitation = None + return { + task: { + "adapter": model_backend, + "effective_max_new_tokens": profile.VLM_BENCHMARK_DATASETS[task].max_new_tokens, + "limitation": limitation, + "requested_max_new_tokens": profile.VLM_BENCHMARK_DATASETS[task].max_new_tokens, + "resolution": resolution, + } + for task in source_tasks + } + + def prepare(args: argparse.Namespace) -> PreparedSuite: """Resolve and validate everything needed before model loading starts.""" profile_name = getattr(args, "profile", None) + if profile_name is not None: + contracts.warn_deprecated_profile(profile_name) profile_contract = contracts.load_profile(profile_name) if profile_name is not None else None suite, source_tasks, profile_task_leaves = _resolve_task_selection(args, profile_contract) - model.verify_checkpoint(args.checkpoint, profile="VLM benchmark") + backend = _backend_policy(profile_contract) + model.verify_checkpoint( + args.checkpoint, + profile="VLM benchmark", + model_backend=str(backend["name"]), + ) + hf_home = _hf_home(args.hf_home) + _verify_profile_model(args.checkpoint, profile_contract, hf_home=hf_home) execution_policy = suites.execution_policy(suite, timeout_seconds=args.timeout_seconds) revisions = {task: profile.VLM_BENCHMARK_DATASETS[task].revision for task in source_tasks} @@ -83,7 +122,6 @@ def prepare(args: argparse.Namespace) -> PreparedSuite: for task in source_tasks: tasks.task_config(profile.VLM_BENCHMARK_DATASETS[task].task_config) - hf_home = _hf_home(args.hf_home) _verify_media_roots(hf_home, source_tasks) dataset_snapshots = { task: suites.offline_dataset_snapshot(hf_home, task, revisions[task]) @@ -138,8 +176,18 @@ def _resolve_task_selection( if profile_task is not None: if profile_contract is None: raise ValueError("--profile-task requires a versioned evaluation profile") - if profile_contract.name not in {"full-v1", "short-all-native-v1"}: - raise ValueError("--profile-task is supported only for full-v1 and short-all-native-v1") + if profile_contract.name not in { + "full-v1", + "core-3_full_r1-native", + "core-3_full_r1-vllm", + "short-all-native-v1", + "judge-free-8_690-examples_r1-native", + }: + raise ValueError( + "--profile-task is supported only for full-v1, core-3_full_r1-native, " + "core-3_full_r1-vllm, short-all-native-v1, and " + "judge-free-8_690-examples_r1-native" + ) if profile_task_shard is not None and profile_task is None: raise ValueError("--profile-task-shard requires --profile-task") @@ -151,6 +199,34 @@ def _resolve_task_selection( return suite, source_tasks, _profile_task_leaves(profile_task, profile_task_shard) +def _verify_profile_model( + checkpoint_path: Path, + profile_contract: contracts.ProfileContract | None, + *, + hf_home: Path, +) -> None: + """Require a model-pinned profile to use its exact local Hub snapshot.""" + if profile_contract is None: + return + model_pin = profile_contract.manifest.get("model") + if not isinstance(model_pin, dict): + return + repository = cast("str", model_pin["repository"]) + revision = cast("str", model_pin["revision"]) + configured_hub_cache = os.environ.get("HF_HUB_CACHE") + hub_cache = ( + Path(configured_hub_cache).expanduser().absolute() + if configured_hub_cache + else (hf_home / "hub").absolute() + ) + repository_cache = f"models--{repository.replace('/', '--')}" + expected_snapshot = hub_cache / repository_cache / "snapshots" / revision + if checkpoint_path.resolve() != expected_snapshot.resolve(): + raise ValueError( + f"{profile_contract.name} requires the exact local Hub snapshot {repository}@{revision}" + ) + + def _row_manifest( args: argparse.Namespace, suite: str, @@ -171,12 +247,11 @@ def _row_manifest( manifest_tasks = cast("dict[str, object]", exact_rows["tasks"]) task_entry = cast("dict[str, object]", manifest_tasks[profile_task]) if profile_task_leaves is not None: - selected_leaves = {f"{profile_task}_{leaf}" for leaf in profile_task_leaves} - rows = cast("list[dict[str, object]]", task_entry["rows"]) - task_entry = { - **task_entry, - "rows": [row for row in rows if row.get("leaf_task") in selected_leaves], - } + task_entry = _shard_exact_row_task( + task_entry, + task=profile_task, + leaves=profile_task_leaves, + ) exact_rows = {**exact_rows, "tasks": {profile_task: task_entry}} return suites.validate_exact_rows_manifest( exact_rows, @@ -190,6 +265,47 @@ def _row_manifest( return None +def _shard_exact_row_task( + entry: dict[str, object], *, task: str, leaves: tuple[str, ...] +) -> dict[str, object]: + """Filter grouped rows and derive a self-consistent sampling audit.""" + selected_leaf_tasks = {f"{task}_{leaf}" for leaf in leaves} + rows = [ + row + for row in cast("list[dict[str, object]]", entry["rows"]) + if row.get("leaf_task") in selected_leaf_tasks + ] + selection = entry.get("selection") + if not isinstance(selection, dict): + return {**entry, "rows": rows} + strata = [ + stratum + for stratum in cast("list[dict[str, object]]", selection["strata"]) + if stratum.get("name") in leaves + ] + indices = sorted(cast("int", row["source_row_index"]) for row in rows) + if not indices: + return {**entry, "rows": rows} + quantiles = { + "method": "lower-order-statistic", + **{ + f"p{percentile}": indices[(len(indices) - 1) * percentile // 100] + for percentile in (0, 25, 50, 75, 100) + }, + } + derived_selection = { + **selection, + "population_rows": sum(cast("int", stratum["population_rows"]) for stratum in strata), + "selected_rows": len(rows), + "strata": strata, + "selected_index_quantiles": quantiles, + "selected_row_identities_sha256": hashlib.sha256( + json.dumps(rows, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest(), + } + return {**entry, "selection": derived_selection, "rows": rows} + + def _profile_task_leaves( profile_task: str | None, shard: tuple[int, int] | None ) -> tuple[str, ...] | None: @@ -294,6 +410,16 @@ def _report( profile_task_shard = getattr(args, "profile_task_shard", None) backend = _backend_policy(profile_contract) model_backend = str(backend["name"]) + output_budget_contract = _output_budget_contract( + source_tasks, + model_backend=model_backend, + ) + profile_population_rows = None + if profile_contract is not None and profile_contract.manifest.get("model") is not None: + profile_tasks = cast("dict[str, dict[str, object]]", profile_contract.manifest["tasks"]) + profile_population_rows = { + task: profile_tasks[task]["population_rows"] for task in source_tasks + } return { "schema": "modelopt.vlm-evaluation-preflight/v1", "profile": suites.EVALUATION_PROFILE, @@ -304,15 +430,30 @@ def _report( "profile_fingerprint": ( profile_contract.fingerprint if profile_contract is not None else None ), + "sample_set": profile_contract.sample_set if profile_contract is not None else None, + "backend_profile": ( + profile_contract.backend_profile if profile_contract is not None else None + ), + "evaluator_profile": ( + profile_contract.evaluator_profile if profile_contract is not None else None + ), + "model_pin": ( + profile_contract.manifest.get("model") if profile_contract is not None else None + ), + "profile_population_rows": profile_population_rows, "suite": suite, "checkpoint": str(args.checkpoint), "lmms_eval_revision": lmms_eval_revision, "model_backend": model_backend, "backend_limitations": ( - ["generic vLLM video messages do not preserve native Qwen 3.5 timestamps"] + [ + "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", + "pinned generic vLLM max_new_tokens is a model-level lower bound", + ] if model_backend == "vllm" else [] ), + "output_budget_contract": output_budget_contract, "source_tasks": list(source_tasks), "profile_task": getattr(args, "profile_task", None), "profile_task_shard": ( @@ -335,6 +476,14 @@ def _report( "quick_selected_rows": ( suites.manifest_selected_rows(quick_manifest) if quick_manifest is not None else None ), + "quick_row_identities": ( + suites.manifest_row_identities(quick_manifest) if quick_manifest is not None else None + ), + "quick_task_denominators": ( + suites.manifest_task_denominators(quick_manifest) + if quick_manifest is not None + else None + ), "judge_free_mmvu_rows": ( [row[0] for row in suites.MMVU_SMOKE_ROWS] if suite == "mmvu-smoke" else None ), @@ -370,7 +519,10 @@ def settings( "batch_size": args.batch_size, "seed": args.seed, "timeout_seconds": execution_policy["timeout_seconds"], - "log_samples": prepared.suite in {"quick", "short", suites.TASK_PREFIX100_REPEAT2_SUITE}, + "log_samples": ( + "mmmu_val" in prepared.source_tasks + or prepared.suite in {"quick", "short", suites.TASK_PREFIX100_REPEAT2_SUITE} + ), "gen_kwargs": { "temperature": generation_policy["temperature"], "do_sample": generation_policy["do_sample"], @@ -393,7 +545,7 @@ def settings( "attn_implementation": backend["attention_implementation"], "device": "cuda", "device_map": "cuda", - "enable_thinking": backend["enable_thinking"], + "enable_thinking": generation_policy["enable_thinking"], "fps": frame_policy["fps"], "max_frames": frame_policy["max_frames"], }, @@ -404,9 +556,18 @@ def settings( "checkpoint_arg": "model", "reasoning_parser": backend.get("reasoning_parser", "qwen3"), "model_args": { + **( + {"attention_config": backend["attention_config"]} + if "attention_config" in backend + else {} + ), "chat_template": str(chat_template), "fps": frame_policy["fps"], + # The pinned adapter takes max(task value, model value). A positive floor of one + # leaves every task-level output budget authoritative. + "max_new_tokens": 1, "max_frame_num": frame_policy["max_frames"], + **({"enforce_eager": backend["enforce_eager"]} if "enforce_eager" in backend else {}), }, } @@ -427,6 +588,6 @@ def _verify_backend_dependencies(model_backend: str) -> None: """Fail before native Qwen evaluation when its vision utilities are unavailable.""" if model_backend == "qwen3_5" and importlib.util.find_spec("qwen_vl_utils") is None: raise RuntimeError( - "native Qwen 3.5 evaluation requires qwen-vl-utils; install the native VLM " - "requirements or use the supported Puzzletron environment" + "native Qwen 3.5 evaluation requires qwen-vl-utils; use the supported " + "Puzzletron worker image" ) diff --git a/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py b/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py index d7e65c51ab9..4c93678d3df 100644 --- a/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py +++ b/examples/puzzletron/evaluation/vlm/preparation/benchmark_data.py @@ -18,6 +18,9 @@ from __future__ import annotations import argparse +import ctypes +import errno +import fcntl import hashlib import json import os @@ -27,9 +30,9 @@ import tarfile import tempfile import zipfile -from contextlib import suppress +from contextlib import contextmanager, suppress from pathlib import Path, PurePosixPath -from typing import IO, TYPE_CHECKING +from typing import IO, TYPE_CHECKING, cast from urllib.parse import urlparse from huggingface_hub import HfApi, get_hf_file_metadata, hf_hub_url, snapshot_download @@ -44,15 +47,19 @@ from examples.puzzletron.evaluation.vlm import profile # noqa: E402 if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Iterable, Iterator _MARKER_NAME = ".modelopt_vlm_benchmark_preparation.json" _RANGE_MARKER_NAME = ".modelopt_vlm_benchmark_range_download.json" +_SNAPSHOT_MARKER_PREFIX = ".modelopt_vlm_benchmark_snapshot_" _TEMPORARY_DIR_NAME = ".modelopt_vlm_benchmark_temporary" +_LOCK_DIRECTORY_NAME = ".modelopt_vlm_benchmark_locks" _COPY_CHUNK_BYTES = 8 * 1024 * 1024 +_AT_FDCWD = -100 +_RENAME_EXCHANGE = 2 -DATASETS = profile.VLM_BENCHMARK_VIDEO_DATASETS +DATASETS = profile.VLM_BENCHMARK_DATASETS _MVBENCH_MEDIA_ROOTS = ( "FunQA_test", @@ -69,6 +76,20 @@ ) +def benchmark_catalog_contract(tasks: Iterable[str]) -> dict[str, dict[str, object]]: + """Return the authoritative preparation contract for selected benchmark tasks.""" + + return { + task: { + "repository": DATASETS[task].repository, + "revision": DATASETS[task].revision, + "requires_media": DATASETS[task].preparation_dir is not None, + "preparation_dir": DATASETS[task].preparation_dir, + } + for task in tasks + } + + def _task_selection(value: str) -> tuple[str, ...]: tasks = tuple(dict.fromkeys(part.strip() for part in value.split(",") if part.strip())) unknown = sorted(set(tasks) - set(DATASETS)) @@ -79,6 +100,10 @@ def _task_selection(value: str) -> tuple[str, ...]: return tasks +def _path_traverses_symlink(path: Path) -> bool: + return any(candidate.is_symlink() for candidate in (path, *path.parents)) + + def _hub_snapshot(hf_home: Path, task: str) -> Path: spec = DATASETS[task] repository_cache = f"datasets--{spec.repository.replace('/', '--')}" @@ -108,6 +133,142 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _inventory_sha256(entries: list[dict[str, object]]) -> str: + payload = json.dumps(entries, separators=(",", ":"), sort_keys=True).encode() + return hashlib.sha256(payload).hexdigest() + + +def _inventory_summary_is_valid(payload: dict[str, object], entries: object) -> bool: + if not isinstance(entries, list) or not entries: + return False + if any( + not isinstance(entry, dict) or not isinstance(entry.get("bytes"), int) for entry in entries + ): + return False + recorded_count = payload.get("file_count") if "file_count" in payload else payload.get("files") + return ( + payload.get("inventory_sha256") == _inventory_sha256(entries) + and recorded_count == len(entries) + and payload.get("bytes") == sum(entry["bytes"] for entry in entries) + ) + + +def _inventory(root: Path, *, repository_cache: Path | None = None) -> list[dict[str, object]]: + entries = [] + for path in sorted(root.rglob("*")): + if path.name == _MARKER_NAME: + continue + if path.is_dir() and not path.is_symlink(): + continue + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + if repository_cache is None: + raise ValueError(f"prepared media inventory contains a symlink: {path}") + resolved = path.resolve(strict=True) + if not resolved.is_relative_to(repository_cache) or not resolved.is_file(): + raise ValueError(f"snapshot inventory path escapes its repository cache: {path}") + stat_result = resolved.stat() + target = resolved.relative_to(repository_cache).as_posix() + blob_name = resolved.name + content_sha256 = _sha256(resolved) + if ( + len(blob_name) == 64 + and all(character in "0123456789abcdef" for character in blob_name.lower()) + and content_sha256 != blob_name.lower() + ): + raise ValueError(f"Hub blob content differs from its SHA-256 identity: {resolved}") + entries.append( + { + "path": relative, + "kind": "hub_blob_symlink", + "target": target, + "bytes": stat_result.st_size, + "mtime_ns": stat_result.st_mtime_ns, + "ctime_ns": stat_result.st_ctime_ns, + "sha256": content_sha256, + } + ) + continue + if not path.is_file(): + raise ValueError(f"inventory path is not a regular file: {path}") + stat_result = path.stat() + entries.append( + { + "path": relative, + "kind": "file", + "bytes": stat_result.st_size, + "mtime_ns": stat_result.st_mtime_ns, + "ctime_ns": stat_result.st_ctime_ns, + "sha256": _sha256(path), + } + ) + if not entries: + raise ValueError(f"inventory root contains no files: {root}") + return entries + + +def _inventory_is_current( + root: Path, + entries: object, + *, + repository_cache: Path | None = None, + verify_content: bool = False, +) -> bool: + if root.is_symlink() or not root.is_dir() or not isinstance(entries, list) or not entries: + return False + expected_paths = [] + for entry in entries: + if not isinstance(entry, dict): + return False + relative = entry.get("path") + if not isinstance(relative, str): + return False + try: + safe_relative = _safe_relative_path(relative) + except ValueError: + return False + path = root.joinpath(*safe_relative.parts) + expected_paths.append(relative) + if entry.get("kind") == "hub_blob_symlink": + if repository_cache is None or not path.is_symlink(): + return False + try: + resolved = path.resolve(strict=True) + except OSError: + return False + if ( + not resolved.is_relative_to(repository_cache) + or not resolved.is_file() + or resolved.relative_to(repository_cache).as_posix() != entry.get("target") + ): + return False + inspected = resolved + elif entry.get("kind") == "file": + if path.is_symlink() or not path.is_file(): + return False + inspected = path + else: + return False + stat_result = inspected.stat() + if ( + stat_result.st_size != entry.get("bytes") + or stat_result.st_mtime_ns != entry.get("mtime_ns") + or stat_result.st_ctime_ns != entry.get("ctime_ns") + ): + return False + expected_sha256 = entry.get("sha256") + if not isinstance(expected_sha256, str) or ( + verify_content and _sha256(inspected) != expected_sha256 + ): + return False + observed_paths = sorted( + path.relative_to(root).as_posix() + for path in root.rglob("*") + if not (path.is_dir() and not path.is_symlink()) and path.name != _MARKER_NAME + ) + return sorted(expected_paths) == observed_paths + + def _lfs_sha256(entry: object) -> str | None: lfs = getattr(entry, "lfs", None) if lfs is None: @@ -125,13 +286,91 @@ def _range_download_marker(hf_home: Path, task: str) -> Path: return hf_home / "hub" / repository_cache / _RANGE_MARKER_NAME +def _snapshot_inventory_marker(hf_home: Path, task: str) -> Path: + snapshot = _hub_snapshot(hf_home, task) + return snapshot.parent.parent / f"{_SNAPSHOT_MARKER_PREFIX}{DATASETS[task].revision}.json" + + +def _snapshot_inventory_report( + hf_home: Path, + task: str, + snapshot: Path, + *, + verify_content: bool = False, +) -> dict[str, object]: + repository_cache = snapshot.parent.parent.resolve() + marker = _snapshot_inventory_marker(hf_home, task) + try: + cached = json.loads(marker.read_text()) + except (OSError, json.JSONDecodeError): + cached = None + expected_identity = { + "schema": "modelopt.vlm-benchmark-snapshot-inventory/v1", + "task": task, + "repository": DATASETS[task].repository, + "revision": DATASETS[task].revision, + "snapshot": str(snapshot), + } + if ( + isinstance(cached, dict) + and all(cached.get(key) == value for key, value in expected_identity.items()) + and _inventory_summary_is_valid(cached, cached.get("files")) + and _inventory_is_current( + snapshot, + cached.get("files"), + repository_cache=repository_cache, + verify_content=verify_content, + ) + ): + return {**cached, "manifest": str(marker)} + + entries = _inventory(snapshot, repository_cache=repository_cache) + report = { + **expected_identity, + "files": entries, + "file_count": len(entries), + "bytes": sum(cast("int", entry["bytes"]) for entry in entries), + "inventory_sha256": _inventory_sha256(entries), + } + _write_json_atomic(marker, report) + return {**report, "manifest": str(marker)} + + +def _snapshot_inventory_is_current(report: object, *, verify_content: bool = True) -> bool: + if not isinstance(report, dict): + return False + snapshot_value = report.get("snapshot") + manifest_value = report.get("manifest") + entries = report.get("files") + if not isinstance(snapshot_value, str) or not isinstance(manifest_value, str): + return False + snapshot = Path(snapshot_value) + repository_cache = snapshot.parent.parent.resolve() + marker = Path(manifest_value) + try: + recorded = json.loads(marker.read_text()) + except (OSError, json.JSONDecodeError): + return False + expected = {key: value for key, value in report.items() if key != "manifest"} + return ( + recorded == expected + and _inventory_summary_is_valid(report, entries) + and _inventory_is_current( + snapshot, + entries, + repository_cache=repository_cache, + verify_content=verify_content, + ) + ) + + def _ensure_directory_path(root: Path, directory: Path) -> None: if not directory.is_relative_to(root): raise ValueError(f"directory escapes its owned root: {directory}") relative = directory.relative_to(root) if ".." in relative.parts: raise ValueError(f"directory escapes its owned root: {directory}") - if root.is_symlink() or not root.is_dir(): + if _path_traverses_symlink(root) or not root.is_dir(): raise ValueError(f"owned root must be a regular directory: {root}") current = root for part in relative.parts: @@ -291,6 +530,8 @@ def _marker_payload(task: str, *, status: str) -> dict[str, object]: "task": task, "repository": spec.repository, "revision": spec.revision, + "requires_media": spec.preparation_dir is not None, + "preparation_dir": spec.preparation_dir, "status": status, } @@ -314,6 +555,30 @@ def _write_marker(target: Path, payload: dict[str, object]) -> None: _write_json_atomic(target / _MARKER_NAME, payload) +@contextmanager +def _task_lock(hf_home: Path, task: str) -> Iterator[None]: + lock_root = hf_home / _LOCK_DIRECTORY_NAME + _ensure_directory_path(hf_home, lock_root) + lock_path = lock_root / f"{task}.lock" + if lock_path.is_symlink(): + raise ValueError(f"benchmark preparation lock must not be a symlink: {lock_path}") + flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(lock_path, flags, 0o600) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ValueError(f"benchmark preparation lock must be a regular file: {lock_path}") + stream = os.fdopen(descriptor, "r+") + except BaseException: + os.close(descriptor) + raise + with stream: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + + def _cleanup_temporary_directory(target: Path) -> None: temporary = target / _TEMPORARY_DIR_NAME if not temporary.exists() and not temporary.is_symlink(): @@ -327,7 +592,73 @@ def _cleanup_temporary_directory(target: Path) -> None: temporary.rmdir() -def _prepare_target(hf_home: Path, task: str) -> tuple[Path, dict[str, object] | None]: +def _remove_owned_media_root(hf_home: Path, target: Path) -> None: + if ( + target.is_symlink() + or not target.is_dir() + or not target.is_relative_to(hf_home) + or target == hf_home + ): + raise ValueError(f"prepared media root is unsafe to repair: {target}") + for entry in target.rglob("*"): + if entry.is_symlink(): + raise ValueError(f"prepared media repair refuses a symlink: {entry}") + shutil.rmtree(target) + + +def _atomic_exchange_directories(first: Path, second: Path) -> bool: + """Atomically exchange two directories when the host provides renameat2.""" + + renameat2 = getattr(ctypes.CDLL(None, use_errno=True), "renameat2", None) + if renameat2 is None: + return False + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + _AT_FDCWD, + os.fsencode(first), + _AT_FDCWD, + os.fsencode(second), + _RENAME_EXCHANGE, + ) + if result == 0: + return True + error_number = ctypes.get_errno() + if error_number in {errno.ENOSYS, errno.EINVAL, errno.ENOTSUP}: + return False + raise OSError(error_number, os.strerror(error_number)) + + +def _media_marker_is_current( + target: Path, + task: str, + observed: object, + *, + verify_content: bool = False, +) -> bool: + if not isinstance(observed, dict): + return False + expected = _marker_payload(task, status="complete") + if any(observed.get(key) != value for key, value in expected.items()): + return False + inventory = observed.get("inventory") + return _inventory_summary_is_valid(observed, inventory) and _inventory_is_current( + target, inventory, verify_content=verify_content + ) + + +def _inspect_prepare_target( + hf_home: Path, + task: str, + *, + verify_content: bool = False, +) -> tuple[Path, dict[str, object] | None]: preparation_dir = DATASETS[task].preparation_dir if preparation_dir is None: raise AssertionError(f"video dataset has no preparation directory: {task}") @@ -342,23 +673,22 @@ def _prepare_target(hf_home: Path, task: str) -> tuple[Path, dict[str, object] | observed = json.loads(marker.read_text()) except (FileNotFoundError, json.JSONDecodeError) as error: raise FileExistsError( - f"refusing a pre-existing media root without a valid marker: {target}" + f"refusing to repair a media root without a readable ownership marker: {target}" ) from error - for key in ("schema", "task", "repository", "revision"): - if observed.get(key) != expected[key]: - raise ValueError(f"preparation marker differs for {task}: {key}") - if observed.get("status") == "complete": + if _media_marker_is_current(target, task, observed, verify_content=verify_content): return target, observed - if observed.get("status") != "in_progress": - raise ValueError(f"preparation marker has an invalid status for {task}") - return target, None - staging = Path(tempfile.mkdtemp(prefix=f".{target.name}.modelopt-staging.", dir=target.parent)) - try: - _write_marker(staging, expected) - os.replace(staging, target) - finally: - with suppress(FileNotFoundError, OSError): - staging.rmdir() + if not isinstance(observed, dict): + raise FileExistsError(f"media root ownership marker must be an object: {marker}") + for key, value in expected.items(): + if key == "status": + continue + if observed.get(key) != value: + raise ValueError(f"refusing to repair media with mismatched ownership: {key}") + if observed.get("status") not in {"in_progress", "complete"}: + raise ValueError("refusing to repair media with an invalid ownership-marker status") + for entry in target.rglob("*"): + if entry.is_symlink(): + raise ValueError(f"prepared media repair refuses a symlink: {entry}") return target, None @@ -623,24 +953,50 @@ def _extract(task: str, snapshot: Path, target: Path) -> list[dict[str, object]] raise ValueError(f"unsupported VLM benchmark data task: {task}") -def _prepare(hf_home: Path, task: str, snapshot: Path) -> dict[str, object]: - target, complete = _prepare_target(hf_home, task) - if complete is not None: - return complete - _cleanup_temporary_directory(target) - archives = _extract(task, snapshot, target) - _cleanup_temporary_directory(target) - files = [path for path in target.rglob("*") if path.is_file() and path.name != _MARKER_NAME] - report = { - **_marker_payload(task, status="complete"), - "snapshot": str(snapshot), - "media_root": str(target), - "archives": archives, - "files": len(files), - "bytes": sum(path.stat().st_size for path in files), - } - _write_marker(target, report) - return report +def _prepare( + hf_home: Path, + task: str, + snapshot: Path, + *, + verify_content: bool = False, +) -> dict[str, object]: + with _task_lock(hf_home, task): + target, complete = _inspect_prepare_target(hf_home, task, verify_content=verify_content) + if complete is not None: + return complete + staging = Path( + tempfile.mkdtemp(prefix=f".{target.name}.modelopt-staging.", dir=target.parent) + ) + staging.chmod(0o755) + try: + _write_marker(staging, _marker_payload(task, status="in_progress")) + archives = _extract(task, snapshot, staging) + _cleanup_temporary_directory(staging) + inventory = _inventory(staging) + report = { + **_marker_payload(task, status="complete"), + "snapshot": str(snapshot), + "media_root": str(target), + "archives": archives, + "files": len(inventory), + "bytes": sum(cast("int", entry["bytes"]) for entry in inventory), + "inventory": inventory, + "inventory_sha256": _inventory_sha256(inventory), + } + _write_marker(staging, report) + if target.exists(): + if not _atomic_exchange_directories(staging, target): + raise RuntimeError( + "atomic media-directory exchange is unavailable; " + "the existing prepared root was preserved" + ) + _remove_owned_media_root(hf_home, staging) + else: + os.replace(staging, target) + return report + finally: + if staging.exists(): + _remove_owned_media_root(hf_home, staging) def _build_parser() -> argparse.ArgumentParser: @@ -661,38 +1017,134 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Use a deterministic single-writer HTTP range download that resumes across jobs", ) + parser.add_argument( + "--verify-content", + action="store_true", + help="Re-hash cached snapshot and prepared-media files instead of trusting unchanged metadata", + ) return parser -def main(argv: list[str] | None = None) -> int: - args = _build_parser().parse_args(argv) - hf_home = args.hf_home.expanduser().absolute() - if hf_home.is_symlink(): +def prepare_benchmark_datasets( + hf_home: Path, + tasks: Iterable[str], + *, + max_workers: int = 8, + range_resume: bool = False, + verify_content: bool = False, + expected_catalog: dict[str, object] | None = None, +) -> list[dict[str, object]]: + """Download pinned benchmark snapshots and prepare media when required.""" + + if max_workers <= 0: + raise ValueError("max_workers must be positive") + if isinstance(tasks, str): + raise TypeError("tasks must be an iterable of task names, not a string") + hf_home = hf_home.expanduser().absolute() + if _path_traverses_symlink(hf_home): raise ValueError(f"HF home must not be a symlink: {hf_home}") hf_home.mkdir(parents=True, exist_ok=True) + raw_tasks = tuple(tasks) + if any(not isinstance(task, str) or not task.strip() for task in raw_tasks): + raise TypeError("tasks must contain non-empty task names") + selected = tuple(task.strip() for task in raw_tasks) + if len(set(selected)) != len(selected): + raise ValueError("VLM benchmark data tasks must be unique") + unknown = sorted(set(selected) - set(DATASETS)) + if not selected: + raise ValueError("at least one VLM benchmark data task is required") + if unknown: + raise ValueError(f"unsupported VLM benchmark data tasks: {unknown}") + catalog = benchmark_catalog_contract(selected) + if expected_catalog is not None and expected_catalog != catalog: + raise ValueError("configured VLM benchmark catalog differs from the authoritative catalog") + reports = [] - for task in args.tasks: + for task in selected: snapshot = ( - _hub_snapshot(hf_home, task) - if args.extract_only - else ( - _range_download(hf_home, task) - if args.range_resume - else _download(hf_home, task, max_workers=args.max_workers) - ) + _range_download(hf_home, task) + if range_resume + else _download(hf_home, task, max_workers=max_workers) ) if not snapshot.is_dir(): raise FileNotFoundError(f"pinned dataset snapshot is missing: {snapshot}") + spec = DATASETS[task] + snapshot_inventory = _snapshot_inventory_report( + hf_home, task, snapshot, verify_content=verify_content + ) report: dict[str, object] = { "task": task, - "repository": DATASETS[task].repository, - "revision": DATASETS[task].revision, + "repository": spec.repository, + "revision": spec.revision, "snapshot": str(snapshot), + "requires_media": spec.preparation_dir is not None, + "preparation_dir": spec.preparation_dir, + "snapshot_inventory": snapshot_inventory, "status": "downloaded", } - if not args.download_only: - report = _prepare(hf_home, task, snapshot) + if spec.preparation_dir is not None: + report = { + **report, + **_prepare(hf_home, task, snapshot, verify_content=verify_content), + } reports.append(report) + return reports + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + hf_home = args.hf_home.expanduser().absolute() + if _path_traverses_symlink(hf_home): + raise ValueError(f"HF home must not be a symlink: {hf_home}") + hf_home.mkdir(parents=True, exist_ok=True) + if args.extract_only: + reports = [] + for task in args.tasks: + snapshot = _hub_snapshot(hf_home, task) + if not snapshot.is_dir(): + raise FileNotFoundError(f"pinned dataset snapshot is missing: {snapshot}") + spec = DATASETS[task] + report: dict[str, object] = { + "task": task, + "repository": spec.repository, + "revision": spec.revision, + "snapshot": str(snapshot), + "requires_media": spec.preparation_dir is not None, + "status": "downloaded", + } + if spec.preparation_dir is not None: + report = { + **report, + **_prepare(hf_home, task, snapshot, verify_content=args.verify_content), + } + reports.append(report) + elif args.download_only: + reports = [] + for task in args.tasks: + snapshot = ( + _range_download(hf_home, task) + if args.range_resume + else _download(hf_home, task, max_workers=args.max_workers) + ) + spec = DATASETS[task] + reports.append( + { + "task": task, + "repository": spec.repository, + "revision": spec.revision, + "snapshot": str(snapshot), + "requires_media": spec.preparation_dir is not None, + "status": "downloaded", + } + ) + else: + reports = prepare_benchmark_datasets( + hf_home, + args.tasks, + max_workers=args.max_workers, + range_resume=args.range_resume, + verify_content=args.verify_content, + ) print(json.dumps({"hf_home": str(hf_home), "tasks": reports}, indent=2, sort_keys=True)) return 0 diff --git a/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm-eager_r1.json b/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm-eager_r1.json new file mode 100644 index 00000000000..05eab6d9138 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm-eager_r1.json @@ -0,0 +1,13 @@ +{ + "schema": "modelopt.vlm-backend-profile/v1", + "name": "anymodel-vllm-eager_r1", + "settings": { + "attention_config": { + "flash_attn_version": 2 + }, + "enable_thinking": false, + "enforce_eager": true, + "name": "vllm", + "reasoning_parser": "qwen3" + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm_r1.json b/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm_r1.json new file mode 100644 index 00000000000..c7d24f57363 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/backends/anymodel-vllm_r1.json @@ -0,0 +1,12 @@ +{ + "schema": "modelopt.vlm-backend-profile/v1", + "name": "anymodel-vllm_r1", + "settings": { + "attention_config": { + "flash_attn_version": 2 + }, + "enable_thinking": false, + "name": "vllm", + "reasoning_parser": "qwen3" + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-native_r1.json b/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-native_r1.json new file mode 100644 index 00000000000..c1b9d493473 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-native_r1.json @@ -0,0 +1,9 @@ +{ + "schema": "modelopt.vlm-backend-profile/v1", + "name": "qwen-3.5-native_r1", + "settings": { + "attention_implementation": "sdpa", + "enable_thinking": false, + "name": "qwen3_5" + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-vllm_r1.json b/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-vllm_r1.json new file mode 100644 index 00000000000..d18c494ab7e --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/backends/qwen-3.5-vllm_r1.json @@ -0,0 +1,9 @@ +{ + "schema": "modelopt.vlm-backend-profile/v1", + "name": "qwen-3.5-vllm_r1", + "settings": { + "enable_thinking": false, + "name": "vllm", + "reasoning_parser": "qwen3" + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-native.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-native.json new file mode 100644 index 00000000000..5803071aa53 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-native.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_24-examples_r1-native", + "sample_set": "core-3_24-examples_r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-vllm.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-vllm.json new file mode 100644 index 00000000000..5f56da0e5dd --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_24-examples_r1-vllm.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_24-examples_r1-vllm", + "sample_set": "core-3_24-examples_r1", + "backend_profile": "anymodel-vllm-eager_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-native.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-native.json new file mode 100644 index 00000000000..6df08f73487 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-native.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_344-examples_r1-native", + "sample_set": "core-3_344-examples_r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-vllm.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-vllm.json new file mode 100644 index 00000000000..eceb3eaccdd --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_344-examples_r1-vllm.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_344-examples_r1-vllm", + "sample_set": "core-3_344-examples_r1", + "backend_profile": "anymodel-vllm_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-native.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-native.json new file mode 100644 index 00000000000..bfe5219df26 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-native.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_full_r1-native", + "sample_set": "core-3_full_r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-vllm.json b/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-vllm.json new file mode 100644 index 00000000000..785e6512df2 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/core-3_full_r1-vllm.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "core-3_full_r1-vllm", + "sample_set": "core-3_full_r1", + "backend_profile": "qwen-3.5-vllm_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-modelopt_r1.json b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-modelopt_r1.json new file mode 100644 index 00000000000..a563853acda --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/evaluators/lmms-eval-modelopt_r1.json @@ -0,0 +1,5 @@ +{ + "schema": "modelopt.vlm-evaluator-profile/v1", + "name": "lmms-eval-modelopt_r1", + "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/full-v1.json b/examples/puzzletron/evaluation/vlm/profiles/full-v1.json index cd8315703de..0497f432f83 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/full-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/full-v1.json @@ -1,77 +1,7 @@ { - "schema": "modelopt.vlm-evaluation-profile/v1", + "schema": "modelopt.vlm-evaluation-profile/v2", "name": "full-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", - "model_family": { - "architecture": "Qwen3_5ForConditionalGeneration", - "model_type": "qwen3_5" - }, - "backend": { - "enable_thinking": false, - "name": "vllm", - "reasoning_parser": "qwen3" - }, - "preprocessing": { - "fps": 2, - "max_frames": 32, - "video_reader": "decord" - }, - "generation": { - "do_sample": false, - "temperature": 0 - }, - "seed": 42, - "repetitions": 1, - "batch_size": 1, - "selection": "all", - "tasks": { - "realworldqa": { - "dataset_repository": "lmms-lab/RealWorldQA", - "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", - "max_new_tokens": 16, - "scoring_task_config": "tasks/realworldqa/realworldqa.yaml" - }, - "mmmu_val": { - "dataset_repository": "lmms-lab/MMMU", - "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", - "max_new_tokens": 128, - "scoring_task_config": "tasks/mmmu/mmmu_val.yaml" - }, - "video_mmmu": { - "dataset_repository": "lmms-lab/VideoMMMU", - "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", - "max_new_tokens": 1024, - "scoring_task_config": "tasks/videommmu/video_mmmu.yaml" - }, - "mvbench": { - "dataset_repository": "OpenGVLab/MVBench", - "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", - "max_new_tokens": 16, - "scoring_task_config": "tasks/mvbench/mvbench.yaml" - }, - "videomme": { - "dataset_repository": "lmms-lab/Video-MME", - "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", - "max_new_tokens": 16, - "scoring_task_config": "tasks/videomme/videomme.yaml" - }, - "longvideobench_val_v": { - "dataset_repository": "longvideobench/LongVideoBench", - "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", - "max_new_tokens": 32, - "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml" - }, - "mlvu_dev": { - "dataset_repository": "sy1998/MLVU_dev", - "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", - "max_new_tokens": 16, - "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml" - }, - "perceptiontest_val_mc": { - "dataset_repository": "lmms-lab/PerceptionTest_Val", - "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", - "max_new_tokens": 16, - "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml" - } - } + "sample_set": "judge-free-8_full_legacy-r1", + "backend_profile": "qwen-3.5-vllm_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" } diff --git a/examples/puzzletron/evaluation/vlm/profiles/judge-free-8_690-examples_r1-native.json b/examples/puzzletron/evaluation/vlm/profiles/judge-free-8_690-examples_r1-native.json new file mode 100644 index 00000000000..81a03c06544 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/judge-free-8_690-examples_r1-native.json @@ -0,0 +1,7 @@ +{ + "schema": "modelopt.vlm-evaluation-profile/v2", + "name": "judge-free-8_690-examples_r1-native", + "sample_set": "judge-free-8_690-examples_r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_24-examples_r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_24-examples_r1.json new file mode 100644 index 00000000000..b234f3b6449 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_24-examples_r1.json @@ -0,0 +1,129 @@ +{ + "extends": "core-3_344-examples_r1", + "schema": "modelopt.vlm-sample-set/v1", + "name": "core-3_24-examples_r1", + "tasks": { + "realworldqa": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "split", + "index_space": "within-stratum", + "population_rows": 765, + "selected_rows": 8, + "strata": [ + {"name": "test", "population_rows": 765, "selected_rows": 8} + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 47, + "p25": 143, + "p50": 334, + "p75": 525, + "p100": 717 + }, + "selected_row_identities_sha256": "18cfff65ce81ca45537a6de3f17184ed51d23c24247e364b5a565a8ef368f70c" + }, + "rows": [ + {"source_row_index": 47, "source_sample_id": "test:47"}, + {"source_row_index": 143, "source_sample_id": "test:143"}, + {"source_row_index": 239, "source_sample_id": "test:239"}, + {"source_row_index": 334, "source_sample_id": "test:334"}, + {"source_row_index": 430, "source_sample_id": "test:430"}, + {"source_row_index": 525, "source_sample_id": "test:525"}, + {"source_row_index": 621, "source_sample_id": "test:621"}, + {"source_row_index": 717, "source_sample_id": "test:717"} + ] + }, + "mmmu_val": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "subject", + "index_space": "within-stratum", + "population_rows": 900, + "selected_rows": 8, + "strata": [ + {"name": "Accounting", "population_rows": 30, "selected_rows": 0}, + {"name": "Agriculture", "population_rows": 30, "selected_rows": 1}, + {"name": "Architecture_and_Engineering", "population_rows": 30, "selected_rows": 0}, + {"name": "Art", "population_rows": 30, "selected_rows": 0}, + {"name": "Art_Theory", "population_rows": 30, "selected_rows": 0}, + {"name": "Basic_Medical_Science", "population_rows": 30, "selected_rows": 1}, + {"name": "Biology", "population_rows": 30, "selected_rows": 0}, + {"name": "Chemistry", "population_rows": 30, "selected_rows": 0}, + {"name": "Clinical_Medicine", "population_rows": 30, "selected_rows": 0}, + {"name": "Computer_Science", "population_rows": 30, "selected_rows": 1}, + {"name": "Design", "population_rows": 30, "selected_rows": 0}, + {"name": "Diagnostics_and_Laboratory_Medicine", "population_rows": 30, "selected_rows": 0}, + {"name": "Economics", "population_rows": 30, "selected_rows": 0}, + {"name": "Electronics", "population_rows": 30, "selected_rows": 1}, + {"name": "Energy_and_Power", "population_rows": 30, "selected_rows": 0}, + {"name": "Finance", "population_rows": 30, "selected_rows": 0}, + {"name": "Geography", "population_rows": 30, "selected_rows": 1}, + {"name": "History", "population_rows": 30, "selected_rows": 0}, + {"name": "Literature", "population_rows": 30, "selected_rows": 0}, + {"name": "Manage", "population_rows": 30, "selected_rows": 0}, + {"name": "Marketing", "population_rows": 30, "selected_rows": 1}, + {"name": "Materials", "population_rows": 30, "selected_rows": 0}, + {"name": "Math", "population_rows": 30, "selected_rows": 0}, + {"name": "Mechanical_Engineering", "population_rows": 30, "selected_rows": 0}, + {"name": "Music", "population_rows": 30, "selected_rows": 1}, + {"name": "Pharmacy", "population_rows": 30, "selected_rows": 0}, + {"name": "Physics", "population_rows": 30, "selected_rows": 0}, + {"name": "Psychology", "population_rows": 30, "selected_rows": 0}, + {"name": "Public_Health", "population_rows": 30, "selected_rows": 1}, + {"name": "Sociology", "population_rows": 30, "selected_rows": 0} + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 15, + "p25": 15, + "p50": 15, + "p75": 15, + "p100": 15 + }, + "selected_row_identities_sha256": "9b2488fac1079e6c920455c9dda9eee5755229868bf710bd1b75902434fdb702" + }, + "rows": [ + {"source_row_index": 45, "source_sample_id": "validation_Agriculture_16"}, + {"source_row_index": 165, "source_sample_id": "validation_Basic_Medical_Science_16"}, + {"source_row_index": 285, "source_sample_id": "validation_Computer_Science_16"}, + {"source_row_index": 405, "source_sample_id": "validation_Electronics_16"}, + {"source_row_index": 495, "source_sample_id": "validation_Geography_16"}, + {"source_row_index": 615, "source_sample_id": "validation_Marketing_16"}, + {"source_row_index": 735, "source_sample_id": "validation_Music_16"}, + {"source_row_index": 855, "source_sample_id": "validation_Public_Health_16"} + ] + }, + "mvbench": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "leaf_task", + "index_space": "within-stratum", + "population_rows": 200, + "selected_rows": 8, + "strata": [ + {"name": "action_sequence", "population_rows": 200, "selected_rows": 8} + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 12, + "p25": 37, + "p50": 87, + "p75": 137, + "p100": 187 + }, + "selected_row_identities_sha256": "45b0dfeef2216c9741aecd10cbd68b7f1c83f0dbbc71c5b3d01752d9191be0b6" + }, + "rows": [ + {"source_row_index": 12, "source_sample_id": "action_sequence:12", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 37, "source_sample_id": "action_sequence:37", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 62, "source_sample_id": "action_sequence:62", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 87, "source_sample_id": "action_sequence:87", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 112, "source_sample_id": "action_sequence:112", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 137, "source_sample_id": "action_sequence:137", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 162, "source_sample_id": "action_sequence:162", "leaf_task": "mvbench_action_sequence"}, + {"source_row_index": 187, "source_sample_id": "action_sequence:187", "leaf_task": "mvbench_action_sequence"} + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_legacy-r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_legacy-r1.json new file mode 100644 index 00000000000..5a940cf34a7 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_legacy-r1.json @@ -0,0 +1,1567 @@ +{ + "schema": "modelopt.vlm-sample-set/v1", + "name": "core-3_344-examples_legacy-r1", + "selection": "exact-rows", + "tasks": { + "realworldqa": { + "dataset_repository": "lmms-lab/RealWorldQA", + "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", + "max_new_tokens": 16, + "scoring_task_config": "tasks/realworldqa/realworldqa.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "test:0" + }, + { + "source_row_index": 1, + "source_sample_id": "test:1" + }, + { + "source_row_index": 2, + "source_sample_id": "test:2" + }, + { + "source_row_index": 3, + "source_sample_id": "test:3" + }, + { + "source_row_index": 4, + "source_sample_id": "test:4" + }, + { + "source_row_index": 5, + "source_sample_id": "test:5" + }, + { + "source_row_index": 6, + "source_sample_id": "test:6" + }, + { + "source_row_index": 7, + "source_sample_id": "test:7" + }, + { + "source_row_index": 8, + "source_sample_id": "test:8" + }, + { + "source_row_index": 9, + "source_sample_id": "test:9" + }, + { + "source_row_index": 10, + "source_sample_id": "test:10" + }, + { + "source_row_index": 11, + "source_sample_id": "test:11" + }, + { + "source_row_index": 12, + "source_sample_id": "test:12" + }, + { + "source_row_index": 13, + "source_sample_id": "test:13" + }, + { + "source_row_index": 14, + "source_sample_id": "test:14" + }, + { + "source_row_index": 15, + "source_sample_id": "test:15" + }, + { + "source_row_index": 16, + "source_sample_id": "test:16" + }, + { + "source_row_index": 17, + "source_sample_id": "test:17" + }, + { + "source_row_index": 18, + "source_sample_id": "test:18" + }, + { + "source_row_index": 19, + "source_sample_id": "test:19" + }, + { + "source_row_index": 20, + "source_sample_id": "test:20" + }, + { + "source_row_index": 21, + "source_sample_id": "test:21" + }, + { + "source_row_index": 22, + "source_sample_id": "test:22" + }, + { + "source_row_index": 23, + "source_sample_id": "test:23" + }, + { + "source_row_index": 24, + "source_sample_id": "test:24" + }, + { + "source_row_index": 25, + "source_sample_id": "test:25" + }, + { + "source_row_index": 26, + "source_sample_id": "test:26" + }, + { + "source_row_index": 27, + "source_sample_id": "test:27" + }, + { + "source_row_index": 28, + "source_sample_id": "test:28" + }, + { + "source_row_index": 29, + "source_sample_id": "test:29" + }, + { + "source_row_index": 30, + "source_sample_id": "test:30" + }, + { + "source_row_index": 31, + "source_sample_id": "test:31" + }, + { + "source_row_index": 32, + "source_sample_id": "test:32" + }, + { + "source_row_index": 33, + "source_sample_id": "test:33" + }, + { + "source_row_index": 34, + "source_sample_id": "test:34" + }, + { + "source_row_index": 35, + "source_sample_id": "test:35" + }, + { + "source_row_index": 36, + "source_sample_id": "test:36" + }, + { + "source_row_index": 37, + "source_sample_id": "test:37" + }, + { + "source_row_index": 38, + "source_sample_id": "test:38" + }, + { + "source_row_index": 39, + "source_sample_id": "test:39" + }, + { + "source_row_index": 40, + "source_sample_id": "test:40" + }, + { + "source_row_index": 41, + "source_sample_id": "test:41" + }, + { + "source_row_index": 42, + "source_sample_id": "test:42" + }, + { + "source_row_index": 43, + "source_sample_id": "test:43" + }, + { + "source_row_index": 44, + "source_sample_id": "test:44" + }, + { + "source_row_index": 45, + "source_sample_id": "test:45" + }, + { + "source_row_index": 46, + "source_sample_id": "test:46" + }, + { + "source_row_index": 47, + "source_sample_id": "test:47" + }, + { + "source_row_index": 48, + "source_sample_id": "test:48" + }, + { + "source_row_index": 49, + "source_sample_id": "test:49" + }, + { + "source_row_index": 50, + "source_sample_id": "test:50" + }, + { + "source_row_index": 51, + "source_sample_id": "test:51" + }, + { + "source_row_index": 52, + "source_sample_id": "test:52" + }, + { + "source_row_index": 53, + "source_sample_id": "test:53" + }, + { + "source_row_index": 54, + "source_sample_id": "test:54" + }, + { + "source_row_index": 55, + "source_sample_id": "test:55" + }, + { + "source_row_index": 56, + "source_sample_id": "test:56" + }, + { + "source_row_index": 57, + "source_sample_id": "test:57" + }, + { + "source_row_index": 58, + "source_sample_id": "test:58" + }, + { + "source_row_index": 59, + "source_sample_id": "test:59" + }, + { + "source_row_index": 60, + "source_sample_id": "test:60" + }, + { + "source_row_index": 61, + "source_sample_id": "test:61" + }, + { + "source_row_index": 62, + "source_sample_id": "test:62" + }, + { + "source_row_index": 63, + "source_sample_id": "test:63" + } + ] + }, + "mmmu_val": { + "dataset_repository": "lmms-lab/MMMU", + "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", + "max_new_tokens": 128, + "scoring_task_config": "tasks/mmmu/mmmu_val.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "validation_Accounting_1" + }, + { + "source_row_index": 7, + "source_sample_id": "validation_Accounting_8" + }, + { + "source_row_index": 15, + "source_sample_id": "validation_Accounting_16" + }, + { + "source_row_index": 22, + "source_sample_id": "validation_Accounting_23" + }, + { + "source_row_index": 30, + "source_sample_id": "validation_Agriculture_1" + }, + { + "source_row_index": 37, + "source_sample_id": "validation_Agriculture_8" + }, + { + "source_row_index": 45, + "source_sample_id": "validation_Agriculture_16" + }, + { + "source_row_index": 52, + "source_sample_id": "validation_Agriculture_23" + }, + { + "source_row_index": 60, + "source_sample_id": "validation_Architecture_and_Engineering_1" + }, + { + "source_row_index": 67, + "source_sample_id": "validation_Architecture_and_Engineering_8" + }, + { + "source_row_index": 75, + "source_sample_id": "validation_Architecture_and_Engineering_16" + }, + { + "source_row_index": 82, + "source_sample_id": "validation_Architecture_and_Engineering_23" + }, + { + "source_row_index": 90, + "source_sample_id": "validation_Art_1" + }, + { + "source_row_index": 97, + "source_sample_id": "validation_Art_8" + }, + { + "source_row_index": 105, + "source_sample_id": "validation_Art_16" + }, + { + "source_row_index": 112, + "source_sample_id": "validation_Art_23" + }, + { + "source_row_index": 120, + "source_sample_id": "validation_Art_Theory_1" + }, + { + "source_row_index": 127, + "source_sample_id": "validation_Art_Theory_8" + }, + { + "source_row_index": 135, + "source_sample_id": "validation_Art_Theory_16" + }, + { + "source_row_index": 142, + "source_sample_id": "validation_Art_Theory_23" + }, + { + "source_row_index": 150, + "source_sample_id": "validation_Basic_Medical_Science_1" + }, + { + "source_row_index": 157, + "source_sample_id": "validation_Basic_Medical_Science_8" + }, + { + "source_row_index": 165, + "source_sample_id": "validation_Basic_Medical_Science_16" + }, + { + "source_row_index": 172, + "source_sample_id": "validation_Basic_Medical_Science_23" + }, + { + "source_row_index": 180, + "source_sample_id": "validation_Biology_1" + }, + { + "source_row_index": 187, + "source_sample_id": "validation_Biology_8" + }, + { + "source_row_index": 195, + "source_sample_id": "validation_Biology_16" + }, + { + "source_row_index": 202, + "source_sample_id": "validation_Biology_23" + }, + { + "source_row_index": 210, + "source_sample_id": "validation_Chemistry_1" + }, + { + "source_row_index": 217, + "source_sample_id": "validation_Chemistry_8" + }, + { + "source_row_index": 225, + "source_sample_id": "validation_Chemistry_16" + }, + { + "source_row_index": 232, + "source_sample_id": "validation_Chemistry_23" + }, + { + "source_row_index": 240, + "source_sample_id": "validation_Clinical_Medicine_1" + }, + { + "source_row_index": 247, + "source_sample_id": "validation_Clinical_Medicine_8" + }, + { + "source_row_index": 255, + "source_sample_id": "validation_Clinical_Medicine_16" + }, + { + "source_row_index": 262, + "source_sample_id": "validation_Clinical_Medicine_23" + }, + { + "source_row_index": 270, + "source_sample_id": "validation_Computer_Science_1" + }, + { + "source_row_index": 277, + "source_sample_id": "validation_Computer_Science_8" + }, + { + "source_row_index": 285, + "source_sample_id": "validation_Computer_Science_16" + }, + { + "source_row_index": 292, + "source_sample_id": "validation_Computer_Science_23" + }, + { + "source_row_index": 300, + "source_sample_id": "validation_Design_1" + }, + { + "source_row_index": 307, + "source_sample_id": "validation_Design_8" + }, + { + "source_row_index": 315, + "source_sample_id": "validation_Design_16" + }, + { + "source_row_index": 322, + "source_sample_id": "validation_Design_23" + }, + { + "source_row_index": 330, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_1" + }, + { + "source_row_index": 337, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_8" + }, + { + "source_row_index": 345, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_16" + }, + { + "source_row_index": 352, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_23" + }, + { + "source_row_index": 360, + "source_sample_id": "validation_Economics_1" + }, + { + "source_row_index": 367, + "source_sample_id": "validation_Economics_8" + }, + { + "source_row_index": 375, + "source_sample_id": "validation_Economics_16" + }, + { + "source_row_index": 382, + "source_sample_id": "validation_Economics_23" + }, + { + "source_row_index": 390, + "source_sample_id": "validation_Electronics_1" + }, + { + "source_row_index": 397, + "source_sample_id": "validation_Electronics_8" + }, + { + "source_row_index": 405, + "source_sample_id": "validation_Electronics_16" + }, + { + "source_row_index": 412, + "source_sample_id": "validation_Electronics_23" + }, + { + "source_row_index": 420, + "source_sample_id": "validation_Energy_and_Power_1" + }, + { + "source_row_index": 427, + "source_sample_id": "validation_Energy_and_Power_8" + }, + { + "source_row_index": 435, + "source_sample_id": "validation_Energy_and_Power_16" + }, + { + "source_row_index": 442, + "source_sample_id": "validation_Energy_and_Power_23" + }, + { + "source_row_index": 450, + "source_sample_id": "validation_Finance_1" + }, + { + "source_row_index": 457, + "source_sample_id": "validation_Finance_8" + }, + { + "source_row_index": 465, + "source_sample_id": "validation_Finance_16" + }, + { + "source_row_index": 472, + "source_sample_id": "validation_Finance_23" + }, + { + "source_row_index": 480, + "source_sample_id": "validation_Geography_1" + }, + { + "source_row_index": 487, + "source_sample_id": "validation_Geography_8" + }, + { + "source_row_index": 495, + "source_sample_id": "validation_Geography_16" + }, + { + "source_row_index": 502, + "source_sample_id": "validation_Geography_23" + }, + { + "source_row_index": 510, + "source_sample_id": "validation_History_1" + }, + { + "source_row_index": 517, + "source_sample_id": "validation_History_8" + }, + { + "source_row_index": 525, + "source_sample_id": "validation_History_16" + }, + { + "source_row_index": 532, + "source_sample_id": "validation_History_23" + }, + { + "source_row_index": 540, + "source_sample_id": "validation_Literature_1" + }, + { + "source_row_index": 547, + "source_sample_id": "validation_Literature_8" + }, + { + "source_row_index": 555, + "source_sample_id": "validation_Literature_16" + }, + { + "source_row_index": 562, + "source_sample_id": "validation_Literature_23" + }, + { + "source_row_index": 570, + "source_sample_id": "validation_Manage_1" + }, + { + "source_row_index": 577, + "source_sample_id": "validation_Manage_8" + }, + { + "source_row_index": 585, + "source_sample_id": "validation_Manage_16" + }, + { + "source_row_index": 592, + "source_sample_id": "validation_Manage_23" + }, + { + "source_row_index": 600, + "source_sample_id": "validation_Marketing_1" + }, + { + "source_row_index": 607, + "source_sample_id": "validation_Marketing_8" + }, + { + "source_row_index": 615, + "source_sample_id": "validation_Marketing_16" + }, + { + "source_row_index": 622, + "source_sample_id": "validation_Marketing_23" + }, + { + "source_row_index": 630, + "source_sample_id": "validation_Materials_1" + }, + { + "source_row_index": 637, + "source_sample_id": "validation_Materials_8" + }, + { + "source_row_index": 645, + "source_sample_id": "validation_Materials_16" + }, + { + "source_row_index": 652, + "source_sample_id": "validation_Materials_23" + }, + { + "source_row_index": 660, + "source_sample_id": "validation_Math_1" + }, + { + "source_row_index": 667, + "source_sample_id": "validation_Math_8" + }, + { + "source_row_index": 675, + "source_sample_id": "validation_Math_16" + }, + { + "source_row_index": 682, + "source_sample_id": "validation_Math_23" + }, + { + "source_row_index": 690, + "source_sample_id": "validation_Mechanical_Engineering_1" + }, + { + "source_row_index": 697, + "source_sample_id": "validation_Mechanical_Engineering_8" + }, + { + "source_row_index": 705, + "source_sample_id": "validation_Mechanical_Engineering_16" + }, + { + "source_row_index": 712, + "source_sample_id": "validation_Mechanical_Engineering_23" + }, + { + "source_row_index": 720, + "source_sample_id": "validation_Music_1" + }, + { + "source_row_index": 727, + "source_sample_id": "validation_Music_8" + }, + { + "source_row_index": 735, + "source_sample_id": "validation_Music_16" + }, + { + "source_row_index": 742, + "source_sample_id": "validation_Music_23" + }, + { + "source_row_index": 750, + "source_sample_id": "validation_Pharmacy_1" + }, + { + "source_row_index": 757, + "source_sample_id": "validation_Pharmacy_8" + }, + { + "source_row_index": 765, + "source_sample_id": "validation_Pharmacy_16" + }, + { + "source_row_index": 772, + "source_sample_id": "validation_Pharmacy_23" + }, + { + "source_row_index": 780, + "source_sample_id": "validation_Physics_1" + }, + { + "source_row_index": 787, + "source_sample_id": "validation_Physics_8" + }, + { + "source_row_index": 795, + "source_sample_id": "validation_Physics_16" + }, + { + "source_row_index": 802, + "source_sample_id": "validation_Physics_23" + }, + { + "source_row_index": 810, + "source_sample_id": "validation_Psychology_1" + }, + { + "source_row_index": 817, + "source_sample_id": "validation_Psychology_8" + }, + { + "source_row_index": 825, + "source_sample_id": "validation_Psychology_16" + }, + { + "source_row_index": 832, + "source_sample_id": "validation_Psychology_23" + }, + { + "source_row_index": 840, + "source_sample_id": "validation_Public_Health_1" + }, + { + "source_row_index": 847, + "source_sample_id": "validation_Public_Health_8" + }, + { + "source_row_index": 855, + "source_sample_id": "validation_Public_Health_16" + }, + { + "source_row_index": 862, + "source_sample_id": "validation_Public_Health_23" + }, + { + "source_row_index": 870, + "source_sample_id": "validation_Sociology_1" + }, + { + "source_row_index": 877, + "source_sample_id": "validation_Sociology_8" + }, + { + "source_row_index": 885, + "source_sample_id": "validation_Sociology_16" + }, + { + "source_row_index": 892, + "source_sample_id": "validation_Sociology_23" + } + ] + }, + "mvbench": { + "dataset_repository": "OpenGVLab/MVBench", + "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mvbench/mvbench.yaml", + "rows": [ + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 0, + "source_sample_id": "action_sequence:0" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 1, + "source_sample_id": "action_sequence:1" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 2, + "source_sample_id": "action_sequence:2" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 3, + "source_sample_id": "action_sequence:3" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 4, + "source_sample_id": "action_sequence:4" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 5, + "source_sample_id": "action_sequence:5" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 6, + "source_sample_id": "action_sequence:6" + }, + { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 7, + "source_sample_id": "action_sequence:7" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 0, + "source_sample_id": "moving_count:0" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 1, + "source_sample_id": "moving_count:1" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 2, + "source_sample_id": "moving_count:2" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 3, + "source_sample_id": "moving_count:3" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 4, + "source_sample_id": "moving_count:4" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 5, + "source_sample_id": "moving_count:5" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 6, + "source_sample_id": "moving_count:6" + }, + { + "leaf_task": "mvbench_moving_count", + "source_row_index": 7, + "source_sample_id": "moving_count:7" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 0, + "source_sample_id": "action_prediction:0" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 1, + "source_sample_id": "action_prediction:1" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 2, + "source_sample_id": "action_prediction:2" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 3, + "source_sample_id": "action_prediction:3" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 4, + "source_sample_id": "action_prediction:4" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 5, + "source_sample_id": "action_prediction:5" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 6, + "source_sample_id": "action_prediction:6" + }, + { + "leaf_task": "mvbench_action_prediction", + "source_row_index": 7, + "source_sample_id": "action_prediction:7" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 0, + "source_sample_id": "episodic_reasoning:0" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 1, + "source_sample_id": "episodic_reasoning:1" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 2, + "source_sample_id": "episodic_reasoning:2" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 3, + "source_sample_id": "episodic_reasoning:3" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 4, + "source_sample_id": "episodic_reasoning:4" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 5, + "source_sample_id": "episodic_reasoning:5" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 6, + "source_sample_id": "episodic_reasoning:6" + }, + { + "leaf_task": "mvbench_episodic_reasoning", + "source_row_index": 7, + "source_sample_id": "episodic_reasoning:7" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 0, + "source_sample_id": "action_antonym:0" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 1, + "source_sample_id": "action_antonym:1" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 2, + "source_sample_id": "action_antonym:2" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 3, + "source_sample_id": "action_antonym:3" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 4, + "source_sample_id": "action_antonym:4" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 5, + "source_sample_id": "action_antonym:5" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 6, + "source_sample_id": "action_antonym:6" + }, + { + "leaf_task": "mvbench_action_antonym", + "source_row_index": 7, + "source_sample_id": "action_antonym:7" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 0, + "source_sample_id": "action_count:0" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 1, + "source_sample_id": "action_count:1" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 2, + "source_sample_id": "action_count:2" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 3, + "source_sample_id": "action_count:3" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 4, + "source_sample_id": "action_count:4" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 5, + "source_sample_id": "action_count:5" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 6, + "source_sample_id": "action_count:6" + }, + { + "leaf_task": "mvbench_action_count", + "source_row_index": 7, + "source_sample_id": "action_count:7" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 0, + "source_sample_id": "scene_transition:0" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 1, + "source_sample_id": "scene_transition:1" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 2, + "source_sample_id": "scene_transition:2" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 3, + "source_sample_id": "scene_transition:3" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 4, + "source_sample_id": "scene_transition:4" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 5, + "source_sample_id": "scene_transition:5" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 6, + "source_sample_id": "scene_transition:6" + }, + { + "leaf_task": "mvbench_scene_transition", + "source_row_index": 7, + "source_sample_id": "scene_transition:7" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 0, + "source_sample_id": "object_shuffle:0" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 1, + "source_sample_id": "object_shuffle:1" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 2, + "source_sample_id": "object_shuffle:2" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 3, + "source_sample_id": "object_shuffle:3" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 4, + "source_sample_id": "object_shuffle:4" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 5, + "source_sample_id": "object_shuffle:5" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 6, + "source_sample_id": "object_shuffle:6" + }, + { + "leaf_task": "mvbench_object_shuffle", + "source_row_index": 7, + "source_sample_id": "object_shuffle:7" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 0, + "source_sample_id": "object_existence:0" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 1, + "source_sample_id": "object_existence:1" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 2, + "source_sample_id": "object_existence:2" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 3, + "source_sample_id": "object_existence:3" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 4, + "source_sample_id": "object_existence:4" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 5, + "source_sample_id": "object_existence:5" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 6, + "source_sample_id": "object_existence:6" + }, + { + "leaf_task": "mvbench_object_existence", + "source_row_index": 7, + "source_sample_id": "object_existence:7" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 0, + "source_sample_id": "fine_grained_pose:0" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 1, + "source_sample_id": "fine_grained_pose:1" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 2, + "source_sample_id": "fine_grained_pose:2" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 3, + "source_sample_id": "fine_grained_pose:3" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 4, + "source_sample_id": "fine_grained_pose:4" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 5, + "source_sample_id": "fine_grained_pose:5" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 6, + "source_sample_id": "fine_grained_pose:6" + }, + { + "leaf_task": "mvbench_fine_grained_pose", + "source_row_index": 7, + "source_sample_id": "fine_grained_pose:7" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 0, + "source_sample_id": "unexpected_action:0" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 1, + "source_sample_id": "unexpected_action:1" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 2, + "source_sample_id": "unexpected_action:2" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 3, + "source_sample_id": "unexpected_action:3" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 4, + "source_sample_id": "unexpected_action:4" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 5, + "source_sample_id": "unexpected_action:5" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 6, + "source_sample_id": "unexpected_action:6" + }, + { + "leaf_task": "mvbench_unexpected_action", + "source_row_index": 7, + "source_sample_id": "unexpected_action:7" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 0, + "source_sample_id": "moving_direction:0" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 1, + "source_sample_id": "moving_direction:1" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 2, + "source_sample_id": "moving_direction:2" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 3, + "source_sample_id": "moving_direction:3" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 4, + "source_sample_id": "moving_direction:4" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 5, + "source_sample_id": "moving_direction:5" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 6, + "source_sample_id": "moving_direction:6" + }, + { + "leaf_task": "mvbench_moving_direction", + "source_row_index": 7, + "source_sample_id": "moving_direction:7" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 0, + "source_sample_id": "state_change:0" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 1, + "source_sample_id": "state_change:1" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 2, + "source_sample_id": "state_change:2" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 3, + "source_sample_id": "state_change:3" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 4, + "source_sample_id": "state_change:4" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 5, + "source_sample_id": "state_change:5" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 6, + "source_sample_id": "state_change:6" + }, + { + "leaf_task": "mvbench_state_change", + "source_row_index": 7, + "source_sample_id": "state_change:7" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 0, + "source_sample_id": "object_interaction:0" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 1, + "source_sample_id": "object_interaction:1" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 2, + "source_sample_id": "object_interaction:2" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 3, + "source_sample_id": "object_interaction:3" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 4, + "source_sample_id": "object_interaction:4" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 5, + "source_sample_id": "object_interaction:5" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 6, + "source_sample_id": "object_interaction:6" + }, + { + "leaf_task": "mvbench_object_interaction", + "source_row_index": 7, + "source_sample_id": "object_interaction:7" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 0, + "source_sample_id": "character_order:0" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 1, + "source_sample_id": "character_order:1" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 2, + "source_sample_id": "character_order:2" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 3, + "source_sample_id": "character_order:3" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 4, + "source_sample_id": "character_order:4" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 5, + "source_sample_id": "character_order:5" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 6, + "source_sample_id": "character_order:6" + }, + { + "leaf_task": "mvbench_character_order", + "source_row_index": 7, + "source_sample_id": "character_order:7" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 0, + "source_sample_id": "action_localization:0" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 1, + "source_sample_id": "action_localization:1" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 2, + "source_sample_id": "action_localization:2" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 3, + "source_sample_id": "action_localization:3" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 4, + "source_sample_id": "action_localization:4" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 5, + "source_sample_id": "action_localization:5" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 6, + "source_sample_id": "action_localization:6" + }, + { + "leaf_task": "mvbench_action_localization", + "source_row_index": 7, + "source_sample_id": "action_localization:7" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 0, + "source_sample_id": "counterfactual_inference:0" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 1, + "source_sample_id": "counterfactual_inference:1" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 2, + "source_sample_id": "counterfactual_inference:2" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 3, + "source_sample_id": "counterfactual_inference:3" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 4, + "source_sample_id": "counterfactual_inference:4" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 5, + "source_sample_id": "counterfactual_inference:5" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 6, + "source_sample_id": "counterfactual_inference:6" + }, + { + "leaf_task": "mvbench_counterfactual_inference", + "source_row_index": 7, + "source_sample_id": "counterfactual_inference:7" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 0, + "source_sample_id": "fine_grained_action:0" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 1, + "source_sample_id": "fine_grained_action:1" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 2, + "source_sample_id": "fine_grained_action:2" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 3, + "source_sample_id": "fine_grained_action:3" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 4, + "source_sample_id": "fine_grained_action:4" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 5, + "source_sample_id": "fine_grained_action:5" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 6, + "source_sample_id": "fine_grained_action:6" + }, + { + "leaf_task": "mvbench_fine_grained_action", + "source_row_index": 7, + "source_sample_id": "fine_grained_action:7" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 0, + "source_sample_id": "moving_attribute:0" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 1, + "source_sample_id": "moving_attribute:1" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 2, + "source_sample_id": "moving_attribute:2" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 3, + "source_sample_id": "moving_attribute:3" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 4, + "source_sample_id": "moving_attribute:4" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 5, + "source_sample_id": "moving_attribute:5" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 6, + "source_sample_id": "moving_attribute:6" + }, + { + "leaf_task": "mvbench_moving_attribute", + "source_row_index": 7, + "source_sample_id": "moving_attribute:7" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 0, + "source_sample_id": "egocentric_navigation:0" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 1, + "source_sample_id": "egocentric_navigation:1" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 2, + "source_sample_id": "egocentric_navigation:2" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 3, + "source_sample_id": "egocentric_navigation:3" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 4, + "source_sample_id": "egocentric_navigation:4" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 5, + "source_sample_id": "egocentric_navigation:5" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 6, + "source_sample_id": "egocentric_navigation:6" + }, + { + "leaf_task": "mvbench_egocentric_navigation", + "source_row_index": 7, + "source_sample_id": "egocentric_navigation:7" + } + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_r1.json new file mode 100644 index 00000000000..c6d4aa32a82 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_344-examples_r1.json @@ -0,0 +1,1872 @@ +{ + "extends": "core-3_344-examples_legacy-r1", + "schema": "modelopt.vlm-sample-set/v1", + "name": "core-3_344-examples_r1", + "sampling": { + "schema": "modelopt.vlm-sampling-audit/v1", + "claim_scope": "deterministic-screening-only", + "generator": { + "name": "systematic-midpoint", + "version": 1 + } + }, + "tasks": { + "realworldqa": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "split", + "index_space": "within-stratum", + "population_rows": 765, + "selected_rows": 64, + "strata": [ + { + "name": "test", + "population_rows": 765, + "selected_rows": 64 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 5, + "p25": 185, + "p50": 376, + "p75": 567, + "p100": 759 + }, + "selected_row_identities_sha256": "c8a03d3811daa83ecd8d82f60d0f6a04736594c1ca3bae2b665b6f6076273b0b" + }, + "rows": [ + { + "source_row_index": 5, + "source_sample_id": "test:5" + }, + { + "source_row_index": 17, + "source_sample_id": "test:17" + }, + { + "source_row_index": 29, + "source_sample_id": "test:29" + }, + { + "source_row_index": 41, + "source_sample_id": "test:41" + }, + { + "source_row_index": 53, + "source_sample_id": "test:53" + }, + { + "source_row_index": 65, + "source_sample_id": "test:65" + }, + { + "source_row_index": 77, + "source_sample_id": "test:77" + }, + { + "source_row_index": 89, + "source_sample_id": "test:89" + }, + { + "source_row_index": 101, + "source_sample_id": "test:101" + }, + { + "source_row_index": 113, + "source_sample_id": "test:113" + }, + { + "source_row_index": 125, + "source_sample_id": "test:125" + }, + { + "source_row_index": 137, + "source_sample_id": "test:137" + }, + { + "source_row_index": 149, + "source_sample_id": "test:149" + }, + { + "source_row_index": 161, + "source_sample_id": "test:161" + }, + { + "source_row_index": 173, + "source_sample_id": "test:173" + }, + { + "source_row_index": 185, + "source_sample_id": "test:185" + }, + { + "source_row_index": 197, + "source_sample_id": "test:197" + }, + { + "source_row_index": 209, + "source_sample_id": "test:209" + }, + { + "source_row_index": 221, + "source_sample_id": "test:221" + }, + { + "source_row_index": 233, + "source_sample_id": "test:233" + }, + { + "source_row_index": 245, + "source_sample_id": "test:245" + }, + { + "source_row_index": 256, + "source_sample_id": "test:256" + }, + { + "source_row_index": 268, + "source_sample_id": "test:268" + }, + { + "source_row_index": 280, + "source_sample_id": "test:280" + }, + { + "source_row_index": 292, + "source_sample_id": "test:292" + }, + { + "source_row_index": 304, + "source_sample_id": "test:304" + }, + { + "source_row_index": 316, + "source_sample_id": "test:316" + }, + { + "source_row_index": 328, + "source_sample_id": "test:328" + }, + { + "source_row_index": 340, + "source_sample_id": "test:340" + }, + { + "source_row_index": 352, + "source_sample_id": "test:352" + }, + { + "source_row_index": 364, + "source_sample_id": "test:364" + }, + { + "source_row_index": 376, + "source_sample_id": "test:376" + }, + { + "source_row_index": 388, + "source_sample_id": "test:388" + }, + { + "source_row_index": 400, + "source_sample_id": "test:400" + }, + { + "source_row_index": 412, + "source_sample_id": "test:412" + }, + { + "source_row_index": 424, + "source_sample_id": "test:424" + }, + { + "source_row_index": 436, + "source_sample_id": "test:436" + }, + { + "source_row_index": 448, + "source_sample_id": "test:448" + }, + { + "source_row_index": 460, + "source_sample_id": "test:460" + }, + { + "source_row_index": 472, + "source_sample_id": "test:472" + }, + { + "source_row_index": 484, + "source_sample_id": "test:484" + }, + { + "source_row_index": 496, + "source_sample_id": "test:496" + }, + { + "source_row_index": 508, + "source_sample_id": "test:508" + }, + { + "source_row_index": 519, + "source_sample_id": "test:519" + }, + { + "source_row_index": 531, + "source_sample_id": "test:531" + }, + { + "source_row_index": 543, + "source_sample_id": "test:543" + }, + { + "source_row_index": 555, + "source_sample_id": "test:555" + }, + { + "source_row_index": 567, + "source_sample_id": "test:567" + }, + { + "source_row_index": 579, + "source_sample_id": "test:579" + }, + { + "source_row_index": 591, + "source_sample_id": "test:591" + }, + { + "source_row_index": 603, + "source_sample_id": "test:603" + }, + { + "source_row_index": 615, + "source_sample_id": "test:615" + }, + { + "source_row_index": 627, + "source_sample_id": "test:627" + }, + { + "source_row_index": 639, + "source_sample_id": "test:639" + }, + { + "source_row_index": 651, + "source_sample_id": "test:651" + }, + { + "source_row_index": 663, + "source_sample_id": "test:663" + }, + { + "source_row_index": 675, + "source_sample_id": "test:675" + }, + { + "source_row_index": 687, + "source_sample_id": "test:687" + }, + { + "source_row_index": 699, + "source_sample_id": "test:699" + }, + { + "source_row_index": 711, + "source_sample_id": "test:711" + }, + { + "source_row_index": 723, + "source_sample_id": "test:723" + }, + { + "source_row_index": 735, + "source_sample_id": "test:735" + }, + { + "source_row_index": 747, + "source_sample_id": "test:747" + }, + { + "source_row_index": 759, + "source_sample_id": "test:759" + } + ] + }, + "mmmu_val": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "subject", + "index_space": "within-stratum", + "population_rows": 900, + "selected_rows": 120, + "strata": [ + { + "name": "Accounting", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Agriculture", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Architecture_and_Engineering", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Art", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Art_Theory", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Basic_Medical_Science", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Biology", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Chemistry", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Clinical_Medicine", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Computer_Science", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Design", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Diagnostics_and_Laboratory_Medicine", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Economics", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Electronics", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Energy_and_Power", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Finance", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Geography", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "History", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Literature", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Manage", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Marketing", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Materials", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Math", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Mechanical_Engineering", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Music", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Pharmacy", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Physics", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Psychology", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Public_Health", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "Sociology", + "population_rows": 30, + "selected_rows": 4 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 3, + "p25": 3, + "p50": 11, + "p75": 18, + "p100": 26 + }, + "selected_row_identities_sha256": "b09e9454c2d03cddf09f32526b390099408a8dabbe24318fb221e5c5fc60556d" + }, + "rows": [ + { + "source_row_index": 3, + "source_sample_id": "validation_Accounting_4" + }, + { + "source_row_index": 11, + "source_sample_id": "validation_Accounting_12" + }, + { + "source_row_index": 18, + "source_sample_id": "validation_Accounting_19" + }, + { + "source_row_index": 26, + "source_sample_id": "validation_Accounting_27" + }, + { + "source_row_index": 33, + "source_sample_id": "validation_Agriculture_4" + }, + { + "source_row_index": 41, + "source_sample_id": "validation_Agriculture_12" + }, + { + "source_row_index": 48, + "source_sample_id": "validation_Agriculture_19" + }, + { + "source_row_index": 56, + "source_sample_id": "validation_Agriculture_27" + }, + { + "source_row_index": 63, + "source_sample_id": "validation_Architecture_and_Engineering_4" + }, + { + "source_row_index": 71, + "source_sample_id": "validation_Architecture_and_Engineering_12" + }, + { + "source_row_index": 78, + "source_sample_id": "validation_Architecture_and_Engineering_19" + }, + { + "source_row_index": 86, + "source_sample_id": "validation_Architecture_and_Engineering_27" + }, + { + "source_row_index": 93, + "source_sample_id": "validation_Art_4" + }, + { + "source_row_index": 101, + "source_sample_id": "validation_Art_12" + }, + { + "source_row_index": 108, + "source_sample_id": "validation_Art_19" + }, + { + "source_row_index": 116, + "source_sample_id": "validation_Art_27" + }, + { + "source_row_index": 123, + "source_sample_id": "validation_Art_Theory_4" + }, + { + "source_row_index": 131, + "source_sample_id": "validation_Art_Theory_12" + }, + { + "source_row_index": 138, + "source_sample_id": "validation_Art_Theory_19" + }, + { + "source_row_index": 146, + "source_sample_id": "validation_Art_Theory_27" + }, + { + "source_row_index": 153, + "source_sample_id": "validation_Basic_Medical_Science_4" + }, + { + "source_row_index": 161, + "source_sample_id": "validation_Basic_Medical_Science_12" + }, + { + "source_row_index": 168, + "source_sample_id": "validation_Basic_Medical_Science_19" + }, + { + "source_row_index": 176, + "source_sample_id": "validation_Basic_Medical_Science_27" + }, + { + "source_row_index": 183, + "source_sample_id": "validation_Biology_4" + }, + { + "source_row_index": 191, + "source_sample_id": "validation_Biology_12" + }, + { + "source_row_index": 198, + "source_sample_id": "validation_Biology_19" + }, + { + "source_row_index": 206, + "source_sample_id": "validation_Biology_27" + }, + { + "source_row_index": 213, + "source_sample_id": "validation_Chemistry_4" + }, + { + "source_row_index": 221, + "source_sample_id": "validation_Chemistry_12" + }, + { + "source_row_index": 228, + "source_sample_id": "validation_Chemistry_19" + }, + { + "source_row_index": 236, + "source_sample_id": "validation_Chemistry_27" + }, + { + "source_row_index": 243, + "source_sample_id": "validation_Clinical_Medicine_4" + }, + { + "source_row_index": 251, + "source_sample_id": "validation_Clinical_Medicine_12" + }, + { + "source_row_index": 258, + "source_sample_id": "validation_Clinical_Medicine_19" + }, + { + "source_row_index": 266, + "source_sample_id": "validation_Clinical_Medicine_27" + }, + { + "source_row_index": 273, + "source_sample_id": "validation_Computer_Science_4" + }, + { + "source_row_index": 281, + "source_sample_id": "validation_Computer_Science_12" + }, + { + "source_row_index": 288, + "source_sample_id": "validation_Computer_Science_19" + }, + { + "source_row_index": 296, + "source_sample_id": "validation_Computer_Science_27" + }, + { + "source_row_index": 303, + "source_sample_id": "validation_Design_4" + }, + { + "source_row_index": 311, + "source_sample_id": "validation_Design_12" + }, + { + "source_row_index": 318, + "source_sample_id": "validation_Design_19" + }, + { + "source_row_index": 326, + "source_sample_id": "validation_Design_27" + }, + { + "source_row_index": 333, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_4" + }, + { + "source_row_index": 341, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_12" + }, + { + "source_row_index": 348, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_19" + }, + { + "source_row_index": 356, + "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_27" + }, + { + "source_row_index": 363, + "source_sample_id": "validation_Economics_4" + }, + { + "source_row_index": 371, + "source_sample_id": "validation_Economics_12" + }, + { + "source_row_index": 378, + "source_sample_id": "validation_Economics_19" + }, + { + "source_row_index": 386, + "source_sample_id": "validation_Economics_27" + }, + { + "source_row_index": 393, + "source_sample_id": "validation_Electronics_4" + }, + { + "source_row_index": 401, + "source_sample_id": "validation_Electronics_12" + }, + { + "source_row_index": 408, + "source_sample_id": "validation_Electronics_19" + }, + { + "source_row_index": 416, + "source_sample_id": "validation_Electronics_27" + }, + { + "source_row_index": 423, + "source_sample_id": "validation_Energy_and_Power_4" + }, + { + "source_row_index": 431, + "source_sample_id": "validation_Energy_and_Power_12" + }, + { + "source_row_index": 438, + "source_sample_id": "validation_Energy_and_Power_19" + }, + { + "source_row_index": 446, + "source_sample_id": "validation_Energy_and_Power_27" + }, + { + "source_row_index": 453, + "source_sample_id": "validation_Finance_4" + }, + { + "source_row_index": 461, + "source_sample_id": "validation_Finance_12" + }, + { + "source_row_index": 468, + "source_sample_id": "validation_Finance_19" + }, + { + "source_row_index": 476, + "source_sample_id": "validation_Finance_27" + }, + { + "source_row_index": 483, + "source_sample_id": "validation_Geography_4" + }, + { + "source_row_index": 491, + "source_sample_id": "validation_Geography_12" + }, + { + "source_row_index": 498, + "source_sample_id": "validation_Geography_19" + }, + { + "source_row_index": 506, + "source_sample_id": "validation_Geography_27" + }, + { + "source_row_index": 513, + "source_sample_id": "validation_History_4" + }, + { + "source_row_index": 521, + "source_sample_id": "validation_History_12" + }, + { + "source_row_index": 528, + "source_sample_id": "validation_History_19" + }, + { + "source_row_index": 536, + "source_sample_id": "validation_History_27" + }, + { + "source_row_index": 543, + "source_sample_id": "validation_Literature_4" + }, + { + "source_row_index": 551, + "source_sample_id": "validation_Literature_12" + }, + { + "source_row_index": 558, + "source_sample_id": "validation_Literature_19" + }, + { + "source_row_index": 566, + "source_sample_id": "validation_Literature_27" + }, + { + "source_row_index": 573, + "source_sample_id": "validation_Manage_4" + }, + { + "source_row_index": 581, + "source_sample_id": "validation_Manage_12" + }, + { + "source_row_index": 588, + "source_sample_id": "validation_Manage_19" + }, + { + "source_row_index": 596, + "source_sample_id": "validation_Manage_27" + }, + { + "source_row_index": 603, + "source_sample_id": "validation_Marketing_4" + }, + { + "source_row_index": 611, + "source_sample_id": "validation_Marketing_12" + }, + { + "source_row_index": 618, + "source_sample_id": "validation_Marketing_19" + }, + { + "source_row_index": 626, + "source_sample_id": "validation_Marketing_27" + }, + { + "source_row_index": 633, + "source_sample_id": "validation_Materials_4" + }, + { + "source_row_index": 641, + "source_sample_id": "validation_Materials_12" + }, + { + "source_row_index": 648, + "source_sample_id": "validation_Materials_19" + }, + { + "source_row_index": 656, + "source_sample_id": "validation_Materials_27" + }, + { + "source_row_index": 663, + "source_sample_id": "validation_Math_4" + }, + { + "source_row_index": 671, + "source_sample_id": "validation_Math_12" + }, + { + "source_row_index": 678, + "source_sample_id": "validation_Math_19" + }, + { + "source_row_index": 686, + "source_sample_id": "validation_Math_27" + }, + { + "source_row_index": 693, + "source_sample_id": "validation_Mechanical_Engineering_4" + }, + { + "source_row_index": 701, + "source_sample_id": "validation_Mechanical_Engineering_12" + }, + { + "source_row_index": 708, + "source_sample_id": "validation_Mechanical_Engineering_19" + }, + { + "source_row_index": 716, + "source_sample_id": "validation_Mechanical_Engineering_27" + }, + { + "source_row_index": 723, + "source_sample_id": "validation_Music_4" + }, + { + "source_row_index": 731, + "source_sample_id": "validation_Music_12" + }, + { + "source_row_index": 738, + "source_sample_id": "validation_Music_19" + }, + { + "source_row_index": 746, + "source_sample_id": "validation_Music_27" + }, + { + "source_row_index": 753, + "source_sample_id": "validation_Pharmacy_4" + }, + { + "source_row_index": 761, + "source_sample_id": "validation_Pharmacy_12" + }, + { + "source_row_index": 768, + "source_sample_id": "validation_Pharmacy_19" + }, + { + "source_row_index": 776, + "source_sample_id": "validation_Pharmacy_27" + }, + { + "source_row_index": 783, + "source_sample_id": "validation_Physics_4" + }, + { + "source_row_index": 791, + "source_sample_id": "validation_Physics_12" + }, + { + "source_row_index": 798, + "source_sample_id": "validation_Physics_19" + }, + { + "source_row_index": 806, + "source_sample_id": "validation_Physics_27" + }, + { + "source_row_index": 813, + "source_sample_id": "validation_Psychology_4" + }, + { + "source_row_index": 821, + "source_sample_id": "validation_Psychology_12" + }, + { + "source_row_index": 828, + "source_sample_id": "validation_Psychology_19" + }, + { + "source_row_index": 836, + "source_sample_id": "validation_Psychology_27" + }, + { + "source_row_index": 843, + "source_sample_id": "validation_Public_Health_4" + }, + { + "source_row_index": 851, + "source_sample_id": "validation_Public_Health_12" + }, + { + "source_row_index": 858, + "source_sample_id": "validation_Public_Health_19" + }, + { + "source_row_index": 866, + "source_sample_id": "validation_Public_Health_27" + }, + { + "source_row_index": 873, + "source_sample_id": "validation_Sociology_4" + }, + { + "source_row_index": 881, + "source_sample_id": "validation_Sociology_12" + }, + { + "source_row_index": 888, + "source_sample_id": "validation_Sociology_19" + }, + { + "source_row_index": 896, + "source_sample_id": "validation_Sociology_27" + } + ] + }, + "mvbench": { + "selection": { + "method": "systematic-midpoint", + "stratified_by": "leaf_task", + "index_space": "within-stratum", + "population_rows": 4000, + "selected_rows": 160, + "strata": [ + { + "name": "action_sequence", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "moving_count", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "action_prediction", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "episodic_reasoning", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "action_antonym", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "action_count", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "scene_transition", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "object_shuffle", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "object_existence", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "fine_grained_pose", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "unexpected_action", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "moving_direction", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "state_change", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "object_interaction", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "character_order", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "action_localization", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "counterfactual_inference", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "fine_grained_action", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "moving_attribute", + "population_rows": 200, + "selected_rows": 8 + }, + { + "name": "egocentric_navigation", + "population_rows": 200, + "selected_rows": 8 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 12, + "p25": 37, + "p50": 87, + "p75": 137, + "p100": 187 + }, + "selected_row_identities_sha256": "e1a4c0665e284828758de4a4f4cb847f98b37df314eb2cb5a9f58631f065b82c" + }, + "rows": [ + { + "source_row_index": 12, + "source_sample_id": "action_sequence:12", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 37, + "source_sample_id": "action_sequence:37", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 62, + "source_sample_id": "action_sequence:62", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 87, + "source_sample_id": "action_sequence:87", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 112, + "source_sample_id": "action_sequence:112", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 137, + "source_sample_id": "action_sequence:137", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 162, + "source_sample_id": "action_sequence:162", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 187, + "source_sample_id": "action_sequence:187", + "leaf_task": "mvbench_action_sequence" + }, + { + "source_row_index": 12, + "source_sample_id": "moving_count:12", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 37, + "source_sample_id": "moving_count:37", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 62, + "source_sample_id": "moving_count:62", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 87, + "source_sample_id": "moving_count:87", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 112, + "source_sample_id": "moving_count:112", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 137, + "source_sample_id": "moving_count:137", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 162, + "source_sample_id": "moving_count:162", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 187, + "source_sample_id": "moving_count:187", + "leaf_task": "mvbench_moving_count" + }, + { + "source_row_index": 12, + "source_sample_id": "action_prediction:12", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 37, + "source_sample_id": "action_prediction:37", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 62, + "source_sample_id": "action_prediction:62", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 87, + "source_sample_id": "action_prediction:87", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 112, + "source_sample_id": "action_prediction:112", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 137, + "source_sample_id": "action_prediction:137", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 162, + "source_sample_id": "action_prediction:162", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 187, + "source_sample_id": "action_prediction:187", + "leaf_task": "mvbench_action_prediction" + }, + { + "source_row_index": 12, + "source_sample_id": "episodic_reasoning:12", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 37, + "source_sample_id": "episodic_reasoning:37", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 62, + "source_sample_id": "episodic_reasoning:62", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 87, + "source_sample_id": "episodic_reasoning:87", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 112, + "source_sample_id": "episodic_reasoning:112", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 137, + "source_sample_id": "episodic_reasoning:137", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 162, + "source_sample_id": "episodic_reasoning:162", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 187, + "source_sample_id": "episodic_reasoning:187", + "leaf_task": "mvbench_episodic_reasoning" + }, + { + "source_row_index": 12, + "source_sample_id": "action_antonym:12", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 37, + "source_sample_id": "action_antonym:37", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 62, + "source_sample_id": "action_antonym:62", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 87, + "source_sample_id": "action_antonym:87", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 112, + "source_sample_id": "action_antonym:112", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 137, + "source_sample_id": "action_antonym:137", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 162, + "source_sample_id": "action_antonym:162", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 187, + "source_sample_id": "action_antonym:187", + "leaf_task": "mvbench_action_antonym" + }, + { + "source_row_index": 12, + "source_sample_id": "action_count:12", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 37, + "source_sample_id": "action_count:37", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 62, + "source_sample_id": "action_count:62", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 87, + "source_sample_id": "action_count:87", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 112, + "source_sample_id": "action_count:112", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 137, + "source_sample_id": "action_count:137", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 162, + "source_sample_id": "action_count:162", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 187, + "source_sample_id": "action_count:187", + "leaf_task": "mvbench_action_count" + }, + { + "source_row_index": 12, + "source_sample_id": "scene_transition:12", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 37, + "source_sample_id": "scene_transition:37", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 62, + "source_sample_id": "scene_transition:62", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 87, + "source_sample_id": "scene_transition:87", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 112, + "source_sample_id": "scene_transition:112", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 137, + "source_sample_id": "scene_transition:137", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 162, + "source_sample_id": "scene_transition:162", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 187, + "source_sample_id": "scene_transition:187", + "leaf_task": "mvbench_scene_transition" + }, + { + "source_row_index": 12, + "source_sample_id": "object_shuffle:12", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 37, + "source_sample_id": "object_shuffle:37", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 62, + "source_sample_id": "object_shuffle:62", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 87, + "source_sample_id": "object_shuffle:87", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 112, + "source_sample_id": "object_shuffle:112", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 137, + "source_sample_id": "object_shuffle:137", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 162, + "source_sample_id": "object_shuffle:162", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 187, + "source_sample_id": "object_shuffle:187", + "leaf_task": "mvbench_object_shuffle" + }, + { + "source_row_index": 12, + "source_sample_id": "object_existence:12", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 37, + "source_sample_id": "object_existence:37", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 62, + "source_sample_id": "object_existence:62", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 87, + "source_sample_id": "object_existence:87", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 112, + "source_sample_id": "object_existence:112", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 137, + "source_sample_id": "object_existence:137", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 162, + "source_sample_id": "object_existence:162", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 187, + "source_sample_id": "object_existence:187", + "leaf_task": "mvbench_object_existence" + }, + { + "source_row_index": 12, + "source_sample_id": "fine_grained_pose:12", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 37, + "source_sample_id": "fine_grained_pose:37", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 62, + "source_sample_id": "fine_grained_pose:62", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 87, + "source_sample_id": "fine_grained_pose:87", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 112, + "source_sample_id": "fine_grained_pose:112", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 137, + "source_sample_id": "fine_grained_pose:137", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 162, + "source_sample_id": "fine_grained_pose:162", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 187, + "source_sample_id": "fine_grained_pose:187", + "leaf_task": "mvbench_fine_grained_pose" + }, + { + "source_row_index": 12, + "source_sample_id": "unexpected_action:12", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 37, + "source_sample_id": "unexpected_action:37", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 62, + "source_sample_id": "unexpected_action:62", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 87, + "source_sample_id": "unexpected_action:87", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 112, + "source_sample_id": "unexpected_action:112", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 137, + "source_sample_id": "unexpected_action:137", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 162, + "source_sample_id": "unexpected_action:162", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 187, + "source_sample_id": "unexpected_action:187", + "leaf_task": "mvbench_unexpected_action" + }, + { + "source_row_index": 12, + "source_sample_id": "moving_direction:12", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 37, + "source_sample_id": "moving_direction:37", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 62, + "source_sample_id": "moving_direction:62", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 87, + "source_sample_id": "moving_direction:87", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 112, + "source_sample_id": "moving_direction:112", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 137, + "source_sample_id": "moving_direction:137", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 162, + "source_sample_id": "moving_direction:162", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 187, + "source_sample_id": "moving_direction:187", + "leaf_task": "mvbench_moving_direction" + }, + { + "source_row_index": 12, + "source_sample_id": "state_change:12", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 37, + "source_sample_id": "state_change:37", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 62, + "source_sample_id": "state_change:62", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 87, + "source_sample_id": "state_change:87", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 112, + "source_sample_id": "state_change:112", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 137, + "source_sample_id": "state_change:137", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 162, + "source_sample_id": "state_change:162", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 187, + "source_sample_id": "state_change:187", + "leaf_task": "mvbench_state_change" + }, + { + "source_row_index": 12, + "source_sample_id": "object_interaction:12", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 37, + "source_sample_id": "object_interaction:37", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 62, + "source_sample_id": "object_interaction:62", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 87, + "source_sample_id": "object_interaction:87", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 112, + "source_sample_id": "object_interaction:112", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 137, + "source_sample_id": "object_interaction:137", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 162, + "source_sample_id": "object_interaction:162", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 187, + "source_sample_id": "object_interaction:187", + "leaf_task": "mvbench_object_interaction" + }, + { + "source_row_index": 12, + "source_sample_id": "character_order:12", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 37, + "source_sample_id": "character_order:37", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 62, + "source_sample_id": "character_order:62", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 87, + "source_sample_id": "character_order:87", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 112, + "source_sample_id": "character_order:112", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 137, + "source_sample_id": "character_order:137", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 162, + "source_sample_id": "character_order:162", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 187, + "source_sample_id": "character_order:187", + "leaf_task": "mvbench_character_order" + }, + { + "source_row_index": 12, + "source_sample_id": "action_localization:12", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 37, + "source_sample_id": "action_localization:37", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 62, + "source_sample_id": "action_localization:62", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 87, + "source_sample_id": "action_localization:87", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 112, + "source_sample_id": "action_localization:112", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 137, + "source_sample_id": "action_localization:137", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 162, + "source_sample_id": "action_localization:162", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 187, + "source_sample_id": "action_localization:187", + "leaf_task": "mvbench_action_localization" + }, + { + "source_row_index": 12, + "source_sample_id": "counterfactual_inference:12", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 37, + "source_sample_id": "counterfactual_inference:37", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 62, + "source_sample_id": "counterfactual_inference:62", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 87, + "source_sample_id": "counterfactual_inference:87", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 112, + "source_sample_id": "counterfactual_inference:112", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 137, + "source_sample_id": "counterfactual_inference:137", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 162, + "source_sample_id": "counterfactual_inference:162", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 187, + "source_sample_id": "counterfactual_inference:187", + "leaf_task": "mvbench_counterfactual_inference" + }, + { + "source_row_index": 12, + "source_sample_id": "fine_grained_action:12", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 37, + "source_sample_id": "fine_grained_action:37", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 62, + "source_sample_id": "fine_grained_action:62", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 87, + "source_sample_id": "fine_grained_action:87", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 112, + "source_sample_id": "fine_grained_action:112", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 137, + "source_sample_id": "fine_grained_action:137", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 162, + "source_sample_id": "fine_grained_action:162", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 187, + "source_sample_id": "fine_grained_action:187", + "leaf_task": "mvbench_fine_grained_action" + }, + { + "source_row_index": 12, + "source_sample_id": "moving_attribute:12", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 37, + "source_sample_id": "moving_attribute:37", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 62, + "source_sample_id": "moving_attribute:62", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 87, + "source_sample_id": "moving_attribute:87", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 112, + "source_sample_id": "moving_attribute:112", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 137, + "source_sample_id": "moving_attribute:137", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 162, + "source_sample_id": "moving_attribute:162", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 187, + "source_sample_id": "moving_attribute:187", + "leaf_task": "mvbench_moving_attribute" + }, + { + "source_row_index": 12, + "source_sample_id": "egocentric_navigation:12", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 37, + "source_sample_id": "egocentric_navigation:37", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 62, + "source_sample_id": "egocentric_navigation:62", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 87, + "source_sample_id": "egocentric_navigation:87", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 112, + "source_sample_id": "egocentric_navigation:112", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 137, + "source_sample_id": "egocentric_navigation:137", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 162, + "source_sample_id": "egocentric_navigation:162", + "leaf_task": "mvbench_egocentric_navigation" + }, + { + "source_row_index": 187, + "source_sample_id": "egocentric_navigation:187", + "leaf_task": "mvbench_egocentric_navigation" + } + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_full_r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_full_r1.json new file mode 100644 index 00000000000..5363b77afc9 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/core-3_full_r1.json @@ -0,0 +1,54 @@ +{ + "schema": "modelopt.vlm-sample-set/v1", + "name": "core-3_full_r1", + "model": { + "repository": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17" + }, + "selection": "all", + "tasks": { + "realworldqa": { + "dataset_repository": "lmms-lab/RealWorldQA", + "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", + "max_new_tokens": 16, + "population_rows": 765, + "scoring_task_config": "tasks/realworldqa/realworldqa.yaml" + }, + "mmmu_val": { + "dataset_repository": "lmms-lab/MMMU", + "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", + "max_new_tokens": 128, + "population_rows": 900, + "scoring_task_config": "tasks/mmmu/mmmu_val.yaml" + }, + "mvbench": { + "dataset_repository": "OpenGVLab/MVBench", + "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", + "leaf_populations": { + "action_sequence": 200, + "moving_count": 200, + "action_prediction": 200, + "episodic_reasoning": 200, + "action_antonym": 200, + "action_count": 200, + "scene_transition": 200, + "object_shuffle": 200, + "object_existence": 200, + "fine_grained_pose": 200, + "unexpected_action": 200, + "moving_direction": 200, + "state_change": 200, + "object_interaction": 200, + "character_order": 200, + "action_localization": 200, + "counterfactual_inference": 200, + "fine_grained_action": 200, + "moving_attribute": 200, + "egocentric_navigation": 200 + }, + "max_new_tokens": 16, + "population_rows": 4000, + "scoring_task_config": "tasks/mvbench/mvbench.yaml" + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_legacy-r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_legacy-r1.json new file mode 100644 index 00000000000..f25f49a97d2 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_legacy-r1.json @@ -0,0 +1,1503 @@ +{ + "extends": "core-3_344-examples_legacy-r1", + "schema": "modelopt.vlm-sample-set/v1", + "name": "judge-free-8_690-examples_legacy-r1", + "tasks": { + "video_mmmu": { + "dataset_repository": "lmms-lab/VideoMMMU", + "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", + "max_new_tokens": 1024, + "scoring_task_config": "tasks/videommmu/video_mmmu.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "adaptation:0", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 13, + "source_sample_id": "adaptation:13", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 26, + "source_sample_id": "adaptation:26", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 39, + "source_sample_id": "adaptation:39", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 52, + "source_sample_id": "adaptation:52", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 65, + "source_sample_id": "adaptation:65", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 78, + "source_sample_id": "adaptation:78", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 91, + "source_sample_id": "adaptation:91", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 104, + "source_sample_id": "adaptation:104", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 117, + "source_sample_id": "adaptation:117", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 130, + "source_sample_id": "adaptation:130", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 143, + "source_sample_id": "adaptation:143", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 156, + "source_sample_id": "adaptation:156", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 169, + "source_sample_id": "adaptation:169", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 182, + "source_sample_id": "adaptation:182", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 195, + "source_sample_id": "adaptation:195", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 208, + "source_sample_id": "adaptation:208", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 221, + "source_sample_id": "adaptation:221", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 234, + "source_sample_id": "adaptation:234", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 247, + "source_sample_id": "adaptation:247", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 260, + "source_sample_id": "adaptation:260", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 273, + "source_sample_id": "adaptation:273", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 286, + "source_sample_id": "adaptation:286", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 299, + "source_sample_id": "adaptation:299", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 0, + "source_sample_id": "comprehension:0", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 13, + "source_sample_id": "comprehension:13", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 26, + "source_sample_id": "comprehension:26", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 39, + "source_sample_id": "comprehension:39", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 52, + "source_sample_id": "comprehension:52", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 65, + "source_sample_id": "comprehension:65", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 78, + "source_sample_id": "comprehension:78", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 91, + "source_sample_id": "comprehension:91", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 104, + "source_sample_id": "comprehension:104", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 117, + "source_sample_id": "comprehension:117", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 130, + "source_sample_id": "comprehension:130", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 143, + "source_sample_id": "comprehension:143", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 156, + "source_sample_id": "comprehension:156", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 169, + "source_sample_id": "comprehension:169", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 182, + "source_sample_id": "comprehension:182", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 195, + "source_sample_id": "comprehension:195", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 208, + "source_sample_id": "comprehension:208", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 221, + "source_sample_id": "comprehension:221", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 234, + "source_sample_id": "comprehension:234", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 247, + "source_sample_id": "comprehension:247", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 260, + "source_sample_id": "comprehension:260", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 273, + "source_sample_id": "comprehension:273", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 286, + "source_sample_id": "comprehension:286", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 299, + "source_sample_id": "comprehension:299", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 0, + "source_sample_id": "perception:0", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 13, + "source_sample_id": "perception:13", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 26, + "source_sample_id": "perception:26", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 39, + "source_sample_id": "perception:39", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 52, + "source_sample_id": "perception:52", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 65, + "source_sample_id": "perception:65", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 78, + "source_sample_id": "perception:78", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 91, + "source_sample_id": "perception:91", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 104, + "source_sample_id": "perception:104", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 117, + "source_sample_id": "perception:117", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 130, + "source_sample_id": "perception:130", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 143, + "source_sample_id": "perception:143", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 156, + "source_sample_id": "perception:156", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 169, + "source_sample_id": "perception:169", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 182, + "source_sample_id": "perception:182", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 195, + "source_sample_id": "perception:195", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 208, + "source_sample_id": "perception:208", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 221, + "source_sample_id": "perception:221", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 234, + "source_sample_id": "perception:234", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 247, + "source_sample_id": "perception:247", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 260, + "source_sample_id": "perception:260", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 273, + "source_sample_id": "perception:273", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 286, + "source_sample_id": "perception:286", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 299, + "source_sample_id": "perception:299", + "leaf_task": "video_mmmu_perception" + } + ] + }, + "videomme": { + "dataset_repository": "lmms-lab/Video-MME", + "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/videomme/videomme.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "videomme:0" + }, + { + "source_row_index": 90, + "source_sample_id": "videomme:90" + }, + { + "source_row_index": 179, + "source_sample_id": "videomme:179" + }, + { + "source_row_index": 269, + "source_sample_id": "videomme:269" + }, + { + "source_row_index": 270, + "source_sample_id": "videomme:270" + }, + { + "source_row_index": 310, + "source_sample_id": "videomme:310" + }, + { + "source_row_index": 349, + "source_sample_id": "videomme:349" + }, + { + "source_row_index": 389, + "source_sample_id": "videomme:389" + }, + { + "source_row_index": 390, + "source_sample_id": "videomme:390" + }, + { + "source_row_index": 440, + "source_sample_id": "videomme:440" + }, + { + "source_row_index": 489, + "source_sample_id": "videomme:489" + }, + { + "source_row_index": 539, + "source_sample_id": "videomme:539" + }, + { + "source_row_index": 540, + "source_sample_id": "videomme:540" + }, + { + "source_row_index": 580, + "source_sample_id": "videomme:580" + }, + { + "source_row_index": 619, + "source_sample_id": "videomme:619" + }, + { + "source_row_index": 659, + "source_sample_id": "videomme:659" + }, + { + "source_row_index": 660, + "source_sample_id": "videomme:660" + }, + { + "source_row_index": 730, + "source_sample_id": "videomme:730" + }, + { + "source_row_index": 799, + "source_sample_id": "videomme:799" + }, + { + "source_row_index": 869, + "source_sample_id": "videomme:869" + }, + { + "source_row_index": 870, + "source_sample_id": "videomme:870" + }, + { + "source_row_index": 880, + "source_sample_id": "videomme:880" + }, + { + "source_row_index": 889, + "source_sample_id": "videomme:889" + }, + { + "source_row_index": 899, + "source_sample_id": "videomme:899" + }, + { + "source_row_index": 900, + "source_sample_id": "videomme:900" + }, + { + "source_row_index": 990, + "source_sample_id": "videomme:990" + }, + { + "source_row_index": 1079, + "source_sample_id": "videomme:1079" + }, + { + "source_row_index": 1169, + "source_sample_id": "videomme:1169" + }, + { + "source_row_index": 1170, + "source_sample_id": "videomme:1170" + }, + { + "source_row_index": 1210, + "source_sample_id": "videomme:1210" + }, + { + "source_row_index": 1249, + "source_sample_id": "videomme:1249" + }, + { + "source_row_index": 1289, + "source_sample_id": "videomme:1289" + }, + { + "source_row_index": 1290, + "source_sample_id": "videomme:1290" + }, + { + "source_row_index": 1340, + "source_sample_id": "videomme:1340" + }, + { + "source_row_index": 1389, + "source_sample_id": "videomme:1389" + }, + { + "source_row_index": 1439, + "source_sample_id": "videomme:1439" + }, + { + "source_row_index": 1440, + "source_sample_id": "videomme:1440" + }, + { + "source_row_index": 1480, + "source_sample_id": "videomme:1480" + }, + { + "source_row_index": 1519, + "source_sample_id": "videomme:1519" + }, + { + "source_row_index": 1559, + "source_sample_id": "videomme:1559" + }, + { + "source_row_index": 1560, + "source_sample_id": "videomme:1560" + }, + { + "source_row_index": 1630, + "source_sample_id": "videomme:1630" + }, + { + "source_row_index": 1699, + "source_sample_id": "videomme:1699" + }, + { + "source_row_index": 1769, + "source_sample_id": "videomme:1769" + }, + { + "source_row_index": 1770, + "source_sample_id": "videomme:1770" + }, + { + "source_row_index": 1780, + "source_sample_id": "videomme:1780" + }, + { + "source_row_index": 1789, + "source_sample_id": "videomme:1789" + }, + { + "source_row_index": 1799, + "source_sample_id": "videomme:1799" + }, + { + "source_row_index": 1800, + "source_sample_id": "videomme:1800" + }, + { + "source_row_index": 1890, + "source_sample_id": "videomme:1890" + }, + { + "source_row_index": 1979, + "source_sample_id": "videomme:1979" + }, + { + "source_row_index": 2069, + "source_sample_id": "videomme:2069" + }, + { + "source_row_index": 2070, + "source_sample_id": "videomme:2070" + }, + { + "source_row_index": 2110, + "source_sample_id": "videomme:2110" + }, + { + "source_row_index": 2149, + "source_sample_id": "videomme:2149" + }, + { + "source_row_index": 2189, + "source_sample_id": "videomme:2189" + }, + { + "source_row_index": 2190, + "source_sample_id": "videomme:2190" + }, + { + "source_row_index": 2240, + "source_sample_id": "videomme:2240" + }, + { + "source_row_index": 2289, + "source_sample_id": "videomme:2289" + }, + { + "source_row_index": 2339, + "source_sample_id": "videomme:2339" + }, + { + "source_row_index": 2340, + "source_sample_id": "videomme:2340" + }, + { + "source_row_index": 2380, + "source_sample_id": "videomme:2380" + }, + { + "source_row_index": 2419, + "source_sample_id": "videomme:2419" + }, + { + "source_row_index": 2459, + "source_sample_id": "videomme:2459" + }, + { + "source_row_index": 2460, + "source_sample_id": "videomme:2460" + }, + { + "source_row_index": 2530, + "source_sample_id": "videomme:2530" + }, + { + "source_row_index": 2599, + "source_sample_id": "videomme:2599" + }, + { + "source_row_index": 2669, + "source_sample_id": "videomme:2669" + }, + { + "source_row_index": 2670, + "source_sample_id": "videomme:2670" + }, + { + "source_row_index": 2680, + "source_sample_id": "videomme:2680" + }, + { + "source_row_index": 2689, + "source_sample_id": "videomme:2689" + }, + { + "source_row_index": 2699, + "source_sample_id": "videomme:2699" + } + ] + }, + "longvideobench_val_v": { + "dataset_repository": "longvideobench/LongVideoBench", + "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", + "max_new_tokens": 32, + "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "longvideobench_val_v:0" + }, + { + "source_row_index": 20, + "source_sample_id": "longvideobench_val_v:20" + }, + { + "source_row_index": 40, + "source_sample_id": "longvideobench_val_v:40" + }, + { + "source_row_index": 60, + "source_sample_id": "longvideobench_val_v:60" + }, + { + "source_row_index": 80, + "source_sample_id": "longvideobench_val_v:80" + }, + { + "source_row_index": 100, + "source_sample_id": "longvideobench_val_v:100" + }, + { + "source_row_index": 120, + "source_sample_id": "longvideobench_val_v:120" + }, + { + "source_row_index": 140, + "source_sample_id": "longvideobench_val_v:140" + }, + { + "source_row_index": 160, + "source_sample_id": "longvideobench_val_v:160" + }, + { + "source_row_index": 179, + "source_sample_id": "longvideobench_val_v:179" + }, + { + "source_row_index": 199, + "source_sample_id": "longvideobench_val_v:199" + }, + { + "source_row_index": 219, + "source_sample_id": "longvideobench_val_v:219" + }, + { + "source_row_index": 239, + "source_sample_id": "longvideobench_val_v:239" + }, + { + "source_row_index": 259, + "source_sample_id": "longvideobench_val_v:259" + }, + { + "source_row_index": 279, + "source_sample_id": "longvideobench_val_v:279" + }, + { + "source_row_index": 299, + "source_sample_id": "longvideobench_val_v:299" + }, + { + "source_row_index": 319, + "source_sample_id": "longvideobench_val_v:319" + }, + { + "source_row_index": 339, + "source_sample_id": "longvideobench_val_v:339" + }, + { + "source_row_index": 359, + "source_sample_id": "longvideobench_val_v:359" + }, + { + "source_row_index": 379, + "source_sample_id": "longvideobench_val_v:379" + }, + { + "source_row_index": 399, + "source_sample_id": "longvideobench_val_v:399" + }, + { + "source_row_index": 419, + "source_sample_id": "longvideobench_val_v:419" + }, + { + "source_row_index": 439, + "source_sample_id": "longvideobench_val_v:439" + }, + { + "source_row_index": 459, + "source_sample_id": "longvideobench_val_v:459" + }, + { + "source_row_index": 479, + "source_sample_id": "longvideobench_val_v:479" + }, + { + "source_row_index": 499, + "source_sample_id": "longvideobench_val_v:499" + }, + { + "source_row_index": 518, + "source_sample_id": "longvideobench_val_v:518" + }, + { + "source_row_index": 538, + "source_sample_id": "longvideobench_val_v:538" + }, + { + "source_row_index": 558, + "source_sample_id": "longvideobench_val_v:558" + }, + { + "source_row_index": 578, + "source_sample_id": "longvideobench_val_v:578" + }, + { + "source_row_index": 598, + "source_sample_id": "longvideobench_val_v:598" + }, + { + "source_row_index": 618, + "source_sample_id": "longvideobench_val_v:618" + }, + { + "source_row_index": 638, + "source_sample_id": "longvideobench_val_v:638" + }, + { + "source_row_index": 658, + "source_sample_id": "longvideobench_val_v:658" + }, + { + "source_row_index": 678, + "source_sample_id": "longvideobench_val_v:678" + }, + { + "source_row_index": 698, + "source_sample_id": "longvideobench_val_v:698" + }, + { + "source_row_index": 718, + "source_sample_id": "longvideobench_val_v:718" + }, + { + "source_row_index": 738, + "source_sample_id": "longvideobench_val_v:738" + }, + { + "source_row_index": 758, + "source_sample_id": "longvideobench_val_v:758" + }, + { + "source_row_index": 778, + "source_sample_id": "longvideobench_val_v:778" + }, + { + "source_row_index": 798, + "source_sample_id": "longvideobench_val_v:798" + }, + { + "source_row_index": 818, + "source_sample_id": "longvideobench_val_v:818" + }, + { + "source_row_index": 837, + "source_sample_id": "longvideobench_val_v:837" + }, + { + "source_row_index": 857, + "source_sample_id": "longvideobench_val_v:857" + }, + { + "source_row_index": 877, + "source_sample_id": "longvideobench_val_v:877" + }, + { + "source_row_index": 897, + "source_sample_id": "longvideobench_val_v:897" + }, + { + "source_row_index": 917, + "source_sample_id": "longvideobench_val_v:917" + }, + { + "source_row_index": 937, + "source_sample_id": "longvideobench_val_v:937" + }, + { + "source_row_index": 957, + "source_sample_id": "longvideobench_val_v:957" + }, + { + "source_row_index": 977, + "source_sample_id": "longvideobench_val_v:977" + }, + { + "source_row_index": 997, + "source_sample_id": "longvideobench_val_v:997" + }, + { + "source_row_index": 1017, + "source_sample_id": "longvideobench_val_v:1017" + }, + { + "source_row_index": 1037, + "source_sample_id": "longvideobench_val_v:1037" + }, + { + "source_row_index": 1057, + "source_sample_id": "longvideobench_val_v:1057" + }, + { + "source_row_index": 1077, + "source_sample_id": "longvideobench_val_v:1077" + }, + { + "source_row_index": 1097, + "source_sample_id": "longvideobench_val_v:1097" + }, + { + "source_row_index": 1117, + "source_sample_id": "longvideobench_val_v:1117" + }, + { + "source_row_index": 1137, + "source_sample_id": "longvideobench_val_v:1137" + }, + { + "source_row_index": 1157, + "source_sample_id": "longvideobench_val_v:1157" + }, + { + "source_row_index": 1176, + "source_sample_id": "longvideobench_val_v:1176" + }, + { + "source_row_index": 1196, + "source_sample_id": "longvideobench_val_v:1196" + }, + { + "source_row_index": 1216, + "source_sample_id": "longvideobench_val_v:1216" + }, + { + "source_row_index": 1236, + "source_sample_id": "longvideobench_val_v:1236" + }, + { + "source_row_index": 1256, + "source_sample_id": "longvideobench_val_v:1256" + }, + { + "source_row_index": 1276, + "source_sample_id": "longvideobench_val_v:1276" + }, + { + "source_row_index": 1296, + "source_sample_id": "longvideobench_val_v:1296" + }, + { + "source_row_index": 1316, + "source_sample_id": "longvideobench_val_v:1316" + }, + { + "source_row_index": 1336, + "source_sample_id": "longvideobench_val_v:1336" + } + ] + }, + "mlvu_dev": { + "dataset_repository": "sy1998/MLVU_dev", + "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "mlvu_dev:0" + }, + { + "source_row_index": 39, + "source_sample_id": "mlvu_dev:39" + }, + { + "source_row_index": 79, + "source_sample_id": "mlvu_dev:79" + }, + { + "source_row_index": 118, + "source_sample_id": "mlvu_dev:118" + }, + { + "source_row_index": 157, + "source_sample_id": "mlvu_dev:157" + }, + { + "source_row_index": 197, + "source_sample_id": "mlvu_dev:197" + }, + { + "source_row_index": 236, + "source_sample_id": "mlvu_dev:236" + }, + { + "source_row_index": 275, + "source_sample_id": "mlvu_dev:275" + }, + { + "source_row_index": 315, + "source_sample_id": "mlvu_dev:315" + }, + { + "source_row_index": 354, + "source_sample_id": "mlvu_dev:354" + }, + { + "source_row_index": 355, + "source_sample_id": "mlvu_dev:355" + }, + { + "source_row_index": 377, + "source_sample_id": "mlvu_dev:377" + }, + { + "source_row_index": 399, + "source_sample_id": "mlvu_dev:399" + }, + { + "source_row_index": 421, + "source_sample_id": "mlvu_dev:421" + }, + { + "source_row_index": 443, + "source_sample_id": "mlvu_dev:443" + }, + { + "source_row_index": 466, + "source_sample_id": "mlvu_dev:466" + }, + { + "source_row_index": 488, + "source_sample_id": "mlvu_dev:488" + }, + { + "source_row_index": 510, + "source_sample_id": "mlvu_dev:510" + }, + { + "source_row_index": 532, + "source_sample_id": "mlvu_dev:532" + }, + { + "source_row_index": 554, + "source_sample_id": "mlvu_dev:554" + }, + { + "source_row_index": 555, + "source_sample_id": "mlvu_dev:555" + }, + { + "source_row_index": 578, + "source_sample_id": "mlvu_dev:578" + }, + { + "source_row_index": 601, + "source_sample_id": "mlvu_dev:601" + }, + { + "source_row_index": 623, + "source_sample_id": "mlvu_dev:623" + }, + { + "source_row_index": 646, + "source_sample_id": "mlvu_dev:646" + }, + { + "source_row_index": 669, + "source_sample_id": "mlvu_dev:669" + }, + { + "source_row_index": 692, + "source_sample_id": "mlvu_dev:692" + }, + { + "source_row_index": 714, + "source_sample_id": "mlvu_dev:714" + }, + { + "source_row_index": 737, + "source_sample_id": "mlvu_dev:737" + }, + { + "source_row_index": 760, + "source_sample_id": "mlvu_dev:760" + }, + { + "source_row_index": 761, + "source_sample_id": "mlvu_dev:761" + }, + { + "source_row_index": 800, + "source_sample_id": "mlvu_dev:800" + }, + { + "source_row_index": 839, + "source_sample_id": "mlvu_dev:839" + }, + { + "source_row_index": 878, + "source_sample_id": "mlvu_dev:878" + }, + { + "source_row_index": 917, + "source_sample_id": "mlvu_dev:917" + }, + { + "source_row_index": 956, + "source_sample_id": "mlvu_dev:956" + }, + { + "source_row_index": 995, + "source_sample_id": "mlvu_dev:995" + }, + { + "source_row_index": 1034, + "source_sample_id": "mlvu_dev:1034" + }, + { + "source_row_index": 1073, + "source_sample_id": "mlvu_dev:1073" + }, + { + "source_row_index": 1112, + "source_sample_id": "mlvu_dev:1112" + }, + { + "source_row_index": 1113, + "source_sample_id": "mlvu_dev:1113" + }, + { + "source_row_index": 1173, + "source_sample_id": "mlvu_dev:1173" + }, + { + "source_row_index": 1233, + "source_sample_id": "mlvu_dev:1233" + }, + { + "source_row_index": 1292, + "source_sample_id": "mlvu_dev:1292" + }, + { + "source_row_index": 1352, + "source_sample_id": "mlvu_dev:1352" + }, + { + "source_row_index": 1412, + "source_sample_id": "mlvu_dev:1412" + }, + { + "source_row_index": 1472, + "source_sample_id": "mlvu_dev:1472" + }, + { + "source_row_index": 1531, + "source_sample_id": "mlvu_dev:1531" + }, + { + "source_row_index": 1591, + "source_sample_id": "mlvu_dev:1591" + }, + { + "source_row_index": 1651, + "source_sample_id": "mlvu_dev:1651" + }, + { + "source_row_index": 1652, + "source_sample_id": "mlvu_dev:1652" + }, + { + "source_row_index": 1681, + "source_sample_id": "mlvu_dev:1681" + }, + { + "source_row_index": 1710, + "source_sample_id": "mlvu_dev:1710" + }, + { + "source_row_index": 1739, + "source_sample_id": "mlvu_dev:1739" + }, + { + "source_row_index": 1768, + "source_sample_id": "mlvu_dev:1768" + }, + { + "source_row_index": 1798, + "source_sample_id": "mlvu_dev:1798" + }, + { + "source_row_index": 1827, + "source_sample_id": "mlvu_dev:1827" + }, + { + "source_row_index": 1856, + "source_sample_id": "mlvu_dev:1856" + }, + { + "source_row_index": 1885, + "source_sample_id": "mlvu_dev:1885" + }, + { + "source_row_index": 1914, + "source_sample_id": "mlvu_dev:1914" + }, + { + "source_row_index": 1915, + "source_sample_id": "mlvu_dev:1915" + }, + { + "source_row_index": 1944, + "source_sample_id": "mlvu_dev:1944" + }, + { + "source_row_index": 1972, + "source_sample_id": "mlvu_dev:1972" + }, + { + "source_row_index": 2001, + "source_sample_id": "mlvu_dev:2001" + }, + { + "source_row_index": 2030, + "source_sample_id": "mlvu_dev:2030" + }, + { + "source_row_index": 2058, + "source_sample_id": "mlvu_dev:2058" + }, + { + "source_row_index": 2087, + "source_sample_id": "mlvu_dev:2087" + }, + { + "source_row_index": 2116, + "source_sample_id": "mlvu_dev:2116" + }, + { + "source_row_index": 2144, + "source_sample_id": "mlvu_dev:2144" + }, + { + "source_row_index": 2173, + "source_sample_id": "mlvu_dev:2173" + } + ] + }, + "perceptiontest_val_mc": { + "dataset_repository": "lmms-lab/PerceptionTest_Val", + "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml", + "rows": [ + { + "source_row_index": 0, + "source_sample_id": "perceptiontest_val_mc:0" + }, + { + "source_row_index": 1, + "source_sample_id": "perceptiontest_val_mc:1" + }, + { + "source_row_index": 3, + "source_sample_id": "perceptiontest_val_mc:3" + }, + { + "source_row_index": 20, + "source_sample_id": "perceptiontest_val_mc:20" + }, + { + "source_row_index": 211, + "source_sample_id": "perceptiontest_val_mc:211" + }, + { + "source_row_index": 544, + "source_sample_id": "perceptiontest_val_mc:544" + }, + { + "source_row_index": 753, + "source_sample_id": "perceptiontest_val_mc:753" + }, + { + "source_row_index": 1672, + "source_sample_id": "perceptiontest_val_mc:1672" + }, + { + "source_row_index": 2310, + "source_sample_id": "perceptiontest_val_mc:2310" + }, + { + "source_row_index": 2386, + "source_sample_id": "perceptiontest_val_mc:2386" + }, + { + "source_row_index": 2846, + "source_sample_id": "perceptiontest_val_mc:2846" + }, + { + "source_row_index": 2894, + "source_sample_id": "perceptiontest_val_mc:2894" + }, + { + "source_row_index": 3857, + "source_sample_id": "perceptiontest_val_mc:3857" + }, + { + "source_row_index": 3905, + "source_sample_id": "perceptiontest_val_mc:3905" + }, + { + "source_row_index": 3957, + "source_sample_id": "perceptiontest_val_mc:3957" + }, + { + "source_row_index": 4004, + "source_sample_id": "perceptiontest_val_mc:4004" + }, + { + "source_row_index": 4031, + "source_sample_id": "perceptiontest_val_mc:4031" + }, + { + "source_row_index": 4053, + "source_sample_id": "perceptiontest_val_mc:4053" + }, + { + "source_row_index": 4167, + "source_sample_id": "perceptiontest_val_mc:4167" + }, + { + "source_row_index": 4228, + "source_sample_id": "perceptiontest_val_mc:4228" + }, + { + "source_row_index": 4994, + "source_sample_id": "perceptiontest_val_mc:4994" + }, + { + "source_row_index": 5673, + "source_sample_id": "perceptiontest_val_mc:5673" + }, + { + "source_row_index": 5898, + "source_sample_id": "perceptiontest_val_mc:5898" + }, + { + "source_row_index": 6414, + "source_sample_id": "perceptiontest_val_mc:6414" + }, + { + "source_row_index": 6502, + "source_sample_id": "perceptiontest_val_mc:6502" + }, + { + "source_row_index": 6598, + "source_sample_id": "perceptiontest_val_mc:6598" + }, + { + "source_row_index": 7494, + "source_sample_id": "perceptiontest_val_mc:7494" + }, + { + "source_row_index": 8074, + "source_sample_id": "perceptiontest_val_mc:8074" + }, + { + "source_row_index": 8617, + "source_sample_id": "perceptiontest_val_mc:8617" + }, + { + "source_row_index": 8763, + "source_sample_id": "perceptiontest_val_mc:8763" + }, + { + "source_row_index": 8865, + "source_sample_id": "perceptiontest_val_mc:8865" + }, + { + "source_row_index": 8873, + "source_sample_id": "perceptiontest_val_mc:8873" + }, + { + "source_row_index": 8981, + "source_sample_id": "perceptiontest_val_mc:8981" + }, + { + "source_row_index": 9350, + "source_sample_id": "perceptiontest_val_mc:9350" + }, + { + "source_row_index": 9617, + "source_sample_id": "perceptiontest_val_mc:9617" + }, + { + "source_row_index": 9749, + "source_sample_id": "perceptiontest_val_mc:9749" + }, + { + "source_row_index": 10470, + "source_sample_id": "perceptiontest_val_mc:10470" + }, + { + "source_row_index": 11697, + "source_sample_id": "perceptiontest_val_mc:11697" + }, + { + "source_row_index": 11745, + "source_sample_id": "perceptiontest_val_mc:11745" + }, + { + "source_row_index": 11798, + "source_sample_id": "perceptiontest_val_mc:11798" + }, + { + "source_row_index": 13051, + "source_sample_id": "perceptiontest_val_mc:13051" + }, + { + "source_row_index": 13244, + "source_sample_id": "perceptiontest_val_mc:13244" + }, + { + "source_row_index": 13581, + "source_sample_id": "perceptiontest_val_mc:13581" + }, + { + "source_row_index": 13929, + "source_sample_id": "perceptiontest_val_mc:13929" + }, + { + "source_row_index": 13999, + "source_sample_id": "perceptiontest_val_mc:13999" + }, + { + "source_row_index": 14073, + "source_sample_id": "perceptiontest_val_mc:14073" + }, + { + "source_row_index": 14200, + "source_sample_id": "perceptiontest_val_mc:14200" + }, + { + "source_row_index": 14884, + "source_sample_id": "perceptiontest_val_mc:14884" + }, + { + "source_row_index": 15201, + "source_sample_id": "perceptiontest_val_mc:15201" + }, + { + "source_row_index": 15432, + "source_sample_id": "perceptiontest_val_mc:15432" + }, + { + "source_row_index": 16191, + "source_sample_id": "perceptiontest_val_mc:16191" + }, + { + "source_row_index": 16280, + "source_sample_id": "perceptiontest_val_mc:16280" + }, + { + "source_row_index": 16741, + "source_sample_id": "perceptiontest_val_mc:16741" + }, + { + "source_row_index": 17109, + "source_sample_id": "perceptiontest_val_mc:17109" + }, + { + "source_row_index": 17820, + "source_sample_id": "perceptiontest_val_mc:17820" + }, + { + "source_row_index": 17866, + "source_sample_id": "perceptiontest_val_mc:17866" + }, + { + "source_row_index": 18478, + "source_sample_id": "perceptiontest_val_mc:18478" + }, + { + "source_row_index": 18489, + "source_sample_id": "perceptiontest_val_mc:18489" + }, + { + "source_row_index": 18867, + "source_sample_id": "perceptiontest_val_mc:18867" + }, + { + "source_row_index": 19021, + "source_sample_id": "perceptiontest_val_mc:19021" + }, + { + "source_row_index": 19102, + "source_sample_id": "perceptiontest_val_mc:19102" + }, + { + "source_row_index": 19135, + "source_sample_id": "perceptiontest_val_mc:19135" + }, + { + "source_row_index": 19137, + "source_sample_id": "perceptiontest_val_mc:19137" + }, + { + "source_row_index": 19139, + "source_sample_id": "perceptiontest_val_mc:19139" + } + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_r1.json new file mode 100644 index 00000000000..32e9e77aa5f --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_690-examples_r1.json @@ -0,0 +1,2557 @@ +{ + "extends": "core-3_344-examples_r1", + "schema": "modelopt.vlm-sample-set/v1", + "name": "judge-free-8_690-examples_r1", + "tasks": { + "video_mmmu": { + "dataset_repository": "lmms-lab/VideoMMMU", + "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", + "max_new_tokens": 1024, + "scoring_task_config": "tasks/videommmu/video_mmmu.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "leaf_task", + "index_space": "within-stratum", + "population_rows": 900, + "selected_rows": 72, + "strata": [ + { + "name": "adaptation", + "population_rows": 300, + "selected_rows": 24 + }, + { + "name": "comprehension", + "population_rows": 300, + "selected_rows": 24 + }, + { + "name": "perception", + "population_rows": 300, + "selected_rows": 24 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 6, + "p25": 68, + "p50": 143, + "p75": 218, + "p100": 293 + }, + "selected_row_identities_sha256": "cccb57bd2f0bd914ef88a2f0ce71a8c2eafd4220bd16b94ba5f18fde6c0ab910" + }, + "rows": [ + { + "source_row_index": 6, + "source_sample_id": "adaptation:6", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 18, + "source_sample_id": "adaptation:18", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 31, + "source_sample_id": "adaptation:31", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 43, + "source_sample_id": "adaptation:43", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 56, + "source_sample_id": "adaptation:56", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 68, + "source_sample_id": "adaptation:68", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 81, + "source_sample_id": "adaptation:81", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 93, + "source_sample_id": "adaptation:93", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 106, + "source_sample_id": "adaptation:106", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 118, + "source_sample_id": "adaptation:118", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 131, + "source_sample_id": "adaptation:131", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 143, + "source_sample_id": "adaptation:143", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 156, + "source_sample_id": "adaptation:156", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 168, + "source_sample_id": "adaptation:168", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 181, + "source_sample_id": "adaptation:181", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 193, + "source_sample_id": "adaptation:193", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 206, + "source_sample_id": "adaptation:206", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 218, + "source_sample_id": "adaptation:218", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 231, + "source_sample_id": "adaptation:231", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 243, + "source_sample_id": "adaptation:243", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 256, + "source_sample_id": "adaptation:256", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 268, + "source_sample_id": "adaptation:268", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 281, + "source_sample_id": "adaptation:281", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 293, + "source_sample_id": "adaptation:293", + "leaf_task": "video_mmmu_adaptation" + }, + { + "source_row_index": 6, + "source_sample_id": "comprehension:6", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 18, + "source_sample_id": "comprehension:18", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 31, + "source_sample_id": "comprehension:31", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 43, + "source_sample_id": "comprehension:43", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 56, + "source_sample_id": "comprehension:56", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 68, + "source_sample_id": "comprehension:68", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 81, + "source_sample_id": "comprehension:81", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 93, + "source_sample_id": "comprehension:93", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 106, + "source_sample_id": "comprehension:106", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 118, + "source_sample_id": "comprehension:118", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 131, + "source_sample_id": "comprehension:131", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 143, + "source_sample_id": "comprehension:143", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 156, + "source_sample_id": "comprehension:156", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 168, + "source_sample_id": "comprehension:168", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 181, + "source_sample_id": "comprehension:181", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 193, + "source_sample_id": "comprehension:193", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 206, + "source_sample_id": "comprehension:206", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 218, + "source_sample_id": "comprehension:218", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 231, + "source_sample_id": "comprehension:231", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 243, + "source_sample_id": "comprehension:243", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 256, + "source_sample_id": "comprehension:256", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 268, + "source_sample_id": "comprehension:268", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 281, + "source_sample_id": "comprehension:281", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 293, + "source_sample_id": "comprehension:293", + "leaf_task": "video_mmmu_comprehension" + }, + { + "source_row_index": 6, + "source_sample_id": "perception:6", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 18, + "source_sample_id": "perception:18", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 31, + "source_sample_id": "perception:31", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 43, + "source_sample_id": "perception:43", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 56, + "source_sample_id": "perception:56", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 68, + "source_sample_id": "perception:68", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 81, + "source_sample_id": "perception:81", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 93, + "source_sample_id": "perception:93", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 106, + "source_sample_id": "perception:106", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 118, + "source_sample_id": "perception:118", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 131, + "source_sample_id": "perception:131", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 143, + "source_sample_id": "perception:143", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 156, + "source_sample_id": "perception:156", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 168, + "source_sample_id": "perception:168", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 181, + "source_sample_id": "perception:181", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 193, + "source_sample_id": "perception:193", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 206, + "source_sample_id": "perception:206", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 218, + "source_sample_id": "perception:218", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 231, + "source_sample_id": "perception:231", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 243, + "source_sample_id": "perception:243", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 256, + "source_sample_id": "perception:256", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 268, + "source_sample_id": "perception:268", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 281, + "source_sample_id": "perception:281", + "leaf_task": "video_mmmu_perception" + }, + { + "source_row_index": 293, + "source_sample_id": "perception:293", + "leaf_task": "video_mmmu_perception" + } + ] + }, + "videomme": { + "dataset_repository": "lmms-lab/Video-MME", + "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/videomme/videomme.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "duration+domain", + "index_space": "within-stratum", + "population_rows": 2700, + "selected_rows": 72, + "strata": [ + { + "name": "long|Artistic Performance", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "long|Film & Television", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "long|Knowledge", + "population_rows": 270, + "selected_rows": 4 + }, + { + "name": "long|Life Record", + "population_rows": 210, + "selected_rows": 4 + }, + { + "name": "long|Multilingual", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "long|Sports Competition", + "population_rows": 150, + "selected_rows": 4 + }, + { + "name": "medium|Artistic Performance", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "medium|Film & Television", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "medium|Knowledge", + "population_rows": 270, + "selected_rows": 4 + }, + { + "name": "medium|Life Record", + "population_rows": 210, + "selected_rows": 4 + }, + { + "name": "medium|Multilingual", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "medium|Sports Competition", + "population_rows": 150, + "selected_rows": 4 + }, + { + "name": "short|Artistic Performance", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "short|Film & Television", + "population_rows": 120, + "selected_rows": 4 + }, + { + "name": "short|Knowledge", + "population_rows": 270, + "selected_rows": 4 + }, + { + "name": "short|Life Record", + "population_rows": 210, + "selected_rows": 4 + }, + { + "name": "short|Multilingual", + "population_rows": 30, + "selected_rows": 4 + }, + { + "name": "short|Sports Competition", + "population_rows": 150, + "selected_rows": 4 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 3, + "p25": 18, + "p50": 56, + "p75": 105, + "p100": 236 + }, + "selected_row_identities_sha256": "fe78a3204ac1d4834fb9acfceaa18a329797cbc2bc840d22212e0de5aa319c81" + }, + "rows": [ + { + "source_row_index": 2355, + "source_sample_id": "videomme:2355", + "sampling_stratum": "long|Artistic Performance", + "source_stratum_index": 15, + "upstream_sample_id": "786-1" + }, + { + "source_row_index": 2385, + "source_sample_id": "videomme:2385", + "sampling_stratum": "long|Artistic Performance", + "source_stratum_index": 45, + "upstream_sample_id": "796-1" + }, + { + "source_row_index": 2415, + "source_sample_id": "videomme:2415", + "sampling_stratum": "long|Artistic Performance", + "source_stratum_index": 75, + "upstream_sample_id": "806-1" + }, + { + "source_row_index": 2445, + "source_sample_id": "videomme:2445", + "sampling_stratum": "long|Artistic Performance", + "source_stratum_index": 105, + "upstream_sample_id": "816-1" + }, + { + "source_row_index": 2085, + "source_sample_id": "videomme:2085", + "sampling_stratum": "long|Film & Television", + "source_stratum_index": 15, + "upstream_sample_id": "696-1" + }, + { + "source_row_index": 2115, + "source_sample_id": "videomme:2115", + "sampling_stratum": "long|Film & Television", + "source_stratum_index": 45, + "upstream_sample_id": "706-1" + }, + { + "source_row_index": 2145, + "source_sample_id": "videomme:2145", + "sampling_stratum": "long|Film & Television", + "source_stratum_index": 75, + "upstream_sample_id": "716-1" + }, + { + "source_row_index": 2175, + "source_sample_id": "videomme:2175", + "sampling_stratum": "long|Film & Television", + "source_stratum_index": 105, + "upstream_sample_id": "726-1" + }, + { + "source_row_index": 1833, + "source_sample_id": "videomme:1833", + "sampling_stratum": "long|Knowledge", + "source_stratum_index": 33, + "upstream_sample_id": "612-1" + }, + { + "source_row_index": 1901, + "source_sample_id": "videomme:1901", + "sampling_stratum": "long|Knowledge", + "source_stratum_index": 101, + "upstream_sample_id": "634-3" + }, + { + "source_row_index": 1968, + "source_sample_id": "videomme:1968", + "sampling_stratum": "long|Knowledge", + "source_stratum_index": 168, + "upstream_sample_id": "657-1" + }, + { + "source_row_index": 2036, + "source_sample_id": "videomme:2036", + "sampling_stratum": "long|Knowledge", + "source_stratum_index": 236, + "upstream_sample_id": "679-3" + }, + { + "source_row_index": 2486, + "source_sample_id": "videomme:2486", + "sampling_stratum": "long|Life Record", + "source_stratum_index": 26, + "upstream_sample_id": "829-3" + }, + { + "source_row_index": 2538, + "source_sample_id": "videomme:2538", + "sampling_stratum": "long|Life Record", + "source_stratum_index": 78, + "upstream_sample_id": "847-1" + }, + { + "source_row_index": 2591, + "source_sample_id": "videomme:2591", + "sampling_stratum": "long|Life Record", + "source_stratum_index": 131, + "upstream_sample_id": "864-3" + }, + { + "source_row_index": 2643, + "source_sample_id": "videomme:2643", + "sampling_stratum": "long|Life Record", + "source_stratum_index": 183, + "upstream_sample_id": "882-1" + }, + { + "source_row_index": 2673, + "source_sample_id": "videomme:2673", + "sampling_stratum": "long|Multilingual", + "source_stratum_index": 3, + "upstream_sample_id": "892-1" + }, + { + "source_row_index": 2681, + "source_sample_id": "videomme:2681", + "sampling_stratum": "long|Multilingual", + "source_stratum_index": 11, + "upstream_sample_id": "894-3" + }, + { + "source_row_index": 2688, + "source_sample_id": "videomme:2688", + "sampling_stratum": "long|Multilingual", + "source_stratum_index": 18, + "upstream_sample_id": "897-1" + }, + { + "source_row_index": 2696, + "source_sample_id": "videomme:2696", + "sampling_stratum": "long|Multilingual", + "source_stratum_index": 26, + "upstream_sample_id": "899-3" + }, + { + "source_row_index": 2208, + "source_sample_id": "videomme:2208", + "sampling_stratum": "long|Sports Competition", + "source_stratum_index": 18, + "upstream_sample_id": "737-1" + }, + { + "source_row_index": 2246, + "source_sample_id": "videomme:2246", + "sampling_stratum": "long|Sports Competition", + "source_stratum_index": 56, + "upstream_sample_id": "749-3" + }, + { + "source_row_index": 2283, + "source_sample_id": "videomme:2283", + "sampling_stratum": "long|Sports Competition", + "source_stratum_index": 93, + "upstream_sample_id": "762-1" + }, + { + "source_row_index": 2321, + "source_sample_id": "videomme:2321", + "sampling_stratum": "long|Sports Competition", + "source_stratum_index": 131, + "upstream_sample_id": "774-3" + }, + { + "source_row_index": 1455, + "source_sample_id": "videomme:1455", + "sampling_stratum": "medium|Artistic Performance", + "source_stratum_index": 15, + "upstream_sample_id": "486-1" + }, + { + "source_row_index": 1485, + "source_sample_id": "videomme:1485", + "sampling_stratum": "medium|Artistic Performance", + "source_stratum_index": 45, + "upstream_sample_id": "496-1" + }, + { + "source_row_index": 1515, + "source_sample_id": "videomme:1515", + "sampling_stratum": "medium|Artistic Performance", + "source_stratum_index": 75, + "upstream_sample_id": "506-1" + }, + { + "source_row_index": 1545, + "source_sample_id": "videomme:1545", + "sampling_stratum": "medium|Artistic Performance", + "source_stratum_index": 105, + "upstream_sample_id": "516-1" + }, + { + "source_row_index": 1185, + "source_sample_id": "videomme:1185", + "sampling_stratum": "medium|Film & Television", + "source_stratum_index": 15, + "upstream_sample_id": "396-1" + }, + { + "source_row_index": 1215, + "source_sample_id": "videomme:1215", + "sampling_stratum": "medium|Film & Television", + "source_stratum_index": 45, + "upstream_sample_id": "406-1" + }, + { + "source_row_index": 1245, + "source_sample_id": "videomme:1245", + "sampling_stratum": "medium|Film & Television", + "source_stratum_index": 75, + "upstream_sample_id": "416-1" + }, + { + "source_row_index": 1275, + "source_sample_id": "videomme:1275", + "sampling_stratum": "medium|Film & Television", + "source_stratum_index": 105, + "upstream_sample_id": "426-1" + }, + { + "source_row_index": 933, + "source_sample_id": "videomme:933", + "sampling_stratum": "medium|Knowledge", + "source_stratum_index": 33, + "upstream_sample_id": "312-1" + }, + { + "source_row_index": 1001, + "source_sample_id": "videomme:1001", + "sampling_stratum": "medium|Knowledge", + "source_stratum_index": 101, + "upstream_sample_id": "334-3" + }, + { + "source_row_index": 1068, + "source_sample_id": "videomme:1068", + "sampling_stratum": "medium|Knowledge", + "source_stratum_index": 168, + "upstream_sample_id": "357-1" + }, + { + "source_row_index": 1136, + "source_sample_id": "videomme:1136", + "sampling_stratum": "medium|Knowledge", + "source_stratum_index": 236, + "upstream_sample_id": "379-3" + }, + { + "source_row_index": 1586, + "source_sample_id": "videomme:1586", + "sampling_stratum": "medium|Life Record", + "source_stratum_index": 26, + "upstream_sample_id": "529-3" + }, + { + "source_row_index": 1638, + "source_sample_id": "videomme:1638", + "sampling_stratum": "medium|Life Record", + "source_stratum_index": 78, + "upstream_sample_id": "547-1" + }, + { + "source_row_index": 1691, + "source_sample_id": "videomme:1691", + "sampling_stratum": "medium|Life Record", + "source_stratum_index": 131, + "upstream_sample_id": "564-3" + }, + { + "source_row_index": 1743, + "source_sample_id": "videomme:1743", + "sampling_stratum": "medium|Life Record", + "source_stratum_index": 183, + "upstream_sample_id": "582-1" + }, + { + "source_row_index": 1773, + "source_sample_id": "videomme:1773", + "sampling_stratum": "medium|Multilingual", + "source_stratum_index": 3, + "upstream_sample_id": "592-1" + }, + { + "source_row_index": 1781, + "source_sample_id": "videomme:1781", + "sampling_stratum": "medium|Multilingual", + "source_stratum_index": 11, + "upstream_sample_id": "594-3" + }, + { + "source_row_index": 1788, + "source_sample_id": "videomme:1788", + "sampling_stratum": "medium|Multilingual", + "source_stratum_index": 18, + "upstream_sample_id": "597-1" + }, + { + "source_row_index": 1796, + "source_sample_id": "videomme:1796", + "sampling_stratum": "medium|Multilingual", + "source_stratum_index": 26, + "upstream_sample_id": "599-3" + }, + { + "source_row_index": 1308, + "source_sample_id": "videomme:1308", + "sampling_stratum": "medium|Sports Competition", + "source_stratum_index": 18, + "upstream_sample_id": "437-1" + }, + { + "source_row_index": 1346, + "source_sample_id": "videomme:1346", + "sampling_stratum": "medium|Sports Competition", + "source_stratum_index": 56, + "upstream_sample_id": "449-3" + }, + { + "source_row_index": 1383, + "source_sample_id": "videomme:1383", + "sampling_stratum": "medium|Sports Competition", + "source_stratum_index": 93, + "upstream_sample_id": "462-1" + }, + { + "source_row_index": 1421, + "source_sample_id": "videomme:1421", + "sampling_stratum": "medium|Sports Competition", + "source_stratum_index": 131, + "upstream_sample_id": "474-3" + }, + { + "source_row_index": 555, + "source_sample_id": "videomme:555", + "sampling_stratum": "short|Artistic Performance", + "source_stratum_index": 15, + "upstream_sample_id": "186-1" + }, + { + "source_row_index": 585, + "source_sample_id": "videomme:585", + "sampling_stratum": "short|Artistic Performance", + "source_stratum_index": 45, + "upstream_sample_id": "196-1" + }, + { + "source_row_index": 615, + "source_sample_id": "videomme:615", + "sampling_stratum": "short|Artistic Performance", + "source_stratum_index": 75, + "upstream_sample_id": "206-1" + }, + { + "source_row_index": 645, + "source_sample_id": "videomme:645", + "sampling_stratum": "short|Artistic Performance", + "source_stratum_index": 105, + "upstream_sample_id": "216-1" + }, + { + "source_row_index": 285, + "source_sample_id": "videomme:285", + "sampling_stratum": "short|Film & Television", + "source_stratum_index": 15, + "upstream_sample_id": "096-1" + }, + { + "source_row_index": 315, + "source_sample_id": "videomme:315", + "sampling_stratum": "short|Film & Television", + "source_stratum_index": 45, + "upstream_sample_id": "106-1" + }, + { + "source_row_index": 345, + "source_sample_id": "videomme:345", + "sampling_stratum": "short|Film & Television", + "source_stratum_index": 75, + "upstream_sample_id": "116-1" + }, + { + "source_row_index": 375, + "source_sample_id": "videomme:375", + "sampling_stratum": "short|Film & Television", + "source_stratum_index": 105, + "upstream_sample_id": "126-1" + }, + { + "source_row_index": 33, + "source_sample_id": "videomme:33", + "sampling_stratum": "short|Knowledge", + "source_stratum_index": 33, + "upstream_sample_id": "012-1" + }, + { + "source_row_index": 101, + "source_sample_id": "videomme:101", + "sampling_stratum": "short|Knowledge", + "source_stratum_index": 101, + "upstream_sample_id": "034-3" + }, + { + "source_row_index": 168, + "source_sample_id": "videomme:168", + "sampling_stratum": "short|Knowledge", + "source_stratum_index": 168, + "upstream_sample_id": "057-1" + }, + { + "source_row_index": 236, + "source_sample_id": "videomme:236", + "sampling_stratum": "short|Knowledge", + "source_stratum_index": 236, + "upstream_sample_id": "079-3" + }, + { + "source_row_index": 686, + "source_sample_id": "videomme:686", + "sampling_stratum": "short|Life Record", + "source_stratum_index": 26, + "upstream_sample_id": "229-3" + }, + { + "source_row_index": 738, + "source_sample_id": "videomme:738", + "sampling_stratum": "short|Life Record", + "source_stratum_index": 78, + "upstream_sample_id": "247-1" + }, + { + "source_row_index": 791, + "source_sample_id": "videomme:791", + "sampling_stratum": "short|Life Record", + "source_stratum_index": 131, + "upstream_sample_id": "264-3" + }, + { + "source_row_index": 843, + "source_sample_id": "videomme:843", + "sampling_stratum": "short|Life Record", + "source_stratum_index": 183, + "upstream_sample_id": "282-1" + }, + { + "source_row_index": 873, + "source_sample_id": "videomme:873", + "sampling_stratum": "short|Multilingual", + "source_stratum_index": 3, + "upstream_sample_id": "292-1" + }, + { + "source_row_index": 881, + "source_sample_id": "videomme:881", + "sampling_stratum": "short|Multilingual", + "source_stratum_index": 11, + "upstream_sample_id": "294-3" + }, + { + "source_row_index": 888, + "source_sample_id": "videomme:888", + "sampling_stratum": "short|Multilingual", + "source_stratum_index": 18, + "upstream_sample_id": "297-1" + }, + { + "source_row_index": 896, + "source_sample_id": "videomme:896", + "sampling_stratum": "short|Multilingual", + "source_stratum_index": 26, + "upstream_sample_id": "299-3" + }, + { + "source_row_index": 408, + "source_sample_id": "videomme:408", + "sampling_stratum": "short|Sports Competition", + "source_stratum_index": 18, + "upstream_sample_id": "137-1" + }, + { + "source_row_index": 446, + "source_sample_id": "videomme:446", + "sampling_stratum": "short|Sports Competition", + "source_stratum_index": 56, + "upstream_sample_id": "149-3" + }, + { + "source_row_index": 483, + "source_sample_id": "videomme:483", + "sampling_stratum": "short|Sports Competition", + "source_stratum_index": 93, + "upstream_sample_id": "162-1" + }, + { + "source_row_index": 521, + "source_sample_id": "videomme:521", + "sampling_stratum": "short|Sports Competition", + "source_stratum_index": 131, + "upstream_sample_id": "174-3" + } + ] + }, + "longvideobench_val_v": { + "dataset_repository": "longvideobench/LongVideoBench", + "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", + "max_new_tokens": 32, + "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "split", + "index_space": "within-stratum", + "population_rows": 1337, + "selected_rows": 68, + "strata": [ + { + "name": "validation", + "population_rows": 1337, + "selected_rows": 68 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 9, + "p25": 324, + "p50": 658, + "p75": 992, + "p100": 1327 + }, + "selected_row_identities_sha256": "124e56d414789cb93e0a8f50d51b2b212972bb6e89f5174663afc55f843c50c2" + }, + "rows": [ + { + "source_row_index": 9, + "source_sample_id": "longvideobench_val_v:9", + "sampling_stratum": "validation", + "source_stratum_index": 9 + }, + { + "source_row_index": 29, + "source_sample_id": "longvideobench_val_v:29", + "sampling_stratum": "validation", + "source_stratum_index": 29 + }, + { + "source_row_index": 49, + "source_sample_id": "longvideobench_val_v:49", + "sampling_stratum": "validation", + "source_stratum_index": 49 + }, + { + "source_row_index": 68, + "source_sample_id": "longvideobench_val_v:68", + "sampling_stratum": "validation", + "source_stratum_index": 68 + }, + { + "source_row_index": 88, + "source_sample_id": "longvideobench_val_v:88", + "sampling_stratum": "validation", + "source_stratum_index": 88 + }, + { + "source_row_index": 108, + "source_sample_id": "longvideobench_val_v:108", + "sampling_stratum": "validation", + "source_stratum_index": 108 + }, + { + "source_row_index": 127, + "source_sample_id": "longvideobench_val_v:127", + "sampling_stratum": "validation", + "source_stratum_index": 127 + }, + { + "source_row_index": 147, + "source_sample_id": "longvideobench_val_v:147", + "sampling_stratum": "validation", + "source_stratum_index": 147 + }, + { + "source_row_index": 167, + "source_sample_id": "longvideobench_val_v:167", + "sampling_stratum": "validation", + "source_stratum_index": 167 + }, + { + "source_row_index": 186, + "source_sample_id": "longvideobench_val_v:186", + "sampling_stratum": "validation", + "source_stratum_index": 186 + }, + { + "source_row_index": 206, + "source_sample_id": "longvideobench_val_v:206", + "sampling_stratum": "validation", + "source_stratum_index": 206 + }, + { + "source_row_index": 226, + "source_sample_id": "longvideobench_val_v:226", + "sampling_stratum": "validation", + "source_stratum_index": 226 + }, + { + "source_row_index": 245, + "source_sample_id": "longvideobench_val_v:245", + "sampling_stratum": "validation", + "source_stratum_index": 245 + }, + { + "source_row_index": 265, + "source_sample_id": "longvideobench_val_v:265", + "sampling_stratum": "validation", + "source_stratum_index": 265 + }, + { + "source_row_index": 285, + "source_sample_id": "longvideobench_val_v:285", + "sampling_stratum": "validation", + "source_stratum_index": 285 + }, + { + "source_row_index": 304, + "source_sample_id": "longvideobench_val_v:304", + "sampling_stratum": "validation", + "source_stratum_index": 304 + }, + { + "source_row_index": 324, + "source_sample_id": "longvideobench_val_v:324", + "sampling_stratum": "validation", + "source_stratum_index": 324 + }, + { + "source_row_index": 344, + "source_sample_id": "longvideobench_val_v:344", + "sampling_stratum": "validation", + "source_stratum_index": 344 + }, + { + "source_row_index": 363, + "source_sample_id": "longvideobench_val_v:363", + "sampling_stratum": "validation", + "source_stratum_index": 363 + }, + { + "source_row_index": 383, + "source_sample_id": "longvideobench_val_v:383", + "sampling_stratum": "validation", + "source_stratum_index": 383 + }, + { + "source_row_index": 403, + "source_sample_id": "longvideobench_val_v:403", + "sampling_stratum": "validation", + "source_stratum_index": 403 + }, + { + "source_row_index": 422, + "source_sample_id": "longvideobench_val_v:422", + "sampling_stratum": "validation", + "source_stratum_index": 422 + }, + { + "source_row_index": 442, + "source_sample_id": "longvideobench_val_v:442", + "sampling_stratum": "validation", + "source_stratum_index": 442 + }, + { + "source_row_index": 462, + "source_sample_id": "longvideobench_val_v:462", + "sampling_stratum": "validation", + "source_stratum_index": 462 + }, + { + "source_row_index": 481, + "source_sample_id": "longvideobench_val_v:481", + "sampling_stratum": "validation", + "source_stratum_index": 481 + }, + { + "source_row_index": 501, + "source_sample_id": "longvideobench_val_v:501", + "sampling_stratum": "validation", + "source_stratum_index": 501 + }, + { + "source_row_index": 521, + "source_sample_id": "longvideobench_val_v:521", + "sampling_stratum": "validation", + "source_stratum_index": 521 + }, + { + "source_row_index": 540, + "source_sample_id": "longvideobench_val_v:540", + "sampling_stratum": "validation", + "source_stratum_index": 540 + }, + { + "source_row_index": 560, + "source_sample_id": "longvideobench_val_v:560", + "sampling_stratum": "validation", + "source_stratum_index": 560 + }, + { + "source_row_index": 580, + "source_sample_id": "longvideobench_val_v:580", + "sampling_stratum": "validation", + "source_stratum_index": 580 + }, + { + "source_row_index": 599, + "source_sample_id": "longvideobench_val_v:599", + "sampling_stratum": "validation", + "source_stratum_index": 599 + }, + { + "source_row_index": 619, + "source_sample_id": "longvideobench_val_v:619", + "sampling_stratum": "validation", + "source_stratum_index": 619 + }, + { + "source_row_index": 639, + "source_sample_id": "longvideobench_val_v:639", + "sampling_stratum": "validation", + "source_stratum_index": 639 + }, + { + "source_row_index": 658, + "source_sample_id": "longvideobench_val_v:658", + "sampling_stratum": "validation", + "source_stratum_index": 658 + }, + { + "source_row_index": 678, + "source_sample_id": "longvideobench_val_v:678", + "sampling_stratum": "validation", + "source_stratum_index": 678 + }, + { + "source_row_index": 697, + "source_sample_id": "longvideobench_val_v:697", + "sampling_stratum": "validation", + "source_stratum_index": 697 + }, + { + "source_row_index": 717, + "source_sample_id": "longvideobench_val_v:717", + "sampling_stratum": "validation", + "source_stratum_index": 717 + }, + { + "source_row_index": 737, + "source_sample_id": "longvideobench_val_v:737", + "sampling_stratum": "validation", + "source_stratum_index": 737 + }, + { + "source_row_index": 756, + "source_sample_id": "longvideobench_val_v:756", + "sampling_stratum": "validation", + "source_stratum_index": 756 + }, + { + "source_row_index": 776, + "source_sample_id": "longvideobench_val_v:776", + "sampling_stratum": "validation", + "source_stratum_index": 776 + }, + { + "source_row_index": 796, + "source_sample_id": "longvideobench_val_v:796", + "sampling_stratum": "validation", + "source_stratum_index": 796 + }, + { + "source_row_index": 815, + "source_sample_id": "longvideobench_val_v:815", + "sampling_stratum": "validation", + "source_stratum_index": 815 + }, + { + "source_row_index": 835, + "source_sample_id": "longvideobench_val_v:835", + "sampling_stratum": "validation", + "source_stratum_index": 835 + }, + { + "source_row_index": 855, + "source_sample_id": "longvideobench_val_v:855", + "sampling_stratum": "validation", + "source_stratum_index": 855 + }, + { + "source_row_index": 874, + "source_sample_id": "longvideobench_val_v:874", + "sampling_stratum": "validation", + "source_stratum_index": 874 + }, + { + "source_row_index": 894, + "source_sample_id": "longvideobench_val_v:894", + "sampling_stratum": "validation", + "source_stratum_index": 894 + }, + { + "source_row_index": 914, + "source_sample_id": "longvideobench_val_v:914", + "sampling_stratum": "validation", + "source_stratum_index": 914 + }, + { + "source_row_index": 933, + "source_sample_id": "longvideobench_val_v:933", + "sampling_stratum": "validation", + "source_stratum_index": 933 + }, + { + "source_row_index": 953, + "source_sample_id": "longvideobench_val_v:953", + "sampling_stratum": "validation", + "source_stratum_index": 953 + }, + { + "source_row_index": 973, + "source_sample_id": "longvideobench_val_v:973", + "sampling_stratum": "validation", + "source_stratum_index": 973 + }, + { + "source_row_index": 992, + "source_sample_id": "longvideobench_val_v:992", + "sampling_stratum": "validation", + "source_stratum_index": 992 + }, + { + "source_row_index": 1012, + "source_sample_id": "longvideobench_val_v:1012", + "sampling_stratum": "validation", + "source_stratum_index": 1012 + }, + { + "source_row_index": 1032, + "source_sample_id": "longvideobench_val_v:1032", + "sampling_stratum": "validation", + "source_stratum_index": 1032 + }, + { + "source_row_index": 1051, + "source_sample_id": "longvideobench_val_v:1051", + "sampling_stratum": "validation", + "source_stratum_index": 1051 + }, + { + "source_row_index": 1071, + "source_sample_id": "longvideobench_val_v:1071", + "sampling_stratum": "validation", + "source_stratum_index": 1071 + }, + { + "source_row_index": 1091, + "source_sample_id": "longvideobench_val_v:1091", + "sampling_stratum": "validation", + "source_stratum_index": 1091 + }, + { + "source_row_index": 1110, + "source_sample_id": "longvideobench_val_v:1110", + "sampling_stratum": "validation", + "source_stratum_index": 1110 + }, + { + "source_row_index": 1130, + "source_sample_id": "longvideobench_val_v:1130", + "sampling_stratum": "validation", + "source_stratum_index": 1130 + }, + { + "source_row_index": 1150, + "source_sample_id": "longvideobench_val_v:1150", + "sampling_stratum": "validation", + "source_stratum_index": 1150 + }, + { + "source_row_index": 1169, + "source_sample_id": "longvideobench_val_v:1169", + "sampling_stratum": "validation", + "source_stratum_index": 1169 + }, + { + "source_row_index": 1189, + "source_sample_id": "longvideobench_val_v:1189", + "sampling_stratum": "validation", + "source_stratum_index": 1189 + }, + { + "source_row_index": 1209, + "source_sample_id": "longvideobench_val_v:1209", + "sampling_stratum": "validation", + "source_stratum_index": 1209 + }, + { + "source_row_index": 1228, + "source_sample_id": "longvideobench_val_v:1228", + "sampling_stratum": "validation", + "source_stratum_index": 1228 + }, + { + "source_row_index": 1248, + "source_sample_id": "longvideobench_val_v:1248", + "sampling_stratum": "validation", + "source_stratum_index": 1248 + }, + { + "source_row_index": 1268, + "source_sample_id": "longvideobench_val_v:1268", + "sampling_stratum": "validation", + "source_stratum_index": 1268 + }, + { + "source_row_index": 1287, + "source_sample_id": "longvideobench_val_v:1287", + "sampling_stratum": "validation", + "source_stratum_index": 1287 + }, + { + "source_row_index": 1307, + "source_sample_id": "longvideobench_val_v:1307", + "sampling_stratum": "validation", + "source_stratum_index": 1307 + }, + { + "source_row_index": 1327, + "source_sample_id": "longvideobench_val_v:1327", + "sampling_stratum": "validation", + "source_stratum_index": 1327 + } + ] + }, + "mlvu_dev": { + "dataset_repository": "sy1998/MLVU_dev", + "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "task_type", + "index_space": "within-stratum", + "population_rows": 2174, + "selected_rows": 70, + "strata": [ + { + "name": "anomaly_reco", + "population_rows": 200, + "selected_rows": 10 + }, + { + "name": "count", + "population_rows": 206, + "selected_rows": 10 + }, + { + "name": "ego", + "population_rows": 352, + "selected_rows": 10 + }, + { + "name": "needle", + "population_rows": 355, + "selected_rows": 10 + }, + { + "name": "order", + "population_rows": 259, + "selected_rows": 10 + }, + { + "name": "plotQA", + "population_rows": 539, + "selected_rows": 10 + }, + { + "name": "topic_reasoning", + "population_rows": 263, + "selected_rows": 10 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 10, + "p25": 70, + "p50": 134, + "p75": 197, + "p100": 512 + }, + "selected_row_identities_sha256": "7951e7c5fd3671e2d29e1634a372958c9de15aa363650a4d72e336ea4b371c80" + }, + "rows": [ + { + "source_row_index": 365, + "source_sample_id": "mlvu_dev:365", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 10, + "upstream_sample_id": "Q365" + }, + { + "source_row_index": 385, + "source_sample_id": "mlvu_dev:385", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 30, + "upstream_sample_id": "Q385" + }, + { + "source_row_index": 405, + "source_sample_id": "mlvu_dev:405", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 50, + "upstream_sample_id": "Q405" + }, + { + "source_row_index": 425, + "source_sample_id": "mlvu_dev:425", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 70, + "upstream_sample_id": "Q425" + }, + { + "source_row_index": 445, + "source_sample_id": "mlvu_dev:445", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 90, + "upstream_sample_id": "Q445" + }, + { + "source_row_index": 465, + "source_sample_id": "mlvu_dev:465", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 110, + "upstream_sample_id": "Q465" + }, + { + "source_row_index": 485, + "source_sample_id": "mlvu_dev:485", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 130, + "upstream_sample_id": "Q485" + }, + { + "source_row_index": 505, + "source_sample_id": "mlvu_dev:505", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 150, + "upstream_sample_id": "Q505" + }, + { + "source_row_index": 525, + "source_sample_id": "mlvu_dev:525", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 170, + "upstream_sample_id": "Q525" + }, + { + "source_row_index": 545, + "source_sample_id": "mlvu_dev:545", + "sampling_stratum": "anomaly_reco", + "source_stratum_index": 190, + "upstream_sample_id": "Q545" + }, + { + "source_row_index": 565, + "source_sample_id": "mlvu_dev:565", + "sampling_stratum": "count", + "source_stratum_index": 10, + "upstream_sample_id": "Q565" + }, + { + "source_row_index": 585, + "source_sample_id": "mlvu_dev:585", + "sampling_stratum": "count", + "source_stratum_index": 30, + "upstream_sample_id": "Q585" + }, + { + "source_row_index": 606, + "source_sample_id": "mlvu_dev:606", + "sampling_stratum": "count", + "source_stratum_index": 51, + "upstream_sample_id": "Q606" + }, + { + "source_row_index": 627, + "source_sample_id": "mlvu_dev:627", + "sampling_stratum": "count", + "source_stratum_index": 72, + "upstream_sample_id": "Q627" + }, + { + "source_row_index": 647, + "source_sample_id": "mlvu_dev:647", + "sampling_stratum": "count", + "source_stratum_index": 92, + "upstream_sample_id": "Q647" + }, + { + "source_row_index": 668, + "source_sample_id": "mlvu_dev:668", + "sampling_stratum": "count", + "source_stratum_index": 113, + "upstream_sample_id": "Q668" + }, + { + "source_row_index": 688, + "source_sample_id": "mlvu_dev:688", + "sampling_stratum": "count", + "source_stratum_index": 133, + "upstream_sample_id": "Q688" + }, + { + "source_row_index": 709, + "source_sample_id": "mlvu_dev:709", + "sampling_stratum": "count", + "source_stratum_index": 154, + "upstream_sample_id": "Q709" + }, + { + "source_row_index": 730, + "source_sample_id": "mlvu_dev:730", + "sampling_stratum": "count", + "source_stratum_index": 175, + "upstream_sample_id": "Q730" + }, + { + "source_row_index": 750, + "source_sample_id": "mlvu_dev:750", + "sampling_stratum": "count", + "source_stratum_index": 195, + "upstream_sample_id": "Q750" + }, + { + "source_row_index": 778, + "source_sample_id": "mlvu_dev:778", + "sampling_stratum": "ego", + "source_stratum_index": 17, + "upstream_sample_id": "Q778" + }, + { + "source_row_index": 813, + "source_sample_id": "mlvu_dev:813", + "sampling_stratum": "ego", + "source_stratum_index": 52, + "upstream_sample_id": "Q813" + }, + { + "source_row_index": 849, + "source_sample_id": "mlvu_dev:849", + "sampling_stratum": "ego", + "source_stratum_index": 88, + "upstream_sample_id": "Q849" + }, + { + "source_row_index": 884, + "source_sample_id": "mlvu_dev:884", + "sampling_stratum": "ego", + "source_stratum_index": 123, + "upstream_sample_id": "Q884" + }, + { + "source_row_index": 919, + "source_sample_id": "mlvu_dev:919", + "sampling_stratum": "ego", + "source_stratum_index": 158, + "upstream_sample_id": "Q919" + }, + { + "source_row_index": 954, + "source_sample_id": "mlvu_dev:954", + "sampling_stratum": "ego", + "source_stratum_index": 193, + "upstream_sample_id": "Q954" + }, + { + "source_row_index": 989, + "source_sample_id": "mlvu_dev:989", + "sampling_stratum": "ego", + "source_stratum_index": 228, + "upstream_sample_id": "Q989" + }, + { + "source_row_index": 1025, + "source_sample_id": "mlvu_dev:1025", + "sampling_stratum": "ego", + "source_stratum_index": 264, + "upstream_sample_id": "Q1025" + }, + { + "source_row_index": 1060, + "source_sample_id": "mlvu_dev:1060", + "sampling_stratum": "ego", + "source_stratum_index": 299, + "upstream_sample_id": "Q1060" + }, + { + "source_row_index": 1095, + "source_sample_id": "mlvu_dev:1095", + "sampling_stratum": "ego", + "source_stratum_index": 334, + "upstream_sample_id": "Q1095" + }, + { + "source_row_index": 17, + "source_sample_id": "mlvu_dev:17", + "sampling_stratum": "needle", + "source_stratum_index": 17, + "upstream_sample_id": "Q17" + }, + { + "source_row_index": 53, + "source_sample_id": "mlvu_dev:53", + "sampling_stratum": "needle", + "source_stratum_index": 53, + "upstream_sample_id": "Q53" + }, + { + "source_row_index": 88, + "source_sample_id": "mlvu_dev:88", + "sampling_stratum": "needle", + "source_stratum_index": 88, + "upstream_sample_id": "Q88" + }, + { + "source_row_index": 124, + "source_sample_id": "mlvu_dev:124", + "sampling_stratum": "needle", + "source_stratum_index": 124, + "upstream_sample_id": "Q124" + }, + { + "source_row_index": 159, + "source_sample_id": "mlvu_dev:159", + "sampling_stratum": "needle", + "source_stratum_index": 159, + "upstream_sample_id": "Q159" + }, + { + "source_row_index": 195, + "source_sample_id": "mlvu_dev:195", + "sampling_stratum": "needle", + "source_stratum_index": 195, + "upstream_sample_id": "Q195" + }, + { + "source_row_index": 230, + "source_sample_id": "mlvu_dev:230", + "sampling_stratum": "needle", + "source_stratum_index": 230, + "upstream_sample_id": "Q230" + }, + { + "source_row_index": 266, + "source_sample_id": "mlvu_dev:266", + "sampling_stratum": "needle", + "source_stratum_index": 266, + "upstream_sample_id": "Q266" + }, + { + "source_row_index": 301, + "source_sample_id": "mlvu_dev:301", + "sampling_stratum": "needle", + "source_stratum_index": 301, + "upstream_sample_id": "Q301" + }, + { + "source_row_index": 337, + "source_sample_id": "mlvu_dev:337", + "sampling_stratum": "needle", + "source_stratum_index": 337, + "upstream_sample_id": "Q337" + }, + { + "source_row_index": 1927, + "source_sample_id": "mlvu_dev:1927", + "sampling_stratum": "order", + "source_stratum_index": 12, + "upstream_sample_id": "Q1927" + }, + { + "source_row_index": 1953, + "source_sample_id": "mlvu_dev:1953", + "sampling_stratum": "order", + "source_stratum_index": 38, + "upstream_sample_id": "Q1953" + }, + { + "source_row_index": 1979, + "source_sample_id": "mlvu_dev:1979", + "sampling_stratum": "order", + "source_stratum_index": 64, + "upstream_sample_id": "Q1979" + }, + { + "source_row_index": 2005, + "source_sample_id": "mlvu_dev:2005", + "sampling_stratum": "order", + "source_stratum_index": 90, + "upstream_sample_id": "Q2005" + }, + { + "source_row_index": 2031, + "source_sample_id": "mlvu_dev:2031", + "sampling_stratum": "order", + "source_stratum_index": 116, + "upstream_sample_id": "Q2031" + }, + { + "source_row_index": 2057, + "source_sample_id": "mlvu_dev:2057", + "sampling_stratum": "order", + "source_stratum_index": 142, + "upstream_sample_id": "Q2057" + }, + { + "source_row_index": 2083, + "source_sample_id": "mlvu_dev:2083", + "sampling_stratum": "order", + "source_stratum_index": 168, + "upstream_sample_id": "Q2083" + }, + { + "source_row_index": 2109, + "source_sample_id": "mlvu_dev:2109", + "sampling_stratum": "order", + "source_stratum_index": 194, + "upstream_sample_id": "Q2109" + }, + { + "source_row_index": 2135, + "source_sample_id": "mlvu_dev:2135", + "sampling_stratum": "order", + "source_stratum_index": 220, + "upstream_sample_id": "Q2135" + }, + { + "source_row_index": 2161, + "source_sample_id": "mlvu_dev:2161", + "sampling_stratum": "order", + "source_stratum_index": 246, + "upstream_sample_id": "Q2161" + }, + { + "source_row_index": 1139, + "source_sample_id": "mlvu_dev:1139", + "sampling_stratum": "plotQA", + "source_stratum_index": 26, + "upstream_sample_id": "Q1139" + }, + { + "source_row_index": 1193, + "source_sample_id": "mlvu_dev:1193", + "sampling_stratum": "plotQA", + "source_stratum_index": 80, + "upstream_sample_id": "Q1193" + }, + { + "source_row_index": 1247, + "source_sample_id": "mlvu_dev:1247", + "sampling_stratum": "plotQA", + "source_stratum_index": 134, + "upstream_sample_id": "Q1247" + }, + { + "source_row_index": 1301, + "source_sample_id": "mlvu_dev:1301", + "sampling_stratum": "plotQA", + "source_stratum_index": 188, + "upstream_sample_id": "Q1301" + }, + { + "source_row_index": 1355, + "source_sample_id": "mlvu_dev:1355", + "sampling_stratum": "plotQA", + "source_stratum_index": 242, + "upstream_sample_id": "Q1355" + }, + { + "source_row_index": 1409, + "source_sample_id": "mlvu_dev:1409", + "sampling_stratum": "plotQA", + "source_stratum_index": 296, + "upstream_sample_id": "Q1409" + }, + { + "source_row_index": 1463, + "source_sample_id": "mlvu_dev:1463", + "sampling_stratum": "plotQA", + "source_stratum_index": 350, + "upstream_sample_id": "Q1463" + }, + { + "source_row_index": 1517, + "source_sample_id": "mlvu_dev:1517", + "sampling_stratum": "plotQA", + "source_stratum_index": 404, + "upstream_sample_id": "Q1517" + }, + { + "source_row_index": 1571, + "source_sample_id": "mlvu_dev:1571", + "sampling_stratum": "plotQA", + "source_stratum_index": 458, + "upstream_sample_id": "Q1571" + }, + { + "source_row_index": 1625, + "source_sample_id": "mlvu_dev:1625", + "sampling_stratum": "plotQA", + "source_stratum_index": 512, + "upstream_sample_id": "Q1625" + }, + { + "source_row_index": 1665, + "source_sample_id": "mlvu_dev:1665", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 13, + "upstream_sample_id": "Q1665" + }, + { + "source_row_index": 1691, + "source_sample_id": "mlvu_dev:1691", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 39, + "upstream_sample_id": "Q1691" + }, + { + "source_row_index": 1717, + "source_sample_id": "mlvu_dev:1717", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 65, + "upstream_sample_id": "Q1717" + }, + { + "source_row_index": 1744, + "source_sample_id": "mlvu_dev:1744", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 92, + "upstream_sample_id": "Q1744" + }, + { + "source_row_index": 1770, + "source_sample_id": "mlvu_dev:1770", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 118, + "upstream_sample_id": "Q1770" + }, + { + "source_row_index": 1796, + "source_sample_id": "mlvu_dev:1796", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 144, + "upstream_sample_id": "Q1796" + }, + { + "source_row_index": 1822, + "source_sample_id": "mlvu_dev:1822", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 170, + "upstream_sample_id": "Q1822" + }, + { + "source_row_index": 1849, + "source_sample_id": "mlvu_dev:1849", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 197, + "upstream_sample_id": "Q1849" + }, + { + "source_row_index": 1875, + "source_sample_id": "mlvu_dev:1875", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 223, + "upstream_sample_id": "Q1875" + }, + { + "source_row_index": 1901, + "source_sample_id": "mlvu_dev:1901", + "sampling_stratum": "topic_reasoning", + "source_stratum_index": 249, + "upstream_sample_id": "Q1901" + } + ] + }, + "perceptiontest_val_mc": { + "dataset_repository": "lmms-lab/PerceptionTest_Val", + "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml", + "selection": { + "method": "systematic-midpoint", + "stratified_by": "area+reasoning", + "index_space": "within-stratum", + "population_rows": 19140, + "selected_rows": 64, + "strata": [ + { + "name": "abstraction|counterfactual", + "population_rows": 83, + "selected_rows": 5 + }, + { + "name": "abstraction|descriptive", + "population_rows": 5268, + "selected_rows": 5 + }, + { + "name": "abstraction|predictive", + "population_rows": 256, + "selected_rows": 5 + }, + { + "name": "memory|counterfactual", + "population_rows": 36, + "selected_rows": 5 + }, + { + "name": "memory|descriptive", + "population_rows": 575, + "selected_rows": 5 + }, + { + "name": "memory|explanatory", + "population_rows": 135, + "selected_rows": 5 + }, + { + "name": "physics|counterfactual", + "population_rows": 31, + "selected_rows": 5 + }, + { + "name": "physics|descriptive", + "population_rows": 5298, + "selected_rows": 5 + }, + { + "name": "physics|explanatory", + "population_rows": 1226, + "selected_rows": 5 + }, + { + "name": "physics|predictive", + "population_rows": 383, + "selected_rows": 5 + }, + { + "name": "semantics|counterfactual", + "population_rows": 207, + "selected_rows": 5 + }, + { + "name": "semantics|descriptive", + "population_rows": 4717, + "selected_rows": 5 + }, + { + "name": "semantics|explanatory", + "population_rows": 925, + "selected_rows": 4 + } + ], + "selected_index_quantiles": { + "method": "lower-order-statistic", + "p0": 3, + "p25": 38, + "p50": 144, + "p75": 578, + "p100": 4768 + }, + "selected_row_identities_sha256": "7e2d17ac3bff117d51cfc27df79cb7d86e3e3ac7c3d32d94da7b1283befe1594" + }, + "rows": [ + { + "source_row_index": 6448, + "source_sample_id": "perceptiontest_val_mc:6448", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 8, + "upstream_sample_id": "video_4364:3" + }, + { + "source_row_index": 6519, + "source_sample_id": "perceptiontest_val_mc:6519", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 24, + "upstream_sample_id": "video_7896:3" + }, + { + "source_row_index": 6598, + "source_sample_id": "perceptiontest_val_mc:6598", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 41, + "upstream_sample_id": "video_10230:3" + }, + { + "source_row_index": 16262, + "source_sample_id": "perceptiontest_val_mc:16262", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 58, + "upstream_sample_id": "video_9279:3" + }, + { + "source_row_index": 16332, + "source_sample_id": "perceptiontest_val_mc:16332", + "sampling_stratum": "abstraction|counterfactual", + "source_stratum_index": 74, + "upstream_sample_id": "video_612:3" + }, + { + "source_row_index": 2460, + "source_sample_id": "perceptiontest_val_mc:2460", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 526, + "upstream_sample_id": "video_3553:2" + }, + { + "source_row_index": 5068, + "source_sample_id": "perceptiontest_val_mc:5068", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 1580, + "upstream_sample_id": "video_11322:2" + }, + { + "source_row_index": 8865, + "source_sample_id": "perceptiontest_val_mc:8865", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 2634, + "upstream_sample_id": "video_1211:2" + }, + { + "source_row_index": 13017, + "source_sample_id": "perceptiontest_val_mc:13017", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 3687, + "upstream_sample_id": "video_7836:1" + }, + { + "source_row_index": 17745, + "source_sample_id": "perceptiontest_val_mc:17745", + "sampling_stratum": "abstraction|descriptive", + "source_stratum_index": 4741, + "upstream_sample_id": "video_4387:1" + }, + { + "source_row_index": 12890, + "source_sample_id": "perceptiontest_val_mc:12890", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 25, + "upstream_sample_id": "video_10565:3" + }, + { + "source_row_index": 13117, + "source_sample_id": "perceptiontest_val_mc:13117", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 76, + "upstream_sample_id": "video_9648:1" + }, + { + "source_row_index": 13244, + "source_sample_id": "perceptiontest_val_mc:13244", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 128, + "upstream_sample_id": "video_8923:1" + }, + { + "source_row_index": 13512, + "source_sample_id": "perceptiontest_val_mc:13512", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 179, + "upstream_sample_id": "video_7597:4" + }, + { + "source_row_index": 18362, + "source_sample_id": "perceptiontest_val_mc:18362", + "sampling_stratum": "abstraction|predictive", + "source_stratum_index": 230, + "upstream_sample_id": "video_2651:5" + }, + { + "source_row_index": 3874, + "source_sample_id": "perceptiontest_val_mc:3874", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 3, + "upstream_sample_id": "video_9066:0" + }, + { + "source_row_index": 3911, + "source_sample_id": "perceptiontest_val_mc:3911", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 10, + "upstream_sample_id": "video_2719:1" + }, + { + "source_row_index": 3957, + "source_sample_id": "perceptiontest_val_mc:3957", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 18, + "upstream_sample_id": "video_9203:0" + }, + { + "source_row_index": 3998, + "source_sample_id": "perceptiontest_val_mc:3998", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 25, + "upstream_sample_id": "video_5154:1" + }, + { + "source_row_index": 4037, + "source_sample_id": "perceptiontest_val_mc:4037", + "sampling_stratum": "memory|counterfactual", + "source_stratum_index": 32, + "upstream_sample_id": "video_7255:0" + }, + { + "source_row_index": 994, + "source_sample_id": "perceptiontest_val_mc:994", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 57, + "upstream_sample_id": "video_3941:1" + }, + { + "source_row_index": 9137, + "source_sample_id": "perceptiontest_val_mc:9137", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 172, + "upstream_sample_id": "video_875:5" + }, + { + "source_row_index": 9350, + "source_sample_id": "perceptiontest_val_mc:9350", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 287, + "upstream_sample_id": "video_4505:3" + }, + { + "source_row_index": 9564, + "source_sample_id": "perceptiontest_val_mc:9564", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 402, + "upstream_sample_id": "video_11549:4" + }, + { + "source_row_index": 17386, + "source_sample_id": "perceptiontest_val_mc:17386", + "sampling_stratum": "memory|descriptive", + "source_stratum_index": 517, + "upstream_sample_id": "video_8337:3" + }, + { + "source_row_index": 8805, + "source_sample_id": "perceptiontest_val_mc:8805", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 13, + "upstream_sample_id": "video_8238:1" + }, + { + "source_row_index": 8893, + "source_sample_id": "perceptiontest_val_mc:8893", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 40, + "upstream_sample_id": "video_4058:1" + }, + { + "source_row_index": 8981, + "source_sample_id": "perceptiontest_val_mc:8981", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 67, + "upstream_sample_id": "video_4183:1" + }, + { + "source_row_index": 16722, + "source_sample_id": "perceptiontest_val_mc:16722", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 94, + "upstream_sample_id": "video_9149:1" + }, + { + "source_row_index": 18569, + "source_sample_id": "perceptiontest_val_mc:18569", + "sampling_stratum": "memory|explanatory", + "source_stratum_index": 121, + "upstream_sample_id": "video_4283:1" + }, + { + "source_row_index": 11717, + "source_sample_id": "perceptiontest_val_mc:11717", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 3, + "upstream_sample_id": "video_9300:3" + }, + { + "source_row_index": 11750, + "source_sample_id": "perceptiontest_val_mc:11750", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 9, + "upstream_sample_id": "video_171:2" + }, + { + "source_row_index": 11798, + "source_sample_id": "perceptiontest_val_mc:11798", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 15, + "upstream_sample_id": "video_4710:3" + }, + { + "source_row_index": 17813, + "source_sample_id": "perceptiontest_val_mc:17813", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 21, + "upstream_sample_id": "video_2589:3" + }, + { + "source_row_index": 17849, + "source_sample_id": "perceptiontest_val_mc:17849", + "sampling_stratum": "physics|counterfactual", + "source_stratum_index": 27, + "upstream_sample_id": "video_6707:3" + }, + { + "source_row_index": 2162, + "source_sample_id": "perceptiontest_val_mc:2162", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 529, + "upstream_sample_id": "video_10714:0" + }, + { + "source_row_index": 6373, + "source_sample_id": "perceptiontest_val_mc:6373", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 1589, + "upstream_sample_id": "video_3840:0" + }, + { + "source_row_index": 10471, + "source_sample_id": "perceptiontest_val_mc:10471", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 2649, + "upstream_sample_id": "video_1567:3" + }, + { + "source_row_index": 13179, + "source_sample_id": "perceptiontest_val_mc:13179", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 3708, + "upstream_sample_id": "video_1768:0" + }, + { + "source_row_index": 17169, + "source_sample_id": "perceptiontest_val_mc:17169", + "sampling_stratum": "physics|descriptive", + "source_stratum_index": 4768, + "upstream_sample_id": "video_2812:0" + }, + { + "source_row_index": 7652, + "source_sample_id": "perceptiontest_val_mc:7652", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 122, + "upstream_sample_id": "video_3751:3" + }, + { + "source_row_index": 8195, + "source_sample_id": "perceptiontest_val_mc:8195", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 367, + "upstream_sample_id": "video_6995:1" + }, + { + "source_row_index": 8619, + "source_sample_id": "perceptiontest_val_mc:8619", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 613, + "upstream_sample_id": "video_1323:1" + }, + { + "source_row_index": 13731, + "source_sample_id": "perceptiontest_val_mc:13731", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 858, + "upstream_sample_id": "video_1625:0" + }, + { + "source_row_index": 16579, + "source_sample_id": "perceptiontest_val_mc:16579", + "sampling_stratum": "physics|explanatory", + "source_stratum_index": 1103, + "upstream_sample_id": "video_617:1" + }, + { + "source_row_index": 4535, + "source_sample_id": "perceptiontest_val_mc:4535", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 38, + "upstream_sample_id": "video_5408:5" + }, + { + "source_row_index": 5180, + "source_sample_id": "perceptiontest_val_mc:5180", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 114, + "upstream_sample_id": "video_2732:4" + }, + { + "source_row_index": 5898, + "source_sample_id": "perceptiontest_val_mc:5898", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 191, + "upstream_sample_id": "video_7998:4" + }, + { + "source_row_index": 13907, + "source_sample_id": "perceptiontest_val_mc:13907", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 268, + "upstream_sample_id": "video_1539:0" + }, + { + "source_row_index": 15729, + "source_sample_id": "perceptiontest_val_mc:15729", + "sampling_stratum": "physics|predictive", + "source_stratum_index": 344, + "upstream_sample_id": "video_356:4" + }, + { + "source_row_index": 81, + "source_sample_id": "perceptiontest_val_mc:81", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 20, + "upstream_sample_id": "video_1482:1" + }, + { + "source_row_index": 1794, + "source_sample_id": "perceptiontest_val_mc:1794", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 62, + "upstream_sample_id": "video_1906:9" + }, + { + "source_row_index": 2310, + "source_sample_id": "perceptiontest_val_mc:2310", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 103, + "upstream_sample_id": "video_10287:11" + }, + { + "source_row_index": 2784, + "source_sample_id": "perceptiontest_val_mc:2784", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 144, + "upstream_sample_id": "video_10446:10" + }, + { + "source_row_index": 14956, + "source_sample_id": "perceptiontest_val_mc:14956", + "sampling_stratum": "semantics|counterfactual", + "source_stratum_index": 186, + "upstream_sample_id": "video_5971:11" + }, + { + "source_row_index": 1137, + "source_sample_id": "perceptiontest_val_mc:1137", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 471, + "upstream_sample_id": "video_6936:0" + }, + { + "source_row_index": 3530, + "source_sample_id": "perceptiontest_val_mc:3530", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 1415, + "upstream_sample_id": "video_5178:4" + }, + { + "source_row_index": 7494, + "source_sample_id": "perceptiontest_val_mc:7494", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 2358, + "upstream_sample_id": "video_2103:0" + }, + { + "source_row_index": 13219, + "source_sample_id": "perceptiontest_val_mc:13219", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 3301, + "upstream_sample_id": "video_1425:2" + }, + { + "source_row_index": 16942, + "source_sample_id": "perceptiontest_val_mc:16942", + "sampling_stratum": "semantics|descriptive", + "source_stratum_index": 4245, + "upstream_sample_id": "video_2446:0" + }, + { + "source_row_index": 1615, + "source_sample_id": "perceptiontest_val_mc:1615", + "sampling_stratum": "semantics|explanatory", + "source_stratum_index": 115, + "upstream_sample_id": "video_3837:1" + }, + { + "source_row_index": 9793, + "source_sample_id": "perceptiontest_val_mc:9793", + "sampling_stratum": "semantics|explanatory", + "source_stratum_index": 346, + "upstream_sample_id": "video_10838:1" + }, + { + "source_row_index": 14460, + "source_sample_id": "perceptiontest_val_mc:14460", + "sampling_stratum": "semantics|explanatory", + "source_stratum_index": 578, + "upstream_sample_id": "video_9722:2" + }, + { + "source_row_index": 17057, + "source_sample_id": "perceptiontest_val_mc:17057", + "sampling_stratum": "semantics|explanatory", + "source_stratum_index": 809, + "upstream_sample_id": "video_665:1" + } + ] + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_full_legacy-r1.json b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_full_legacy-r1.json new file mode 100644 index 00000000000..112b8dac3c9 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/profiles/sample_sets/judge-free-8_full_legacy-r1.json @@ -0,0 +1,55 @@ +{ + "schema": "modelopt.vlm-sample-set/v1", + "name": "judge-free-8_full_legacy-r1", + "selection": "all", + "tasks": { + "realworldqa": { + "dataset_repository": "lmms-lab/RealWorldQA", + "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", + "max_new_tokens": 16, + "scoring_task_config": "tasks/realworldqa/realworldqa.yaml" + }, + "mmmu_val": { + "dataset_repository": "lmms-lab/MMMU", + "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", + "max_new_tokens": 128, + "scoring_task_config": "tasks/mmmu/mmmu_val.yaml" + }, + "video_mmmu": { + "dataset_repository": "lmms-lab/VideoMMMU", + "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", + "max_new_tokens": 1024, + "scoring_task_config": "tasks/videommmu/video_mmmu.yaml" + }, + "mvbench": { + "dataset_repository": "OpenGVLab/MVBench", + "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mvbench/mvbench.yaml" + }, + "videomme": { + "dataset_repository": "lmms-lab/Video-MME", + "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/videomme/videomme.yaml" + }, + "longvideobench_val_v": { + "dataset_repository": "longvideobench/LongVideoBench", + "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", + "max_new_tokens": 32, + "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml" + }, + "mlvu_dev": { + "dataset_repository": "sy1998/MLVU_dev", + "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", + "max_new_tokens": 16, + "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml" + }, + "perceptiontest_val_mc": { + "dataset_repository": "lmms-lab/PerceptionTest_Val", + "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", + "max_new_tokens": 16, + "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml" + } + } +} diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json index cb70fabb241..dc9467f21ae 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-all-native-v1.json @@ -1,1509 +1,7 @@ { - "extends": "short-v1", - "schema": "modelopt.vlm-evaluation-profile/v1", + "schema": "modelopt.vlm-evaluation-profile/v2", "name": "short-all-native-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", - "backend": { - "attention_implementation": "sdpa", - "enable_thinking": false, - "name": "qwen3_5" - }, - "tasks": { - "video_mmmu": { - "dataset_repository": "lmms-lab/VideoMMMU", - "dataset_revision": "d1c35ac933123d79e877b7f1b9506afb0309cf1b", - "max_new_tokens": 1024, - "scoring_task_config": "tasks/videommmu/video_mmmu.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "adaptation:0", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 13, - "source_sample_id": "adaptation:13", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 26, - "source_sample_id": "adaptation:26", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 39, - "source_sample_id": "adaptation:39", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 52, - "source_sample_id": "adaptation:52", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 65, - "source_sample_id": "adaptation:65", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 78, - "source_sample_id": "adaptation:78", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 91, - "source_sample_id": "adaptation:91", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 104, - "source_sample_id": "adaptation:104", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 117, - "source_sample_id": "adaptation:117", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 130, - "source_sample_id": "adaptation:130", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 143, - "source_sample_id": "adaptation:143", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 156, - "source_sample_id": "adaptation:156", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 169, - "source_sample_id": "adaptation:169", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 182, - "source_sample_id": "adaptation:182", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 195, - "source_sample_id": "adaptation:195", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 208, - "source_sample_id": "adaptation:208", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 221, - "source_sample_id": "adaptation:221", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 234, - "source_sample_id": "adaptation:234", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 247, - "source_sample_id": "adaptation:247", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 260, - "source_sample_id": "adaptation:260", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 273, - "source_sample_id": "adaptation:273", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 286, - "source_sample_id": "adaptation:286", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 299, - "source_sample_id": "adaptation:299", - "leaf_task": "video_mmmu_adaptation" - }, - { - "source_row_index": 0, - "source_sample_id": "comprehension:0", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 13, - "source_sample_id": "comprehension:13", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 26, - "source_sample_id": "comprehension:26", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 39, - "source_sample_id": "comprehension:39", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 52, - "source_sample_id": "comprehension:52", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 65, - "source_sample_id": "comprehension:65", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 78, - "source_sample_id": "comprehension:78", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 91, - "source_sample_id": "comprehension:91", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 104, - "source_sample_id": "comprehension:104", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 117, - "source_sample_id": "comprehension:117", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 130, - "source_sample_id": "comprehension:130", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 143, - "source_sample_id": "comprehension:143", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 156, - "source_sample_id": "comprehension:156", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 169, - "source_sample_id": "comprehension:169", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 182, - "source_sample_id": "comprehension:182", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 195, - "source_sample_id": "comprehension:195", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 208, - "source_sample_id": "comprehension:208", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 221, - "source_sample_id": "comprehension:221", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 234, - "source_sample_id": "comprehension:234", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 247, - "source_sample_id": "comprehension:247", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 260, - "source_sample_id": "comprehension:260", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 273, - "source_sample_id": "comprehension:273", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 286, - "source_sample_id": "comprehension:286", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 299, - "source_sample_id": "comprehension:299", - "leaf_task": "video_mmmu_comprehension" - }, - { - "source_row_index": 0, - "source_sample_id": "perception:0", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 13, - "source_sample_id": "perception:13", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 26, - "source_sample_id": "perception:26", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 39, - "source_sample_id": "perception:39", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 52, - "source_sample_id": "perception:52", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 65, - "source_sample_id": "perception:65", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 78, - "source_sample_id": "perception:78", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 91, - "source_sample_id": "perception:91", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 104, - "source_sample_id": "perception:104", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 117, - "source_sample_id": "perception:117", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 130, - "source_sample_id": "perception:130", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 143, - "source_sample_id": "perception:143", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 156, - "source_sample_id": "perception:156", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 169, - "source_sample_id": "perception:169", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 182, - "source_sample_id": "perception:182", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 195, - "source_sample_id": "perception:195", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 208, - "source_sample_id": "perception:208", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 221, - "source_sample_id": "perception:221", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 234, - "source_sample_id": "perception:234", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 247, - "source_sample_id": "perception:247", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 260, - "source_sample_id": "perception:260", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 273, - "source_sample_id": "perception:273", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 286, - "source_sample_id": "perception:286", - "leaf_task": "video_mmmu_perception" - }, - { - "source_row_index": 299, - "source_sample_id": "perception:299", - "leaf_task": "video_mmmu_perception" - } - ] - }, - "videomme": { - "dataset_repository": "lmms-lab/Video-MME", - "dataset_revision": "ead1408f75b618502df9a1d8e0950166bf0a2a0b", - "max_new_tokens": 16, - "scoring_task_config": "tasks/videomme/videomme.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "videomme:0" - }, - { - "source_row_index": 90, - "source_sample_id": "videomme:90" - }, - { - "source_row_index": 179, - "source_sample_id": "videomme:179" - }, - { - "source_row_index": 269, - "source_sample_id": "videomme:269" - }, - { - "source_row_index": 270, - "source_sample_id": "videomme:270" - }, - { - "source_row_index": 310, - "source_sample_id": "videomme:310" - }, - { - "source_row_index": 349, - "source_sample_id": "videomme:349" - }, - { - "source_row_index": 389, - "source_sample_id": "videomme:389" - }, - { - "source_row_index": 390, - "source_sample_id": "videomme:390" - }, - { - "source_row_index": 440, - "source_sample_id": "videomme:440" - }, - { - "source_row_index": 489, - "source_sample_id": "videomme:489" - }, - { - "source_row_index": 539, - "source_sample_id": "videomme:539" - }, - { - "source_row_index": 540, - "source_sample_id": "videomme:540" - }, - { - "source_row_index": 580, - "source_sample_id": "videomme:580" - }, - { - "source_row_index": 619, - "source_sample_id": "videomme:619" - }, - { - "source_row_index": 659, - "source_sample_id": "videomme:659" - }, - { - "source_row_index": 660, - "source_sample_id": "videomme:660" - }, - { - "source_row_index": 730, - "source_sample_id": "videomme:730" - }, - { - "source_row_index": 799, - "source_sample_id": "videomme:799" - }, - { - "source_row_index": 869, - "source_sample_id": "videomme:869" - }, - { - "source_row_index": 870, - "source_sample_id": "videomme:870" - }, - { - "source_row_index": 880, - "source_sample_id": "videomme:880" - }, - { - "source_row_index": 889, - "source_sample_id": "videomme:889" - }, - { - "source_row_index": 899, - "source_sample_id": "videomme:899" - }, - { - "source_row_index": 900, - "source_sample_id": "videomme:900" - }, - { - "source_row_index": 990, - "source_sample_id": "videomme:990" - }, - { - "source_row_index": 1079, - "source_sample_id": "videomme:1079" - }, - { - "source_row_index": 1169, - "source_sample_id": "videomme:1169" - }, - { - "source_row_index": 1170, - "source_sample_id": "videomme:1170" - }, - { - "source_row_index": 1210, - "source_sample_id": "videomme:1210" - }, - { - "source_row_index": 1249, - "source_sample_id": "videomme:1249" - }, - { - "source_row_index": 1289, - "source_sample_id": "videomme:1289" - }, - { - "source_row_index": 1290, - "source_sample_id": "videomme:1290" - }, - { - "source_row_index": 1340, - "source_sample_id": "videomme:1340" - }, - { - "source_row_index": 1389, - "source_sample_id": "videomme:1389" - }, - { - "source_row_index": 1439, - "source_sample_id": "videomme:1439" - }, - { - "source_row_index": 1440, - "source_sample_id": "videomme:1440" - }, - { - "source_row_index": 1480, - "source_sample_id": "videomme:1480" - }, - { - "source_row_index": 1519, - "source_sample_id": "videomme:1519" - }, - { - "source_row_index": 1559, - "source_sample_id": "videomme:1559" - }, - { - "source_row_index": 1560, - "source_sample_id": "videomme:1560" - }, - { - "source_row_index": 1630, - "source_sample_id": "videomme:1630" - }, - { - "source_row_index": 1699, - "source_sample_id": "videomme:1699" - }, - { - "source_row_index": 1769, - "source_sample_id": "videomme:1769" - }, - { - "source_row_index": 1770, - "source_sample_id": "videomme:1770" - }, - { - "source_row_index": 1780, - "source_sample_id": "videomme:1780" - }, - { - "source_row_index": 1789, - "source_sample_id": "videomme:1789" - }, - { - "source_row_index": 1799, - "source_sample_id": "videomme:1799" - }, - { - "source_row_index": 1800, - "source_sample_id": "videomme:1800" - }, - { - "source_row_index": 1890, - "source_sample_id": "videomme:1890" - }, - { - "source_row_index": 1979, - "source_sample_id": "videomme:1979" - }, - { - "source_row_index": 2069, - "source_sample_id": "videomme:2069" - }, - { - "source_row_index": 2070, - "source_sample_id": "videomme:2070" - }, - { - "source_row_index": 2110, - "source_sample_id": "videomme:2110" - }, - { - "source_row_index": 2149, - "source_sample_id": "videomme:2149" - }, - { - "source_row_index": 2189, - "source_sample_id": "videomme:2189" - }, - { - "source_row_index": 2190, - "source_sample_id": "videomme:2190" - }, - { - "source_row_index": 2240, - "source_sample_id": "videomme:2240" - }, - { - "source_row_index": 2289, - "source_sample_id": "videomme:2289" - }, - { - "source_row_index": 2339, - "source_sample_id": "videomme:2339" - }, - { - "source_row_index": 2340, - "source_sample_id": "videomme:2340" - }, - { - "source_row_index": 2380, - "source_sample_id": "videomme:2380" - }, - { - "source_row_index": 2419, - "source_sample_id": "videomme:2419" - }, - { - "source_row_index": 2459, - "source_sample_id": "videomme:2459" - }, - { - "source_row_index": 2460, - "source_sample_id": "videomme:2460" - }, - { - "source_row_index": 2530, - "source_sample_id": "videomme:2530" - }, - { - "source_row_index": 2599, - "source_sample_id": "videomme:2599" - }, - { - "source_row_index": 2669, - "source_sample_id": "videomme:2669" - }, - { - "source_row_index": 2670, - "source_sample_id": "videomme:2670" - }, - { - "source_row_index": 2680, - "source_sample_id": "videomme:2680" - }, - { - "source_row_index": 2689, - "source_sample_id": "videomme:2689" - }, - { - "source_row_index": 2699, - "source_sample_id": "videomme:2699" - } - ] - }, - "longvideobench_val_v": { - "dataset_repository": "longvideobench/LongVideoBench", - "dataset_revision": "60d1c89c1919a198b73be39c2babb213b29d6a5c", - "max_new_tokens": 32, - "scoring_task_config": "tasks/longvideobench/longvideobench_val_v.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "longvideobench_val_v:0" - }, - { - "source_row_index": 20, - "source_sample_id": "longvideobench_val_v:20" - }, - { - "source_row_index": 40, - "source_sample_id": "longvideobench_val_v:40" - }, - { - "source_row_index": 60, - "source_sample_id": "longvideobench_val_v:60" - }, - { - "source_row_index": 80, - "source_sample_id": "longvideobench_val_v:80" - }, - { - "source_row_index": 100, - "source_sample_id": "longvideobench_val_v:100" - }, - { - "source_row_index": 120, - "source_sample_id": "longvideobench_val_v:120" - }, - { - "source_row_index": 140, - "source_sample_id": "longvideobench_val_v:140" - }, - { - "source_row_index": 160, - "source_sample_id": "longvideobench_val_v:160" - }, - { - "source_row_index": 179, - "source_sample_id": "longvideobench_val_v:179" - }, - { - "source_row_index": 199, - "source_sample_id": "longvideobench_val_v:199" - }, - { - "source_row_index": 219, - "source_sample_id": "longvideobench_val_v:219" - }, - { - "source_row_index": 239, - "source_sample_id": "longvideobench_val_v:239" - }, - { - "source_row_index": 259, - "source_sample_id": "longvideobench_val_v:259" - }, - { - "source_row_index": 279, - "source_sample_id": "longvideobench_val_v:279" - }, - { - "source_row_index": 299, - "source_sample_id": "longvideobench_val_v:299" - }, - { - "source_row_index": 319, - "source_sample_id": "longvideobench_val_v:319" - }, - { - "source_row_index": 339, - "source_sample_id": "longvideobench_val_v:339" - }, - { - "source_row_index": 359, - "source_sample_id": "longvideobench_val_v:359" - }, - { - "source_row_index": 379, - "source_sample_id": "longvideobench_val_v:379" - }, - { - "source_row_index": 399, - "source_sample_id": "longvideobench_val_v:399" - }, - { - "source_row_index": 419, - "source_sample_id": "longvideobench_val_v:419" - }, - { - "source_row_index": 439, - "source_sample_id": "longvideobench_val_v:439" - }, - { - "source_row_index": 459, - "source_sample_id": "longvideobench_val_v:459" - }, - { - "source_row_index": 479, - "source_sample_id": "longvideobench_val_v:479" - }, - { - "source_row_index": 499, - "source_sample_id": "longvideobench_val_v:499" - }, - { - "source_row_index": 518, - "source_sample_id": "longvideobench_val_v:518" - }, - { - "source_row_index": 538, - "source_sample_id": "longvideobench_val_v:538" - }, - { - "source_row_index": 558, - "source_sample_id": "longvideobench_val_v:558" - }, - { - "source_row_index": 578, - "source_sample_id": "longvideobench_val_v:578" - }, - { - "source_row_index": 598, - "source_sample_id": "longvideobench_val_v:598" - }, - { - "source_row_index": 618, - "source_sample_id": "longvideobench_val_v:618" - }, - { - "source_row_index": 638, - "source_sample_id": "longvideobench_val_v:638" - }, - { - "source_row_index": 658, - "source_sample_id": "longvideobench_val_v:658" - }, - { - "source_row_index": 678, - "source_sample_id": "longvideobench_val_v:678" - }, - { - "source_row_index": 698, - "source_sample_id": "longvideobench_val_v:698" - }, - { - "source_row_index": 718, - "source_sample_id": "longvideobench_val_v:718" - }, - { - "source_row_index": 738, - "source_sample_id": "longvideobench_val_v:738" - }, - { - "source_row_index": 758, - "source_sample_id": "longvideobench_val_v:758" - }, - { - "source_row_index": 778, - "source_sample_id": "longvideobench_val_v:778" - }, - { - "source_row_index": 798, - "source_sample_id": "longvideobench_val_v:798" - }, - { - "source_row_index": 818, - "source_sample_id": "longvideobench_val_v:818" - }, - { - "source_row_index": 837, - "source_sample_id": "longvideobench_val_v:837" - }, - { - "source_row_index": 857, - "source_sample_id": "longvideobench_val_v:857" - }, - { - "source_row_index": 877, - "source_sample_id": "longvideobench_val_v:877" - }, - { - "source_row_index": 897, - "source_sample_id": "longvideobench_val_v:897" - }, - { - "source_row_index": 917, - "source_sample_id": "longvideobench_val_v:917" - }, - { - "source_row_index": 937, - "source_sample_id": "longvideobench_val_v:937" - }, - { - "source_row_index": 957, - "source_sample_id": "longvideobench_val_v:957" - }, - { - "source_row_index": 977, - "source_sample_id": "longvideobench_val_v:977" - }, - { - "source_row_index": 997, - "source_sample_id": "longvideobench_val_v:997" - }, - { - "source_row_index": 1017, - "source_sample_id": "longvideobench_val_v:1017" - }, - { - "source_row_index": 1037, - "source_sample_id": "longvideobench_val_v:1037" - }, - { - "source_row_index": 1057, - "source_sample_id": "longvideobench_val_v:1057" - }, - { - "source_row_index": 1077, - "source_sample_id": "longvideobench_val_v:1077" - }, - { - "source_row_index": 1097, - "source_sample_id": "longvideobench_val_v:1097" - }, - { - "source_row_index": 1117, - "source_sample_id": "longvideobench_val_v:1117" - }, - { - "source_row_index": 1137, - "source_sample_id": "longvideobench_val_v:1137" - }, - { - "source_row_index": 1157, - "source_sample_id": "longvideobench_val_v:1157" - }, - { - "source_row_index": 1176, - "source_sample_id": "longvideobench_val_v:1176" - }, - { - "source_row_index": 1196, - "source_sample_id": "longvideobench_val_v:1196" - }, - { - "source_row_index": 1216, - "source_sample_id": "longvideobench_val_v:1216" - }, - { - "source_row_index": 1236, - "source_sample_id": "longvideobench_val_v:1236" - }, - { - "source_row_index": 1256, - "source_sample_id": "longvideobench_val_v:1256" - }, - { - "source_row_index": 1276, - "source_sample_id": "longvideobench_val_v:1276" - }, - { - "source_row_index": 1296, - "source_sample_id": "longvideobench_val_v:1296" - }, - { - "source_row_index": 1316, - "source_sample_id": "longvideobench_val_v:1316" - }, - { - "source_row_index": 1336, - "source_sample_id": "longvideobench_val_v:1336" - } - ] - }, - "mlvu_dev": { - "dataset_repository": "sy1998/MLVU_dev", - "dataset_revision": "96207eb9aa7101e2a495dd147684a7e618c79e12", - "max_new_tokens": 16, - "scoring_task_config": "tasks/mlvu/mlvu_dev.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "mlvu_dev:0" - }, - { - "source_row_index": 39, - "source_sample_id": "mlvu_dev:39" - }, - { - "source_row_index": 79, - "source_sample_id": "mlvu_dev:79" - }, - { - "source_row_index": 118, - "source_sample_id": "mlvu_dev:118" - }, - { - "source_row_index": 157, - "source_sample_id": "mlvu_dev:157" - }, - { - "source_row_index": 197, - "source_sample_id": "mlvu_dev:197" - }, - { - "source_row_index": 236, - "source_sample_id": "mlvu_dev:236" - }, - { - "source_row_index": 275, - "source_sample_id": "mlvu_dev:275" - }, - { - "source_row_index": 315, - "source_sample_id": "mlvu_dev:315" - }, - { - "source_row_index": 354, - "source_sample_id": "mlvu_dev:354" - }, - { - "source_row_index": 355, - "source_sample_id": "mlvu_dev:355" - }, - { - "source_row_index": 377, - "source_sample_id": "mlvu_dev:377" - }, - { - "source_row_index": 399, - "source_sample_id": "mlvu_dev:399" - }, - { - "source_row_index": 421, - "source_sample_id": "mlvu_dev:421" - }, - { - "source_row_index": 443, - "source_sample_id": "mlvu_dev:443" - }, - { - "source_row_index": 466, - "source_sample_id": "mlvu_dev:466" - }, - { - "source_row_index": 488, - "source_sample_id": "mlvu_dev:488" - }, - { - "source_row_index": 510, - "source_sample_id": "mlvu_dev:510" - }, - { - "source_row_index": 532, - "source_sample_id": "mlvu_dev:532" - }, - { - "source_row_index": 554, - "source_sample_id": "mlvu_dev:554" - }, - { - "source_row_index": 555, - "source_sample_id": "mlvu_dev:555" - }, - { - "source_row_index": 578, - "source_sample_id": "mlvu_dev:578" - }, - { - "source_row_index": 601, - "source_sample_id": "mlvu_dev:601" - }, - { - "source_row_index": 623, - "source_sample_id": "mlvu_dev:623" - }, - { - "source_row_index": 646, - "source_sample_id": "mlvu_dev:646" - }, - { - "source_row_index": 669, - "source_sample_id": "mlvu_dev:669" - }, - { - "source_row_index": 692, - "source_sample_id": "mlvu_dev:692" - }, - { - "source_row_index": 714, - "source_sample_id": "mlvu_dev:714" - }, - { - "source_row_index": 737, - "source_sample_id": "mlvu_dev:737" - }, - { - "source_row_index": 760, - "source_sample_id": "mlvu_dev:760" - }, - { - "source_row_index": 761, - "source_sample_id": "mlvu_dev:761" - }, - { - "source_row_index": 800, - "source_sample_id": "mlvu_dev:800" - }, - { - "source_row_index": 839, - "source_sample_id": "mlvu_dev:839" - }, - { - "source_row_index": 878, - "source_sample_id": "mlvu_dev:878" - }, - { - "source_row_index": 917, - "source_sample_id": "mlvu_dev:917" - }, - { - "source_row_index": 956, - "source_sample_id": "mlvu_dev:956" - }, - { - "source_row_index": 995, - "source_sample_id": "mlvu_dev:995" - }, - { - "source_row_index": 1034, - "source_sample_id": "mlvu_dev:1034" - }, - { - "source_row_index": 1073, - "source_sample_id": "mlvu_dev:1073" - }, - { - "source_row_index": 1112, - "source_sample_id": "mlvu_dev:1112" - }, - { - "source_row_index": 1113, - "source_sample_id": "mlvu_dev:1113" - }, - { - "source_row_index": 1173, - "source_sample_id": "mlvu_dev:1173" - }, - { - "source_row_index": 1233, - "source_sample_id": "mlvu_dev:1233" - }, - { - "source_row_index": 1292, - "source_sample_id": "mlvu_dev:1292" - }, - { - "source_row_index": 1352, - "source_sample_id": "mlvu_dev:1352" - }, - { - "source_row_index": 1412, - "source_sample_id": "mlvu_dev:1412" - }, - { - "source_row_index": 1472, - "source_sample_id": "mlvu_dev:1472" - }, - { - "source_row_index": 1531, - "source_sample_id": "mlvu_dev:1531" - }, - { - "source_row_index": 1591, - "source_sample_id": "mlvu_dev:1591" - }, - { - "source_row_index": 1651, - "source_sample_id": "mlvu_dev:1651" - }, - { - "source_row_index": 1652, - "source_sample_id": "mlvu_dev:1652" - }, - { - "source_row_index": 1681, - "source_sample_id": "mlvu_dev:1681" - }, - { - "source_row_index": 1710, - "source_sample_id": "mlvu_dev:1710" - }, - { - "source_row_index": 1739, - "source_sample_id": "mlvu_dev:1739" - }, - { - "source_row_index": 1768, - "source_sample_id": "mlvu_dev:1768" - }, - { - "source_row_index": 1798, - "source_sample_id": "mlvu_dev:1798" - }, - { - "source_row_index": 1827, - "source_sample_id": "mlvu_dev:1827" - }, - { - "source_row_index": 1856, - "source_sample_id": "mlvu_dev:1856" - }, - { - "source_row_index": 1885, - "source_sample_id": "mlvu_dev:1885" - }, - { - "source_row_index": 1914, - "source_sample_id": "mlvu_dev:1914" - }, - { - "source_row_index": 1915, - "source_sample_id": "mlvu_dev:1915" - }, - { - "source_row_index": 1944, - "source_sample_id": "mlvu_dev:1944" - }, - { - "source_row_index": 1972, - "source_sample_id": "mlvu_dev:1972" - }, - { - "source_row_index": 2001, - "source_sample_id": "mlvu_dev:2001" - }, - { - "source_row_index": 2030, - "source_sample_id": "mlvu_dev:2030" - }, - { - "source_row_index": 2058, - "source_sample_id": "mlvu_dev:2058" - }, - { - "source_row_index": 2087, - "source_sample_id": "mlvu_dev:2087" - }, - { - "source_row_index": 2116, - "source_sample_id": "mlvu_dev:2116" - }, - { - "source_row_index": 2144, - "source_sample_id": "mlvu_dev:2144" - }, - { - "source_row_index": 2173, - "source_sample_id": "mlvu_dev:2173" - } - ] - }, - "perceptiontest_val_mc": { - "dataset_repository": "lmms-lab/PerceptionTest_Val", - "dataset_revision": "c5e520d8c4167fb1f135c36e9d6e67312b4f8e6b", - "max_new_tokens": 16, - "scoring_task_config": "tasks/perceptiontest/val/perceptiontest_mc.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "perceptiontest_val_mc:0" - }, - { - "source_row_index": 1, - "source_sample_id": "perceptiontest_val_mc:1" - }, - { - "source_row_index": 3, - "source_sample_id": "perceptiontest_val_mc:3" - }, - { - "source_row_index": 20, - "source_sample_id": "perceptiontest_val_mc:20" - }, - { - "source_row_index": 211, - "source_sample_id": "perceptiontest_val_mc:211" - }, - { - "source_row_index": 544, - "source_sample_id": "perceptiontest_val_mc:544" - }, - { - "source_row_index": 753, - "source_sample_id": "perceptiontest_val_mc:753" - }, - { - "source_row_index": 1672, - "source_sample_id": "perceptiontest_val_mc:1672" - }, - { - "source_row_index": 2310, - "source_sample_id": "perceptiontest_val_mc:2310" - }, - { - "source_row_index": 2386, - "source_sample_id": "perceptiontest_val_mc:2386" - }, - { - "source_row_index": 2846, - "source_sample_id": "perceptiontest_val_mc:2846" - }, - { - "source_row_index": 2894, - "source_sample_id": "perceptiontest_val_mc:2894" - }, - { - "source_row_index": 3857, - "source_sample_id": "perceptiontest_val_mc:3857" - }, - { - "source_row_index": 3905, - "source_sample_id": "perceptiontest_val_mc:3905" - }, - { - "source_row_index": 3957, - "source_sample_id": "perceptiontest_val_mc:3957" - }, - { - "source_row_index": 4004, - "source_sample_id": "perceptiontest_val_mc:4004" - }, - { - "source_row_index": 4031, - "source_sample_id": "perceptiontest_val_mc:4031" - }, - { - "source_row_index": 4053, - "source_sample_id": "perceptiontest_val_mc:4053" - }, - { - "source_row_index": 4167, - "source_sample_id": "perceptiontest_val_mc:4167" - }, - { - "source_row_index": 4228, - "source_sample_id": "perceptiontest_val_mc:4228" - }, - { - "source_row_index": 4994, - "source_sample_id": "perceptiontest_val_mc:4994" - }, - { - "source_row_index": 5673, - "source_sample_id": "perceptiontest_val_mc:5673" - }, - { - "source_row_index": 5898, - "source_sample_id": "perceptiontest_val_mc:5898" - }, - { - "source_row_index": 6414, - "source_sample_id": "perceptiontest_val_mc:6414" - }, - { - "source_row_index": 6502, - "source_sample_id": "perceptiontest_val_mc:6502" - }, - { - "source_row_index": 6598, - "source_sample_id": "perceptiontest_val_mc:6598" - }, - { - "source_row_index": 7494, - "source_sample_id": "perceptiontest_val_mc:7494" - }, - { - "source_row_index": 8074, - "source_sample_id": "perceptiontest_val_mc:8074" - }, - { - "source_row_index": 8617, - "source_sample_id": "perceptiontest_val_mc:8617" - }, - { - "source_row_index": 8763, - "source_sample_id": "perceptiontest_val_mc:8763" - }, - { - "source_row_index": 8865, - "source_sample_id": "perceptiontest_val_mc:8865" - }, - { - "source_row_index": 8873, - "source_sample_id": "perceptiontest_val_mc:8873" - }, - { - "source_row_index": 8981, - "source_sample_id": "perceptiontest_val_mc:8981" - }, - { - "source_row_index": 9350, - "source_sample_id": "perceptiontest_val_mc:9350" - }, - { - "source_row_index": 9617, - "source_sample_id": "perceptiontest_val_mc:9617" - }, - { - "source_row_index": 9749, - "source_sample_id": "perceptiontest_val_mc:9749" - }, - { - "source_row_index": 10470, - "source_sample_id": "perceptiontest_val_mc:10470" - }, - { - "source_row_index": 11697, - "source_sample_id": "perceptiontest_val_mc:11697" - }, - { - "source_row_index": 11745, - "source_sample_id": "perceptiontest_val_mc:11745" - }, - { - "source_row_index": 11798, - "source_sample_id": "perceptiontest_val_mc:11798" - }, - { - "source_row_index": 13051, - "source_sample_id": "perceptiontest_val_mc:13051" - }, - { - "source_row_index": 13244, - "source_sample_id": "perceptiontest_val_mc:13244" - }, - { - "source_row_index": 13581, - "source_sample_id": "perceptiontest_val_mc:13581" - }, - { - "source_row_index": 13929, - "source_sample_id": "perceptiontest_val_mc:13929" - }, - { - "source_row_index": 13999, - "source_sample_id": "perceptiontest_val_mc:13999" - }, - { - "source_row_index": 14073, - "source_sample_id": "perceptiontest_val_mc:14073" - }, - { - "source_row_index": 14200, - "source_sample_id": "perceptiontest_val_mc:14200" - }, - { - "source_row_index": 14884, - "source_sample_id": "perceptiontest_val_mc:14884" - }, - { - "source_row_index": 15201, - "source_sample_id": "perceptiontest_val_mc:15201" - }, - { - "source_row_index": 15432, - "source_sample_id": "perceptiontest_val_mc:15432" - }, - { - "source_row_index": 16191, - "source_sample_id": "perceptiontest_val_mc:16191" - }, - { - "source_row_index": 16280, - "source_sample_id": "perceptiontest_val_mc:16280" - }, - { - "source_row_index": 16741, - "source_sample_id": "perceptiontest_val_mc:16741" - }, - { - "source_row_index": 17109, - "source_sample_id": "perceptiontest_val_mc:17109" - }, - { - "source_row_index": 17820, - "source_sample_id": "perceptiontest_val_mc:17820" - }, - { - "source_row_index": 17866, - "source_sample_id": "perceptiontest_val_mc:17866" - }, - { - "source_row_index": 18478, - "source_sample_id": "perceptiontest_val_mc:18478" - }, - { - "source_row_index": 18489, - "source_sample_id": "perceptiontest_val_mc:18489" - }, - { - "source_row_index": 18867, - "source_sample_id": "perceptiontest_val_mc:18867" - }, - { - "source_row_index": 19021, - "source_sample_id": "perceptiontest_val_mc:19021" - }, - { - "source_row_index": 19102, - "source_sample_id": "perceptiontest_val_mc:19102" - }, - { - "source_row_index": 19135, - "source_sample_id": "perceptiontest_val_mc:19135" - }, - { - "source_row_index": 19137, - "source_sample_id": "perceptiontest_val_mc:19137" - }, - { - "source_row_index": 19139, - "source_sample_id": "perceptiontest_val_mc:19139" - } - ] - } - } + "sample_set": "judge-free-8_690-examples_legacy-r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" } diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json index 604e5fcc853..41f6b548ff3 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-native-v1.json @@ -1,11 +1,7 @@ { - "extends": "short-v1", - "schema": "modelopt.vlm-evaluation-profile/v1", + "schema": "modelopt.vlm-evaluation-profile/v2", "name": "short-native-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", - "backend": { - "attention_implementation": "sdpa", - "enable_thinking": false, - "name": "qwen3_5" - } + "sample_set": "core-3_344-examples_legacy-r1", + "backend_profile": "qwen-3.5-native_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" } diff --git a/examples/puzzletron/evaluation/vlm/profiles/short-v1.json b/examples/puzzletron/evaluation/vlm/profiles/short-v1.json index 68d0e500aca..ca10add87ca 100644 --- a/examples/puzzletron/evaluation/vlm/profiles/short-v1.json +++ b/examples/puzzletron/evaluation/vlm/profiles/short-v1.json @@ -1,1589 +1,7 @@ { - "schema": "modelopt.vlm-evaluation-profile/v1", + "schema": "modelopt.vlm-evaluation-profile/v2", "name": "short-v1", - "lmms_eval_revision": "3e675904f8cba6793de12b91979b04d91754bdf3", - "model_family": { - "architecture": "Qwen3_5ForConditionalGeneration", - "model_type": "qwen3_5" - }, - "backend": { - "enable_thinking": false, - "name": "vllm", - "reasoning_parser": "qwen3" - }, - "preprocessing": { - "fps": 2, - "max_frames": 32, - "video_reader": "decord" - }, - "generation": { - "do_sample": false, - "temperature": 0 - }, - "seed": 42, - "repetitions": 1, - "batch_size": 1, - "selection": "exact-rows", - "tasks": { - "realworldqa": { - "dataset_repository": "lmms-lab/RealWorldQA", - "dataset_revision": "907c4e5228fd1703c710ed937601cb5f89ab8d5c", - "max_new_tokens": 16, - "scoring_task_config": "tasks/realworldqa/realworldqa.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "test:0" - }, - { - "source_row_index": 1, - "source_sample_id": "test:1" - }, - { - "source_row_index": 2, - "source_sample_id": "test:2" - }, - { - "source_row_index": 3, - "source_sample_id": "test:3" - }, - { - "source_row_index": 4, - "source_sample_id": "test:4" - }, - { - "source_row_index": 5, - "source_sample_id": "test:5" - }, - { - "source_row_index": 6, - "source_sample_id": "test:6" - }, - { - "source_row_index": 7, - "source_sample_id": "test:7" - }, - { - "source_row_index": 8, - "source_sample_id": "test:8" - }, - { - "source_row_index": 9, - "source_sample_id": "test:9" - }, - { - "source_row_index": 10, - "source_sample_id": "test:10" - }, - { - "source_row_index": 11, - "source_sample_id": "test:11" - }, - { - "source_row_index": 12, - "source_sample_id": "test:12" - }, - { - "source_row_index": 13, - "source_sample_id": "test:13" - }, - { - "source_row_index": 14, - "source_sample_id": "test:14" - }, - { - "source_row_index": 15, - "source_sample_id": "test:15" - }, - { - "source_row_index": 16, - "source_sample_id": "test:16" - }, - { - "source_row_index": 17, - "source_sample_id": "test:17" - }, - { - "source_row_index": 18, - "source_sample_id": "test:18" - }, - { - "source_row_index": 19, - "source_sample_id": "test:19" - }, - { - "source_row_index": 20, - "source_sample_id": "test:20" - }, - { - "source_row_index": 21, - "source_sample_id": "test:21" - }, - { - "source_row_index": 22, - "source_sample_id": "test:22" - }, - { - "source_row_index": 23, - "source_sample_id": "test:23" - }, - { - "source_row_index": 24, - "source_sample_id": "test:24" - }, - { - "source_row_index": 25, - "source_sample_id": "test:25" - }, - { - "source_row_index": 26, - "source_sample_id": "test:26" - }, - { - "source_row_index": 27, - "source_sample_id": "test:27" - }, - { - "source_row_index": 28, - "source_sample_id": "test:28" - }, - { - "source_row_index": 29, - "source_sample_id": "test:29" - }, - { - "source_row_index": 30, - "source_sample_id": "test:30" - }, - { - "source_row_index": 31, - "source_sample_id": "test:31" - }, - { - "source_row_index": 32, - "source_sample_id": "test:32" - }, - { - "source_row_index": 33, - "source_sample_id": "test:33" - }, - { - "source_row_index": 34, - "source_sample_id": "test:34" - }, - { - "source_row_index": 35, - "source_sample_id": "test:35" - }, - { - "source_row_index": 36, - "source_sample_id": "test:36" - }, - { - "source_row_index": 37, - "source_sample_id": "test:37" - }, - { - "source_row_index": 38, - "source_sample_id": "test:38" - }, - { - "source_row_index": 39, - "source_sample_id": "test:39" - }, - { - "source_row_index": 40, - "source_sample_id": "test:40" - }, - { - "source_row_index": 41, - "source_sample_id": "test:41" - }, - { - "source_row_index": 42, - "source_sample_id": "test:42" - }, - { - "source_row_index": 43, - "source_sample_id": "test:43" - }, - { - "source_row_index": 44, - "source_sample_id": "test:44" - }, - { - "source_row_index": 45, - "source_sample_id": "test:45" - }, - { - "source_row_index": 46, - "source_sample_id": "test:46" - }, - { - "source_row_index": 47, - "source_sample_id": "test:47" - }, - { - "source_row_index": 48, - "source_sample_id": "test:48" - }, - { - "source_row_index": 49, - "source_sample_id": "test:49" - }, - { - "source_row_index": 50, - "source_sample_id": "test:50" - }, - { - "source_row_index": 51, - "source_sample_id": "test:51" - }, - { - "source_row_index": 52, - "source_sample_id": "test:52" - }, - { - "source_row_index": 53, - "source_sample_id": "test:53" - }, - { - "source_row_index": 54, - "source_sample_id": "test:54" - }, - { - "source_row_index": 55, - "source_sample_id": "test:55" - }, - { - "source_row_index": 56, - "source_sample_id": "test:56" - }, - { - "source_row_index": 57, - "source_sample_id": "test:57" - }, - { - "source_row_index": 58, - "source_sample_id": "test:58" - }, - { - "source_row_index": 59, - "source_sample_id": "test:59" - }, - { - "source_row_index": 60, - "source_sample_id": "test:60" - }, - { - "source_row_index": 61, - "source_sample_id": "test:61" - }, - { - "source_row_index": 62, - "source_sample_id": "test:62" - }, - { - "source_row_index": 63, - "source_sample_id": "test:63" - } - ] - }, - "mmmu_val": { - "dataset_repository": "lmms-lab/MMMU", - "dataset_revision": "364f2e2eb107b36e07ff4c5a15f5947a759cef47", - "max_new_tokens": 128, - "scoring_task_config": "tasks/mmmu/mmmu_val.yaml", - "rows": [ - { - "source_row_index": 0, - "source_sample_id": "validation_Accounting_1" - }, - { - "source_row_index": 7, - "source_sample_id": "validation_Accounting_8" - }, - { - "source_row_index": 15, - "source_sample_id": "validation_Accounting_16" - }, - { - "source_row_index": 22, - "source_sample_id": "validation_Accounting_23" - }, - { - "source_row_index": 30, - "source_sample_id": "validation_Agriculture_1" - }, - { - "source_row_index": 37, - "source_sample_id": "validation_Agriculture_8" - }, - { - "source_row_index": 45, - "source_sample_id": "validation_Agriculture_16" - }, - { - "source_row_index": 52, - "source_sample_id": "validation_Agriculture_23" - }, - { - "source_row_index": 60, - "source_sample_id": "validation_Architecture_and_Engineering_1" - }, - { - "source_row_index": 67, - "source_sample_id": "validation_Architecture_and_Engineering_8" - }, - { - "source_row_index": 75, - "source_sample_id": "validation_Architecture_and_Engineering_16" - }, - { - "source_row_index": 82, - "source_sample_id": "validation_Architecture_and_Engineering_23" - }, - { - "source_row_index": 90, - "source_sample_id": "validation_Art_1" - }, - { - "source_row_index": 97, - "source_sample_id": "validation_Art_8" - }, - { - "source_row_index": 105, - "source_sample_id": "validation_Art_16" - }, - { - "source_row_index": 112, - "source_sample_id": "validation_Art_23" - }, - { - "source_row_index": 120, - "source_sample_id": "validation_Art_Theory_1" - }, - { - "source_row_index": 127, - "source_sample_id": "validation_Art_Theory_8" - }, - { - "source_row_index": 135, - "source_sample_id": "validation_Art_Theory_16" - }, - { - "source_row_index": 142, - "source_sample_id": "validation_Art_Theory_23" - }, - { - "source_row_index": 150, - "source_sample_id": "validation_Basic_Medical_Science_1" - }, - { - "source_row_index": 157, - "source_sample_id": "validation_Basic_Medical_Science_8" - }, - { - "source_row_index": 165, - "source_sample_id": "validation_Basic_Medical_Science_16" - }, - { - "source_row_index": 172, - "source_sample_id": "validation_Basic_Medical_Science_23" - }, - { - "source_row_index": 180, - "source_sample_id": "validation_Biology_1" - }, - { - "source_row_index": 187, - "source_sample_id": "validation_Biology_8" - }, - { - "source_row_index": 195, - "source_sample_id": "validation_Biology_16" - }, - { - "source_row_index": 202, - "source_sample_id": "validation_Biology_23" - }, - { - "source_row_index": 210, - "source_sample_id": "validation_Chemistry_1" - }, - { - "source_row_index": 217, - "source_sample_id": "validation_Chemistry_8" - }, - { - "source_row_index": 225, - "source_sample_id": "validation_Chemistry_16" - }, - { - "source_row_index": 232, - "source_sample_id": "validation_Chemistry_23" - }, - { - "source_row_index": 240, - "source_sample_id": "validation_Clinical_Medicine_1" - }, - { - "source_row_index": 247, - "source_sample_id": "validation_Clinical_Medicine_8" - }, - { - "source_row_index": 255, - "source_sample_id": "validation_Clinical_Medicine_16" - }, - { - "source_row_index": 262, - "source_sample_id": "validation_Clinical_Medicine_23" - }, - { - "source_row_index": 270, - "source_sample_id": "validation_Computer_Science_1" - }, - { - "source_row_index": 277, - "source_sample_id": "validation_Computer_Science_8" - }, - { - "source_row_index": 285, - "source_sample_id": "validation_Computer_Science_16" - }, - { - "source_row_index": 292, - "source_sample_id": "validation_Computer_Science_23" - }, - { - "source_row_index": 300, - "source_sample_id": "validation_Design_1" - }, - { - "source_row_index": 307, - "source_sample_id": "validation_Design_8" - }, - { - "source_row_index": 315, - "source_sample_id": "validation_Design_16" - }, - { - "source_row_index": 322, - "source_sample_id": "validation_Design_23" - }, - { - "source_row_index": 330, - "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_1" - }, - { - "source_row_index": 337, - "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_8" - }, - { - "source_row_index": 345, - "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_16" - }, - { - "source_row_index": 352, - "source_sample_id": "validation_Diagnostics_and_Laboratory_Medicine_23" - }, - { - "source_row_index": 360, - "source_sample_id": "validation_Economics_1" - }, - { - "source_row_index": 367, - "source_sample_id": "validation_Economics_8" - }, - { - "source_row_index": 375, - "source_sample_id": "validation_Economics_16" - }, - { - "source_row_index": 382, - "source_sample_id": "validation_Economics_23" - }, - { - "source_row_index": 390, - "source_sample_id": "validation_Electronics_1" - }, - { - "source_row_index": 397, - "source_sample_id": "validation_Electronics_8" - }, - { - "source_row_index": 405, - "source_sample_id": "validation_Electronics_16" - }, - { - "source_row_index": 412, - "source_sample_id": "validation_Electronics_23" - }, - { - "source_row_index": 420, - "source_sample_id": "validation_Energy_and_Power_1" - }, - { - "source_row_index": 427, - "source_sample_id": "validation_Energy_and_Power_8" - }, - { - "source_row_index": 435, - "source_sample_id": "validation_Energy_and_Power_16" - }, - { - "source_row_index": 442, - "source_sample_id": "validation_Energy_and_Power_23" - }, - { - "source_row_index": 450, - "source_sample_id": "validation_Finance_1" - }, - { - "source_row_index": 457, - "source_sample_id": "validation_Finance_8" - }, - { - "source_row_index": 465, - "source_sample_id": "validation_Finance_16" - }, - { - "source_row_index": 472, - "source_sample_id": "validation_Finance_23" - }, - { - "source_row_index": 480, - "source_sample_id": "validation_Geography_1" - }, - { - "source_row_index": 487, - "source_sample_id": "validation_Geography_8" - }, - { - "source_row_index": 495, - "source_sample_id": "validation_Geography_16" - }, - { - "source_row_index": 502, - "source_sample_id": "validation_Geography_23" - }, - { - "source_row_index": 510, - "source_sample_id": "validation_History_1" - }, - { - "source_row_index": 517, - "source_sample_id": "validation_History_8" - }, - { - "source_row_index": 525, - "source_sample_id": "validation_History_16" - }, - { - "source_row_index": 532, - "source_sample_id": "validation_History_23" - }, - { - "source_row_index": 540, - "source_sample_id": "validation_Literature_1" - }, - { - "source_row_index": 547, - "source_sample_id": "validation_Literature_8" - }, - { - "source_row_index": 555, - "source_sample_id": "validation_Literature_16" - }, - { - "source_row_index": 562, - "source_sample_id": "validation_Literature_23" - }, - { - "source_row_index": 570, - "source_sample_id": "validation_Manage_1" - }, - { - "source_row_index": 577, - "source_sample_id": "validation_Manage_8" - }, - { - "source_row_index": 585, - "source_sample_id": "validation_Manage_16" - }, - { - "source_row_index": 592, - "source_sample_id": "validation_Manage_23" - }, - { - "source_row_index": 600, - "source_sample_id": "validation_Marketing_1" - }, - { - "source_row_index": 607, - "source_sample_id": "validation_Marketing_8" - }, - { - "source_row_index": 615, - "source_sample_id": "validation_Marketing_16" - }, - { - "source_row_index": 622, - "source_sample_id": "validation_Marketing_23" - }, - { - "source_row_index": 630, - "source_sample_id": "validation_Materials_1" - }, - { - "source_row_index": 637, - "source_sample_id": "validation_Materials_8" - }, - { - "source_row_index": 645, - "source_sample_id": "validation_Materials_16" - }, - { - "source_row_index": 652, - "source_sample_id": "validation_Materials_23" - }, - { - "source_row_index": 660, - "source_sample_id": "validation_Math_1" - }, - { - "source_row_index": 667, - "source_sample_id": "validation_Math_8" - }, - { - "source_row_index": 675, - "source_sample_id": "validation_Math_16" - }, - { - "source_row_index": 682, - "source_sample_id": "validation_Math_23" - }, - { - "source_row_index": 690, - "source_sample_id": "validation_Mechanical_Engineering_1" - }, - { - "source_row_index": 697, - "source_sample_id": "validation_Mechanical_Engineering_8" - }, - { - "source_row_index": 705, - "source_sample_id": "validation_Mechanical_Engineering_16" - }, - { - "source_row_index": 712, - "source_sample_id": "validation_Mechanical_Engineering_23" - }, - { - "source_row_index": 720, - "source_sample_id": "validation_Music_1" - }, - { - "source_row_index": 727, - "source_sample_id": "validation_Music_8" - }, - { - "source_row_index": 735, - "source_sample_id": "validation_Music_16" - }, - { - "source_row_index": 742, - "source_sample_id": "validation_Music_23" - }, - { - "source_row_index": 750, - "source_sample_id": "validation_Pharmacy_1" - }, - { - "source_row_index": 757, - "source_sample_id": "validation_Pharmacy_8" - }, - { - "source_row_index": 765, - "source_sample_id": "validation_Pharmacy_16" - }, - { - "source_row_index": 772, - "source_sample_id": "validation_Pharmacy_23" - }, - { - "source_row_index": 780, - "source_sample_id": "validation_Physics_1" - }, - { - "source_row_index": 787, - "source_sample_id": "validation_Physics_8" - }, - { - "source_row_index": 795, - "source_sample_id": "validation_Physics_16" - }, - { - "source_row_index": 802, - "source_sample_id": "validation_Physics_23" - }, - { - "source_row_index": 810, - "source_sample_id": "validation_Psychology_1" - }, - { - "source_row_index": 817, - "source_sample_id": "validation_Psychology_8" - }, - { - "source_row_index": 825, - "source_sample_id": "validation_Psychology_16" - }, - { - "source_row_index": 832, - "source_sample_id": "validation_Psychology_23" - }, - { - "source_row_index": 840, - "source_sample_id": "validation_Public_Health_1" - }, - { - "source_row_index": 847, - "source_sample_id": "validation_Public_Health_8" - }, - { - "source_row_index": 855, - "source_sample_id": "validation_Public_Health_16" - }, - { - "source_row_index": 862, - "source_sample_id": "validation_Public_Health_23" - }, - { - "source_row_index": 870, - "source_sample_id": "validation_Sociology_1" - }, - { - "source_row_index": 877, - "source_sample_id": "validation_Sociology_8" - }, - { - "source_row_index": 885, - "source_sample_id": "validation_Sociology_16" - }, - { - "source_row_index": 892, - "source_sample_id": "validation_Sociology_23" - } - ] - }, - "mvbench": { - "dataset_repository": "OpenGVLab/MVBench", - "dataset_revision": "a776e554280b99b70f00cc3eacd69a65e0727efc", - "max_new_tokens": 16, - "scoring_task_config": "tasks/mvbench/mvbench.yaml", - "rows": [ - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 0, - "source_sample_id": "action_sequence:0" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 1, - "source_sample_id": "action_sequence:1" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 2, - "source_sample_id": "action_sequence:2" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 3, - "source_sample_id": "action_sequence:3" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 4, - "source_sample_id": "action_sequence:4" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 5, - "source_sample_id": "action_sequence:5" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 6, - "source_sample_id": "action_sequence:6" - }, - { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 7, - "source_sample_id": "action_sequence:7" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 0, - "source_sample_id": "moving_count:0" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 1, - "source_sample_id": "moving_count:1" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 2, - "source_sample_id": "moving_count:2" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 3, - "source_sample_id": "moving_count:3" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 4, - "source_sample_id": "moving_count:4" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 5, - "source_sample_id": "moving_count:5" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 6, - "source_sample_id": "moving_count:6" - }, - { - "leaf_task": "mvbench_moving_count", - "source_row_index": 7, - "source_sample_id": "moving_count:7" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 0, - "source_sample_id": "action_prediction:0" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 1, - "source_sample_id": "action_prediction:1" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 2, - "source_sample_id": "action_prediction:2" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 3, - "source_sample_id": "action_prediction:3" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 4, - "source_sample_id": "action_prediction:4" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 5, - "source_sample_id": "action_prediction:5" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 6, - "source_sample_id": "action_prediction:6" - }, - { - "leaf_task": "mvbench_action_prediction", - "source_row_index": 7, - "source_sample_id": "action_prediction:7" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 0, - "source_sample_id": "episodic_reasoning:0" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 1, - "source_sample_id": "episodic_reasoning:1" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 2, - "source_sample_id": "episodic_reasoning:2" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 3, - "source_sample_id": "episodic_reasoning:3" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 4, - "source_sample_id": "episodic_reasoning:4" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 5, - "source_sample_id": "episodic_reasoning:5" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 6, - "source_sample_id": "episodic_reasoning:6" - }, - { - "leaf_task": "mvbench_episodic_reasoning", - "source_row_index": 7, - "source_sample_id": "episodic_reasoning:7" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 0, - "source_sample_id": "action_antonym:0" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 1, - "source_sample_id": "action_antonym:1" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 2, - "source_sample_id": "action_antonym:2" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 3, - "source_sample_id": "action_antonym:3" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 4, - "source_sample_id": "action_antonym:4" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 5, - "source_sample_id": "action_antonym:5" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 6, - "source_sample_id": "action_antonym:6" - }, - { - "leaf_task": "mvbench_action_antonym", - "source_row_index": 7, - "source_sample_id": "action_antonym:7" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 0, - "source_sample_id": "action_count:0" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 1, - "source_sample_id": "action_count:1" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 2, - "source_sample_id": "action_count:2" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 3, - "source_sample_id": "action_count:3" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 4, - "source_sample_id": "action_count:4" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 5, - "source_sample_id": "action_count:5" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 6, - "source_sample_id": "action_count:6" - }, - { - "leaf_task": "mvbench_action_count", - "source_row_index": 7, - "source_sample_id": "action_count:7" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 0, - "source_sample_id": "scene_transition:0" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 1, - "source_sample_id": "scene_transition:1" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 2, - "source_sample_id": "scene_transition:2" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 3, - "source_sample_id": "scene_transition:3" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 4, - "source_sample_id": "scene_transition:4" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 5, - "source_sample_id": "scene_transition:5" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 6, - "source_sample_id": "scene_transition:6" - }, - { - "leaf_task": "mvbench_scene_transition", - "source_row_index": 7, - "source_sample_id": "scene_transition:7" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 0, - "source_sample_id": "object_shuffle:0" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 1, - "source_sample_id": "object_shuffle:1" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 2, - "source_sample_id": "object_shuffle:2" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 3, - "source_sample_id": "object_shuffle:3" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 4, - "source_sample_id": "object_shuffle:4" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 5, - "source_sample_id": "object_shuffle:5" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 6, - "source_sample_id": "object_shuffle:6" - }, - { - "leaf_task": "mvbench_object_shuffle", - "source_row_index": 7, - "source_sample_id": "object_shuffle:7" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 0, - "source_sample_id": "object_existence:0" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 1, - "source_sample_id": "object_existence:1" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 2, - "source_sample_id": "object_existence:2" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 3, - "source_sample_id": "object_existence:3" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 4, - "source_sample_id": "object_existence:4" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 5, - "source_sample_id": "object_existence:5" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 6, - "source_sample_id": "object_existence:6" - }, - { - "leaf_task": "mvbench_object_existence", - "source_row_index": 7, - "source_sample_id": "object_existence:7" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 0, - "source_sample_id": "fine_grained_pose:0" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 1, - "source_sample_id": "fine_grained_pose:1" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 2, - "source_sample_id": "fine_grained_pose:2" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 3, - "source_sample_id": "fine_grained_pose:3" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 4, - "source_sample_id": "fine_grained_pose:4" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 5, - "source_sample_id": "fine_grained_pose:5" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 6, - "source_sample_id": "fine_grained_pose:6" - }, - { - "leaf_task": "mvbench_fine_grained_pose", - "source_row_index": 7, - "source_sample_id": "fine_grained_pose:7" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 0, - "source_sample_id": "unexpected_action:0" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 1, - "source_sample_id": "unexpected_action:1" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 2, - "source_sample_id": "unexpected_action:2" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 3, - "source_sample_id": "unexpected_action:3" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 4, - "source_sample_id": "unexpected_action:4" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 5, - "source_sample_id": "unexpected_action:5" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 6, - "source_sample_id": "unexpected_action:6" - }, - { - "leaf_task": "mvbench_unexpected_action", - "source_row_index": 7, - "source_sample_id": "unexpected_action:7" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 0, - "source_sample_id": "moving_direction:0" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 1, - "source_sample_id": "moving_direction:1" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 2, - "source_sample_id": "moving_direction:2" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 3, - "source_sample_id": "moving_direction:3" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 4, - "source_sample_id": "moving_direction:4" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 5, - "source_sample_id": "moving_direction:5" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 6, - "source_sample_id": "moving_direction:6" - }, - { - "leaf_task": "mvbench_moving_direction", - "source_row_index": 7, - "source_sample_id": "moving_direction:7" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 0, - "source_sample_id": "state_change:0" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 1, - "source_sample_id": "state_change:1" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 2, - "source_sample_id": "state_change:2" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 3, - "source_sample_id": "state_change:3" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 4, - "source_sample_id": "state_change:4" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 5, - "source_sample_id": "state_change:5" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 6, - "source_sample_id": "state_change:6" - }, - { - "leaf_task": "mvbench_state_change", - "source_row_index": 7, - "source_sample_id": "state_change:7" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 0, - "source_sample_id": "object_interaction:0" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 1, - "source_sample_id": "object_interaction:1" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 2, - "source_sample_id": "object_interaction:2" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 3, - "source_sample_id": "object_interaction:3" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 4, - "source_sample_id": "object_interaction:4" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 5, - "source_sample_id": "object_interaction:5" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 6, - "source_sample_id": "object_interaction:6" - }, - { - "leaf_task": "mvbench_object_interaction", - "source_row_index": 7, - "source_sample_id": "object_interaction:7" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 0, - "source_sample_id": "character_order:0" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 1, - "source_sample_id": "character_order:1" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 2, - "source_sample_id": "character_order:2" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 3, - "source_sample_id": "character_order:3" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 4, - "source_sample_id": "character_order:4" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 5, - "source_sample_id": "character_order:5" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 6, - "source_sample_id": "character_order:6" - }, - { - "leaf_task": "mvbench_character_order", - "source_row_index": 7, - "source_sample_id": "character_order:7" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 0, - "source_sample_id": "action_localization:0" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 1, - "source_sample_id": "action_localization:1" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 2, - "source_sample_id": "action_localization:2" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 3, - "source_sample_id": "action_localization:3" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 4, - "source_sample_id": "action_localization:4" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 5, - "source_sample_id": "action_localization:5" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 6, - "source_sample_id": "action_localization:6" - }, - { - "leaf_task": "mvbench_action_localization", - "source_row_index": 7, - "source_sample_id": "action_localization:7" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 0, - "source_sample_id": "counterfactual_inference:0" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 1, - "source_sample_id": "counterfactual_inference:1" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 2, - "source_sample_id": "counterfactual_inference:2" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 3, - "source_sample_id": "counterfactual_inference:3" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 4, - "source_sample_id": "counterfactual_inference:4" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 5, - "source_sample_id": "counterfactual_inference:5" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 6, - "source_sample_id": "counterfactual_inference:6" - }, - { - "leaf_task": "mvbench_counterfactual_inference", - "source_row_index": 7, - "source_sample_id": "counterfactual_inference:7" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 0, - "source_sample_id": "fine_grained_action:0" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 1, - "source_sample_id": "fine_grained_action:1" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 2, - "source_sample_id": "fine_grained_action:2" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 3, - "source_sample_id": "fine_grained_action:3" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 4, - "source_sample_id": "fine_grained_action:4" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 5, - "source_sample_id": "fine_grained_action:5" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 6, - "source_sample_id": "fine_grained_action:6" - }, - { - "leaf_task": "mvbench_fine_grained_action", - "source_row_index": 7, - "source_sample_id": "fine_grained_action:7" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 0, - "source_sample_id": "moving_attribute:0" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 1, - "source_sample_id": "moving_attribute:1" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 2, - "source_sample_id": "moving_attribute:2" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 3, - "source_sample_id": "moving_attribute:3" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 4, - "source_sample_id": "moving_attribute:4" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 5, - "source_sample_id": "moving_attribute:5" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 6, - "source_sample_id": "moving_attribute:6" - }, - { - "leaf_task": "mvbench_moving_attribute", - "source_row_index": 7, - "source_sample_id": "moving_attribute:7" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 0, - "source_sample_id": "egocentric_navigation:0" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 1, - "source_sample_id": "egocentric_navigation:1" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 2, - "source_sample_id": "egocentric_navigation:2" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 3, - "source_sample_id": "egocentric_navigation:3" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 4, - "source_sample_id": "egocentric_navigation:4" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 5, - "source_sample_id": "egocentric_navigation:5" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 6, - "source_sample_id": "egocentric_navigation:6" - }, - { - "leaf_task": "mvbench_egocentric_navigation", - "source_row_index": 7, - "source_sample_id": "egocentric_navigation:7" - } - ] - } - } + "sample_set": "core-3_344-examples_legacy-r1", + "backend_profile": "qwen-3.5-vllm_r1", + "evaluator_profile": "lmms-eval-modelopt_r1" } diff --git a/examples/puzzletron/evaluation/vlm/suites.py b/examples/puzzletron/evaluation/vlm/suites.py index e8f2fbc7d0a..32a887b6fa5 100644 --- a/examples/puzzletron/evaluation/vlm/suites.py +++ b/examples/puzzletron/evaluation/vlm/suites.py @@ -52,8 +52,10 @@ "execution_policy", "generation_kwargs", "load_quick_manifest", + "manifest_row_identities", "manifest_selected_rows", "manifest_sha256", + "manifest_task_denominators", "offline_dataset_snapshot", "source_tasks", "task_name", @@ -131,6 +133,7 @@ class FramePolicy(TypedDict): class GenerationPolicy(TypedDict): """Generation fields shared by provenance and execution.""" + enable_thinking: bool temperature: int do_sample: bool @@ -195,7 +198,7 @@ def execution_policy(suite: str, *, timeout_seconds: float | None) -> ExecutionP limit = None return { "frame": {"reader": "decord", "fps": 2, "max_frames": 32}, - "generation": {"temperature": 0, "do_sample": False}, + "generation": {"enable_thinking": False, "temperature": 0, "do_sample": False}, "limit": limit, "repetitions": 2 if suite in {"short", TASK_PREFIX100_REPEAT2_SUITE} else 1, "timeout_seconds": ( @@ -319,6 +322,42 @@ def manifest_selected_rows(manifest: dict[str, object]) -> int: return sum(len(cast("list[object]", entry["rows"])) for entry in manifest_tasks.values()) +def manifest_row_identities(manifest: dict[str, object]) -> dict[str, list[dict[str, object]]]: + """Project the exact selected row identities in task and row order.""" + + manifest_tasks = cast("dict[str, dict[str, object]]", manifest["tasks"]) + return { + task: [ + { + key: row[key] + for key in ("source_row_index", "source_sample_id", "leaf_task") + if key in row + } + for row in cast("list[dict[str, object]]", entry["rows"]) + ] + for task, entry in manifest_tasks.items() + } + + +def manifest_task_denominators( + manifest: dict[str, object], +) -> dict[str, dict[str, int | None]]: + """Project selected and population row denominators for every exact task.""" + + manifest_tasks = cast("dict[str, dict[str, object]]", manifest["tasks"]) + denominators = {} + for task, entry in manifest_tasks.items(): + selection = entry.get("selection", {}) + if not isinstance(selection, dict): + raise ValueError(f"exact-row manifest task selection must be an object: {task}") + population_rows = selection.get("population_rows", entry.get("population_rows")) + denominators[task] = { + "selected_rows": len(cast("list[object]", entry["rows"])), + "population_rows": cast("int | None", population_rows), + } + return denominators + + def offline_dataset_snapshot(hf_home: Path, task: str, revision: str) -> Path: """Resolve an exact local Hub dataset snapshot without dereferencing mount aliases.""" configured_hub_cache = os.environ.get("HF_HUB_CACHE") diff --git a/examples/puzzletron/evaluation/vlm/tasks.py b/examples/puzzletron/evaluation/vlm/tasks.py index 8b12a1b9e81..2bf1d63bcaa 100644 --- a/examples/puzzletron/evaluation/vlm/tasks.py +++ b/examples/puzzletron/evaluation/vlm/tasks.py @@ -62,6 +62,7 @@ def _write_task_config( process_docs: str | None = None, process_docs_module: str = "modelopt_quick_selection", process_results: str | None = None, + process_results_module: str = "modelopt_mmvu_guard", ) -> None: values = { "include": str(include), @@ -80,29 +81,58 @@ def _write_task_config( if doc_to_visual is not None: lines.append(f"doc_to_visual: !function modelopt_video_paths.{doc_to_visual}") if process_results is not None: - lines.append(f"process_results: !function modelopt_mmvu_guard.{process_results}") + lines.append(f"process_results: !function {process_results_module}.{process_results}") checkpoint.write_generated(path, "\n".join(lines) + "\n") +def _manifest_group_leaves( + manifest: dict[str, object], task: str, leaves: tuple[str, ...] +) -> tuple[str, ...]: + manifest_tasks = cast("dict[str, dict[str, object]]", manifest["tasks"]) + rows = cast("list[dict[str, object]]", manifest_tasks[task]["rows"]) + selected_leaf_tasks = {cast("str", row["leaf_task"]) for row in rows} + selected = tuple(leaf for leaf in leaves if f"{task}_{leaf}" in selected_leaf_tasks) + if not selected: + raise ValueError(f"exact-row manifest selects no leaves for grouped task {task}") + return selected + + def _write_quick_selection_module(tasks_root: Path, manifest: dict[str, object]) -> None: entries: dict[str, dict[str, object]] = {} functions: list[str] = [] manifest_tasks = cast("dict[str, dict[str, object]]", manifest["tasks"]) for task in manifest_tasks: - rows = cast("list[dict[str, object]]", manifest_tasks[task]["rows"]) + task_entry = manifest_tasks[task] + rows = cast("list[dict[str, object]]", task_entry["rows"]) + sampling = task_entry.get("selection") + audited = cast("dict[str, object]", sampling) if isinstance(sampling, dict) else None if task in {"mvbench", "video_mmmu"}: - leaves = ( + all_leaves = ( suites.MVBENCH_LEAF_TASKS if task == "mvbench" else suites.VIDEO_MMMU_LEAF_TASKS ) + leaves = _manifest_group_leaves(manifest, task, all_leaves) for leaf in leaves: leaf_task = f"{task}_{leaf}" selected = [row for row in rows if row["leaf_task"] == leaf_task] key = suites.task_name(task, leaf=leaf) + stratum = ( + next( + item + for item in cast("list[dict[str, object]]", audited["strata"]) + if item["name"] == leaf + ) + if audited is not None + else None + ) entries[key] = { "kind": task, "config": leaf, "indices": [row["source_row_index"] for row in selected], "source_ids": [row["source_sample_id"] for row in selected], + "population_rows": stratum["population_rows"] if stratum else None, + "strata": {leaf: stratum["population_rows"]} if stratum else None, + "upstream_ids": [row.get("upstream_sample_id") for row in selected], + "sampling_positions": None, } functions.append( f"def select_{key}(documents):\n return _select(documents, {key!r})\n" @@ -113,20 +143,101 @@ def _write_quick_selection_module(tasks_root: Path, manifest: dict[str, object]) "kind": task, "indices": [row["source_row_index"] for row in rows], "source_ids": [row["source_sample_id"] for row in rows], + "population_rows": audited.get("population_rows") if audited else None, + "strata": ( + { + item["name"]: item["population_rows"] + for item in cast("list[dict[str, object]]", audited["strata"]) + } + if audited + else None + ), + "upstream_ids": [row.get("upstream_sample_id") for row in rows], + "sampling_positions": ( + [(row["sampling_stratum"], row["source_stratum_index"]) for row in rows] + if all( + "sampling_stratum" in row and "source_stratum_index" in row for row in rows + ) + else None + ), } functions.append( f"def select_{key}(documents):\n return _select(documents, {key!r})\n" ) source = f'''"""Generated exact-row selectors for a VLM benchmark profile.""" +from collections import Counter + _SELECTIONS = {entries!r} +def _column(documents, name): + values = documents[name] + if len(values) != len(documents): + raise ValueError(f"exact-row manifest source column {{name}} has the wrong length") + return values + + +def _source_strata(documents, kind): + if kind == "mmmu_val": + return [value.removeprefix("validation_").rsplit("_", 1)[0] for value in _column(documents, "id")] + if kind == "videomme": + return [f"{{duration}}|{{domain}}" for duration, domain in zip( + _column(documents, "duration"), _column(documents, "domain"), strict=True + )] + if kind == "mlvu_dev": + return list(_column(documents, "task_type")) + if kind == "perceptiontest_val_mc": + return [f"{{area}}|{{reasoning}}" for area, reasoning in zip( + _column(documents, "area"), _column(documents, "reasoning"), strict=True + )] + return None + + +def _verify_population(documents, name, selection): + expected_rows = selection["population_rows"] + if expected_rows is None: + return None + if len(documents) != expected_rows: + raise ValueError( + f"exact-row manifest source population drifted for {{name}}: " + f"{{len(documents)}} != {{expected_rows}}" + ) + observed_strata = _source_strata(documents, selection["kind"]) + if observed_strata is not None and Counter(observed_strata) != selection["strata"]: + raise ValueError(f"exact-row manifest source strata drifted for {{name}}") + return observed_strata + + +def _stratum_ranks(strata): + counts = Counter() + ranks = [] + for stratum in strata: + ranks.append(counts[stratum]) + counts[stratum] += 1 + return ranks + + +def _upstream_id(document, kind): + if kind in {{"videomme", "mlvu_dev"}}: + return str(document["question_id"]) + if kind == "perceptiontest_val_mc": + return f"{{document['video_name']}}:{{document['question_id']}}" + return None + + def _select(documents, name): selection = _SELECTIONS[name] + observed_strata = _verify_population(documents, name, selection) indices = selection["indices"] + expected_positions = selection["sampling_positions"] + if observed_strata is not None and expected_positions is not None: + ranks = _stratum_ranks(observed_strata) + observed_positions = [(observed_strata[index], ranks[index]) for index in indices] + if observed_positions != expected_positions: + raise ValueError(f"exact-row manifest source sampling positions drifted for {{name}}") observed = [] - for index in indices: + for position, index in enumerate(indices): if index >= len(documents): raise ValueError(f"exact-row manifest row {{index}} is outside {{name}}") document = documents[index] @@ -138,6 +249,9 @@ def _select(documents, name): observed.append(f"{{selection['config']}}:{{index}}") else: observed.append(f"{{selection['kind']}}:{{index}}") + expected_upstream_id = selection["upstream_ids"][position] + if expected_upstream_id is not None and _upstream_id(document, selection["kind"]) != expected_upstream_id: + raise ValueError(f"exact-row manifest upstream identity drifted for {{name}}") if observed != selection["source_ids"]: raise ValueError(f"exact-row manifest source identities drifted for {{name}}") return documents.select(indices) @@ -218,6 +332,64 @@ def process_results(document, results): ) +def _write_mmmu_audit(tasks_root: Path) -> None: + checkpoint.write_generated( + tasks_root / "modelopt_mmmu_audit.py", + '''"""Expose whether the pinned MMMU parser used its random fallback.""" + +import ast +import random + +from lmms_eval.tasks.mmmu import utils as _upstream + + +class _TrackedChoices(list): + def __init__(self, choices): + super().__init__(choices) + self.used_random_fallback = False + + def __getitem__(self, index): + self.used_random_fallback = True + return super().__getitem__(index) + + +def _multiple_choice_status(document, response): + index_to_answer, choices = _upstream.get_multi_choice_info( + ast.literal_eval(document["options"]) + ) + tracked_choices = _TrackedChoices(choices) + random_state = random.getstate() + try: + _upstream.parse_multi_choice_response(response, tracked_choices, index_to_answer) + finally: + random.setstate(random_state) + return "fallback_random" if tracked_choices.used_random_fallback else "parsed" + + +def process_results(document, results): + processed = _upstream.mmmu_process_results(document, results) + accuracy = processed.get("mmmu_acc") + if not isinstance(accuracy, dict): + raise RuntimeError("MMMU result is missing its per-sample accuracy leaf") + parsed_predictions = accuracy.get("parsed_pred") + if not isinstance(parsed_predictions, list) or len(parsed_predictions) != len(results): + raise RuntimeError("MMMU result has invalid parsed predictions") + if document.get("question_type") == "multiple-choice": + statuses = [ + _multiple_choice_status(document, response) + for response in results + ] + else: + statuses = [ + "parsed_open" if str(prediction).strip() else "invalid_open" + for prediction in parsed_predictions + ] + accuracy["parser_status"] = statuses + return processed +''', + ) + + def _write_video_path_adapter(tasks_root: Path) -> None: alias_root = tasks_root / "video_path_aliases" checkpoint.write_generated( @@ -275,6 +447,7 @@ def verify_offline( hf_home: Path, timeout_seconds: float, model_name: str = "vllm", + expected_populations: dict[str, int] | None = None, ) -> dict[str, object]: """Instantiate every generated task with network access disabled.""" script = """ @@ -285,7 +458,8 @@ def verify_offline( from lmms_eval.tasks import TaskManager -model_name, root, *tasks = sys.argv[1:] +model_name, root, expected_json, *tasks = sys.argv[1:] +expected_populations = json.loads(expected_json) credential_names = ("HF_TOKEN", "HUGGINGFACEHUB_API_TOKEN", "HUGGING_FACE_HUB_TOKEN") if inherited := [name for name in credential_names if name in os.environ]: raise RuntimeError(f"offline task preflight inherited Hub credentials: {inherited}") @@ -311,10 +485,16 @@ def task_objects(value): "modelopt_vlm_benchmark_mmmu_val", } media_documents = 0 +document_counts = {} +seen_task_objects = set() for task in task_objects(loaded): - task_name = task.config.task - if task_name in image_tasks: + task_object_id = id(task) + if task_object_id in seen_task_objects: continue + seen_task_objects.add(task_object_id) + task_name = task.config.task + if task_name in document_counts: + raise RuntimeError(f"distinct task objects share configured task name: {task_name}") if task.has_test_docs(): documents = task.test_docs() elif task.has_validation_docs(): @@ -322,7 +502,11 @@ def task_objects(value): elif task.has_training_docs(): documents = task.training_docs() else: - raise RuntimeError(f"configured media task has no evaluation split: {task_name}") + raise RuntimeError(f"configured task has no evaluation split: {task_name}") + document_counts[task_name] = len(documents) + if task_name in image_tasks: + _ = len(documents) + continue for document in documents: visuals = task.doc_to_visual(document) if not visuals: @@ -337,11 +521,21 @@ def task_objects(value): ) media_documents += 1 +observed_populations = { + task_name: document_counts.get(task_name, 0) for task_name in expected_populations +} +if observed_populations != expected_populations: + raise RuntimeError( + f"configured task population mismatch: {observed_populations} != {expected_populations}" + ) + print( json.dumps( { "configured_tasks": tasks, + "document_counts": document_counts, "media_documents": media_documents, + "observed_populations": observed_populations, "status": "passed", }, sort_keys=True, @@ -361,7 +555,15 @@ def task_objects(value): # A child interpreter is required to import lmms-eval in a clean offline environment. The # fixed interpreter/script and argument-vector invocation avoid shell parsing or interpolation. completed = subprocess.run( - [sys.executable, "-c", script, model_name, str(tasks_root), *configured_tasks], + [ + sys.executable, + "-c", + script, + model_name, + str(tasks_root), + json.dumps(expected_populations or {}, sort_keys=True), + *configured_tasks, + ], check=False, capture_output=True, env=env, @@ -399,6 +601,8 @@ def prepare( _write_mmvu_smoke_selection_module(tasks_root) if suite == "full": _write_mmvu_guard(tasks_root) + if "mmmu_val" in source_tasks: + _write_mmmu_audit(tasks_root) if set(source_tasks) & {"videomme", "perceptiontest_val_mc"}: _write_video_path_adapter(tasks_root) @@ -449,6 +653,8 @@ def _write_task_group( dataset_path: Path, quick_manifest: dict[str, object] | None, ) -> str: + if quick_manifest is not None: + leaves = _manifest_group_leaves(quick_manifest, task, leaves) generated_leaves = [] for leaf in leaves: leaf_task = suites.task_name(task, leaf=leaf) @@ -498,6 +704,13 @@ def _write_single_task( doc_to_visual=doc_to_visual, process_docs=process_docs, process_docs_module=process_docs_module, - process_results="process_results" if suite == "full" and task == "mmvu_val" else None, + process_results=( + "process_results" + if task == "mmmu_val" or (suite == "full" and task == "mmvu_val") + else None + ), + process_results_module=( + "modelopt_mmmu_audit" if task == "mmmu_val" else "modelopt_mmvu_guard" + ), ) return configured_task diff --git a/examples/puzzletron/patches/lmms_eval_wandb_0_29.patch b/examples/puzzletron/patches/lmms_eval_compat_3e675904.patch similarity index 88% rename from examples/puzzletron/patches/lmms_eval_wandb_0_29.patch rename to examples/puzzletron/patches/lmms_eval_compat_3e675904.patch index 6ddc4f76ca7..98022f7e2da 100644 --- a/examples/puzzletron/patches/lmms_eval_wandb_0_29.patch +++ b/examples/puzzletron/patches/lmms_eval_compat_3e675904.patch @@ -23,6 +23,12 @@ index 89978777..57a31642 100644 @@ -55,2 +40,0 @@ class WandbLogger: - self.printer = get_wandb_printer() - +diff --git a/lmms_eval/models/simple/vllm.py b/lmms_eval/models/simple/vllm.py +index 22946997..e0af27ef 100644 +--- a/lmms_eval/models/simple/vllm.py ++++ b/lmms_eval/models/simple/vllm.py +@@ -521 +520,0 @@ class VLLM(lmms): +- sampling_params = SamplingParams(**params) diff --git a/lmms_eval/tasks/emma/utils.py b/lmms_eval/tasks/emma/utils.py index 001d70c8..d90a88fe 100644 --- a/lmms_eval/tasks/emma/utils.py diff --git a/modelopt/torch/puzzletron/evaluation/lmms.py b/modelopt/torch/puzzletron/evaluation/lmms.py index 72efcc2fd6b..30d94bf8a53 100644 --- a/modelopt/torch/puzzletron/evaluation/lmms.py +++ b/modelopt/torch/puzzletron/evaluation/lmms.py @@ -43,6 +43,7 @@ { "dtype", "gpu_memory_utilization", + "attention_config", "chat_template", "max_model_len", "trust_remote_code", @@ -609,6 +610,7 @@ def _result_payload(output_path: Path) -> tuple[dict[str, Any], Path]: def _write_streams(output_path: Path, result: _ProcessResult) -> dict[str, str]: + output_path.mkdir(parents=True, exist_ok=True) stream_paths = {} for stream_name, text in (("stdout", result.stdout), ("stderr", result.stderr)): stream_path = output_path / f"{stream_name}.txt" @@ -769,17 +771,17 @@ def run_lmms_eval_checkpoint( checkpoint=str(checkpoint_path), output_path=output, ) - command_path = _atomic_json( - output / "command.json", - { - "argv": argv, - "env_overrides": sorted(str(key) for key in dict(settings.get("env") or {})), - "timeout": timeout, - }, - ) + command_payload = { + "argv": argv, + "env_overrides": sorted(str(key) for key in dict(settings.get("env") or {})), + "timeout": timeout, + } + command_path = _atomic_json(output / "command.json", command_payload) try: result = _run_process(argv, cwd=str(output), env=env, timeout=timeout) except LmmsEvalTimeoutError as error: + if not command_path.is_file(): + command_path = _atomic_json(output / "command.json", command_payload) captured = _ProcessResult(argv, -1, error.output, error.stderr) stream_paths = _write_streams(output, captured) _annotate_error(error, command_path=command_path, stream_paths=stream_paths) diff --git a/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py b/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py index da4ae62ff4e..af129c0d18f 100644 --- a/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py +++ b/tests/unit/torch/puzzletron/evaluation/test_checkpoint.py @@ -45,7 +45,7 @@ def test_load_runner_restores_import_state(monkeypatch): assert "modelopt.torch.puzzletron.evaluation.lmms" not in sys.modules -def test_verify_lmms_eval_revision_rejects_unpatched_vcs_install(monkeypatch): +def test_verify_lmms_eval_revision_rejects_unpatched_current_vcs_install(monkeypatch): provenance = { "url": checkpoint.LMMS_EVAL_SOURCE["repository"], "vcs_info": {"commit_id": checkpoint.LMMS_EVAL_REVISION}, @@ -59,6 +59,11 @@ def test_verify_lmms_eval_revision_rejects_unpatched_vcs_install(monkeypatch): checkpoint.verify_lmms_eval_revision() +def test_verify_lmms_eval_revision_rejects_unsupported_revision(): + with pytest.raises(RuntimeError, match="unsupported lmms-eval revision"): + checkpoint.verify_lmms_eval_revision("different") + + def test_verify_lmms_eval_revision_accepts_clean_pinned_editable_checkout(monkeypatch, tmp_path): provenance = {"dir_info": {"editable": True}, "url": tmp_path.as_uri()} monkeypatch.setattr(checkpoint, "_imported_lmms_eval_revision", lambda: None) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py index 1f1ef0f57b1..6a83bab18b9 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/preparation/test_benchmark_data.py @@ -18,7 +18,9 @@ import hashlib import io import json +import stat import tarfile +import threading import zipfile from pathlib import Path from types import SimpleNamespace @@ -28,6 +30,16 @@ from examples.puzzletron.evaluation.vlm.preparation import benchmark_data as preparation _EXPECTED_DATASETS = { + "realworldqa": ( + "lmms-lab/RealWorldQA", + "907c4e5228fd1703c710ed937601cb5f89ab8d5c", + None, + ), + "mmmu_val": ( + "lmms-lab/MMMU", + "364f2e2eb107b36e07ff4c5a15f5947a759cef47", + None, + ), "video_mmmu": ( "lmms-lab/VideoMMMU", "d1c35ac933123d79e877b7f1b9506afb0309cf1b", @@ -64,6 +76,29 @@ def _write_zip(path: Path, members: dict[str, bytes]) -> None: archive.writestr(name, payload) +def _emulate_atomic_exchange(first: Path, second: Path) -> bool: + displaced = second.with_name(f".{second.name}.test-exchange") + second.rename(displaced) + first.rename(second) + displaced.rename(first) + return True + + +def test_atomic_exchange_directories_when_supported(tmp_path): + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + (first / "identity").write_text("first") + (second / "identity").write_text("second") + + if not preparation._atomic_exchange_directories(first, second): + pytest.skip("atomic directory exchange is unavailable on this host") + + assert (first / "identity").read_text() == "second" + assert (second / "identity").read_text() == "first" + + @pytest.mark.parametrize( ("task", "repository", "revision", "directory"), [(task, *values) for task, values in _EXPECTED_DATASETS.items()], @@ -76,6 +111,69 @@ def test_every_preparation_contract_is_explicitly_pinned(task, repository, revis revision, directory, ) + assert preparation.benchmark_catalog_contract((task,))[task] == { + "repository": repository, + "revision": revision, + "requires_media": directory is not None, + "preparation_dir": directory, + } + + +def test_prepare_benchmark_datasets_dispatches_media_only_for_media_tasks(tmp_path, monkeypatch): + hf_home = tmp_path / "hf-home" + prepared = [] + + def download(root, task, *, max_workers): + del max_workers + snapshot = preparation._hub_snapshot(root, task) + snapshot.mkdir(parents=True) + (snapshot / "dataset-info.json").write_text("{}") + return snapshot + + def prepare(root, task, snapshot, *, verify_content=False): + assert not verify_content + prepared.append(task) + media_root = root / preparation.DATASETS[task].preparation_dir + media_root.mkdir(parents=True) + (media_root / "sample.mp4").write_bytes(b"video") + payload = { + **preparation._marker_payload(task, status="complete"), + "snapshot": str(snapshot), + "media_root": str(media_root), + "files": 1, + "bytes": 5, + } + preparation._write_marker(media_root, payload) + return payload + + monkeypatch.setattr(preparation, "_download", download) + monkeypatch.setattr(preparation, "_prepare", prepare) + + reports = preparation.prepare_benchmark_datasets( + hf_home, ["realworldqa", "mmmu_val", "mvbench"], max_workers=3 + ) + + assert [report["task"] for report in reports] == ["realworldqa", "mmmu_val", "mvbench"] + assert [report["requires_media"] for report in reports] == [False, False, True] + assert prepared == ["mvbench"] + + +def test_prepare_benchmark_datasets_rejects_symlinked_hf_home(tmp_path): + target = tmp_path / "target" + target.mkdir() + alias = tmp_path / "hf-home" + alias.symlink_to(target, target_is_directory=True) + + with pytest.raises(ValueError, match="must not be a symlink"): + preparation.prepare_benchmark_datasets(alias, ("realworldqa",)) + + +def test_prepare_benchmark_datasets_rejects_dangling_symlinked_hf_home(tmp_path): + alias = tmp_path / "hf-home" + alias.symlink_to(tmp_path / "missing", target_is_directory=True) + + with pytest.raises(ValueError, match="must not be a symlink"): + preparation.prepare_benchmark_datasets(alias, ("realworldqa",)) def test_zip_preparation_is_revision_bound_idempotent_and_byte_verified(tmp_path): @@ -90,6 +188,7 @@ def test_zip_preparation_is_revision_bound_idempotent_and_byte_verified(tmp_path assert second == first assert first["status"] == "complete" assert first["files"] == 1 + assert stat.S_IMODE((hf_home / "mmvu").stat().st_mode) == 0o755 marker = json.loads((hf_home / "mmvu" / preparation._MARKER_NAME).read_text()) assert marker["revision"] == _EXPECTED_DATASETS["mmvu_val"][1] @@ -104,6 +203,226 @@ def test_zip_preparation_is_revision_bound_idempotent_and_byte_verified(tmp_path preparation._extract_zip(archive, target) +def test_prepared_media_reuses_metadata_unless_content_verification_is_requested( + tmp_path, monkeypatch +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + expected = preparation._prepare(hf_home, "mmvu_val", snapshot) + original_sha256 = preparation._sha256 + hashed = [] + + def record_hash(path): + hashed.append(path) + return original_sha256(path) + + monkeypatch.setattr(preparation, "_sha256", record_hash) + assert preparation._prepare(hf_home, "mmvu_val", snapshot) == expected + assert not hashed + + assert preparation._prepare(hf_home, "mmvu_val", snapshot, verify_content=True) == expected + assert hashed == [hf_home / "mmvu/videos/sample.mp4"] + + +@pytest.mark.parametrize("damage", ["missing", "corrupt", "unexpected"]) +def test_complete_media_marker_repairs_owned_root_from_pinned_snapshot( + tmp_path, monkeypatch, damage +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + preparation._prepare(hf_home, "mmvu_val", snapshot) + monkeypatch.setattr(preparation, "_atomic_exchange_directories", _emulate_atomic_exchange) + target = hf_home / "mmvu" + media = target / "videos/sample.mp4" + + if damage == "missing": + media.unlink() + elif damage == "corrupt": + media.write_bytes(b"wrong") + elif damage == "unexpected": + (target / "unexpected.bin").write_bytes(b"stale") + report = preparation._prepare(hf_home, "mmvu_val", snapshot) + + assert report["status"] == "complete" + assert media.read_bytes() == b"video" + assert not (target / "unexpected.bin").exists() + assert preparation._media_marker_is_current( + target, + "mmvu_val", + json.loads((target / preparation._MARKER_NAME).read_text()), + ) + + +def test_repair_without_atomic_exchange_preserves_live_root(monkeypatch, tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + preparation._prepare(hf_home, "mmvu_val", snapshot) + target = hf_home / "mmvu" + media = target / "videos/sample.mp4" + media.write_bytes(b"wrong") + monkeypatch.setattr(preparation, "_atomic_exchange_directories", lambda *_args: False) + + with pytest.raises(RuntimeError, match="atomic media-directory exchange is unavailable"): + preparation._prepare(hf_home, "mmvu_val", snapshot) + + assert target.is_dir() + assert media.read_bytes() == b"wrong" + assert not tuple(target.parent.glob(f".{target.name}.modelopt-staging.*")) + assert not tuple(target.parent.glob(f".{target.name}.modelopt-replaced.*")) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + (None, None, "readable ownership marker"), + ("corrupt", "{", "readable ownership marker"), + ("revision", "other-revision", "mismatched ownership: revision"), + ("status", "unknown", "invalid ownership-marker status"), + ], +) +def test_missing_or_mismatched_media_marker_preserves_unproven_root( + tmp_path, field, value, message +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + preparation._prepare(hf_home, "mmvu_val", snapshot) + target = hf_home / "mmvu" + media = target / "videos/sample.mp4" + marker = target / preparation._MARKER_NAME + if field is None: + marker.unlink() + elif field == "corrupt": + marker.write_text(value) + else: + payload = json.loads(marker.read_text()) + payload[field] = value + marker.write_text(json.dumps(payload)) + + with pytest.raises((FileExistsError, ValueError), match=message): + preparation._prepare(hf_home, "mmvu_val", snapshot) + + assert target.is_dir() + assert media.read_bytes() == b"video" + + +def test_media_repair_rejects_symlinks_in_owned_root(tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) + preparation._prepare(hf_home, "mmvu_val", snapshot) + target = hf_home / "mmvu" + (target / "videos/sample.mp4").unlink() + (target / "videos/sample.mp4").symlink_to(tmp_path / "outside") + + with pytest.raises(ValueError, match="repair refuses a symlink"): + preparation._prepare(hf_home, "mmvu_val", snapshot) + + +def test_snapshot_inventory_rejects_partial_and_same_size_corruption(tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "realworldqa") + snapshot.mkdir(parents=True) + first = snapshot / "first.json" + second = snapshot / "second.json" + first.write_bytes(b"one") + second.write_bytes(b"two") + report = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + + assert preparation._snapshot_inventory_is_current(report) + second.unlink() + assert not preparation._snapshot_inventory_is_current(report) + second.write_bytes(b"two") + report = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + assert preparation._snapshot_inventory_is_current(report) + first.write_bytes(b"bad") + assert not preparation._snapshot_inventory_is_current(report) + + +def test_snapshot_inventory_reuses_metadata_unless_content_verification_is_requested( + tmp_path, monkeypatch +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "realworldqa") + snapshot.mkdir(parents=True) + sample = snapshot / "sample.json" + sample.write_bytes(b"one") + expected = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + original_sha256 = preparation._sha256 + hashed = [] + + def record_hash(path): + hashed.append(path) + return original_sha256(path) + + monkeypatch.setattr(preparation, "_sha256", record_hash) + assert preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) == expected + assert not hashed + + assert ( + preparation._snapshot_inventory_report( + hf_home, "realworldqa", snapshot, verify_content=True + ) + == expected + ) + assert hashed == [sample] + + hashed.clear() + sample.write_bytes(b"two") + refreshed = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + assert hashed == [sample] + assert refreshed["files"][0]["sha256"] == hashlib.sha256(b"two").hexdigest() + + +def test_snapshot_inventory_seals_and_validates_hub_blob_symlink(tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "realworldqa") + snapshot.mkdir(parents=True) + payload = b"pinned blob" + blob_sha256 = hashlib.sha256(payload).hexdigest() + blob = snapshot.parent.parent / "blobs" / blob_sha256 + blob.parent.mkdir() + blob.write_bytes(payload) + (snapshot / "dataset.parquet").symlink_to(Path("../../blobs") / blob_sha256) + + report = preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + + assert preparation._snapshot_inventory_is_current(report) + blob.write_bytes(b"broken blob") + assert not preparation._snapshot_inventory_is_current(report) + + +def test_snapshot_inventory_rejects_blob_whose_content_differs_from_sha_name(tmp_path): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "realworldqa") + snapshot.mkdir(parents=True) + blob = snapshot.parent.parent / "blobs" / ("0" * 64) + blob.parent.mkdir() + blob.write_bytes(b"not the named content") + (snapshot / "dataset.parquet").symlink_to(Path("../../blobs") / blob.name) + + with pytest.raises(ValueError, match="differs from its SHA-256 identity"): + preparation._snapshot_inventory_report(hf_home, "realworldqa", snapshot) + + +def test_prepare_benchmark_datasets_rejects_catalog_pin_drift(tmp_path): + catalog = preparation.benchmark_catalog_contract(("realworldqa",)) + catalog["realworldqa"]["revision"] = "stale" + + with pytest.raises(ValueError, match="differs from the authoritative catalog"): + preparation.prepare_benchmark_datasets( + tmp_path / "hf-home", ("realworldqa",), expected_catalog=catalog + ) + + @pytest.mark.parametrize("member", ["../escape.mp4", "/absolute.mp4"]) def test_archive_extraction_rejects_paths_outside_owned_root(tmp_path, member): archive = tmp_path / "unsafe.zip" @@ -152,24 +471,92 @@ def test_archive_extraction_rejects_links_and_streams_multipart_tar(tmp_path): def test_interrupted_initialization_leaves_target_retryable(monkeypatch, tmp_path): hf_home = tmp_path / "hf-home" - hf_home.mkdir() + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + _write_zip(snapshot / "videos.zip", {"videos/sample.mp4": b"video"}) target = hf_home / preparation.DATASETS["mmvu_val"].preparation_dir - write_marker = preparation._write_marker + extract = preparation._extract - def interrupt(staging, payload): - write_marker(staging, payload) + def interrupt(*_args): raise RuntimeError("interrupted") - monkeypatch.setattr(preparation, "_write_marker", interrupt) + monkeypatch.setattr(preparation, "_extract", interrupt) with pytest.raises(RuntimeError, match="interrupted"): - preparation._prepare_target(hf_home, "mmvu_val") + preparation._prepare(hf_home, "mmvu_val", snapshot) assert not target.exists() + assert not tuple(target.parent.glob(f".{target.name}.modelopt-staging.*")) + + monkeypatch.setattr(preparation, "_extract", extract) + report = preparation._prepare(hf_home, "mmvu_val", snapshot) + assert report["status"] == "complete" + assert (target / "videos/sample.mp4").read_bytes() == b"video" - monkeypatch.setattr(preparation, "_write_marker", write_marker) - prepared, complete = preparation._prepare_target(hf_home, "mmvu_val") - assert prepared == target - assert complete is None - assert json.loads((target / preparation._MARKER_NAME).read_text())["status"] == "in_progress" + +def test_task_lock_can_be_reacquired_after_body_failure(tmp_path): + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + + with ( + pytest.raises(RuntimeError, match="failed while locked"), + preparation._task_lock(hf_home, "mmvu_val"), + ): + raise RuntimeError("failed while locked") + + with preparation._task_lock(hf_home, "mmvu_val"): + pass + + +def test_concurrent_media_preparation_is_task_locked_and_publishes_only_complete_root( + monkeypatch, tmp_path +): + hf_home = tmp_path / "hf-home" + snapshot = preparation._hub_snapshot(hf_home, "mmvu_val") + snapshot.mkdir(parents=True) + target = hf_home / "mmvu" + entered = threading.Event() + release = threading.Event() + second_done = threading.Event() + extraction_count = 0 + results = [] + errors = [] + + def extract(_task, _snapshot, staging): + nonlocal extraction_count + extraction_count += 1 + (staging / "videos").mkdir() + (staging / "videos/sample.mp4").write_bytes(b"video") + entered.set() + assert release.wait(timeout=5) + return [] + + def run(*, second=False): + try: + results.append(preparation._prepare(hf_home, "mmvu_val", snapshot)) + except BaseException as error: # pragma: no cover - asserted below + errors.append(error) + finally: + if second: + second_done.set() + + monkeypatch.setattr(preparation, "_extract", extract) + first = threading.Thread(target=run) + first.start() + assert entered.wait(timeout=5) + second = threading.Thread(target=run, kwargs={"second": True}) + second.start() + + assert not second_done.wait(timeout=0.1) + assert not target.exists() + release.set() + first.join(timeout=5) + second.join(timeout=5) + + assert not errors + assert not first.is_alive() and not second.is_alive() + assert extraction_count == 1 + assert len(results) == 2 + assert all(result["status"] == "complete" for result in results) + assert (target / "videos/sample.mp4").read_bytes() == b"video" def test_range_download_resumes_without_forwarding_credentials_and_verifies_hash( diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py new file mode 100644 index 00000000000..3294dc38cf2 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_contracts.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for composed VLM profiles and sample-set contracts.""" + +import json +import shutil + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import contracts, profile, suites +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import _quick_manifest + + +def test_manifest_task_denominators_rejects_non_mapping_selection(): + manifest = contracts.load_profile("core-3_344-examples_r1-native").exact_rows + assert manifest is not None + manifest["tasks"]["realworldqa"]["selection"] = "invalid" + + with pytest.raises(ValueError, match="task selection must be an object"): + suites.manifest_task_denominators(manifest) + + +def test_versioned_profile_contracts_pin_backends_and_fingerprints(): + profiles = {name: contracts.load_profile(name) for name in contracts.PROFILE_NAMES} + assert {name: contract.fingerprint for name, contract in profiles.items()} == { + "short-v1": "f111b49238fd5a1843a22afb6ea7da02c2562e198e6401217b7d41d69b3df49f", + "short-native-v1": "578358d3644fe5d71ee26f8c5a2e8a03f2fad662e40ae8e4eefe19e316185aee", + "core-3_344-examples_r1-native": "2017656d093de7d95d25c7e34241b1d708150157f0c4e6a0bf6bd48649c2191a", + "core-3_344-examples_r1-vllm": "859908fdb32b6bcaddb5400cd4430f4c9026264db38c7a8b56a98f42109c1f78", + "core-3_24-examples_r1-native": "0e51e27d57e27f0c5e4943d077308766387fa739b2d1c413b7b951327358cefc", + "core-3_24-examples_r1-vllm": "9c68168f05003e695258dc119351610b4e98bafb3e4f3e773c4e64ce5d17835a", + "short-all-native-v1": "91a0ec543e9ddf055502ff42fbe98e15125d1174d9133f8e60317fdb1b7b77e0", + "judge-free-8_690-examples_r1-native": "78457702288ba2d9d7b903366f7030302936377690b0ec37a0704e3eda8fd851", + "full-v1": "544a5c5cd5d91248ccf2d2fbe92df5f99f7c4e93a8741a2de8e6bba91aaea5a4", + "core-3_full_r1-native": "976efbd056fecb686e64b912ed50251b1c16e5efe1d3ae3d179f208cc587c0a7", + "core-3_full_r1-vllm": "40fd44fbb4812bd927d3f82e33d6eddec5b4641736c65c48250b8ea77acaec81", + } + + current_short = profiles["core-3_344-examples_r1-native"] + smoke = profiles["core-3_24-examples_r1-native"] + materialized_short = profiles["core-3_344-examples_r1-vllm"] + materialized_smoke = profiles["core-3_24-examples_r1-vllm"] + assert current_short.manifest["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION + assert current_short.manifest["backend"]["name"] == "qwen3_5" + assert materialized_short.manifest["backend"]["name"] == "vllm" + assert materialized_short.sample_set == current_short.sample_set + assert materialized_short.backend_profile != current_short.backend_profile + assert materialized_short.evaluator_profile == current_short.evaluator_profile + assert materialized_short.exact_rows == current_short.exact_rows + assert materialized_smoke.manifest["backend"]["name"] == "vllm" + assert materialized_smoke.manifest["backend"]["enforce_eager"] is True + assert materialized_smoke.exact_rows == smoke.exact_rows + assert suites.manifest_selected_rows(current_short.exact_rows) == 344 + assert suites.manifest_selected_rows(smoke.exact_rows) == 24 + assert ( + suites.manifest_selected_rows(profiles["judge-free-8_690-examples_r1-native"].exact_rows) + == 690 + ) + assert profiles["full-v1"].exact_rows is None + + +def test_compatibility_profiles_warn_until_downstream_callers_migrate(): + for name in ("short-v1", "short-native-v1", "short-all-native-v1", "full-v1"): + with pytest.warns(FutureWarning, match=rf"{name} is a deprecated compatibility profile"): + contracts.warn_deprecated_profile(name) + assert contracts.load_profile(name).manifest["lmms_eval_revision"] == ( + checkpoint.LMMS_EVAL_REVISION + ) + + +@pytest.mark.parametrize( + ("name", "backend"), + [ + ( + "core-3_full_r1-native", + { + "attention_implementation": "sdpa", + "enable_thinking": False, + "name": "qwen3_5", + }, + ), + ( + "core-3_full_r1-vllm", + {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, + ), + ], +) +def test_core3_full_teacher_profiles_pin_paired_population_and_runtime(name, backend): + contract = contracts.load_profile(name) + + assert contract.manifest["model"] == { + "repository": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17", + } + assert contract.manifest["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION + assert contract.manifest["backend"] == backend + assert contract.manifest["generation"] == {"do_sample": False, "temperature": 0} + assert contract.manifest["seed"] == 42 + assert contract.manifest["repetitions"] == 1 + assert contract.manifest["batch_size"] == 1 + assert contract.manifest["selection"] == "all" + assert contract.exact_rows is None + assert { + task: entry["population_rows"] for task, entry in contract.manifest["tasks"].items() + } == {"realworldqa": 765, "mmmu_val": 900, "mvbench": 4000} + assert contract.manifest["tasks"]["mvbench"]["leaf_populations"] == dict.fromkeys( + suites.MVBENCH_LEAF_TASKS, 200 + ) + assert { + task: entry["dataset_revision"] for task, entry in contract.manifest["tasks"].items() + } == { + task: profile.VLM_BENCHMARK_DATASETS[task].revision + for task in ("realworldqa", "mmmu_val", "mvbench") + } + + +@pytest.mark.parametrize( + ("name", "field", "value", "message"), + [ + ( + "core-3_full_r1-native", + "backend", + {"enable_thinking": False, "name": "vllm", "reasoning_parser": "qwen3"}, + "backend profile differs", + ), + ( + "core-3_full_r1-vllm", + "model", + {"repository": "Qwen/Qwen3.5-0.8B", "revision": "different"}, + "model pin differs", + ), + ("core-3_full_r1-native", "population", 764, "population differs"), + ], +) +def test_core3_full_teacher_profiles_reject_contract_overrides( + monkeypatch, tmp_path, name, field, value, message +): + shutil.copytree(contracts._PROFILE_ROOT, tmp_path, dirs_exist_ok=True) + if field == "backend": + manifest_path = tmp_path / "backends" / "qwen-3.5-native_r1.json" + else: + manifest_path = tmp_path / "sample_sets" / "core-3_full_r1.json" + manifest = json.loads(manifest_path.read_text()) + if field == "population": + manifest["tasks"]["realworldqa"]["population_rows"] = value + elif field == "backend": + manifest["settings"] = value + else: + manifest[field] = value + manifest_path.write_text(json.dumps(manifest)) + monkeypatch.setattr(contracts, "_PROFILE_ROOT", tmp_path) + + with pytest.raises(RuntimeError, match=message): + contracts.load_profile(name) + + +def test_audited_profile_rejects_rows_that_drift_from_systematic_selection(monkeypatch, tmp_path): + shutil.copytree(contracts._PROFILE_ROOT, tmp_path, dirs_exist_ok=True) + manifest_path = tmp_path / "sample_sets" / "core-3_344-examples_r1.json" + manifest = json.loads(manifest_path.read_text()) + manifest["tasks"]["realworldqa"]["rows"][0]["source_row_index"] = 6 + manifest_path.write_text(json.dumps(manifest)) + monkeypatch.setattr(contracts, "_PROFILE_ROOT", tmp_path) + + with pytest.raises(RuntimeError, match="rows differ from its sampling audit"): + contracts.load_profile("core-3_344-examples_r1-native") + + +def test_quick_manifest_requires_exact_pins_counts_and_leaf_balance(tmp_path): + path = _quick_manifest(tmp_path / "quick.json") + suites.load_quick_manifest(path) + + manifest = json.loads(path.read_text()) + manifest["tasks"]["mmmu_val"]["rows"].pop() + path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="exactly 120 rows"): + suites.load_quick_manifest(path) + + path = _quick_manifest(path) + manifest = json.loads(path.read_text()) + manifest["tasks"]["mvbench"]["rows"][-1] = { + "leaf_task": "mvbench_action_sequence", + "source_row_index": 8, + "source_sample_id": "action_sequence:8", + } + path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="exactly 8 rows per leaf task"): + suites.load_quick_manifest(path) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py new file mode 100644 index 00000000000..bb989efe5db --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_evaluator.py @@ -0,0 +1,356 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for VLM evaluation execution and result recovery.""" + +import hashlib +import json + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import evaluator, preflight, profile, suites +from examples.puzzletron.evaluation.vlm import run as evaluation +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import ( + _use_offline_fakes, + _write_checkpoint, + _write_fake_mmmu_artifacts, + _write_lmms_tasks, +) + + +def test_chat_template_fingerprint_accepts_file_and_inline_content(tmp_path): + content = "{% if messages %}{{ messages[0]['content'] }}{% endif %}" + template_path = tmp_path / "chat_template.jinja" + template_path.write_text(content) + expected = hashlib.sha256(content.encode()).hexdigest() + + assert ( + evaluator._chat_template_sha256({"model_args": {"chat_template": str(template_path)}}) + == expected + ) + assert evaluator._chat_template_sha256({"model_args": {"chat_template": content}}) == expected + + +def test_mmmu_parser_audit_is_attached_to_normalized_result(tmp_path): + attempt = tmp_path / "attempt" + attempt.mkdir() + task_name = suites.task_name("mmmu_val") + result_path = attempt / "summary.json" + result_path.write_text(json.dumps({"sample_counts": {task_name: 2}})) + raw_result_path = attempt / "20260903_120000_results.json" + raw_result_path.write_text("{}\n") + sample_path = attempt / f"20260903_120000_samples_{task_name}.jsonl" + sample_path.write_text( + "\n".join( + json.dumps({"mmmu_acc": {"parser_status": [status]}}) + for status in ("parsed", "fallback_random") + ) + + "\n" + ) + (attempt / f"20260903_110000_samples_{task_name}.jsonl").write_text("{not-json}\n") + + evaluator._attach_mmmu_parser_audit( + {"raw_result_path": str(raw_result_path), "result_path": str(result_path)} + ) + + audit = json.loads(result_path.read_text())["mmmu_parser_audit"] + assert audit["sample_count"] == 2 + assert audit["status_counts"] == {"fallback_random": 1, "parsed": 1} + assert audit["sample_logs"] == [ + { + "path": sample_path.name, + "sha256": hashlib.sha256(sample_path.read_bytes()).hexdigest(), + "size": sample_path.stat().st_size, + } + ] + + +def test_mmmu_parser_audit_rejects_unlabeled_sample(tmp_path): + task_name = suites.task_name("mmmu_val") + result_path = tmp_path / "summary.json" + result_path.write_text(json.dumps({"sample_counts": {task_name: 1}})) + raw_result_path = tmp_path / "new_results.json" + raw_result_path.write_text("{}\n") + (tmp_path / f"old_samples_{task_name}.jsonl").write_text( + json.dumps({"mmmu_acc": {"parser_status": ["parsed"]}}) + "\n" + ) + (tmp_path / f"new_samples_{task_name}.jsonl").write_text( + json.dumps({"mmmu_acc": {"parsed_pred": ["A"]}}) + "\n" + ) + + with pytest.raises(RuntimeError, match="no valid parser status"): + evaluator._attach_mmmu_parser_audit( + {"raw_result_path": str(raw_result_path), "result_path": str(result_path)} + ) + + +def test_short_profile_preserves_default_vllm_backend(monkeypatch, tmp_path, capsys): + model = _write_checkpoint(tmp_path) + source_tasks = ("realworldqa", "mmmu_val") + lmms_root = _write_lmms_tasks(tmp_path, source_tasks) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + output = tmp_path / "results" + calls = [] + + def fake_runner(checkpoint_path, *, output_root, settings): + calls.append( + { + "checkpoint": checkpoint_path, + "output_root": output_root, + "settings": settings, + } + ) + result_path = output_root / "result.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + raw_result_path = _write_fake_mmmu_artifacts(result_path) + return { + "attempt": len(calls), + "metrics": {"accuracy": len(calls) / 10}, + "output_root": str(output_root), + "raw_result_path": str(raw_result_path), + "result_path": str(result_path), + } + + monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) + argv = [ + "--checkpoint", + str(model), + "--output-dir", + str(output), + "--suite", + "short", + "--hf-home", + str(hf_home), + ] + + assert evaluation.main(argv) == 0 + + result = json.loads(capsys.readouterr().out) + report = result["preflight"] + generated = json.loads( + (output / "task_configs/modelopt_vlm_benchmark_realworldqa.yaml").read_text() + ) + assert generated["dataset_path"].endswith( + profile.VLM_BENCHMARK_DATASETS["realworldqa"].revision + ) + assert generated["generation_kwargs"]["max_new_tokens"] == 16 + mmmu_text = (output / "task_configs/modelopt_vlm_benchmark_mmmu_val.yaml").read_text() + assert '"max_new_tokens": 128' in mmmu_text + expected_tasks = ( + "modelopt_vlm_benchmark_realworldqa", + "modelopt_vlm_benchmark_mmmu_val", + ) + assert [call["checkpoint"] for call in calls] == [model, model] + assert [call["output_root"] for call in calls] == [ + output / "short-repetition-1", + output / "short-repetition-2", + ] + assert all(call["settings"]["tasks"] == ",".join(expected_tasks) for call in calls) + assert [run["attempt"] for run in result["runs"]] == [1, 2] + settings = calls[0]["settings"] + assert settings["model"] == "vllm" + assert report["backend_limitations"] == [ + "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", + "pinned generic vLLM max_new_tokens is a model-level lower bound", + ] + for task, expected_tokens in {"mmmu_val": 128, "realworldqa": 16}.items(): + budget = report["output_budget_contract"][task] + assert budget["adapter"] == "vllm" + assert budget["requested_max_new_tokens"] == expected_tokens + assert budget["effective_max_new_tokens"] == expected_tokens + assert budget["limitation"] is not None + assert budget["resolution"] == "max(task_max_new_tokens, model_max_new_tokens_floor=1)" + + +def test_repeated_profile_resumes_completed_repetitions(monkeypatch, tmp_path): + suite = suites.TASK_PREFIX100_REPEAT2_SUITE + model = _write_checkpoint(tmp_path) + source_tasks = ("realworldqa", "mmmu_val") + lmms_root = _write_lmms_tasks(tmp_path, source_tasks) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + output = tmp_path / "results" + calls = [] + + def fake_runner(checkpoint_path, *, output_root, settings): + calls.append(output_root) + result_path = output_root / "attempt" / "summary.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + raw_result_path = _write_fake_mmmu_artifacts(result_path) + return { + "metrics": {"accuracy": len(calls) / 10}, + "raw_result_path": str(raw_result_path), + "result_path": str(result_path), + } + + monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(output), + "--suite", + suite, + "--hf-home", + str(hf_home), + ] + ) + + first = evaluation.evaluate(args) + second = evaluation.evaluate(args) + + assert len(calls) == 2 + assert second["runs"] == first["runs"] + for repetition in (1, 2): + completed = json.loads( + (output / f"{suite}-repetition-{repetition}" / "completed_run.json").read_text() + ) + assert completed["schema"] == "modelopt.vlm-evaluation-completed-run/v1" + assert completed["identity"]["repetition"] == repetition + assert completed["identity"]["checkpoint"]["fingerprint"] + assert completed["identity"]["profile"]["suite"] == suite + + +@pytest.mark.parametrize( + ("corruption", "expected_calls"), + [("checkpoint", 4), ("artifact", 3), ("result", 3), ("profile", 4)], +) +def test_short_profile_reruns_stale_completed_repetitions( + monkeypatch, + tmp_path, + corruption, + expected_calls, +): + model = _write_checkpoint(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa", "mmmu_val")) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + output = tmp_path / "results" + calls = [] + + def fake_runner(checkpoint_path, *, output_root, settings): + calls.append(output_root) + result_path = output_root / "attempt" / "summary.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + raw_result_path = _write_fake_mmmu_artifacts(result_path) + return { + "metrics": {"accuracy": 0.5}, + "raw_result_path": str(raw_result_path), + "result_path": str(result_path), + } + + monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(output), + "--suite", + "short", + "--hf-home", + str(hf_home), + ] + ) + evaluation.evaluate(args) + + if corruption == "checkpoint": + (model / "preprocessor_config.json").write_text('{"changed": true}\n') + elif corruption == "artifact": + (output / "short-repetition-1" / "attempt" / "run_results.json").unlink() + elif corruption == "result": + (output / "short-repetition-1" / "attempt" / "summary.json").unlink() + else: + original_backend_policy = preflight._backend_policy + + def changed_backend_policy(profile_contract): + return { + **original_backend_policy(profile_contract), + "enforce_eager": True, + } + + monkeypatch.setattr(preflight, "_backend_policy", changed_backend_policy) + + evaluation.evaluate(args) + assert len(calls) == expected_calls + + +@pytest.mark.parametrize( + "record", + [ + "{\n", + json.dumps( + { + "identity": {}, + "result": {"metrics": []}, + "schema": "modelopt.vlm-evaluation-completed-run/v1", + } + ), + ], +) +def test_completed_repetition_records_fail_closed_when_malformed(tmp_path, record): + output = tmp_path / "results" + output.mkdir() + (output / "completed_run.json").write_text(record) + + with pytest.raises(RuntimeError, match="invalid completed VLM evaluation"): + evaluator._load_completed_run(output, identity={}) + + +def test_realworldqa_mmmu_prefix100_policy_is_explicit_and_repeated(): + suite = suites.TASK_PREFIX100_REPEAT2_SUITE + assert suites.source_tasks(suite) == ("realworldqa", "mmmu_val") + policy = suites.execution_policy(suite, timeout_seconds=14400) + assert policy["limit"] == 100 + assert policy["repetitions"] == 2 + assert policy["generation"] == { + "enable_thinking": False, + "temperature": 0, + "do_sample": False, + } + assert suites.execution_policy("full", timeout_seconds=None)["limit"] is None + + +@pytest.mark.parametrize("alias", suites.DEPRECATED_SUITE_ALIASES) +def test_deprecated_suite_alias_records_the_canonical_identity(monkeypatch, tmp_path, alias): + model = _write_checkpoint(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa", "mmmu_val")) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--suite", + alias, + "--hf-home", + str(hf_home), + ] + ) + + with pytest.warns(FutureWarning, match="is deprecated"): + prepared = preflight.prepare(args) + + assert prepared.suite == suites.TASK_PREFIX100_REPEAT2_SUITE + assert prepared.report["suite"] == suites.TASK_PREFIX100_REPEAT2_SUITE diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py new file mode 100644 index 00000000000..404c2f06a68 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_model.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for VLM checkpoint and prompt-template validation.""" + +import json + +import pytest + +from examples.puzzletron.evaluation.vlm import model as vlm_model +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import _write_checkpoint + + +def _homogeneous_qwen_block_configs() -> list[dict[str, object]]: + block = { + "subblock_configs": [ + { + "kind": "attention", + "name": "attention", + "no_op": False, + "num_kv_heads": 2, + "num_query_heads": 8, + }, + { + "kind": "ffn", + "name": "ffn", + "no_op": False, + "intermediate_size": 3584, + }, + ] + } + return [json.loads(json.dumps(block)) for _ in range(24)] + + +def test_no_think_template_is_local_and_requires_checkpoint_switch(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + tasks_root = tmp_path / "tasks" + tasks_root.mkdir() + + (checkpoint_path / "chat_template.jinja").write_text( + "{% if enable_thinking is defined and enable_thinking is true %}" + "\n{% else %}\n\n\n\n{% endif %}" + ) + generated = vlm_model.no_think_chat_template(checkpoint_path, tasks_root) + assert generated.parent == tasks_root + (checkpoint_path / "chat_template.jinja").write_text("unsupported\n") + with pytest.raises(ValueError, match="cannot disable thinking"): + vlm_model.no_think_chat_template(checkpoint_path, tasks_root) + + +def test_no_think_template_rejects_unsafe_checkpoint_expression(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + (checkpoint_path / "chat_template.jinja").write_text( + "{{ ''.__class__.__mro__ }}" + "{% if enable_thinking is defined and enable_thinking is false %}" + "\n\n\n\n{% endif %}" + ) + tasks_root = tmp_path / "tasks" + tasks_root.mkdir() + + with pytest.raises(ValueError, match="chat template is invalid"): + vlm_model.no_think_chat_template(checkpoint_path, tasks_root) + + +def test_checkpoint_contract_accepts_only_matching_realized_anymodel(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + config_path = checkpoint_path / "config.json" + config = json.loads(config_path.read_text()) + config.update( + architectures=["AnyModel"], + base_architecture="Qwen3_5ForConditionalGeneration", + block_configs=_homogeneous_qwen_block_configs(), + ) + config_path.write_text(json.dumps(config) + "\n") + + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config.pop("block_configs") + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="cannot prove.*homogeneous"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config["block_configs"] = _homogeneous_qwen_block_configs() + config["base_architecture"] = "OtherForConditionalGeneration" + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="AnyModel base_architecture"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config.update( + architectures=["AnyModel", "Qwen3_5ForConditionalGeneration"], + base_architecture="Qwen3_5ForConditionalGeneration", + ) + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="AnyModel base_architecture"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + +def test_checkpoint_contract_routes_heterogeneous_anymodel_to_vllm(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + config_path = checkpoint_path / "config.json" + config = json.loads(config_path.read_text()) + config.update( + architectures=["AnyModel"], + base_architecture="Qwen3_5ForConditionalGeneration", + block_configs=_homogeneous_qwen_block_configs(), + ) + config["block_configs"][19]["subblock_configs"][0]["num_query_heads"] = 6 + config["text_config"]["per_layer_config"] = { + "19": {"num_attention_heads": 6, "num_key_value_heads": 2} + } + config_path.write_text(json.dumps(config) + "\n") + + with pytest.raises(ValueError, match="native qwen3_5 backend cannot load"): + vlm_model.verify_checkpoint( + checkpoint_path, + profile="VLM benchmark", + model_backend="qwen3_5", + ) + + config["text_config"].pop("per_layer_config") + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="native qwen3_5 backend cannot load"): + vlm_model.verify_checkpoint( + checkpoint_path, + profile="VLM benchmark", + model_backend="qwen3_5", + ) + + vlm_model.verify_checkpoint( + checkpoint_path, + profile="VLM benchmark", + model_backend="vllm", + ) + + +def test_checkpoint_contract_accepts_other_positive_qwen35_geometry(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + config_path = checkpoint_path / "config.json" + config = json.loads(config_path.read_text()) + config["text_config"].update( + hidden_size=2560, + intermediate_size=9728, + num_attention_heads=20, + num_hidden_layers=40, + num_key_value_heads=4, + ) + config_path.write_text(json.dumps(config) + "\n") + + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config["text_config"]["hidden_size"] = 0 + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="invalid Qwen 3.5 geometry"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + +@pytest.mark.parametrize("processor_content", [None, "[]\n", b"\xff"]) +def test_checkpoint_contract_requires_valid_local_processor_assets(tmp_path, processor_content): + checkpoint_path = _write_checkpoint(tmp_path) + processor_path = checkpoint_path / "preprocessor_config.json" + if processor_content is None: + processor_path.unlink() + elif isinstance(processor_content, bytes): + processor_path.write_bytes(processor_content) + else: + processor_path.write_text(processor_content) + + with pytest.raises(ValueError, match="processor asset"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + +def test_checkpoint_contract_rejects_malformed_companion_processor_asset(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + (checkpoint_path / "video_preprocessor_config.json").write_text("{\n") + + with pytest.raises(ValueError, match="video_preprocessor_config.json"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py new file mode 100644 index 00000000000..4de3cf79497 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_post_mip.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for VLM post-MIP evaluation adapters.""" + +import json +from pathlib import Path + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import post_mip, suites +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import ( + _use_offline_fakes, + _write_checkpoint, + _write_lmms_tasks, +) + + +def test_post_mip_realworldqa_adapter_runs_pinned_profile(monkeypatch, tmp_path): + model = _write_checkpoint(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + monkeypatch.setenv("HF_HOME", str(hf_home)) + output = tmp_path / "output" + captured = {} + + def fake_runner(checkpoint_path, *, output_root, settings): + report = json.loads((output / "profile.json").read_text()) + assert report["configured_tasks"] == ["modelopt_vlm_benchmark_realworldqa"] + assert report["sample_limit"] == 2 + captured.update( + checkpoint=checkpoint_path, + output_root=output_root, + settings=settings, + ) + result_path = output_root / "result.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text("{}\n") + return { + "metrics": {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5}, + "result_path": str(result_path), + } + + monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) + result = post_mip.evaluate_realworldqa_checkpoint( + model, + output_root=output, + settings={ + "batch_size": 1, + "timeout_seconds": 900, + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + }, + ) + + assert captured["checkpoint"] == model + assert captured["output_root"] == output + assert captured["settings"]["tasks"] == "modelopt_vlm_benchmark_realworldqa" + assert captured["settings"]["limit"] == 2 + assert captured["settings"]["timeout_seconds"] == 900 + assert captured["settings"]["dtype"] == "bfloat16" + assert captured["settings"]["topology"] == {"tensor_parallel_size": 1} + assert result["metrics"] == {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5} + assert result["profile_path"] == str(output / "profile.json") + + +def test_post_mip_prefix100_adapter_averages_repeated_bounded_tasks( + monkeypatch, + tmp_path, +): + model = tmp_path / "model" + model.mkdir() + output = tmp_path / "output" + captured = {"invocations": 0} + + def fake_evaluate(args, *, settings_overrides, preflight_callback): + captured["invocations"] += 1 + captured.update(args=args, settings_overrides=settings_overrides) + preflight_callback({"profile": suites.EVALUATION_PROFILE, "sample_limit": None}) + runs = [] + score_offset = (captured["invocations"] - 1) * 0.2 + for index, realworldqa_score in enumerate( + (0.4 + score_offset, 0.6 + score_offset), start=1 + ): + result_path = tmp_path / f"run-{index}.json" + result_path.write_text( + json.dumps( + { + "sample_counts": {"realworldqa": 100, "mmmu_val": 100}, + "mmmu_parser_audit": { + "sample_count": 100, + "status_counts": {"parsed": 90, "fallback_random": 10}, + }, + } + ) + ) + runs.append( + { + "metrics": { + "modelopt_vlm_benchmark_realworldqa.exact_match_none": (realworldqa_score), + "modelopt_vlm_benchmark_mmmu_val.mmmu_acc_none": 0.3, + }, + "result_path": str(result_path), + } + ) + return {"runs": runs} + + monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) + result = post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( + model, + output_root=output, + settings={ + "batch_size": 1, + "timeout_seconds": 14400, + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + }, + ) + + assert captured["args"].suite == suites.TASK_PREFIX100_REPEAT2_SUITE + assert captured["args"].batch_size == 1 + assert captured["args"].seed == 42 + assert captured["settings_overrides"] == { + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + } + assert result["metrics"] == { + "modelopt_vlm_benchmark_mmmu_val.mmmu_acc_none": 0.3, + "modelopt_vlm_benchmark_realworldqa.exact_match_none": 0.5, + } + assert result["profile"] == post_mip.TASK_PREFIX100_REPEAT2_PROFILE + summary = json.loads(Path(result["result_path"]).read_text()) + assert summary["suite"] == suites.TASK_PREFIX100_REPEAT2_SUITE + assert summary["profile"] == post_mip.TASK_PREFIX100_REPEAT2_PROFILE + assert summary["metrics"] == result["metrics"] + assert summary["result_paths"] == result["run_result_paths"] + assert summary["sample_counts"] == {"mmmu_val": 200, "realworldqa": 200} + assert summary["mmmu_parser_audit"] == { + "sample_count": 200, + "status_counts": {"fallback_random": 20, "parsed": 180}, + } + + refreshed = post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( + model, + output_root=output, + settings={ + "batch_size": 1, + "timeout_seconds": 14400, + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + }, + ) + assert refreshed["metrics"][ + "modelopt_vlm_benchmark_realworldqa.exact_match_none" + ] == pytest.approx(0.7) + assert json.loads(Path(refreshed["result_path"]).read_text())["metrics"] == refreshed["metrics"] + + +def test_post_mip_prefix100_rejects_different_repetition_metrics( + monkeypatch, + tmp_path, +): + def fake_evaluate(args, *, settings_overrides, preflight_callback): + return { + "runs": [ + {"metrics": {"realworldqa.accuracy": 0.5}, "result_path": "first.json"}, + {"metrics": {"mmmu.accuracy": 0.5}, "result_path": "second.json"}, + ] + } + + monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) + + with pytest.raises(RuntimeError, match="produced different metrics"): + post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( + tmp_path / "model", + output_root=tmp_path / "output", + settings={}, + ) + + +def test_deprecated_post_mip_profile_alias_forwards_to_canonical(monkeypatch, tmp_path): + expected = {"metrics": {"accuracy": 0.5}} + monkeypatch.setattr( + post_mip, + "evaluate_realworldqa_mmmu_prefix100_checkpoint", + lambda *_args, **_kwargs: expected, + ) + + with pytest.warns(FutureWarning, match="is deprecated"): + result = post_mip.evaluate_e2e_full_eval_checkpoint( + tmp_path / "model", + output_root=tmp_path / "output", + settings={}, + ) + + assert result is expected diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py new file mode 100644 index 00000000000..e44d4dba73f --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_preflight.py @@ -0,0 +1,509 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for VLM profile preflight and runtime settings.""" + +import json +import os +from pathlib import Path + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import ( + contracts, + evaluator, + preflight, + profile, + suites, + tasks, +) +from examples.puzzletron.evaluation.vlm import run as evaluation +from modelopt.torch.puzzletron.evaluation import lmms +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import ( + _full_inputs, + _use_offline_fakes, + _write_checkpoint, + _write_checkpoint_at, + _write_core3_teacher_snapshot, + _write_lmms_tasks, +) + + +def test_versioned_profile_preflight_reports_immutable_contract(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + name = "core-3_344-examples_r1-native" + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + name, + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + + contract = contracts.load_profile(name) + assert prepared.report["profile_name"] == name + assert prepared.report["profile_fingerprint"] == contract.fingerprint + assert prepared.report["sample_set"] == contract.sample_set + assert prepared.report["backend_profile"] == contract.backend_profile + assert prepared.report["evaluator_profile"] == contract.evaluator_profile + assert prepared.report["source_tasks"] == list(contract.source_tasks) + assert prepared.report["quick_selected_rows"] == 344 + assert prepared.report["quick_row_identities"] == suites.manifest_row_identities( + prepared.quick_manifest + ) + assert prepared.report["quick_task_denominators"] == suites.manifest_task_denominators( + prepared.quick_manifest + ) + + +def test_native_profile_builds_qwen35_backend_settings(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: object()) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "judge-free-8_690-examples_r1-native", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + (tmp_path / "tasks").mkdir() + settings = preflight.settings( + args, + tasks_root=tmp_path / "tasks", + configured_tasks=("modelopt_vlm_benchmark_mvbench",), + prepared=prepared, + ) + + assert prepared.report["model_backend"] == "qwen3_5" + assert settings["model"] == "qwen3_5" + assert settings["checkpoint_arg"] == "pretrained" + assert settings["model_args"] == { + "attn_implementation": "sdpa", + "device": "cuda", + "device_map": "cuda", + "enable_thinking": False, + "fps": 2, + "max_frames": 32, + } + assert "reasoning_parser" not in settings + assert not (tmp_path / "tasks/modelopt_qwen35_no_think.jinja").exists() + + +@pytest.mark.parametrize( + ("profile_name", "expected_eager"), + [("core-3_24-examples_r1-vllm", True), ("core-3_344-examples_r1-vllm", None)], +) +def test_vllm_profile_forwards_runtime_settings( + monkeypatch, tmp_path, profile_name, expected_eager +): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + profile_name, + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + tasks_root = tmp_path / "tasks" + tasks_root.mkdir() + settings = preflight.settings( + args, + tasks_root=tasks_root, + configured_tasks=("modelopt_vlm_benchmark_realworldqa",), + prepared=prepared, + ) + argv, _, _ = lmms._build_command( + settings, + checkpoint=str(model), + output_path=tmp_path / "lmms-results", + ) + model_args = argv[argv.index("--model_args") + 1] + + assert settings["model_args"].get("enforce_eager") is expected_eager + assert ("enforce_eager=True" in model_args) is (expected_eager is True) + assert settings["model_args"]["attention_config"] == {"flash_attn_version": 2} + assert 'attention_config={"flash_attn_version":2}' in model_args + + +def test_core3_full_teacher_profile_population_expectations_follow_group_shard( + monkeypatch, tmp_path +): + model, hf_home = _write_core3_teacher_snapshot(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("mvbench",)) + _use_offline_fakes(monkeypatch, lmms_root) + media = hf_home / profile.VLM_BENCHMARK_DATASETS["mvbench"].media_dir + media.mkdir(parents=True) + (media / "sample").write_bytes(b"media") + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_full_r1-native", + "--profile-task", + "mvbench", + "--profile-task-shard", + "3/8", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + expected_leaves = ("episodic_reasoning", "moving_direction", "egocentric_navigation") + + assert prepared.profile_task_leaves == expected_leaves + assert evaluator._expected_task_populations(prepared, ("modelopt_vlm_benchmark_mvbench",)) == { + suites.task_name("mvbench", leaf=leaf): 200 for leaf in expected_leaves + } + + +def test_core3_full_teacher_profile_accepts_snapshot_symlink(monkeypatch, tmp_path): + snapshot, hf_home = _write_core3_teacher_snapshot(tmp_path) + checkpoint_alias = tmp_path / "teacher" + checkpoint_alias.symlink_to(snapshot, target_is_directory=True) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) + _use_offline_fakes(monkeypatch, lmms_root) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(checkpoint_alias), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_full_r1-native", + "--profile-task", + "realworldqa", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + + assert prepared.source_tasks == ("realworldqa",) + assert prepared.report["model_pin"]["revision"] == snapshot.name + + +@pytest.mark.parametrize("kind", ["untracked-copy", "wrong-revision"]) +def test_core3_full_teacher_profile_rejects_unpinned_checkpoint(monkeypatch, tmp_path, kind): + monkeypatch.delenv("HF_HUB_CACHE", raising=False) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + if kind == "untracked-copy": + model = _write_checkpoint(tmp_path) + else: + parent = hf_home / "hub/models--Qwen--Qwen3.5-0.8B/snapshots" + parent.mkdir(parents=True) + model = _write_checkpoint_at(parent / "different") + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_full_r1-native", + "--profile-task", + "realworldqa", + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match="requires the exact local Hub snapshot"): + preflight.prepare(args) + + +def test_core3_full_teacher_profile_rejects_settings_override(monkeypatch, tmp_path): + model, hf_home = _write_core3_teacher_snapshot(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) + _use_offline_fakes(monkeypatch, lmms_root) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_full_r1-native", + "--profile-task", + "realworldqa", + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match="do not allow settings overrides"): + evaluator.evaluate(args, settings_overrides={"model": "vllm"}) + + +def test_deprecated_short_profile_preserves_vllm_backend(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "short-v1", + "--hf-home", + str(hf_home), + ] + ) + + with pytest.warns(FutureWarning, match="short-v1 is a deprecated compatibility profile"): + prepared = preflight.prepare(args) + (tmp_path / "tasks").mkdir() + settings = preflight.settings( + args, + tasks_root=tmp_path / "tasks", + configured_tasks=("modelopt_vlm_benchmark_realworldqa",), + prepared=prepared, + ) + + assert prepared.report["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION + assert settings["model"] == "vllm" + assert settings["checkpoint_arg"] == "model" + assert settings["reasoning_parser"] == "qwen3" + assert settings["model_args"]["max_frame_num"] == 32 + assert Path(settings["model_args"]["chat_template"]).exists() + + +@pytest.mark.parametrize( + ("option", "value"), + [("--seed", "7"), ("--batch-size", "8")], +) +def test_versioned_profile_rejects_runtime_override(monkeypatch, tmp_path, option, value): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_344-examples_r1-native", + option, + value, + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match=f"{option} cannot override"): + preflight.prepare(args) + + +def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "judge-free-8_690-examples_r1-native", + "--profile-task", + "mvbench", + "--profile-task-shard", + "3/8", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + + expected_leaves = ("episodic_reasoning", "moving_direction", "egocentric_navigation") + assert prepared.profile_task_leaves == expected_leaves + assert prepared.quick_manifest is not None + manifest_rows = prepared.quick_manifest["tasks"]["mvbench"]["rows"] + assert len(manifest_rows) == 24 + assert {row["leaf_task"] for row in manifest_rows} == { + f"mvbench_{leaf}" for leaf in expected_leaves + } + assert prepared.report["quick_task_denominators"] == { + "mvbench": {"population_rows": 600, "selected_rows": 24} + } + assert ( + evaluator._expected_task_populations(prepared, ("modelopt_vlm_benchmark_mvbench",)) is None + ) + tasks_root, _ = tasks.prepare( + tmp_path / "results", + suite=prepared.suite, + source_tasks=prepared.source_tasks, + profile_task_leaves=prepared.profile_task_leaves, + dataset_snapshots=prepared.dataset_snapshots, + quick_manifest=prepared.quick_manifest, + ) + group = json.loads((tasks_root / "modelopt_vlm_benchmark_mvbench.yaml").read_text()) + assert group["task"] == [f"modelopt_vlm_benchmark_mvbench_{leaf}" for leaf in expected_leaves] + + +def test_empty_exact_row_leaf_filter_reaches_manifest_validation(): + contract = contracts.load_profile("core-3_344-examples_r1-native") + manifest = contract.exact_rows + assert manifest is not None + entry = manifest["tasks"]["mvbench"] + manifest["tasks"] = { + "mvbench": preflight._shard_exact_row_task( + entry, + task="mvbench", + leaves=("not-selected",), + ) + } + + with pytest.raises(ValueError, match="must select at least one row"): + suites.validate_exact_rows_manifest( + manifest, + expected_revision=str(contract.manifest["lmms_eval_revision"]), + expected_tasks=("mvbench",), + ) + + +@pytest.mark.parametrize( + ("selection", "message"), + [ + (("--suite", "short", "--profile-task", "realworldqa"), "requires"), + ( + ( + "--profile", + "core-3_344-examples_r1-native", + "--profile-task", + "realworldqa", + ), + "supported only", + ), + (("--profile", "core-3_full_r1-native", "--profile-task-shard", "0/8"), "requires"), + ( + ( + "--profile", + "core-3_full_r1-native", + "--profile-task", + "realworldqa", + "--profile-task-shard", + "0/8", + ), + "supports only", + ), + ], +) +def test_profile_task_rejects_invalid_parent(monkeypatch, tmp_path, selection, message): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + *selection, + "--hf-home", + str(hf_home), + ] + ) + + with pytest.raises(ValueError, match=message): + preflight.prepare(args) + + +@pytest.mark.parametrize( + ("extra", "environment", "message"), + [ + ([], {"OPENAI_API_KEY": "token"}, "explicit --allow-judge-calls"), + (["--allow-judge-calls"], {}, "judge credentials are missing"), + ], +) +def test_full_profile_fails_closed_without_judge_authorization_or_credentials( + monkeypatch, tmp_path, extra, environment, message +): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + for name in ("OPENAI_API_KEY", "AZURE_API_KEY", "AZURE_ENDPOINT"): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--suite", + "full", + "--hf-home", + str(hf_home), + "--mmvu-judge-api-type", + "openai", + "--mmvu-judge-model", + "judge", + *extra, + ] + ) + with pytest.raises(ValueError, match=message): + preflight.prepare(args) + + +def test_video_reader_validation_is_limited_to_video_suites(monkeypatch): + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: None) + + preflight._verify_video_reader(("realworldqa", "mmmu_val")) + with pytest.raises(RuntimeError, match="decord-compatible reader"): + preflight._verify_video_reader(("mvbench",)) + + +def test_native_backend_validation_requires_qwen_vision_utilities(monkeypatch): + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: None) + + preflight._verify_backend_dependencies("vllm") + with pytest.raises(RuntimeError, match="qwen-vl-utils"): + preflight._verify_backend_dependencies("qwen3_5") + + +def test_credential_scope_restores_inherited_values(monkeypatch): + expected = { + name: f"secret-{index}" + for index, name in enumerate(checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) + } + for name, value in expected.items(): + monkeypatch.setenv(name, value) + with checkpoint.without_huggingface_credentials(): + assert all(name not in os.environ for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) + assert {name: os.environ.get(name) for name in expected} == expected diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py index ed6171c6114..ba43a7c1e2f 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py @@ -13,49 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""High-value behavior tests for local VLM evaluation workflows.""" +"""Tests for the VLM evaluation command-line entry point.""" -import hashlib -import importlib.util -import json -import os import subprocess import sys -from collections import Counter -from hashlib import sha256 from pathlib import Path -from types import ModuleType -import pytest - -from examples.puzzletron.evaluation import checkpoint -from examples.puzzletron.evaluation.vlm import ( - contracts, - evaluator, - post_mip, - preflight, - profile, - suites, - tasks, -) -from examples.puzzletron.evaluation.vlm import model as vlm_model from examples.puzzletron.evaluation.vlm import run as evaluation -_QWEN_CONFIG = { - "architectures": ["Qwen3_5ForConditionalGeneration"], - "model_type": "qwen3_5", - "text_config": { - "hidden_size": 1024, - "intermediate_size": 3584, - "model_type": "qwen3_5_text", - "num_attention_heads": 8, - "num_hidden_layers": 24, - "num_key_value_heads": 2, - "vocab_size": 248320, - }, -} -_TASK_CONFIGS = {name: item.task_config for name, item in profile.VLM_BENCHMARK_DATASETS.items()} - def test_direct_launcher_does_not_shadow_standard_library_profile(): script = Path(evaluation.__file__).absolute() @@ -77,1213 +42,6 @@ def test_direct_launcher_does_not_shadow_standard_library_profile(): ) -def _write_checkpoint(root: Path) -> Path: - model = root / "model" - model.mkdir() - (model / "config.json").write_text(json.dumps(_QWEN_CONFIG) + "\n") - (model / "preprocessor_config.json").write_text("{}\n") - (model / "chat_template.jinja").write_text( - "{% if enable_thinking is defined and enable_thinking is false %}" - "\n\n\n\n{% else %}\n{% endif %}" - ) - return model - - -def test_no_think_template_is_local_and_requires_checkpoint_switch(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - tasks_root = tmp_path / "tasks" - tasks_root.mkdir() - - (checkpoint_path / "chat_template.jinja").write_text( - "{% if enable_thinking is defined and enable_thinking is true %}" - "\n{% else %}\n\n\n\n{% endif %}" - ) - generated = vlm_model.no_think_chat_template(checkpoint_path, tasks_root) - assert generated.parent == tasks_root - (checkpoint_path / "chat_template.jinja").write_text("unsupported\n") - with pytest.raises(ValueError, match="cannot disable thinking"): - vlm_model.no_think_chat_template(checkpoint_path, tasks_root) - - -def test_no_think_template_rejects_unsafe_checkpoint_expression(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - (checkpoint_path / "chat_template.jinja").write_text( - "{{ ''.__class__.__mro__ }}" - "{% if enable_thinking is defined and enable_thinking is false %}" - "\n\n\n\n{% endif %}" - ) - tasks_root = tmp_path / "tasks" - tasks_root.mkdir() - - with pytest.raises(ValueError, match="chat template is invalid"): - vlm_model.no_think_chat_template(checkpoint_path, tasks_root) - - -def test_chat_template_fingerprint_accepts_file_and_inline_content(tmp_path): - content = "{% if messages %}{{ messages[0]['content'] }}{% endif %}" - template_path = tmp_path / "chat_template.jinja" - template_path.write_text(content) - expected = sha256(content.encode()).hexdigest() - - assert ( - evaluator._chat_template_sha256({"model_args": {"chat_template": str(template_path)}}) - == expected - ) - assert evaluator._chat_template_sha256({"model_args": {"chat_template": content}}) == expected - - -def test_checkpoint_contract_accepts_only_matching_realized_anymodel(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - config_path = checkpoint_path / "config.json" - config = json.loads(config_path.read_text()) - config.update( - architectures=["AnyModel"], - base_architecture="Qwen3_5ForConditionalGeneration", - ) - config_path.write_text(json.dumps(config) + "\n") - - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - config["base_architecture"] = "OtherForConditionalGeneration" - config_path.write_text(json.dumps(config) + "\n") - with pytest.raises(ValueError, match="AnyModel base_architecture"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - config.update( - architectures=["AnyModel", "Qwen3_5ForConditionalGeneration"], - base_architecture="Qwen3_5ForConditionalGeneration", - ) - config_path.write_text(json.dumps(config) + "\n") - with pytest.raises(ValueError, match="AnyModel base_architecture"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - -def test_checkpoint_contract_accepts_other_positive_qwen35_geometry(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - config_path = checkpoint_path / "config.json" - config = json.loads(config_path.read_text()) - config["text_config"].update( - hidden_size=2560, - intermediate_size=9728, - num_attention_heads=20, - num_hidden_layers=40, - num_key_value_heads=4, - ) - config_path.write_text(json.dumps(config) + "\n") - - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - config["text_config"]["hidden_size"] = 0 - config_path.write_text(json.dumps(config) + "\n") - with pytest.raises(ValueError, match="invalid Qwen 3.5 geometry"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - -@pytest.mark.parametrize("processor_content", [None, "", "[]\n", "{\n", b"\xff"]) -def test_checkpoint_contract_requires_valid_local_processor_assets(tmp_path, processor_content): - checkpoint_path = _write_checkpoint(tmp_path) - processor_path = checkpoint_path / "preprocessor_config.json" - if processor_content is None: - processor_path.unlink() - elif isinstance(processor_content, bytes): - processor_path.write_bytes(processor_content) - else: - processor_path.write_text(processor_content) - - with pytest.raises(ValueError, match="processor asset"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - -def test_checkpoint_contract_rejects_malformed_companion_processor_asset(tmp_path): - checkpoint_path = _write_checkpoint(tmp_path) - (checkpoint_path / "video_preprocessor_config.json").write_text("{\n") - - with pytest.raises(ValueError, match="video_preprocessor_config.json"): - vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") - - -def _write_lmms_tasks(root: Path, tasks: tuple[str, ...]) -> Path: - lmms_root = root / "lmms_eval" - for task in tasks: - task_configs = [_TASK_CONFIGS[task]] - if task == "video_mmmu": - task_configs.extend( - f"tasks/videommmu/{leaf}.yaml" for leaf in suites.VIDEO_MMMU_LEAF_TASKS - ) - elif task == "mvbench": - task_configs.extend( - f"tasks/mvbench/mvbench_{leaf}.yaml" for leaf in suites.MVBENCH_LEAF_TASKS - ) - for relative_path in task_configs: - config = lmms_root / relative_path - config.parent.mkdir(parents=True, exist_ok=True) - config.write_text(f"task: {task}\n") - return lmms_root - - -def _use_offline_fakes(monkeypatch, lmms_root: Path) -> None: - monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) - monkeypatch.setattr( - checkpoint, - "verify_lmms_eval_revision", - lambda expected=checkpoint.LMMS_EVAL_REVISION: expected, - ) - monkeypatch.setattr( - suites, - "offline_dataset_snapshot", - lambda hf_home, task, revision: hf_home / ".snapshots" / task / revision, - ) - monkeypatch.setattr( - tasks, - "verify_offline", - lambda _root, tasks, **_kwargs: { - "configured_tasks": list(tasks), - "status": "passed", - }, - ) - - -def _full_inputs(monkeypatch, tmp_path): - model = _write_checkpoint(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, profile.VLM_BENCHMARK_TASKS) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - for dataset in profile.VLM_BENCHMARK_VIDEO_DATASETS.values(): - media = hf_home / dataset.media_dir - media.mkdir(parents=True, exist_ok=True) - (media / "sample").write_bytes(b"media") - return model, hf_home - - -def _quick_manifest(path: Path) -> Path: - counts = {"realworldqa": 64, "mmmu_val": 120} - tasks = { - task: { - "dataset_revision": profile.VLM_BENCHMARK_DATASETS[task].revision, - "rows": [ - { - "source_row_index": index, - "source_sample_id": ( - f"test:{index}" if task == "realworldqa" else f"question-{index}" - ), - } - for index in range(count) - ], - } - for task, count in counts.items() - } - tasks["mvbench"] = { - "dataset_revision": profile.VLM_BENCHMARK_DATASETS["mvbench"].revision, - "rows": [ - { - "leaf_task": f"mvbench_{leaf}", - "source_row_index": index, - "source_sample_id": f"{leaf}:{index}", - } - for leaf in suites.MVBENCH_LEAF_TASKS - for index in range(8) - ], - } - path.write_text( - json.dumps( - { - "schema": "modelopt.vlm-benchmark-quick/v1", - "lmms_eval_revision": checkpoint.LMMS_EVAL_REVISION, - "tasks": tasks, - } - ) - + "\n" - ) - return path - - -def test_short_profile_materializes_pinned_tasks_and_vllm_backend(monkeypatch, tmp_path, capsys): - model = _write_checkpoint(tmp_path) - source_tasks = ("realworldqa", "mmmu_val") - lmms_root = _write_lmms_tasks(tmp_path, source_tasks) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - output = tmp_path / "results" - for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES: - monkeypatch.setenv(name, f"inherited-{name.lower()}") - calls = [] - - def fake_runner(checkpoint_path, *, output_root, settings): - calls.append( - { - "checkpoint": checkpoint_path, - "credentials": { - name: os.environ.get(name) for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES - }, - "output_root": output_root, - "settings": settings, - } - ) - result_path = output_root / "result.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text("{}\n") - return { - "attempt": len(calls), - "metrics": {"accuracy": len(calls) / 10}, - "output_root": str(output_root), - "result_path": str(result_path), - } - - monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) - argv = [ - "--checkpoint", - str(model), - "--output-dir", - str(output), - "--suite", - "short", - "--hf-home", - str(hf_home), - ] - - assert evaluation.main(argv) == 0 - - result = json.loads(capsys.readouterr().out) - assert result["schema"] == "modelopt.vlm-evaluation-result/v1" - report = result["preflight"] - assert report["source_tasks"] == list(source_tasks) - assert report["short_repetitions"] == 2 - assert report["lmms_eval_revision"] == checkpoint.LMMS_EVAL_REVISION - generated = json.loads( - (output / "task_configs/modelopt_vlm_benchmark_realworldqa.yaml").read_text() - ) - assert generated["dataset_path"].endswith( - profile.VLM_BENCHMARK_DATASETS["realworldqa"].revision - ) - expected_tasks = ( - "modelopt_vlm_benchmark_realworldqa", - "modelopt_vlm_benchmark_mmmu_val", - ) - assert [call["checkpoint"] for call in calls] == [model, model] - assert [call["output_root"] for call in calls] == [ - output / "short-repetition-1", - output / "short-repetition-2", - ] - assert all(not any(call["credentials"].values()) for call in calls) - assert all(call["settings"]["tasks"] == ",".join(expected_tasks) for call in calls) - assert [run["attempt"] for run in result["runs"]] == [1, 2] - for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES: - assert os.environ[name] == f"inherited-{name.lower()}" - settings = calls[0]["settings"] - assert settings["model"] == "vllm" - assert settings["log_samples"] is True - assert settings["checkpoint_arg"] == "model" - assert settings["reasoning_parser"] == "qwen3" - chat_template = Path(settings["model_args"]["chat_template"]) - assert chat_template.read_text().startswith("{%- set enable_thinking = false %}\n") - assert "topology" not in settings - assert settings["env"]["HF_HUB_OFFLINE"] == "1" - assert settings["env"]["API_TYPE"] == "openai" - assert settings["env"]["MODEL_VERSION"] == "modelopt-disabled-lmms-eval-judge" - assert settings["env"]["OPENAI_API_KEY"] == "modelopt-disabled-lmms-eval-judge" - assert settings["env"]["OPENAI_API_URL"] == "http://127.0.0.1:9" - assert report["model_backend"] == settings["model"] - assert report["backend_limitations"] == [ - "generic vLLM video messages do not preserve native Qwen 3.5 timestamps", - ] - assert report["sample_limit"] == settings["limit"] - assert report["timeout_seconds"] == settings["timeout_seconds"] - assert report["frame_policy"] == { - "reader": settings["env"]["FORCE_QWENVL_VIDEO_READER"], - "fps": settings["model_args"]["fps"], - "max_frames": settings["model_args"]["max_frame_num"], - } - assert report["generation_policy"] == settings["gen_kwargs"] - - -@pytest.mark.parametrize("suite", ["short", suites.TASK_PREFIX100_REPEAT2_SUITE]) -def test_repeated_profile_resumes_completed_repetitions(monkeypatch, tmp_path, suite): - model = _write_checkpoint(tmp_path) - source_tasks = ("realworldqa", "mmmu_val") - lmms_root = _write_lmms_tasks(tmp_path, source_tasks) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - output = tmp_path / "results" - calls = [] - - def fake_runner(checkpoint_path, *, output_root, settings): - calls.append(output_root) - result_path = output_root / "attempt" / "summary.json" - raw_result_path = output_root / "attempt" / "samples.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text("{}\n") - raw_result_path.write_text("{}\n") - return { - "metrics": {"accuracy": len(calls) / 10}, - "raw_result_path": str(raw_result_path), - "result_path": str(result_path), - } - - monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(output), - "--suite", - suite, - "--hf-home", - str(hf_home), - ] - ) - - first = evaluation.evaluate(args) - second = evaluation.evaluate(args) - - assert len(calls) == 2 - assert second["runs"] == first["runs"] - for repetition in (1, 2): - completed = json.loads( - (output / f"{suite}-repetition-{repetition}" / "completed_run.json").read_text() - ) - assert completed["schema"] == "modelopt.vlm-evaluation-completed-run/v1" - assert completed["identity"]["repetition"] == repetition - assert completed["identity"]["checkpoint"]["fingerprint"] - assert completed["identity"]["profile"]["suite"] == suite - - -@pytest.mark.parametrize( - ("corruption", "expected_calls"), - [("checkpoint", 4), ("artifact", 3), ("result", 3), ("profile", 4)], -) -def test_short_profile_reruns_stale_completed_repetitions( - monkeypatch, - tmp_path, - corruption, - expected_calls, -): - model = _write_checkpoint(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa", "mmmu_val")) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - output = tmp_path / "results" - calls = [] - - def fake_runner(checkpoint_path, *, output_root, settings): - calls.append(output_root) - result_path = output_root / "attempt" / "summary.json" - raw_result_path = output_root / "attempt" / "samples.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text("{}\n") - raw_result_path.write_text("{}\n") - return { - "metrics": {"accuracy": 0.5}, - "raw_result_path": str(raw_result_path), - "result_path": str(result_path), - } - - monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(output), - "--suite", - "short", - "--hf-home", - str(hf_home), - ] - ) - evaluation.evaluate(args) - - if corruption == "checkpoint": - (model / "preprocessor_config.json").write_text('{"changed": true}\n') - elif corruption == "artifact": - (output / "short-repetition-1" / "attempt" / "samples.json").unlink() - elif corruption == "result": - (output / "short-repetition-1" / "attempt" / "summary.json").unlink() - else: - - def changed_chat_template(_checkpoint_path, output_directory): - target = output_directory / "modelopt_qwen35_no_think.jinja" - target.write_text("changed template\n") - return target - - monkeypatch.setattr(vlm_model, "no_think_chat_template", changed_chat_template) - - evaluation.evaluate(args) - assert len(calls) == expected_calls - - -@pytest.mark.parametrize( - "record", - [ - "{\n", - json.dumps( - { - "identity": {}, - "result": {"metrics": []}, - "schema": "modelopt.vlm-evaluation-completed-run/v1", - } - ), - ], -) -def test_completed_repetition_records_fail_closed_when_malformed(tmp_path, record): - output = tmp_path / "results" - output.mkdir() - (output / "completed_run.json").write_text(record) - - with pytest.raises(RuntimeError, match="invalid completed VLM evaluation"): - evaluator._load_completed_run(output, identity={}) - - -def test_realworldqa_mmmu_prefix100_policy_is_explicit_and_repeated(): - suite = suites.TASK_PREFIX100_REPEAT2_SUITE - assert suites.source_tasks(suite) == ("realworldqa", "mmmu_val") - policy = suites.execution_policy(suite, timeout_seconds=14400) - assert policy["limit"] == 100 - assert policy["repetitions"] == 2 - assert policy["generation"] == {"temperature": 0, "do_sample": False} - assert suites.execution_policy("full", timeout_seconds=None)["limit"] is None - assert suites.execution_policy("full-v1", timeout_seconds=None)["limit"] is None - - -@pytest.mark.parametrize("alias", suites.DEPRECATED_SUITE_ALIASES) -def test_deprecated_suite_alias_records_the_canonical_identity(monkeypatch, tmp_path, alias): - model = _write_checkpoint(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa", "mmmu_val")) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--suite", - alias, - "--hf-home", - str(hf_home), - ] - ) - - with pytest.warns(FutureWarning, match="is deprecated"): - prepared = preflight.prepare(args) - - assert prepared.suite == suites.TASK_PREFIX100_REPEAT2_SUITE - assert prepared.report["suite"] == suites.TASK_PREFIX100_REPEAT2_SUITE - - -def test_versioned_profile_contracts_pin_selection_and_fingerprints(tmp_path): - profiles = {name: contracts.load_profile(name) for name in contracts.PROFILE_NAMES} - assert {name: contract.fingerprint for name, contract in profiles.items()} == { - "short-v1": "3b0803c0deff0873d2c8e0963f1167dfbbf2ef7309b5eb33f089aa2048f6cf91", - "short-native-v1": "aca78320b188c4c6f41a7e5ec0017a0c738b98ec797e9f2f6f9f1aabce34dfa7", - "short-all-native-v1": "88074cec92cd6aa972cd2e48ddc6adbaa84ab26b24e36be2e3842b71317cdd7e", - "full-v1": "5b0849975f65e4bbbdc93d52ff2866d54968f5cc3de6d0928077e3d6ab320e6a", - } - - short = profiles["short-v1"] - short_all_native = profiles["short-all-native-v1"] - assert { - task: len(entry["rows"]) for task, entry in short_all_native.manifest["tasks"].items() - } == { - "realworldqa": 64, - "mmmu_val": 120, - "mvbench": 160, - "video_mmmu": 72, - "videomme": 72, - "longvideobench_val_v": 68, - "mlvu_dev": 70, - "perceptiontest_val_mc": 64, - } - assert short.exact_rows is not None - assert profiles["full-v1"].exact_rows is None - mmmu_rows = short.manifest["tasks"]["mmmu_val"]["rows"] - assert Counter(row["source_row_index"] // 30 for row in mmmu_rows) == Counter( - dict.fromkeys(range(30), 4) - ) - - exact_rows = short.exact_rows - assert exact_rows is not None - path = tmp_path / "short-v1-rows.json" - path.write_text(json.dumps(exact_rows)) - validated = suites.load_quick_manifest(path) - assert suites.manifest_sha256(validated) - - -def test_short_all_native_profile_builds_grouped_and_single_selectors(tmp_path): - contract = contracts.load_profile("short-all-native-v1") - exact_rows = contract.exact_rows - assert exact_rows is not None - validated = suites.validate_exact_rows_manifest( - exact_rows, - expected_revision=checkpoint.LMMS_EVAL_REVISION, - expected_tasks=contract.source_tasks, - ) - - tasks._write_quick_selection_module(tmp_path, validated) - spec = importlib.util.spec_from_file_location( - "short_all_selectors", tmp_path / "modelopt_quick_selection.py" - ) - assert spec is not None and spec.loader is not None - selectors = importlib.util.module_from_spec(spec) - spec.loader.exec_module(selectors) - - class Documents: - def __init__(self, size): - self.size = size - - def __len__(self): - return self.size - - def __getitem__(self, _index): - return {} - - def select(self, indices): - return list(indices) - - tasks_manifest = validated["tasks"] - adaptation = [ - row["source_row_index"] - for row in tasks_manifest["video_mmmu"]["rows"] - if row["leaf_task"] == "video_mmmu_adaptation" - ] - assert ( - selectors.select_modelopt_vlm_benchmark_video_mmmu_adaptation(Documents(300)) == adaptation - ) - videomme = [row["source_row_index"] for row in tasks_manifest["videomme"]["rows"]] - assert selectors.select_modelopt_vlm_benchmark_videomme(Documents(2700)) == videomme - - -def test_versioned_profile_preflight_reports_immutable_contract(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - name = "short-v1" - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - name, - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - - contract = contracts.load_profile(name) - assert prepared.report["profile_name"] == name - assert prepared.report["profile_fingerprint"] == contract.fingerprint - assert prepared.report["source_tasks"] == list(contract.source_tasks) - assert prepared.report["quick_selected_rows"] == 344 - - -def test_native_profile_builds_qwen35_backend_settings(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: object()) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "short-native-v1", - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - settings = preflight.settings( - args, - tasks_root=tmp_path / "tasks", - configured_tasks=("modelopt_vlm_benchmark_mvbench",), - prepared=prepared, - ) - - assert prepared.report["model_backend"] == "qwen3_5" - assert settings["model"] == "qwen3_5" - assert settings["checkpoint_arg"] == "pretrained" - assert settings["model_args"] == { - "attn_implementation": "sdpa", - "device": "cuda", - "device_map": "cuda", - "enable_thinking": False, - "fps": 2, - "max_frames": 32, - } - assert "reasoning_parser" not in settings - assert not (tmp_path / "tasks/modelopt_qwen35_no_think.jinja").exists() - - -def test_versioned_profile_rejects_seed_override(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "short-v1", - "--seed", - "7", - "--hf-home", - str(hf_home), - ] - ) - - with pytest.raises(ValueError, match="--seed cannot override"): - preflight.prepare(args) - - -def test_versioned_profile_rejects_batch_size_override(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "short-v1", - "--batch-size", - "8", - "--hf-home", - str(hf_home), - ] - ) - - with pytest.raises(ValueError, match="--batch-size cannot override"): - preflight.prepare(args) - - -def test_all_row_profile_task_preserves_contract_identity(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - profile_name = "full-v1" - task = "realworldqa" - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - profile_name, - "--profile-task", - task, - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - - assert prepared.source_tasks == (task,) - assert prepared.report["quick_selected_rows"] is None - assert ( - prepared.report["profile_fingerprint"] == contracts.load_profile(profile_name).fingerprint - ) - - -def test_exact_row_profile_group_shard_partitions_rows_and_leaves(monkeypatch, tmp_path): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--profile", - "short-all-native-v1", - "--profile-task", - "mvbench", - "--profile-task-shard", - "3/8", - "--hf-home", - str(hf_home), - ] - ) - - prepared = preflight.prepare(args) - - expected_leaves = ("episodic_reasoning", "moving_direction", "egocentric_navigation") - assert prepared.profile_task_leaves == expected_leaves - assert prepared.quick_manifest is not None - manifest_rows = prepared.quick_manifest["tasks"]["mvbench"]["rows"] - assert len(manifest_rows) == 24 - assert {row["leaf_task"] for row in manifest_rows} == { - f"mvbench_{leaf}" for leaf in expected_leaves - } - assert prepared.report["quick_selected_rows"] == 24 - assert prepared.report["quick_manifest_sha256"] == suites.manifest_sha256( - prepared.quick_manifest - ) - assert ( - prepared.report["profile_fingerprint"] - == contracts.load_profile("short-all-native-v1").fingerprint - ) - tasks_root, _ = tasks.prepare( - tmp_path / "results", - suite=prepared.suite, - source_tasks=prepared.source_tasks, - profile_task_leaves=prepared.profile_task_leaves, - dataset_snapshots=prepared.dataset_snapshots, - quick_manifest=prepared.quick_manifest, - ) - group = json.loads((tasks_root / "modelopt_vlm_benchmark_mvbench.yaml").read_text()) - assert group["task"] == [f"modelopt_vlm_benchmark_mvbench_{leaf}" for leaf in expected_leaves] - - -@pytest.mark.parametrize( - ("selection", "message"), - [ - (("--suite", "short", "--profile-task", "realworldqa"), "requires"), - (("--profile", "short-v1", "--profile-task", "realworldqa"), "supported only"), - (("--profile", "full-v1", "--profile-task-shard", "0/8"), "requires"), - ( - ( - "--profile", - "full-v1", - "--profile-task", - "realworldqa", - "--profile-task-shard", - "0/8", - ), - "supports only", - ), - ], -) -def test_profile_task_rejects_invalid_parent(monkeypatch, tmp_path, selection, message): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - *selection, - "--hf-home", - str(hf_home), - ] - ) - - with pytest.raises(ValueError, match=message): - preflight.prepare(args) - - -def test_post_mip_realworldqa_adapter_runs_pinned_profile(monkeypatch, tmp_path): - model = _write_checkpoint(tmp_path) - lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) - _use_offline_fakes(monkeypatch, lmms_root) - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - monkeypatch.setenv("HF_HOME", str(hf_home)) - output = tmp_path / "output" - captured = {} - - def fake_runner(checkpoint_path, *, output_root, settings): - report = json.loads((output / "profile.json").read_text()) - assert report["configured_tasks"] == ["modelopt_vlm_benchmark_realworldqa"] - assert report["sample_limit"] == 2 - captured.update( - checkpoint=checkpoint_path, - output_root=output_root, - settings=settings, - ) - result_path = output_root / "result.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text("{}\n") - return { - "metrics": {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5}, - "result_path": str(result_path), - } - - monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) - result = post_mip.evaluate_realworldqa_checkpoint( - model, - output_root=output, - settings={ - "batch_size": 1, - "timeout_seconds": 900, - "dtype": "bfloat16", - "topology": {"tensor_parallel_size": 1}, - }, - ) - - assert captured["checkpoint"] == model - assert captured["output_root"] == output - assert captured["settings"]["tasks"] == "modelopt_vlm_benchmark_realworldqa" - assert captured["settings"]["limit"] == 2 - assert captured["settings"]["timeout_seconds"] == 900 - assert captured["settings"]["dtype"] == "bfloat16" - assert captured["settings"]["topology"] == {"tensor_parallel_size": 1} - assert result["metrics"] == {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5} - assert result["profile_path"] == str(output / "profile.json") - - -def test_post_mip_prefix100_adapter_averages_repeated_bounded_tasks( - monkeypatch, - tmp_path, -): - model = tmp_path / "model" - model.mkdir() - output = tmp_path / "output" - captured = {"invocations": 0} - - def fake_evaluate(args, *, settings_overrides, preflight_callback): - captured["invocations"] += 1 - captured.update(args=args, settings_overrides=settings_overrides) - preflight_callback({"profile": suites.EVALUATION_PROFILE, "sample_limit": None}) - runs = [] - score_offset = (captured["invocations"] - 1) * 0.2 - for index, realworldqa_score in enumerate( - (0.4 + score_offset, 0.6 + score_offset), start=1 - ): - result_path = tmp_path / f"run-{index}.json" - result_path.write_text("{}") - runs.append( - { - "metrics": { - "modelopt_vlm_benchmark_realworldqa.exact_match_none": (realworldqa_score), - "modelopt_vlm_benchmark_mmmu_val.mmmu_acc_none": 0.3, - }, - "result_path": str(result_path), - } - ) - return {"runs": runs} - - monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) - result = post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( - model, - output_root=output, - settings={ - "batch_size": 1, - "timeout_seconds": 14400, - "dtype": "bfloat16", - "topology": {"tensor_parallel_size": 1}, - }, - ) - - assert captured["args"].suite == suites.TASK_PREFIX100_REPEAT2_SUITE - assert captured["args"].batch_size == 1 - assert captured["args"].seed == 42 - assert captured["settings_overrides"] == { - "dtype": "bfloat16", - "topology": {"tensor_parallel_size": 1}, - } - assert result["metrics"] == { - "modelopt_vlm_benchmark_mmmu_val.mmmu_acc_none": 0.3, - "modelopt_vlm_benchmark_realworldqa.exact_match_none": 0.5, - } - assert result["profile"] == post_mip.TASK_PREFIX100_REPEAT2_PROFILE - summary = json.loads(Path(result["result_path"]).read_text()) - assert summary["suite"] == suites.TASK_PREFIX100_REPEAT2_SUITE - assert summary["profile"] == post_mip.TASK_PREFIX100_REPEAT2_PROFILE - assert summary["metrics"] == result["metrics"] - assert summary["result_paths"] == result["run_result_paths"] - - refreshed = post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( - model, - output_root=output, - settings={ - "batch_size": 1, - "timeout_seconds": 14400, - "dtype": "bfloat16", - "topology": {"tensor_parallel_size": 1}, - }, - ) - assert refreshed["metrics"][ - "modelopt_vlm_benchmark_realworldqa.exact_match_none" - ] == pytest.approx(0.7) - assert json.loads(Path(refreshed["result_path"]).read_text())["metrics"] == refreshed["metrics"] - - -def test_post_mip_prefix100_rejects_different_repetition_metrics( - monkeypatch, - tmp_path, -): - def fake_evaluate(args, *, settings_overrides, preflight_callback): - return { - "runs": [ - {"metrics": {"realworldqa.accuracy": 0.5}, "result_path": "first.json"}, - {"metrics": {"mmmu.accuracy": 0.5}, "result_path": "second.json"}, - ] - } - - monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) - - with pytest.raises(RuntimeError, match="produced different metrics"): - post_mip.evaluate_realworldqa_mmmu_prefix100_checkpoint( - tmp_path / "model", - output_root=tmp_path / "output", - settings={}, - ) - - -def test_deprecated_post_mip_profile_alias_forwards_to_canonical(monkeypatch, tmp_path): - expected = {"metrics": {"accuracy": 0.5}} - monkeypatch.setattr( - post_mip, - "evaluate_realworldqa_mmmu_prefix100_checkpoint", - lambda *_args, **_kwargs: expected, - ) - - with pytest.warns(FutureWarning, match="is deprecated"): - result = post_mip.evaluate_e2e_full_eval_checkpoint( - tmp_path / "model", - output_root=tmp_path / "output", - settings={}, - ) - - assert result is expected - - -def test_mmvu_guard_is_limited_to_full_suite(monkeypatch, tmp_path): - snapshot = tmp_path / "snapshot" - snapshot.mkdir() - lmms_root = _write_lmms_tasks(tmp_path, profile.VLM_BENCHMARK_TASKS) - monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) - - tasks_root, configured_tasks = tasks.prepare( - tmp_path / "results", - suite="mmvu-smoke", - dataset_snapshots={"mmvu_val": snapshot}, - quick_manifest=None, - ) - - assert configured_tasks == ("modelopt_vlm_benchmark_mmvu_val",) - assert not (tasks_root / "modelopt_mmvu_guard.py").exists() - generated = (tasks_root / "modelopt_vlm_benchmark_mmvu_val.yaml").read_text() - assert "\nprocess_results:" not in generated - assert ( - "process_docs: !function " - "modelopt_mmvu_smoke_selection.select_modelopt_vlm_benchmark_mmvu_val\n" in generated - ) - assert (tasks_root / "modelopt_mmvu_smoke_selection.py").is_file() - assert not (tasks_root / "modelopt_quick_selection.py").exists() - - full_root, _ = tasks.prepare( - tmp_path / "full-results", - suite="full", - dataset_snapshots=dict.fromkeys(profile.VLM_BENCHMARK_TASKS, snapshot), - quick_manifest=None, - ) - assert (full_root / "modelopt_mmvu_guard.py").is_file() - full_generated = (full_root / "modelopt_vlm_benchmark_mmvu_val.yaml").read_text() - assert "\nprocess_results: !function modelopt_mmvu_guard.process_results\n" in full_generated - - -def test_quick_manifest_requires_exact_pins_counts_and_leaf_balance(tmp_path): - path = _quick_manifest(tmp_path / "quick.json") - assert suites.manifest_sha256(suites.load_quick_manifest(path)) - - manifest = json.loads(path.read_text()) - manifest["tasks"]["mmmu_val"]["rows"].pop() - path.write_text(json.dumps(manifest)) - with pytest.raises(ValueError, match="exactly 120 rows"): - suites.load_quick_manifest(path) - - path = _quick_manifest(path) - manifest = json.loads(path.read_text()) - manifest["tasks"]["mvbench"]["rows"][-1] = { - "leaf_task": "mvbench_action_sequence", - "source_row_index": 8, - "source_sample_id": "action_sequence:8", - } - path.write_text(json.dumps(manifest)) - with pytest.raises(ValueError, match="exactly 8 rows per leaf task"): - suites.load_quick_manifest(path) - - -@pytest.mark.parametrize( - ("extra", "environment", "message"), - [ - ([], {"OPENAI_API_KEY": "token"}, "explicit --allow-judge-calls"), - (["--allow-judge-calls"], {}, "judge credentials are missing"), - ], -) -def test_full_profile_fails_closed_without_judge_authorization_or_credentials( - monkeypatch, tmp_path, extra, environment, message -): - model, hf_home = _full_inputs(monkeypatch, tmp_path) - for name in ("OPENAI_API_KEY", "AZURE_API_KEY", "AZURE_ENDPOINT"): - monkeypatch.delenv(name, raising=False) - for name, value in environment.items(): - monkeypatch.setenv(name, value) - args = evaluation._build_parser().parse_args( - [ - "--checkpoint", - str(model), - "--output-dir", - str(tmp_path / "results"), - "--suite", - "full", - "--hf-home", - str(hf_home), - "--mmvu-judge-api-type", - "openai", - "--mmvu-judge-model", - "judge", - *extra, - ] - ) - with pytest.raises(ValueError, match=message): - preflight.prepare(args) - - -def test_offline_preflight_scrubs_credentials_and_traverses_media(monkeypatch, tmp_path): - for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES: - monkeypatch.setenv(name, "must-not-reach-child") - tasks_root = tmp_path / "tasks" - package = tasks_root / "lmms_eval/tasks" - package.mkdir(parents=True) - (tasks_root / "lmms_eval/__init__.py").write_text("") - (package / "__init__.py").write_text("""import os -class Config: task = "modelopt_vlm_benchmark_mvbench" -class Task: - config = Config() - def has_test_docs(self): return True - def has_validation_docs(self): return False - def has_training_docs(self): return False - def test_docs(self): return [{"video": os.environ["FAKE_MEDIA_PATH"]}] - def doc_to_visual(self, document): return [document["video"]] -class Group: group_name = "modelopt_vlm_benchmark_mvbench" -class TaskManager: - def __init__(self, include_path, model_name): - assert model_name == "qwen3_5" - assert os.environ["HF_DATASETS_OFFLINE"] == "1" - assert os.environ["HF_HUB_OFFLINE"] == "1" - assert os.environ["API_TYPE"] == "openai" - assert os.environ["MODEL_VERSION"] == "modelopt-disabled-lmms-eval-judge" - assert os.environ["OPENAI_API_KEY"] == "modelopt-disabled-lmms-eval-judge" - assert os.environ["OPENAI_API_URL"] == "http://127.0.0.1:9" - credential_names = ( - "HF_TOKEN", - "HUGGINGFACEHUB_API_TOKEN", - "HUGGING_FACE_HUB_TOKEN", - ) - assert all(name not in os.environ for name in credential_names) - def load_task_or_group(self, tasks): return {Group(): {"leaf": Task()}} -""") - hf_home = tmp_path / "hf-home" - hf_home.mkdir() - media = tmp_path / "sample.mp4" - media.write_bytes(b"video") - monkeypatch.setenv("FAKE_MEDIA_PATH", str(media)) - - report = tasks.verify_offline( - tasks_root, - ("modelopt_vlm_benchmark_mvbench",), - hf_home=hf_home, - timeout_seconds=123, - model_name="qwen3_5", - ) - - assert report["media_documents"] == 1 - assert report["status"] == "passed" - - -def test_video_adapter_normalizes_supported_suffixes_and_rejects_unknown(monkeypatch, tmp_path): - uppercase = tmp_path / "sample.MP4" - matroska = tmp_path / "sample.mkv" - unknown = tmp_path / "sample.webm" - for path in (uppercase, matroska, unknown): - path.write_bytes(b"video") - videomme = ModuleType("lmms_eval.tasks.videomme.utils") - videomme.videomme_doc_to_visual = lambda _document: [str(matroska)] - perception = ModuleType("lmms_eval.tasks.perceptiontest.val.utils") - perception.perceptiontest_val_doc_to_visual = lambda _document: [str(uppercase)] - packages = { - "lmms_eval": ModuleType("lmms_eval"), - "lmms_eval.tasks": ModuleType("lmms_eval.tasks"), - "lmms_eval.tasks.videomme": ModuleType("lmms_eval.tasks.videomme"), - "lmms_eval.tasks.videomme.utils": videomme, - "lmms_eval.tasks.perceptiontest": ModuleType("lmms_eval.tasks.perceptiontest"), - "lmms_eval.tasks.perceptiontest.val": ModuleType("lmms_eval.tasks.perceptiontest.val"), - "lmms_eval.tasks.perceptiontest.val.utils": perception, - } - packages["lmms_eval.tasks.videomme"].utils = videomme - packages["lmms_eval.tasks.perceptiontest.val"].utils = perception - for name, module in packages.items(): - monkeypatch.setitem(sys.modules, name, module) - - tasks._write_video_path_adapter(tmp_path) - spec = importlib.util.spec_from_file_location( - "adapter_test", tmp_path / "modelopt_video_paths.py" - ) - assert spec is not None and spec.loader is not None - adapter = importlib.util.module_from_spec(spec) - spec.loader.exec_module(adapter) - - for generated, source in ( - (adapter.videomme_doc_to_visual({})[0], matroska), - (adapter.perceptiontest_doc_to_visual({})[0], uppercase), - ): - alias = Path(generated) - assert alias.suffix == ".mp4" - assert alias.resolve() == source - with pytest.raises(ValueError, match="unsupported Qwen 3.5 video suffix"): - adapter._normalize([str(unknown)]) - - -def test_profile_contract_pins_every_task_and_revision(): - assert profile.VLM_BENCHMARK_TASKS == ( - "realworldqa", - "mmmu_val", - "video_mmmu", - "mvbench", - "mmvu_val", - "videomme", - "longvideobench_val_v", - "mlvu_dev", - "perceptiontest_val_mc", - ) - assert all(len(item.revision) == 40 for item in profile.VLM_BENCHMARK_DATASETS.values()) - - -def test_video_reader_validation_is_limited_to_video_suites(monkeypatch): - monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: None) - - preflight._verify_video_reader(("realworldqa", "mmmu_val")) - with pytest.raises(RuntimeError, match="decord-compatible reader"): - preflight._verify_video_reader(("mvbench",)) - - -def test_native_backend_validation_requires_qwen_vision_utilities(monkeypatch): - monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: None) - - preflight._verify_backend_dependencies("vllm") - with pytest.raises(RuntimeError, match="qwen-vl-utils"): - preflight._verify_backend_dependencies("qwen3_5") - - -def test_requirements_pin_matches_runtime_lmms_eval_revision(): - requirements = (checkpoint.REPOSITORY_ROOT / "examples/puzzletron/requirements.txt").read_text() - assert "lmms-eval.git" not in requirements - assert 'eva-decord==0.6.1; platform_system == "Linux"' in requirements.splitlines() - assert "wandb==0.29.0" in requirements.splitlines() - environment = json.loads( - (checkpoint.REPOSITORY_ROOT / "examples/puzzletron/ci_environment.json").read_text() - ) - assert environment["lmms_eval"]["commit"] == checkpoint.LMMS_EVAL_REVISION - patch = ( - checkpoint.REPOSITORY_ROOT - / "examples/puzzletron/patches" - / environment["lmms_eval"]["compatibility_patch"] - ) - assert ( - hashlib.sha256(patch.read_bytes()).hexdigest() - == environment["lmms_eval"]["compatibility_patch_sha256"] - ) - - def test_vlm_parser_exposes_only_suite_owned_sample_limits(): help_text = evaluation._build_parser().format_help() assert "--limit" not in help_text @@ -1294,16 +52,3 @@ def test_vlm_parser_exposes_only_suite_owned_sample_limits(): def test_vlm_parser_defaults_to_short_suite(): assert evaluation._build_parser().get_default("suite") == "short" - - -def test_huggingface_dependency_supports_range_metadata_api(): - pyproject = (checkpoint.REPOSITORY_ROOT / "pyproject.toml").read_text() - assert '"huggingface_hub>=0.30.0",' in pyproject - - -def test_credential_scope_restores_inherited_values(monkeypatch): - for index, name in enumerate(checkpoint.HUGGINGFACE_CREDENTIAL_NAMES): - monkeypatch.setenv(name, f"secret-{index}") - with checkpoint.without_huggingface_credentials(): - assert all(name not in os.environ for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) - assert all(name in os.environ for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py new file mode 100644 index 00000000000..871dac1f4a2 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_tasks.py @@ -0,0 +1,473 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for generated lmms-eval task adapters and selectors.""" + +import importlib.util +import json +import random +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import contracts, preflight, profile, suites, tasks +from examples.puzzletron.evaluation.vlm import run as evaluation +from tests.unit.torch.puzzletron.evaluation.vlm.vlm_test_utils import ( + _full_inputs, + _write_lmms_tasks, +) + + +def test_judge_free_profile_builds_grouped_and_single_selectors(tmp_path): + contract = contracts.load_profile("judge-free-8_690-examples_r1-native") + exact_rows = contract.exact_rows + assert exact_rows is not None + validated = suites.validate_exact_rows_manifest( + exact_rows, + expected_revision=checkpoint.LMMS_EVAL_REVISION, + expected_tasks=contract.source_tasks, + ) + + tasks._write_quick_selection_module(tmp_path, validated) + spec = importlib.util.spec_from_file_location( + "short_all_selectors", tmp_path / "modelopt_quick_selection.py" + ) + assert spec is not None and spec.loader is not None + selectors = importlib.util.module_from_spec(spec) + spec.loader.exec_module(selectors) + + class Documents: + def __init__(self, size, *, columns=None, rows=None): + self.size = size + self.columns = columns or {} + self.rows = rows or {} + + def __len__(self): + return self.size + + def __getitem__(self, index): + if isinstance(index, str): + return self.columns[index] + return self.rows.get(index, {}) + + def select(self, indices): + return list(indices) + + tasks_manifest = validated["tasks"] + adaptation = [ + row["source_row_index"] + for row in tasks_manifest["video_mmmu"]["rows"] + if row["leaf_task"] == "video_mmmu_adaptation" + ] + assert ( + selectors.select_modelopt_vlm_benchmark_video_mmmu_adaptation(Documents(300)) == adaptation + ) + with pytest.raises(ValueError, match="source population drifted"): + selectors.select_modelopt_vlm_benchmark_video_mmmu_adaptation(Documents(299)) + with pytest.raises(ValueError, match="source population drifted"): + selectors.select_modelopt_vlm_benchmark_realworldqa(Documents(764)) + with pytest.raises(ValueError, match="source population drifted"): + selectors.select_modelopt_vlm_benchmark_mvbench_action_sequence(Documents(199)) + + mmmu_task = tasks_manifest["mmmu_val"] + mmmu_ids = [ + f"validation_{stratum['name']}_{index + 1}" + for stratum in mmmu_task["selection"]["strata"] + for index in range(stratum["population_rows"]) + ] + mmmu_rows = { + row["source_row_index"]: {"id": row["source_sample_id"]} for row in mmmu_task["rows"] + } + mmmu_documents = Documents(900, columns={"id": mmmu_ids}, rows=mmmu_rows) + assert selectors.select_modelopt_vlm_benchmark_mmmu_val(mmmu_documents) == [ + row["source_row_index"] for row in mmmu_task["rows"] + ] + mmmu_ids[0] = mmmu_ids[30] + with pytest.raises(ValueError, match="source strata drifted"): + selectors.select_modelopt_vlm_benchmark_mmmu_val(mmmu_documents) + + assert callable(selectors.select_modelopt_vlm_benchmark_videomme) + + +@pytest.mark.parametrize( + ("task", "drift", "expected_rank", "observed_strata"), + [ + ("videomme", None, 1, ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"]), + ("mlvu_dev", None, 1, ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"]), + ( + "perceptiontest_val_mc", + None, + 1, + ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"], + ), + ( + "videomme", + "selected stratum", + 1, + ["alpha|kind", "beta|kind", "beta|kind", "alpha|kind"], + ), + ( + "videomme", + "local rank", + 0, + ["alpha|kind", "beta|kind", "alpha|kind", "beta|kind"], + ), + ], +) +def test_audited_selector_checks_selected_stratum_and_local_rank( + tmp_path, task, drift, expected_rank, observed_strata +): + expected_stratum = "alpha" if task == "mlvu_dev" else "alpha|kind" + other_stratum = "beta" if task == "mlvu_dev" else "beta|kind" + upstream_id = "video:q" if task == "perceptiontest_val_mc" else "q" + manifest = { + "tasks": { + task: { + "rows": [ + { + "source_row_index": 2, + "source_sample_id": f"{task}:2", + "sampling_stratum": expected_stratum, + "source_stratum_index": expected_rank, + "upstream_sample_id": upstream_id, + } + ], + "selection": { + "population_rows": 4, + "strata": [ + {"name": expected_stratum, "population_rows": 2}, + {"name": other_stratum, "population_rows": 2}, + ], + }, + } + } + } + tasks._write_quick_selection_module(tmp_path, manifest) + spec = importlib.util.spec_from_file_location( + f"sampling_position_{task}_{drift}", tmp_path / "modelopt_quick_selection.py" + ) + assert spec is not None and spec.loader is not None + selectors = importlib.util.module_from_spec(spec) + spec.loader.exec_module(selectors) + + class Documents: + def __len__(self): + return 4 + + def __getitem__(self, index): + if isinstance(index, str): + if task == "mlvu_dev": + return [value.split("|", 1)[0] for value in observed_strata] + column = 0 if index in {"duration", "area"} else 1 + return [value.split("|", 1)[column] for value in observed_strata] + if task == "perceptiontest_val_mc": + return {"video_name": "video", "question_id": "q"} + return {"question_id": "q"} + + def select(self, indices): + return list(indices) + + selector = getattr(selectors, f"select_{suites.task_name(task)}") + if drift is None: + assert selector(Documents()) == [2] + else: + with pytest.raises(ValueError, match="source sampling positions drifted"): + selector(Documents()) + + +def test_smoke_profile_generates_only_manifest_backed_mvbench_leaves(monkeypatch, tmp_path): + model, hf_home = _full_inputs(monkeypatch, tmp_path) + args = evaluation._build_parser().parse_args( + [ + "--checkpoint", + str(model), + "--output-dir", + str(tmp_path / "results"), + "--profile", + "core-3_24-examples_r1-native", + "--hf-home", + str(hf_home), + ] + ) + + prepared = preflight.prepare(args) + tasks_root, configured_tasks = tasks.prepare( + args.output_dir, + suite=prepared.suite, + source_tasks=prepared.source_tasks, + profile_task_leaves=prepared.profile_task_leaves, + dataset_snapshots=prepared.dataset_snapshots, + quick_manifest=prepared.quick_manifest, + ) + + assert configured_tasks == ( + "modelopt_vlm_benchmark_realworldqa", + "modelopt_vlm_benchmark_mmmu_val", + "modelopt_vlm_benchmark_mvbench", + ) + group = json.loads((tasks_root / "modelopt_vlm_benchmark_mvbench.yaml").read_text()) + assert group["task"] == ["modelopt_vlm_benchmark_mvbench_action_sequence"] + assert (tasks_root / "modelopt_vlm_benchmark_mvbench_action_sequence.yaml").is_file() + assert not (tasks_root / "modelopt_vlm_benchmark_mvbench_egocentric_navigation.yaml").exists() + + spec = importlib.util.spec_from_file_location( + "smoke_selectors", tasks_root / "modelopt_quick_selection.py" + ) + assert spec is not None and spec.loader is not None + selectors = importlib.util.module_from_spec(spec) + spec.loader.exec_module(selectors) + assert hasattr(selectors, "select_modelopt_vlm_benchmark_mvbench_action_sequence") + assert not hasattr(selectors, "select_modelopt_vlm_benchmark_mvbench_egocentric_navigation") + + +def test_mmvu_guard_is_limited_to_full_suite(monkeypatch, tmp_path): + snapshot = tmp_path / "snapshot" + snapshot.mkdir() + lmms_root = _write_lmms_tasks(tmp_path, profile.VLM_BENCHMARK_TASKS) + monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) + + tasks_root, configured_tasks = tasks.prepare( + tmp_path / "results", + suite="mmvu-smoke", + dataset_snapshots={"mmvu_val": snapshot}, + quick_manifest=None, + ) + + assert configured_tasks == ("modelopt_vlm_benchmark_mmvu_val",) + assert not (tasks_root / "modelopt_mmvu_guard.py").exists() + generated = (tasks_root / "modelopt_vlm_benchmark_mmvu_val.yaml").read_text() + assert "\nprocess_results:" not in generated + assert ( + "process_docs: !function " + "modelopt_mmvu_smoke_selection.select_modelopt_vlm_benchmark_mmvu_val\n" in generated + ) + assert (tasks_root / "modelopt_mmvu_smoke_selection.py").is_file() + assert not (tasks_root / "modelopt_quick_selection.py").exists() + + full_root, _ = tasks.prepare( + tmp_path / "full-results", + suite="full", + dataset_snapshots=dict.fromkeys(profile.VLM_BENCHMARK_TASKS, snapshot), + quick_manifest=None, + ) + assert (full_root / "modelopt_mmvu_guard.py").is_file() + full_generated = (full_root / "modelopt_vlm_benchmark_mmvu_val.yaml").read_text() + assert "\nprocess_results: !function modelopt_mmvu_guard.process_results\n" in full_generated + + +def test_mmmu_adapter_labels_parser_fallback_without_changing_prediction(monkeypatch, tmp_path): + snapshot = tmp_path / "snapshot" + snapshot.mkdir() + lmms_root = _write_lmms_tasks(tmp_path, ("mmmu_val",)) + monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) + tasks_root, _ = tasks.prepare( + tmp_path / "results", + suite="short", + source_tasks=("mmmu_val",), + dataset_snapshots={"mmmu_val": snapshot}, + quick_manifest=None, + ) + + upstream = ModuleType("lmms_eval.tasks.mmmu.utils") + + def get_multi_choice_info(options): + choices = [chr(ord("A") + index) for index in range(len(options))] + return dict(zip(choices, options, strict=True)), choices + + def parse_multi_choice_response(response, all_choices, _index_to_answer): + return random.choice(all_choices) if response == "unparseable" else "A" + + def mmmu_process_results(document, results): + if document["question_type"] == "multiple-choice": + index_to_answer, choices = get_multi_choice_info(json.loads(document["options"])) + parsed = [ + parse_multi_choice_response(response, choices, index_to_answer) + for response in results + ] + else: + parsed = [""] * len(results) + accuracy = {"parsed_pred": parsed} + return {"mmmu_acc": accuracy, "mmmu_acc_pass_at_k": accuracy} + + upstream.get_multi_choice_info = get_multi_choice_info + upstream.parse_multi_choice_response = parse_multi_choice_response + upstream.mmmu_process_results = mmmu_process_results + package_modules = { + "lmms_eval": ModuleType("lmms_eval"), + "lmms_eval.tasks": ModuleType("lmms_eval.tasks"), + "lmms_eval.tasks.mmmu": ModuleType("lmms_eval.tasks.mmmu"), + "lmms_eval.tasks.mmmu.utils": upstream, + } + package_modules["lmms_eval.tasks.mmmu"].utils = upstream + for name, module in package_modules.items(): + monkeypatch.setitem(sys.modules, name, module) + spec = importlib.util.spec_from_file_location( + "modelopt_mmmu_audit", tasks_root / "modelopt_mmmu_audit.py" + ) + assert spec is not None and spec.loader is not None + audit_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(audit_module) + document = { + "options": json.dumps(["first", "second"]), + "question_type": "multiple-choice", + } + + parsed = audit_module.process_results(document, ["(A)"]) + random.seed(123) + parse_multi_choice_response("unparseable", ["A", "B"], {}) + expected_random_state = random.getstate() + random.seed(123) + fallback = audit_module.process_results(document, ["unparseable"]) + invalid_open = audit_module.process_results( + {"question_type": "open"}, + ["unparseable"], + ) + + assert parsed["mmmu_acc"] == { + "parsed_pred": ["A"], + "parser_status": ["parsed"], + } + assert fallback["mmmu_acc"]["parsed_pred"][0] in {"A", "B"} + assert fallback["mmmu_acc"]["parser_status"] == ["fallback_random"] + assert random.getstate() == expected_random_state + assert invalid_open["mmmu_acc"]["parser_status"] == ["invalid_open"] + generated = (tasks_root / f"{suites.task_name('mmmu_val')}.yaml").read_text() + assert "process_results: !function modelopt_mmmu_audit.process_results\n" in generated + + +def test_offline_preflight_scrubs_credentials_and_traverses_media(monkeypatch, tmp_path): + for name in checkpoint.HUGGINGFACE_CREDENTIAL_NAMES: + monkeypatch.setenv(name, "must-not-reach-child") + tasks_root = tmp_path / "tasks" + package = tasks_root / "lmms_eval/tasks" + package.mkdir(parents=True) + (tasks_root / "lmms_eval/__init__.py").write_text("") + (package / "__init__.py").write_text("""import os +class Config: task = "modelopt_vlm_benchmark_mvbench_action_sequence" +class Task: + config = Config() + def has_test_docs(self): return True + def has_validation_docs(self): return False + def has_training_docs(self): return False + def test_docs(self): return [{"video": os.environ["FAKE_MEDIA_PATH"]}] + def doc_to_visual(self, document): return [document["video"]] +class Group: group_name = "modelopt_vlm_benchmark_mvbench" +class TaskManager: + def __init__(self, include_path, model_name): + assert model_name == "qwen3_5" + assert os.environ["HF_DATASETS_OFFLINE"] == "1" + assert os.environ["HF_HUB_OFFLINE"] == "1" + assert os.environ["API_TYPE"] == "openai" + assert os.environ["MODEL_VERSION"] == "modelopt-disabled-lmms-eval-judge" + assert os.environ["OPENAI_API_KEY"] == "modelopt-disabled-lmms-eval-judge" + assert os.environ["OPENAI_API_URL"] == "http://127.0.0.1:9" + credential_names = ( + "HF_TOKEN", + "HUGGINGFACEHUB_API_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + ) + assert all(name not in os.environ for name in credential_names) + def load_task_or_group(self, tasks): + first = Task() + if os.environ.get("FAKE_DISTINCT_DUPLICATE"): + return {Group(): {"first": first, "second": Task()}} + return {Group(): {"first": first, "repeat": first}} +""") + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + media = tmp_path / "sample.mp4" + media.write_bytes(b"video") + monkeypatch.setenv("FAKE_MEDIA_PATH", str(media)) + + report = tasks.verify_offline( + tasks_root, + ("modelopt_vlm_benchmark_mvbench",), + hf_home=hf_home, + timeout_seconds=123, + model_name="qwen3_5", + expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 1}, + ) + + assert report["document_counts"] == {"modelopt_vlm_benchmark_mvbench_action_sequence": 1} + assert report["media_documents"] == 1 + assert report["observed_populations"] == {"modelopt_vlm_benchmark_mvbench_action_sequence": 1} + assert report["status"] == "passed" + + with pytest.raises(RuntimeError, match="configured task population mismatch"): + tasks.verify_offline( + tasks_root, + ("modelopt_vlm_benchmark_mvbench",), + hf_home=hf_home, + timeout_seconds=123, + model_name="qwen3_5", + expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 2}, + ) + + monkeypatch.setenv("FAKE_DISTINCT_DUPLICATE", "1") + with pytest.raises(RuntimeError, match="distinct task objects share configured task name"): + tasks.verify_offline( + tasks_root, + ("modelopt_vlm_benchmark_mvbench",), + hf_home=hf_home, + timeout_seconds=123, + model_name="qwen3_5", + expected_populations={"modelopt_vlm_benchmark_mvbench_action_sequence": 1}, + ) + + +def test_video_adapter_normalizes_supported_suffixes_and_rejects_unknown(monkeypatch, tmp_path): + uppercase = tmp_path / "sample.MP4" + matroska = tmp_path / "sample.mkv" + unknown = tmp_path / "sample.webm" + for path in (uppercase, matroska, unknown): + path.write_bytes(b"video") + videomme = ModuleType("lmms_eval.tasks.videomme.utils") + videomme.videomme_doc_to_visual = lambda _document: [str(matroska)] + perception = ModuleType("lmms_eval.tasks.perceptiontest.val.utils") + perception.perceptiontest_val_doc_to_visual = lambda _document: [str(uppercase)] + packages = { + "lmms_eval": ModuleType("lmms_eval"), + "lmms_eval.tasks": ModuleType("lmms_eval.tasks"), + "lmms_eval.tasks.videomme": ModuleType("lmms_eval.tasks.videomme"), + "lmms_eval.tasks.videomme.utils": videomme, + "lmms_eval.tasks.perceptiontest": ModuleType("lmms_eval.tasks.perceptiontest"), + "lmms_eval.tasks.perceptiontest.val": ModuleType("lmms_eval.tasks.perceptiontest.val"), + "lmms_eval.tasks.perceptiontest.val.utils": perception, + } + packages["lmms_eval.tasks.videomme"].utils = videomme + packages["lmms_eval.tasks.perceptiontest.val"].utils = perception + for name, module in packages.items(): + monkeypatch.setitem(sys.modules, name, module) + + tasks._write_video_path_adapter(tmp_path) + spec = importlib.util.spec_from_file_location( + "adapter_test", tmp_path / "modelopt_video_paths.py" + ) + assert spec is not None and spec.loader is not None + adapter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(adapter) + + for generated, source in ( + (adapter.videomme_doc_to_visual({})[0], matroska), + (adapter.perceptiontest_doc_to_visual({})[0], uppercase), + ): + alias = Path(generated) + assert alias.suffix == ".mp4" + assert alias.resolve() == source + with pytest.raises(ValueError, match="unsupported Qwen 3.5 video suffix"): + adapter._normalize([str(unknown)]) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/vlm_test_utils.py b/tests/unit/torch/puzzletron/evaluation/vlm/vlm_test_utils.py new file mode 100644 index 00000000000..18a1715ab59 --- /dev/null +++ b/tests/unit/torch/puzzletron/evaluation/vlm/vlm_test_utils.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared local fixtures for VLM evaluation unit tests.""" + +import json +from pathlib import Path + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import preflight, profile, suites, tasks + +_QWEN_CONFIG = { + "architectures": ["Qwen3_5ForConditionalGeneration"], + "model_type": "qwen3_5", + "text_config": { + "hidden_size": 1024, + "intermediate_size": 3584, + "model_type": "qwen3_5_text", + "num_attention_heads": 8, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "vocab_size": 248320, + }, +} +_TASK_CONFIGS = {name: item.task_config for name, item in profile.VLM_BENCHMARK_DATASETS.items()} + + +def _write_checkpoint_at(model: Path) -> Path: + model.mkdir() + (model / "config.json").write_text(json.dumps(_QWEN_CONFIG) + "\n") + (model / "preprocessor_config.json").write_text("{}\n") + (model / "chat_template.jinja").write_text( + "{% if enable_thinking is defined and enable_thinking is false %}" + "\n\n\n\n{% else %}\n{% endif %}" + ) + return model + + +def _write_checkpoint(root: Path) -> Path: + return _write_checkpoint_at(root / "model") + + +def _write_core3_teacher_snapshot(root: Path) -> tuple[Path, Path]: + hf_home = root / "hf-home" + snapshot = ( + hf_home + / "hub/models--Qwen--Qwen3.5-0.8B/snapshots" + / "2fc06364715b967f1860aea9cf38778875588b17" + ) + snapshot.parent.mkdir(parents=True) + return _write_checkpoint_at(snapshot), hf_home + + +def _write_lmms_tasks(root: Path, tasks: tuple[str, ...]) -> Path: + lmms_root = root / "lmms_eval" + for task in tasks: + task_configs = [_TASK_CONFIGS[task]] + if task == "video_mmmu": + task_configs.extend( + f"tasks/videommmu/{leaf}.yaml" for leaf in suites.VIDEO_MMMU_LEAF_TASKS + ) + elif task == "mvbench": + task_configs.extend( + f"tasks/mvbench/mvbench_{leaf}.yaml" for leaf in suites.MVBENCH_LEAF_TASKS + ) + for relative_path in task_configs: + config = lmms_root / relative_path + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text(f"task: {task}\n") + return lmms_root + + +def _use_offline_fakes(monkeypatch, lmms_root: Path) -> None: + monkeypatch.delenv("HF_HUB_CACHE", raising=False) + monkeypatch.setattr(tasks, "_lmms_eval_root", lambda: lmms_root) + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda _name: object()) + monkeypatch.setattr( + checkpoint, + "verify_lmms_eval_revision", + lambda expected=checkpoint.LMMS_EVAL_REVISION: expected, + ) + monkeypatch.setattr( + suites, + "offline_dataset_snapshot", + lambda hf_home, task, revision: hf_home / ".snapshots" / task / revision, + ) + monkeypatch.setattr( + tasks, + "verify_offline", + lambda _root, tasks, **_kwargs: { + "configured_tasks": list(tasks), + "status": "passed", + }, + ) + + +def _write_fake_mmmu_artifacts(result_path: Path) -> Path: + result_path.write_text(json.dumps({"sample_counts": {suites.task_name("mmmu_val"): 1}}) + "\n") + raw_result_path = result_path.parent / "run_results.json" + raw_result_path.write_text("{}\n") + sample_path = result_path.parent / f"run_samples_{suites.task_name('mmmu_val')}.jsonl" + sample_path.write_text(json.dumps({"mmmu_acc": {"parser_status": ["parsed"]}}) + "\n") + return raw_result_path + + +def _full_inputs(monkeypatch, tmp_path): + model = _write_checkpoint(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, profile.VLM_BENCHMARK_TASKS) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + for dataset in profile.VLM_BENCHMARK_VIDEO_DATASETS.values(): + media = hf_home / dataset.media_dir + media.mkdir(parents=True, exist_ok=True) + (media / "sample").write_bytes(b"media") + return model, hf_home + + +def _quick_manifest(path: Path) -> Path: + counts = {"realworldqa": 64, "mmmu_val": 120} + tasks = { + task: { + "dataset_revision": profile.VLM_BENCHMARK_DATASETS[task].revision, + "rows": [ + { + "source_row_index": index, + "source_sample_id": ( + f"test:{index}" if task == "realworldqa" else f"question-{index}" + ), + } + for index in range(count) + ], + } + for task, count in counts.items() + } + tasks["mvbench"] = { + "dataset_revision": profile.VLM_BENCHMARK_DATASETS["mvbench"].revision, + "rows": [ + { + "leaf_task": f"mvbench_{leaf}", + "source_row_index": index, + "source_sample_id": f"{leaf}:{index}", + } + for leaf in suites.MVBENCH_LEAF_TASKS + for index in range(8) + ], + } + path.write_text( + json.dumps( + { + "schema": "modelopt.vlm-benchmark-quick/v1", + "lmms_eval_revision": checkpoint.LMMS_EVAL_REVISION, + "tasks": tasks, + } + ) + + "\n" + ) + return path diff --git a/tests/unit/torch/puzzletron/test_ci_image_contract.py b/tests/unit/torch/puzzletron/test_ci_image_contract.py index 61c7331ad70..715a6fc8fef 100644 --- a/tests/unit/torch/puzzletron/test_ci_image_contract.py +++ b/tests/unit/torch/puzzletron/test_ci_image_contract.py @@ -18,6 +18,7 @@ import hashlib import json import re +import subprocess import yaml @@ -119,6 +120,7 @@ def test_lmms_eval_compatibility_patch_reconciles_worker_dependencies(project_ro "- printer = get_printer(Settings()._jupyter)", "- return printer", "- self.printer = get_wandb_printer()", + "- sampling_params = SamplingParams(**params)", "-from latex2sympy2 import latex2sympy", "+from latex2sympy2_extended import latex2sympy", "-from latex2sympy2 import latex2sympy", @@ -132,6 +134,7 @@ def test_lmms_eval_compatibility_patch_reconciles_worker_dependencies(project_ro ] assert lmms_source["compatibility_patch_files"] == [ "lmms_eval/loggers/wandb_logger.py", + "lmms_eval/models/simple/vllm.py", "lmms_eval/tasks/emma/utils.py", "lmms_eval/tasks/mathvision/eval_utils.py", "lmms_eval/tasks/stare/utils.py", @@ -145,6 +148,60 @@ def test_lmms_eval_compatibility_patch_reconciles_worker_dependencies(project_ro assert 'python -m pip install -e "${LMMS_EVAL_ROOT}[qwen]"' in dockerfile +def test_lmms_eval_vllm_patch_preserves_task_sampling(project_root_path, tmp_path): + puzzletron_root = project_root_path / "examples/puzzletron" + environment = json.loads((puzzletron_root / "ci_environment.json").read_text()) + patch_text = ( + puzzletron_root / "patches" / environment["lmms_eval"]["compatibility_patch"] + ).read_text() + marker = "diff --git a/lmms_eval/models/simple/vllm.py b/lmms_eval/models/simple/vllm.py" + start = patch_text.index(marker) + end = patch_text.index("\ndiff --git ", start + len(marker)) + 1 + vllm_patch = tmp_path / "vllm.patch" + vllm_patch.write_text(patch_text[start:end]) + + correct_sampling = ( + " sampling_params = " + "SamplingParams(**self._build_sampling_params_dict(gen_kwargs))" + ) + undefined_overwrite = " sampling_params = SamplingParams(**params)" + + def write_fixture(root): + source = root / "lmms_eval/models/simple/vllm.py" + source.parent.mkdir(parents=True) + lines = ["# pinned upstream fixture"] * 522 + lines[474] = ( + ' gen_kwargs["max_new_tokens"] = ' + 'self._select_max_new_tokens(gen_kwargs.get("max_new_tokens"))' + ) + lines[477] = correct_sampling + lines[520] = undefined_overwrite + lines[521] = ( + ' self._write_watchdog_heartbeat("chat_start", ' + "batch_idx=batch_idx, batch_requests=batch_requests)" + ) + source.write_text("\n".join(lines) + "\n") + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + return source + + exact_checkout = tmp_path / "exact" + exact_checkout.mkdir() + exact_source = write_fixture(exact_checkout) + subprocess.run( + ["git", "apply", "--unidiff-zero", "--check", str(vllm_patch)], + cwd=exact_checkout, + check=True, + ) + subprocess.run( + ["git", "apply", "--unidiff-zero", str(vllm_patch)], + cwd=exact_checkout, + check=True, + ) + patched_source = exact_source.read_text() + assert correct_sampling in patched_source + assert undefined_overwrite not in patched_source + + def test_image_checks_native_lmms_eval_contract(project_root_path): puzzletron_root = project_root_path / "examples/puzzletron" environment = json.loads((puzzletron_root / "ci_environment.json").read_text()) diff --git a/tests/unit/torch/puzzletron/test_lmms_evaluation.py b/tests/unit/torch/puzzletron/test_lmms_evaluation.py index 1ba168edd9c..bd86a120693 100644 --- a/tests/unit/torch/puzzletron/test_lmms_evaluation.py +++ b/tests/unit/torch/puzzletron/test_lmms_evaluation.py @@ -17,6 +17,7 @@ import json import os +import shutil import subprocess import sys from pathlib import Path @@ -500,6 +501,36 @@ def time_out(argv, **_kwargs): assert Path(error.stderr_path).read_text() == "evaluation timed out\n" +def test_run_checkpoint_preserves_timeout_artifacts_if_attempt_directory_disappears( + monkeypatch, tmp_path +): + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + + def time_out(argv, *, cwd, **_kwargs): + shutil.rmtree(cwd) + raise lmms.LmmsEvalTimeoutError( + argv, + 7, + output="partial evaluator output\n", + stderr="evaluation timed out\n", + ) + + monkeypatch.setattr(lmms, "_run_process", time_out) + + with pytest.raises(lmms.LmmsEvalTimeoutError) as exc_info: + lmms.run_lmms_eval_checkpoint( + checkpoint, + output_root=tmp_path / "results", + settings=_settings("ifeval"), + ) + + error = exc_info.value + assert Path(error.command_path).is_file() + assert Path(error.stdout_path).read_text() == "partial evaluator output\n" + assert Path(error.stderr_path).read_text() == "evaluation timed out\n" + + def test_completion_validates_resolved_task_expansion(): sample_counts = lmms._validate_completion( { diff --git a/tests/unit/torch/puzzletron/test_orchestration_compiler.py b/tests/unit/torch/puzzletron/test_orchestration_compiler.py index 5adab0934da..3008a023cff 100644 --- a/tests/unit/torch/puzzletron/test_orchestration_compiler.py +++ b/tests/unit/torch/puzzletron/test_orchestration_compiler.py @@ -571,10 +571,17 @@ def test_post_mip_compiler_topologically_orders_serialized_nodes() -> None: "best": { "type": "filter", "input": "final_eval", + "mode": "top_k", "metric": "final_eval.kl_div", + "top_k": 1, }, "final_eval": {"type": "evaluation", "input": "initial"}, - "initial": {"type": "filter", "metric": "mip.score"}, + "initial": { + "type": "filter", + "mode": "top_k", + "metric": "mip.score", + "top_k": 1, + }, }, } } diff --git a/tests/unit/torch/puzzletron/test_post_mip_runner.py b/tests/unit/torch/puzzletron/test_post_mip_runner.py index 4ccb6cfde4a..9dac6795131 100644 --- a/tests/unit/torch/puzzletron/test_post_mip_runner.py +++ b/tests/unit/torch/puzzletron/test_post_mip_runner.py @@ -512,22 +512,26 @@ def fake_evaluate(args, *, settings_overrides, preflight_callback): return {"runs": [{"metrics": {"accuracy": 0.5}, "result_path": "result.json"}]} monkeypatch.setattr(post_mip, "evaluate", fake_evaluate) - result = post_mip.evaluate_short_v1_checkpoint( - checkpoint, - output_root=tmp_path / "output", - settings={ - "row_manifest": str(manifest), - "row_manifest_sha256": "a" * 64, - "batch_size": 1, - }, - ) + with pytest.warns(DeprecationWarning, match="qwen35_vlm_short_v1 is deprecated"): + result = post_mip.evaluate_short_v1_checkpoint( + checkpoint, + output_root=tmp_path / "output", + settings={ + "row_manifest": str(manifest), + "row_manifest_sha256": "a" * 64, + "batch_size": 1, + }, + ) assert captured["args"].suite == "quick" assert captured["args"].quick_manifest == manifest assert captured["settings"] == {} assert result["checkpoint"] == str(checkpoint) - with pytest.raises(ValueError, match="differs from the campaign identity"): + with ( + pytest.warns(DeprecationWarning, match="qwen35_vlm_short_v1 is deprecated"), + pytest.raises(ValueError, match="differs from the profile identity"), + ): post_mip.evaluate_short_v1_checkpoint( checkpoint, output_root=tmp_path / "mismatch",