Add reproducible Qwen 3.5 VLM evaluation profiles

examples/puzzletron/Dockerfile

@@ -72,8 +72,9 @@ RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \
 
 # Install the native Qwen 3.5 evaluator in the shared worker environment. The
 # tracked patch removes lmms-eval's unused private WandB printer integration,
-# reconciles its WandB range with AutoModel, and replaces the legacy LaTeX
-# parser whose ANTLR pin conflicts with Puzzletron's Hydra stack.
+# reconciles its WandB range with AutoModel, replaces the legacy LaTeX parser
+# whose ANTLR pin conflicts with Puzzletron's Hydra stack, and preserves the
+# vLLM adapter's task-specific sampling parameters.
 RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \
     lmms_patch="$(pin lmms_eval.compatibility_patch)" && \
     echo "$(pin lmms_eval.compatibility_patch_sha256)  /opt/puzzletron/patches/${lmms_patch}" | \

examples/puzzletron/ci_environment.json

@@ -35,16 +35,17 @@
     "base_version": "0.7.2",
     "repository": "https://github.com/EvolvingLMMs-Lab/lmms-eval.git",
     "commit": "3e675904f8cba6793de12b91979b04d91754bdf3",
-    "compatibility_patch": "lmms_eval_wandb_0_29.patch",
+    "compatibility_patch": "lmms_eval_compat_3e675904.patch",
     "compatibility_patch_context_lines": 0,
     "compatibility_patch_files": [
       "lmms_eval/loggers/wandb_logger.py",
+      "lmms_eval/models/simple/vllm.py",
       "lmms_eval/tasks/emma/utils.py",
       "lmms_eval/tasks/mathvision/eval_utils.py",
       "lmms_eval/tasks/stare/utils.py",
       "pyproject.toml"
     ],
-    "compatibility_patch_sha256": "960ef2f0d5cea85ccb67e22ab28356ae935f8109876b2f65f88693795549de08",
+    "compatibility_patch_sha256": "c8cbb9614cd80c2a58c08e669fdd5e94558bf9f0bc78a740af0e086cc69faec2",
     "required_paths": [
       "models/chat/qwen3_5.py",
       "tasks/gsm8k/gsm8k.yaml",

examples/puzzletron/docs/worker_image.md

@@ -4,6 +4,9 @@ The [`Dockerfile`](../Dockerfile) contains the worker installation steps.
 [`ci_environment.json`](../ci_environment.json) stores the versions, source
 revisions, CUDA targets, and downloaded-file checksums used by those steps.
 The Dockerfile reads the same file for installation and its build-time checks.
+Rebuild the image whenever the Dockerfile, `ci_environment.json`, or a patch
+referenced by that manifest changes. Patches are applied while the image is
+built, so an existing image does not acquire a later patch automatically.
 
 ## Build
 

examples/puzzletron/evaluation/checkpoint.py

@@ -151,6 +151,8 @@ def positive_float(value: str) -> float:
 
 def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> str:
     """Return the imported evaluator revision after matching its source and patch pin."""
+    if expected_revision != LMMS_EVAL_REVISION:
+        raise RuntimeError(f"unsupported lmms-eval revision: {expected_revision}")
     revision = _imported_lmms_eval_revision()
     if revision is not None:
         if revision != expected_revision:
@@ -172,7 +174,7 @@ def verify_lmms_eval_revision(expected_revision: str = LMMS_EVAL_REVISION) -> st
             ci_environment.verify_installed_vcs_source("lmms-eval", LMMS_EVAL_SOURCE)
         except (OSError, subprocess.SubprocessError) as error:
             raise RuntimeError("installed lmms-eval source provenance is unavailable") from error
-        revision = LMMS_EVAL_REVISION
+        revision = expected_revision
     else:
         revision = None
     if revision != expected_revision: