From e6d9de4965c2f4697520d31e5bc597196ffa18e0 Mon Sep 17 00:00:00 2001 From: yuming Date: Tue, 22 Sep 2026 00:48:26 +0800 Subject: [PATCH 01/22] docs(harness): establish project guidance and ignore local worktrees Establish project guidance and worktree ignore rules. --- .gitignore | 6 ++ AGENTS.md | 85 ++++++++-------------- ARCHITECTURE.md | 35 +++++++++ FORCE_CONSTRAIN.md | 31 ++++++++ KEY_INFO_REMINDER.md | 9 +++ harness/constraints/COMMIT.md | 9 +++ harness/design-docs/index.md | 7 ++ harness/exec-plans/active/.gitkeep | 0 harness/exec-plans/completed/.gitkeep | 0 harness/exec-plans/product-debt-tracker.md | 5 ++ harness/exec-plans/tech-debt-tracker.md | 5 ++ harness/generated/.gitkeep | 0 harness/product-specs/index.md | 7 ++ harness/references/index.md | 7 ++ 14 files changed, 150 insertions(+), 56 deletions(-) create mode 100644 ARCHITECTURE.md create mode 100644 FORCE_CONSTRAIN.md create mode 100644 KEY_INFO_REMINDER.md create mode 100644 harness/constraints/COMMIT.md create mode 100644 harness/design-docs/index.md create mode 100644 harness/exec-plans/active/.gitkeep create mode 100644 harness/exec-plans/completed/.gitkeep create mode 100644 harness/exec-plans/product-debt-tracker.md create mode 100644 harness/exec-plans/tech-debt-tracker.md create mode 100644 harness/generated/.gitkeep create mode 100644 harness/product-specs/index.md create mode 100644 harness/references/index.md diff --git a/.gitignore b/.gitignore index 940df9af4b..9d113fa9ab 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,9 @@ __pycache__/ *.cpio.xz *.cpio* .direnv/ + +# worktrees +*worktree*/ + +# Claude Code agent worktrees (local isolation dirs, never for VCS) +.claude/worktrees/ diff --git a/AGENTS.md b/AGENTS.md index 7385bce354..b6034e236f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,56 +1,29 @@ -# README for AI Agents - -## 项目简介 - -DragonOS是一个面向云计算轻量化场景的，完全自主内核的，提供Linux二进制兼容性的64位操作系统，旨在为容器化工作负载提供轻量级、高性能的解决方案。 - -## 设计思想 - -- Linux兼容性：系统调用接口/procfs/sysfs/devfs等的行为应当符合Linux语义。参考Linux 6.6的行为进行实现。 -- 轻量：简化复杂的抽象设计，保留合理的、简洁、符合Rust开发最佳实践的的抽象，提升系统性能。 -- 安全：注重内存安全、并发安全 - - -## 开发准则 - -### 交互语言规范 - -- 所有AI助手在回应DragonOS仓库内的请求时必须使用简体中文。 -- 即便用户使用其他语言输入，也需以中文回答，除非更高优先级的指令明确要求使用其他语言。 - -### 项目目录结构： - -- 文档：在`docs/`目录下 -- 内核：在`kernel/`目录下 -- 自行编写的单元测试程序：在`user/apps/c_unitest`目录下 -- gvisor系统调用测试程序：根据用户给出的程序代码片段来读取测试代码。如果用户没有提供，就尝试寻找，如果找不到，则从 https://cnb.cool/DragonOS-Community/gvisor/-/tree/dragonos/release-20250616.0/test/syscalls/linux 下面获取（下载文件然后再尝试读取） - - -### 开发最佳实践 - -- 三思而后行！深度研究，掌握解决问题所必要的信息，然后再动手开发/修复。 -- 设计要具有合理抽象，避免过度抽象。并且要注意代码复用。 -- 实现代码的时候，多问问自己：这代码写在这里合理吗？（架构、正确性等方面） -- 高内聚、低耦合 -- 符合Linux语义 -- 不得使用workaround的方法绕过问题。要从本质解决问题！ - - -**测试修复相关** - -- 符合Linux 6.6的语义 -- 结合测例报错、测例代码、DragonOS代码、Linux行为实现来深入分析 - -### 开发时的一些常见命令 - -- 编译内核：在项目根目录下运行`make kernel`. 当你想检查你编辑的代码有没有语法错误的时候，请执行这个命令 -- Nix开发环境（推荐）：`nix develop`激活环境后可使用上述命令；`nix run .#yolo-x86_64`一键编译+启动QEMU。详见`docs/introduction/develop_nix.md` - - -## WHERE TO LOOK - -| 任务 | 位置 | 备注 | -|------|------|------| -| 运行 DragonOS | `docs/introduction/develop_nix.md` | Nix 一键启动流程 | -| OOM Killer | `kernel/src/mm/oom.rs` | 全局状态机，缺页路径触发，详见代码注释 | -| 进程管理重构后 | `kernel/src/process/state.rs`（ProcessFlags）、`kernel/src/process/manager/exit.rs`（exit路径） | mod.rs 已拆分为子模块 | +# AGENTS.md — DragonOS 文档 Harness 地图 + +> DragonOS 是面向云计算轻量化场景、提供 Linux 二进制兼容性的 64 位自主内核操作系统，目标是为容器化工作负载提供轻量、高性能的运行环境。 + +本文件只负责路由 Harness 文档。源码布局、工程约束和操作命令分别由下列文档维护，按任务需要读取： + +```text +AGENTS.md ← Harness 文档总地图 +ARCHITECTURE.md ← 系统边界、目录职责与阅读路由 +FORCE_CONSTRAIN.md ← 交互、兼容性、安全与开发强约束 +KEY_INFO_REMINDER.md ← 构建、Nix 环境与 QEMU 关键命令 +harness/ ← 深层 Harness 文档 +├── constraints/ ← 局部操作约束入口 +│ └── COMMIT.md ← 提交上下文与权威规范入口 +├── product-specs/ ← 产品需求文档 +│ └── index.md ← 问题、价值、范围与需求索引 +├── design-docs/ ← 设计意图文档 +│ └── index.md ← 技术方案与权衡索引 +├── exec-plans/ ← 执行计划与债务跟踪 +│ ├── active/ ← 进行中的实施计划 +│ ├── completed/ ← 已完成的实施计划 +│ ├── tech-debt-tracker.md ← 技术债跟踪 +│ └── product-debt-tracker.md ← 产品债跟踪 +├── references/ ← 可复用参考资料 +│ └── index.md ← 参考资料索引 +└── generated/ ← 生成型文档预留目录 +``` + +以上路径均相对仓库根目录，按任务读取对应权威文档。 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000000..97f279ddd3 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,35 @@ +# DragonOS 架构与源码地图 + +## 系统概览 + +DragonOS 是一个使用自主内核的 64 位操作系统，面向云计算轻量化场景并提供 Linux 二进制兼容性。系统以容器化工作负载的轻量、高性能运行需求为目标；具体实现必须同时遵守 [FORCE_CONSTRAIN.md](FORCE_CONSTRAIN.md) 中的 Linux 语义、轻量化和安全约束。 + +## 仓库目录职责 + +根目录的 `Makefile` 统一编排 `kernel`、`user`、`tools` 和 `build-scripts`。阅读或修改代码时，先按下列职责定位，避免把实现放入职责不相符的目录： + +| 路径 | 职责 | +| --- | --- | +| `kernel/` | DragonOS 内核实现及其内核侧构建配置。 | +| `user/` | 用户空间应用、rootfs 组成和用户程序构建配置。 | +| `build-scripts/` | 构建过程使用的 Rust 工具和构建辅助程序。 | +| `config/` | 应用选择、rootfs manifest 等仓库级构建配置。 | +| `tools/` | 构建、镜像制作、运行、调试及开发环境辅助工具。 | +| `docs/` | 项目文档与开发指南。 | +| `harness/` | 二级 Harness 文档，包括规格、设计说明、执行计划、约束入口和参考资料。 | + +## 测试代码路由 + +自行编写的单元测试程序位于 `user/apps/c_unitest/`。 + +分析 gVisor 系统调用测试时，优先读取用户提供的测试程序代码片段；用户未提供时，先在仓库及现有材料中查找。仍然找不到时，再从 [DragonOS-Community/gvisor 的指定测试目录](https://cnb.cool/DragonOS-Community/gvisor/-/tree/dragonos/release-20250616.0/test/syscalls/linux) 获取对应文件，下载后再读取和分析。这个顺序可以保证分析优先基于调用上下文和本地证据，同时保留外部测试源作为最后的权威补充。 + +## 高频阅读路由 + +| 任务 | 入口 | 局部上下文 | +| --- | --- | --- | +| 运行 DragonOS | [docs/introduction/develop_nix.md](docs/introduction/develop_nix.md) | Nix 开发环境和启动流程；常用命令另见 [KEY_INFO_REMINDER.md](KEY_INFO_REMINDER.md)。 | +| 理解 OOM Killer | `kernel/src/mm/oom.rs` | 全局状态机及缺页路径触发逻辑，细节以源码注释和对应内存管理文档为准。 | +| 理解进程管理重构后的状态与退出路径 | `kernel/src/process/state.rs`、`kernel/src/process/manager/exit.rs` | `ProcessFlags` 位于前者，退出路径位于后者；原 `mod.rs` 职责已拆分到子模块。 | + +本文只提供稳定的阅读地图，不展开各子系统内部设计；进入具体模块后，应继续以对应源码和专题文档为依据。 diff --git a/FORCE_CONSTRAIN.md b/FORCE_CONSTRAIN.md new file mode 100644 index 0000000000..50cff84015 --- /dev/null +++ b/FORCE_CONSTRAIN.md @@ -0,0 +1,31 @@ +# DragonOS 强约束 + +本文件是仓库内开发行为与工程取舍的权威约束。它保留无法仅从代码或构建脚本稳定推导的理由和适用范围；具体源码位置由 [ARCHITECTURE.md](ARCHITECTURE.md) 路由。 + +## 交互语言 + +所有 AI 助手回应 DragonOS 仓库内的请求时必须使用简体中文。即使用户使用其他语言输入，也仍应使用简体中文回答，除非更高优先级的指令明确要求使用其他语言。该约束作用于仓库相关的解释、计划、检查结果和交付说明，目的是保持项目协作语言一致。 + +## 兼容性、轻量化与安全 + +系统调用接口以及 `/proc`、`/sys`、`devfs` 等用户可见行为必须符合 Linux 语义，并以 Linux 6.6 的行为作为实现参考。兼容性判断应落实到可观察行为，不能只追求接口名称或表面返回值相似。 + +轻量化要求实现采用合理、简洁并符合 Rust 最佳实践的抽象，避免为了形式完整而过度抽象。抽象应服务于复用、清晰边界和性能，而不是增加无必要的层次。 + +安全性要求设计和实现同时重视内存安全与并发安全。涉及共享状态、生命周期或并发路径时，必须把安全性作为架构与正确性的一部分分析，不能留给事后补救。 + +## 开发与修复方法 + +开始开发或修复前，必须先研究完成判断所需的代码、测试和外部语义证据。这样做是为了避免在未理解调用关系和行为契约时引入局部修补。 + +设计应保持高内聚、低耦合，优先复用已有的合理能力，并持续检查代码所在模块是否符合架构职责。既要避免重复实现，也要避免为了复用而制造与问题规模不相称的抽象。 + +禁止使用 workaround 绕过根因。修复必须解释问题为何发生，并在正确的职责边界内解决；如果问题涉及兼容行为，应以 Linux 语义而不是测例偶然通过作为完成标准。 + +测试修复必须结合测例报错、测例代码、DragonOS 实现和 Linux 行为进行分析，并符合 Linux 6.6 语义。该要求适用于系统调用及相关内核行为的缺陷定位，目的是防止只针对单个输入打补丁。 + +## 文档职责 + +需求的权威入口是 [harness/product-specs/index.md](harness/product-specs/index.md)，设计意图和权衡的权威入口是 [harness/design-docs/index.md](harness/design-docs/index.md)，执行步骤位于 `harness/exec-plans/`。新增内容应进入对应职责的文档，避免在多个位置形成竞争定义。 + +提交消息规则不在本文件重复定义。需要提交相关上下文时，先读 [harness/constraints/COMMIT.md](harness/constraints/COMMIT.md)，再进入其中链接的权威贡献规范。 diff --git a/KEY_INFO_REMINDER.md b/KEY_INFO_REMINDER.md new file mode 100644 index 0000000000..2b8b92ce8c --- /dev/null +++ b/KEY_INFO_REMINDER.md @@ -0,0 +1,9 @@ +# DragonOS 关键命令 + +## 构建与运行 + +在仓库根目录执行 `make kernel` 编译内核；需要检查内核代码是否存在编译或语法问题时，也使用该命令。 + +推荐先执行 `nix develop` 进入 Nix 开发环境，再运行仓库构建命令。需要一键完成 x86_64 构建并启动 QEMU 时，执行 `nix run .#yolo-x86_64`。 + +Nix 安装、flake 配置、分步构建和 QEMU 操作方式以 [docs/introduction/develop_nix.md](docs/introduction/develop_nix.md) 为准。可用 flake 目标可能随仓库演进而变化，需要发现其他目标时在根目录执行 `nix flake show`，不要在本文复制完整目标清单。 diff --git a/harness/constraints/COMMIT.md b/harness/constraints/COMMIT.md new file mode 100644 index 0000000000..d0be7e1985 --- /dev/null +++ b/harness/constraints/COMMIT.md @@ -0,0 +1,9 @@ +# 提交约束入口 + +本文件只提供 Agent 执行提交相关工作时需要的局部上下文，不另行定义或复制一套提交规则。 + +DragonOS 的提交消息采用约定式提交，并对内核破坏性变更标记、squash 工作流、scope 和小写格式有项目特定说明。需要拟定或检查提交消息时，必须阅读权威文档 [代码提交规范](../../docs/community/code_contribution/conventional-commit.md)，以其中当前内容为准。 + +当前仓库没有记录独立的 Archive Commit 规则，因此不得推断或虚构归档提交格式、必填内容或额外流程。 + +当前没有待机械化的提交约束条目。 diff --git a/harness/design-docs/index.md b/harness/design-docs/index.md new file mode 100644 index 0000000000..c52bc9ebb3 --- /dev/null +++ b/harness/design-docs/index.md @@ -0,0 +1,7 @@ +# 设计文档索引 + +设计文档负责说明设计意图、技术方案和关键权衡，是这些语义的权威归属。待解决的问题、用户价值、范围、需求和验收标准应写入 [产品规格](../product-specs/index.md)，不要在设计文档中重复定义。 + +把设计落实为具体活动、顺序和进度记录时，应创建或更新 [执行计划](../exec-plans/)（`harness/exec-plans/`），而不是把执行细节混入设计意图。 + +当前没有设计文档条目。 diff --git a/harness/exec-plans/active/.gitkeep b/harness/exec-plans/active/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/harness/exec-plans/completed/.gitkeep b/harness/exec-plans/completed/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/harness/exec-plans/product-debt-tracker.md b/harness/exec-plans/product-debt-tracker.md new file mode 100644 index 0000000000..c4357e827d --- /dev/null +++ b/harness/exec-plans/product-debt-tracker.md @@ -0,0 +1,5 @@ +# 产品债跟踪 + +本文件记录尚未解决的产品语义、用户体验债务和开放设计问题。 + +当前没有已知的产品债条目。 diff --git a/harness/exec-plans/tech-debt-tracker.md b/harness/exec-plans/tech-debt-tracker.md new file mode 100644 index 0000000000..8b0cafe330 --- /dev/null +++ b/harness/exec-plans/tech-debt-tracker.md @@ -0,0 +1,5 @@ +# 技术债跟踪 + +本文件记录已确认的技术债及其解决状态。 + +当前没有已知的活动或已解决技术债条目。 diff --git a/harness/generated/.gitkeep b/harness/generated/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/harness/product-specs/index.md b/harness/product-specs/index.md new file mode 100644 index 0000000000..22ff64b206 --- /dev/null +++ b/harness/product-specs/index.md @@ -0,0 +1,7 @@ +# 产品规格索引 + +产品规格负责定义问题、用户价值、范围、需求和验收标准，是这些语义的权威归属。技术方案、设计意图及关键权衡应写入 [设计文档](../design-docs/index.md)，不要在产品规格中形成竞争定义。 + +规格落地所需的实施步骤和进度记录属于 [执行计划](../exec-plans/)（`harness/exec-plans/`），不属于产品规格本身。 + +当前没有产品规格条目。 diff --git a/harness/references/index.md b/harness/references/index.md new file mode 100644 index 0000000000..dd74dc121d --- /dev/null +++ b/harness/references/index.md @@ -0,0 +1,7 @@ +# 参考资料索引 + +本索引收录对 Agent 任务具有稳定复用价值的仓库内参考资料。具体任务仍应核对文档描述的适用范围和当前仓库状态。 + +| 参考资料 | 用途 | +| --- | --- | +| [DragonOS 发布助手能力说明](../../docs/agents/release_agent_capability.md) | 调研版本差异、生成发行日志及更新 ChangeLog 时的工作流参考。 | From d8eb817b3e49715cf4eaf842065901e4a1557289 Mon Sep 17 00:00:00 2001 From: yuming Date: Tue, 22 Sep 2026 00:48:26 +0800 Subject: [PATCH 02/22] docs(lmbench): define runner architecture and full-suite implementation plans Introduce runner abstraction and metadata extraction designs and plans before implementation. --- .../2026-07-29-lmbench-runner-abstraction.md | 741 ++++++++++++++++++ ...07-30-lmbench-fullsuite-meta-extraction.md | 590 ++++++++++++++ ...07-29-lmbench-runner-abstraction-design.md | 167 ++++ ...mbench-fullsuite-meta-extraction-design.md | 86 ++ 4 files changed, 1584 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-29-lmbench-runner-abstraction.md create mode 100644 docs/superpowers/plans/2026-07-30-lmbench-fullsuite-meta-extraction.md create mode 100644 docs/superpowers/specs/2026-07-29-lmbench-runner-abstraction-design.md create mode 100644 docs/superpowers/specs/2026-07-30-lmbench-fullsuite-meta-extraction-design.md diff --git a/docs/superpowers/plans/2026-07-29-lmbench-runner-abstraction.md b/docs/superpowers/plans/2026-07-29-lmbench-runner-abstraction.md new file mode 100644 index 0000000000..316c5e3ce2 --- /dev/null +++ b/docs/superpowers/plans/2026-07-29-lmbench-runner-abstraction.md @@ -0,0 +1,741 @@ +# LMbench Runner 抽象与目录分层 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 在扩到 48 测例前，把 `user/apps/tests/benchmark/lmbench/` 按 `runner/`（guest）、`orchestrator/`（host）、根（用户配置 + 包元信息）分层，给 guest runner 一个参数化入口 `run.sh`，删死代码 `run_test_case.sh`，顺手修 `default.nix` 误装/漏装问题。 + +**Architecture:** 源码分层是 host 侧组织，guest 运行时仍扁平——DADK/nix install 时把 `runner/` 内文件 + 根 `config`/`whitelist.txt` 摊平到 rootfs `/opt/tests/benchmark/lmbench/`。`run.sh` 加 `parse_args()` 实现 `--samples/--timeout/--warmup/--whitelist/--config/--only/--list`，无参数 = 现状行为。输出 contract、rcS 调用方式、CI 制品路径全不变。 + +**Tech Stack:** BusyBox `sh`（POSIX，guest）、Python 3（host collector）、GNU make（DADK install）、Nix（可选打包）、JSON Schema（结果校验）。 + +## Global Constraints + +- DADK 硬约束：`Makefile` 必须留包根 `user/apps/tests/benchmark/lmbench/`（`user/dadk/config/all/lmbench_benchmark_tests-1.0.0.toml` 用 `build-from-source` + `source-path` 指向包根 + `build-command = "make install"`）。 +- Nix 硬约束：`default.nix` / `flake.nix` / `flake.lock` 必须留包根（`user/apps/default.nix:66` 用 `callPackage ./tests/benchmark/lmbench {}`）。 +- `toggle_compile_lmbench.sh` 留包根：其 `../../../../..` 相对路径依赖当前层级。 +- guest 运行时布局必须扁平：DADK 与 Nix 两条 install 路径都摊平到 `/opt/tests/benchmark/lmbench/`，`run.sh` 用 `$SCRIPT_DIR/{test_cases,config,whitelist.txt}` 定位。 +- `run.sh` 只在 guest 运行时执行；host 单测用 `LMBENCH_RUNNER_NO_MAIN=1` source，不走 main。 +- 输出 contract 不变：`===LMBENCH_RUN_BEGIN===` / `LMBENCH_META` / `LMBENCH_JSON` / `LMBENCH_SUMMARY` / `===LMBENCH_RUN_END===` / `benchmark测试完成`。 +- 代码注释用英文；commit message 用 `refactor(lmbench):` / `feat(lmbench):` / `docs(lmbench):` 前缀。 +- 用户已授权写完计划直接执行，e2e 步骤若需覆盖 `bin/disk-image-x86_64.img` 仍需明确授权（本计划把 e2e 单列为 Task 7，标注需授权）。 + +--- + +## File Structure + +迁移后的目标结构（spec §目录布局）： + +``` +user/apps/tests/benchmark/lmbench/ + runner/ # guest 内运行 + run.sh # 参数化入口（原 run_tests.sh） + init.sh / env.sh / clean_up.sh + test_cases/ + *.sh / *.meta + test_tcp_loopback_lat_parser.sh # 现有回归 + test_args_parser.sh # 新增：parse_args 回归 + orchestrator/ # host 侧 + monitor_test_results.sh + collect_results.py + schema/lmbench-run.schema.json + config # 留根 + whitelist.txt # 留根 + Makefile # 留根，install 改为从 runner/ 拷 + default.nix / flake.nix / flake.lock # 留根 + toggle_compile_lmbench.sh # 留根 + README.md / .gitignore / results/ +``` + +各文件职责： +- `runner/run.sh` — guest runner 主入口；内含 `kv_get`/`load_config`/`extract_value`/`compute_stats`/`metric_head`/`run_one_case`/`parse_args`/`run_main`；输出 JSONL contract。 +- `runner/init.sh` / `env.sh` / `clean_up.sh` — guest 环境生命周期钩子，被 `run_main` 调起。 +- `runner/test_cases/.sh` + `.meta` — 单测例包装 + 抽取规则元数据。 +- `runner/test_cases/test_*_parser.sh` — host 侧 `busybox sh` 回归测试，source `run.sh`。 +- `orchestrator/monitor_test_results.sh` — host 串口轮询 + 超时判定。 +- `orchestrator/collect_results.py` — host 解析串口 → schema 校验 → 持久化。 +- `orchestrator/schema/lmbench-run.schema.json` — 结果 schema。 +- `Makefile` — DADK install：从 `runner/` 摊平安装到 rootfs。 +- `default.nix` / `flake.nix` — Nix 打包。 +- `config` / `whitelist.txt` — 用户面向顶层配置。 +- `toggle_compile_lmbench.sh` — 构建开关（改 `config/app-blocklist.toml`）。 + +--- + +### Task 1: guest 脚本移入 runner/ + 重命名 + 删死代码 + +**Files:** +- Move: `run_tests.sh` → `runner/run.sh` +- Move: `init.sh` / `env.sh` / `clean_up.sh` → `runner/` +- Move: `test_cases/` → `runner/test_cases/` +- Delete: `run_test_case.sh` +- Modify: `runner/test_cases/test_tcp_loopback_lat_parser.sh`（`RUNNER` 路径） + +**Interfaces:** +- Produces: `runner/run.sh`，被回归测试以 `LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh` source，暴露 `kv_get`/`extract_value`/`run_one_case`（本任务不改这些函数，仅移动 + 重命名）。 + +- [ ] **Step 1: 用 git mv 移动 guest 脚本到 runner/** + +```sh +cd user/apps/tests/benchmark/lmbench +mkdir -p runner +git mv run_tests.sh runner/run.sh +git mv init.sh runner/init.sh +git mv env.sh runner/env.sh +git mv clean_up.sh runner/clean_up.sh +git mv test_cases runner/test_cases +git rm run_test_case.sh +``` + +- [ ] **Step 2: 修回归测试里的 RUNNER 路径** + +编辑 `runner/test_cases/test_tcp_loopback_lat_parser.sh`，把： +```sh +RUNNER="$SCRIPT_DIR/../run_tests.sh" +``` +改为： +```sh +RUNNER="$SCRIPT_DIR/../run.sh" +``` + +- [ ] **Step 3: 跑回归测试验证移动不破坏现有功能** + +Run: +```sh +cd user/apps/tests/benchmark/lmbench +busybox sh runner/test_cases/test_tcp_loopback_lat_parser.sh +``` +Expected: 输出 `tcp_loopback_lat parser regression: PASS`，退出码 0。 + +- [ ] **Step 4: 静态确认无残留旧引用** + +Run: +```sh +cd user/apps/tests/benchmark/lmbench +grep -rn "run_test_case" . --exclude-dir=results --exclude-dir=__pycache__ || true +grep -rn "run_tests\.sh" . --exclude-dir=results --exclude-dir=__pycache__ || true +``` +Expected: `run_test_case` 零命中；`run_tests.sh` 仅在 README.md / Makefile / default.nix / flake.nix（这些在后续 Task 修）。Task 1 范围内文件（runner/）无旧名残留。 + +- [ ] **Step 5: Commit** + +```sh +git add -A user/apps/tests/benchmark/lmbench +git commit -m "refactor(lmbench): move guest scripts into runner/ and drop dead run_test_case.sh" +``` + +--- + +### Task 2: run.sh 参数化（TDD） + +**Files:** +- Create: `runner/test_cases/test_args_parser.sh` +- Modify: `runner/run.sh`（加 `usage`/`parse_args`/`apply_cli_overrides`，改 `run_main`） + +**Interfaces:** +- Consumes: Task 1 产出的 `runner/run.sh`（已含 `load_config`/`run_one_case`）。 +- Produces: `runner/run.sh` 新增 `parse_args()`（`return 2` on unknown arg，可被 source 单测捕获）、`apply_cli_overrides()`、`usage()`；`run_main` 开头调 `parse_args "$@" || exit $?`；`--only` 通过写临时单行 whitelist 文件复用 `run_one_case`（不引入新 dispatch 分支）；`--list` 列 `test_cases/*.sh` 名后 `return 0`。 + +- [ ] **Step 1: 写失败的参数解析回归测试** + +创建 `runner/test_cases/test_args_parser.sh`： +```sh +#!/bin/sh +# Regression test for run.sh argument parsing. +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +RUNNER="$SCRIPT_DIR/../run.sh" + +LMBENCH_RUNNER_NO_MAIN=1 +export LMBENCH_RUNNER_NO_MAIN +. "$RUNNER" + +reset_cli() { + CLI_SAMPLES=""; CLI_TIMEOUT=""; CLI_WARMUP="" + CLI_WHITELIST=""; CLI_CONFIG=""; ONLY_NAME=""; LIST_ONLY="" +} + +# --samples / --only parsed into CLI_* / ONLY_NAME +reset_cli +parse_args --samples 7 --only foo +[ "$CLI_SAMPLES" = "7" ] || { echo "FAIL: CLI_SAMPLES=$CLI_SAMPLES" >&2; exit 1; } +[ "$ONLY_NAME" = "foo" ] || { echo "FAIL: ONLY_NAME=$ONLY_NAME" >&2; exit 1; } + +# --timeout / --warmup / --whitelist / --config / --list +reset_cli +parse_args --timeout 30 --warmup 2 --whitelist /tmp/wl --config /tmp/cfg --list +[ "$CLI_TIMEOUT" = "30" ] || { echo "FAIL: CLI_TIMEOUT" >&2; exit 1; } +[ "$CLI_WARMUP" = "2" ] || { echo "FAIL: CLI_WARMUP" >&2; exit 1; } +[ "$CLI_WHITELIST" = "/tmp/wl" ] || { echo "FAIL: CLI_WHITELIST" >&2; exit 1; } +[ "$CLI_CONFIG" = "/tmp/cfg" ] || { echo "FAIL: CLI_CONFIG" >&2; exit 1; } +[ "$LIST_ONLY" = "1" ] || { echo "FAIL: LIST_ONLY" >&2; exit 1; } + +# unknown arg returns 2 (does not exit the sourced shell) +reset_cli +set +e +parse_args --bogus 2>/dev/null +rc=$? +set -e +[ "$rc" = "2" ] || { echo "FAIL: unknown arg rc=$rc" >&2; exit 1; } + +echo "args parser regression: PASS" +``` + +- [ ] **Step 2: 跑测试确认失败** + +Run: +```sh +cd user/apps/tests/benchmark/lmbench +busybox sh runner/test_cases/test_args_parser.sh +``` +Expected: FAIL（`parse_args: not found` 或类似），因为 `run.sh` 还没定义 `parse_args`。 + +- [ ] **Step 3: 在 run.sh 实现 usage / parse_args / apply_cli_overrides** + +在 `runner/run.sh` 的 `log()` 函数定义之后、`kv_get()` 之前，插入： +```sh +usage() { + cat >&2 <<'EOF' +usage: run.sh [--samples N] [--timeout S] [--warmup N] + [--whitelist FILE] [--config FILE] [--only NAME] [--list] +EOF +} + +# Parse CLI args into CLI_* intent variables (applied after load_config). +# Returns 2 on unknown arg (caller exits); does not exit the shell itself +# so it can be unit-tested via `LMBENCH_RUNNER_NO_MAIN=1 . run.sh`. +parse_args() { + while [ $# -gt 0 ]; do + case "$1" in + --samples) CLI_SAMPLES=$2; shift 2 ;; + --timeout) CLI_TIMEOUT=$2; shift 2 ;; + --warmup) CLI_WARMUP=$2; shift 2 ;; + --whitelist) CLI_WHITELIST=$2; shift 2 ;; + --config) CLI_CONFIG=$2; shift 2 ;; + --only) ONLY_NAME=$2; shift 2 ;; + --list) LIST_ONLY=1; shift ;; + -h|--help) usage; return 0 ;; + *) usage; return 2 ;; + esac + done +} + +# Apply CLI overrides on top of config-file values. +apply_cli_overrides() { + [ -n "$CLI_SAMPLES" ] && SAMPLES=$CLI_SAMPLES + [ -n "$CLI_TIMEOUT" ] && TIMEOUT_SEC=$CLI_TIMEOUT + [ -n "$CLI_WARMUP" ] && WARMUP=$CLI_WARMUP + [ -n "$CLI_WHITELIST" ] && WHITELIST_FILE=$CLI_WHITELIST +} +``` + +- [ ] **Step 4: 改 run_main 支持 --config / --only / --list** + +把 `runner/run.sh` 末尾的 `run_main()` 改为： +```sh +run_main() { + parse_args "$@" || exit $? + + # --config may redirect the config file before load_config reads it. + [ -n "$CLI_CONFIG" ] && CONFIG_FILE=$CLI_CONFIG + load_config + apply_cli_overrides + mkdir -p "$WORK_TMP" + + if [ "$LIST_ONLY" = "1" ]; then + ls "$CASES_DIR"/*.sh 2>/dev/null | sed 's|.*/||; s|\.sh$||' + return 0 + fi + + # --only: replace whitelist with a single-line file, reusing run_one_case + # unchanged (no second dispatch branch). + if [ -n "$ONLY_NAME" ]; then + wl_tmp="$WORK_TMP/only_whitelist" + printf '%s\n' "$ONLY_NAME" > "$wl_tmp" + WHITELIST_FILE="$wl_tmp" + fi + + log "LMbench benchmark run starting" + echo "===LMBENCH_RUN_BEGIN===" + printf 'LMBENCH_META {"suite":"lmbench","suite_version":"%s","samples":%s,"timeout_sec":%s,"warmup":%s}\n' \ + "$SUITE_VERSION" "$SAMPLES" "$TIMEOUT_SEC" "$WARMUP" + + if [ -f "$SCRIPT_DIR/init.sh" ]; then + log "initializing test environment (init.sh)..." + if ! sh "$SCRIPT_DIR/init.sh"; then + log "ERROR: test environment initialization failed" + printf 'LMBENCH_SUMMARY {"total":0,"ok":0,"failed":0,"skipped":0}\n' + echo "===LMBENCH_RUN_END===" + echo "benchmark测试完成" + exit 1 + fi + fi + + if [ ! -f "$WHITELIST_FILE" ]; then + log "ERROR: whitelist not found: $WHITELIST_FILE" + echo "===LMBENCH_RUN_END===" + echo "benchmark测试完成" + exit 1 + fi + + total=0; ok=0; failed=0; skipped=0 + while read -r line; do + case "$line" in ''|\#*) continue ;; esac + total=$((total + 1)) + log "dispatching $line" + run_one_case "$line" + case $? in + 0) ok=$((ok + 1)) ;; + 2) skipped=$((skipped + 1)) ;; + *) failed=$((failed + 1)) ;; + esac + echo "---" + done < "$WHITELIST_FILE" + + printf 'LMBENCH_SUMMARY {"total":%s,"ok":%s,"failed":%s,"skipped":%s}\n' \ + "$total" "$ok" "$failed" "$skipped" + echo "===LMBENCH_RUN_END===" + + if [ -f "$SCRIPT_DIR/clean_up.sh" ]; then + sh "$SCRIPT_DIR/clean_up.sh" >/dev/null 2>&1 || true + fi + + log "done: total=$total ok=$ok failed=$failed skipped=$skipped" + echo "benchmark测试完成" +} +``` + +并在 `run.sh` 顶部全局 config 区（`WARMUP=0` 之后）加 CLI 意图变量初值： +```sh +# CLI intent variables (set by parse_args, applied after load_config). +CLI_SAMPLES=""; CLI_TIMEOUT=""; CLI_WARMUP="" +CLI_WHITELIST=""; CLI_CONFIG=""; ONLY_NAME=""; LIST_ONLY="" +``` + +- [ ] **Step 5: 跑参数解析测试确认通过** + +Run: +```sh +cd user/apps/tests/benchmark/lmbench +busybox sh runner/test_cases/test_args_parser.sh +``` +Expected: `args parser regression: PASS`，退出码 0。 + +- [ ] **Step 6: 跑 parser 回归测试确认未破坏** + +Run: +```sh +busybox sh runner/test_cases/test_tcp_loopback_lat_parser.sh +``` +Expected: `tcp_loopback_lat parser regression: PASS`。 + +- [ ] **Step 7: Commit** + +```sh +git add runner/run.sh runner/test_cases/test_args_parser.sh +git commit -m "feat(lmbench): parameterize runner entrypoint with parse_args" +``` + +--- + +### Task 3: host 编排移入 orchestrator/ + +**Files:** +- Move: `monitor_test_results.sh` / `collect_results.py` → `orchestrator/` +- Move: `schema/` → `orchestrator/schema/` +- Modify: `orchestrator/collect_results.py`（`--outdir` 默认 + docstring） +- Modify: `Makefile:325,327`（根 Makefile，加 `orchestrator/` 前缀） + +**Interfaces:** +- Produces: `orchestrator/collect_results.py` 的 `--outdir` 默认指向 `lmbench/results/`（即 `HERE/../results`）；`--schema` 默认 `HERE/schema/...` 不变（schema 跟随移动）。 + +- [ ] **Step 1: 移动 host 编排文件** + +```sh +cd user/apps/tests/benchmark/lmbench +mkdir -p orchestrator +git mv monitor_test_results.sh orchestrator/monitor_test_results.sh +git mv collect_results.py orchestrator/collect_results.py +git mv schema orchestrator/schema +``` + +- [ ] **Step 2: 改 collect_results.py 的 outdir 默认值与 docstring** + +编辑 `orchestrator/collect_results.py`： + +第 4 行 docstring `The guest runner (run_tests.sh) prints` → `The guest runner (run.sh) prints`。 + +第 269 行： +```python + ap.add_argument("--outdir", default=os.path.join(HERE, "results")) +``` +改为： +```python + ap.add_argument("--outdir", default=os.path.join(HERE, "..", "results")) +``` +（`--schema` 默认 `os.path.join(HERE, "schema", "lmbench-run.schema.json")` 不用改——schema 已跟随移到 `orchestrator/schema/`。） + +- [ ] **Step 3: 改根 Makefile 的 monitor/collect 路径** + +编辑 `Makefile` 第 325、327 行： +```makefile + bash user/apps/tests/benchmark/lmbench/monitor_test_results.sh || status=$$?; \ + if [ $$status -eq 0 ]; then \ + python3 user/apps/tests/benchmark/lmbench/collect_results.py || status=$$?; \ +``` +改为： +```makefile + bash user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh || status=$$?; \ + if [ $$status -eq 0 ]; then \ + python3 user/apps/tests/benchmark/lmbench/orchestrator/collect_results.py || status=$$?; \ +``` +（第 312、315 行的 `toggle_compile_lmbench.sh` 路径不变——toggle 留根。） + +- [ ] **Step 4: 语法检查 collect_results.py** + +Run: +```sh +cd user/apps/tests/benchmark/lmbench +python3 -c "import ast; ast.parse(open('orchestrator/collect_results.py').read()); print('syntax ok')" +``` +Expected: `syntax ok`。 + +- [ ] **Step 5: 静态确认无残留旧 host 路径** + +Run: +```sh +grep -rn "lmbench/monitor_test_results\|lmbench/collect_results\|lmbench/schema" Makefile .github/ \ + || true +``` +Expected: 零命中（已全部改为 `orchestrator/` 前缀）。 + +- [ ] **Step 6: Commit** + +```sh +git add -A user/apps/tests/benchmark/lmbench Makefile +git commit -m "refactor(lmbench): move host orchestration into orchestrator/" +``` + +--- + +### Task 4: 打包文件更新（Makefile / default.nix / flake.nix） + +**Files:** +- Modify: `user/apps/tests/benchmark/lmbench/Makefile`（install 从 `runner/` 拷，删 `run_test_case.sh`） +- Modify: `user/apps/tests/benchmark/lmbench/default.nix`（sourceByRegex + installPhase + 补装 config/whitelist + run.sh） +- Modify: `user/apps/tests/benchmark/lmbench/flake.nix:41`（`run_tests.sh` → `run.sh`） + +**Interfaces:** +- Produces: DADK 与 Nix 两条 install 路径都摊平到 `/opt/tests/benchmark/lmbench/`，产物含 `run.sh`/`init.sh`/`env.sh`/`clean_up.sh`/`test_cases/`/`config`/`whitelist.txt`（扁平）。 + +- [ ] **Step 1: 改 lmbench/Makefile 的 install 目标** + +把 `user/apps/tests/benchmark/lmbench/Makefile` 第 11-30 行： +```makefile +# Files needed inside the guest at runtime. +GUEST_SCRIPTS = run_tests.sh init.sh env.sh clean_up.sh run_test_case.sh +GUEST_DATA = whitelist.txt config +``` +改为： +```makefile +# Files needed inside the guest at runtime. Sources live under runner/; +# `install` flattens them into INSTALL_DIR (guest runtime layout is flat). +RUNNER_DIR = runner +GUEST_SCRIPTS = $(RUNNER_DIR)/run.sh $(RUNNER_DIR)/init.sh $(RUNNER_DIR)/env.sh $(RUNNER_DIR)/clean_up.sh +GUEST_DATA = whitelist.txt config +``` + +并把 install 目标： +```makefile +install: + @echo "[lmbench] installing to $(INSTALL_DIR)" + @mkdir -p $(INSTALL_DIR) + @install -m755 $(GUEST_SCRIPTS) $(INSTALL_DIR)/ + @install -m644 $(GUEST_DATA) $(INSTALL_DIR)/ + @cp -r test_cases $(INSTALL_DIR)/ + @chmod +x $(INSTALL_DIR)/test_cases/*.sh 2>/dev/null || true + @echo "[lmbench] install done" +``` +改为： +```makefile +install: + @echo "[lmbench] installing to $(INSTALL_DIR)" + @mkdir -p $(INSTALL_DIR) + @install -m755 $(GUEST_SCRIPTS) $(INSTALL_DIR)/ + @install -m644 $(GUEST_DATA) $(INSTALL_DIR)/ + @cp -r $(RUNNER_DIR)/test_cases $(INSTALL_DIR)/ + @chmod +x $(INSTALL_DIR)/test_cases/*.sh 2>/dev/null || true + @echo "[lmbench] install done" +``` + +- [ ] **Step 2: 改 default.nix 的 sourceByRegex + installPhase** + +把 `default.nix` 第 17-28 行： +```nix + src = lib.sourceByRegex ./. [ + "^test_cases" + "^.*\.sh$" + ]; + + installPhase = '' + mkdir -p $out/${installDir} + + install -m755 *.sh $out/${installDir}/ + cp -r test_cases $out/${installDir}/ + chmod +x $out/${installDir}/test_cases/*.sh + ''; +``` +改为： +```nix + src = lib.sourceByRegex ./. [ + "^runner" + "^runner/.*\.sh$" + "^runner/test_cases" + "^runner/test_cases/.*" + "^config$" + "^whitelist\.txt$" + ]; + + installPhase = '' + mkdir -p $out/${installDir} + + install -m755 runner/*.sh $out/${installDir}/ + install -m644 config whitelist.txt $out/${installDir}/ + cp -r runner/test_cases $out/${installDir}/ + chmod +x $out/${installDir}/test_cases/*.sh + ''; +``` +（顺手修两个既有问题：原 `^.*\.sh$` 会误装 host 脚本进 nix 输出；原 installPhase 漏装 `config`/`whitelist.txt`。） + +- [ ] **Step 3: 改 flake.nix 的 program 路径** + +把 `flake.nix` 第 41 行： +```nix + program = "${lmbench}/${defaultInstallDir}/run_tests.sh"; +``` +改为： +```nix + program = "${lmbench}/${defaultInstallDir}/run.sh"; +``` + +- [ ] **Step 4: 验证 DADK Makefile install 摊平布局** + +Run: +```sh +cd user/apps/tests/benchmark/lmbench +rm -rf /tmp/lmbench-install-test +make install INSTALL_DIR=/tmp/lmbench-install-test +ls /tmp/lmbench-install-test +ls /tmp/lmbench-install-test/test_cases | head +``` +Expected: `/tmp/lmbench-install-test` 下扁平出现 `run.sh` `init.sh` `env.sh` `clean_up.sh` `config` `whitelist.txt` `test_cases/`；`test_cases/` 下有各 `.sh`/`.meta`。**不应**出现 `run_tests.sh`、`run_test_case.sh`、`runner/` 子目录。 + +- [ ] **Step 5: 清理测试产物** + +```sh +rm -rf /tmp/lmbench-install-test user/apps/tests/benchmark/lmbench/install +``` + +- [ ] **Step 6: Commit** + +```sh +git add user/apps/tests/benchmark/lmbench/Makefile user/apps/tests/benchmark/lmbench/default.nix user/apps/tests/benchmark/lmbench/flake.nix +git commit -m "build(lmbench): install from runner/ and fix nix sourceByRegex" +``` + +--- + +### Task 5: 外部调用点 + 文档 + +**Files:** +- Modify: `user/sysconfig/etc/init.d/rcS:52`（`run_tests.sh` → `run.sh`） +- Modify: `user/apps/tests/benchmark/lmbench/README.md`（目录结构 + 路径 + 参数说明 + 删 `run_test_case.sh`） +- Modify: `orchestrator/collect_results.py` docstring（已在 Task 3 改，此处仅核对） +- Modify: `user/apps/tests/benchmark/lmbench/Makefile` 注释（`run_tests.sh` → `run.sh` 若有） + +**Interfaces:** +- Produces: rcS 调 `sh $BENCHMARK_TEST_DIR/run.sh`；README 反映新布局与 `run.sh` 参数面。 + +- [ ] **Step 1: 改 rcS 的 runner 入口名** + +编辑 `user/sysconfig/etc/init.d/rcS` 第 52 行： +```sh + /bin/busybox sh $BENCHMARK_TEST_DIR/run_tests.sh +``` +改为： +```sh + /bin/busybox sh $BENCHMARK_TEST_DIR/run.sh +``` + +- [ ] **Step 2: 更新 README.md 目录结构图** + +把 `README.md` 的 `## 目录结构` 代码块替换为： +``` +``` +runner/ guest 端 runner（AUTO_TEST=benchmark 入口） + run.sh 参数化入口(原 run_tests.sh) + init.sh / env.sh / clean_up.sh 测试环境准备/变量/清理 + test_cases/.sh 测例包装脚本(调用 lmbench 二进制) + test_cases/.meta 该测例的抽取规则 + 单位 + 方向元数据 + test_cases/test_*_parser.sh host 侧回归测试(busybox sh,source run.sh) +orchestrator/ host 端编排 + monitor_test_results.sh 串口监控 + 超时/完成判定 + collect_results.py 解析串口 → 校验 → 持久化 + schema/lmbench-run.schema.json 结果 JSON Schema(draft-07) +config 全局配置(SAMPLES / TIMEOUT_SEC / WARMUP) +whitelist.txt 要运行的测例清单(每行一个) +toggle_compile_lmbench.sh 切换是否把本套件编进 rootfs +Makefile DADK build-from-source 的安装脚本 +default.nix / flake.nix Nix 打包(可选) +results/ 运行产物(gitignore;作为 CI 制品归档) + /-.json 每次运行的完整快照(canonical) + history.jsonl 每条指标一行(时序友好) + github-benchmark/data.json github-action-benchmark 兼容格式(为可视化预留) +``` +``` + +- [ ] **Step 3: 更新 README.md 快速开始里的脚本名** + +把 `## 快速开始` 区域里所有 `run_tests.sh` 引用改为 `run.sh`（如架构图里的 `run_tests.sh` → `run.sh`）。 + +- [ ] **Step 4: 在 README.md 添加 run.sh 参数说明** + +在 `## 添加/启用一个测例` 之前插入新章节： +```markdown +## runner 入口参数 + +`runner/run.sh` 支持以下参数（无参数 = 读 `config` + `whitelist.txt` 全跑，与 rcS 调用一致）： + +``` +run.sh [--samples N] [--timeout S] [--warmup N] + [--whitelist FILE] [--config FILE] [--only NAME] [--list] +``` + +- `--samples N` / `--timeout S` / `--warmup N`：覆盖 `config` 的对应项。 +- `--whitelist FILE` / `--config FILE`：覆盖默认路径（`$SCRIPT_DIR/whitelist.txt` / `$SCRIPT_DIR/config`）。 +- `--only NAME`：只跑单个测例（用于手动调试），等价于临时 whitelist 只含 NAME。 +- `--list`：列出 `test_cases/*.sh` 可用测例名，不跑。 + +优先级：命令行参数 > `--config` 指定文件 > 内置默认。 +``` + +- [ ] **Step 5: 从 README 删除 run_test_case.sh 条目** + +`README.md` 原 `## 目录结构` 里有 `run_test_case.sh 手动单测例运行(init→run→cleanup)` 行——已在 Step 2 整块替换时移除。确认 README 全文无 `run_test_case` 残留： +```sh +grep -n "run_test_case" user/apps/tests/benchmark/lmbench/README.md || true +``` +Expected: 零命中。 + +- [ ] **Step 6: 全仓静态扫描门禁** + +Run: +```sh +cd /home/yuming/project_ongoing/DragonOSDev/DragonOS/.worktrees/feat-lmbench +grep -rn "run_tests\.sh\|run_test_case" . \ + --exclude-dir=.git --exclude-dir=results --exclude-dir=__pycache__ --exclude-dir=target \ + --exclude='*.json' || true +``` +Expected: 命中只在 `docs/superpowers/specs/` 与 `docs/superpowers/plans/`（设计/计划文档引用旧名是历史记录，可接受）以及 `results/` 之外无源码/配置/CI 命中。若有源码命中，回到对应 Task 修复。 + +- [ ] **Step 7: Commit** + +```sh +git add user/sysconfig/etc/init.d/rcS user/apps/tests/benchmark/lmbench/README.md +git commit -m "docs(lmbench): update rcS entrypoint name and README for new layout" +``` + +--- + +### Task 6: 构建冒烟验证 + +**Files:** 无（仅验证） + +- [ ] **Step 1: 跑两个 host 回归测试** + +Run: +```sh +cd user/apps/tests/benchmark/lmbench +busybox sh runner/test_cases/test_tcp_loopback_lat_parser.sh +busybox sh runner/test_cases/test_args_parser.sh +``` +Expected: 两个都 PASS，退出码 0。 + +- [ ] **Step 2: DADK 构建冒烟（不覆盖 disk image）** + +Run: +```sh +cd /home/yuming/project_ongoing/DragonOSDev/DragonOS/.worktrees/feat-lmbench +make all -j$(nproc) +``` +Expected: 构建成功。验证 sysroot 下扁平布局： +```sh +find bin -path '*opt/tests/benchmark/lmbench*' -maxdepth 8 2>/dev/null | head -20 +``` +Expected: 出现 `.../opt/tests/benchmark/lmbench/run.sh`、`.../init.sh`、`.../test_cases/`、`.../config`、`.../whitelist.txt`，且**无** `run_tests.sh`、`run_test_case.sh`、`runner/` 子目录、`monitor_test_results.sh`、`collect_results.py`（后两者不装进 rootfs）。 + +- [ ] **Step 3: 记录验证结果到 commit（可选）** + +若一切通过，无需额外 commit（本任务是验证）。若发现缺失，回到对应 Task 修复后再跑。 + +--- + +### Task 7: 端到端验证（需用户授权覆盖 disk image） + +**Files:** 无（仅验证） + +**授权要求：** 本任务执行 `make test-benchmark`，会覆盖 `bin/disk-image-x86_64.img`。需用户明确授权该确切路径。用户已表示"请求的话，我就通过"——执行前向用户确认一次即可。 + +- [ ] **Step 1: 请求用户授权覆盖 bin/disk-image-x86_64.img** + +向用户确认："`make test-benchmark` 会覆盖 `bin/disk-image-x86_64.img`，是否授权执行？"得到肯定答复后继续。 + +- [ ] **Step 2: 跑 make test-benchmark** + +Run: +```sh +cd /home/yuming/project_ongoing/DragonOSDev/DragonOS/.worktrees/feat-lmbench +make test-benchmark +``` +Expected: 8 个指标全部 `ok`，`LMBENCH_SUMMARY {"total":8,"ok":8,"failed":0,"skipped":0}`，`[collect] schema validation: PASS`，退出码 0。 + +- [ ] **Step 3: 确认产物落盘** + +Run: +```sh +ls -t user/apps/tests/benchmark/lmbench/results/x86_64/*.json | head -1 +tail -n 8 user/apps/tests/benchmark/lmbench/results/history.jsonl +``` +Expected: 新 run 的 canonical snapshot 已写入；history.jsonl 末尾有 8 条新指标行。 + +- [ ] **Step 4: 确认 blocklist 已恢复** + +Run: +```sh +grep -n "lmbench benchmark tests" config/app-blocklist.toml +``` +Expected: 该行存在且未被注释（`make test-benchmark` 的 trap cleanup 已 disable）。 + +--- + +## Self-Review + +**1. Spec coverage：** +- spec §目标1（分层）→ Task 1（runner/）+ Task 3（orchestrator/）+ Task 5（rcS/README）。 +- spec §目标2（参数化入口）→ Task 2。 +- spec §目标3（修 default.nix 误装/漏装）→ Task 4 Step 2。 +- spec §目标4（外部行为零变化）→ Task 6 + Task 7 验证；Task 3/4 保持 contract/调用方式/CI 路径不变。 +- spec §硬约束（Makefile/nix/toggle 留根）→ Task 1 只移动 guest 脚本，Task 3 只移动 host 编排，Makefile/nix/toggle 未移动。 +- spec §guest 运行时扁平 → Task 4 Step 4 验证摊平布局。 +- spec §删除 run_test_case.sh → Task 1 Step 1 `git rm` + Task 5 Step 5 确认 README 无残留。 +- spec §验证（静态扫描/回归/构建冒烟/e2e）→ Task 5 Step 6 + Task 6 + Task 7。 +- spec §风险缓解（--only 不走新分支）→ Task 2 Step 4 用临时单行 whitelist 文件复用 run_one_case。 +- spec §不做（YAGNI）→ 计划未包含拆 lib / host 参数化 / CI 门控 / 扩 48 测例。✓ + +**2. Placeholder scan：** 无 TBD/TODO；每个 step 有具体命令或代码块。✓ + +**3. Type consistency：** `parse_args` 在 Task 2 定义、Task 5/6 不再调用；`CLI_SAMPLES`/`ONLY_NAME`/`LIST_ONLY` 在 Task 2 定义并被 test_args_parser.sh 断言；`runner/run.sh` 路径在 Task 1 产出，Task 2/4/5 引用一致；`orchestrator/` 路径在 Task 3 产出，根 Makefile 引用一致。✓ + +## Execution Handoff + +用户已授权写完计划直接执行，不请求审批。采用 Inline Execution（任务间路径耦合强、机械迁移为主，单 context 持续跟踪已改路径比 fresh subagent 更可靠），直接按 Task 1→7 顺序执行。Task 7 执行前需就 `bin/disk-image-x86_64.img` 覆盖向用户确认一次（用户已预告会通过）。 \ No newline at end of file diff --git a/docs/superpowers/plans/2026-07-30-lmbench-fullsuite-meta-extraction.md b/docs/superpowers/plans/2026-07-30-lmbench-fullsuite-meta-extraction.md new file mode 100644 index 0000000000..25dac71276 --- /dev/null +++ b/docs/superpowers/plans/2026-07-30-lmbench-fullsuite-meta-extraction.md @@ -0,0 +1,590 @@ +# LMbench 全量测例 .meta 提取 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 在 host 用 Docker ubuntu:24.04 容器跑 guest 同款 lmbench archive 二进制，为 48 个测例提取/复核 `.meta`，扩 whitelist 全量，产出探查报告。 + +**Architecture:** 一次性手操探查——下载 lmbench archive 解压，docker 24.04 容器内直接跑 lmbench 二进制（参数参考各 `.sh`），按"延迟类/带宽类/ops 表类"三类映射规则把输出格式写成 `.meta`。不启 QEMU、不改任何 guest 文件（`env.sh`/`.sh`/`init.sh`）、不做 QEMU 验证。 + +**Tech Stack:** Docker ubuntu:24.04 容器、lmbench 3.0-a9 预编译二进制（Ubuntu 24.04 glibc 2.39）、BusyBox `sh`（host 侧 `.meta` 抽取验证用 `runner/run.sh` 的 `extract_value`）。 + +## Global Constraints + +- host glibc 2.35 跑不了 24.04 二进制 → 必须用 `docker run ubuntu:24.04` 容器。 +- lmbench 二进制来源：`user/dadk/config/all/lmbench_bin_ubuntu2404.toml` 的 `source-path`（`https://mirrors.dragonos.org.cn/pub/third_party/lmbench/lmbench-ubuntu2404-202511301014-36c2fb2d084343e098b5e343576d4fc0.tar.xz`），解压后含 `lib/lmbench/bin/x86_64-linux-gnu/*` + `lib/x86_64-linux-gnu/*` glibc 2.39。 +- `env.sh`/`.sh`/`init.sh` 一律不改（`git diff` 最终无 guest 文件改动）。 +- QEMU 验证不纳入本轮。 +- 代码注释用英文；commit 用 `feat(lmbench):` / `docs(lmbench):` 前缀。 +- `.meta` 路径：`user/apps/tests/benchmark/lmbench/runner/test_cases/.meta`。 + +--- + +## File Structure + +- **Create/Modify**: `runner/test_cases/.meta` × 40 个新写 + 8 个复核（`mem_copy_bw`/`process_fork_lat`/`pipe_lat`/`unix_lat`/`vfs_open_lat`/`tcp_loopback_lat`/`signal_catch_lat`/`ext4_create_delete_files_10k_ops`）。 +- **Modify**: `whitelist.txt`（扩到全量 48）。 +- **Create**: `docs/superpowers/specs/2026-07-30-lmbench-fullsuite-survey.md`（探查报告）。 +- **不改**: `env.sh` / `*.sh` / `init.sh` / `run.sh` / `collect_results.py`。 + +### `.meta` 通用模板 + +``` +# Extraction/metadata for (KEY=VALUE; parsed by run.sh) +CATEGORY= +BINARY= +METRIC_TYPE=latency|bandwidth|ops +UNIT=microseconds|MB/s|ops/sec +BIGGER_IS_BETTER=0|1 +SEARCH_PATTERN=<锚定词，跑后从输出填> +RESULT_INDEX=NF-1|NF|<数字> +NTH_OCCURRENCE=1 +DESCRIPTION=<一句描述> +``` + +### 映射规则（三类） + +- **延迟类**（输出 `<描述>: microseconds`）：`SEARCH_PATTERN=<描述特征词>`, `RESULT_INDEX=NF-1`, `BIGGER_IS_BETTER=0`, `UNIT=microseconds`, `METRIC_TYPE=latency`。 +- **带宽类**（输出 ` ` 或 `... MB/sec`）：`SEARCH_PATTERN=^[0-9]` 或锚定词, `RESULT_INDEX=NF`, `BIGGER_IS_BETTER=1`, `UNIT=MB/s`, `METRIC_TYPE=bandwidth`。 +- **ops/表类**（`lat_fs` 多 size 表）：`SEARCH_PATTERN=^`, `RESULT_INDEX=<列号>`, `BIGGER_IS_BETTER=1`, `UNIT=ops/sec`, `METRIC_TYPE=ops`。 + +### banner 规避 + +`SEARCH_PATTERN` 锚定 lmbench 二进制输出的特征词（如 `Pipe latency`/`Signal handler`/`open/close`），不要用可能撞 `.sh` banner `=== Running XXX ===` 的宽泛词。TCP 那个 bug 的教训：原 `TCP latency` 撞 banner `=== Running TCP latency test ===` 取了 `test`，修成 `^TCP latency using`。 + +### host 侧 `.meta` 抽取验证（每组 Task 用） + +`runner/run.sh` 支持 `LMBENCH_RUNNER_NO_MAIN=1` source 复用 `kv_get`/`extract_value`。验证一个 `.meta` 抽取正确： +```sh +LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh +META=runner/test_cases/.meta +PAT=$(kv_get "$META" SEARCH_PATTERN) +IDX=$(kv_get "$META" RESULT_INDEX) +NTH=$(kv_get "$META" NTH_OCCURRENCE) +# 用容器观察到的真实输出行构造 tmp 文件，extract_value 应返回数值 +printf '<真实输出行>\n' > /tmp/probe.out +extract_value /tmp/probe.out # 应打印数值，非空 +``` + +--- + +### Task 1: 环境准备——下载 archive + 启动 24.04 容器 + 验证一个二进制能跑 + +**Files:** 无（仅环境准备） + +- [ ] **Step 1: 下载 lmbench archive** + +```sh +mkdir -p /tmp/lmbench-survey +cd /tmp/lmbench-survey +curl -fL -o lmbench.tar.xz "https://mirrors.dragonos.org.cn/pub/third_party/lmbench/lmbench-ubuntu2404-202511301014-36c2fb2d084343e098b5e343576d4fc0.tar.xz" +tar xf lmbench.tar.xz +ls lib/lmbench/bin/x86_64-linux-gnu/ | head +``` +Expected: 解压后 `lib/lmbench/bin/x86_64-linux-gnu/` 下有 `lat_fifo`/`lat_pipe`/`bw_mem` 等二进制。 + +- [ ] **Step 2: 启动 24.04 容器并验证一个二进制能跑** + +```sh +docker run --rm -v /tmp/lmbench-survey:/srv -it ubuntu:24.04 bash +# 容器内： +/srv/lib/lmbench/bin/x86_64-linux-gnu/lat_pipe -P 1 +``` +Expected: 打印类似 `Pipe latency: microseconds`（证明 glibc 2.39 兼容、二进制可跑）。记录输出格式。 + +- [ ] **Step 3: 在容器内建等价环境变量与 dummy 文件** + +容器内执行（后续 Task 复用）： +```sh +export BIN=/srv/lib/lmbench/bin/x86_64-linux-gnu +export WORK=/srv/work +mkdir -p $WORK +dd if=/dev/zero of=$WORK/test_file bs=1M count=512 2>/dev/null +dd if=/dev/zero of=$WORK/zero_file bs=1M count=512 2>/dev/null +``` +Expected: `$WORK/test_file`、`$WORK/zero_file` 存在。 + +- [ ] **Step 4: 保持容器开启，进入 Task 2** + +容器保持运行（用 `-it` 交互）。后续 Task 在同一容器内跑。若退出需重启：`docker run --rm -v /tmp/lmbench-survey:/srv -it ubuntu:24.04 bash` + 重做 Step 3 的 export/建文件。 + +--- + +### Task 2: memory 组提取 .meta + +**测例**（6）：`mem_copy_bw`（已有，复核）、`mem_read_bw`、`mem_write_bw`、`mem_mmap_bw`、`mem_mmap_lat`、`mem_pagefault_lat`。 + +**lmbench 调用**（容器内，`$BIN` 已 export）： +| 测例 | 容器命令 | +|---|---| +| mem_copy_bw | `$BIN/bw_mem -P 1 -N 3 64m fcp` | +| mem_read_bw | `$BIN/bw_mem -P 1 -N 50 512m frd` | +| mem_write_bw | `$BIN/bw_mem -P 1 -N 50 512m fwr` | +| mem_mmap_bw | `$BIN/bw_mmap_rd -W 30 -N 300 256m mmap_only $WORK/test_file` | +| mem_mmap_lat | `sudo $BIN/lat_mmap 4m $WORK/test_file`（容器内可能无 sudo，改 `$BIN/lat_mmap 4m $WORK/test_file`） | +| mem_pagefault_lat | `$BIN/lat_pagefault -P 1 $WORK/test_file` | + +- [ ] **Step 1: 逐个跑并记录输出** + +容器内依次跑上表命令，记录每个的输出行格式。`bw_mem`/`bw_mmap_rd` 属带宽类（` `），`lat_mmap`/`lat_pagefault` 属延迟类（`...: microseconds`）。 + +- [ ] **Step 2: 写/复核 6 个 .meta** + +按映射规则写 `runner/test_cases/.meta`。带宽类 `RESULT_INDEX=NF`，延迟类 `RESULT_INDEX=NF-1`，`SEARCH_PATTERN` 用观察到的特征词。`mem_copy_bw.meta` 复核现有规则与容器输出一致。 + +- [ ] **Step 3: host 侧抽样验证抽取** + +退出容器或新开 host shell： +```sh +cd /home/yuming/project_ongoing/DragonOSDev/DragonOS/.worktrees/feat-lmbench +LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh +for m in mem_read_bw mem_mmap_lat mem_pagefault_lat; do + PAT=$(kv_get runner/test_cases/$m.meta SEARCH_PATTERN) + IDX=$(kv_get runner/test_cases/$m.meta RESULT_INDEX) + NTH=$(kv_get runner/test_cases/$m.meta NTH_OCCURRENCE) + printf '<该测例容器输出行>\n' > /tmp/probe.out + printf '%s -> %s\n' "$m" "$(extract_value /tmp/probe.out)" +done +``` +Expected: 每个 `extract_value` 打印数值（非空），证明 `.meta` 规则正确。 + +- [ ] **Step 4: Commit** + +```sh +git add runner/test_cases/mem_read_bw.meta runner/test_cases/mem_write_bw.meta runner/test_cases/mem_mmap_bw.meta runner/test_cases/mem_mmap_lat.meta runner/test_cases/mem_pagefault_lat.meta +# mem_copy_bw.meta 若复核有改动也 add +git commit -m "feat(lmbench): add .meta for memory bandwidth/latency tests" +``` + +--- + +### Task 3: process 组提取 .meta + +**测例**（5）：`process_fork_lat`（已有，复核）、`process_exec_lat`、`process_ctx_lat`、`process_getppid_lat`、`process_shell_lat`。 + +**lmbench 调用**： +| 测例 | 容器命令 | +|---|---| +| process_fork_lat | `$BIN/lat_proc -P 1 fork` | +| process_exec_lat | `$BIN/lat_proc -P 1 exec` | +| process_shell_lat | `$BIN/lat_proc -P 1 shell` | +| process_ctx_lat | `$BIN/lat_ctx -P 1 18` | +| process_getppid_lat | `$BIN/lat_syscall -P 1 null` | + +- [ ] **Step 1: 逐个跑并记录输出** + +`lat_proc fork/exec/shell` 输出 `... fork/exec/shell latency: microseconds`（延迟类）。`lat_ctx` 输出 `Context switch latency: ...`（延迟类）。`lat_syscall null` 输出 `Null syscall latency: ...`（延迟类，但测例名是 getppid——记录实际输出，DESCRIPTION 按实际写）。 + +- [ ] **Step 2: 写/复核 5 个 .meta** + +全部延迟类：`RESULT_INDEX=NF-1`，`SEARCH_PATTERN` 锚定输出特征词（如 `fork latency`/`exec latency`/`shell latency`/`Context switch`/`Null syscall`）。 + +- [ ] **Step 3: host 侧抽样验证** + +```sh +LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh +for m in process_exec_lat process_ctx_lat process_getppid_lat; do + PAT=$(kv_get runner/test_cases/$m.meta SEARCH_PATTERN) + IDX=$(kv_get runner/test_cases/$m.meta RESULT_INDEX) + NTH=$(kv_get runner/test_cases/$m.meta NTH_OCCURRENCE) + printf '<容器输出行>\n' > /tmp/probe.out + printf '%s -> %s\n' "$m" "$(extract_value /tmp/probe.out)" +done +``` +Expected: 每个打印数值。 + +- [ ] **Step 4: Commit** + +```sh +git add runner/test_cases/process_exec_lat.meta runner/test_cases/process_ctx_lat.meta runner/test_cases/process_getppid_lat.meta runner/test_cases/process_shell_lat.meta +git commit -m "feat(lmbench): add .meta for process latency tests" +``` + +--- + +### Task 4: ipc 组提取 .meta + +**测例**（7）：`pipe_lat`（已有，复核）、`pipe_bw`、`fifo_lat`、`unix_lat`（已有，复核）、`unix_bw`、`unix_connect_lat`、`semaphore_lat`。 + +**lmbench 调用**： +| 测例 | 容器命令 | +|---|---| +| pipe_lat | `$BIN/lat_pipe -P 1` | +| pipe_bw | `$BIN/bw_pipe -P 1` | +| fifo_lat | `$BIN/lat_fifo -P 1` | +| unix_lat | `$BIN/lat_unix -P 1` | +| unix_bw | `$BIN/bw_unix -P 1` | +| unix_connect_lat | `$BIN/lat_unix_connect -s &`（server 后台），然后 `$BIN/lat_unix_connect 127.0.0.1`（client；实际参数跑后确认） | +| semaphore_lat | `$BIN/lat_sem -P 1 -N 21` | + +- [ ] **Step 1: 逐个跑并记录输出** + +`lat_pipe`/`lat_fifo`/`lat_unix`/`lat_sem` 延迟类（`...: microseconds`）。`bw_pipe`/`bw_unix` 带宽类（` `）。`lat_unix_connect` 需起 server 后台再跑 client，记录 client 输出。 + +- [ ] **Step 2: 写/复核 7 个 .meta** + +延迟类 `RESULT_INDEX=NF-1`，带宽类 `RESULT_INDEX=NF`。`SEARCH_PATTERN` 锚定特征词（如 `Pipe latency`/`FIFO latency`/`Unix socket latency`/`Semaphore latency`）。复核 `pipe_lat.meta`/`unix_lat.meta`。 + +- [ ] **Step 3: host 侧抽样验证** + +```sh +LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh +for m in pipe_bw fifo_lat unix_bw unix_connect_lat semaphore_lat; do + PAT=$(kv_get runner/test_cases/$m.meta SEARCH_PATTERN) + IDX=$(kv_get runner/test_cases/$m.meta RESULT_INDEX) + NTH=$(kv_get runner/test_cases/$m.meta NTH_OCCURRENCE) + printf '<容器输出行>\n' > /tmp/probe.out + printf '%s -> %s\n' "$m" "$(extract_value /tmp/probe.out)" +done +``` +Expected: 每个打印数值。 + +- [ ] **Step 4: Commit** + +```sh +git add runner/test_cases/pipe_bw.meta runner/test_cases/fifo_lat.meta runner/test_cases/unix_bw.meta runner/test_cases/unix_connect_lat.meta runner/test_cases/semaphore_lat.meta +git commit -m "feat(lmbench): add .meta for ipc tests" +``` + +--- + +### Task 5: signal 组提取 .meta + +**测例**（3）：`signal_catch_lat`（已有，复核）、`signal_install_lat`、`signal_prot_lat`。 + +**lmbench 调用**： +| 测例 | 容器命令 | +|---|---| +| signal_catch_lat | `$BIN/lat_sig -P 1 catch` | +| signal_install_lat | `$BIN/lat_sig -P 1 install` | +| signal_prot_lat | `$BIN/lat_sig -W 30 -N 300 prot $WORK/test_file` | + +- [ ] **Step 1: 逐个跑并记录输出** + +`lat_sig catch/install/prot` 延迟类（`Signal handler overhead: microseconds` 或类似）。记录各模式实际输出特征词。 + +- [ ] **Step 2: 写/复核 3 个 .meta** + +延迟类 `RESULT_INDEX=NF-1`，`SEARCH_PATTERN` 锚定各模式输出特征词。复核 `signal_catch_lat.meta`。 + +- [ ] **Step 3: host 侧抽样验证** + +```sh +LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh +for m in signal_install_lat signal_prot_lat; do + PAT=$(kv_get runner/test_cases/$m.meta SEARCH_PATTERN) + IDX=$(kv_get runner/test_cases/$m.meta RESULT_INDEX) + NTH=$(kv_get runner/test_cases/$m.meta NTH_OCCURRENCE) + printf '<容器输出行>\n' > /tmp/probe.out + printf '%s -> %s\n' "$m" "$(extract_value /tmp/probe.out)" +done +``` +Expected: 每个打印数值。 + +- [ ] **Step 4: Commit** + +```sh +git add runner/test_cases/signal_install_lat.meta runner/test_cases/signal_prot_lat.meta +git commit -m "feat(lmbench): add .meta for signal latency tests" +``` + +--- + +### Task 6: vfs 组提取 .meta + +**测例**（8）：`vfs_open_lat`（已有，复核）、`vfs_read_lat`、`vfs_write_lat`、`vfs_fstat_lat`、`vfs_stat_lat`、`vfs_fcntl_lat`、`vfs_select_lat`、`vfs_read_pagecache_bw`。 + +**lmbench 调用**： +| 测例 | 容器命令 | +|---|---| +| vfs_open_lat | `$BIN/lat_syscall -P 1 -W 1 -N 3 open "$testfile"`（需先 `touch /tmp/testfile; testfile=/tmp/testfile`） | +| vfs_read_lat | `$BIN/lat_syscall -P 1 read` | +| vfs_write_lat | `$BIN/lat_syscall -P 1 write` | +| vfs_fstat_lat | `$BIN/lat_syscall -P 1 fstat test_file`（需 `test_file` 存在） | +| vfs_stat_lat | `$BIN/lat_syscall -P 1 -W 1000 -N 1000 stat testfile`（需 `testfile` 存在） | +| vfs_fcntl_lat | `$BIN/lat_fcntl -P 1 -W 30 -N 200` | +| vfs_select_lat | `$BIN/lat_select -P 1 file` | +| vfs_read_pagecache_bw | `$BIN/bw_file_rd -P 1 -W 30 -N 300 512m io_only $WORK/test_file` | + +- [ ] **Step 1: 建所需文件并逐个跑** + +```sh +touch /tmp/testfile +cd /tmp +``` +然后跑上表命令记录输出。`lat_syscall *` 延迟类，`lat_fcntl`/`lat_select` 延迟类，`bw_file_rd` 带宽类。 + +- [ ] **Step 2: 写/复核 8 个 .meta** + +延迟类 `RESULT_INDEX=NF-1`（`lat_syscall open` 锚定 `open/close`，`read` 锚定 `read`，`write` 锚定 `write`，`fstat` 锚定 `fstat`，`stat` 锚定 `stat`；`lat_fcntl`/`lat_select` 按实际输出特征词）。`bw_file_rd` 带宽类 `RESULT_INDEX=NF`。复核 `vfs_open_lat.meta`。 + +- [ ] **Step 3: host 侧抽样验证** + +```sh +LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh +for m in vfs_read_lat vfs_write_lat vfs_fstat_lat vfs_stat_lat vfs_fcntl_lat vfs_select_lat vfs_read_pagecache_bw; do + PAT=$(kv_get runner/test_cases/$m.meta SEARCH_PATTERN) + IDX=$(kv_get runner/test_cases/$m.meta RESULT_INDEX) + NTH=$(kv_get runner/test_cases/$m.meta NTH_OCCURRENCE) + printf '<容器输出行>\n' > /tmp/probe.out + printf '%s -> %s\n' "$m" "$(extract_value /tmp/probe.out)" +done +``` +Expected: 每个打印数值。 + +- [ ] **Step 4: Commit** + +```sh +git add runner/test_cases/vfs_read_lat.meta runner/test_cases/vfs_write_lat.meta runner/test_cases/vfs_fstat_lat.meta runner/test_cases/vfs_stat_lat.meta runner/test_cases/vfs_fcntl_lat.meta runner/test_cases/vfs_select_lat.meta runner/test_cases/vfs_read_pagecache_bw.meta +git commit -m "feat(lmbench): add .meta for vfs tests" +``` + +--- + +### Task 7: filesystem 组提取 .meta + +**测例**（6）：`ext4_create_delete_files_10k_ops`（已有，复核）、`ext4_create_delete_files_0k_ops`、`ramfs_create_delete_files_10k_ops`、`ramfs_create_delete_files_0k_ops`、`ext4_copy_files_bw`、`ramfs_copy_files_bw`。 + +**lmbench 调用**： +| 测例 | 容器命令 | +|---|---| +| ext4_create_delete_files_0k_ops | `$BIN/lat_fs -s 0k -P 1 $WORK` | +| ext4_create_delete_files_10k_ops | `$BIN/lat_fs -s 10k -P 1 $WORK` | +| ramfs_create_delete_files_0k_ops | `$BIN/lat_fs -s 0k -P 1 -W 30 -N 200 /dev/shm`（用 tmpfs 替代 ramfs） | +| ramfs_create_delete_files_10k_ops | `$BIN/lat_fs -s 10k -P 1 -W 30 -N 300 /dev/shm` | +| ext4_copy_files_bw | `$BIN/lmdd if=$WORK/zero_file of=$WORK/test_file`（去 sudo，容器内 root） | +| ramfs_copy_files_bw | `$BIN/lmdd if=/dev/shm/zero_file of=/dev/shm/test_file`（先 `dd` 建 /dev/shm 文件） | + +- [ ] **Step 1: 准备 /dev/shm 文件并逐个跑** + +```sh +dd if=/dev/zero of=/dev/shm/zero_file bs=1M count=512 2>/dev/null +dd if=/dev/zero of=/dev/shm/test_file bs=1M count=512 2>/dev/null +``` +跑上表命令记录输出。`lat_fs` 输出多 size 表（ops 表类，`^` 行 + 列号）。`lmdd` 带宽类。 + +- [ ] **Step 2: 写/复核 6 个 .meta** + +`lat_fs` ops 表类：`SEARCH_PATTERN=^0k` 或 `^10k`，`RESULT_INDEX=`，`UNIT=ops/sec`，`METRIC_TYPE=ops`。`lmdd` 带宽类。复核 `ext4_create_delete_files_10k_ops.meta`（原 `RESULT_INDEX=3`，跑后确认列号）。 + +- [ ] **Step 3: host 侧抽样验证** + +```sh +LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh +for m in ext4_create_delete_files_0k_ops ramfs_create_delete_files_10k_ops ramfs_create_delete_files_0k_ops ext4_copy_files_bw ramfs_copy_files_bw; do + PAT=$(kv_get runner/test_cases/$m.meta SEARCH_PATTERN) + IDX=$(kv_get runner/test_cases/$m.meta RESULT_INDEX) + NTH=$(kv_get runner/test_cases/$m.meta NTH_OCCURRENCE) + printf '<容器输出行>\n' > /tmp/probe.out + printf '%s -> %s\n' "$m" "$(extract_value /tmp/probe.out)" +done +``` +Expected: 每个打印数值。 + +- [ ] **Step 4: Commit** + +```sh +git add runner/test_cases/ext4_create_delete_files_0k_ops.meta runner/test_cases/ramfs_create_delete_files_10k_ops.meta runner/test_cases/ramfs_create_delete_files_0k_ops.meta runner/test_cases/ext4_copy_files_bw.meta runner/test_cases/ramfs_copy_files_bw.meta +git commit -m "feat(lmbench): add .meta for filesystem tests" +``` + +--- + +### Task 8: network 组提取 .meta + +**测例**（11）：`tcp_loopback_lat`（已有，复核）、`tcp_loopback_bw_128`、`tcp_loopback_bw_4k`、`tcp_loopback_bw_64k`、`tcp_loopback_connect_lat`、`tcp_loopback_select_lat`、`tcp_loopback_http_bw`、`tcp_virtio_lat`、`tcp_virtio_bw_128`、`tcp_virtio_bw_64k`、`tcp_virtio_connect_lat`、`udp_loopback_lat`、`udp_virtio_lat`。 + +> 注：实际 13 个（含 udp 2 个）。本组是最大组。 + +**lmbench 调用**（server+client 类：先起 server 后台，再跑 client；容器内全用 127.0.0.1）： +| 测例 | 容器命令（server + client） | +|---|---| +| tcp_loopback_lat | `$BIN/lat_tcp -s 127.0.0.1 -b 1 &` ; `$BIN/lat_tcp 127.0.0.1` | +| tcp_loopback_bw_128 | `$BIN/bw_tcp -s 127.0.0.1 -b 1 &` ; `$BIN/bw_tcp 127.0.0.1`（client 参数跑后确认） | +| tcp_loopback_bw_4k | 同上 | +| tcp_loopback_bw_64k | 同上 | +| tcp_loopback_connect_lat | `$BIN/lat_connect -s 127.0.0.1 &` ; `$BIN/lat_connect 127.0.0.1` | +| tcp_loopback_select_lat | `$BIN/lat_select -P 1 tcp` | +| tcp_loopback_http_bw | `$BIN/lmhttp &` ; `$BIN/bw_tcp 127.0.0.1`（http server + bw client；参数跑后确认） | +| tcp_virtio_lat | 用 127.0.0.1 替代 10.0.2.15：`$BIN/lat_tcp -s 127.0.0.1 -b 1 &` ; `$BIN/lat_tcp 127.0.0.1` | +| tcp_virtio_bw_128 | `$BIN/bw_tcp -s 127.0.0.1 -b 1 &` ; `$BIN/bw_tcp 127.0.0.1` | +| tcp_virtio_bw_64k | 同上 | +| tcp_virtio_connect_lat | `$BIN/lat_connect -s 127.0.0.1 -b 1000 &` ; `$BIN/lat_connect 127.0.0.1` | +| udp_loopback_lat | `$BIN/lat_udp -s 127.0.0.1 &` ; `$BIN/lat_udp 127.0.0.1` | +| udp_virtio_lat | `$BIN/lat_udp -s 127.0.0.1 &` ; `$BIN/lat_udp 127.0.0.1` | + +- [ ] **Step 1: 逐个跑（server 后台 + client）并记录 client 输出** + +`lat_tcp`/`lat_connect`/`lat_udp` 延迟类（`...: microseconds`）。`bw_tcp` 带宽类。`lat_select tcp` 延迟类。`lmhttp`+`bw_tcp` http 带宽类。注意：server 起后台后等 1 秒再跑 client（`sleep 1`）。每个测例跑完 `kill %1` 清 server。 + +- [ ] **Step 2: 写/复核 13 个 .meta** + +延迟类 `RESULT_INDEX=NF-1`，带宽类 `RESULT_INDEX=NF`。`SEARCH_PATTERN` 锚定特征词（`lat_tcp` 用 `^TCP latency using`——复核 `tcp_loopback_lat.meta`；`lat_connect` 锚定 `Connection latency`；`lat_udp` 锚定 `UDP latency`；`lat_select` 锚定 `Select latency`；`bw_tcp` 按输出格式）。virtio 类与 loopback 类输出格式相同（IP 不影响格式），`.meta` 的 SEARCH_PATTERN 可相同，DESCRIPTION 区分 virtio/loopback。 + +- [ ] **Step 3: host 侧抽样验证** + +```sh +LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh +for m in tcp_loopback_bw_128 tcp_loopback_connect_lat tcp_loopback_select_lat tcp_virtio_lat udp_loopback_lat udp_virtio_lat; do + PAT=$(kv_get runner/test_cases/$m.meta SEARCH_PATTERN) + IDX=$(kv_get runner/test_cases/$m.meta RESULT_INDEX) + NTH=$(kv_get runner/test_cases/$m.meta NTH_OCCURRENCE) + printf '<容器输出行>\n' > /tmp/probe.out + printf '%s -> %s\n' "$m" "$(extract_value /tmp/probe.out)" +done +``` +Expected: 每个打印数值。 + +- [ ] **Step 4: Commit** + +```sh +git add runner/test_cases/tcp_loopback_bw_128.meta runner/test_cases/tcp_loopback_bw_4k.meta runner/test_cases/tcp_loopback_bw_64k.meta runner/test_cases/tcp_loopback_connect_lat.meta runner/test_cases/tcp_loopback_select_lat.meta runner/test_cases/tcp_loopback_http_bw.meta runner/test_cases/tcp_virtio_lat.meta runner/test_cases/tcp_virtio_bw_128.meta runner/test_cases/tcp_virtio_bw_64k.meta runner/test_cases/tcp_virtio_connect_lat.meta runner/test_cases/udp_loopback_lat.meta runner/test_cases/udp_virtio_lat.meta +git commit -m "feat(lmbench): add .meta for network tests" +``` + +--- + +### Task 9: whitelist 扩全量 + 探查报告 + 最终验证 + +**Files:** +- Modify: `whitelist.txt` +- Create: `docs/superpowers/specs/2026-07-30-lmbench-fullsuite-survey.md` + +- [ ] **Step 1: 扩 whitelist 到全量 48** + +把 `whitelist.txt` 替换为全量 48 个测例名（每行一个，按类别分组注释，参考现有 8 个的格式）： +``` +# memory +mem_copy_bw +mem_read_bw +mem_write_bw +mem_mmap_bw +mem_mmap_lat +mem_pagefault_lat + +# process +process_fork_lat +process_exec_lat +process_ctx_lat +process_getppid_lat +process_shell_lat + +# ipc +pipe_lat +pipe_bw +fifo_lat +unix_lat +unix_bw +unix_connect_lat +semaphore_lat + +# signal +signal_catch_lat +signal_install_lat +signal_prot_lat + +# vfs +vfs_open_lat +vfs_read_lat +vfs_write_lat +vfs_fstat_lat +vfs_stat_lat +vfs_fcntl_lat +vfs_select_lat +vfs_read_pagecache_bw + +# filesystem +ext4_create_delete_files_0k_ops +ext4_create_delete_files_10k_ops +ramfs_create_delete_files_0k_ops +ramfs_create_delete_files_10k_ops +ext4_copy_files_bw +ramfs_copy_files_bw + +# network +tcp_loopback_lat +tcp_loopback_bw_128 +tcp_loopback_bw_4k +tcp_loopback_bw_64k +tcp_loopback_connect_lat +tcp_loopback_select_lat +tcp_loopback_http_bw +tcp_virtio_lat +tcp_virtio_bw_128 +tcp_virtio_bw_64k +tcp_virtio_connect_lat +udp_loopback_lat +udp_virtio_lat +``` + +- [ ] **Step 2: 验证 .meta 齐全** + +```sh +cd /home/yuming/project_ongoing/DragonOSDev/DragonOS/.worktrees/feat-lmbench/user/apps/tests/benchmark/lmbench +echo "meta count: $(ls runner/test_cases/*.meta | wc -l)" +echo "sh count: $(ls runner/test_cases/*.sh | grep -v '^runner/test_cases/test_' | wc -l)" +``` +Expected: 两者都 = 48。 + +- [ ] **Step 3: 验证每个 whitelist 条目有对应 .meta** + +```sh +while read -r line; do + case "$line" in ''|\#*) continue ;; esac + [ -f "runner/test_cases/$line.meta" ] || echo "MISSING meta: $line" +done < whitelist.txt +echo "check done" +``` +Expected: 只打印 `check done`，无 `MISSING meta`。 + +- [ ] **Step 4: 写探查报告** + +创建 `docs/superpowers/specs/2026-07-30-lmbench-fullsuite-survey.md`，每测例一条：lmbench 工具 + 容器跑法（哪组等价环境）+ 容器实际输出样本（一行）+ `.meta` 抽取规则（SEARCH_PATTERN/RESULT_INDEX）+ guest 校准状态（已校准/待 guest 校准/guest 可能不支持）。guest 校准状态： +- 已校准：原 8 个（`mem_copy_bw`/`process_fork_lat`/`pipe_lat`/`unix_lat`/`vfs_open_lat`/`tcp_loopback_lat`/`signal_catch_lat`/`ext4_create_delete_files_10k_ops`）。 +- 待 guest 校准：host 提取的 40 个。 +- guest 可能不支持：基于工具用的 syscall 推断标注（如 `lat_connect`/`lat_select`/`lat_fcntl` 可能依赖内核未实现特性），留后续 QEMU 验证确认。 + +- [ ] **Step 5: 验证未改 guest 文件** + +```sh +cd /home/yuming/project_ongoing/DragonOSDev/DragonOS/.worktrees/feat-lmbench +git diff --name-only HEAD~9 HEAD -- user/apps/tests/benchmark/lmbench/runner/run.sh user/apps/tests/benchmark/lmbench/runner/env.sh user/apps/tests/benchmark/lmbench/runner/init.sh user/apps/tests/benchmark/lmbench/runner/clean_up.sh user/apps/tests/benchmark/lmbench/runner/test_cases/*.sh +``` +Expected: 空输出（未改 `run.sh`/`env.sh`/`init.sh`/`clean_up.sh`/任何 `.sh`）。 + +- [ ] **Step 6: Commit whitelist + 报告** + +```sh +git add whitelist.txt docs/superpowers/specs/2026-07-30-lmbench-fullsuite-survey.md +git commit -m "feat(lmbench): enable full whitelist and add survey report" +``` + +- [ ] **Step 7: 清理临时环境** + +```sh +rm -rf /tmp/lmbench-survey +docker system prune -f 2>/dev/null || true +``` + +--- + +## Self-Review + +**1. Spec coverage：** +- spec §目标1（48 个 .meta）→ Task 2-8 覆盖 6 组（memory/process/ipc/signal/vfs/filesystem/network），每组写 + 复核；Task 9 Step 2 验证齐全。 +- spec §目标2（探查报告）→ Task 9 Step 4。 +- spec §目标3（whitelist 全量）→ Task 9 Step 1。 +- spec §硬约束（24.04 容器、guest 同款 archive、不改 guest 文件、无 QEMU）→ Global Constraints + Task 9 Step 5 验证未改 guest 文件。 +- spec §流程（下载/解压/容器手操/写 .meta）→ Task 1 环境 + Task 2-8 各组。 +- spec §等价环境分组 → Task 2-8 按组 + Task 7（/dev/shm 替代 ramfs）+ Task 8（127.0.0.1 替代 virtio）。 +- spec §映射规则三类 → File Structure 映射规则 + 各 Task step 2。 +- spec §banner 规避 → File Structure banner 规避 + Task 8 复核 TCP。 +- spec §验证（48 .meta 齐全 + 报告 + 不改 guest）→ Task 9 Step 2/3/5。 +- spec §不做（QEMU/内核/.sh 修复）→ Global Constraints + 无对应 Task。✓ + +**2. Placeholder scan：** `.meta` 的 `SEARCH_PATTERN` 实际值标为"跑后从输出填"——这是探查任务的本质（必须跑后观察），非"TODO"占位；每步有确切跑命令 + .meta 模板 + 映射规则 + 验证。✓ + +**3. Type consistency：** `LMBENCH_RUNNER_NO_MAIN=1 . runner/run.sh` + `kv_get`/`extract_value` 在各 Task 一致；`$BIN`/`$WORK` 环境变量在 Task 1 定义、Task 2-8 复用；`.meta` 字段名（SEARCH_PATTERN/RESULT_INDEX/NTH_OCCURRENCE）与 `run.sh` 的 `kv_get` 一致。✓ + +## Execution Handoff + +计划已保存。鉴于本任务手操性质（docker 内跑 lmbench、观察输出、填 `.meta`），**Inline Execution** 更合适——需要交互式观察输出填值，fresh subagent 难处理。建议 inline 执行。 \ No newline at end of file diff --git a/docs/superpowers/specs/2026-07-29-lmbench-runner-abstraction-design.md b/docs/superpowers/specs/2026-07-29-lmbench-runner-abstraction-design.md new file mode 100644 index 0000000000..eb9fed38a5 --- /dev/null +++ b/docs/superpowers/specs/2026-07-29-lmbench-runner-abstraction-design.md @@ -0,0 +1,167 @@ +# LMbench Runner 抽象与目录分层设计 + +- 日期：2026-07-29 +- 分支：`feat/lmbench-suite` +- 状态：设计已用户确认，待 review + +## Context + +`make test-benchmark` 已达成 MVP 8/8 端到端验证。在扩到全部 48 个测例之前，`user/apps/tests/benchmark/lmbench/` 目录里的脚本角色混杂：runner 组成部分、host 编排、打包脚本混放同一层，且存在死代码 `run_test_case.sh`。本设计在扩展前先清边界：给 guest runner 一个参数化入口，按职责分层目录，底层实现暂不拆 lib。 + +## 目标 + +1. 把脚本按 `runner/`（guest 内运行）、`orchestrator/`（host 编排）、根（用户配置 + 包元信息）分层，让"加测例/调 host/改打包"三条工作路径互不干扰。 +2. 给 guest runner 一个参数化 CLI 入口，屏蔽底层实现；新增测例或调采样只跟接口打交道。 +3. 顺手修 `default.nix` 误装 host 脚本、漏装 config/whitelist 的既有问题。 +4. 不改输出 contract、不改 rcS 调用方式、不改 CI 制品路径——`make test-benchmark` 外部行为零变化。 + +## 硬约束（查证所得） + +- `Makefile` 必须留包根：DADK `lmbench_benchmark_tests-1.0.0.toml` 用 `build-from-source` + `source-path = "user/apps/tests/benchmark/lmbench"` + `build-command = "make install"`，在包根找 Makefile。 +- `default.nix` / `flake.nix` / `flake.lock` 必须留包根：`user/apps/default.nix:66` 用 `callPackage ./tests/benchmark/lmbench {}` 找 default.nix。 +- `toggle_compile_lmbench.sh` 留包根：其 `../../../../..` 相对路径依赖当前层级。 + +## runner 定义 + +**窄定义**：runner = 仅 guest 内运行部分（`run.sh` + init/env/clean_up + test_cases）。host 侧 `monitor_test_results.sh` / `collect_results.py` 是 orchestrator（协作者，不是 runner），因为它们的"运行"对象是 QEMU 进程 + 串口流，不是测例；且 host(py+sh) 与 guest(busybox sh) 无共享实现层，强行归一个"runner"会再次模糊边界。 + +## 目录布局 + +``` +lmbench/ + runner/ # guest 内运行（DADK/nix install 时摊平到 rootfs） + run.sh # 参数化入口（原 run_tests.sh，重命名） + init.sh / env.sh / clean_up.sh + test_cases/ + .sh / .meta + test_tcp_loopback_lat_parser.sh # 回归测试随迁 + orchestrator/ # host 侧编排 + monitor_test_results.sh + collect_results.py + schema/lmbench-run.schema.json + config # 留根：用户调参（SAMPLES/TIMEOUT_SEC/WARMUP） + whitelist.txt # 留根：用户选测例 + Makefile # 留根：install 改为从 runner/ 拷 + default.nix / flake.nix / flake.lock # 留根：sourceByRegex 改为只抓 runner/ + toggle_compile_lmbench.sh # 留根（构建开关，路径不动） + README.md / .gitignore / results/ +``` + +分层逻辑：`runner/` = 纯执行机制 + test_cases（runner 直接 dispatch 的对象，且 48×2 文件放根会淹没根目录）；根 = 用户面向配置（config/whitelist）+ 包元信息（Makefile/nix/toggle/README）。 + +### guest 运行时布局：扁平 + +Makefile/nix install 时把 `runner/` 内文件 + 根的 `config`/`whitelist.txt` 全摊平到 rootfs `/opt/tests/benchmark/lmbench/`，`runner/test_cases/` → `/opt/tests/benchmark/lmbench/test_cases/`。`orchestrator/` 与 `schema/` 不装进 rootfs（只 host 用）。 + +源码分层是 host 侧组织，guest 运行时仍扁平。`run.sh` 用 `$SCRIPT_DIR/{test_cases,config,whitelist.txt}` 定位——运行时同目录，天然兼容；`init.sh` 的 `$SCRIPT_DIR/env.sh`、`$SCRIPT_DIR/ext4.img` 同理。 + +注：`run.sh` 只在 guest 运行时执行，源码树里不直接跑（无 lmbench 二进制与 guest 环境）。源码层面 `$SCRIPT_DIR/config` 指向 `runner/config`（不存在）的"错位"无实际影响；host 单测用 `LMBENCH_RUNNER_NO_MAIN=1` source 不走 main。 + +## 参数化接口 + +`run.sh` 的 CLI 面（busybox sh POSIX 兼容，手写 `while [ $# -gt 0 ]; do case` 解析，不用 getopt）： + +``` +run.sh [--samples N] [--timeout S] [--warmup N] + [--whitelist FILE] [--config FILE] [--only NAME] [--list] +``` + +| 参数 | 语义 | 覆盖 | +|---|---|---| +| `--samples N` | 每测例采样次数 | config `SAMPLES` | +| `--timeout S` | 单测例墙钟超时秒 | config `TIMEOUT_SEC` | +| `--warmup N` | 丢弃的预热轮数 | config `WARMUP` | +| `--whitelist FILE` | 测例清单路径 | `$SCRIPT_DIR/whitelist.txt` | +| `--config FILE` | 配置文件路径 | `$SCRIPT_DIR/config` | +| `--only NAME` | 只跑单个测例（取代 `run_test_case.sh`） | 临时 whitelist 只含 NAME | +| `--list` | 列出 `test_cases/*.sh` 可用测例名，不跑 | — | + +**优先级**：命令行参数 > `--config` 指定文件 > 内置默认（`SAMPLES=5` 等）。实现：先 `load_config()`，再让命令行覆盖。 + +**向后兼容**：无参数 = 现状行为（读 `config` + `whitelist.txt` 全跑）。rcS 仅文件名 `run_tests.sh` → `run.sh`，调用方式不变。 + +**`--only NAME` 实现 constraint**：仅作用于 whitelist 读取环节（构造单行清单），下游 `run_one_case` 不改——不引入第二条 dispatch 分支。NAME 无对应 `.sh` 时走现有 skipped 路径，不 abort 整轮。 + +**`--list` 实现**：`ls test_cases/*.sh` 去后缀去路径打印。 + +**错误处理**：未知参数 → stderr 打 usage、exit 2。 + +**输出 contract 不变**：`===LMBENCH_RUN_BEGIN===` / `LMBENCH_META` / `LMBENCH_JSON` / `LMBENCH_SUMMARY` / `===LMBENCH_RUN_END===` / `benchmark测试完成` 全部保持。host `collect_results.py` / `monitor_test_results.sh` 零改动。 + +## 迁移影响面 + +### A. Guest 侧（runner/） + +| 文件 | 改动 | +|---|---| +| `run_tests.sh` → `runner/run.sh` | 重命名 + 加 `parse_args()`；`LMBENCH_RUNNER_NO_MAIN` 保留；内部 `$SCRIPT_DIR/{test_cases,config,whitelist.txt}` 运行时扁平仍对 | +| `init.sh`/`env.sh`/`clean_up.sh` → `runner/` | 移位，内容不动 | +| `test_cases/` → `runner/test_cases/` | 整目录移；`test_tcp_loopback_lat_parser.sh` 内 `RUNNER="$SCRIPT_DIR/../run_tests.sh"` → `../run.sh` | +| `run_test_case.sh` | 删除（死代码：在 Makefile GUEST_SCRIPTS 里被装进 rootfs，但 guest 内无人调；host 上跑不通因无 lmbench 二进制） | + +### B. 根目录配置（留根） + +| 文件 | 改动 | +|---|---| +| `config` / `whitelist.txt` | 留根，内容不动；Makefile/nix install 时摊平进 rootfs | + +### C. Host 编排（orchestrator/） + +| 文件 | 改动 | +|---|---| +| `monitor_test_results.sh` → `orchestrator/` | 移位，内容不动（全用 env var，无内部相对路径） | +| `collect_results.py` → `orchestrator/` | `--schema` 默认 `HERE/schema/...`（schema 跟移，仍对）；`--outdir` 默认 `HERE/../results`（results 留根）；docstring `run_tests.sh` → `run.sh` | +| `schema/` → `orchestrator/schema/` | 跟 collect 移 | + +### D. 打包 / 构建（留根） + +| 文件 | 改动 | +|---|---| +| `Makefile` | `GUEST_SCRIPTS` 从 `runner/` 取（`runner/run.sh` 等，`install` 自动取 basename 摊平）；删 `run_test_case.sh`；`GUEST_DATA = whitelist.txt config`；`cp -r runner/test_cases` | +| `default.nix` | `sourceByRegex` 改为只抓 `runner/` 下 `.sh` + `runner/test_cases`；installPhase 从 `runner/` 拷 + 补装 `config`/`whitelist.txt`（当前漏装，顺手修）；`run_tests.sh` → `run.sh` | +| `flake.nix:41` | `program` 路径 `run_tests.sh` → `run.sh` | +| `toggle_compile_lmbench.sh` | 留根不动 | + +### E. 外部调用点 + +| 文件 | 改动 | +|---|---| +| `user/sysconfig/etc/init.d/rcS:52` | `run_tests.sh` → `run.sh` | +| 根 `Makefile:325/327` | `monitor_test_results.sh`/`collect_results.py` 路径加 `orchestrator/` 前缀；toggle 路径不变 | +| `.github/workflows/benchmark.yml` | `results/**` 路径不变（results 留根）；无内部脚本引用 | + +### F. 文档 + +| 文件 | 改动 | +|---|---| +| `README.md` | 目录结构图 + 所有脚本路径引用同步新布局；新增 `run.sh` 参数说明；删 `run_test_case.sh` 条目 | +| `Makefile` 注释 / `collect_results.py` docstring | `run_tests.sh` → `run.sh` | + +## 验证 + +1. **静态引用扫描**（门禁，先于 e2e）：迁移完成后 `grep -rn "run_tests.sh\|run_test_case\.sh"` 全仓应为零命中（除 spec/commit message）；`grep -rn "monitor_test_results.sh\|collect_results.py"` 确认只剩 `orchestrator/` 前缀引用。 + +2. **host 侧回归测试**（`busybox sh`，不依赖 guest）： + - `test_tcp_loopback_lat_parser.sh`（现有，随迁）：source `run.sh`，断言 `extract_value` 返回 `4745.8345`。 + - `test_args_parser.sh`（新增）：source `run.sh`，断言 `--samples 7 --only foo` 解析后 `SAMPLES=7` 且 whitelist 等效单行 `foo`；断言未知参数 exit 2。 + +3. **构建冒烟**：`make all` 确认 DADK `make install` 从 `runner/` 摊平安装不报错，sysroot 下 `/opt/tests/benchmark/lmbench/` 出现扁平 `run.sh`/`init.sh`/`test_cases/`/`config`/`whitelist.txt`。 + +4. **端到端**（需用户授权覆盖 `bin/disk-image-x86_64.img`）：`make test-benchmark` 跑通 8/8，schema 校验通过，确认 rcS→`run.sh`、orchestrator 路径、摊平布局全链路工作。 + +## 风险与缓解 + +| 风险 | 缓解 | +|---|---| +| DADK/nix 两条安装路径摊平布局不一致，guest 运行时分叉 | spec 明确两条 install 都摊平到 `/opt/tests/benchmark/lmbench/` 扁平；构建冒烟验证 DADK 输出；nix 输出由 `default.nix` installPhase 对齐 | +| `--only` 悄悄引入第二条 dispatch 分支 | 实现 constraint：`--only NAME` 仅作用于 whitelist 读取环节，下游 `run_one_case` 不改 | +| busybox sh 参数解析兼容性 | 手写 `while/case`，不用 getopt/getopts；回归测试覆盖 | +| 源码树 `run.sh` 的 `$SCRIPT_DIR/config` 指向 `runner/config`（不存在）造成困惑 | spec 注明 run.sh 只在 guest 运行时执行；host 单测用 `LMBENCH_RUNNER_NO_MAIN=1` source | +| 迁移漏改旧路径导致 e2e 失败 | 静态 grep 扫描作为门禁，先于 e2e | + +## 不做（YAGNI） + +- 不拆 `run.sh` 内部 lib（`kv_get`/`extract_value`/`compute_stats` 等留在单文件）。 +- 不参数化 host 侧（`monitor`/`collect` 已被 Makefile 串好，收益小）。 +- 不加 CI 回归门控/可视化。 +- 不扩 48 测例——本设计是扩展前的边界清理，扩展是后续独立轮次。 \ No newline at end of file diff --git a/docs/superpowers/specs/2026-07-30-lmbench-fullsuite-meta-extraction-design.md b/docs/superpowers/specs/2026-07-30-lmbench-fullsuite-meta-extraction-design.md new file mode 100644 index 0000000000..3f9e420b3b --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-lmbench-fullsuite-meta-extraction-design.md @@ -0,0 +1,86 @@ +# LMbench 全量测例 .meta 提取设计 + +- 日期：2026-07-30 +- 分支：`feat/lmbench` +- 状态：设计已用户确认，待 review + +## Context + +`runner/test_cases/` 下有 48 个测例 `.sh`，但仅 8 个有 `.meta` 抽取规则——这 8 个是 MVP whitelist 已跑通的。剩余 40 个无 `.meta`，`run.sh` 对它们走默认抽取（`PAT='^[0-9]'`）会 failed，且 `raw_tail` 仅 300 字节尾部不足以可靠看清输出格式。在扩全量 whitelist 前必须为这 40 个补 `.meta`。 + +`.meta` 抽取规则的核心是 lmbench 二进制的**输出格式**，而输出格式由二进制自身决定、不依赖 guest 内核。因此可在 host 侧（Docker ubuntu:24.04 容器，提供 lmbench 24.04 二进制所需的 glibc 2.39）跑 lmbench 工具看完整输出、提取 `.meta`，不必启 QEMU、不必覆盖 disk image、不必授权。 + +## 目标 + +1. 为 48 个测例补全 `.meta`（含已有 8 个复核）。 +2. 产出探查报告：每测例的 lmbench 工具、容器输出样本、`.meta` 规则、guest 校准状态。 +3. `whitelist.txt` 保留全量 48 个。 +4. 不改任何 guest 生产代码（`env.sh`/`.sh`/`init.sh`）；不做 QEMU 验证。 + +## 硬约束 + +- host glibc 2.35（Ubuntu 22.04），跑不了 24.04 glibc 2.39 编译的 lmbench 二进制 → 必须用 Docker ubuntu:24.04 容器（已确认 docker 29.6.2 可用）。 +- lmbench 二进制来源：用 guest 同款 archive（toml 里 `lmbench-ubuntu2404-...tar.xz`），保证 `.meta` 抽取规则和 guest 完全一致。 +- 一次性探查工作，手操即可，不为它改 guest 文件。 +- QEMU 验证不纳入本轮：guest 内核兼容性问题可能蔓延，导致核心任务"获取 .meta"延期。 + +## 流程 + +1. 下载 `lmbench-ubuntu2404-...tar.xz`（URL 见 `user/dadk/config/all/lmbench_bin_ubuntu2404.toml` 的 `source-path`），解压到 `/tmp/lmbench-survey/`（含 `lib/lmbench/bin/x86_64-linux-gnu/*` + `lib/x86_64-linux-gnu/*` glibc 2.39）。 +2. `docker run --rm -v /tmp/lmbench-survey:/srv ubuntu:24.04 bash`，容器内直接跑 lmbench 二进制（路径 `/srv/lib/lmbench/bin/x86_64-linux-gnu/`，参数参考对应 `.sh` 里的调用），捕获完整 stdout/stderr。 +3. server+client 类（`tcp_loopback_*`/`unix_*`/`tcp_virtio_*`/`udp_*` 等）手操起 server 后台 + 跑 client，看输出。 +4. 据输出写 `runner/test_cases/.meta`。 +5. `env.sh`/`.sh`/`init.sh` 一律不动。`.sh` 的完整性（如 `tcp_virtio_lat.sh` 只 client 没 server）是后续 guest 修复问题，不影响 host 看输出格式提取 `.meta`。 + +## 容器等价环境分组 + +48 个测例按 lmbench 工具的环境依赖分 4 组，容器里用等价环境跑（只看输出格式，数值不重要）： + +| 组 | 测例 | 容器跑法 | +|---|---|---| +| 无依赖 | `lat_pipe`/`lat_fifo`/`lat_unix`/`bw_unix`/`lat_proc *`/`lat_sig *`/`lat_syscall *`/`bw_mem *`/`lat_mem`/`lat_ctx`/`lat_pagefault`/`bw_mmap_rd`(mmap_only) | 直接跑，容器内 cwd | +| 需文件/目录 | `bw_file_rd *`(pagecache)/`bw_mmap_rd`(file)/`lat_fs *`(ext4/ramfs create+delete) | 容器内 `dd` 建 dummy 文件 / `mkdir` 建目录；ext4/ramfs 用容器普通目录/tmpfs 替代（格式不依赖 fs 类型） | +| 需 server+client | `lat_tcp`/`bw_tcp *`/`lat_connect`(tcp)/`unix_connect_lat` | 容器内 127.0.0.1：起 server 后台 `lat_tcp -s &`，再跑 client `lat_tcp 127.0.0.1` | +| virtio 特定 | `tcp_virtio_*`/`udp_virtio_*` | 容器用 127.0.0.1 替代 `10.0.2.15`（格式相同，IP 只影响数值）；`.sh` 里只 client 没 server 的，容器手操补 server 看输出 | + +## 输出→`.meta` 映射规则 + +三类格式（基于已有 8 个 `.meta` 验证）： + +- **延迟类**（`lat_pipe`/`lat_sig`/`lat_syscall`/`lat_tcp`/`lat_proc`/`lat_unix`/`lat_fifo`/`lat_sem`/`lat_connect`/`lat_ctx`/`lat_pagefault`/`lat_select`）：输出 `<描述>: microseconds` → `SEARCH_PATTERN=<锚定词>`, `RESULT_INDEX=NF-1`（"microseconds"前的值）, `BIGGER_IS_BETTER=0`, `UNIT=microseconds`, `METRIC_TYPE=latency`。 +- **带宽类**（`bw_mem`/`bw_unix`/`bw_tcp`/`bw_mmap_rd`/`bw_file_rd`）：输出 ` ` 或 `... MB/sec` → `SEARCH_PATTERN=^[0-9]` 或锚定词, `RESULT_INDEX=NF`（最后字段=带宽）, `BIGGER_IS_BETTER=1`, `UNIT=MB/s`, `METRIC_TYPE=bandwidth`。 +- **ops/表类**（`lat_fs`）：多 size 表 → `SEARCH_PATTERN=^`, `RESULT_INDEX=<列号>`, `BIGGER_IS_BETTER=1`, `UNIT=ops/sec`, `METRIC_TYPE=ops`。 + +每个 `.meta` 含 `CATEGORY`/`BINARY`/`METRIC_TYPE`/`UNIT`/`BIGGER_IS_BETTER`/`SEARCH_PATTERN`/`RESULT_INDEX`/`NTH_OCCURRENCE`/`DESCRIPTION`。 + +## banner 规避 + +`.sh` 打的 `=== Running XXX test ===` banner 不是结果行。`SEARCH_PATTERN` 必须锚定 lmbench 二进制实际输出的特征词（`Pipe latency`/`Signal handler`/`open/close` 这种二进制才会打的描述），避免误匹配 banner。 + +教训：`tcp_loopback_lat.meta` 原 `SEARCH_PATTERN=TCP latency` 会先撞 banner `=== Running TCP latency test ===`，`RESULT_INDEX=NF-1` 取了 `test` 而非数字；修成 `^TCP latency using` 才跑通。延迟类尤其注意；带宽类 `^[0-9]` 因 banner 不以数字开头，天然安全。 + +## whitelist + +`whitelist.txt` 保留全量 48 个测例（本轮补全了 `.meta`，不再恢复 8 个）。`.meta` 是 host 提取、未在 guest 校准，正式 `make test-benchmark` 跑全量可能仍有部分 failed（guest 输出与 host 差异，或 guest 内核不支持某些 syscall）——这是有价值的校准信号，不视为问题。 + +## 报告 + +落到 `docs/superpowers/specs/2026-07-30-lmbench-fullsuite-survey.md`，每测例一条： +- lmbench 工具 + 容器跑法（哪组等价环境） +- 容器实际输出样本（一行） +- `.meta` 抽取规则（SEARCH_PATTERN/RESULT_INDEX） +- guest 校准状态：**已校准**（原 8 个）/ **待 guest 校准**（host 提取，40 个）/ **guest 可能不支持**（基于工具用的 syscall 推断，如 `lat_connect`/`lat_select` 等可能依赖内核未实现特性，留后续 QEMU 验证确认） + +## 不做（YAGNI / 留后续） + +- 不做 QEMU 验证（guest 内核兼容性问题可能蔓延，导致 .meta 提取延期）。 +- 不改 guest 生产代码（`env.sh`/`.sh`/`init.sh`）。 +- 不修内核限制、不补 `.sh` server 逻辑、不扩 `init.sh` 环境——这些留后续 guest 校准轮。 +- 不要求 48 个在 guest 全部正确出数——本轮只交付 host 产物（48 `.meta` + 报告）。 + +## 验证 + +- 48 个 `.meta` 齐全（`ls runner/test_cases/*.meta | wc -l` = 48）。 +- 报告每测例含容器输出样本 + 抽取规则。 +- `whitelist.txt` 含 48 行有效测例名。 +- 不改 `env.sh`/`.sh`/`init.sh`（`git diff` 无 guest 文件改动）。 \ No newline at end of file From d373a8c9ad5674534f4cd4af2fb76e605b45a6a3 Mon Sep 17 00:00:00 2001 From: yuming Date: Tue, 22 Sep 2026 00:48:26 +0800 Subject: [PATCH 03/22] feat(lmbench): add portable full-suite runner and regression tests Add the full suite, portable runners, regression tests and metadata survey. Document runner arguments, Linux execution, case authoring and calibration alongside the implementation. Co-authored-by: Samuka007 Co-authored-by: sparkzky --- .../2026-07-30-lmbench-fullsuite-survey.md | 136 ++++++ user/apps/tests/README.md | 5 + user/apps/tests/benchmark/README.md | 0 user/apps/tests/benchmark/lmbench/.gitignore | 7 + user/apps/tests/benchmark/lmbench/README.md | 101 ++++ user/apps/tests/benchmark/lmbench/config | 11 + .../apps/tests/benchmark/lmbench/run_linux.sh | 191 ++++++++ .../benchmark/lmbench/runner/clean_up.sh | 25 + .../tests/benchmark/lmbench/runner/env.sh | 18 + .../benchmark/lmbench/runner/fifo_cleanup.sh | 64 +++ .../tests/benchmark/lmbench/runner/init.sh | 70 +++ .../tests/benchmark/lmbench/runner/result.sh | 180 ++++++++ .../tests/benchmark/lmbench/runner/run.sh | 433 ++++++++++++++++++ .../runner/test_cases/ext4_copy_files_bw.meta | 11 + .../runner/test_cases/ext4_copy_files_bw.sh | 23 + .../ext4_create_delete_files_0k_ops.meta | 12 + .../ext4_create_delete_files_0k_ops.sh | 25 + .../ext4_create_delete_files_10k_ops.meta | 13 + .../ext4_create_delete_files_10k_ops.sh | 23 + .../lmbench/runner/test_cases/fifo_lat.meta | 12 + .../lmbench/runner/test_cases/fifo_lat.sh | 25 + .../runner/test_cases/mem_copy_bw.meta | 11 + .../lmbench/runner/test_cases/mem_copy_bw.sh | 21 + .../runner/test_cases/mem_mmap_bw.meta | 11 + .../lmbench/runner/test_cases/mem_mmap_bw.sh | 26 ++ .../runner/test_cases/mem_mmap_lat.meta | 11 + .../lmbench/runner/test_cases/mem_mmap_lat.sh | 21 + .../runner/test_cases/mem_pagefault_lat.meta | 11 + .../runner/test_cases/mem_pagefault_lat.sh | 35 ++ .../runner/test_cases/mem_read_bw.meta | 11 + .../lmbench/runner/test_cases/mem_read_bw.sh | 21 + .../runner/test_cases/mem_write_bw.meta | 11 + .../lmbench/runner/test_cases/mem_write_bw.sh | 21 + .../lmbench/runner/test_cases/pipe_bw.meta | 11 + .../lmbench/runner/test_cases/pipe_bw.sh | 21 + .../lmbench/runner/test_cases/pipe_lat.meta | 11 + .../lmbench/runner/test_cases/pipe_lat.sh | 21 + .../runner/test_cases/process_ctx_lat.meta | 12 + .../runner/test_cases/process_ctx_lat.sh | 21 + .../runner/test_cases/process_exec_lat.meta | 11 + .../runner/test_cases/process_exec_lat.sh | 21 + .../runner/test_cases/process_fork_lat.meta | 11 + .../runner/test_cases/process_fork_lat.sh | 21 + .../test_cases/process_getppid_lat.meta | 13 + .../runner/test_cases/process_getppid_lat.sh | 21 + .../runner/test_cases/process_shell_lat.meta | 12 + .../runner/test_cases/process_shell_lat.sh | 21 + .../test_cases/ramfs_copy_files_bw.meta | 11 + .../runner/test_cases/ramfs_copy_files_bw.sh | 21 + .../ramfs_create_delete_files_0k_ops.meta | 11 + .../ramfs_create_delete_files_0k_ops.sh | 25 + .../ramfs_create_delete_files_10k_ops.meta | 11 + .../ramfs_create_delete_files_10k_ops.sh | 23 + .../runner/test_cases/semaphore_lat.meta | 12 + .../runner/test_cases/semaphore_lat.sh | 22 + .../runner/test_cases/signal_catch_lat.meta | 11 + .../runner/test_cases/signal_catch_lat.sh | 21 + .../runner/test_cases/signal_install_lat.meta | 11 + .../runner/test_cases/signal_install_lat.sh | 21 + .../runner/test_cases/signal_prot_lat.meta | 12 + .../runner/test_cases/signal_prot_lat.sh | 21 + .../test_cases/tcp_loopback_bw_128.meta | 12 + .../runner/test_cases/tcp_loopback_bw_128.sh | 48 ++ .../runner/test_cases/tcp_loopback_bw_4k.meta | 10 + .../runner/test_cases/tcp_loopback_bw_4k.sh | 45 ++ .../test_cases/tcp_loopback_bw_64k.meta | 10 + .../runner/test_cases/tcp_loopback_bw_64k.sh | 45 ++ .../test_cases/tcp_loopback_connect_lat.meta | 11 + .../test_cases/tcp_loopback_connect_lat.sh | 45 ++ .../test_cases/tcp_loopback_http_bw.meta | 11 + .../runner/test_cases/tcp_loopback_http_bw.sh | 56 +++ .../runner/test_cases/tcp_loopback_lat.meta | 12 + .../runner/test_cases/tcp_loopback_lat.sh | 45 ++ .../test_cases/tcp_loopback_select_lat.meta | 11 + .../test_cases/tcp_loopback_select_lat.sh | 21 + .../runner/test_cases/tcp_virtio_bw_128.meta | 12 + .../runner/test_cases/tcp_virtio_bw_128.sh | 48 ++ .../runner/test_cases/tcp_virtio_bw_64k.meta | 12 + .../runner/test_cases/tcp_virtio_bw_64k.sh | 48 ++ .../test_cases/tcp_virtio_connect_lat.meta | 10 + .../test_cases/tcp_virtio_connect_lat.sh | 48 ++ .../runner/test_cases/tcp_virtio_lat.meta | 11 + .../runner/test_cases/tcp_virtio_lat.sh | 48 ++ .../runner/test_cases/test_args_parser.sh | 41 ++ .../test_tcp_loopback_lat_parser.sh | 43 ++ .../test_cases/test_wrapper_portability.sh | 206 +++++++++ .../runner/test_cases/udp_loopback_lat.meta | 11 + .../runner/test_cases/udp_loopback_lat.sh | 45 ++ .../runner/test_cases/udp_virtio_lat.meta | 10 + .../runner/test_cases/udp_virtio_lat.sh | 46 ++ .../lmbench/runner/test_cases/unix_bw.meta | 12 + .../lmbench/runner/test_cases/unix_bw.sh | 21 + .../runner/test_cases/unix_connect_lat.meta | 12 + .../runner/test_cases/unix_connect_lat.sh | 46 ++ .../lmbench/runner/test_cases/unix_lat.meta | 11 + .../lmbench/runner/test_cases/unix_lat.sh | 21 + .../runner/test_cases/vfs_fcntl_lat.meta | 11 + .../runner/test_cases/vfs_fcntl_lat.sh | 21 + .../runner/test_cases/vfs_fstat_lat.meta | 11 + .../runner/test_cases/vfs_fstat_lat.sh | 29 ++ .../runner/test_cases/vfs_open_lat.meta | 11 + .../lmbench/runner/test_cases/vfs_open_lat.sh | 25 + .../runner/test_cases/vfs_read_lat.meta | 11 + .../lmbench/runner/test_cases/vfs_read_lat.sh | 21 + .../test_cases/vfs_read_pagecache_bw.meta | 11 + .../test_cases/vfs_read_pagecache_bw.sh | 23 + .../runner/test_cases/vfs_select_lat.meta | 11 + .../runner/test_cases/vfs_select_lat.sh | 21 + .../runner/test_cases/vfs_stat_lat.meta | 11 + .../lmbench/runner/test_cases/vfs_stat_lat.sh | 29 ++ .../runner/test_cases/vfs_write_lat.meta | 11 + .../runner/test_cases/vfs_write_lat.sh | 21 + .../lmbench/tests/test_fifo_driver.sh | 32 ++ .../benchmark/lmbench/tests/test_fixtures.sh | 37 ++ .../benchmark/lmbench/tests/test_fs_binary.py | 66 +++ .../lmbench/tests/test_linux_entry.sh | 238 ++++++++++ .../benchmark/lmbench/tests/test_runner.sh | 282 ++++++++++++ .../tests/benchmark/lmbench/whitelist.txt | 70 +++ 118 files changed, 4204 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-30-lmbench-fullsuite-survey.md create mode 100644 user/apps/tests/README.md create mode 100644 user/apps/tests/benchmark/README.md create mode 100644 user/apps/tests/benchmark/lmbench/.gitignore create mode 100644 user/apps/tests/benchmark/lmbench/README.md create mode 100644 user/apps/tests/benchmark/lmbench/config create mode 100755 user/apps/tests/benchmark/lmbench/run_linux.sh create mode 100755 user/apps/tests/benchmark/lmbench/runner/clean_up.sh create mode 100755 user/apps/tests/benchmark/lmbench/runner/env.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/fifo_cleanup.sh create mode 100755 user/apps/tests/benchmark/lmbench/runner/init.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/result.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/run.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_copy_files_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_copy_files_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_0k_ops.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_0k_ops.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_10k_ops.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_10k_ops.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/fifo_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/fifo_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_copy_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_copy_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_pagefault_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_pagefault_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_read_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_read_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_write_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/mem_write_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/process_ctx_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/process_ctx_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/process_exec_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/process_exec_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/process_fork_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/process_fork_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/process_getppid_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/process_getppid_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/process_shell_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/process_shell_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_copy_files_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_copy_files_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_0k_ops.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_0k_ops.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_10k_ops.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_10k_ops.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/semaphore_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/semaphore_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/signal_catch_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/signal_catch_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/signal_install_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/signal_install_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/signal_prot_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/signal_prot_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_128.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_128.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_4k.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_4k.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_64k.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_64k.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_connect_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_connect_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_http_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_http_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_select_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_select_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_128.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_128.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_64k.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_64k.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_connect_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_connect_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/test_args_parser.sh create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/test_tcp_loopback_lat_parser.sh create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/test_wrapper_portability.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/udp_loopback_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/udp_loopback_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/udp_virtio_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/udp_virtio_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/unix_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/unix_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/unix_connect_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/unix_connect_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/unix_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/unix_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fcntl_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fcntl_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fstat_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fstat_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_open_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_open_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_pagecache_bw.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_pagecache_bw.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_select_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_select_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_stat_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_stat_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_write_lat.meta create mode 100755 user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_write_lat.sh create mode 100644 user/apps/tests/benchmark/lmbench/tests/test_fifo_driver.sh create mode 100644 user/apps/tests/benchmark/lmbench/tests/test_fixtures.sh create mode 100644 user/apps/tests/benchmark/lmbench/tests/test_fs_binary.py create mode 100755 user/apps/tests/benchmark/lmbench/tests/test_linux_entry.sh create mode 100644 user/apps/tests/benchmark/lmbench/tests/test_runner.sh create mode 100644 user/apps/tests/benchmark/lmbench/whitelist.txt diff --git a/docs/superpowers/specs/2026-07-30-lmbench-fullsuite-survey.md b/docs/superpowers/specs/2026-07-30-lmbench-fullsuite-survey.md new file mode 100644 index 0000000000..f8ae95d612 --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-lmbench-fullsuite-survey.md @@ -0,0 +1,136 @@ +# LMbench 全量测例 .meta 提取探查报告 + +- 日期：2026-07-30 +- 分支：feat/lmbench +- 对应 spec：`2026-07-30-lmbench-fullsuite-meta-extraction-design.md` + +## 概述 + +为 48 个 LMbench 测例提取/复核 `.meta` 抽取规则，扩 whitelist 全量。 + +**方法**：host（WSL2 glibc 2.35）跑不了 24.04 glibc 2.39 编译的 lmbench 二进制，改用 archive 自带的 `ld-linux-x86-64.so.2` + `--library-path` 直接在 host 跑（`/tmp/lmbench-survey/run.sh` helper），不启 QEMU、不依赖 docker（docker.io 拉取超时）。 + +- 能跑通的工具：直接跑看完整输出。 +- host 跑不通/卡死的工具（lat_fifo/bw_unix/lat_sem/lat_unix_connect/lat_syscall null/bw_tcp 等）：用 `strings` 从二进制提取输出格式字符串 + lmbench 标准格式推断，标注"待 guest 校准"。 +- 每个新 `.meta` 用 host 侧 `extract_value`（busybox sh，和 guest 一致）抽样验证抽取正确。 + +## .meta 提取结果（按组） + +格式列说明：`SEARCH_PATTERN` / `RESULT_INDEX`。校准状态：**已校准**=原 8 个 e2e 验证；**host 确认**=host 跑通看输出；**strings 推断**=host 跑不通，基于 strings + lmbench 标准格式。 + +### memory（6） +| 测例 | 工具 | 输出样本 | SEARCH_PATTERN / IDX | 状态 | +| ----------------- | -------------------- | ---------------------------------------- | -------------------- | --------- | +| mem_copy_bw | bw_mem fcp | `67.11 16445.12` | `^[0-9]` / NF | 已校准 | +| mem_read_bw | bw_mem frd | `536.87 33961.94` | `^[0-9]` / NF | host 确认 | +| mem_write_bw | bw_mem fwr | `536.87 10045.11` | `^[0-9]` / NF | host 确认 | +| mem_mmap_bw | bw_mmap_rd mmap_only | `67.11 54358.60` | `^[0-9]` / NF | host 确认 | +| mem_mmap_lat | lat_mmap | `4.194304 20` | `^[0-9]` / NF | host 确认 | +| mem_pagefault_lat | lat_pagefault | `Pagefaults on : 0.1149 microseconds` | `^Pagefaults` / NF-1 | host 确认 | + +### process（5） +| 测例 | 工具 | 输出样本 | SEARCH_PATTERN / IDX | 状态 | +| ------------------- | ---------------- | ---------------------------------------------- | --------------------- | ---------------------------------------- | +| process_fork_lat | lat_proc fork | `Process fork+exit: 143.03 microseconds` | `Process fork` / NF-1 | 已校准 | +| process_exec_lat | lat_proc exec | `Process fork+execve: 156.85 microseconds` | `Process fork` / NF-1 | host 确认 | +| process_shell_lat | lat_proc shell | `Process fork+/bin/sh -c: 936.22 microseconds` | `Process fork` / NF-1 | host 确认（需 `/var/tmp/lmbench/hello`） | +| process_ctx_lat | lat_ctx 18 | `18 28.51` | `^[0-9]` / NF | host 确认 | +| process_getppid_lat | lat_syscall null | `Simple null: N microseconds`（推断） | `Simple null` / NF-1 | strings 推断（host 卡死） | + +### ipc（7） +| 测例 | 工具 | 输出样本 | SEARCH_PATTERN / IDX | 状态 | +| ---------------- | ---------------- | ------------------------------------------------- | -------------------------------------- | ------------------------- | +| pipe_lat | lat_pipe | `Pipe latency: 48.27 microseconds` | `Pipe latency` / NF-1 | 已校准 | +| pipe_bw | bw_pipe | `Pipe bandwidth: 3065.49 MB/sec` | `Pipe bandwidth` / NF-1 | host 确认 | +| fifo_lat | lat_fifo | `Fifo latency: N microseconds` | `Fifo latency` / NF-1 | strings 推断（host 报错） | +| unix_lat | lat_unix | `AF_UNIX sock stream latency: 47.97 microseconds` | `AF_UNIX` / NF-1 | 已校准 | +| unix_bw | bw_unix | `AF_UNIX sock stream bandwidth: N MB/sec` | `AF_UNIX sock stream bandwidth` / NF-1 | strings 推断（host 卡死） | +| unix_connect_lat | lat_unix_connect | `UNIX connection cost: N microseconds` | `UNIX connection cost` / NF-1 | strings 推断（host 报错） | +| semaphore_lat | lat_sem | `Semaphore latency: N microseconds` | `Semaphore latency` / NF-1 | strings 推断（host 卡死） | + +### signal（3） +| 测例 | 工具 | 输出样本 | SEARCH_PATTERN / IDX | 状态 | +| ------------------ | --------------- | --------------------------------------------- | ------------------------------------ | ---------------------------- | +| signal_catch_lat | lat_sig catch | `Signal handler overhead: N microseconds` | `Signal handler` / NF-1 | 已校准 | +| signal_install_lat | lat_sig install | `Signal handler installation: N microseconds` | `Signal handler installation` / NF-1 | strings 确认 | +| signal_prot_lat | lat_sig prot | `Protection fault: N microseconds` | `Protection fault` / NF-1 | strings 确认（需 test file） | + +### vfs（8） +| 测例 | 工具 | 输出样本 | SEARCH_PATTERN / IDX | 状态 | +| --------------------- | ------------------ | ------------------------------------ | --------------------------- | ------------ | +| vfs_open_lat | lat_syscall open | `Simple open/close: N microseconds` | `open/close` / NF-1 | 已校准 | +| vfs_read_lat | lat_syscall read | `Simple read: 0.10 microseconds` | `Simple read` / NF-1 | strings 确认 | +| vfs_write_lat | lat_syscall write | `Simple write: N microseconds` | `Simple write` / NF-1 | strings 确认 | +| vfs_fstat_lat | lat_syscall fstat | `Simple fstat: N microseconds` | `Simple fstat` / NF-1 | strings 确认 | +| vfs_stat_lat | lat_syscall stat | `Simple stat: N microseconds` | `Simple stat` / NF-1 | strings 确认 | +| vfs_fcntl_lat | lat_fcntl | `Fcntl lock latency: N microseconds` | `Fcntl lock latency` / NF-1 | strings 确认 | +| vfs_select_lat | lat_select file | `Select on N fd's: M microseconds` | `Select on` / NF-1 | strings 确认 | +| vfs_read_pagecache_bw | bw_file_rd io_only | ` ` | `^[0-9]` / NF | strings 确认 | + +### filesystem（6） +| 测例 | 工具 | 输出样本 | SEARCH_PATTERN / IDX | 状态 | +| --------------------------------- | ---------- | -------------------------------------- | -------------------- | -------------------------- | +| ext4_create_delete_files_10k_ops | lat_fs 10k | `10k\tN1\tN2\tN3` | `^10k` / 3 | 已校准 | +| ext4_create_delete_files_0k_ops | lat_fs 0k | `0k\tN1\tN2\tN3` | `^0k` / 3 | host 确认 | +| ramfs_create_delete_files_0k_ops | lat_fs 0k | 同上 | `^0k` / 3 | host 确认（用 tmpfs 替代） | +| ramfs_create_delete_files_10k_ops | lat_fs 10k | 同上 | `^10k` / 3 | host 确认 | +| ext4_copy_files_bw | lmdd | `67.11 MB in 0.19 secs, 352.97 MB/sec` | `MB in` / NF-1 | host 确认 | +| ramfs_copy_files_bw | lmdd | 同上 | `MB in` / NF-1 | host 确认 | + +### network（13） +| 测例 | 工具 | 输出样本 | SEARCH_PATTERN / IDX | 状态 | +| -------------------------- | --------------------- | --------------------------------------------- | ------------------------------- | -------------------------------- | +| tcp_loopback_lat | lat_tcp | `TCP latency using 127.0.0.1: N microseconds` | `^TCP latency using` / NF-1 | 已校准 | +| tcp_loopback_bw_128/4k/64k | bw_tcp | `socket connection: N MB/sec` | `socket connection` / NF-1 | strings 推断（host 卡死） | +| tcp_loopback_connect_lat | lat_connect | `TCP/IP connection cost to H: N microseconds` | `TCP/IP connection cost` / NF-1 | strings 推断 | +| tcp_loopback_select_lat | lat_select tcp | `Select on N tcp fd's: M microseconds` | `Select on` / NF-1 | strings 确认 | +| tcp_loopback_http_bw | lat_http+lmhttp | `Avg xfer: NKB, X ms, Y KB/sec` | `Avg xfer` / NF-1 | strings 推断（.sh 缺 file_list） | +| tcp_virtio_lat | lat_tcp 10.0.2.15 | 同 loopback 格式 | `^TCP latency using` / NF-1 | strings 推断（.sh 缺 server） | +| tcp_virtio_bw_128/64k | bw_tcp 10.0.2.15 | 同 loopback | `socket connection` / NF-1 | strings 推断 | +| tcp_virtio_connect_lat | lat_connect 10.0.2.15 | 同 loopback | `TCP/IP connection cost` / NF-1 | strings 推断 | +| udp_loopback_lat | lat_udp | `UDP latency using H: N microseconds` | `^UDP latency using` / NF-1 | strings 确认 | +| udp_virtio_lat | lat_udp 10.0.2.15 | 同 loopback | `^UDP latency using` / NF-1 | strings 推断 | + +## guest 校准状态分类 + +- **已校准（8）**：mem_copy_bw, process_fork_lat, pipe_lat, unix_lat, vfs_open_lat, tcp_loopback_lat, signal_catch_lat, ext4_create_delete_files_10k_ops——e2e 8/8 验证。 +- **host 确认（~18）**：host 跑通看完整输出，格式确定。 +- **strings 推断（~22）**：host 跑不通（卡死/报错），基于 `strings` 提取的输出格式字符串 + lmbench 标准格式。需 guest 校准确认（guest raw_tail 会暴露真实格式）。 + +## guest 可能不支持的测例（留 QEMU 验证确认） + +基于工具用的 syscall / 特性推断，可能受 DragonOS 内核限制： +- `lat_connect` / `lat_select tcp` / `lat_unix_connect`：依赖 socket connect/select，内核未实现的特性可能 failed。 +- `lat_sem`：semaphore syscall。 +- `lat_fcntl`：fcntl lock。 +- `lat_syscall null`：host 已卡死，guest 行为待确认。 +- `tcp_virtio_*` / `udp_virtio_*`：`.sh` 只 client 没 server，guest 跑前需补 server 逻辑。 +- `tcp_loopback_http_bw`：`.sh` 用 `lat_http < file_list` 但未建 file_list，guest 跑前需补。 + +这些是 guest 侧 `.sh`/环境问题，不属于本轮 `.meta` 提取范围；`.meta` 已写，待 guest 校准轮确认或修 `.sh`。 + +## 验证 + +- `.meta` 数：48（`ls runner/test_cases/*.meta | wc -l` = 48）。 +- `.sh`（excl test_ parser）：48，每个都有对应 `.meta`。 +- whitelist：48 条，每条都有 `.meta`。 +- guest 文件未改：`git diff HEAD~7 HEAD -- runner/run.sh runner/env.sh runner/init.sh runner/clean_up.sh 'runner/test_cases/*.sh'` 为空。 +- 每个新 `.meta` 用 host 侧 `extract_value`（busybox sh）抽样验证抽取正确数值。 + +## 不做（留后续） + +- QEMU guest 校准（guest 内核兼容性可能蔓延，本轮只交付 host 产物）。 +- 修 `.sh` server 逻辑（tcp_virtio/udp_virtio 缺 server）。 +- 补 `init.sh` 环境（ramfs 挂载、test_file 创建、hello 二进制）。 +- 修 `tcp_loopback_http_bw.sh` 的 file_list。 +- 内核限制修复。 + +## 容器复核补充（docker 24.04 代理配好后补测） + +`.meta` 提取完成后，docker 代理可用，用 ubuntu:24.04 容器（原生 glibc 2.39，非 archive LD hack）补测 host 跑不通的工具： + +- **容器跑通，格式确认**： + - `lat_syscall null`（process_getppid_lat）：输出 `Simple syscall: microseconds`——**不是** `Simple null`。已修正 `process_getppid_lat.meta` 的 SEARCH_PATTERN 为 `Simple syscall`。 + - `lat_connect`（tcp_loopback/virtio_connect_lat）：输出 `TCP/IP connection cost to : microseconds`，与 strings 推断一致，`.meta` 不变。 +- **容器仍卡死（无输出，timeout rc=124）**：`lat_fifo`/`bw_unix`/`lat_sem`/`bw_tcp`/`lat_select tcp`/`lat_udp`/`lat_unix_connect`——这些用 lmbench `benchmp` 框架（fork + SIGCHLD + waitpid + 共享内存同步），容器与 host 共享同一内核，`benchmp` 的信号竞争死锁在容器里同样发生。`.meta` 保留 strings 推断的 SEARCH_PATTERN（锚定二进制真实输出 prefix），待 guest 校准。 +- **网络工具依赖 libtirpc**：容器原生跑需 `apt install libtirpc3t64`（archive sysroot 自带，host LD 方式不缺；guest DADK 打包会带上依赖）。 \ No newline at end of file diff --git a/user/apps/tests/README.md b/user/apps/tests/README.md new file mode 100644 index 0000000000..59b7d25359 --- /dev/null +++ b/user/apps/tests/README.md @@ -0,0 +1,5 @@ +Test Tookit Overview +========================= + +## Brief Introduction +This Directory contains all using test toolkit and source for DragonOS, including **compatible test** and **performance benchmarks**. \ No newline at end of file diff --git a/user/apps/tests/benchmark/README.md b/user/apps/tests/benchmark/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/user/apps/tests/benchmark/lmbench/.gitignore b/user/apps/tests/benchmark/lmbench/.gitignore new file mode 100644 index 0000000000..ae441bf08f --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/.gitignore @@ -0,0 +1,7 @@ +# Runtime artifacts (kept out of git; results are archived as CI artifacts instead) +/results/ +/install/ +/result +ext4.img +*.out +__pycache__/ diff --git a/user/apps/tests/benchmark/lmbench/README.md b/user/apps/tests/benchmark/lmbench/README.md new file mode 100644 index 0000000000..b9aac83c6a --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/README.md @@ -0,0 +1,101 @@ +# LMbench 性能基准测试套件 + +## 在 Linux 上运行（POSIX sh） + +在仓库根目录执行： + +```sh +sh user/apps/tests/benchmark/lmbench/run_linux.sh --samples 3 + +# 指定另一套 lmbench 二进制，或使用 BusyBox ash 执行 runner 和每个用例： +LMBENCH_BIN_DIR=/path/to/lmbench/bin/x86_64-linux-gnu \ +LMBENCH_SH=/usr/bin/busybox \ + sh user/apps/tests/benchmark/lmbench/run_linux.sh --samples 1 +``` + +入口复用 `runner/run.sh` 及全部白名单用例。默认二进制目录为仓库的 +`bin/sysroot/usr/lib/lmbench/bin/x86_64-linux-gnu`。Linux 需要 `bwrap`、允许非特权用户 +创建 namespace、GNU `timeout`、`findmnt`、`ps` 和常用 POSIX 工具；输出目录必须位于 ext4。 +GNU `timeout` 默认使用 `/usr/bin/timeout`，可用 `LMBENCH_TIMEOUT` 指定绝对路径。 +入口在每次运行开始时调用所选二进制目录中的 `enough`、`timing_o` 和 `loop_o` +完成一次有界校准，并把三项值及其来源写入 `environment.txt` 后传入隔离环境。 +调用者也可显式设置 `ENOUGH`、`TIMING_O`、`LOOP_O`；其中 `ENOUGH` 必须为正整数， +两项 overhead 可为零，显式值会校验、保留并标记为 `explicit`。 +入口在私有 PID、网络、IPC 和挂载 namespace 内连续运行整套用例，ext4 用例使用该输出目录 +下的专用 fixture，内存文件系统用例使用私有 tmpfs；无需 root，也不会挂载宿主 loop 设备。 + +所有用例都使用 `LMBENCH_BIN_DIR` 下同一套预编译二进制，运行时不下载源码、打补丁或编译。 +DADK 与 Nix 的包来源保持一致，输入哈希随结果保存。 + +Linux 入口为原版 `lat_fifo` 启用 `runner/fifo_cleanup.sh`：收到 writer 的 EOF 诊断后， +通过 `/proc` 确认它属于本次 benchmark 的子进程，且其两条 FIFO 都已被 unlink，才终止 +无法响应 SIGTERM 的 writer。该阶段测量数据已交回 controller；驱动不介入计时循环， +由原 controller 正常输出结果并退出。已确认清理后的重复 EOF 诊断会合并计数；其他输出保留。 +仍要求真实退出码为 0 且 latency 有效，超时、测量中失败不会算成功。 +该辅助仅由 Linux 入口启用，依赖 Linux `/proc` 和 `readlink`。 + +原版 `lat_fs` 的零大小用例使用默认大小扫描，并由 metadata 只抽取 `0k` 行； +10k 用例使用 `-s 10k`。四个 wrapper 均设置 `TMPDIR`，确保测量落在指定文件系统。 +零大小调用还会运行 1k、4k、10k，但这些行不计入零大小指标。 + +默认产物写入 `harness/references/lmbench-linux/-sh-validation/`，可通过 +`LMBENCH_LINUX_OUTPUT` 指定新目录；已有目录会被拒绝覆盖。仅保留测试结果： +`results.jsonl`（保留 runner 协议前缀）、每次采样的 `raw/.<轮次>.out` 和 `.rc`、环境、参数及输入哈希。源码快照、构建文件和过程日志使用临时目录，退出时清理。运行结束后会检查存活的 benchmark 进程和 FIFO 文件。 +任一用例失败、缺失或有效样本不足时，入口返回非零。单次采样默认超时 120 秒，整套默认 +上限 7200 秒（`LMBENCH_SUITE_TIMEOUT`）。 + +`stat`、信号保护、`fcntl` 和 page-cache 读取使用 11 次 lmbench 内部采样 +（上游 `TRIES` 的常用值）。原来的 200～1000 次在 `ENOUGH=1000000` 时会耗时数分钟， +超过默认用例期限。`--samples` 控制外层独立运行次数，与内部 `-N` 分开记录。 + +Linux 入口将网络客户端指向私有 namespace 的 `127.0.0.1`；保留 `virtio` 名称的用例在这里 +验证本地协议路径与脚本行为，结果不能解释为 virtio 网卡性能。 + +runner 和用例也支持通过 `LMBENCH_BIN_DIR`、`LMBENCH_TMP_DIR`、`LMBENCH_EXT4_DIR` 覆盖路径。 +提供现有文件系统目录时设置 `LMBENCH_MANAGE_EXT4=0`，并使用专用测试目录；初始化会写入 +fixture。直接运行网络用例时，`LMBENCH_NET_SERVER` 应指向本机地址，默认仍为 guest 的 +`10.0.2.15`。 + +## runner 入口参数 + +`runner/run.sh` 支持以下参数（无参数 = 读 `config` + `whitelist.txt` 全跑，与 rcS 调用一致）： + +``` +run.sh [--samples N] [--timeout S] [--warmup N] + [--whitelist FILE] [--config FILE] [--only NAME] [--list] +``` + +- `--samples N` / `--timeout S` / `--warmup N`：覆盖 `config` 的对应项。 +- `--whitelist FILE` / `--config FILE`：覆盖默认路径（`$SCRIPT_DIR/whitelist.txt` / `$SCRIPT_DIR/config`）。 +- `--only NAME`：只跑单个测例（用于手动调试），等价于临时 whitelist 只含 NAME。 +- `--list`：列出具有 `test_cases/*.meta` 的可用测例名，不跑。 + +优先级：命令行参数 > `--config` 指定文件 > 内置默认。 + +## 添加/启用一个测例 + +1. 在 `whitelist.txt` 里加上测例名（对应 `test_cases/.sh`）。 +2. 确保存在 `test_cases/.sh`（调用某个 lmbench 二进制）。 +3. 写 `test_cases/.meta` 描述如何从输出抽取数值： + +``` +CATEGORY=memory # memory|process|ipc|vfs|network|signal|filesystem|other +BINARY=bw_mem # 底层 lmbench 二进制 +METRIC_TYPE=bandwidth # latency|bandwidth|ops|other +UNIT=MB/s +BIGGER_IS_BETTER=1 # 1=越大越好(带宽/吞吐),0=越小越好(延迟) +SEARCH_PATTERN=^[0-9] # awk 正则,定位结果行 +RESULT_INDEX=NF # awk 字段:数字 | NF | NF-1(取 "microseconds" 前的值) +NTH_OCCURRENCE=1 # 匹配的第几行 +DESCRIPTION=Memory copy bandwidth via bw_mem fcp +# SAMPLES=5 # 可选:覆盖全局采样次数 +``` + +抽取等价于 `awk "/SEARCH_PATTERN/ {print \$(RESULT_INDEX)}"` 取第 `NTH_OCCURRENCE` 个匹配。 +延迟类输出多为 `... : VALUE microseconds`，统一用 `RESULT_INDEX=NF-1`；`bw_mem` 输出 ` `，用 `NF`。 + +## 校准提示 + +各 lmbench 工具输出格式稳定，`.meta` 的抽取规则起点移植自 Asterinas 已验证配置。 +但 `lat_fs`（ext4/ramfs create/delete）等的列布局需在**一次真实运行**后按 `raw_tail` 核对确认 +（相关 `.meta` 已标注 `CALIBRATE`）。当前白名单已包含全部 48 个测例。HTTP 带宽单位按 `lat_http` 的实际输出记为 `MB/s`。 diff --git a/user/apps/tests/benchmark/lmbench/config b/user/apps/tests/benchmark/lmbench/config new file mode 100644 index 0000000000..9937a85af7 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/config @@ -0,0 +1,11 @@ +# LMbench runner 全局配置(KEY=VALUE,被 run.sh 解析;勿加空格/引号) +# 单个测例可用其 .meta 里的 SAMPLES= 覆盖 SAMPLES。 + +# 每个测例重复采样次数(用于算 mean/median/stddev/min/max) +SAMPLES=1 + +# 单个测例单次运行的墙钟超时(秒);超时记为 failed,不阻断整轮 +TIMEOUT_SEC=120 + +# 采样前丢弃的预热轮数(0 表示不额外预热;lmbench 自身也有 warmup) +WARMUP=0 diff --git a/user/apps/tests/benchmark/lmbench/run_linux.sh b/user/apps/tests/benchmark/lmbench/run_linux.sh new file mode 100755 index 0000000000..e12e72a887 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/run_linux.sh @@ -0,0 +1,191 @@ +#!/bin/sh +# Run the same POSIX-sh suite in private Linux namespaces, without host mounts. +set -eu +SUITE=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +# Informational flags do not need binaries, namespaces, builds or result files. +skip_value=0 +for arg do + if [ "$skip_value" = 1 ]; then skip_value=0; continue; fi + case "$arg" in + --samples|--timeout|--warmup|--config|--whitelist|--only) skip_value=1 ;; + --help|-h|--list) exec sh "$SUITE/runner/run.sh" "$@" ;; + esac +done +REPO=$(git -C "$SUITE" rev-parse --show-toplevel) +LMBENCH_BIN_DIR=${LMBENCH_BIN_DIR:-$REPO/bin/sysroot/usr/lib/lmbench/bin/x86_64-linux-gnu} +LMBENCH_SH=${LMBENCH_SH:-/bin/sh} +LINUX_TIMEOUT=${LMBENCH_TIMEOUT:-/usr/bin/timeout} +[ -x "$LINUX_TIMEOUT" ] || { echo "Set LMBENCH_TIMEOUT to GNU timeout" >&2; exit 1; } +LMBENCH_LINUX_OUTPUT=${LMBENCH_LINUX_OUTPUT:-$REPO/harness/references/lmbench-linux/$(date -u +%Y%m%dT%H%M%SZ)-sh-validation} +for tool in bwrap timeout findmnt sha256sum ps awk find; do + command -v "$tool" >/dev/null 2>&1 || { echo "Missing Linux dependency: $tool" >&2; exit 1; } +done +[ -x "$LMBENCH_BIN_DIR/lat_syscall" ] || { echo "Set LMBENCH_BIN_DIR to the lmbench binaries" >&2; exit 1; } +[ -x "$LMBENCH_SH" ] || { echo "Shell is not executable: $LMBENCH_SH" >&2; exit 1; } + +is_positive_integer() { + printf '%s\n' "$1" | awk ' + NR == 1 && /^[0-9]+$/ && $0 !~ /^0+$/ { valid = 1 } + END { exit !(NR == 1 && valid) } + ' +} + +is_nonnegative_decimal() { + printf '%s\n' "$1" | awk ' + NR == 1 && /^[0-9]+([.][0-9]+)?$/ { valid = 1 } + END { exit !(NR == 1 && valid) } + ' +} + +calibrate() { + calibration_tool=$1 + [ -x "$LMBENCH_BIN_DIR/$calibration_tool" ] || { + echo "Calibration tool is not executable: $LMBENCH_BIN_DIR/$calibration_tool" >&2 + return 1 + } + calibration_value=$( + "$LINUX_TIMEOUT" -k 2s 60s "$LMBENCH_BIN_DIR/$calibration_tool" + ) || { + calibration_rc=$? + echo "Calibration tool failed: $calibration_tool (exit $calibration_rc)" >&2 + return 1 + } + printf '%s\n' "$calibration_value" +} + +# Determine the complete calibration environment once, before entering the +# clearenv namespace. Explicit values are authoritative but still validated. +if [ "${ENOUGH+x}" = x ]; then + is_positive_integer "$ENOUGH" || { echo "Invalid explicit ENOUGH: $ENOUGH" >&2; exit 1; } + ENOUGH_SOURCE=explicit +else + ENOUGH=$(calibrate enough) || exit 1 + is_positive_integer "$ENOUGH" || { echo "Invalid output from enough: $ENOUGH" >&2; exit 1; } + ENOUGH_SOURCE=calibrated +fi +export ENOUGH +if [ "${TIMING_O+x}" = x ]; then + is_nonnegative_decimal "$TIMING_O" || { echo "Invalid explicit TIMING_O: $TIMING_O" >&2; exit 1; } + TIMING_O_SOURCE=explicit +else + TIMING_O=$(calibrate timing_o) || exit 1 + is_nonnegative_decimal "$TIMING_O" || { echo "Invalid output from timing_o: $TIMING_O" >&2; exit 1; } + TIMING_O_SOURCE=calibrated +fi +export TIMING_O +if [ "${LOOP_O+x}" = x ]; then + is_nonnegative_decimal "$LOOP_O" || { echo "Invalid explicit LOOP_O: $LOOP_O" >&2; exit 1; } + LOOP_O_SOURCE=explicit +else + LOOP_O=$(calibrate loop_o) || exit 1 + is_nonnegative_decimal "$LOOP_O" || { echo "Invalid output from loop_o: $LOOP_O" >&2; exit 1; } + LOOP_O_SOURCE=calibrated +fi +export ENOUGH TIMING_O LOOP_O +# Refuse to overwrite a previous (including partial) run. +mkdir -p "$(dirname -- "$LMBENCH_LINUX_OUTPUT")" +mkdir "$LMBENCH_LINUX_OUTPUT" +OUT=$(CDPATH='' cd -- "$LMBENCH_LINUX_OUTPUT" && pwd) +printf 'Linux lmbench output: %s\n' "$OUT" +WORK=$(mktemp -d "${TMPDIR:-/tmp}/lmbench-linux.XXXXXX") +cleanup() { rm -rf "$OUT/fixtures" "$WORK"; } +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +mkdir "$OUT/fixtures" "$OUT/raw" "$WORK/source" +# The ext4-labelled cases must actually use ext4; private /tmp is tmpfs. +fs_type=$(findmnt -n -o FSTYPE -T "$OUT/fixtures") +[ "$fs_type" = ext4 ] || { echo "Output directory must be on ext4 (found $fs_type)" >&2; exit 1; } +printf 'runner arguments:' > "$OUT/arguments.txt" +printf ' <%s>' "$@" >> "$OUT/arguments.txt" +printf '\n' >> "$OUT/arguments.txt" +# Resolve and archive host files before /tmp and the working directory change. +# Rotate only the original arguments, preserving spaces without eval or arrays. +remaining=$# +while [ "$remaining" -gt 0 ]; do + arg=$1; shift + remaining=$((remaining - 1)) + case "$arg" in + --config|--whitelist) + [ "$remaining" -gt 0 ] || { echo "Missing value for $arg" >&2; exit 2; } + input_file=$1; shift + remaining=$((remaining - 1)) + selected=selected-${arg#--} + cp -- "$input_file" "$OUT/$selected" + set -- "$@" "$arg" "/tmp/lmbench-output/$selected" + ;; + *) set -- "$@" "$arg" ;; + esac +done +# Execute a snapshot so concurrent checkout edits cannot change a running suite. +cp -R "$SUITE/runner" "$SUITE/config" "$SUITE/whitelist.txt" \ + "$SUITE/run_linux.sh" "$SUITE/README.md" "$WORK/source/" +{ + date -u '+timestamp=%Y-%m-%dT%H:%M:%SZ' + uname -a + printf 'shell=%s\nbinary_dir=%s\nfifo_binary=%s\nfilesystem=%s\n' \ + "$LMBENCH_SH" "$LMBENCH_BIN_DIR" "$LMBENCH_BIN_DIR/lat_fifo" "$fs_type" + printf 'filesystem_binary=%s\n' "$LMBENCH_BIN_DIR/lat_fs" + printf 'network=private namespace; all clients use local addresses, not a virtio NIC\n' + printf 'ENOUGH=%s\nENOUGH_SOURCE=%s\n' "$ENOUGH" "$ENOUGH_SOURCE" + printf 'TIMING_O=%s\nTIMING_O_SOURCE=%s\n' "$TIMING_O" "$TIMING_O_SOURCE" + printf 'LOOP_O=%s\nLOOP_O_SOURCE=%s\n' "$LOOP_O" "$LOOP_O_SOURCE" + cat "$OUT/arguments.txt" + git -C "$REPO" rev-parse HEAD + getconf GNU_LIBC_VERSION + findmnt -T "$OUT" -o TARGET,SOURCE,FSTYPE,OPTIONS +} > "$OUT/environment.txt" +# Hash the exact scripts and the selected, unmodified binary package. +{ + sha256sum "$WORK/source"/runner/*.sh "$WORK/source"/runner/test_cases/*.sh \ + "$WORK/source"/runner/test_cases/*.meta "$WORK/source/config" "$WORK/source/whitelist.txt" \ + "$WORK/source/run_linux.sh" "$LMBENCH_SH" + for selected_file in "$OUT"/selected-*; do + [ ! -f "$selected_file" ] || sha256sum "$selected_file" + done + for binary in "$LMBENCH_BIN_DIR"/*; do + [ ! -f "$binary" ] || sha256sum "$binary" + done +} > "$OUT/inputs.sha256" +cp "$WORK/source/whitelist.txt" "$OUT/whitelist.txt" +# Mount inputs below private /tmp; inputs originally under host /tmp also work. +# The PID namespace contains any daemon that outlives a broken wrapper. +rc=0 +"$LINUX_TIMEOUT" -k 5s "${LMBENCH_SUITE_TIMEOUT:-7200}s" \ + bwrap --unshare-user --uid 0 --gid 0 --unshare-pid --unshare-net \ + --unshare-ipc --unshare-uts --die-with-parent \ + --ro-bind / / --dev /dev --proc /proc --tmpfs /tmp --tmpfs /var/tmp --tmpfs /run \ + --ro-bind "$WORK/source" /tmp/lmbench-suite --ro-bind "$LMBENCH_BIN_DIR" /tmp/lmbench-bin \ + --ro-bind "$LINUX_TIMEOUT" /tmp/lmbench-timeout \ + --ro-bind "$LMBENCH_SH" /tmp/lmbench-shell/sh --bind "$OUT" /tmp/lmbench-output \ + --chdir /tmp --clearenv --setenv PATH /usr/bin:/bin --setenv LC_ALL C \ + --setenv ENOUGH "$ENOUGH" --setenv TIMING_O "$TIMING_O" --setenv LOOP_O "$LOOP_O" \ + --setenv LMBENCH_SH /tmp/lmbench-shell/sh \ + --setenv LMBENCH_TIMEOUT /tmp/lmbench-timeout \ + --setenv LMBENCH_BIN_DIR /tmp/lmbench-bin --setenv LMBENCH_FIFO_CLEANUP 1 \ + --setenv LMBENCH_EXT4_DIR /tmp/lmbench-output/fixtures --setenv LMBENCH_TMP_DIR /tmp \ + --setenv LMBENCH_RUN_TMP /tmp/lmbench-output/raw --setenv LMBENCH_MANAGE_EXT4 0 \ + --setenv LMBENCH_NET_SERVER 127.0.0.1 \ + /tmp/lmbench-shell/sh -c ' + rc=0 + "$LMBENCH_SH" /tmp/lmbench-suite/runner/run.sh "$@" || rc=$? + ps -eo pid,ppid,stat,comm > /tmp/lmbench-processes-after.txt + if ! awk '\''$3 !~ /^Z/ && $4 ~ /^(lat_|bw_|lmhttp|lmdd|enough|lmbench-fifo|lmbench-fs)/ { found=1 } END { exit found }'\'' \ + /tmp/lmbench-processes-after.txt; then + echo "ERROR: benchmark processes survived cleanup" >&2 + rc=1 + fi + find /tmp /var/tmp -type p > /tmp/lmbench-fifos-after.txt + if [ -s /tmp/lmbench-fifos-after.txt ]; then + echo "ERROR: FIFO files survived cleanup" >&2 + rc=1 + fi + exit "$rc" + ' sh "$@" > "$WORK/runner.log" 2>&1 || rc=$? +printf '%s\n' "$rc" > "$OUT/exit-status.txt" +date -u '+%Y-%m-%dT%H:%M:%SZ' > "$OUT/finished-at.txt" +grep '^LMBENCH_' "$WORK/runner.log" > "$OUT/results.jsonl" || true +grep '^LMBENCH_SUMMARY ' "$WORK/runner.log" || tail -n 20 "$WORK/runner.log" +printf 'Linux lmbench finished: exit=%s, output=%s\n' "$rc" "$OUT" +exit "$rc" diff --git a/user/apps/tests/benchmark/lmbench/runner/clean_up.sh b/user/apps/tests/benchmark/lmbench/runner/clean_up.sh new file mode 100755 index 0000000000..3dacbb7737 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/clean_up.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Clean only resources recorded by init.sh for this run. +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +. "$SCRIPT_DIR/env.sh" +STATE_DIR=${LMBENCH_RUN_TMP:-/tmp/lmbench_run} + +if [ -f "$STATE_DIR/created_files" ]; then + while IFS= read -r file_path || [ -n "$file_path" ]; do + [ -n "$file_path" ] && rm -f "$file_path" + done < "$STATE_DIR/created_files" + rm -f "$STATE_DIR/created_files" +fi +if [ -f "$STATE_DIR/mounted_ext4" ]; then + umount "$LMBENCH_EXT4_DIR" || exit 1 + rm -f "$STATE_DIR/mounted_ext4" +fi +if [ -f "$STATE_DIR/created_ext4_dir" ]; then + rmdir "$LMBENCH_EXT4_DIR" || exit 1 + rm -f "$STATE_DIR/created_ext4_dir" +fi +if [ -f "$STATE_DIR/created_ext4_image" ]; then + IFS= read -r image < "$STATE_DIR/created_ext4_image" + rm -f "$image" "$STATE_DIR/created_ext4_image" +fi +echo 'lmbench test environment cleaned' diff --git a/user/apps/tests/benchmark/lmbench/runner/env.sh b/user/apps/tests/benchmark/lmbench/runner/env.sh new file mode 100755 index 0000000000..2bfd567c56 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/env.sh @@ -0,0 +1,18 @@ +# Defaults for DragonOS; callers may select Linux binaries and isolated fixtures. +export LMBENCH_BIN_DIR="${LMBENCH_BIN_DIR:-/lib/lmbench/bin/x86_64-linux-gnu}" +export LMBENCH_EXT4_DIR="${LMBENCH_EXT4_DIR:-/ext4}" +export LMBENCH_TMP_DIR="${LMBENCH_TMP_DIR:-/tmp}" +export LMBENCH_TEST_FILE="${LMBENCH_TEST_FILE:-test_file}" +export LMBENCH_ZERO_FILE="${LMBENCH_ZERO_FILE:-zero_file}" +export LMBENCH_CREATE_TEST_FILES="${LMBENCH_CREATE_TEST_FILES:-1}" +export LMBENCH_NET_SERVER="${LMBENCH_NET_SERVER:-10.0.2.15}" +# Set to 0 when supplying an existing filesystem directory (the Linux entry). +export LMBENCH_MANAGE_EXT4="${LMBENCH_MANAGE_EXT4:-1}" + +# SCRIPT_DIR is the wrapper's test_cases directory during direct invocation, +# and the runner directory when sourced by run.sh. +if [ -f "$SCRIPT_DIR/../result.sh" ]; then + . "$SCRIPT_DIR/../result.sh" +elif [ -f "$SCRIPT_DIR/result.sh" ]; then + . "$SCRIPT_DIR/result.sh" +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/fifo_cleanup.sh b/user/apps/tests/benchmark/lmbench/runner/fifo_cleanup.sh new file mode 100644 index 0000000000..0c8772bbcc --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/fifo_cleanup.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# Linux driver for the unmodified packaged lat_fifo (-P 1). +# Its writer inherits a SIGTERM handler and loops on EOF during cleanup. +# Only reap that writer AFTER both FIFO paths are unlinked. The benchmark +# has already sent its measured samples to its controller at that point. +# The caller supervises this entire process group (runner/result.sh). +set -eu +# Private Linux tmpfs, not persistent raw results: namespace teardown also +# reclaims this pipe if the supervisor must escalate to SIGKILL. +work=$(mktemp -d "${LMBENCH_TMP_DIR:-/tmp}/fifo-cleanup.XXXXXXXX") +trap 'rm -rf "$work"' 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +mkfifo "$work/output" +"$1" -P 1 > "$work/output" 2>&1 & +bench=$! +reaped=0 +repeated=0 +invalid=0 +while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + '(w) read/write on pipe:'*) + if [ "$reaped" = 1 ]; then + repeated=$((repeated + 1)) + continue + fi + printf '%s\n' "$line" + # Restrict action to this invocation's worker and its own child; + # a diagnostic alone is never sufficient to terminate a process. + workers=$(cat "/proc/$bench/task/$bench/children" 2>/dev/null) || workers= + for worker in $workers; do + writers=$(cat "/proc/$worker/task/$worker/children" 2>/dev/null) || writers= + for writer in $writers; do + # BusyBox readlink accepts one path per invocation. + links=$(for fd in /proc/"$writer"/fd/*; do + readlink "$fd" 2>/dev/null || true + done) + case "$links" in *"/var/tmp/lmbench/lmbench_f1.$worker (deleted)"*) ;; *) continue ;; esac + case "$links" in *"/var/tmp/lmbench/lmbench_f2.$worker (deleted)"*) ;; *) continue ;; esac + kill -KILL "$writer" + reaped=1 + printf 'lmbench: reaped FIFO writer %s after both FIFO paths were unlinked\n' "$writer" + done + done + if [ "$reaped" = 0 ]; then + printf 'lmbench: error: writer diagnostic outside verified FIFO cleanup\n' + invalid=1 + fi + ;; + '(r) read/write on pipe:'*|'(i) read/write on pipe:'*) + invalid=1; printf '%s\n' "$line" ;; + *) printf '%s\n' "$line" ;; + esac +done < "$work/output" +rc=0 +wait "$bench" || rc=$? +if [ "$repeated" -gt 0 ]; then + printf 'lmbench: coalesced %s repeated writer EOF diagnostics after verified cleanup\n' "$repeated" +fi +# Keep the real controller's exit status; result.sh additionally requires a +# positive latency and rejects fatal diagnostics. No timeout becomes success. +[ "$invalid" = 0 ] || rc=1 +exit "$rc" diff --git a/user/apps/tests/benchmark/lmbench/runner/init.sh b/user/apps/tests/benchmark/lmbench/runner/init.sh new file mode 100755 index 0000000000..0061768fed --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/init.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# Prepare fixtures; record ownership so cleanup never unmounts a supplied FS. +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +. "$SCRIPT_DIR/env.sh" +STATE_DIR=${LMBENCH_RUN_TMP:-/tmp/lmbench_run} + +create_ext4_fs() { + if [ "$LMBENCH_MANAGE_EXT4" = 0 ]; then + [ -d "$LMBENCH_EXT4_DIR" ] && [ -w "$LMBENCH_EXT4_DIR" ] + return $? + fi + # A caller-provided existing mount is not owned by this run. + [ -d "$LMBENCH_EXT4_DIR" ] && return 0 + [ "$(id -u)" -eq 0 ] || { + echo 'Creating an ext4 loop mount requires root privileges' >&2; return 1; + } + mkdir -p "$LMBENCH_EXT4_DIR" || return 1 + : > "$STATE_DIR/created_ext4_dir" + image=${LMBENCH_EXT4_IMAGE:-$SCRIPT_DIR/ext4.img} + if [ -e "$image" ] || [ -L "$image" ]; then + echo "Refusing to overwrite existing image: $image" >&2 + return 1 + fi + (set -C; : > "$image") || return 1 + printf '%s\n' "$image" > "$STATE_DIR/created_ext4_image" || return 1 + dd if=/dev/zero of="$image" bs=1M count=1024 || return 1 + mkfs.ext4 -F "$image" || return 1 + mount -o loop "$image" "$LMBENCH_EXT4_DIR" || return 1 + : > "$STATE_DIR/mounted_ext4" +} + +create_one_test_file() { + file_path=$1 + # Never claim or truncate a pre-existing caller file (including symlinks). + if [ -e "$file_path" ] || [ -L "$file_path" ]; then + echo "Refusing to overwrite existing fixture: $file_path" >&2 + return 1 + fi + (set -C; : > "$file_path") || return 1 + printf '%s\n' "$file_path" >> "$STATE_DIR/created_files" || { + rm -f "$file_path"; return 1; + } + dd if=/dev/zero of="$file_path" bs=1M count=64 || return 1 + [ "$(wc -c < "$file_path")" -eq 67108864 ] +} + +main() { + mkdir -p "$STATE_DIR" "$LMBENCH_TMP_DIR" || return 1 + create_ext4_fs || return 1 + if [ "$LMBENCH_CREATE_TEST_FILES" = 1 ]; then + for base in "$LMBENCH_EXT4_DIR" "$LMBENCH_TMP_DIR"; do + create_one_test_file "$base/$LMBENCH_ZERO_FILE" || return 1 + create_one_test_file "$base/$LMBENCH_TEST_FILE" || return 1 + done + fi + # Upstream lat_proc and lat_unix_connect use this fixed directory. + mkdir -p /var/tmp/lmbench || return 1 + if [ ! -e /var/tmp/lmbench/hello ]; then + printf '%s\n' /var/tmp/lmbench/hello >> "$STATE_DIR/created_files" || return 1 + cp "$LMBENCH_BIN_DIR/hello" /var/tmp/lmbench/hello || return 1 + chmod +x /var/tmp/lmbench/hello || return 1 + fi +} + +if main "$@"; then + echo 'lmbench test environment initialized' +else + echo 'lmbench test environment initialization failed' >&2 + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/result.sh b/user/apps/tests/benchmark/lmbench/runner/result.sh new file mode 100644 index 0000000000..fd6e2f3dff --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/result.sh @@ -0,0 +1,180 @@ +#!/bin/sh +# Shared LMbench output validation for direct wrappers and the suite runner. + +lmbench_kv_get() { + _lm_file=$1 _lm_key=$2 + [ -f "$_lm_file" ] || return 0 + while IFS= read -r _lm_line || [ -n "$_lm_line" ]; do + case "$_lm_line" in + "$_lm_key="*) printf '%s\n' "${_lm_line#*=}"; return 0 ;; + esac + done < "$_lm_file" +} + +lmbench_extract_value() { + _lm_out=$1 _lm_pat=$2 _lm_idx=$3 _lm_nth=$4 + _lm_line=$(grep -E "$_lm_pat" "$_lm_out" 2>/dev/null | sed -n "${_lm_nth}p") + [ -n "$_lm_line" ] || return 1 + set -- $_lm_line + case "$_lm_idx" in + NF) eval '_lm_value=${'$#'}' ;; + NF-1) + _lm_field=$(($# - 1)); [ "$_lm_field" -gt 0 ] || return 1 + eval '_lm_value=${'$_lm_field'}' + ;; + *[!0-9]*|'') return 1 ;; + *) [ "$_lm_idx" -le "$#" ] || return 1; eval '_lm_value=${'"$_lm_idx"'}' ;; + esac + printf '%s\n' "$_lm_value" +} + +lmbench_awk() { + if command -v awk >/dev/null 2>&1; then command awk "$@" + else /bin/busybox awk "$@"; fi +} + +lmbench_is_positive_finite() { + case "$1" in ''|*[!0-9.eE+-]*) return 1 ;; esac + printf '%s\n' "$1" | lmbench_awk ' + /^[+-]?[0-9]+([.][0-9]+)?([eE][+-]?[0-9]+)?$/ { + value = $1 + 0 + rendered = tolower(sprintf("%g", value)) + if (value > 0 && value == value && rendered !~ /(inf|nan)/) exit 0 + } + { exit 1 } + ' +} + +lmbench_output_has_fatal() { + grep -Eiq '(^|[^[:alpha:]])(assert(ion)?([^[:alpha:]]|$).*fail|(error|fatal)[[:space:]]*:|error[[:space:]]+on[[:space:]][^:[:cntrl:]]+:|(send|recv)[[:space:]]+failed[[:space:]]*:|benchmark[^[:cntrl:]]*fail|segmentation fault|core dumped|function not implemented|not implemented([^[:alpha:]]|$)|enosys([^[:alpha:]]|$)|internal[[:space:]]+time[- ]?out|timed[[:space:]]+out|capture truncated)' "$1" +} + +lmbench_validate_output() { + _lm_out=$1 _lm_pat=$2 _lm_idx=$3 _lm_nth=$4 + lmbench_output_has_fatal "$_lm_out" && return 1 + _lm_value=$(lmbench_extract_value "$_lm_out" "$_lm_pat" "$_lm_idx" "$_lm_nth") || return 1 + _lm_value=$(printf '%s' "$_lm_value" | tr -d ' \t\r') + lmbench_is_positive_finite "$_lm_value" +} + +lmbench_cleanup_capture() { + if [ -n "${LMBENCH_COMMAND_GROUP:-}" ]; then + kill -TERM "-$LMBENCH_COMMAND_GROUP" 2>/dev/null || true + kill -KILL "-$LMBENCH_COMMAND_GROUP" 2>/dev/null || true + fi + if [ -n "${LMBENCH_COMMAND_PID:-}" ]; then + wait "$LMBENCH_COMMAND_PID" 2>/dev/null || true + fi + LMBENCH_COMMAND_PID= + LMBENCH_COMMAND_GROUP= + # With the supervised command group gone, all inherited FIFO writers are + # closed. The pump's current dd observes EOF and is reaped with the pump. + if [ -n "${LMBENCH_PUMP_PID:-}" ]; then + wait "$LMBENCH_PUMP_PID" 2>/dev/null || true + fi + LMBENCH_PUMP_PID= + [ -z "${LMBENCH_CAPTURE_FIFO:-}" ] || rm -f "$LMBENCH_CAPTURE_FIFO" + [ -z "${LMBENCH_CAPTURE_FILE:-}" ] || rm -f "$LMBENCH_CAPTURE_FILE" + [ -z "${LMBENCH_CAPTURE_CHUNK:-}" ] || rm -f "$LMBENCH_CAPTURE_CHUNK" + LMBENCH_CAPTURE_FIFO= + LMBENCH_CAPTURE_FILE= + LMBENCH_CAPTURE_CHUNK= +} + +lmbench_stream_capture() { + trap - 0 HUP INT TERM + _lm_fifo=$1 _lm_capture=$2 _lm_chunk=$3 + _lm_output_limit=1048576 + _lm_capture_data_limit=65500 + _lm_emitted=0 _lm_captured=0 _lm_total=0 + : > "$_lm_capture" || return 1 + exec 3< "$_lm_fifo" || return 1 + while :; do + dd bs=4096 count=1 <&3 > "$_lm_chunk" 2>/dev/null || true + _lm_bytes=$(wc -c < "$_lm_chunk") + [ "$_lm_bytes" -gt 0 ] || break + _lm_total=$((_lm_total + _lm_bytes)) + if [ "$_lm_emitted" -lt "$_lm_output_limit" ]; then + _lm_room=$((_lm_output_limit - _lm_emitted)) + if [ "$_lm_bytes" -le "$_lm_room" ]; then + cat "$_lm_chunk" + _lm_emitted=$((_lm_emitted + _lm_bytes)) + else + dd if="$_lm_chunk" bs=1 count="$_lm_room" 2>/dev/null + _lm_emitted=$_lm_output_limit + fi + if [ "$_lm_emitted" -eq "$_lm_output_limit" ]; then + printf '\nlmbench: output truncated after 1048576 bytes\n' + fi + fi + if [ "$_lm_captured" -lt "$_lm_capture_data_limit" ]; then + _lm_room=$((_lm_capture_data_limit - _lm_captured)) + if [ "$_lm_bytes" -le "$_lm_room" ]; then + cat "$_lm_chunk" >> "$_lm_capture" + _lm_captured=$((_lm_captured + _lm_bytes)) + else + dd if="$_lm_chunk" bs=1 count="$_lm_room" 2>/dev/null >> "$_lm_capture" + _lm_captured=$_lm_capture_data_limit + fi + fi + done + exec 3<&- + if [ "$_lm_total" -gt "$_lm_capture_data_limit" ]; then + printf '\nlmbench: capture truncated\n' >> "$_lm_capture" + fi + rm -f "$_lm_chunk" +} + +run_lmbench_measurement() { + trap 'lmbench_cleanup_capture; exit 129' HUP + trap 'lmbench_cleanup_capture; exit 130' INT + trap 'lmbench_cleanup_capture; exit 143' TERM + _lm_script=${0##*/} + _lm_meta="$SCRIPT_DIR/${_lm_script%.sh}.meta" + _lm_pat=$(lmbench_kv_get "$_lm_meta" SEARCH_PATTERN); [ -n "$_lm_pat" ] || _lm_pat='^[0-9]' + _lm_idx=$(lmbench_kv_get "$_lm_meta" RESULT_INDEX); [ -n "$_lm_idx" ] || _lm_idx=NF + _lm_nth=$(lmbench_kv_get "$_lm_meta" NTH_OCCURRENCE); [ -n "$_lm_nth" ] || _lm_nth=1 + _lm_dir=${LMBENCH_RUN_TMP:-${LMBENCH_TMP_DIR:-/tmp}} + mkdir -p "$_lm_dir" || return 1 + LMBENCH_CAPTURE_FILE="$_lm_dir/.lmbench-measurement.$$" + LMBENCH_CAPTURE_FIFO="$_lm_dir/.lmbench-stream.$$" + LMBENCH_CAPTURE_CHUNK="$_lm_dir/.lmbench-chunk.$$" + rm -f "$LMBENCH_CAPTURE_FILE" "$LMBENCH_CAPTURE_FIFO" "$LMBENCH_CAPTURE_CHUNK" + mkfifo "$LMBENCH_CAPTURE_FIFO" || { lmbench_cleanup_capture; return 1; } + if [ -n "${LMBENCH_TIMEOUT:-}" ]; then + _lm_supervisor=$LMBENCH_TIMEOUT + elif [ -x /usr/local/bin/timeout ]; then + _lm_supervisor=/usr/local/bin/timeout + elif [ -x /usr/bin/timeout ]; then + _lm_supervisor=/usr/bin/timeout + else + _lm_supervisor=timeout + fi + command -v "$_lm_supervisor" >/dev/null 2>&1 || { lmbench_cleanup_capture; return 1; } + exec 4<&0 || { lmbench_cleanup_capture; return 1; } + lmbench_stream_capture "$LMBENCH_CAPTURE_FIFO" "$LMBENCH_CAPTURE_FILE" "$LMBENCH_CAPTURE_CHUNK" & + LMBENCH_PUMP_PID=$! + # GNU timeout creates a process group without setsid. A zero duration adds + # no deadline; it only gives this helper a bounded command-family target. + "$_lm_supervisor" 0 "$@" <&4 >"$LMBENCH_CAPTURE_FIFO" 2>&1 & + LMBENCH_COMMAND_PID=$! + LMBENCH_COMMAND_GROUP=$LMBENCH_COMMAND_PID + exec 4<&- + if wait "$LMBENCH_COMMAND_PID"; then _lm_rc=0; else _lm_rc=$?; fi + LMBENCH_COMMAND_PID= + kill -TERM "-$LMBENCH_COMMAND_GROUP" 2>/dev/null || true + kill -KILL "-$LMBENCH_COMMAND_GROUP" 2>/dev/null || true + LMBENCH_COMMAND_GROUP= + if wait "$LMBENCH_PUMP_PID"; then :; elif [ "$_lm_rc" -eq 0 ]; then _lm_rc=1; fi + LMBENCH_PUMP_PID= + rm -f "$LMBENCH_CAPTURE_FIFO" + LMBENCH_CAPTURE_FIFO= + if [ "$_lm_rc" -eq 0 ] && ! lmbench_validate_output "$LMBENCH_CAPTURE_FILE" "$_lm_pat" "$_lm_idx" "$_lm_nth"; then + printf '%s\n' 'lmbench: invalid measurement output' >&2 + _lm_rc=1 + fi + rm -f "$LMBENCH_CAPTURE_FILE" "$LMBENCH_CAPTURE_CHUNK" + LMBENCH_CAPTURE_FILE= + LMBENCH_CAPTURE_CHUNK= + return "$_lm_rc" +} diff --git a/user/apps/tests/benchmark/lmbench/runner/run.sh b/user/apps/tests/benchmark/lmbench/runner/run.sh new file mode 100644 index 0000000000..7b4d01fc3e --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/run.sh @@ -0,0 +1,433 @@ +#!/bin/sh +# Shared POSIX-sh LMbench benchmark runner. +# +# Entry point invoked by /etc/init.d/rcS when AUTO_TEST=benchmark. Runs each +# whitelisted case N times, extracts a numeric result per test_cases/.meta, +# computes summary statistics, and emits ONE JSON line per metric (prefixed with +# "LMBENCH_JSON ") to the serial console for the host-side collector to harvest. +# +# Output contract (host collect_results.py depends on it): +# ===LMBENCH_RUN_BEGIN=== +# LMBENCH_META {json} # run-level info known inside the guest +# LMBENCH_JSON {json} # one per metric (JSONL, single line each) +# LMBENCH_SUMMARY {json} # totals +# ===LMBENCH_RUN_END=== +# benchmark测试完成 # completion marker (mirrors gvisor "测试完成") +set -u + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +CASES_DIR="$SCRIPT_DIR/test_cases" +CONFIG_FILE="$SCRIPT_DIR/config" +WHITELIST_FILE="$SCRIPT_DIR/whitelist.txt" +# Support both the installed flat layout and running directly from the checkout. +[ -f "$CONFIG_FILE" ] || CONFIG_FILE="$SCRIPT_DIR/../config" +[ -f "$WHITELIST_FILE" ] || WHITELIST_FILE="$SCRIPT_DIR/../whitelist.txt" +WORK_TMP="${LMBENCH_RUN_TMP:-/tmp/lmbench_run}" +LMBENCH_SH=${LMBENCH_SH:-sh} +CLEANUP_DONE=0 +CLEANUP_STATUS=0 +# Prefer the guest's GNU coreutils installation; Linux normally uses PATH. +if [ -z "${LMBENCH_TIMEOUT:-}" ]; then + if [ -x /usr/local/bin/timeout ]; then + LMBENCH_TIMEOUT=/usr/local/bin/timeout + elif [ -x /usr/bin/timeout ]; then + # An absolute path avoids BusyBox ash selecting its timeout applet. + LMBENCH_TIMEOUT=/usr/bin/timeout + else + LMBENCH_TIMEOUT=timeout + fi +fi + +# ---- global config (overridable via config file) ---- +SAMPLES=5 +TIMEOUT_SEC=120 +WARMUP=0 +SUITE_VERSION="3.0-a9" + +# CLI intent variables (set by parse_args, applied after load_config). +CLI_SAMPLES=""; CLI_TIMEOUT=""; CLI_WARMUP="" +CLI_WHITELIST=""; CLI_CONFIG=""; ONLY_NAME=""; LIST_ONLY=""; HELP_ONLY="" + +log() { echo "[lmbench-runner] $*"; } + +usage() { + cat >&2 <<'EOF' +usage: run.sh [--samples N] [--timeout S] [--warmup N] + [--whitelist FILE] [--config FILE] [--only NAME] [--list] +EOF +} + +# Parse CLI args into CLI_* intent variables (applied after load_config). +# Returns 2 on unknown arg (caller exits); does not exit the shell itself +# so it can be unit-tested via `LMBENCH_RUNNER_NO_MAIN=1 . run.sh`. +parse_args() { + while [ $# -gt 0 ]; do + case "$1" in + --samples|--timeout|--warmup|--whitelist|--config|--only) + if [ "$#" -lt 2 ] || [ -z "$2" ]; then usage; return 2; fi ;; + esac + case "$1" in + --samples) CLI_SAMPLES=$2; shift 2 ;; + --timeout) CLI_TIMEOUT=$2; shift 2 ;; + --warmup) CLI_WARMUP=$2; shift 2 ;; + --whitelist) CLI_WHITELIST=$2; shift 2 ;; + --config) CLI_CONFIG=$2; shift 2 ;; + --only) ONLY_NAME=$2; shift 2 ;; + --list) LIST_ONLY=1; shift ;; + -h|--help) HELP_ONLY=1; usage; return 0 ;; + *) usage; return 2 ;; + esac + done +} + +# Apply CLI overrides on top of config-file values. +apply_cli_overrides() { + [ -n "$CLI_SAMPLES" ] && SAMPLES=$CLI_SAMPLES + [ -n "$CLI_TIMEOUT" ] && TIMEOUT_SEC=$CLI_TIMEOUT + [ -n "$CLI_WARMUP" ] && WARMUP=$CLI_WARMUP + [ -n "$CLI_WHITELIST" ] && WHITELIST_FILE=$CLI_WHITELIST +} + +is_positive_integer() { + case "$1" in ''|0*|*[!0-9]*) return 1 ;; *) return 0 ;; esac +} + +# Read KEY=VALUE from a file without sourcing it (avoids clobbering runner state). +# Use shell builtins only: DragonOS can stall when repeatedly constructing short +# grep|head|cut pipelines during early boot. +kv_get() { + _kv_file=$1 + _kv_key=$2 + [ -f "$_kv_file" ] || return 0 + while IFS= read -r _kv_line || [ -n "$_kv_line" ]; do + case "$_kv_line" in + "$_kv_key="*) + printf '%s\n' "${_kv_line#*=}" + return 0 + ;; + esac + done < "$_kv_file" +} + +load_config() { + [ -f "$CONFIG_FILE" ] || return 0 + v=$(kv_get "$CONFIG_FILE" SAMPLES); [ -n "$v" ] && SAMPLES=$v + v=$(kv_get "$CONFIG_FILE" TIMEOUT_SEC); [ -n "$v" ] && TIMEOUT_SEC=$v + v=$(kv_get "$CONFIG_FILE" WARMUP); [ -n "$v" ] && WARMUP=$v +} + +# Minimal JSON string escaper: backslash, quote, then collapse control chars. +json_escape() { + printf '%s' "$1" \ + | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' \ + | tr '\n\r\t' ' ' +} + +# Is the argument a plain decimal number? +is_number() { + case "$1" in + ''|*[!0-9.eE+-]*) return 1 ;; + *) printf '%s\n' "$1" | grep -Eq '^[-+]?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?$' ;; + esac +} + +# Run "$@" in a dedicated process group with a wall-clock timeout. +# GNU timeout uses setpgid (not setsid), avoiding DragonOS's setsid fault while +# still terminating background servers spawned by a benchmark wrapper. +run_with_timeout() { + secs=$1; outf=$2; shift 2 + "$LMBENCH_TIMEOUT" -k 2s "${secs}s" "$@" >"$outf" 2>&1 + rc=$? + case "$rc" in + 124|137) return 124 ;; + *) return "$rc" ;; + esac +} + +# Extract the metric value from a captured-output file. +# PAT/IDX/NTH are metadata globals. Keep field selection in shell instead of +# interpolating $(NF-1) into awk; DragonOS BusyBox awk rejects that construct. +extract_value() { + lmbench_extract_value "$1" "$PAT" "$IDX" "$NTH" +} + +# Some DragonOS images install BusyBox without standalone applet links. +runner_sort() { + if command -v sort >/dev/null 2>&1; then command sort "$@" + else /bin/busybox sort "$@"; fi +} +runner_awk() { + if command -v awk >/dev/null 2>&1; then command awk "$@" + else /bin/busybox awk "$@"; fi +} + +# Compute stats from space-separated numbers on $1. Echoes: +# count mean median stddev min max cv +# sqrt is done via Newton's method so we never depend on busybox awk's optional +# libm (CONFIG_FEATURE_AWK_LIBM); only + - * / are used. +compute_stats() { + printf '%s\n' $1 | runner_sort -n | runner_awk ' + function nsqrt(x, g, i) { + if (x <= 0) return 0 + g = x + for (i = 0; i < 60; i++) g = (g + x / g) / 2 + return g + } + { v[NR] = $1 + 0; sum += $1 } + END { + n = NR; if (n == 0) { exit } + mean = sum / n + for (i = 1; i <= n; i++) { d = v[i] - mean; ss += d * d } + sd = (n > 1) ? nsqrt(ss / (n - 1)) : 0 + median = (n % 2) ? v[(n + 1) / 2] : (v[n / 2] + v[n / 2 + 1]) / 2 + cv = (mean != 0) ? sd / mean : 0 + printf "%d %.6f %.6f %.6f %.6f %.6f %.6f", n, mean, median, sd, v[1], v[n], cv + }' +} + +# Common metric JSON prefix built from .meta fields (no trailing brace). +metric_head() { + _big=false; [ "$BIGGER" = "1" ] && _big=true + printf '{"name":"%s","category":"%s","binary":"%s","metric_type":"%s","unit":"%s","bigger_is_better":%s,"description":"%s"' \ + "$NAME" "$CATEGORY" "$BINARY" "$MTYPE" "$UNIT" "$_big" "$(json_escape "$DESC")" +} + +# Run one whitelisted case; emit its LMBENCH_JSON line. +# Returns 0=ok, 1=failed, 2=skipped. +run_one_case() { + NAME=$1 + case_sh="$CASES_DIR/$NAME.sh" + meta="$CASES_DIR/$NAME.meta" + + if [ ! -f "$case_sh" ]; then + log "SKIP $NAME: test_cases/$NAME.sh not found" + printf 'LMBENCH_JSON {"name":"%s","metric_type":"other","unit":"","bigger_is_better":false,"status":"skipped","error":"missing script"}\n' "$NAME" + return 2 + fi + + # metadata (with defaults) + log "loading metadata for $NAME" + CATEGORY=$(kv_get "$meta" CATEGORY); [ -n "$CATEGORY" ] || CATEGORY=other + BINARY=$(kv_get "$meta" BINARY); [ -n "$BINARY" ] || BINARY="" + MTYPE=$(kv_get "$meta" METRIC_TYPE); [ -n "$MTYPE" ] || MTYPE=other + UNIT=$(kv_get "$meta" UNIT); [ -n "$UNIT" ] || UNIT="" + BIGGER=$(kv_get "$meta" BIGGER_IS_BETTER); [ -n "$BIGGER" ] || BIGGER=0 + PAT=$(kv_get "$meta" SEARCH_PATTERN); [ -n "$PAT" ] || PAT='^[0-9]' + IDX=$(kv_get "$meta" RESULT_INDEX); [ -n "$IDX" ] || IDX=NF + NTH=$(kv_get "$meta" NTH_OCCURRENCE); [ -n "$NTH" ] || NTH=1 + DESC=$(kv_get "$meta" DESCRIPTION) + n_samples=$(kv_get "$meta" SAMPLES); [ -n "$n_samples" ] || n_samples=$SAMPLES + + if ! is_positive_integer "$n_samples"; then + printf 'LMBENCH_JSON %s,"status":"failed","error":"invalid sample count"}\n' "$(metric_head)" + log "FAIL $NAME: invalid sample count" + return 1 + fi + log "RUN $NAME (samples=$n_samples, timeout=${TIMEOUT_SEC}s)" + + samples="" + last_out="$WORK_TMP/$NAME.out" + last_rc=0 + valid_samples=0 + i=1 + total_iter=$((WARMUP + n_samples)) + idx_iter=0 + while [ "$idx_iter" -lt "$total_iter" ]; do + idx_iter=$((idx_iter + 1)) + last_out="$WORK_TMP/$NAME.$idx_iter.out" + run_with_timeout "$TIMEOUT_SEC" "$last_out" "$LMBENCH_SH" "$case_sh" + last_rc=$? + printf '%s\n' "$last_rc" > "$WORK_TMP/$NAME.$idx_iter.rc" + [ "$idx_iter" -le "$WARMUP" ] && continue # discard warmup rounds + val=$(extract_value "$last_out") + val=$(printf '%s' "$val" | tr -d ' \t\r') + if [ "$last_rc" -eq 0 ] && ! lmbench_output_has_fatal "$last_out" && lmbench_is_positive_finite "$val"; then + samples="$samples $val" + valid_samples=$((valid_samples + 1)) + else + log " $NAME sample $i: no valid numeric result (rc=$last_rc)" + fi + i=$((i + 1)) + done + + samples=$(printf '%s' "$samples" | sed 's/^ *//') + + if [ "$valid_samples" -ne "$n_samples" ]; then + err="incomplete samples ($valid_samples/$n_samples)" + [ "$last_rc" -eq 124 ] && err="$err: timeout" + raw=$(tail -c 300 "$last_out" 2>/dev/null) + printf 'LMBENCH_JSON %s,"status":"failed","error":"%s","raw_tail":"%s"}\n' \ + "$(metric_head)" "$err" "$(json_escape "$raw")" + log "FAIL $NAME: $err" + return 1 + fi + + stats=$(compute_stats "$samples") + if [ -z "$stats" ]; then + printf 'LMBENCH_JSON %s,"status":"failed","error":"stats computation failed"}\n' "$(metric_head)" + log "FAIL $NAME: stats computation failed" + return 1 + fi + set -- $stats + s_count=$1; s_mean=$2; s_median=$3; s_sd=$4; s_min=$5; s_max=$6; s_cv=$7 + samples_json="[$(printf '%s' "$samples" | tr ' ' ',')]" + printf 'LMBENCH_JSON %s,"status":"ok","samples":%s,"stats":{"count":%s,"mean":%s,"median":%s,"stddev":%s,"min":%s,"max":%s,"cv":%s}}\n' \ + "$(metric_head)" "$samples_json" \ + "$s_count" "$s_mean" "$s_median" "$s_sd" "$s_min" "$s_max" "$s_cv" + log "OK $NAME: mean=$s_mean $UNIT (n=$s_count, cv=$s_cv)" + return 0 +} + +# ============================== main ============================== +run_main() { + parse_args "$@" || return $? + [ "$HELP_ONLY" = 1 ] && return 0 + + # --config may redirect the config file before load_config reads it. + if [ -n "$CLI_CONFIG" ]; then + CONFIG_FILE=$CLI_CONFIG + [ -f "$CONFIG_FILE" ] || { log "ERROR: config not found: $CONFIG_FILE"; return 2; } + fi + load_config + apply_cli_overrides + for positive in "$SAMPLES" "$TIMEOUT_SEC"; do + is_positive_integer "$positive" || { usage; return 2; } + done + if [ "$WARMUP" != 0 ] && ! is_positive_integer "$WARMUP"; then usage; return 2; fi + mkdir -p "$WORK_TMP" || return 1 + + if [ "$LIST_ONLY" = "1" ]; then + ls "$CASES_DIR"/*.meta 2>/dev/null | sed 's|.*/||; s|\.meta$||' + return 0 + fi + + # Resolve caller-supplied relative paths before moving to the fixture CWD. + if [ -f "$WHITELIST_FILE" ]; then + WHITELIST_FILE=$(CDPATH='' cd -- "$(dirname -- "$WHITELIST_FILE")" && pwd)/$(basename -- "$WHITELIST_FILE") + fi + + # --only: replace whitelist with a single-line file, reusing run_one_case + # unchanged (no second dispatch branch). + if [ -n "$ONLY_NAME" ]; then + wl_tmp="$WORK_TMP/only_whitelist" + printf '%s\n' "$ONLY_NAME" > "$wl_tmp" + WHITELIST_FILE="$wl_tmp" + fi + + . "$SCRIPT_DIR/env.sh" + export LMBENCH_RUN_TMP="$WORK_TMP" + command -v "$LMBENCH_TIMEOUT" >/dev/null 2>&1 || { + log "ERROR: timeout command not found: $LMBENCH_TIMEOUT"; return 1; + } + trap 'runner_cleanup' 0 + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + +log "LMbench benchmark run starting" +echo "===LMBENCH_RUN_BEGIN===" +printf 'LMBENCH_META {"suite":"lmbench","suite_version":"%s","samples":%s,"timeout_sec":%s,"warmup":%s}\n' \ + "$SUITE_VERSION" "$SAMPLES" "$TIMEOUT_SEC" "$WARMUP" + + if [ -f "$SCRIPT_DIR/init.sh" ]; then + log "initializing test environment (init.sh)..." + if ! "$LMBENCH_SH" "$SCRIPT_DIR/init.sh"; then + log "ERROR: test environment initialization failed" + printf 'LMBENCH_SUMMARY {"total":0,"ok":0,"failed":0,"skipped":0}\n' + echo "===LMBENCH_RUN_END===" + perform_cleanup || true +echo "benchmark测试完成" + exit 1 + fi + fi + + cd "$LMBENCH_TMP_DIR" || return 1 + + # Pre-compute ENOUGH once, as lmbench's scripts/config-run does. Keep the + # timeout bounded and fall back to REAL_SHORT when calibration fails. + if [ -z "${ENOUGH:-}" ]; then + . "$SCRIPT_DIR/env.sh" 2>/dev/null || true + if [ -x "${LMBENCH_BIN_DIR:-}/enough" ]; then + if calibrated_enough=$("$LMBENCH_TIMEOUT" -k 2s 10s "${LMBENCH_BIN_DIR}/enough" 2>/dev/null); then + case "$calibrated_enough" in + '') + ENOUGH=50000 + enough_note='fallback: enough tool returned empty output' + ;; + *[!0-9]*) + ENOUGH=50000 + enough_note='fallback: enough tool returned non-decimal output' + ;; + *[!0]*) + ENOUGH=$calibrated_enough + enough_note='calibrated by enough tool' + ;; + *) + ENOUGH=50000 + enough_note='fallback: enough tool returned zero' + ;; + esac + else + ENOUGH=50000 + enough_note='fallback: enough tool failed' + fi + else + ENOUGH=50000 + enough_note='fallback: enough tool not found' + fi + export ENOUGH + log "ENOUGH=$ENOUGH ($enough_note)" + fi + +if [ ! -f "$WHITELIST_FILE" ]; then + log "ERROR: whitelist not found: $WHITELIST_FILE" + echo "===LMBENCH_RUN_END===" + perform_cleanup || true +echo "benchmark测试完成" + exit 1 +fi + + total=0; ok=0; failed=0; skipped=0 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in ''|\#*) continue ;; esac + total=$((total + 1)) + log "dispatching $line" + run_one_case "$line" + case $? in + 0) ok=$((ok + 1)) ;; + 2) skipped=$((skipped + 1)) ;; + *) failed=$((failed + 1)) ;; + esac + echo "---" +done < "$WHITELIST_FILE" + +printf 'LMBENCH_SUMMARY {"total":%s,"ok":%s,"failed":%s,"skipped":%s}\n' \ + "$total" "$ok" "$failed" "$skipped" +echo "===LMBENCH_RUN_END===" + +log "done: total=$total ok=$ok failed=$failed skipped=$skipped" +perform_cleanup || true +echo "benchmark测试完成" +[ "$total" -gt 0 ] && [ "$failed" -eq 0 ] && [ "$skipped" -eq 0 ] +} + +perform_cleanup() { + if [ "$CLEANUP_DONE" -eq 0 ]; then + CLEANUP_DONE=1 + if [ -f "$SCRIPT_DIR/clean_up.sh" ]; then + "$LMBENCH_SH" "$SCRIPT_DIR/clean_up.sh" || CLEANUP_STATUS=$? + [ "$CLEANUP_STATUS" -eq 0 ] || log "ERROR: cleanup failed" + fi + fi + return "$CLEANUP_STATUS" +} + +runner_cleanup() { + cleanup_rc=$? + trap - 0 HUP INT TERM + if ! perform_cleanup && [ "$cleanup_rc" -eq 0 ]; then cleanup_rc=1; fi + exit "$cleanup_rc" +} + +# Allow sourcing for host-side unit tests without executing the run. +[ "${LMBENCH_RUNNER_NO_MAIN:-0}" = "1" ] || run_main "$@" diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_copy_files_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_copy_files_bw.meta new file mode 100644 index 0000000000..7dea6ce8e7 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_copy_files_bw.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for ext4_copy_files_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=filesystem +BINARY=lmdd +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# lmdd prints " MB in secs, MB/sec"; value is the field before MB/sec. +SEARCH_PATTERN=MB in +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=ext4 file copy bandwidth via lmdd \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_copy_files_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_copy_files_bw.sh new file mode 100755 index 0000000000..1489da148f --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_copy_files_bw.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Test: ext4_copy_files_bw +# Binary: lmdd +# Description: Copy files on ext4 filesystem bandwidth test + + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + + +echo "=== Running ext4_copy_files_bw test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lmdd" if="${LMBENCH_EXT4_DIR}/${LMBENCH_ZERO_FILE}" of="${LMBENCH_EXT4_DIR}/${LMBENCH_TEST_FILE}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_0k_ops.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_0k_ops.meta new file mode 100644 index 0000000000..c8ada05d5d --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_0k_ops.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for ext4_create_delete_files_0k_ops (KEY=VALUE; parsed by run.sh) +CATEGORY=filesystem +BINARY=lat_fs +METRIC_TYPE=ops +UNIT=ops/sec +BIGGER_IS_BETTER=1 +# lat_fs prints a per-size row "\t\t\t"; take column 3 (matches +# the ext4_create_delete_files_10k_ops convention validated in e2e). +SEARCH_PATTERN=^0k +RESULT_INDEX=3 +NTH_OCCURRENCE=1 +DESCRIPTION=ext4 create+delete file ops/sec for 0k-byte files via lat_fs \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_0k_ops.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_0k_ops.sh new file mode 100755 index 0000000000..9e304c5faa --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_0k_ops.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Test: ext4_create_delete_files_0k_ops +# Binary: lat_fs +# Description: Create and delete 0k files on ext4 filesystem + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running ext4_create_delete_files_0k_ops test ===" +# tempnam prefers TMPDIR over its directory argument. Keep both on the fixture. +export TMPDIR="$LMBENCH_EXT4_DIR" +# Unmodified lat_fs includes 0k in its default size sweep; metadata selects +# only that row. Do not rely on -s 0k being distinguished from no size option. +run_lmbench_measurement "$LMBENCH_BIN_DIR/lat_fs" -P 1 "${LMBENCH_EXT4_DIR}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_10k_ops.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_10k_ops.meta new file mode 100644 index 0000000000..2306bc9372 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_10k_ops.meta @@ -0,0 +1,13 @@ +# Extraction/metadata for ext4_create_delete_files_10k_ops +CATEGORY=filesystem +BINARY=lat_fs +METRIC_TYPE=ops +UNIT=ops/sec +BIGGER_IS_BETTER=1 +# CALIBRATE: lat_fs prints a per-size table; column layout must be confirmed against +# real DragonOS output (runner captures raw_tail on no-match to make this trivial). +# Best guess: the "10k" size row, creates/sec column. +SEARCH_PATTERN=^10k +RESULT_INDEX=3 +NTH_OCCURRENCE=1 +DESCRIPTION=ext4 create+delete throughput for 10k-byte files via lat_fs diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_10k_ops.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_10k_ops.sh new file mode 100755 index 0000000000..a11c78aff8 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ext4_create_delete_files_10k_ops.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Test: ext4_create_delete_files_10k_ops +# Binary: lat_fs +# Description: Create and delete 10k files on ext4 filesystem + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running ext4_create_delete_files_10k_ops test ===" +# tempnam prefers TMPDIR over its directory argument. Keep both on the fixture. +export TMPDIR="$LMBENCH_EXT4_DIR" +run_lmbench_measurement "$LMBENCH_BIN_DIR/lat_fs" -s 10k -P 1 "${LMBENCH_EXT4_DIR}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/fifo_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/fifo_lat.meta new file mode 100644 index 0000000000..25c2c72c58 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/fifo_lat.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for fifo_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=ipc +BINARY=lat_fifo +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_fifo prints "Fifo latency: microseconds" (prefix confirmed via strings; +# lat_fifo did not run on host, to be confirmed in guest). +SEARCH_PATTERN=Fifo latency +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=FIFO latency via lat_fifo \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/fifo_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/fifo_lat.sh new file mode 100755 index 0000000000..effb2cc04a --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/fifo_lat.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Test: fifo_lat +# Binary: lat_fifo +# Description: FIFO latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running fifo_lat test ===" +if [ "${LMBENCH_FIFO_CLEANUP:-0}" = 1 ]; then + run_lmbench_measurement "${LMBENCH_SH:-sh}" "$SCRIPT_DIR/../fifo_cleanup.sh" "$LMBENCH_BIN_DIR/lat_fifo" +else + run_lmbench_measurement "$LMBENCH_BIN_DIR/lat_fifo" -P 1 +fi + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_copy_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_copy_bw.meta new file mode 100644 index 0000000000..9fbc05d521 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_copy_bw.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for mem_copy_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=memory +BINARY=bw_mem +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_mem prints " " to stderr; take the bandwidth (last field). +SEARCH_PATTERN=^[0-9] +RESULT_INDEX=NF +NTH_OCCURRENCE=1 +DESCRIPTION=Memory copy bandwidth via bw_mem fcp (64MB) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_copy_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_copy_bw.sh new file mode 100755 index 0000000000..37fd973693 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_copy_bw.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: mem_copy_bw +# Binary: bw_mem +# Description: Memory copy bandwidth test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running mem_copy_bw test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_mem" -P 1 -N 3 64m fcp + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_bw.meta new file mode 100644 index 0000000000..ec74e533a6 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_bw.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for mem_mmap_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=memory +BINARY=bw_mmap_rd +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_mmap_rd prints " "; take the bandwidth (last field). +SEARCH_PATTERN=^[0-9] +RESULT_INDEX=NF +NTH_OCCURRENCE=1 +DESCRIPTION=Memory mmap read bandwidth via bw_mmap_rd mmap_only (16MB) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_bw.sh new file mode 100755 index 0000000000..a7244a4994 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_bw.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Test: mem_mmap_bw +# Binary: bw_mmap_rd +# Description: Memory mmap bandwidth test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running mem_mmap_bw test ===" +# size must be <= test file size (see bw_mmap_rd.c: nbytes > st_size +# => " out of range!"), and >= MINSZ (512B). Use 16m (not 64m): full-size +# 64MB mmap rounds repeatedly slow the guest kernel down (mmap/page-cache +# state), stalling later process_* fork tests. 16m still covers DRAM size for +# bandwidth purposes and keeps the suite green. +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_mmap_rd" 16m mmap_only "${LMBENCH_EXT4_DIR}/${LMBENCH_TEST_FILE}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_lat.meta new file mode 100644 index 0000000000..ef1d1c7a44 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for mem_mmap_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=memory +BINARY=lat_mmap +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_mmap prints " "; take the latency (last field). +SEARCH_PATTERN=^[0-9] +RESULT_INDEX=NF +NTH_OCCURRENCE=1 +DESCRIPTION=Memory mmap latency via lat_mmap (4MB) \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_lat.sh new file mode 100755 index 0000000000..f3bff08536 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_mmap_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: mem_mmap_lat +# Binary: lat_mmap +# Description: Memory mmap latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running mem_mmap_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_mmap" 4m "${LMBENCH_EXT4_DIR}/${LMBENCH_TEST_FILE}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_pagefault_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_pagefault_lat.meta new file mode 100644 index 0000000000..8dcb3b930c --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_pagefault_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for mem_pagefault_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=memory +BINARY=lat_pagefault +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_pagefault prints "Pagefaults on : microseconds". +SEARCH_PATTERN=^Pagefaults +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Page fault latency via lat_pagefault \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_pagefault_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_pagefault_lat.sh new file mode 100755 index 0000000000..de5e639686 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_pagefault_lat.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Test: mem_pagefault_lat +# Binary: lat_pagefault +# Description: Memory page fault latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running mem_pagefault_lat test ===" +# Pagefault the whole passed file; the 64MB fixture would touch every page of +# the full ext4 image-backed file each sample and stall later fork tests. +# Use a small runner-owned file instead (lat_pagefault requires >= 1MB). +# A unique directory avoids overwriting or deleting an existing user file. +PFAULT_PARENT=${LMBENCH_RUN_TMP:-${LMBENCH_TMP_DIR:-/tmp}} +PFAULT_DIR=$PFAULT_PARENT/pagefault.$$ +mkdir "$PFAULT_DIR" +PFAULT_FILE=$PFAULT_DIR/input +cleanup_pagefault_fixture() { + rm -f "$PFAULT_FILE" + rmdir "$PFAULT_DIR" 2>/dev/null || true +} +trap cleanup_pagefault_fixture 0 +dd if=/dev/zero of="$PFAULT_FILE" bs=1M count=8 2>/dev/null +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_pagefault" -P 1 "$PFAULT_FILE" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_read_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_read_bw.meta new file mode 100644 index 0000000000..8c14c7bbe9 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_read_bw.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for mem_read_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=memory +BINARY=bw_mem +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_mem prints " "; take the bandwidth (last field). +SEARCH_PATTERN=^[0-9] +RESULT_INDEX=NF +NTH_OCCURRENCE=1 +DESCRIPTION=Memory read bandwidth via bw_mem frd (512MB) \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_read_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_read_bw.sh new file mode 100755 index 0000000000..8d1ea9aa7d --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_read_bw.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: mem_read_bw +# Binary: bw_mem +# Description: Memory read bandwidth test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running mem_read_bw test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_mem" -P 1 -N 50 512m frd + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_write_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_write_bw.meta new file mode 100644 index 0000000000..82f2bc0503 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_write_bw.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for mem_write_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=memory +BINARY=bw_mem +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_mem prints " "; take the bandwidth (last field). +SEARCH_PATTERN=^[0-9] +RESULT_INDEX=NF +NTH_OCCURRENCE=1 +DESCRIPTION=Memory write bandwidth via bw_mem fwr (512MB) \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_write_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_write_bw.sh new file mode 100755 index 0000000000..98cf19afa9 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/mem_write_bw.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: mem_write_bw +# Binary: bw_mem +# Description: Memory write bandwidth test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running mem_write_bw test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_mem" -P 1 -N 50 512m fwr + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_bw.meta new file mode 100644 index 0000000000..0c8e1c9e07 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_bw.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for pipe_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=ipc +BINARY=bw_pipe +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_pipe prints "Pipe bandwidth: MB/sec"; value is the field before MB/sec. +SEARCH_PATTERN=Pipe bandwidth +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Pipe bandwidth via bw_pipe \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_bw.sh new file mode 100755 index 0000000000..a842f2736c --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_bw.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: pipe_bw +# Binary: bw_pipe +# Description: Pipe bandwidth test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running pipe_bw test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_pipe" -P 1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_lat.meta new file mode 100644 index 0000000000..2f4b0114fe --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for pipe_lat +CATEGORY=ipc +BINARY=lat_pipe +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_pipe prints "Pipe latency: microseconds". +SEARCH_PATTERN=Pipe latency +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Pipe round-trip latency via lat_pipe diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_lat.sh new file mode 100755 index 0000000000..e483030f9e --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/pipe_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: pipe_lat +# Binary: lat_pipe +# Description: Pipe latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running pipe_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_pipe" -P 1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_ctx_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_ctx_lat.meta new file mode 100644 index 0000000000..0767d9be0a --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_ctx_lat.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for process_ctx_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=process +BINARY=lat_ctx +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_ctx prints a header "size=0k ovr=..." then a data row " ". +# The data row is the only one starting with a digit; take the last field. +SEARCH_PATTERN=^[0-9] +RESULT_INDEX=NF +NTH_OCCURRENCE=1 +DESCRIPTION=Context switch latency via lat_ctx (18 procs) \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_ctx_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_ctx_lat.sh new file mode 100755 index 0000000000..63f49a697d --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_ctx_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: process_ctx_lat +# Binary: lat_ctx +# Description: Process context switch latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running process_ctx_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_ctx" -P 1 18 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_exec_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_exec_lat.meta new file mode 100644 index 0000000000..77838f91e0 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_exec_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for process_exec_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=process +BINARY=lat_proc +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_proc exec prints "Process fork+execve: microseconds"; anchor "Process fork". +SEARCH_PATTERN=Process fork +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Process fork+execve latency via lat_proc exec \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_exec_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_exec_lat.sh new file mode 100755 index 0000000000..f01438b10a --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_exec_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: process_exec_lat +# Binary: lat_proc +# Description: Process exec latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running process_exec_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_proc" -P 1 exec + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_fork_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_fork_lat.meta new file mode 100644 index 0000000000..3fd07f65ff --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_fork_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for process_fork_lat +CATEGORY=process +BINARY=lat_proc +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_proc fork prints "Process fork+exit: microseconds"; value is the field before the unit. +SEARCH_PATTERN=Process fork +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Process fork+exit latency via lat_proc fork diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_fork_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_fork_lat.sh new file mode 100755 index 0000000000..7602080593 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_fork_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: process_fork_lat +# Binary: lat_proc +# Description: Process fork latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running process_fork_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_proc" -P 1 fork + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_getppid_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_getppid_lat.meta new file mode 100644 index 0000000000..ef5cdea9b9 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_getppid_lat.meta @@ -0,0 +1,13 @@ +# Extraction/metadata for process_getppid_lat (KEY=VALUE; parsed by run.sh) +# Note: the .sh invokes `lat_syscall null` (null syscall), not getppid directly. +CATEGORY=process +BINARY=lat_syscall +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_syscall null prints "Simple syscall: microseconds" (confirmed in 24.04 +# container; null mode timed out on host with archive-glibc LD hack). +SEARCH_PATTERN=Simple syscall +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Null syscall latency via lat_syscall null (named getppid_lat) \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_getppid_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_getppid_lat.sh new file mode 100755 index 0000000000..91087d0d27 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_getppid_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: process_getppid_lat +# Binary: lat_syscall +# Description: Process getppid syscall latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running process_getppid_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_syscall" -P 1 null + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_shell_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_shell_lat.meta new file mode 100644 index 0000000000..49955ca4dc --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_shell_lat.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for process_shell_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=process +BINARY=lat_proc +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_proc shell prints "Process fork+/bin/sh -c: microseconds"; anchor "Process fork". +# Requires /var/tmp/lmbench/hello (lmbench hello binary) at guest runtime. +SEARCH_PATTERN=Process fork +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Process fork+shell latency via lat_proc shell \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/process_shell_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_shell_lat.sh new file mode 100755 index 0000000000..9fc219caae --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/process_shell_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: process_shell_lat +# Binary: lat_proc +# Description: Process shell latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running process_shell_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_proc" -P 1 shell + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_copy_files_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_copy_files_bw.meta new file mode 100644 index 0000000000..991dbac659 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_copy_files_bw.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for ramfs_copy_files_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=filesystem +BINARY=lmdd +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# lmdd prints " MB in secs, MB/sec"; value is the field before MB/sec. +SEARCH_PATTERN=MB in +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=ramfs file copy bandwidth via lmdd \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_copy_files_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_copy_files_bw.sh new file mode 100755 index 0000000000..0867e9e4b0 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_copy_files_bw.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: ramfs_copy_files_bw +# Binary: lmdd +# Description: Copy files on ramfs bandwidth test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running ramfs_copy_files_bw test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lmdd" if="${LMBENCH_TMP_DIR}/${LMBENCH_ZERO_FILE}" of="${LMBENCH_TMP_DIR}/${LMBENCH_TEST_FILE}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_0k_ops.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_0k_ops.meta new file mode 100644 index 0000000000..7df0ecb546 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_0k_ops.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for ramfs_create_delete_files_0k_ops (KEY=VALUE; parsed by run.sh) +CATEGORY=filesystem +BINARY=lat_fs +METRIC_TYPE=ops +UNIT=ops/sec +BIGGER_IS_BETTER=1 +# lat_fs prints a per-size row "\t\t\t"; take column 3. +SEARCH_PATTERN=^0k +RESULT_INDEX=3 +NTH_OCCURRENCE=1 +DESCRIPTION=ramfs create+delete file ops/sec for 0k-byte files via lat_fs \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_0k_ops.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_0k_ops.sh new file mode 100755 index 0000000000..ce9f7e19e8 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_0k_ops.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Test: ramfs_create_delete_files_0k_ops +# Binary: lat_fs +# Description: Create and delete 0k files on ramfs + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running ramfs_create_delete_files_0k_ops test ===" +# tempnam prefers TMPDIR over its directory argument. Keep both on the fixture. +export TMPDIR="$LMBENCH_TMP_DIR" +# Unmodified lat_fs includes 0k in its default size sweep; metadata selects +# only that row. Do not rely on -s 0k being distinguished from no size option. +run_lmbench_measurement "$LMBENCH_BIN_DIR/lat_fs" -P 1 "${LMBENCH_TMP_DIR}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_10k_ops.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_10k_ops.meta new file mode 100644 index 0000000000..5b269503e8 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_10k_ops.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for ramfs_create_delete_files_10k_ops (KEY=VALUE; parsed by run.sh) +CATEGORY=filesystem +BINARY=lat_fs +METRIC_TYPE=ops +UNIT=ops/sec +BIGGER_IS_BETTER=1 +# lat_fs prints a per-size row "\t\t\t"; take column 3. +SEARCH_PATTERN=^10k +RESULT_INDEX=3 +NTH_OCCURRENCE=1 +DESCRIPTION=ramfs create+delete file ops/sec for 10k-byte files via lat_fs \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_10k_ops.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_10k_ops.sh new file mode 100755 index 0000000000..ea4f2d2fcc --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/ramfs_create_delete_files_10k_ops.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Test: ramfs_create_delete_files_10k_ops +# Binary: lat_fs +# Description: Create and delete 10k files on ramfs + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running ramfs_create_delete_files_10k_ops test ===" +# tempnam prefers TMPDIR over its directory argument. Keep both on the fixture. +export TMPDIR="$LMBENCH_TMP_DIR" +run_lmbench_measurement "$LMBENCH_BIN_DIR/lat_fs" -s 10k -P 1 "${LMBENCH_TMP_DIR}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/semaphore_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/semaphore_lat.meta new file mode 100644 index 0000000000..40f3994bd6 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/semaphore_lat.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for semaphore_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=ipc +BINARY=lat_sem +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_sem prints "Semaphore latency: microseconds" (prefix confirmed via +# strings; lat_sem hung on host, to be confirmed in guest). +SEARCH_PATTERN=Semaphore latency +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Semaphore latency via lat_sem \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/semaphore_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/semaphore_lat.sh new file mode 100755 index 0000000000..a1fc95c36f --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/semaphore_lat.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Test: semaphore_lat +# Binary: lat_sem +# Description: Semaphore latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running semaphore_lat test ===" +# The runner bounds the whole sample, including calibration and cleanup. +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_sem" -P 1 -N 21 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_catch_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_catch_lat.meta new file mode 100644 index 0000000000..1047b202ba --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_catch_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for signal_catch_lat +CATEGORY=signal +BINARY=lat_sig +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_sig catch prints "Signal handler overhead: microseconds". +SEARCH_PATTERN=Signal handler +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Signal catch/handler overhead via lat_sig catch diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_catch_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_catch_lat.sh new file mode 100755 index 0000000000..6302995553 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_catch_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: signal_catch_lat +# Binary: lat_sig +# Description: Signal catch latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running signal_catch_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_sig" -P 1 catch + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_install_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_install_lat.meta new file mode 100644 index 0000000000..6d3b98a21f --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_install_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for signal_install_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=signal +BINARY=lat_sig +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_sig install prints "Signal handler installation: microseconds". +SEARCH_PATTERN=Signal handler installation +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Signal handler installation latency via lat_sig install \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_install_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_install_lat.sh new file mode 100755 index 0000000000..8b767269b0 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_install_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: signal_install_lat +# Binary: lat_sig +# Description: Signal install latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running signal_install_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_sig" -P 1 install + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_prot_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_prot_lat.meta new file mode 100644 index 0000000000..d234bb0d36 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_prot_lat.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for signal_prot_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=signal +BINARY=lat_sig +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_sig prot prints "Protection fault: microseconds" (prefix confirmed via +# strings; requires a test file at runtime per the .sh). +SEARCH_PATTERN=Protection fault +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=Signal protection fault latency via lat_sig prot \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_prot_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_prot_lat.sh new file mode 100755 index 0000000000..44eaf5effa --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/signal_prot_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: signal_prot_lat +# Binary: lat_sig +# Description: Signal protection latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running signal_prot_lat test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_sig" -W 30 -N 11 prot "${LMBENCH_EXT4_DIR}/${LMBENCH_TEST_FILE}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_128.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_128.meta new file mode 100644 index 0000000000..4cd39fdd11 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_128.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for tcp_loopback_bw_128 (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=bw_tcp +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_tcp client prints " MB/sec" as its final result line; +# anchor on the trailing unit (banners in the .sh never contain "MB/sec"). +SEARCH_PATTERN=MB/sec$ +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP loopback bandwidth via bw_tcp (128B) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_128.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_128.sh new file mode 100755 index 0000000000..87acef7c1f --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_128.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# Test: tcp_loopback_bw_128 +# Binary: bw_tcp +# Description: TCP loopback bandwidth test with 128 byte messages + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + # Explicitly stop the daemon: -s double-forks, so $! is the launcher (already + # dead) and kill "$SERVER_PID" misses the daemon. -S is the canonical shutdown + # and is a harmless no-op if the daemon already exited. + "${LMBENCH_BIN_DIR}/bw_tcp" -S 127.0.0.1 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting TCP server ===" +"${LMBENCH_BIN_DIR}/bw_tcp" -s 127.0.0.1 -b 1 & +SERVER_PID=$! +sleep 2 + +echo "=== Running TCP bandwidth test (128 bytes) ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_tcp" -m 128 -P 1 127.0.0.1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_4k.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_4k.meta new file mode 100644 index 0000000000..77e026f7fa --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_4k.meta @@ -0,0 +1,10 @@ +# Extraction/metadata for tcp_loopback_bw_4k (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=bw_tcp +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +SEARCH_PATTERN=MB/sec$ +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP loopback bandwidth via bw_tcp (4KB) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_4k.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_4k.sh new file mode 100755 index 0000000000..2b4186c526 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_4k.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Test: tcp_loopback_bw_4k +# Binary: bw_tcp +# Description: TCP loopback bandwidth test with 4k messages + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/bw_tcp" -S 127.0.0.1 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting TCP server ===" +"${LMBENCH_BIN_DIR}/bw_tcp" -s 127.0.0.1 -b 1 & +SERVER_PID=$! +sleep 2 + +echo "=== Running TCP bandwidth test (4k) ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_tcp" -m 4096 -P 1 127.0.0.1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_64k.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_64k.meta new file mode 100644 index 0000000000..ca51b11805 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_64k.meta @@ -0,0 +1,10 @@ +# Extraction/metadata for tcp_loopback_bw_64k (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=bw_tcp +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +SEARCH_PATTERN=MB/sec$ +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP loopback bandwidth via bw_tcp (64KB) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_64k.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_64k.sh new file mode 100755 index 0000000000..98b04c1e0b --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_bw_64k.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Test: tcp_loopback_bw_64k +# Binary: bw_tcp +# Description: TCP loopback bandwidth test with 64k messages + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/bw_tcp" -S 127.0.0.1 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting TCP server ===" +"${LMBENCH_BIN_DIR}/bw_tcp" -s 127.0.0.1 -b 1 & +SERVER_PID=$! +sleep 2 + +echo "=== Running TCP bandwidth test (64k) ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_tcp" -m 65536 -P 1 127.0.0.1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_connect_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_connect_lat.meta new file mode 100644 index 0000000000..2539e57eb7 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_connect_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for tcp_loopback_connect_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=lat_connect +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_connect prints "TCP/IP connection cost to : microseconds". +SEARCH_PATTERN=TCP/IP connection cost +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP loopback connect latency via lat_connect (127.0.0.1) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_connect_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_connect_lat.sh new file mode 100755 index 0000000000..e2afac1c3b --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_connect_lat.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Test: tcp_loopback_connect_lat +# Binary: lat_connect +# Description: TCP loopback connection latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/lat_connect" -S 127.0.0.1 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting TCP server ===" +"${LMBENCH_BIN_DIR}/lat_connect" -s 127.0.0.1 & +SERVER_PID=$! +sleep 2 + +echo "=== Running TCP connection latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_connect" 127.0.0.1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_http_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_http_bw.meta new file mode 100644 index 0000000000..e5975fb99d --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_http_bw.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for tcp_loopback_http_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=lat_http +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# lat_http prints "Avg xfer: ... MB/sec"; extract decimal megabytes/sec. +SEARCH_PATTERN=Avg xfer +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP loopback HTTP bandwidth via lat_http (lmhttp server) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_http_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_http_bw.sh new file mode 100755 index 0000000000..3726f237d7 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_http_bw.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# Test: tcp_loopback_http_bw +# Binary: lmhttp, lat_http +# Description: TCP loopback HTTP bandwidth test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" +WEB_DIR="${LMBENCH_TMP_DIR:-/tmp}/lmbench_http" +PORT=8080 + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/lat_http" -S 127.0.0.1 $PORT 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +# Rebuild a small web root. lmhttp serves files relative to its CWD (or +# $DOCROOT); lat_http reads one relative file name per line from stdin. +rm -rf "$WEB_DIR" +mkdir -p "$WEB_DIR" +dd if=/dev/zero of="$WEB_DIR/file1k" bs=1024 count=1 2>/dev/null +dd if=/dev/zero of="$WEB_DIR/file4k" bs=1024 count=4 2>/dev/null +dd if=/dev/zero of="$WEB_DIR/file16k" bs=1024 count=16 2>/dev/null +printf 'file1k\nfile4k\nfile16k\n' > "$WEB_DIR/file_list" + +echo "=== Starting HTTP server ===" +DOCROOT="$WEB_DIR" "${LMBENCH_BIN_DIR}/lmhttp" $PORT & +SERVER_PID=$! +sleep 2 + +echo "=== Running HTTP bandwidth test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_http" 127.0.0.1 $PORT < "$WEB_DIR/file_list" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_lat.meta new file mode 100644 index 0000000000..f428382da8 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_lat.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for tcp_loopback_lat +CATEGORY=network +BINARY=lat_tcp +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_tcp prints "TCP latency using 127.0.0.1: microseconds". +# Anchor the result line so the wrapper banner "Running TCP latency test" cannot match. +SEARCH_PATTERN=^TCP latency using +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP loopback round-trip latency via lat_tcp diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_lat.sh new file mode 100755 index 0000000000..4cf23c3afd --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_lat.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Test: tcp_loopback_lat +# Binary: lat_tcp +# Description: TCP loopback latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/lat_tcp" -S 127.0.0.1 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting TCP server ===" +"${LMBENCH_BIN_DIR}/lat_tcp" -s 127.0.0.1 -b 1 & +SERVER_PID=$! +sleep 2 + +echo "=== Running TCP latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_tcp" -P 1 127.0.0.1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_select_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_select_lat.meta new file mode 100644 index 0000000000..01db6a8aed --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_select_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for tcp_loopback_select_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=lat_select +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_select tcp prints "Select on tcp fd's: microseconds". +SEARCH_PATTERN=Select on +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=select on tcp fd latency via lat_select tcp diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_select_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_select_lat.sh new file mode 100755 index 0000000000..d732640b48 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_loopback_select_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: tcp_loopback_select_lat +# Binary: lat_select +# Description: TCP loopback select latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running TCP select latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_select" -P 1 tcp + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_128.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_128.meta new file mode 100644 index 0000000000..9ba4212ea2 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_128.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for tcp_virtio_bw_128 (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=bw_tcp +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_tcp prints " MB/sec" e.g. "0.000128 0.09 MB/sec"; +# "socket connection" only appears in the perror on connect failure. +SEARCH_PATTERN=^[0-9] +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP virtio bandwidth via bw_tcp (128B, configured local address) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_128.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_128.sh new file mode 100755 index 0000000000..ebc19dc5d1 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_128.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# Test: tcp_virtio_bw_128 +# Binary: bw_tcp +# Description: TCP virtio bandwidth test with 128 byte messages + +set -e + +# Load environment variables +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/bw_tcp" -S "$LMBENCH_NET_SERVER" 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting TCP server ===" +# lmbench server mode (-s) double-forks a daemon and the launcher exits 0 +# immediately; any arguments after -s are ignored. The daemon binds +# 0.0.0.0:31236 (fixed port) and must be stopped via `bw_tcp -S`. +"${LMBENCH_BIN_DIR}/bw_tcp" -s & +SERVER_PID=$! +sleep 2 + +echo "=== Running TCP bandwidth test (128 bytes) ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_tcp" -m 128 -P 1 "$LMBENCH_NET_SERVER" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_64k.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_64k.meta new file mode 100644 index 0000000000..d1de733ca7 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_64k.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for tcp_virtio_bw_64k (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=bw_tcp +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_tcp prints " MB/sec" e.g. "0.065536 8.88 MB/sec"; +# "socket connection" only appears in the perror on connect failure. +SEARCH_PATTERN=^[0-9] +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP virtio bandwidth via bw_tcp (64KB, configured local address) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_64k.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_64k.sh new file mode 100755 index 0000000000..03b3f2ff5d --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_bw_64k.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# Test: tcp_virtio_bw_64k +# Binary: bw_tcp +# Description: TCP virtio bandwidth test with 64k messages + +set -e + +# Load environment variables +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/bw_tcp" -S "$LMBENCH_NET_SERVER" 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting TCP server ===" +# lmbench server mode (-s) double-forks a daemon and the launcher exits 0 +# immediately; any arguments after -s are ignored. The daemon binds +# 0.0.0.0:31236 (fixed port) and must be stopped via `bw_tcp -S`. +"${LMBENCH_BIN_DIR}/bw_tcp" -s & +SERVER_PID=$! +sleep 2 + +echo "=== Running TCP bandwidth test (64k) ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_tcp" -m 65536 -P 1 "$LMBENCH_NET_SERVER" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_connect_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_connect_lat.meta new file mode 100644 index 0000000000..405c543f68 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_connect_lat.meta @@ -0,0 +1,10 @@ +# Extraction/metadata for tcp_virtio_connect_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=lat_connect +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +SEARCH_PATTERN=TCP/IP connection cost +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP virtio connect latency via lat_connect (configured local address) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_connect_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_connect_lat.sh new file mode 100755 index 0000000000..19040a5c49 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_connect_lat.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# Test: tcp_virtio_connect_lat +# Binary: lat_connect +# Description: TCP virtio connection latency test + +set -e + +# Load environment variables +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/lat_connect" -S "$LMBENCH_NET_SERVER" 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting TCP server ===" +# lmbench server mode (-s) double-forks a daemon and the launcher exits 0 +# immediately; any arguments after -s are ignored. The daemon binds +# 0.0.0.0:31237 (fixed port) and must be stopped via `lat_connect -S`. +"${LMBENCH_BIN_DIR}/lat_connect" -s & +SERVER_PID=$! +sleep 2 + +echo "=== Running TCP connection latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_connect" "$LMBENCH_NET_SERVER" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_lat.meta new file mode 100644 index 0000000000..179a9bdf57 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for tcp_virtio_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=lat_tcp +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_tcp prints "TCP latency using : microseconds"; virtio uses configured local address. +SEARCH_PATTERN=^TCP latency using +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=TCP virtio latency via lat_tcp (configured local address) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_lat.sh new file mode 100755 index 0000000000..d4d965c2f6 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/tcp_virtio_lat.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# Test: tcp_virtio_lat +# Binary: lat_tcp +# Description: TCP virtio latency test + +set -e + +# Load environment variables +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/lat_tcp" -S "$LMBENCH_NET_SERVER" 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting TCP server ===" +# lmbench server mode (-s) double-forks a daemon and the launcher exits 0 +# immediately; any arguments after -s are ignored. The daemon binds +# 0.0.0.0:31234 (fixed port) and must be stopped via `lat_tcp -S`. +"${LMBENCH_BIN_DIR}/lat_tcp" -s & +SERVER_PID=$! +sleep 2 + +echo "=== Running TCP latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_tcp" -P 1 "$LMBENCH_NET_SERVER" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/test_args_parser.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/test_args_parser.sh new file mode 100644 index 0000000000..391594ad6f --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/test_args_parser.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Regression test for run.sh argument parsing. + +set -eu + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +RUNNER="$SCRIPT_DIR/../run.sh" + +LMBENCH_RUNNER_NO_MAIN=1 +export LMBENCH_RUNNER_NO_MAIN +. "$RUNNER" + +reset_cli() { + CLI_SAMPLES=""; CLI_TIMEOUT=""; CLI_WARMUP="" + CLI_WHITELIST=""; CLI_CONFIG=""; ONLY_NAME=""; LIST_ONLY="" +} + +# --samples / --only parsed into CLI_* / ONLY_NAME +reset_cli +parse_args --samples 7 --only foo +[ "$CLI_SAMPLES" = "7" ] || { echo "FAIL: CLI_SAMPLES=$CLI_SAMPLES" >&2; exit 1; } +[ "$ONLY_NAME" = "foo" ] || { echo "FAIL: ONLY_NAME=$ONLY_NAME" >&2; exit 1; } + +# --timeout / --warmup / --whitelist / --config / --list +reset_cli +parse_args --timeout 30 --warmup 2 --whitelist /tmp/wl --config /tmp/cfg --list +[ "$CLI_TIMEOUT" = "30" ] || { echo "FAIL: CLI_TIMEOUT" >&2; exit 1; } +[ "$CLI_WARMUP" = "2" ] || { echo "FAIL: CLI_WARMUP" >&2; exit 1; } +[ "$CLI_WHITELIST" = "/tmp/wl" ] || { echo "FAIL: CLI_WHITELIST" >&2; exit 1; } +[ "$CLI_CONFIG" = "/tmp/cfg" ] || { echo "FAIL: CLI_CONFIG" >&2; exit 1; } +[ "$LIST_ONLY" = "1" ] || { echo "FAIL: LIST_ONLY" >&2; exit 1; } + +# unknown arg returns 2 (does not exit the sourced shell) +reset_cli +set +e +parse_args --bogus 2>/dev/null +rc=$? +set -e +[ "$rc" = "2" ] || { echo "FAIL: unknown arg rc=$rc" >&2; exit 1; } + +echo "args parser regression: PASS" \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/test_tcp_loopback_lat_parser.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/test_tcp_loopback_lat_parser.sh new file mode 100755 index 0000000000..b6c7319e2f --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/test_tcp_loopback_lat_parser.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Regression test for tcp_loopback_lat result extraction. +# Metadata globals below are consumed by the sourced extract_value function. +# shellcheck disable=SC2034 + +set -eu + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +RUNNER="$SCRIPT_DIR/../run.sh" +META="$SCRIPT_DIR/tcp_loopback_lat.meta" +TMPDIR=${TMPDIR:-/tmp} +OUTPUT_FILE=$(mktemp "$TMPDIR/lmbench-tcp-parser.XXXXXX") + +cleanup() { + rm -f "$OUTPUT_FILE" +} +trap cleanup EXIT HUP INT TERM + +cat > "$OUTPUT_FILE" <<'EOF' +timeout: warning: timer_create: Invalid argument +=== Starting TCP server === +=== Running TCP latency test === +TCP latency using 127.0.0.1: 4745.8345 microseconds +=== Shutting down server === +Test completed successfully +EOF + +LMBENCH_RUNNER_NO_MAIN=1 +export LMBENCH_RUNNER_NO_MAIN +. "$SCRIPT_DIR/../result.sh" +. "$RUNNER" + +PAT=$(kv_get "$META" SEARCH_PATTERN) +IDX=$(kv_get "$META" RESULT_INDEX) +NTH=$(kv_get "$META" NTH_OCCURRENCE) +value=$(extract_value "$OUTPUT_FILE") + +[ "$value" = "4745.8345" ] || { + printf 'expected 4745.8345, got %s\n' "${value:-}" >&2 + exit 1 +} + +printf 'tcp_loopback_lat parser regression: PASS\n' diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/test_wrapper_portability.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/test_wrapper_portability.sh new file mode 100755 index 0000000000..77f864b07a --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/test_wrapper_portability.sh @@ -0,0 +1,206 @@ +#!/bin/sh +# Exercise wrapper arguments, cleanup and exit status without benchmarking. +# Usage: sh test_wrapper_portability.sh [shell [shell-options ...]] +set -eu + +TEST_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/lmbench-wrappers.XXXXXX") +trap 'rm -rf "$WORK_DIR"' 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +[ "$#" -gt 0 ] || set -- sh + +mkdir -p "$WORK_DIR/runner/test_cases" "$WORK_DIR/bin" "$WORK_DIR/tmp" "$WORK_DIR/ext4" "$WORK_DIR/cwd" +cp "$TEST_DIR"/*.sh "$WORK_DIR/runner/test_cases/" +cp "$TEST_DIR"/*.meta "$WORK_DIR/runner/test_cases/" +[ ! -f "$TEST_DIR/../result.sh" ] || cp "$TEST_DIR/../result.sh" "$WORK_DIR/runner/result.sh" +# Wrappers still source env.sh; isolate defaults while retaining shared result +# validation exactly as the installed flat layout does. +cat > "$WORK_DIR/runner/env.sh" <<'ENV' +. "$SCRIPT_DIR/../result.sh" +ENV +export LMBENCH_BIN_DIR="$WORK_DIR/bin" LMBENCH_TMP_DIR="$WORK_DIR/tmp" LMBENCH_EXT4_DIR="$WORK_DIR/ext4" +export LMBENCH_TEST_FILE=test_file LMBENCH_ZERO_FILE=zero_file LMBENCH_NET_SERVER=192.0.2.7 +export WRAPPER_LOG="$WORK_DIR/invocations" +export FORK_PID_FILE="$WORK_DIR/forked-writer.pid" +cat > "$WORK_DIR/bin/benchmark-stub" <<'STUB' +#!/bin/sh +printf '%s' "${0##*/}" >> "$WRAPPER_LOG" +for arg do printf ' <%s>' "$arg" >> "$WRAPPER_LOG"; done +printf '\n' >> "$WRAPPER_LOG" +case "${1:-}" in -s|-S) exit 0 ;; esac +if [ "${NOISY_HANG:-0}" = 1 ]; then + while :; do printf '0123456789abcdef'; done +fi +if [ "${FORK_WRITER:-0}" = 1 ]; then + ( + trap '' TERM + while :; do printf 'forked writer still holds stdout\n'; sleep 1; done + ) & + printf '%s\n' "$!" > "$FORK_PID_FILE" +fi +if [ "${FAIL_CLIENT:-0}" != 0 ]; then + printf 'RAW_FAILURE_MARKER\n' + exit 23 +fi +case "${BAD_OUTPUT:-}" in + assertion) printf 'assertion failed: clock went backwards\n' ;; + error) printf 'error: benchmark child failed\n' ;; + semaphore_read_error) printf '(r) error on semaphore: Invalid argument\n' ;; + semaphore_write_error) printf '(w) error on initial semaphore: Permission denied\n' ;; + send_failed) printf 'lat_udp client: send failed: Permission denied\n' ;; + recv_failed) printf 'lat_udp client: recv failed: Invalid argument\n' ;; + timeout) printf 'UDP benchmark internal timeout\n' ;; + enosys) printf 'Function not implemented\n' ;; + missing) exit 0 ;; +esac +case "${0##*/}" in + lat_sem) printf 'Semaphore latency: 1.0 microseconds\n' ;; + lat_udp) printf 'UDP latency using 192.0.2.7: 1.0 microseconds\n' ;; + lat_tcp) printf 'TCP latency using 127.0.0.1: 1.0 microseconds\n' ;; + lat_unix_connect) printf 'UNIX connection cost: 1.0 microseconds\n' ;; + bw_tcp) printf '0.000128 1.0 MB/sec\n' ;; + lat_connect) printf 'TCP/IP connection cost to 192.0.2.7: 1.0 microseconds\n' ;; + lat_syscall) + case " $* " in + *' fstat '*) printf 'Simple fstat: 1.0 microseconds\n' ;; + *) printf 'Simple stat: 1.0 microseconds\n' ;; + esac + ;; + lat_http) + http_input=$(cat) + [ "$http_input" = "file1k +file4k +file16k" ] || { printf 'error: HTTP file list stdin missing\n'; exit 42; } + printf 'Avg xfer: 0.5 MB/sec\n' + ;; + *) printf 'benchmark result: 1.0 microseconds\n' ;; +esac +STUB +chmod +x "$WORK_DIR/bin/benchmark-stub" +for binary in lat_sem lat_unix_connect lat_udp lat_tcp bw_tcp lat_connect lat_syscall lat_http lmhttp; do + ln -s benchmark-stub "$WORK_DIR/bin/$binary" +done +ln -s /bin/true "$WORK_DIR/bin/sleep" +PATH="$WORK_DIR/bin:$PATH" +export PATH +cd "$WORK_DIR/cwd" +FAILURES=0 +fail() { printf 'FAIL: %s\n' "$*" >&2; FAILURES=$((FAILURES + 1)); } +run_wrapper() { + wrapper_name=$1; shift + : > "$WRAPPER_LOG" + WRAPPER_RC=0 + "$@" "$WORK_DIR/runner/test_cases/$wrapper_name.sh" > "$WORK_DIR/output" 2>&1 || WRAPPER_RC=$? +} + +for wrapper in semaphore_lat unix_connect_lat; do + run_wrapper "$wrapper" "$@" + [ "$WRAPPER_RC" = 0 ] || fail "$wrapper must use the runner timeout and return success (got $WRAPPER_RC)" +done + +grep -q '^lat_unix_connect <-S>$' "$WRAPPER_LOG" || fail 'Unix connection wrapper must stop its forked server' + +for wrapper in tcp_virtio_lat tcp_virtio_bw_128 tcp_virtio_bw_64k tcp_virtio_connect_lat udp_virtio_lat; do + run_wrapper "$wrapper" "$@" + [ "$WRAPPER_RC" = 0 ] || fail "$wrapper failed ($WRAPPER_RC)" + grep -q '<192.0.2.7>' "$WRAPPER_LOG" || fail "$wrapper must honor the configured target" + grep -q ' <-S> <192.0.2.7>$' "$WRAPPER_LOG" || fail "$wrapper must stop its server" + if [ "$wrapper" = udp_virtio_lat ]; then + grep -q '^lat_udp <-P> <1> <192.0.2.7>$' "$WRAPPER_LOG" || fail 'UDP must run a latency client' + fi +done + +for scenario in assertion error semaphore_read_error semaphore_write_error send_failed recv_failed timeout enosys missing; do + BAD_OUTPUT=$scenario; export BAD_OUTPUT + run_wrapper semaphore_lat "$@" + [ "$WRAPPER_RC" -ne 0 ] || fail "direct wrapper accepted $scenario output" +done +unset BAD_OUTPUT + +# The HTTP wrapper has distinct multi-field output and stdin handling. +run_wrapper tcp_loopback_http_bw "$@" +[ "$WRAPPER_RC" = 0 ] || fail "HTTP wrapper rejected a valid metric ($WRAPPER_RC)" +grep -q '^lat_http <127.0.0.1> <8080>$' "$WRAPPER_LOG" || fail 'HTTP measurement command did not consume file_list stdin' +rm -rf "$LMBENCH_TMP_DIR/lmbench_http" + +for wrapper in vfs_stat_lat vfs_fstat_lat; do + printf 'existing fixture\n' > test_file + printf 'existing fixture\n' > testfile + run_wrapper "$wrapper" "$@" + [ "$WRAPPER_RC" = 0 ] || fail "$wrapper failed ($WRAPPER_RC)" + [ -f test_file ] && [ -f testfile ] || fail "$wrapper must not remove existing files in the caller directory" + [ -z "$(ls -A "$LMBENCH_TMP_DIR")" ] || fail "$wrapper must remove its temporary file" +done + +FAIL_CLIENT=1; export FAIL_CLIENT +run_wrapper tcp_loopback_lat "$@" +[ "$WRAPPER_RC" = 23 ] || fail "cleanup must preserve the client exit code (got $WRAPPER_RC)" +grep -q 'RAW_FAILURE_MARKER' "$WORK_DIR/output" || fail 'failed client raw diagnostics must be replayed' +if find "$LMBENCH_TMP_DIR" -name '.lmbench-*' | grep -q .; then + fail 'failed client left measurement capture files' +fi +grep -q '^lat_tcp <-S> <127.0.0.1>$' "$WRAPPER_LOG" || fail 'failed client must stop its server' +unset FAIL_CLIENT + +NOISY_HANG=1; export NOISY_HANG +WRAPPER_RC=0 +"${LMBENCH_TIMEOUT:-/usr/bin/timeout}" -k 2s 1s "$@" "$WORK_DIR/runner/test_cases/semaphore_lat.sh" > "$WORK_DIR/noisy-output" 2>&1 || WRAPPER_RC=$? +[ "$WRAPPER_RC" = 124 ] || fail "noisy hung wrapper must time out (got $WRAPPER_RC)" +[ "$(wc -c < "$WORK_DIR/noisy-output")" -le 1100000 ] || fail 'noisy wrapper output exceeded its bound' +grep -q 'output truncated after 1048576 bytes' "$WORK_DIR/noisy-output" || fail 'noisy wrapper did not report output truncation' +if find "$LMBENCH_TMP_DIR" -name '.lmbench-*' | grep -q .; then + find "$LMBENCH_TMP_DIR" -name '.lmbench-*' >&2 + fail 'timed-out wrapper left measurement capture files' +fi +if ps -eo args= | grep -F "$WORK_DIR" | grep -v 'grep -F' | grep -q .; then + ps -eo pid=,args= | grep -F "$WORK_DIR" | grep -v 'grep -F' >&2 || true + fail 'timed-out wrapper left measurement or pump processes' +fi +unset NOISY_HANG + +FORK_WRITER=1; export FORK_WRITER +WRAPPER_RC=0 +"${LMBENCH_TIMEOUT:-/usr/bin/timeout}" -k 2s 3s "$@" "$WORK_DIR/runner/test_cases/semaphore_lat.sh" > "$WORK_DIR/fork-output" 2>&1 || WRAPPER_RC=$? +[ "$WRAPPER_RC" = 0 ] || fail "forking measurement wrapper did not finish cleanly (got $WRAPPER_RC)" +forked_pid=$(cat "$WORK_DIR/forked-writer.pid") +if kill -0 "$forked_pid" 2>/dev/null; then fail 'forked FIFO writer survived wrapper completion'; fi +if find "$LMBENCH_TMP_DIR" -name '.lmbench-*' | grep -q .; then + fail 'forking measurement left helper paths' +fi +if ps -eo args= | grep -F "$WORK_DIR" | grep -v 'grep -F' | grep -q .; then + fail 'forking measurement left command-family or pump processes' +fi +unset FORK_WRITER + +NOISY_HANG=1; export NOISY_HANG +: > "$WRAPPER_LOG" +WRAPPER_RC=0 +"$@" "$WORK_DIR/runner/test_cases/tcp_loopback_lat.sh" > "$WORK_DIR/output" 2>&1 & +wrapper_pid=$! +i=0 +while [ "$i" -lt 50 ] && ! find "$LMBENCH_TMP_DIR" -name '.lmbench-stream.*' | grep -q .; do + /bin/sleep 0.02 + i=$((i + 1)) +done +kill -TERM "$wrapper_pid" +wait "$wrapper_pid" || WRAPPER_RC=$? +[ "$WRAPPER_RC" = 143 ] || fail "TERM must terminate the wrapper (got $WRAPPER_RC)" +grep -q '^lat_tcp <-S> <127.0.0.1>$' "$WRAPPER_LOG" || fail 'TERM must stop the server' +if find "$LMBENCH_TMP_DIR" -name '.lmbench-*' | grep -q .; then + fail 'signaled wrapper left measurement capture files' +fi +if ps -eo args= | grep -F "$WORK_DIR" | grep -v 'grep -F' | grep -q .; then + fail 'signaled wrapper left measurement or pump processes' +fi +unset NOISY_HANG + +mv "$WORK_DIR/bin" "$WORK_DIR/bin tools" +LMBENCH_BIN_DIR="$WORK_DIR/bin tools" +export LMBENCH_BIN_DIR +run_wrapper semaphore_lat "$@" +[ "$WRAPPER_RC" = 0 ] || fail 'binary paths with spaces must work' + +[ "$FAILURES" = 0 ] || exit 1 +printf 'wrapper portability regression: PASS\n' diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_loopback_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_loopback_lat.meta new file mode 100644 index 0000000000..ed7a016c7b --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_loopback_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for udp_loopback_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=lat_udp +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_udp prints "UDP latency using : microseconds". +SEARCH_PATTERN=^UDP latency using +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=UDP loopback latency via lat_udp (127.0.0.1) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_loopback_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_loopback_lat.sh new file mode 100755 index 0000000000..2e1b841030 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_loopback_lat.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Test: udp_loopback_lat +# Binary: lat_udp +# Description: UDP loopback latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/lat_udp" -S 127.0.0.1 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting UDP server ===" +"${LMBENCH_BIN_DIR}/lat_udp" -s 127.0.0.1 & +SERVER_PID=$! +sleep 2 + +echo "=== Running UDP latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_udp" -P 1 127.0.0.1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_virtio_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_virtio_lat.meta new file mode 100644 index 0000000000..b4dd610bea --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_virtio_lat.meta @@ -0,0 +1,10 @@ +# Extraction/metadata for udp_virtio_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=network +BINARY=lat_udp +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +SEARCH_PATTERN=^UDP latency using +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=UDP virtio latency via lat_udp (configured local address) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_virtio_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_virtio_lat.sh new file mode 100755 index 0000000000..fa0bb28dc0 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/udp_virtio_lat.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Test: udp_virtio_lat +# Binary: lat_udp +# Description: UDP virtio latency test + +set -e + +# Load environment variables +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/lat_udp" -S "$LMBENCH_NET_SERVER" 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting UDP server ===" +# The server forks a daemon; stop it via lat_udp -S during cleanup. +"${LMBENCH_BIN_DIR}/lat_udp" -s & +SERVER_PID=$! +sleep 2 + +echo "=== Running UDP latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_udp" -P 1 "$LMBENCH_NET_SERVER" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_bw.meta new file mode 100644 index 0000000000..4f87bdea47 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_bw.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for unix_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=ipc +BINARY=bw_unix +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_unix prints "AF_UNIX sock stream bandwidth: MB/sec" (prefix confirmed via +# strings; bw_unix hung on host, to be confirmed in guest). +SEARCH_PATTERN=AF_UNIX sock stream bandwidth +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=AF_UNIX socket bandwidth via bw_unix \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_bw.sh new file mode 100755 index 0000000000..514c9d24db --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_bw.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: unix_bw +# Binary: bw_unix +# Description: Unix domain socket bandwidth test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running Unix domain socket bandwidth test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_unix" -P 1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_connect_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_connect_lat.meta new file mode 100644 index 0000000000..b9abff7752 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_connect_lat.meta @@ -0,0 +1,12 @@ +# Extraction/metadata for unix_connect_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=ipc +BINARY=lat_unix_connect +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_unix_connect prints "UNIX connection cost: microseconds" (prefix confirmed +# via strings; tool did not run on host, to be confirmed in guest). +SEARCH_PATTERN=UNIX connection cost +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=AF_UNIX connect latency via lat_unix_connect \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_connect_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_connect_lat.sh new file mode 100755 index 0000000000..5bb2c78c3e --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_connect_lat.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Test: unix_connect_lat +# Binary: lat_unix_connect +# Description: Unix domain socket connection latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +SERVER_PID="" + +cleanup() { + cleanup_rc=$? + lmbench_cleanup_capture + trap - 0 HUP INT TERM + "${LMBENCH_BIN_DIR}/lat_unix_connect" -S 2>/dev/null || true + if [ ! -z "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + exit "$cleanup_rc" +} + +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Starting Unix socket server ===" +# The server forks; cleanup must use -S to stop the daemon. +"${LMBENCH_BIN_DIR}/lat_unix_connect" -s & +SERVER_PID=$! +sleep 2 + +echo "=== Running Unix socket connection latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_unix_connect" -P 1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_lat.meta new file mode 100644 index 0000000000..506d60d9f0 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for unix_lat +CATEGORY=ipc +BINARY=lat_unix +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_unix prints "AF_UNIX sock stream latency: microseconds". +SEARCH_PATTERN=AF_UNIX +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=AF_UNIX stream socket latency via lat_unix diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_lat.sh new file mode 100755 index 0000000000..15c8e97f8f --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/unix_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: unix_lat +# Binary: lat_unix +# Description: Unix domain socket latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running Unix domain socket latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_unix" -P 1 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fcntl_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fcntl_lat.meta new file mode 100644 index 0000000000..64073c76c8 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fcntl_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for vfs_fcntl_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=vfs +BINARY=lat_fcntl +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_fcntl prints "Fcntl lock latency: microseconds". +SEARCH_PATTERN=Fcntl lock latency +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=fcntl lock latency via lat_fcntl \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fcntl_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fcntl_lat.sh new file mode 100755 index 0000000000..56fc8f5d22 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fcntl_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: vfs_fcntl_lat +# Binary: lat_fcntl +# Description: VFS fcntl latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running VFS fcntl latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_fcntl" -P 1 -W 30 -N 11 + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fstat_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fstat_lat.meta new file mode 100644 index 0000000000..4b6f888a75 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fstat_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for vfs_fstat_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=vfs +BINARY=lat_syscall +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_syscall fstat prints "Simple fstat: microseconds". +SEARCH_PATTERN=Simple fstat +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=fstat syscall latency via lat_syscall fstat \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fstat_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fstat_lat.sh new file mode 100755 index 0000000000..e6daaa35f8 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_fstat_lat.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Test: vfs_fstat_lat +# Binary: lat_syscall +# Description: VFS fstat syscall latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +testfile="$LMBENCH_TMP_DIR/vfs_fstat_lat.$$" +cleanup() { lmbench_cleanup_capture; rm -f "$testfile"; } +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Running VFS fstat latency test ===" +touch "$testfile" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_syscall" -P 1 fstat "$testfile" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_open_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_open_lat.meta new file mode 100644 index 0000000000..5ec4c7b5b6 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_open_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for vfs_open_lat +CATEGORY=vfs +BINARY=lat_syscall +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_syscall open prints "Simple open/close: microseconds". +SEARCH_PATTERN=open/close +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=open/close syscall latency via lat_syscall open diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_open_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_open_lat.sh new file mode 100755 index 0000000000..ccb8a3b4a0 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_open_lat.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Test: vfs_open_lat +# Binary: lat_syscall +# Description: VFS open syscall latency test + +set -e + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +testfile="$LMBENCH_TMP_DIR/vfs_open_lat.$$" +cleanup() { + lmbench_cleanup_capture + rm -f "$testfile" +} +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Running VFS open latency test ===" +touch "$testfile" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_syscall" -P 1 -W 1 -N 3 open "$testfile" +echo "Test completed successfully" diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_lat.meta new file mode 100644 index 0000000000..ca089550cb --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for vfs_read_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=vfs +BINARY=lat_syscall +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_syscall read prints "Simple read: microseconds". +SEARCH_PATTERN=Simple read +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=read syscall latency via lat_syscall read \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_lat.sh new file mode 100755 index 0000000000..9aadb362b9 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: vfs_read_lat +# Binary: lat_syscall +# Description: VFS read syscall latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running VFS read latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_syscall" -P 1 read + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_pagecache_bw.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_pagecache_bw.meta new file mode 100644 index 0000000000..59b34bc19a --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_pagecache_bw.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for vfs_read_pagecache_bw (KEY=VALUE; parsed by run.sh) +CATEGORY=vfs +BINARY=bw_file_rd +METRIC_TYPE=bandwidth +UNIT=MB/s +BIGGER_IS_BETTER=1 +# bw_file_rd prints " "; take the bandwidth (last field). +SEARCH_PATTERN=^[0-9] +RESULT_INDEX=NF +NTH_OCCURRENCE=1 +DESCRIPTION=File read pagecache bandwidth via bw_file_rd io_only (64MB) diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_pagecache_bw.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_pagecache_bw.sh new file mode 100755 index 0000000000..dfe5bef60f --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_read_pagecache_bw.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Test: vfs_read_pagecache_bw +# Binary: bw_file_rd +# Description: VFS read page cache bandwidth test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running VFS read page cache bandwidth test ===" +# size must be <= test file size (64MB fixture recreated by init.sh); +# bw_file_rd validates against the file and fails with perror("x") otherwise. +run_lmbench_measurement "${LMBENCH_BIN_DIR}/bw_file_rd" -P 1 -W 30 -N 11 64m io_only "${LMBENCH_EXT4_DIR}/${LMBENCH_TEST_FILE}" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_select_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_select_lat.meta new file mode 100644 index 0000000000..58bf8c4094 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_select_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for vfs_select_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=vfs +BINARY=lat_select +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_select file prints "Select on fd's: microseconds". +SEARCH_PATTERN=Select on +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=select on file fd latency via lat_select file \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_select_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_select_lat.sh new file mode 100755 index 0000000000..56a67eba3e --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_select_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: vfs_select_lat +# Binary: lat_select +# Description: VFS select latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running VFS select latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_select" -P 1 file + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_stat_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_stat_lat.meta new file mode 100644 index 0000000000..6b2723c200 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_stat_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for vfs_stat_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=vfs +BINARY=lat_syscall +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_syscall stat prints "Simple stat: microseconds". +SEARCH_PATTERN=Simple stat +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=stat syscall latency via lat_syscall stat \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_stat_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_stat_lat.sh new file mode 100755 index 0000000000..7003cdadad --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_stat_lat.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Test: vfs_stat_lat +# Binary: lat_syscall +# Description: VFS stat syscall latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +testfile="$LMBENCH_TMP_DIR/vfs_stat_lat.$$" +cleanup() { lmbench_cleanup_capture; rm -f "$testfile"; } +trap cleanup 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +echo "=== Running VFS stat latency test ===" +touch "$testfile" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_syscall" -P 1 -W 1000 -N 11 stat "$testfile" + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_write_lat.meta b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_write_lat.meta new file mode 100644 index 0000000000..03d958890b --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_write_lat.meta @@ -0,0 +1,11 @@ +# Extraction/metadata for vfs_write_lat (KEY=VALUE; parsed by run.sh) +CATEGORY=vfs +BINARY=lat_syscall +METRIC_TYPE=latency +UNIT=microseconds +BIGGER_IS_BETTER=0 +# lat_syscall write prints "Simple write: microseconds". +SEARCH_PATTERN=Simple write +RESULT_INDEX=NF-1 +NTH_OCCURRENCE=1 +DESCRIPTION=write syscall latency via lat_syscall write \ No newline at end of file diff --git a/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_write_lat.sh b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_write_lat.sh new file mode 100755 index 0000000000..fb9e3460d4 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/runner/test_cases/vfs_write_lat.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Test: vfs_write_lat +# Binary: lat_syscall +# Description: VFS write syscall latency test + +set -e + +# 加载环境变量 +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +ENV_PATH="$SCRIPT_DIR/../env.sh" +. "$ENV_PATH" + +echo "=== Running VFS write latency test ===" +run_lmbench_measurement "${LMBENCH_BIN_DIR}/lat_syscall" -P 1 write + +if [ $? -eq 0 ]; then + echo "Test completed successfully" +else + echo "Test failed" + exit 1 +fi diff --git a/user/apps/tests/benchmark/lmbench/tests/test_fifo_driver.sh b/user/apps/tests/benchmark/lmbench/tests/test_fifo_driver.sh new file mode 100644 index 0000000000..6af704c1e7 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/tests/test_fifo_driver.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Failures must not be converted to success by the FIFO cleanup driver. +set -eu +SUITE=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +TEST_SHELL=${TEST_SHELL:-/bin/sh} +work=$(mktemp -d) +trap 'rm -rf "$work"' 0 +cat > "$work/lat_fifo" <<'STUB' +#!/bin/sh +case "$MODE" in + ok) echo 'Fifo latency: 1.0 microseconds' ;; + exit) echo 'Fifo latency: 1.0 microseconds'; exit 23 ;; + writer) echo '(w) read/write on pipe: Invalid argument'; echo 'Fifo latency: 1.0 microseconds' ;; + reader) echo '(r) read/write on pipe: Invalid argument'; echo 'Fifo latency: 1.0 microseconds' ;; +esac +STUB +chmod +x "$work/lat_fifo" +for MODE in ok exit writer reader; do + export MODE + rc=0 + LMBENCH_TMP_DIR="$work" "$TEST_SHELL" "$SUITE/runner/fifo_cleanup.sh" "$work/lat_fifo" > "$work/$MODE.log" 2>&1 || rc=$? + case "$MODE" in + ok) [ "$rc" = 0 ] ;; + exit) [ "$rc" = 23 ] ;; + writer|reader) [ "$rc" != 0 ] ;; + esac + ! grep -q 'reaped FIFO writer' "$work/$MODE.log" +done +# No helper pipe/directory survives any of these paths. +[ "$(find "$work" -type p | wc -l)" -eq 0 ] +[ "$(find "$work" -type d | wc -l)" -eq 1 ] +echo "FIFO driver failure regression: PASS ($TEST_SHELL)" diff --git a/user/apps/tests/benchmark/lmbench/tests/test_fixtures.sh b/user/apps/tests/benchmark/lmbench/tests/test_fixtures.sh new file mode 100644 index 0000000000..71ce31b226 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/tests/test_fixtures.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Run inside a private mount namespace: no host /tmp or /var/tmp fixtures. +set -eu +SUITE=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +TEST_SHELL=${TEST_SHELL:-/bin/sh} +exec bwrap --unshare-user --unshare-pid --unshare-net --die-with-parent \ + --ro-bind / / --dev /dev --proc /proc --tmpfs /tmp --tmpfs /var/tmp \ + --ro-bind "$TEST_SHELL" /tmp/test-shell/sh /tmp/test-shell/sh -c ' +set -eu +mkdir -p /tmp/ext4 /tmp/ram /tmp/log /tmp/bin +printf "#!/bin/sh\nexit 0\n" > /tmp/bin/hello +chmod +x /tmp/bin/hello +: > /tmp/ext4/unrelated +export LMBENCH_MANAGE_EXT4=0 LMBENCH_EXT4_DIR=/tmp/ext4 +export LMBENCH_TMP_DIR=/tmp/ram LMBENCH_RUN_TMP=/tmp/log LMBENCH_BIN_DIR=/tmp/bin +export LMBENCH_ZERO_FILE=fixture_zero LMBENCH_TEST_FILE=fixture_test +"$1" "$2/runner/init.sh" +for base in /tmp/ext4 /tmp/ram; do + [ "$(wc -c < "$base/fixture_zero")" -eq 67108864 ] + [ "$(wc -c < "$base/fixture_test")" -eq 67108864 ] +done +[ -x /var/tmp/lmbench/hello ] +"$1" "$2/runner/clean_up.sh" +[ -f /tmp/ext4/unrelated ] +[ ! -f /tmp/ext4/fixture_zero ] +[ ! -f /tmp/ram/fixture_test ] +# Repeated cleanup must be harmless. +"$1" "$2/runner/clean_up.sh" +# A supplied filesystem can contain unrelated data with the fixture name. +printf "keep me\n" > /tmp/ext4/fixture_zero +if "$1" "$2/runner/init.sh"; then + echo "FAIL: existing fixture was overwritten"; exit 1 +fi +"$1" "$2/runner/clean_up.sh" +[ "$(cat /tmp/ext4/fixture_zero)" = "keep me" ] +echo "fixture regression: PASS" +' sh /tmp/test-shell/sh "$SUITE" diff --git a/user/apps/tests/benchmark/lmbench/tests/test_fs_binary.py b/user/apps/tests/benchmark/lmbench/tests/test_fs_binary.py new file mode 100644 index 0000000000..71833dc6c4 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/tests/test_fs_binary.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Check filesystem wrappers against the unmodified binary package on Linux.""" +import os +from pathlib import Path +import re +import resource +import signal +import subprocess +import sys +import tempfile + + +def limit_output(): + # Trace/output files are bounded even if the benchmark misbehaves. + resource.setrlimit(resource.RLIMIT_FSIZE, (65536, 65536)) + + +binary = Path(sys.argv[1]).resolve() +suite = Path(__file__).resolve().parents[1] +shell = os.environ.get("TEST_SHELL", "/bin/sh") +errors = [] +with tempfile.TemporaryDirectory(prefix="lmbench-fs-test.") as tmp: + root = Path(tmp) + env = dict(os.environ, ENOUGH="10000", TIMING_O="0", LOOP_O="0") + env["LMBENCH_BIN_DIR"] = str(binary.parent) + env["LMBENCH_RUN_TMP"] = str(root) + env["LMBENCH_SH"] = shell + for name, expected in [ + ("ext4_create_delete_files_0k_ops", ["0", "1", "4", "10"]), + ("ramfs_create_delete_files_0k_ops", ["0", "1", "4", "10"]), + ("ext4_create_delete_files_10k_ops", ["10"]), + ("ramfs_create_delete_files_10k_ops", ["10"]), + ]: + target = root / name + target.mkdir() + env["LMBENCH_EXT4_DIR"] = env["LMBENCH_TMP_DIR"] = str(target) + # Deliberately wrong incoming TMPDIR: each wrapper must select its fixture. + env["TMPDIR"] = str(root / "wrong-directory") + trace = root / (name + ".trace") + output = root / (name + ".output") + with output.open("wb") as stream: + proc = subprocess.Popen( + ["strace", "-f", "-e", "trace=mkdir", "-o", str(trace), + shell, str(suite / "runner/test_cases" / (name + ".sh"))], + stdout=stream, stderr=stream, env=env, + start_new_session=True, preexec_fn=limit_output, + ) + try: + rc = proc.wait(timeout=60) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + errors.append(name + ": timeout") + continue + text = output.read_text() + sizes = re.findall(r"^(\d+)k\s", text, re.M) + paths = re.findall(r'mkdir\("([^\"]+)",[^\n]*\) += 0', trace.read_text()) + if rc != 0 or sizes != expected: + errors.append(f"{name}: rc={rc}, sizes={sizes}, expected={expected}") + if not paths or any(not p.startswith(str(target) + "/") for p in paths): + errors.append(name + ": benchmark did not use the requested directory") + if list(target.iterdir()): + errors.append(name + ": benchmark left fixtures behind") +if errors: + raise SystemExit("\n".join(errors)) +print("lat_fs binary regression: PASS (four wrappers, original size sweep, target directory, cleanup)") diff --git a/user/apps/tests/benchmark/lmbench/tests/test_linux_entry.sh b/user/apps/tests/benchmark/lmbench/tests/test_linux_entry.sh new file mode 100755 index 0000000000..a7b9e28124 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/tests/test_linux_entry.sh @@ -0,0 +1,238 @@ +#!/bin/sh +# Linux entry-point functionality only: benchmark executables below are stubs. +# This verifies CLI/path mapping and exit status, not performance or lmbench. +set -eu +SUITE=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +TEST_SHELL=${TEST_SHELL:-/bin/sh} +TEST_ROOT=$(mktemp -d /tmp/lmbench-linux-entry.XXXXXX) +trap 'rm -rf "$TEST_ROOT"' 0 +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +unset ENOUGH TIMING_O LOOP_O +# Running the selected binary package must never download or compile sources. +export LMBENCH_SOURCE_ARCHIVE=/nonexistent/lmbench-source-must-not-be-needed +mkdir -p "$TEST_ROOT/bin" "$TEST_ROOT/inputs with spaces" +cat > "$TEST_ROOT/bin/lat_syscall" <<'STUB' +#!/bin/sh +printf 'called\n' >> "$LMBENCH_RUN_TMP/stub-calls" +printf 'ENOUGH=%s\nTIMING_O=%s\nLOOP_O=%s\n' "$ENOUGH" "$TIMING_O" "$LOOP_O" \ + > "$LMBENCH_RUN_TMP/stub-calibration" +printf 'Simple syscall: 1.2500 microseconds\n' +STUB +cat > "$TEST_ROOT/bin/enough" <<'STUB' +#!/bin/sh +[ "${CALIBRATION_TIMEOUT_ARGS:-}" = '-k 2s 60s' ] || exit 80 +printf 'enough\n' >> "$CALIBRATION_CALLS" +printf '1000000\n' +STUB +cat > "$TEST_ROOT/bin/timing_o" <<'STUB' +#!/bin/sh +[ "${ENOUGH:-}" = 1000000 ] || exit 81 +printf 'timing_o\n' >> "$CALIBRATION_CALLS" +printf '0\n' +STUB +cat > "$TEST_ROOT/bin/loop_o" <<'STUB' +#!/bin/sh +[ "${ENOUGH:-}" = 1000000 ] || exit 82 +[ "${TIMING_O:-}" = 0 ] || exit 83 +printf 'loop_o\n' >> "$CALIBRATION_CALLS" +printf '0.00000000\n' +STUB +cat > "$TEST_ROOT/timeout-stub" <<'STUB' +#!/bin/sh +CALIBRATION_TIMEOUT_ARGS="$1 $2 $3" +export CALIBRATION_TIMEOUT_ARGS +exec /usr/bin/timeout "$@" +STUB +printf '#!/bin/sh\nexit 0\n' > "$TEST_ROOT/bin/hello" +printf '#!/bin/sh\nexit 0\n' > "$TEST_ROOT/bin/lat_fifo" +cat > "$TEST_ROOT/bin/lat_fs" <<'STUB' +#!/bin/sh +set -eu +for last do :; done +[ "$TMPDIR" = "$LMBENCH_EXT4_DIR" ] && [ "$last" = "$LMBENCH_EXT4_DIR" ] || exit 9 +printf 'called\n' >> "$LMBENCH_RUN_TMP/fs-stub-calls" +printf '0k\t1\t2\t3\n' +STUB +chmod +x "$TEST_ROOT/bin/lat_fs" +chmod +x "$TEST_ROOT/bin/lat_syscall" "$TEST_ROOT/bin/hello" "$TEST_ROOT/bin/lat_fifo" \ + "$TEST_ROOT/bin/enough" "$TEST_ROOT/bin/timing_o" "$TEST_ROOT/bin/loop_o" \ + "$TEST_ROOT/timeout-stub" +printf 'SAMPLES=2\nTIMEOUT_SEC=3\nWARMUP=0\n' > "$TEST_ROOT/inputs with spaces/config" +printf 'process_getppid_lat\n' > "$TEST_ROOT/inputs with spaces/whitelist" +FAILURES=0 +fail() { printf 'FAIL: %s\n' "$*" >&2; FAILURES=$((FAILURES + 1)); } + +run_entry() { + ENTRY_NAME=$1; shift + ENTRY_OUT="$TEST_ROOT/$ENTRY_NAME" + ENTRY_RC=0 + if [ "${ENTRY_EXPLICIT_CALIBRATION:-0}" -eq 1 ]; then + ENOUGH=$ENTRY_ENOUGH TIMING_O=$ENTRY_TIMING_O LOOP_O=$ENTRY_LOOP_O \ + CALIBRATION_CALLS="$TEST_ROOT/calibration-calls" \ + LMBENCH_BIN_DIR="$TEST_ROOT/bin" \ + LMBENCH_TIMEOUT="$TEST_ROOT/timeout-stub" \ + LMBENCH_SH="$TEST_SHELL" LMBENCH_LINUX_OUTPUT="$ENTRY_OUT" \ + LMBENCH_SUITE_TIMEOUT=30 \ + "$TEST_SHELL" "$SUITE/run_linux.sh" "$@" > "$TEST_ROOT/$ENTRY_NAME.log" 2>&1 || ENTRY_RC=$? + else + CALIBRATION_CALLS="$TEST_ROOT/calibration-calls" \ + LMBENCH_BIN_DIR="$TEST_ROOT/bin" \ + LMBENCH_TIMEOUT="$TEST_ROOT/timeout-stub" \ + LMBENCH_SH="$TEST_SHELL" LMBENCH_LINUX_OUTPUT="$ENTRY_OUT" \ + LMBENCH_SUITE_TIMEOUT=30 \ + "$TEST_SHELL" "$SUITE/run_linux.sh" "$@" > "$TEST_ROOT/$ENTRY_NAME.log" 2>&1 || ENTRY_RC=$? + fi +} + +assert_configured_run() { + if [ "$ENTRY_RC" -ne 0 ]; then + fail "$ENTRY_NAME: expected successful mapped run, got exit $ENTRY_RC" + return + fi + grep -q '"total":1,"ok":1,"failed":0,"skipped":0' "$ENTRY_OUT/results.jsonl" || \ + fail "$ENTRY_NAME: selected whitelist did not produce exactly one successful case" + grep -q '"samples":2,"timeout_sec":3,"warmup":0' "$ENTRY_OUT/results.jsonl" || \ + fail "$ENTRY_NAME: host config was not applied" + if [ ! -f "$ENTRY_OUT/raw/stub-calls" ]; then + fail "$ENTRY_NAME: benchmark stub was not invoked" + elif [ "$(wc -l < "$ENTRY_OUT/raw/stub-calls")" -ne 2 ]; then + fail "$ENTRY_NAME: expected exactly two samples from host config" + fi + [ ! -d "$ENTRY_OUT/fixtures" ] || fail "$ENTRY_NAME: fixture directory was not cleaned" +} + +assert_no_benchmark() { + [ "$ENTRY_RC" -eq 0 ] || fail "$ENTRY_NAME: informational invocation exited $ENTRY_RC" + [ ! -f "$ENTRY_OUT/raw/stub-calls" ] || fail "$ENTRY_NAME: started a benchmark" + for log_file in "$TEST_ROOT/$ENTRY_NAME.log" "$ENTRY_OUT/runner.log"; do + if [ -f "$log_file" ] && grep -q '^===LMBENCH_RUN_BEGIN===' "$log_file"; then + fail "$ENTRY_NAME: initialized the benchmark runner" + fi + done +} + +# Both absolute files live below the host /tmp, which the sandbox replaces. +run_entry absolute --config "$TEST_ROOT/inputs with spaces/config" \ + --whitelist "$TEST_ROOT/inputs with spaces/whitelist" +assert_configured_run +grep -q '^ENOUGH=1000000$' "$ENTRY_OUT/raw/stub-calibration" || fail 'calibrated ENOUGH was not injected' +grep -q '^TIMING_O=0$' "$ENTRY_OUT/raw/stub-calibration" || fail 'calibrated TIMING_O=0 was not injected' +grep -q '^LOOP_O=0.00000000$' "$ENTRY_OUT/raw/stub-calibration" || fail 'calibrated LOOP_O=0.00000000 was not injected' +[ "$(grep -c '^enough$' "$TEST_ROOT/calibration-calls")" -eq 1 ] || fail 'enough was calibrated more than once' +[ "$(grep -c '^timing_o$' "$TEST_ROOT/calibration-calls")" -eq 1 ] || fail 'timing_o was calibrated more than once' +[ "$(grep -c '^loop_o$' "$TEST_ROOT/calibration-calls")" -eq 1 ] || fail 'loop_o was calibrated more than once' +for expected in \ + 'ENOUGH=1000000' 'ENOUGH_SOURCE=calibrated' \ + 'TIMING_O=0' 'TIMING_O_SOURCE=calibrated' \ + 'LOOP_O=0.00000000' 'LOOP_O_SOURCE=calibrated'; do + grep -q "^$expected$" "$ENTRY_OUT/environment.txt" || fail "missing provenance: $expected" +done + +# A binary under host /tmp must remain available after the namespace hides /tmp. +run_entry filesystem --only ext4_create_delete_files_0k_ops --samples 1 --timeout 3 +[ "$ENTRY_RC" -eq 0 ] || fail 'filesystem binary mapping failed' +[ -f "$ENTRY_OUT/raw/fs-stub-calls" ] || fail 'selected filesystem binary was not invoked' +grep -q '^filesystem_binary=' "$ENTRY_OUT/environment.txt" || fail 'filesystem provenance missing' +fs_hash=$(sha256sum "$TEST_ROOT/bin/lat_fs" | awk '{print $1}') +grep -q "^$fs_hash $TEST_ROOT/bin/lat_fs\$" "$ENTRY_OUT/inputs.sha256" || \ + fail 'selected filesystem binary hash missing' + +# Explicit valid values, including zero overheads, are preserved and bypass tools. +: > "$TEST_ROOT/calibration-calls" +ENTRY_EXPLICIT_CALIBRATION=1 ENTRY_ENOUGH=765432 ENTRY_TIMING_O=0 ENTRY_LOOP_O=0.00000000 +export ENTRY_EXPLICIT_CALIBRATION ENTRY_ENOUGH ENTRY_TIMING_O ENTRY_LOOP_O +run_entry explicit --only process_getppid_lat --samples 1 +[ "$ENTRY_RC" -eq 0 ] || fail 'explicit calibration run failed' +[ ! -s "$TEST_ROOT/calibration-calls" ] || fail 'explicit values unexpectedly ran calibration tools' +grep -q '^ENOUGH=765432$' "$ENTRY_OUT/raw/stub-calibration" || fail 'explicit ENOUGH was not injected' +grep -q '^TIMING_O=0$' "$ENTRY_OUT/raw/stub-calibration" || fail 'explicit TIMING_O=0 was not injected' +grep -q '^LOOP_O=0.00000000$' "$ENTRY_OUT/raw/stub-calibration" || fail 'explicit LOOP_O=0 was not injected' +grep -q '^ENOUGH_SOURCE=explicit$' "$ENTRY_OUT/environment.txt" || fail 'explicit ENOUGH source missing' +grep -q '^TIMING_O_SOURCE=explicit$' "$ENTRY_OUT/environment.txt" || fail 'explicit TIMING_O source missing' +grep -q '^LOOP_O_SOURCE=explicit$' "$ENTRY_OUT/environment.txt" || fail 'explicit LOOP_O source missing' +unset ENTRY_EXPLICIT_CALIBRATION ENTRY_ENOUGH ENTRY_TIMING_O ENTRY_LOOP_O + +# Calibration failures are fatal and never claim calibrated provenance. +cat > "$TEST_ROOT/bin/timing_o" <<'STUB' +#!/bin/sh +exit 7 +STUB +run_entry calibration_failed --only process_getppid_lat --samples 1 +[ "$ENTRY_RC" -ne 0 ] || fail 'failed timing_o calibration was accepted' +grep -q 'Calibration tool failed: timing_o (exit 7)' "$TEST_ROOT/calibration_failed.log" || \ + fail 'timing_o calibration failure was not explained' +[ ! -e "$ENTRY_OUT/environment.txt" ] || fail 'failed calibration wrote provenance as if usable' +cat > "$TEST_ROOT/bin/timing_o" <<'STUB' +#!/bin/sh +[ "${ENOUGH:-}" = 1000000 ] || exit 81 +printf 'timing_o\n' >> "$CALIBRATION_CALLS" +printf '0\n' +STUB +chmod +x "$TEST_ROOT/bin/timing_o" + +# Every explicit value is validated before any result directory is created. +for invalid_case in enough timing loop; do + ENTRY_EXPLICIT_CALIBRATION=1 ENTRY_ENOUGH=765432 ENTRY_TIMING_O=0 ENTRY_LOOP_O=0.00000000 + case "$invalid_case" in + enough) ENTRY_ENOUGH=0 ;; + timing) ENTRY_TIMING_O=invalid ;; + loop) ENTRY_LOOP_O=-1 ;; + esac + export ENTRY_EXPLICIT_CALIBRATION ENTRY_ENOUGH ENTRY_TIMING_O ENTRY_LOOP_O + run_entry "invalid_$invalid_case" --only process_getppid_lat --samples 1 + [ "$ENTRY_RC" -ne 0 ] || fail "invalid explicit $invalid_case calibration was accepted" + [ ! -e "$ENTRY_OUT/environment.txt" ] || fail "invalid explicit $invalid_case wrote provenance" +done +unset ENTRY_EXPLICIT_CALIBRATION ENTRY_ENOUGH ENTRY_TIMING_O ENTRY_LOOP_O + +# A missing config must not silently fall back to the suite's default samples. +run_entry config_only --config "$TEST_ROOT/inputs with spaces/config" --only process_getppid_lat +assert_configured_run + +# Resolve relative paths against the caller's CWD before entering namespaces. +cd "$TEST_ROOT" +run_entry relative --config 'inputs with spaces/config' --whitelist 'inputs with spaces/whitelist' +assert_configured_run + +run_entry help --help +assert_no_benchmark +run_entry list --list +assert_no_benchmark + +mkdir "$TEST_ROOT/existing" +printf 'keep this data\n' > "$TEST_ROOT/existing/keep" +run_entry existing --only process_getppid_lat --samples 1 +[ "$ENTRY_RC" -ne 0 ] || fail 'existing output directory must be rejected' +[ "$(cat "$TEST_ROOT/existing/keep")" = 'keep this data' ] || fail 'existing output data was changed' +[ "$(ls -A "$TEST_ROOT/existing")" = keep ] || fail 'existing output directory was populated' + +# A failed benchmark must propagate failure through runner, bwrap and entry. +printf '#!/bin/sh\nexit 7\n' > "$TEST_ROOT/bin/lat_syscall" +run_entry failed --only process_getppid_lat --samples 1 --timeout 3 +[ "$ENTRY_RC" -ne 0 ] || fail 'failed benchmark was reported as a successful Linux run' +if [ -f "$ENTRY_OUT/results.jsonl" ]; then + grep -q '"total":1,"ok":0,"failed":1,"skipped":0' "$ENTRY_OUT/results.jsonl" || \ + fail 'failed benchmark did not retain its failed summary' +else + fail 'failed benchmark did not produce results.jsonl' +fi +[ ! -d "$ENTRY_OUT/fixtures" ] || fail 'failed benchmark left its fixture directory' + +# Even a printed latency cannot make a hung FIFO controller a success. +cat > "$TEST_ROOT/bin/lat_fifo" <<'STUB' +#!/bin/sh +printf 'Fifo latency: 1.0 microseconds\n' +sleep 60 +STUB +run_entry fifo_timeout --only fifo_lat --samples 1 --timeout 1 +[ "$ENTRY_RC" -ne 0 ] || fail 'FIFO timeout was accepted as a valid measurement' +grep -q '"total":1,"ok":0,"failed":1,"skipped":0' "$ENTRY_OUT/results.jsonl" || \ + fail 'FIFO timeout did not retain a failed summary' +[ "$(cat "$ENTRY_OUT/raw/fifo_lat.1.rc")" = 124 ] || fail 'FIFO timeout status was lost' +[ "$(find "$ENTRY_OUT" -type p | wc -l)" -eq 0 ] || fail 'FIFO scratch pipe persisted in results' +[ "$(find "$ENTRY_OUT/raw" -type d | wc -l)" -eq 1 ] || fail 'FIFO scratch directory persisted in results' + +[ "$FAILURES" -eq 0 ] || exit 1 +printf 'Linux entry functionality regression: PASS (%s; no performance measurements)\n' "$TEST_SHELL" diff --git a/user/apps/tests/benchmark/lmbench/tests/test_runner.sh b/user/apps/tests/benchmark/lmbench/tests/test_runner.sh new file mode 100644 index 0000000000..dd96ef9262 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/tests/test_runner.sh @@ -0,0 +1,282 @@ +#!/bin/sh +# Exercise the actual runner with tiny benchmarks, without mounts or lmbench. +set -eu +SUITE=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +TEST_SHELL=${TEST_SHELL:-/bin/sh} +TEST_ROOT=$(mktemp -d) +trap 'rm -rf "$TEST_ROOT"' 0 +trap 'exit 130' INT +trap 'exit 143' TERM +mkdir -p "$TEST_ROOT/suite/test_cases" "$TEST_ROOT/work" +cp "$SUITE/runner/run.sh" "$SUITE/runner/env.sh" "$SUITE/runner/result.sh" "$TEST_ROOT/suite/" +export LMBENCH_RUN_TMP="$TEST_ROOT/work" ENOUGH=50000 LC_ALL=C +export LMBENCH_SH="$TEST_SHELL" +printf 'SAMPLES=2\nTIMEOUT_SEC=2\nWARMUP=0' > "$TEST_ROOT/suite/config" +printf 'first\nlast' > "$TEST_ROOT/suite/whitelist.txt" +for name in first last; do + printf '#!/bin/sh\nprintf "value 12.5\\n"\n' > "$TEST_ROOT/suite/test_cases/$name.sh" + printf 'SEARCH_PATTERN=^value\nRESULT_INDEX=2\nUNIT=us' > "$TEST_ROOT/suite/test_cases/$name.meta" +done +if ! "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" > "$TEST_ROOT/run.log" 2>&1; then + cat "$TEST_ROOT/run.log" + exit 1 +fi +grep -q '"total":2,"ok":2,"failed":0,"skipped":0' "$TEST_ROOT/run.log" || { + cat "$TEST_ROOT/run.log"; echo 'FAIL: every whitelist entry must run'; exit 1; +} +grep -q '"unit":"us"' "$TEST_ROOT/run.log" +[ -f "$LMBENCH_RUN_TMP/first.1.out" ] +[ -f "$LMBENCH_RUN_TMP/first.2.out" ] + +printf '#!/bin/sh\nprintf "timeout: warning: timer_create: Invalid argument\\nvalue 12.5\\n"\n' > "$TEST_ROOT/suite/test_cases/first.sh" +"$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only first --samples 1 > "$TEST_ROOT/timer-warning.log" 2>&1 +grep -q '"status":"ok"' "$TEST_ROOT/timer-warning.log" || { + cat "$TEST_ROOT/timer-warning.log"; echo 'FAIL: nonfatal timer warning rejected'; exit 1; +} + +# A sample is successful only when its output contains one finite, positive +# metric and no fatal benchmark diagnostic, even if the wrapper exits zero. +assert_runner_rejects() { + fixture=$1 + printf '#!/bin/sh\n%s\n' "$fixture" > "$TEST_ROOT/suite/test_cases/first.sh" + if "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only first --samples 1 > "$TEST_ROOT/rejected.log" 2>&1; then + cat "$TEST_ROOT/rejected.log" + echo "FAIL: runner accepted invalid output: $fixture" + exit 1 + fi + grep -q '"status":"failed"' "$TEST_ROOT/rejected.log" + grep -q '"total":1,"ok":0,"failed":1,"skipped":0' "$TEST_ROOT/rejected.log" +} +assert_runner_rejects 'printf "assertion failed: clock went backwards\\nvalue 12.5\\n"' +assert_runner_rejects 'printf "error: benchmark child failed\\nvalue 12.5\\n"' +assert_runner_rejects 'printf "(r) error on semaphore: Invalid argument\\nvalue 12.5\\n"' +assert_runner_rejects 'printf "lat_udp client: send failed: Permission denied\\nvalue 12.5\\n"' +assert_runner_rejects 'printf "lat_udp client: recv failed: Invalid argument\\nvalue 12.5\\n"' +assert_runner_rejects 'printf "lmbench: internal timeout\\nvalue 12.5\\n"' +assert_runner_rejects 'printf "Function not implemented\\nvalue 12.5\\n"' +assert_runner_rejects 'printf "value 0\\n"' +assert_runner_rejects 'printf "value nan\\n"' +assert_runner_rejects 'printf "value 1e999\\n"' +assert_runner_rejects 'printf "benchmark produced no metric\\n"' + +cat > "$TEST_ROOT/suite/test_cases/first.sh" <<'CASE' +#!/bin/sh +if [ -f "$LMBENCH_RUN_TMP/attempted" ]; then exit 1; fi +: > "$LMBENCH_RUN_TMP/attempted" +printf 'value 12.5\n' +CASE +if "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only first > "$TEST_ROOT/partial.log" 2>&1; then + cat "$TEST_ROOT/partial.log"; echo 'FAIL: partial samples must fail the run'; exit 1 +fi +grep -q '"status":"failed"' "$TEST_ROOT/partial.log" +# Caller-selected binaries and fixtures must survive sourcing env.sh. +LMBENCH_BIN_DIR=/chosen/bin LMBENCH_TMP_DIR="$TEST_ROOT" "$TEST_SHELL" -c ' + . "$1" + [ "$LMBENCH_BIN_DIR" = /chosen/bin ] && [ "$LMBENCH_TMP_DIR" = "$2" ] +' sh "$SUITE/runner/env.sh" "$TEST_ROOT" +# Help/invalid arguments must finish before initialization. +"$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --help > "$TEST_ROOT/help.log" 2>&1 +! grep -q LMBENCH_RUN_BEGIN "$TEST_ROOT/help.log" +if "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --samples 0 > /dev/null 2>&1; then + echo 'FAIL: zero samples accepted'; exit 1 +fi +for count in 00 01 08; do + if "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --samples "$count" > /dev/null 2>&1; then + echo "FAIL: noncanonical sample count accepted: $count"; exit 1 + fi +done +printf '\nSAMPLES=0\n' >> "$TEST_ROOT/suite/test_cases/first.meta" +if "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only first > /dev/null 2>&1; then + echo 'FAIL: metadata accepted zero samples'; exit 1 +fi +printf '#!/bin/sh\nsleep 5\n' > "$TEST_ROOT/suite/test_cases/last.sh" +if "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only last --samples 1 --timeout 1 > "$TEST_ROOT/timeout.log" 2>&1; then + echo 'FAIL: timed-out sample accepted'; exit 1 +fi +grep -q 'timeout' "$TEST_ROOT/timeout.log" +[ "$(cat "$LMBENCH_RUN_TMP/last.1.rc")" -eq 124 ] +if "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --config "$TEST_ROOT/missing" > /dev/null 2>&1; then + echo 'FAIL: missing explicit config silently ignored'; exit 1 +fi +# Completion must be emitted after cleanup, so the guest monitor may stop QEMU. +cat > "$TEST_ROOT/suite/clean_up.sh" <<'CLEANUP' +#!/bin/sh +echo CLEANUP_COMPLETE +CLEANUP +printf '#!/bin/sh\nprintf "value 12.5\\n"\n' > "$TEST_ROOT/suite/test_cases/last.sh" +unset ENOUGH +"$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only last --samples 1 > "$TEST_ROOT/order.log" 2>&1 +awk '/CLEANUP_COMPLETE/ { cleaned=1 } /benchmark测试完成/ { if (!cleaned) exit 1 }' "$TEST_ROOT/order.log" || { + cat "$TEST_ROOT/order.log"; echo 'FAIL: completion emitted before cleanup'; exit 1; +} +# Re-sourcing env.sh during default ENOUGH calibration must not replace the +# runner's EXIT cleanup trap. +cat > "$TEST_ROOT/suite/clean_up.sh" <<'CLEANUP' +#!/bin/sh +: > "$LMBENCH_RUN_TMP/resourced-env-cleaned" +CLEANUP +trap_rc=0 +"$TEST_SHELL" -c ' + LMBENCH_RUNNER_NO_MAIN=1 + export LMBENCH_RUNNER_NO_MAIN + . "$1/run.sh" + SCRIPT_DIR=$1 + LMBENCH_SH=$2 + LMBENCH_RUN_TMP=$3 + export LMBENCH_RUN_TMP + trap runner_cleanup 0 + . "$1/env.sh" + exit 7 +' sh "$TEST_ROOT/suite" "$TEST_SHELL" "$LMBENCH_RUN_TMP" || trap_rc=$? +[ "$trap_rc" -eq 7 ] +[ -f "$LMBENCH_RUN_TMP/resourced-env-cleaned" ] || { + echo 'FAIL: re-sourcing env.sh replaced runner cleanup trap'; exit 1; +} +# Automatic calibration must preserve a valid value returned by lmbench's +# enough tool, including its 1000000 SHORT result. +mkdir -p "$TEST_ROOT/fake-bin" +cat > "$TEST_ROOT/fake-bin/enough" <<'ENOUGH' +#!/bin/sh +printf '1000000\n' +ENOUGH +chmod +x "$TEST_ROOT/fake-bin/enough" +cat > "$TEST_ROOT/suite/test_cases/last.sh" <<'CASE' +#!/bin/sh +printf '%s\n' "$ENOUGH" > "$LMBENCH_RUN_TMP/observed-enough" +printf 'value 12.5\n' +CASE +unset ENOUGH +LMBENCH_BIN_DIR="$TEST_ROOT/fake-bin" \ + "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only last --samples 1 \ + > "$TEST_ROOT/enough-valid.log" 2>&1 +[ "$(cat "$LMBENCH_RUN_TMP/observed-enough")" = 1000000 ] || { + cat "$TEST_ROOT/enough-valid.log" + echo 'FAIL: valid automatic ENOUGH value was not preserved' + exit 1 +} +grep -q 'ENOUGH=1000000 (calibrated by enough tool)' "$TEST_ROOT/enough-valid.log" || { + cat "$TEST_ROOT/enough-valid.log" + echo 'FAIL: successful ENOUGH calibration was not identified' + exit 1 +} +# Failed and invalid calibration results must use the bounded fallback and say +# why. These cases also guard against validating ENOUGH with shell arithmetic. +assert_enough_fallback() { + label=$1 + reason=$2 + rm -f "$LMBENCH_RUN_TMP/observed-enough" + unset ENOUGH + LMBENCH_BIN_DIR="$TEST_ROOT/fake-bin" \ + "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only last --samples 1 \ + > "$TEST_ROOT/enough-$label.log" 2>&1 + [ "$(cat "$LMBENCH_RUN_TMP/observed-enough")" = 50000 ] || { + cat "$TEST_ROOT/enough-$label.log" + echo "FAIL: $label calibration did not use ENOUGH=50000" + exit 1 + } + grep -q "ENOUGH=50000 (fallback: $reason)" "$TEST_ROOT/enough-$label.log" || { + cat "$TEST_ROOT/enough-$label.log" + echo "FAIL: $label fallback reason was not identified" + exit 1 + } +} +cat > "$TEST_ROOT/fake-bin/enough" <<'ENOUGH' +#!/bin/sh +exit 1 +ENOUGH +assert_enough_fallback failed 'enough tool failed' +cat > "$TEST_ROOT/fake-bin/enough" <<'ENOUGH' +#!/bin/sh +: +ENOUGH +assert_enough_fallback empty 'enough tool returned empty output' +cat > "$TEST_ROOT/fake-bin/enough" <<'ENOUGH' +#!/bin/sh +printf 'not-a-number\n' +ENOUGH +assert_enough_fallback nonnumeric 'enough tool returned non-decimal output' +cat > "$TEST_ROOT/fake-bin/enough" <<'ENOUGH' +#!/bin/sh +printf '0\n' +ENOUGH +assert_enough_fallback zero 'enough tool returned zero' + +# A caller-provided ENOUGH value is authoritative and bypasses calibration. +cat > "$TEST_ROOT/fake-bin/enough" <<'ENOUGH' +#!/bin/sh +: > "$LMBENCH_RUN_TMP/enough-called" +printf '999\n' +ENOUGH +rm -f "$LMBENCH_RUN_TMP/enough-called" "$LMBENCH_RUN_TMP/observed-enough" +ENOUGH=765432 LMBENCH_BIN_DIR="$TEST_ROOT/fake-bin" \ + "$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only last --samples 1 \ + > "$TEST_ROOT/enough-explicit.log" 2>&1 +[ "$(cat "$LMBENCH_RUN_TMP/observed-enough")" = 765432 ] || { + cat "$TEST_ROOT/enough-explicit.log" + echo 'FAIL: explicit ENOUGH value was not preserved' + exit 1 +} +[ ! -e "$LMBENCH_RUN_TMP/enough-called" ] || { + cat "$TEST_ROOT/enough-explicit.log" + echo 'FAIL: explicit ENOUGH unexpectedly ran calibration' + exit 1 +} +# Hangup must run cleanup and retain its conventional signal exit status. +cat > "$TEST_ROOT/suite/init.sh" <<'INIT' +#!/bin/sh +kill -HUP "$PPID" +INIT +cat > "$TEST_ROOT/suite/clean_up.sh" <<'CLEANUP' +#!/bin/sh +: > "$LMBENCH_RUN_TMP/hup-cleaned" +CLEANUP +signal_rc=0 +"$TEST_SHELL" "$TEST_ROOT/suite/run.sh" --only last > "$TEST_ROOT/hup.log" 2>&1 || signal_rc=$? +[ "$signal_rc" -eq 129 ] +[ -f "$LMBENCH_RUN_TMP/hup-cleaned" ] +# DragonOS may provide BusyBox without standalone awk/sort applet links. +"$TEST_SHELL" -c ' + LMBENCH_RUNNER_NO_MAIN=1 + . "$1" + PATH=/no-applet-links + stats=$(compute_stats "1 2 3") + case "$stats" in "3 2.000000 2.000000 "*) ;; *) exit 1 ;; esac +' sh "$SUITE/runner/run.sh" + +# Published working-set descriptions must match the wrapper arguments. +grep -q ' 16m mmap_only ' "$SUITE/runner/test_cases/mem_mmap_bw.sh" +grep -q 'DESCRIPTION=.*(16MB)' "$SUITE/runner/test_cases/mem_mmap_bw.meta" +grep -q ' 64m io_only ' "$SUITE/runner/test_cases/vfs_read_pagecache_bw.sh" +grep -q 'DESCRIPTION=.*(64MB)' "$SUITE/runner/test_cases/vfs_read_pagecache_bw.meta" + +# The pagefault wrapper owns a unique per-invocation fixture and removes it, +# while preserving an unrelated legacy path that may belong to the user. +PAGEFAULT_TMP="$TEST_ROOT/pagefault-tmp" +PAGEFAULT_RUN="$TEST_ROOT/pagefault-run" +PAGEFAULT_BIN="$TEST_ROOT/pagefault-bin" +mkdir -p "$PAGEFAULT_TMP" "$PAGEFAULT_RUN" "$PAGEFAULT_BIN" +printf 'keep me\n' > "$PAGEFAULT_TMP/pagefault_file" +cat > "$PAGEFAULT_BIN/lat_pagefault" <<'PAGEFAULT' +#!/bin/sh +for last do :; done +[ -f "$last" ] && [ "$(wc -c < "$last")" -eq 8388608 ] || exit 9 +printf '%s\n' "$last" > "$LMBENCH_RUN_TMP/pagefault-observed-path" +printf 'Pagefaults on %s: 1.25 microseconds\n' "$last" +PAGEFAULT +chmod +x "$PAGEFAULT_BIN/lat_pagefault" +LMBENCH_BIN_DIR="$PAGEFAULT_BIN" LMBENCH_TMP_DIR="$PAGEFAULT_TMP" \ + LMBENCH_RUN_TMP="$PAGEFAULT_RUN" LMBENCH_TIMEOUT=/usr/bin/timeout \ + "$TEST_SHELL" "$SUITE/runner/test_cases/mem_pagefault_lat.sh" \ + > "$TEST_ROOT/pagefault.log" 2>&1 || { + cat "$TEST_ROOT/pagefault.log" + echo 'FAIL: pagefault wrapper invocation failed' + exit 1 + } +[ "$(cat "$PAGEFAULT_TMP/pagefault_file")" = 'keep me' ] || { + echo 'FAIL: pagefault wrapper overwrote a pre-existing file'; exit 1; +} +observed_pagefault_path=$(cat "$PAGEFAULT_RUN/pagefault-observed-path") +[ ! -e "$observed_pagefault_path" ] || { + echo 'FAIL: pagefault wrapper left its owned fixture'; exit 1; +} +printf 'runner regression: PASS (%s)\n' "$TEST_SHELL" diff --git a/user/apps/tests/benchmark/lmbench/whitelist.txt b/user/apps/tests/benchmark/lmbench/whitelist.txt new file mode 100644 index 0000000000..2896466a96 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/whitelist.txt @@ -0,0 +1,70 @@ +# LMbench 基准测试白名单 +# 每行一个测例名(对应 runner/test_cases/.sh,不带路径/后缀) +# 以 # 开头为注释,空行忽略。runner 只跑此列表中的测例。 +# +# 全量启用:48 个测例覆盖各类别。.meta 在 host 侧用 guest 同款 lmbench 二进制 +# 提取(见 docs/superpowers/specs/2026-07-30-lmbench-fullsuite-meta-extraction-design.md); +# 部分工具在 host 跑不通(lat_fifo/bw_unix/lat_sem/lat_syscall null/bw_tcp 等), +# 其 .meta 基于 strings 提取的输出格式 + lmbench 标准格式推断,标注待 guest 校准。 + +# memory +mem_copy_bw +mem_read_bw +mem_write_bw +mem_mmap_bw +mem_mmap_lat +mem_pagefault_lat + +# process +process_fork_lat +process_exec_lat +process_ctx_lat +process_getppid_lat +process_shell_lat + +# ipc +pipe_lat +pipe_bw +fifo_lat +unix_lat +unix_bw +unix_connect_lat +semaphore_lat + +# signal +signal_catch_lat +signal_install_lat +signal_prot_lat + +# vfs +vfs_open_lat +vfs_read_lat +vfs_write_lat +vfs_fstat_lat +vfs_stat_lat +vfs_fcntl_lat +vfs_select_lat +vfs_read_pagecache_bw + +# filesystem +ext4_create_delete_files_0k_ops +ext4_create_delete_files_10k_ops +ramfs_create_delete_files_0k_ops +ramfs_create_delete_files_10k_ops +ext4_copy_files_bw +ramfs_copy_files_bw + +# network +tcp_loopback_lat +tcp_loopback_bw_128 +tcp_loopback_bw_4k +tcp_loopback_bw_64k +tcp_loopback_connect_lat +tcp_loopback_select_lat +tcp_loopback_http_bw +tcp_virtio_lat +tcp_virtio_bw_128 +tcp_virtio_bw_64k +tcp_virtio_connect_lat +udp_loopback_lat +udp_virtio_lat \ No newline at end of file From bd9d1a132a2b0df92eb36703549a50133317d9f6 Mon Sep 17 00:00:00 2001 From: yuming Date: Tue, 22 Sep 2026 00:48:26 +0800 Subject: [PATCH 04/22] build(lmbench): package benchmark binaries and guest scripts Package the benchmark binaries and guest scripts through Nix and DADK. Co-authored-by: Samuka007 --- config/app-blocklist.toml | 1 + user/apps/default.nix | 4 + user/apps/tests/benchmark/lmbench/Makefile | 41 ++++++++++ user/apps/tests/benchmark/lmbench/default.nix | 80 +++++++++++++++++++ user/apps/tests/benchmark/lmbench/flake.lock | 27 +++++++ user/apps/tests/benchmark/lmbench/flake.nix | 44 ++++++++++ .../all/lmbench_benchmark_tests-1.0.0.toml | 40 ++++++++++ .../config/all/lmbench_bin_ubuntu2404.toml | 43 ++++++++++ .../lmbench_benchmark_tests-1.0.0.toml | 1 + .../sets/default/lmbench_bin_ubuntu2404.toml | 1 + 10 files changed, 282 insertions(+) create mode 100644 user/apps/tests/benchmark/lmbench/Makefile create mode 100644 user/apps/tests/benchmark/lmbench/default.nix create mode 100644 user/apps/tests/benchmark/lmbench/flake.lock create mode 100644 user/apps/tests/benchmark/lmbench/flake.nix create mode 100644 user/dadk/config/all/lmbench_benchmark_tests-1.0.0.toml create mode 100644 user/dadk/config/all/lmbench_bin_ubuntu2404.toml create mode 120000 user/dadk/config/sets/default/lmbench_benchmark_tests-1.0.0.toml create mode 120000 user/dadk/config/sets/default/lmbench_bin_ubuntu2404.toml diff --git a/config/app-blocklist.toml b/config/app-blocklist.toml index 3e1eb3a318..21c1bddf38 100644 --- a/config/app-blocklist.toml +++ b/config/app-blocklist.toml @@ -64,6 +64,7 @@ log_skipped = true # 屏蔽gvisor系统调用测试 blocked_apps = [ { name = "gvisor syscall tests", reason = "由于文件较大，因此屏蔽。如果要允许系统调用测试，则把这行取消注释即可" }, + # { name = "lmbench benchmark tests", reason = "性能基准套件，默认屏蔽以免拖慢常规构建；make test-benchmark 会临时启用此行（注释即可启用）" }, { name = "test_ebpf_new", reason = "2025.11.17,aya上游发版有问题，导致ci过不了，暂时禁用" }, { name = "test_ebpf_tp", reason = "2025.11.17,aya上游发版有问题，导致ci过不了，暂时禁用" }, { name = "runcell", reason = "依赖项依赖github，可能网络问题导致编不过" }, diff --git a/user/apps/default.nix b/user/apps/default.nix index 97c90eb618..e6b7347f5f 100644 --- a/user/apps/default.nix +++ b/user/apps/default.nix @@ -62,6 +62,8 @@ let installDir = testOpt.dunitest.testDir; } ); + + lmbench-benchmark-tests = (pkgs.callPackage ./tests/benchmark/lmbench { }); in [ (static.busybox.override { @@ -84,6 +86,8 @@ in ++ lib.optionals (target == "x86_64" && testOpt.syscall.enable) [ # gvisor test case only included on x86_64 gvisor-syscall-tests + # TODO(lmbench): move under a dedicated testOpt.benchmark.enable toggle (Phase 3) + lmbench-benchmark-tests # TODO: Add debian libcxx deps or FHS ] ++ lib.optionals (target == "x86_64" && testOpt.dunitest.enable) [ diff --git a/user/apps/tests/benchmark/lmbench/Makefile b/user/apps/tests/benchmark/lmbench/Makefile new file mode 100644 index 0000000000..25ef27fcb0 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/Makefile @@ -0,0 +1,41 @@ +# LMbench benchmark test suite Makefile. +# Shell-only: no compilation, just install the guest-side files into the rootfs. +# Invoked by DADK (build-from-source) with DADK_CURRENT_BUILD_DIR set. + +ifdef DADK_CURRENT_BUILD_DIR +INSTALL_DIR = $(DADK_CURRENT_BUILD_DIR) +else +INSTALL_DIR = ./install +endif + +# Files needed inside the guest at runtime. Sources live under runner/; +# `install` flattens them into INSTALL_DIR (guest runtime layout is flat). +RUNNER_DIR = runner +GUEST_SCRIPTS = $(RUNNER_DIR)/run.sh $(RUNNER_DIR)/init.sh $(RUNNER_DIR)/env.sh $(RUNNER_DIR)/clean_up.sh $(RUNNER_DIR)/result.sh +GUEST_DATA = whitelist.txt config + +.PHONY: all build install clean help + +all: install + +# No build step (shell scripts only); kept so DADK's default flow is happy. +build: + @true + +install: + @echo "[lmbench] installing to $(INSTALL_DIR)" + @mkdir -p $(INSTALL_DIR) + @rm -f $(INSTALL_DIR)/run_tests.sh $(INSTALL_DIR)/run_test_case.sh + @install -m755 $(GUEST_SCRIPTS) $(INSTALL_DIR)/ + @install -m644 $(GUEST_DATA) $(INSTALL_DIR)/ + @cp -r $(RUNNER_DIR)/test_cases $(INSTALL_DIR)/ + @chmod +x $(INSTALL_DIR)/test_cases/*.sh 2>/dev/null || true + @echo "[lmbench] install done" + +clean: + @rm -rf ./install + +help: + @echo "lmbench benchmark Makefile targets:" + @echo " install - copy runner + test cases into INSTALL_DIR (DADK build dir)" + @echo " clean - remove local ./install" diff --git a/user/apps/tests/benchmark/lmbench/default.nix b/user/apps/tests/benchmark/lmbench/default.nix new file mode 100644 index 0000000000..12d21b78bf --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/default.nix @@ -0,0 +1,80 @@ +{ + lib, + pkgs, + installDir ? "/opt/lmbench", +}: + +let + lmbenchBin = pkgs.fetchurl { + url = "https://mirrors.dragonos.org.cn/pub/third_party/lmbench/lmbench-ubuntu2404-202511301014-36c2fb2d084343e098b5e343576d4fc0.tar.xz"; + sha256 = "sha256-/iP7oKc2n0CXpA0OgVQX86quNfbtjXBZ83RBl1MEBhE="; + }; + + testScript = pkgs.stdenv.mkDerivation { + pname = "lmbench-test-script"; + version = "3.0-a9"; + + src = lib.sourceByRegex ./. [ + "^runner" + "^runner/.*\.sh$" + "^runner/test_cases" + "^runner/test_cases/.*" + "^config$" + "^whitelist\.txt$" + ]; + + installPhase = '' + mkdir -p $out/${installDir} + + install -m755 runner/*.sh $out/${installDir}/ + install -m644 config whitelist.txt $out/${installDir}/ + cp -r runner/test_cases $out/${installDir}/ + chmod +x $out/${installDir}/test_cases/*.sh + ''; + }; + + lmbench = pkgs.stdenv.mkDerivation { + pname = "lmbench"; + version = "3.0-a9"; + + src = lmbenchBin; + + nativeBuildInputs = [ pkgs.autoPatchelfHook ]; + + buildInputs = [ + pkgs.stdenv.cc.cc.lib + pkgs.glibc + pkgs.bzip2 + ]; + + sourceRoot = "."; + + installPhase = '' + # Keep Ubuntu package's original directory structure, but flatten sysroot + mkdir -p $out + + # If there's a sysroot directory, flatten it + if [ -d sysroot ]; then + cp -r sysroot/* $out/ + else + cp -r * $out/ + fi + + # Make all binaries executable + find $out -type f -executable -exec chmod +x {} \; + + # Remove broken symlinks (mainly documentation files from sysroot) + find $out -xtype l -delete + ''; + + # autoPatchelfHook will automatically run after installPhase + # to patch all ELF binaries with correct library paths + }; +in +pkgs.symlinkJoin { + name = "lmbench-with-tests"; + paths = [ + lmbench + testScript + ]; +} diff --git a/user/apps/tests/benchmark/lmbench/flake.lock b/user/apps/tests/benchmark/lmbench/flake.lock new file mode 100644 index 0000000000..00eeb0412d --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1767767207, + "narHash": "sha256-Mj3d3PfwltLmukFal5i3fFt27L6NiKXdBezC1EBuZs4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5912c1772a44e31bf1c63c0390b90501e5026886", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/user/apps/tests/benchmark/lmbench/flake.nix b/user/apps/tests/benchmark/lmbench/flake.nix new file mode 100644 index 0000000000..91c73ed393 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/flake.nix @@ -0,0 +1,44 @@ +{ + description = "LMbench benchmark suite packaged for nix (x86_64-linux only)"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + }; + + outputs = + { self, nixpkgs }: + let + system = "x86_64-linux"; + pkgs = nixpkgs.legacyPackages.${system}; + + # Default installation directory + defaultInstallDir = "tests/benchmark/lmbench"; + + # Import the package definition + lmbench = import ./default.nix { + inherit (pkgs) lib pkgs; + installDir = defaultInstallDir; + }; + in + { + packages.${system} = { + default = lmbench; + lmbench = lmbench; + }; + + # Allow overriding installDir + lib.${system}.mkLmbench = + { + installDir ? defaultInstallDir, + }: + import ./default.nix { + inherit (pkgs) lib pkgs; + inherit installDir; + }; + + apps.${system}.default = { + type = "app"; + program = "${lmbench}/${defaultInstallDir}/run.sh"; + }; + }; +} diff --git a/user/dadk/config/all/lmbench_benchmark_tests-1.0.0.toml b/user/dadk/config/all/lmbench_benchmark_tests-1.0.0.toml new file mode 100644 index 0000000000..736009e440 --- /dev/null +++ b/user/dadk/config/all/lmbench_benchmark_tests-1.0.0.toml @@ -0,0 +1,40 @@ +# 用户程序名称（必须与 config/app-blocklist.toml 中的条目一致） +name = "lmbench benchmark tests" +# 版本号 +version = "1.0.0" +# 用户程序描述信息 +description = "LMbench 性能基准测试套件（测例脚本 + guest 端 runner）" +# （可选）是否只构建一次 +build-once = false +# （可选）是否只安装一次 +install-once = false +# 目标架构 +target-arch = ["x86_64"] + +# 任务源 +[task-source] +# 构建类型 +type = "build-from-source" +# 构建来源 +source = "local" +# 路径或URL +source-path = "user/apps/tests/benchmark/lmbench" + +# 构建相关信息 +[build] +# 仅安装 shell 脚本/元数据到 rootfs（无需编译） +build-command = "make install" + +# 安装相关信息 +[install] +# 安装到DragonOS的路径 +in-dragonos-path = "/opt/tests/benchmark/lmbench" + +# 清除相关信息 +[clean] +clean-command = "make clean" + +# 依赖:lmbench 二进制（会连带其 glibc 依赖一起被 DADK 构建/安装） +[[depends]] +name = "lmbench_bin_ubuntu2404" +version = "3.0" diff --git a/user/dadk/config/all/lmbench_bin_ubuntu2404.toml b/user/dadk/config/all/lmbench_bin_ubuntu2404.toml new file mode 100644 index 0000000000..024084f7db --- /dev/null +++ b/user/dadk/config/all/lmbench_bin_ubuntu2404.toml @@ -0,0 +1,43 @@ +# 用户程序名称 +name = "lmbench_bin_ubuntu2404" +# 版本号 +version = "3.0" +# 用户程序描述信息 +description = "LMbench - Tools for Performance Analysis from Ubuntu 24.04" +# （可选）默认: false 是否只构建一次，如果为true，DADK会在构建成功后，将构建结果缓存起来，下次构建时，直接使用缓存的构建结果 +build-once = false +# (可选) 默认: false 是否只安装一次，如果为true，DADK会在安装成功后，不再重复安装 +install-once = false +# 目标架构 +# 可选值："x86_64", "aarch64", "riscv64", "loongarch64" +target-arch = ["x86_64"] +# 任务源 +[task-source] +# 构建类型 +# 可选值："build-from-source", "install-from-prebuilt" +type = "install-from-prebuilt" +# 构建来源 +# "build_from_source" 可选值："git", "local", "archive" +# "install_from_prebuilt" 可选值："local", "archive" +source = "archive" +# 本地目录路径 - 指向已解压的 sysroot 目录 +source-path = "https://mirrors.dragonos.org.cn/pub/third_party/lmbench/lmbench-ubuntu2404-202511301014-36c2fb2d084343e098b5e343576d4fc0.tar.xz" +archive-rootdir = "sysroot/" + +[build] + +# 安装相关信息 +[install] +# （可选）安装到DragonOS的路径 +in-dragonos-path = "/" +# 清除相关信息 +[clean] +# （可选）清除命令 +clean-command = "" +# 依赖项 +[[depends]] +name = "glibc_bin_ubuntu2404" +version = "2.39" + +# （可选）环境变量 +# 注意：如果没有环境变量，忽略此项，不允许只留一个[[envs]] diff --git a/user/dadk/config/sets/default/lmbench_benchmark_tests-1.0.0.toml b/user/dadk/config/sets/default/lmbench_benchmark_tests-1.0.0.toml new file mode 120000 index 0000000000..f54c0d1565 --- /dev/null +++ b/user/dadk/config/sets/default/lmbench_benchmark_tests-1.0.0.toml @@ -0,0 +1 @@ +../../all/lmbench_benchmark_tests-1.0.0.toml \ No newline at end of file diff --git a/user/dadk/config/sets/default/lmbench_bin_ubuntu2404.toml b/user/dadk/config/sets/default/lmbench_bin_ubuntu2404.toml new file mode 120000 index 0000000000..d7a37811e3 --- /dev/null +++ b/user/dadk/config/sets/default/lmbench_bin_ubuntu2404.toml @@ -0,0 +1 @@ +../../all/lmbench_bin_ubuntu2404.toml \ No newline at end of file From e379f61ee69accb2488f8d7ac0bd52ba144ce45d Mon Sep 17 00:00:00 2001 From: yuming Date: Tue, 22 Sep 2026 00:48:26 +0800 Subject: [PATCH 05/22] feat(lmbench): integrate qemu execution and result collection Add QEMU execution, monitoring, result collection and schema validation. Extend the suite README with the end-to-end workflow, guest prerequisites, architecture and result schema. --- Makefile | 21 ++ tools/run-qemu.sh | 5 +- user/apps/tests/benchmark/lmbench/README.md | 95 +++++ .../lmbench/orchestrator/collect_results.py | 335 ++++++++++++++++++ .../orchestrator/monitor_test_results.sh | 139 ++++++++ .../schema/lmbench-run.schema.json | 180 ++++++++++ .../lmbench/tests/test_collect_results.py | 101 ++++++ .../lmbench/toggle_compile_lmbench.sh | 87 +++++ user/sysconfig/etc/init.d/rcS | 2 + 9 files changed, 964 insertions(+), 1 deletion(-) create mode 100644 user/apps/tests/benchmark/lmbench/orchestrator/collect_results.py create mode 100644 user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh create mode 100644 user/apps/tests/benchmark/lmbench/orchestrator/schema/lmbench-run.schema.json create mode 100644 user/apps/tests/benchmark/lmbench/tests/test_collect_results.py create mode 100755 user/apps/tests/benchmark/lmbench/toggle_compile_lmbench.sh diff --git a/Makefile b/Makefile index 6199bba7db..ef1afced6f 100644 --- a/Makefile +++ b/Makefile @@ -313,6 +313,27 @@ test-syscall: prepare_rootfs_manifest exit $$status; \ } +test-benchmark: prepare_rootfs_manifest + @set -e; \ + cleanup() { bash user/apps/tests/benchmark/lmbench/toggle_compile_lmbench.sh disable; }; \ + trap cleanup EXIT; \ + echo "构建运行并执行lmbench性能基准测试"; \ + bash user/apps/tests/benchmark/lmbench/toggle_compile_lmbench.sh enable; \ + $(MAKE) all -j $(NPROCS); \ + if [ "$(DISK_SAVE_MODE)" = "1" ]; then \ + echo "磁盘节省模式启用，正在清理用户程序构建缓存..."; \ + $(DADK) -f $(ROOT_PATH)/dadk-manifest.generated.toml user clean --level in-src -w $(ROOT_PATH); \ + fi; \ + SKIP_GRUB=1 $(MAKE) write_diskimage; \ + $(MAKE) qemu-nographic AUTO_TEST=benchmark BENCHMARK_TEST_DIR=/opt/tests/benchmark/lmbench & \ + sleep 5; \ + status=0; \ + bash user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh || status=$$?; \ + if [ $$status -eq 0 ]; then \ + python3 user/apps/tests/benchmark/lmbench/orchestrator/collect_results.py || status=$$?; \ + fi; \ + exit $$status + test-dunit: prepare_rootfs_manifest @echo "构建运行并执行dunitest测试" $(MAKE) all -j $(NPROCS) diff --git a/tools/run-qemu.sh b/tools/run-qemu.sh index 10d850fc41..f12f5c4d5b 100755 --- a/tools/run-qemu.sh +++ b/tools/run-qemu.sh @@ -480,6 +480,7 @@ KERNEL_CMDLINE=" rw " # - none: 不进行自动测试 # - syscall: 进行gvisor系统调用测试 # - dunit: 进行dunitest测试 +# - benchmark: 进行lmbench性能基准测试 AUTO_TEST=${AUTO_TEST:=none} # gvisor测试目录 SYSCALL_TEST_DIR=${SYSCALL_TEST_DIR:=/opt/tests/gvisor} @@ -487,6 +488,8 @@ SYSCALL_TEST_DIR=${SYSCALL_TEST_DIR:=/opt/tests/gvisor} DUNITEST_DIR=${DUNITEST_DIR:=/opt/tests/dunitest} # dunitest pattern过滤条件 DUNITEST_PATTERN=${DUNITEST_PATTERN:=} +# lmbench基准测试目录 +BENCHMARK_TEST_DIR=${BENCHMARK_TEST_DIR:=/opt/tests/benchmark/lmbench} BIOS_TYPE="" #这个变量为true则使用virtio磁盘 @@ -601,7 +604,7 @@ while true;do setup_kernel_init_program() { if [ ${ARCH} == "x86_64" ]; then - KERNEL_CMDLINE+=" init=/bin/busybox init AUTO_TEST=${AUTO_TEST} SYSCALL_TEST_DIR=${SYSCALL_TEST_DIR} DUNITEST_DIR=${DUNITEST_DIR} DUNITEST_PATTERN=${DUNITEST_PATTERN} " + KERNEL_CMDLINE+=" init=/bin/busybox init AUTO_TEST=${AUTO_TEST} SYSCALL_TEST_DIR=${SYSCALL_TEST_DIR} DUNITEST_DIR=${DUNITEST_DIR} DUNITEST_PATTERN=${DUNITEST_PATTERN} BENCHMARK_TEST_DIR=${BENCHMARK_TEST_DIR} " # KERNEL_CMDLINE+=" init=/bin/dragonreach " elif [ ${ARCH} == "riscv64" ]; then KERNEL_CMDLINE+=" init=/bin/riscv_rust_init " diff --git a/user/apps/tests/benchmark/lmbench/README.md b/user/apps/tests/benchmark/lmbench/README.md index b9aac83c6a..7e071d23a1 100644 --- a/user/apps/tests/benchmark/lmbench/README.md +++ b/user/apps/tests/benchmark/lmbench/README.md @@ -1,5 +1,51 @@ # LMbench 性能基准测试套件 +DragonOS 的性能基准评测套件，基于用户态移植的 [LMbench](http://lmbench.sourceforge.net/) 二进制， +参考隔壁 Asterinas 的 benchmark runner 设计。它自动启动 QEMU、在 guest 内跑白名单测例、抽取性能数值、 +计算统计量，并在 host 侧把结果按统一 schema 持久化。 + +设计上对齐现有的 gVisor syscall 测试套件（`user/apps/tests/syscall/gvisor`）：host 侧 monitor 轮询串口、 +guest 侧 runner 由 `AUTO_TEST` 触发。 + +## 快速开始 + +```bash +# 在仓库根目录:构建 DragonOS → 启动 QEMU → 跑基准 → 收集结果 +make test-benchmark +``` + +该目标会： +1. `toggle_compile_lmbench.sh enable` 临时把基准套件编进 rootfs（默认屏蔽以免拖慢常规构建）； +2. `make all` + `write_diskimage`； +3. 后台启动 `qemu-nographic AUTO_TEST=benchmark`（串口 → `serial_opt.txt`）； +4. `monitor_test_results.sh` 轮询串口，判定启动/超时/完成； +5. `collect_results.py` 解析串口、补 run 级元数据、校验 schema、落盘到 `results/`； +6. 恢复 blocklist（disable）。 + +结果位于 `results//-.json`。 + +当前白名单包含 48 个测例；每轮以实际结果中的成功、失败和跳过数判断覆盖情况。GNU `timeout` 调用 `timer_create(CLOCK_REALTIME)` 的内核兼容性问题由 [#2159](https://github.com/DragonOS-Community/DragonOS/issues/2159) 独立跟踪；它不属于 LMbench 结果解析修复。 + +## DragonOS guest 环境前置条件 + +`LMBENCH_NET_SERVER` 默认值 `10.0.2.15` 只是网络用例的连接目标，不会为 guest 配置地址或路由。运行网络用例前，guest 必须实际拥有所选地址及可用路由；网卡名应从该 guest 的 `ip link` 输出确定，不能硬编码为 `eth0`。以下检查均在 **guest 内**执行，并应在测试前通过： + +```sh +net_server=${LMBENCH_NET_SERVER:-10.0.2.15} +ip -4 addr show +ip -4 route show +ip -4 route get "$net_server" +getent hosts localhost +``` + +若需要静态配置，只能在私有 guest 镜像或一次性 overlay 中，由调用者明确选择接口和地址；不得修改宿主网络。`tcp_loopback_select_lat` 使用的 `lat_select tcp` 依赖 libc 将 `localhost` 解析为 `127.0.0.1`。制作私有 guest 镜像时应保留已有 `/etc/hosts` 内容，并确定性地提供该映射；不得复制或修改宿主 `/etc/hosts`。guest 缺少 `getent` 时，应使用调用 guest libc 的等价解析探针验证，而不能只检查文件内容。 + +五个 `*_virtio_*` wrapper 会在同一 guest 内启动 server 和 client。默认目标 `.15` 是 guest 本机地址时，结果只覆盖本机非 loopback 地址上的 TCP/UDP 协议路径，不代表 guest 与 host 或外部端点之间的 virtio 数据传输性能。 + +`runner/run.sh` 可直接使用 `--only NAME` 或 `--whitelist FILE` 做定向调试；`make test-benchmark` 经 `rcS` 无参数调用 runner，不会转发这些命令行参数。需要筛选时可在私有 guest 中直接调用 runner，或安装私有入口和 whitelist，无需修改通用 `rcS`。 + +结果归档应随测试数据简要保存 guest 地址/路由/解析现场、校准值、各层超时，以及 kernel、suite 安装树和私有 whitelist 的哈希；若将精简结果纳入 references，过程日志与镜像仍留在 references 之外。完整验收须保留正式 whitelist 的全部 48 个测例及原参数。若为减少已知 panic 对后续用例的污染而在私有 whitelist 中将 `signal_prot_lat` 移到最后，应记录实际顺序并验证测例集合不变；该用例仍计为失败，不能据此宣称问题已修复或基线稳定。 + ## 在 Linux 上运行（POSIX sh） 在仓库根目录执行： @@ -56,6 +102,49 @@ runner 和用例也支持通过 `LMBENCH_BIN_DIR`、`LMBENCH_TMP_DIR`、`LMBENCH fixture。直接运行网络用例时，`LMBENCH_NET_SERVER` 应指向本机地址，默认仍为 guest 的 `10.0.2.15`。 +## 架构 + +``` +make test-benchmark (host) + │ build + write disk + qemu-nographic AUTO_TEST=benchmark & + │ monitor_test_results.sh (轮询 serial_opt.txt,超时/完成判定) + │ collect_results.py (解析 → 校验 schema → 持久化) + ▼ +guest: /etc/init.d/rcS → $BENCHMARK_TEST_DIR/run.sh + · init.sh 建 ext4 loop fs + 测试文件(一次) + · 读 whitelist.txt + config,逐测例跑 N 次 + · 按 test_cases/.meta 的规则从输出抽取数值 + · 算 mean/median/stddev/min/max/cv (busybox awk,自实现 sqrt) + · 逐条打印 "LMBENCH_JSON {...}" (JSONL) 到串口 + · 末尾打印 "benchmark测试完成" +``` + +## 目录结构 + +``` +run_linux.sh Linux 隔离运行入口(POSIX sh) +tests/ runner/fixture 回归检查 +runner/ 共用 runner(AUTO_TEST=benchmark 入口) + run.sh 参数化入口(原 run_tests.sh) + init.sh / env.sh / clean_up.sh 测试环境准备/变量/清理 + test_cases/.sh 测例包装脚本(调用 lmbench 二进制) + test_cases/.meta 该测例的抽取规则 + 单位 + 方向元数据 + test_cases/test_*.sh host 侧解析及包装脚本回归测试 +orchestrator/ host 端编排 + monitor_test_results.sh 串口监控 + 超时/完成判定 + collect_results.py 解析串口 → 校验 → 持久化 + schema/lmbench-run.schema.json 结果 JSON Schema(draft-07) +config 全局配置(SAMPLES / TIMEOUT_SEC / WARMUP) +whitelist.txt 要运行的测例清单(每行一个) +toggle_compile_lmbench.sh 切换是否把本套件编进 rootfs +Makefile DADK build-from-source 的安装脚本 +default.nix / flake.nix Nix 打包(可选) +results/ 运行产物(gitignore;作为 CI 制品归档) + /-.json 每次运行的完整快照(canonical) + history.jsonl 每条指标一行(时序友好) + github-benchmark/data.json github-action-benchmark 兼容格式(为可视化预留) +``` + ## runner 入口参数 `runner/run.sh` 支持以下参数（无参数 = 读 `config` + `whitelist.txt` 全跑，与 rcS 调用一致）： @@ -94,6 +183,12 @@ DESCRIPTION=Memory copy bandwidth via bw_mem fcp 抽取等价于 `awk "/SEARCH_PATTERN/ {print \$(RESULT_INDEX)}"` 取第 `NTH_OCCURRENCE` 个匹配。 延迟类输出多为 `... : VALUE microseconds`，统一用 `RESULT_INDEX=NF-1`；`bw_mem` 输出 ` `，用 `NF`。 +## 结果 Schema + +见 [orchestrator/schema/lmbench-run.schema.json](orchestrator/schema/lmbench-run.schema.json)。每次运行一个 JSON 文档：run 级元数据（run_id/时间/git/arch/host/config） ++ `metrics[]`（每指标含 `samples[]` 原始样本与 `stats{mean,median,stddev,min,max,cv}`）+ `summary`。 +指标 `status` 为 `ok|failed|skipped`；失败时带 `error` 与 `raw_tail`（便于按真实输出校准抽取规则）。 + ## 校准提示 各 lmbench 工具输出格式稳定，`.meta` 的抽取规则起点移植自 Asterinas 已验证配置。 diff --git a/user/apps/tests/benchmark/lmbench/orchestrator/collect_results.py b/user/apps/tests/benchmark/lmbench/orchestrator/collect_results.py new file mode 100644 index 0000000000..82a5723d2e --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/orchestrator/collect_results.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Collect an LMbench benchmark run from the guest serial log into persisted JSON. + +The guest runner (run.sh) prints one ``LMBENCH_JSON`` line per metric plus +``LMBENCH_META`` / ``LMBENCH_SUMMARY`` framing. This tool parses the LAST run in the +serial log, enriches it with host/git/QEMU context, validates it against +schema/lmbench-run.schema.json, and writes: + + results//-.json canonical per-run snapshot + results/history.jsonl one line per metric (time-series friendly) + results/github-benchmark/data.json github-action-benchmark compatible array + +Exit codes: 0 ok; 2 no results parsed; 3 schema validation failed. +""" +import argparse +import json +import os +import re +import socket +import subprocess +import sys +from datetime import datetime, timezone + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCHEMA_VERSION = "1.0" + +RUN_BEGIN = "===LMBENCH_RUN_BEGIN===" +RUN_END = "===LMBENCH_RUN_END===" +P_JSON = "LMBENCH_JSON " +P_META = "LMBENCH_META " +P_SUMMARY = "LMBENCH_SUMMARY " + + +def eprint(*a): + print(*a, file=sys.stderr) + + +def run_git(root, *args): + try: + out = subprocess.run(["git", "-C", root, *args], + capture_output=True, text=True, timeout=15) + if out.returncode == 0: + return out.stdout.strip() + except Exception: + pass + return None + + +def gather_git(root): + commit = run_git(root, "rev-parse", "HEAD") + if not commit: + return None + branch = run_git(root, "rev-parse", "--abbrev-ref", "HEAD") + porcelain = run_git(root, "status", "--porcelain") + return { + "commit": commit, + "branch": branch or "", + "dirty": bool(porcelain), + } + + +def qemu_version(arch): + bin_ = f"qemu-system-{arch}" + try: + out = subprocess.run([bin_, "--version"], capture_output=True, text=True, timeout=10) + if out.returncode == 0: + m = re.search(r"version\s+([0-9][0-9A-Za-z.\-]*)", out.stdout) + return m.group(1) if m else out.stdout.splitlines()[0].strip() + except Exception: + pass + return None + + +def gather_host(arch): + host = {"hostname": socket.gethostname()} + qv = qemu_version(arch) + if qv: + host["qemu"] = qv + if os.path.exists("/dev/kvm"): + host["kvm"] = True + for key, env in (("cpus", "QEMU_SMP"), ("mem_mb", "QEMU_MEMORY_MB")): + val = os.environ.get(env) + if val and val.isdigit(): + host[key] = int(val) + return host + + +def parse_last_run(serial_path): + """Return the last run and explicit frame-presence metadata.""" + with open(serial_path, "r", encoding="utf-8", errors="replace") as f: + lines = f.readlines() + # Locate the last RUN_BEGIN so a serial log spanning several boots is unambiguous. + begin = None + for i, ln in enumerate(lines): + if RUN_BEGIN in ln: + begin = i + framed = begin is not None + if begin is None: + begin = 0 + meta, summary, metrics = None, None, [] + meta_count = summary_count = 0 + ended = False + for ln in lines[begin:]: + s = ln.rstrip("\n") + idx = s.find(P_JSON) + if idx != -1: + metrics.append(_loads(s[idx + len(P_JSON):], "metric")) + continue + idx = s.find(P_META) + if idx != -1: + meta = _loads(s[idx + len(P_META):], "meta") + meta_count += 1 + continue + idx = s.find(P_SUMMARY) + if idx != -1: + summary = _loads(s[idx + len(P_SUMMARY):], "summary") + summary_count += 1 + if RUN_END in s: + ended = True + break + metrics = [m for m in metrics if m is not None] + frame = { + "begin": framed, + "end": ended, + "meta_count": meta_count, + "summary_count": summary_count, + } + return meta, metrics, summary, frame + + +def protocol_errors(meta, metrics, summary, frame): + errors = [] + if not frame["begin"]: + errors.append("missing RUN_BEGIN frame") + if not frame["end"]: + errors.append("incomplete frame: missing RUN_END") + if frame["meta_count"] != 1 or meta is None: + errors.append("frame must contain exactly one valid META") + if frame["summary_count"] != 1 or summary is None: + errors.append("frame must contain exactly one valid SUMMARY") + if summary is not None: + expected = { + "total": len(metrics), + "ok": sum(1 for m in metrics if m.get("status") == "ok"), + "failed": sum(1 for m in metrics if m.get("status") == "failed"), + "skipped": sum(1 for m in metrics if m.get("status") == "skipped"), + } + if any(summary.get(key) != value for key, value in expected.items()): + errors.append(f"summary counts do not match metrics: expected {expected}") + return errors + + +def _loads(text, what): + try: + return json.loads(text) + except json.JSONDecodeError as e: + eprint(f"[collect] WARN: skipping malformed {what} JSON: {e}: {text[:120]}") + return None + + +def guest_kernel_version(serial_path): + """Best-effort: pull a DragonOS version banner out of the serial log.""" + try: + with open(serial_path, "r", encoding="utf-8", errors="replace") as f: + for ln in f: + m = re.search(r"DragonOS\s+v?[0-9][0-9A-Za-z.\-_]*", ln) + if m: + return m.group(0).strip() + except Exception: + pass + return None + + +def build_run(meta, metrics, summary, args, serial_path): + now = datetime.now(timezone.utc) + ts = now.strftime("%Y-%m-%dT%H:%M:%SZ") + git = gather_git(args.root) + short = (git["commit"][:8] if git else "nogit") + run_id = now.strftime("%Y%m%dT%H%M%SZ") + "-" + short + + if not summary: + ok = sum(1 for m in metrics if m.get("status") == "ok") + failed = sum(1 for m in metrics if m.get("status") == "failed") + skipped = sum(1 for m in metrics if m.get("status") == "skipped") + summary = {"total": len(metrics), "ok": ok, "failed": failed, "skipped": skipped} + + cfg = {"samples": (meta or {}).get("samples", 0), + "timeout_sec": (meta or {}).get("timeout_sec", 0)} + if meta and "warmup" in meta: + cfg["warmup"] = meta["warmup"] + + run = { + "schema_version": SCHEMA_VERSION, + "run_id": run_id, + "timestamp": ts, + "arch": args.arch, + "suite": (meta or {}).get("suite", "lmbench"), + "config": cfg, + "metrics": metrics, + "summary": summary, + } + if meta and meta.get("suite_version"): + run["suite_version"] = meta["suite_version"] + if git: + run["git"] = git + kv = guest_kernel_version(serial_path) + if kv: + run["kernel_version"] = kv + host = gather_host(args.arch) + if host: + run["host"] = host + return run + + +def validate(run, schema_path): + try: + import jsonschema + except ImportError: + eprint("[collect] WARN: jsonschema not installed; skipping schema validation") + return True + with open(schema_path) as f: + schema = json.load(f) + try: + jsonschema.validate(run, schema) + return True + except jsonschema.ValidationError as e: + eprint("[collect] SCHEMA VALIDATION FAILED:") + eprint(" path:", "/".join(str(p) for p in e.absolute_path) or "") + eprint(" msg :", e.message) + return False + + +def write_outputs(run, outdir): + arch = run["arch"] + arch_dir = os.path.join(outdir, arch) + os.makedirs(arch_dir, exist_ok=True) + snapshot = os.path.join(arch_dir, f"{run['run_id']}.json") + with open(snapshot, "w", encoding="utf-8") as f: + json.dump(run, f, indent=2, ensure_ascii=False) + f.write("\n") + + # history.jsonl: one flat record per metric + hist = os.path.join(outdir, "history.jsonl") + base = {"run_id": run["run_id"], "timestamp": run["timestamp"], "arch": arch, + "commit": run.get("git", {}).get("commit")} + with open(hist, "a", encoding="utf-8") as f: + for m in run["metrics"]: + rec = dict(base) + rec.update({ + "name": m.get("name"), "category": m.get("category"), + "metric_type": m.get("metric_type"), "unit": m.get("unit"), + "bigger_is_better": m.get("bigger_is_better"), + "status": m.get("status"), + }) + st = m.get("stats") or {} + for k in ("mean", "median", "stddev", "min", "max", "cv"): + rec[k] = st.get(k) + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + + # github-action-benchmark compatible array (ok metrics only) + gh_dir = os.path.join(outdir, "github-benchmark") + os.makedirs(gh_dir, exist_ok=True) + gh = [] + for m in run["metrics"]: + if m.get("status") != "ok": + continue + st = m["stats"] + direction = "bigger is better" if m["bigger_is_better"] else "smaller is better" + gh.append({ + "name": m["name"], + "unit": m["unit"], + "value": st["mean"], + "extra": (f"category={m.get('category')} median={st['median']} " + f"stddev={st['stddev']} cv={st['cv']} n={st['count']} ({direction})"), + }) + with open(os.path.join(gh_dir, "data.json"), "w", encoding="utf-8") as f: + json.dump(gh, f, indent=2, ensure_ascii=False) + f.write("\n") + return snapshot + + +def print_summary(run): + s = run["summary"] + print(f"[collect] run_id={run['run_id']} arch={run['arch']} " + f"commit={run.get('git', {}).get('commit', 'n/a')[:8]}") + print(f"[collect] metrics: total={s['total']} ok={s['ok']} " + f"failed={s['failed']} skipped={s['skipped']}") + print(f"{'METRIC':<34} {'STATUS':<8} {'MEAN':>14} {'UNIT':<12} {'CV':>8}") + for m in run["metrics"]: + st = m.get("stats") or {} + mean = f"{st['mean']:.4f}" if "mean" in st else "-" + cv = f"{st['cv']:.4f}" if "cv" in st else "-" + print(f"{m.get('name', '?'):<34} {m.get('status', '?'):<8} " + f"{mean:>14} {m.get('unit', ''):<12} {cv:>8}") + + +def main(): + ap = argparse.ArgumentParser(description="Collect an LMbench run from serial into JSON") + ap.add_argument("--serial", default=os.environ.get("SERIAL_FILE", "serial_opt.txt"), + help="guest serial log to parse") + ap.add_argument("--schema", default=os.path.join(HERE, "schema", "lmbench-run.schema.json")) + ap.add_argument("--outdir", default=os.path.join(HERE, "..", "results")) + ap.add_argument("--arch", default=os.environ.get("ARCH", "x86_64")) + ap.add_argument("--root", default=os.environ.get("ROOT_PATH", os.getcwd()), + help="repo root for git metadata") + args = ap.parse_args() + + if not os.path.isfile(args.serial): + eprint(f"[collect] ERROR: serial log not found: {args.serial}") + return 2 + + meta, metrics, summary, frame = parse_last_run(args.serial) + errors = protocol_errors(meta, metrics, summary, frame) + if errors: + for error in errors: + eprint(f"[collect] ERROR: {error}") + return 2 + if not metrics: + eprint("[collect] ERROR: no LMBENCH_JSON metrics found in serial log " + "(run did not produce results)") + return 2 + + run = build_run(meta, metrics, summary, args, args.serial) + if not validate(run, args.schema): + return 3 + print("[collect] schema validation: PASS") + snapshot = write_outputs(run, args.outdir) + print_summary(run) + print(f"[collect] wrote {snapshot}") + print(f"[collect] appended {os.path.join(args.outdir, 'history.jsonl')}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh b/user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh new file mode 100644 index 0000000000..908bad177b --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh @@ -0,0 +1,139 @@ +#!/bin/busybox sh +# Host-side monitor for the LMbench benchmark run. +# +# Boots are driven by the Makefile `test-benchmark` target, which starts QEMU in +# the background (serial -> serial_opt.txt, PID -> $VMSTATE_DIR/pid) and then runs +# this script. We poll the serial log for boot / run-begin / progress / completion +# and enforce timeouts, then tear QEMU down. Unlike the syscall suite this is a +# performance run, so individual metric failures are DATA, not harness failures: +# we exit 0 as long as the run reaches "benchmark测试完成". + +if [ -z "${ROOT_PATH}" ]; then + echo "[monitor] 错误: ROOT_PATH 未设置(请通过 Makefile 运行)"; exit 1 +fi +if [ -z "${VMSTATE_DIR}" ]; then + echo "[monitor] 错误: VMSTATE_DIR 未设置(请通过 Makefile 运行)"; exit 1 +fi + +SERIAL_FILE="${SERIAL_FILE:-serial_opt.txt}" +BOOT_TIMEOUT="${BENCH_BOOT_TIMEOUT:-300}" # DragonOS boot +RUN_START_TIMEOUT="${BENCH_RUN_START_TIMEOUT:-600}" # until run begins +TOTAL_TIMEOUT="${BENCH_TOTAL_TIMEOUT:-3600}" # whole run (benchmarks are slow) +IDLE_TIMEOUT="${BENCH_IDLE_TIMEOUT:-600}" # no serial output + +get_qemu_pid() { [ -f "${VMSTATE_DIR}/pid" ] && cat "${VMSTATE_DIR}/pid" || echo ""; } + +clean_up() { + echo "[monitor] 清理资源..." + if [ -f "${VMSTATE_DIR}/pid" ]; then + QEMU_PID=$(cat "${VMSTATE_DIR}/pid") + if [ -n "$QEMU_PID" ] && sudo kill -0 "$QEMU_PID" 2>/dev/null; then + echo "[monitor] 终止 QEMU (PID: $QEMU_PID)" + sudo kill -TERM "$QEMU_PID" 2>/dev/null + sleep 3 + sudo kill -0 "$QEMU_PID" 2>/dev/null && sudo kill -9 "$QEMU_PID" 2>/dev/null + fi + rm -f "${VMSTATE_DIR}/pid" + fi + pkill -P $$ 2>/dev/null + stty sane 2>/dev/null +} + +show_diag() { + echo "[monitor] ===== 诊断 =====" + echo "[monitor] 已运行: $(($(date +%s) - START_TIME))s boot=$BOOT_DONE run=$RUN_STARTED" + if [ -f "$SERIAL_FILE" ]; then + echo "[monitor] 已产出指标: $(grep -ac 'LMBENCH_JSON ' "$SERIAL_FILE" 2>/dev/null)" + echo "[monitor] 串口尾部:"; tail -n 15 "$SERIAL_FILE" 2>/dev/null | sed 's/^/[monitor] /' + fi + echo "[monitor] ================" +} + +check_qemu_alive() { + p=$(get_qemu_pid); [ -n "$p" ] && sudo kill -0 "$p" 2>/dev/null +} +check_boot_done() { + [ -f "$SERIAL_FILE" ] && ( grep -aq "\[rcS\] Running system init script" "$SERIAL_FILE" 2>/dev/null || \ + grep -aq "===LMBENCH_RUN_BEGIN===" "$SERIAL_FILE" 2>/dev/null ) +} +check_run_started() { + [ -f "$SERIAL_FILE" ] && ( grep -aq "===LMBENCH_RUN_BEGIN===" "$SERIAL_FILE" 2>/dev/null || \ + grep -aq "LMBENCH_JSON " "$SERIAL_FILE" 2>/dev/null ) +} +check_done() { + [ -f "$SERIAL_FILE" ] && grep -aq "benchmark测试完成" "$SERIAL_FILE" 2>/dev/null +} + +# ---- wait for QEMU PID ---- +echo "[monitor] 等待 QEMU 写入 PID 文件..." +WAITED=0 +QEMU_PID=$(get_qemu_pid) +while [ -z "$QEMU_PID" ] && [ $WAITED -lt 30 ]; do + sleep 1; WAITED=$((WAITED + 1)); QEMU_PID=$(get_qemu_pid) +done +if [ -z "$QEMU_PID" ]; then echo "[monitor] 错误: 超时未找到 PID 文件"; exit 1; fi +if ! sudo kill -0 "$QEMU_PID" 2>/dev/null; then echo "[monitor] 错误: QEMU (PID $QEMU_PID) 不存在"; exit 1; fi +echo "[monitor] QEMU PID: $QEMU_PID" + +START_TIME=$(date +%s) +LAST_OUTPUT_TIME=$START_TIME +LAST_LINES=0 +BOOT_DONE=false +RUN_STARTED=false + +trap 'clean_up; exit 1' INT TERM + +echo "[monitor] 开始监控 (boot=${BOOT_TIMEOUT}s run_start=${RUN_START_TIMEOUT}s total=${TOTAL_TIMEOUT}s idle=${IDLE_TIMEOUT}s)" +while true; do + NOW=$(date +%s); ELAPSED=$((NOW - START_TIME)) + + if [ "$ELAPSED" -gt "$TOTAL_TIMEOUT" ]; then + echo "[monitor] 错误: 总超时 (${TOTAL_TIMEOUT}s)"; show_diag; clean_up; exit 1 + fi + if ! check_qemu_alive; then + # QEMU may have exited right after printing completion; re-check done. + if check_done; then break; fi + echo "[monitor] 错误: QEMU 进程已退出"; show_diag; clean_up; exit 1 + fi + + if [ "$BOOT_DONE" = false ]; then + if check_boot_done; then BOOT_DONE=true; echo "[monitor] 系统已启动"; LAST_OUTPUT_TIME=$NOW + elif [ "$ELAPSED" -gt "$BOOT_TIMEOUT" ]; then + echo "[monitor] 错误: 启动超时 (${BOOT_TIMEOUT}s)"; show_diag; clean_up; exit 1 + fi + fi + if [ "$BOOT_DONE" = true ] && [ "$RUN_STARTED" = false ]; then + if check_run_started; then RUN_STARTED=true; echo "[monitor] 基准运行已开始"; LAST_OUTPUT_TIME=$NOW + elif [ "$ELAPSED" -gt "$RUN_START_TIMEOUT" ]; then + echo "[monitor] 错误: 基准程序启动超时 (${RUN_START_TIMEOUT}s)"; show_diag; clean_up; exit 1 + fi + fi + + # idle detection + if [ -f "$SERIAL_FILE" ]; then + CUR_LINES=$(wc -l < "$SERIAL_FILE" 2>/dev/null || echo 0) + MTIME=$(stat -c %Y "$SERIAL_FILE" 2>/dev/null || echo 0) + if [ "$CUR_LINES" -gt "$LAST_LINES" ] || [ "$MTIME" -gt "$((NOW - 5))" ]; then + LAST_OUTPUT_TIME=$NOW; LAST_LINES=$CUR_LINES + elif [ "$((NOW - LAST_OUTPUT_TIME))" -gt "$IDLE_TIMEOUT" ]; then + echo "[monitor] 错误: ${IDLE_TIMEOUT}s 无输出,疑似卡死"; show_diag; clean_up; exit 1 + fi + fi + + if check_done; then echo "[monitor] 检测到 benchmark测试完成"; break; fi + + if [ $((ELAPSED % 60)) -eq 0 ]; then + n=$(grep -ac 'LMBENCH_JSON ' "$SERIAL_FILE" 2>/dev/null || echo 0) + echo "[monitor] 运行中... (${ELAPSED}s, 已产出 ${n} 个指标)" + fi + sleep 10 +done + +# summary from serial (informational; does not gate exit) +SUMMARY=$(grep -a 'LMBENCH_SUMMARY ' "$SERIAL_FILE" 2>/dev/null | tail -n1) +echo "[monitor] 运行完成,用时 $(($(date +%s) - START_TIME))s" +[ -n "$SUMMARY" ] && echo "[monitor] ${SUMMARY}" +clean_up +stty sane 2>/dev/null +echo "[monitor] OK: 基准运行结束,结果待 collect_results.py 收集" +exit 0 diff --git a/user/apps/tests/benchmark/lmbench/orchestrator/schema/lmbench-run.schema.json b/user/apps/tests/benchmark/lmbench/orchestrator/schema/lmbench-run.schema.json new file mode 100644 index 0000000000..c030b4dfc5 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/orchestrator/schema/lmbench-run.schema.json @@ -0,0 +1,180 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://dragonos.org/schema/lmbench-run.schema.json", + "title": "DragonOS LMbench Benchmark Run", + "description": "One benchmark run: environment metadata plus per-metric samples and statistics. Produced by the host-side collector from guest serial output.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "run_id", + "timestamp", + "arch", + "config", + "metrics", + "summary" + ], + "properties": { + "schema_version": { + "type": "string", + "description": "Semantic version of THIS schema; bump on breaking shape changes.", + "pattern": "^[0-9]+\\.[0-9]+$" + }, + "run_id": { + "type": "string", + "description": "Unique id for the run, e.g. -.", + "minLength": 1 + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "Run start time, UTC, ISO-8601 (e.g. 2026-07-27T12:00:00Z)." + }, + "arch": { + "type": "string", + "enum": ["x86_64", "riscv64", "aarch64", "loongarch64"] + }, + "suite": { + "type": "string", + "description": "Benchmark suite identifier.", + "default": "lmbench" + }, + "suite_version": { + "type": "string", + "description": "Version of the lmbench binaries under test (e.g. 3.0-a9)." + }, + "git": { + "type": "object", + "additionalProperties": false, + "description": "Source revision of DragonOS under test.", + "properties": { + "commit": { "type": "string" }, + "branch": { "type": "string" }, + "dirty": { "type": "boolean" } + } + }, + "kernel_version": { + "type": "string", + "description": "DragonOS kernel version/build string, if captured." + }, + "host": { + "type": "object", + "description": "Host/QEMU environment that produced the numbers (affects comparability).", + "properties": { + "qemu": { "type": "string" }, + "kvm": { "type": "boolean" }, + "cpus": { "type": "integer", "minimum": 1 }, + "mem_mb": { "type": "integer", "minimum": 1 }, + "hostname": { "type": "string" } + } + }, + "config": { + "type": "object", + "additionalProperties": false, + "description": "Runner configuration in effect for this run.", + "required": ["samples", "timeout_sec"], + "properties": { + "samples": { "type": "integer", "minimum": 1, "description": "Repetitions per metric." }, + "timeout_sec": { "type": "integer", "minimum": 1, "description": "Per-metric wall-clock timeout." }, + "warmup": { "type": "integer", "minimum": 0, "description": "Warmup repetitions discarded before sampling." } + } + }, + "metrics": { + "type": "array", + "items": { "$ref": "#/definitions/metric" } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": ["total", "ok", "failed", "skipped"], + "properties": { + "total": { "type": "integer", "minimum": 0 }, + "ok": { "type": "integer", "minimum": 0 }, + "failed": { "type": "integer", "minimum": 0 }, + "skipped": { "type": "integer", "minimum": 0 } + } + } + }, + "definitions": { + "metric": { + "type": "object", + "additionalProperties": false, + "required": ["name", "metric_type", "unit", "bigger_is_better", "status"], + "properties": { + "name": { + "type": "string", + "description": "Stable metric id, matches the test_cases/.sh basename.", + "pattern": "^[a-z0-9_]+$" + }, + "category": { + "type": "string", + "description": "Grouping for charts, e.g. memory, process, ipc, vfs, network, signal, filesystem.", + "enum": ["memory", "process", "ipc", "vfs", "network", "signal", "filesystem", "other"] + }, + "binary": { "type": "string", "description": "Underlying lmbench binary, e.g. bw_mem, lat_pipe." }, + "command": { "type": "string", "description": "Exact command line executed inside the guest." }, + "description": { "type": "string" }, + "metric_type": { + "type": "string", + "enum": ["latency", "bandwidth", "ops", "other"] + }, + "unit": { + "type": "string", + "description": "Human unit, e.g. microseconds, MB/s, ops/sec. May be empty for skipped/failed metrics that never resolved a unit." + }, + "bigger_is_better": { + "type": "boolean", + "description": "Direction of goodness; drives regression alerts and chart legends." + }, + "status": { + "type": "string", + "enum": ["ok", "failed", "skipped"] + }, + "samples": { + "type": "array", + "description": "Raw per-repetition values, in run order.", + "items": { "type": "number" } + }, + "stats": { "$ref": "#/definitions/stats" }, + "error": { + "type": "string", + "description": "Present when status is failed/skipped: reason (timeout, non-zero exit, no-match, etc.)." + }, + "raw_tail": { + "type": "string", + "description": "Tail of captured output; emitted on extraction failure to aid calibration." + } + }, + "allOf": [ + { + "if": { "properties": { "status": { "const": "ok" } } }, + "then": { + "required": ["samples", "stats"], + "properties": { + "unit": { "type": "string", "minLength": 1 }, + "samples": { "minItems": 1 } + } + } + } + ] + }, + "stats": { + "type": "object", + "additionalProperties": false, + "required": ["count", "mean", "median", "stddev", "min", "max"], + "properties": { + "count": { "type": "integer", "minimum": 1 }, + "mean": { "type": "number" }, + "median": { "type": "number" }, + "stddev": { "type": "number", "minimum": 0 }, + "min": { "type": "number" }, + "max": { "type": "number" }, + "cv": { + "type": "number", + "minimum": 0, + "description": "Coefficient of variation = stddev/mean; a stability indicator." + } + } + } + } +} diff --git a/user/apps/tests/benchmark/lmbench/tests/test_collect_results.py b/user/apps/tests/benchmark/lmbench/tests/test_collect_results.py new file mode 100644 index 0000000000..9cb39c0727 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/tests/test_collect_results.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +import json +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SUITE = Path(__file__).resolve().parent.parent +COLLECTOR = SUITE / "orchestrator" / "collect_results.py" + + +def metric(name="process_getppid_lat", complete=True): + value = { + "name": name, + "category": "process", + "metric_type": "latency", + "unit": "microseconds", + "bigger_is_better": False, + "status": "ok", + } + if complete: + value.update({ + "samples": [1.25], + "stats": {"count": 1, "mean": 1.25, "median": 1.25, + "stddev": 0.0, "min": 1.25, "max": 1.25, "cv": 0.0}, + }) + return value + + +def frame(metrics, summary=None, end=True): + if summary is None: + summary = {"total": len(metrics), "ok": len(metrics), "failed": 0, "skipped": 0} + lines = [ + "===LMBENCH_RUN_BEGIN===", + 'LMBENCH_META ' + json.dumps({"suite": "lmbench", "suite_version": "3.0-a9", + "samples": 1, "timeout_sec": 3, "warmup": 0}), + ] + lines.extend("LMBENCH_JSON " + json.dumps(item) for item in metrics) + lines.append("LMBENCH_SUMMARY " + json.dumps(summary)) + if end: + lines.append("===LMBENCH_RUN_END===") + return "\n".join(lines) + "\n" + + +class CollectorProtocolTest(unittest.TestCase): + def run_collector(self, serial_text): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + root = Path(temp.name) + serial = root / "serial.log" + outdir = root / "out" + serial.write_text(serial_text, encoding="utf-8") + proc = subprocess.run( + ["python3", str(COLLECTOR), "--serial", str(serial), "--outdir", str(outdir), + "--root", str(SUITE)], + text=True, capture_output=True, + ) + return proc, outdir + + def assert_no_outputs(self, outdir): + self.assertFalse((outdir / "history.jsonl").exists()) + self.assertFalse((outdir / "github-benchmark" / "data.json").exists()) + self.assertFalse((outdir / "x86_64").exists()) + + def test_complete_frame_is_persisted(self): + proc, outdir = self.run_collector(frame([metric()])) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertTrue((outdir / "history.jsonl").is_file()) + self.assertEqual(len(list((outdir / "x86_64").glob("*.json"))), 1) + + def test_truncated_last_frame_does_not_fall_back_to_old_complete_frame(self): + serial = frame([metric("first")]) + frame([metric("last")], end=False) + proc, outdir = self.run_collector(serial) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("incomplete", proc.stderr.lower()) + self.assert_no_outputs(outdir) + + def test_summary_counts_must_match_metric_statuses(self): + wrong = {"total": 2, "ok": 1, "failed": 1, "skipped": 0} + proc, outdir = self.run_collector(frame([metric()], wrong)) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("summary", proc.stderr.lower()) + self.assert_no_outputs(outdir) + + def test_schema_failure_does_not_write_any_output(self): + proc, outdir = self.run_collector(frame([metric(complete=False)])) + self.assertEqual(proc.returncode, 3, proc.stderr) + self.assertIn("schema validation failed", proc.stderr.lower()) + self.assert_no_outputs(outdir) + + def test_unframed_input_is_rejected(self): + serial = "LMBENCH_JSON " + json.dumps(metric()) + "\n" + proc, outdir = self.run_collector(serial) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("frame", proc.stderr.lower()) + self.assert_no_outputs(outdir) + + +if __name__ == "__main__": + unittest.main() diff --git a/user/apps/tests/benchmark/lmbench/toggle_compile_lmbench.sh b/user/apps/tests/benchmark/lmbench/toggle_compile_lmbench.sh new file mode 100755 index 0000000000..4315a1a259 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/toggle_compile_lmbench.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Toggle whether the "lmbench benchmark tests" app is compiled into the rootfs, +# by (un)commenting its line in config/app-blocklist.toml. Mirrors +# tools/../gvisor/toggle_compile_gvisor.sh. +# +# enable comment the blocklist line -> app is BUILT +# disable uncomment the blocklist line -> app is BLOCKED + +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +用法: toggle_compile_lmbench.sh + enable 注释 lmbench 基准配置行，允许编译 + disable 取消注释 lmbench 基准配置行，继续屏蔽 +EOF + exit 1 +} + +ACTION="${1:-}" +shift || true +case "$ACTION" in + enable|disable) ;; + *) usage ;; +esac + +SCRIPT_DIR="$(cd -- "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/../../../../.." && pwd)" +CONFIG_FILE="$REPO_ROOT/config/app-blocklist.toml" +TMP_FILE="${CONFIG_FILE}.tmp" +TARGET_PATTERN='name[[:space:]]*=[[:space:]]*"lmbench benchmark tests"' + +if [ ! -f "$CONFIG_FILE" ]; then + echo "错误：配置文件不存在: $CONFIG_FILE" >&2 + exit 1 +fi +if [ ! -w "$CONFIG_FILE" ]; then + echo "错误：配置文件不可写: $CONFIG_FILE" >&2 + exit 1 +fi + +cleanup() { rm -f "$TMP_FILE"; } +trap cleanup EXIT + +if ! awk -v action="$ACTION" -v pattern="$TARGET_PATTERN" ' + function comment_line(line, indent, rest) { + if (line ~ /^[[:space:]]*$/) { return line } + if (line ~ /^[[:space:]]*#/) { return line } + match(line, /^[[:space:]]*/) + indent = substr(line, 1, RLENGTH) + rest = substr(line, RLENGTH + 1) + if (rest == "") { return indent "#" } + return indent "# " rest + } + function uncomment_line(line, indent, rest) { + if (line ~ /^[[:space:]]*$/) { return line } + match(line, /^[[:space:]]*/) + indent = substr(line, 1, RLENGTH) + rest = substr(line, RLENGTH + 1) + while (rest ~ /^#/) { sub(/^# ?/, "", rest) } + return indent rest + } + BEGIN { matched = 0 } + { + line = $0 + tmp = line + sub(/^[[:space:]]*# ?/, "", tmp) + if (!matched && tmp ~ pattern) { + if (action == "enable") { + if (line ~ /^[[:space:]]*#/) { print line } else { print comment_line(line) } + } else { + if (line ~ /^[[:space:]]*#/) { print uncomment_line(line) } else { print line } + } + matched = 1 + next + } + print line + } + END { if (!matched) { exit 1 } } +' "$CONFIG_FILE" > "$TMP_FILE"; then + echo "错误：未找到 lmbench 基准配置行" >&2 + exit 1 +fi + +mv "$TMP_FILE" "$CONFIG_FILE" +trap - EXIT +cleanup diff --git a/user/sysconfig/etc/init.d/rcS b/user/sysconfig/etc/init.d/rcS index e48d599c73..3703417a1c 100755 --- a/user/sysconfig/etc/init.d/rcS +++ b/user/sysconfig/etc/init.d/rcS @@ -64,4 +64,6 @@ if [ "$AUTO_TEST" = "syscall" ]; then /bin/busybox sh $SYSCALL_TEST_DIR/run_tests.sh elif [ "$AUTO_TEST" = "dunit" ]; then /bin/busybox sh $DUNITEST_DIR/run_tests.sh +elif [ "$AUTO_TEST" = "benchmark" ]; then + /bin/busybox sh $BENCHMARK_TEST_DIR/run.sh fi From b412e232972bb1c7a2babff0940bf82b684ba385 Mon Sep 17 00:00:00 2001 From: yuming Date: Tue, 22 Sep 2026 00:48:26 +0800 Subject: [PATCH 06/22] ci(lmbench): add benchmark workflow Run the benchmark workflow and publish result artifacts. --- .github/workflows/benchmark.yml | 64 +++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..0ee67a579b --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,64 @@ +name: LMbench Benchmark x86_64 + +# MVP scope: build DragonOS, run the lmbench performance suite under QEMU, and +# archive the per-run result JSON + serial log. No regression gating / charts yet. +on: + workflow_dispatch: + push: + branches: ["feat/lmbench*", "feat-lmbench*"] + +env: + ARCH: x86_64 + HOME: /root + RUSTUP_DIST_SERVER: "https://rsproxy.cn" + RUSTUP_UPDATE_ROOT: "https://rsproxy.cn/rustup" + +jobs: + benchmark: + name: LMbench + runs-on: ubuntu-latest + # Benchmarks are slow: monitor allows up to a 60-minute run, plus build/cleanup. + timeout-minutes: 90 + container: + image: dragonos/dragonos-dev:v1.23 + options: --privileged -v /dev:/dev + steps: + - name: Checkout DragonOS code + uses: actions/checkout@v4 + + - name: Change source + run: | + find . -type f \( -name "*.toml" -o -name "Makefile" \) -exec sed -i 's/git\.mirrors\.dragonos\.org\.cn/github\.com/g' {} + + + - name: Ensure jsonschema for the result collector + run: pip3 install --quiet jsonschema || true + + - name: Build DragonOS + shell: bash -ileo pipefail {0} + run: | + make clean + make -j$(nproc) all + + - name: Run lmbench benchmark + shell: bash -ileo pipefail {0} + env: + DISK_SAVE_MODE: "1" + run: | + make test-benchmark + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: lmbench-results-x86_64-${{ github.run_id }}-${{ github.run_attempt }} + path: | + user/apps/tests/benchmark/lmbench/results/** + if-no-files-found: warn + + - name: Preserve benchmark serial log + if: always() + uses: actions/upload-artifact@v4 + with: + name: lmbench-serial-x86_64-${{ github.run_id }}-${{ github.run_attempt }} + path: serial_opt.txt + if-no-files-found: error From 99190c075bc6752ff6fbdb6eea8dc0114f3555f8 Mon Sep 17 00:00:00 2001 From: yuming Date: Tue, 22 Sep 2026 00:48:26 +0800 Subject: [PATCH 07/22] docs(lmbench): record recovery validation and remaining limitations Record wrapper and guest recovery validation, scope boundaries and remaining failures. --- .../2026-09-19-lmbench-wrapper-results.md | 26 +++++++ ...2026-09-20-lmbench-environment-recovery.md | 35 ++++++++++ .../completed/2026-09-20-lmbench-recovery.md | 70 +++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md create mode 100644 harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md create mode 100644 harness/exec-plans/completed/2026-09-20-lmbench-recovery.md diff --git a/harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md b/harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md new file mode 100644 index 0000000000..c4f23899d6 --- /dev/null +++ b/harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md @@ -0,0 +1,26 @@ +# LMbench wrapper 结果判定修复 + +所有 48 个 wrapper 与 runner 共享结果校验：命令成功退出、提取到有限正数且没有断言或致命错误，才记为成功。测量输出有大小上限，实际非零退出码保留；测量进程组和临时 FIFO 在退出时清理。内核计时与协议问题不在此次修复范围。 + +## 验证 + +- 六项 shell 回归分别在 dash 与实际 BusyBox ash 下通过，共 12/12；覆盖有效值、断言、错误伴随数值、内部超时、ENOSYS、缺失指标、退出码和清理。 +- 63 个脚本在两种 shell 下通过语法检查，安装后的布局与 help 验证通过。 +- 最后一次错误分类修正后，两种 shell 的 runner/wrapper 覆盖回归通过。 +- 只读检查历史 48 项输出，分类符合预期（13 项有效）；该检查不代表本轮实际运行。 +- 2026-09-19 在单台 DragonOS VM 中调用完整 suite 一次，29 项返回结果（11 成功、18 失败）。第 30 项 ext4_create_delete_files_0k_ops 超过 600 秒客体时限后仍未返回，690 秒宿主保护终止运行，后续 18 项未执行。该次运行未完成全部 48 项，随后对白名单剩余项连续补跑。结果见 [本轮数据](../../references/lmbench-dragonos/20260919T144815Z-single-suite/README.md)。 +- 客体没有执行最终 sync，磁盘导出的部分原始输出和退出码缺失；收到的 29 项 runner 结果完整保留，不用缺失数据推断成功或失败。 + +## 结果存储 + +按用户要求，Linux 与 DragonOS 参考目录仅保留测试结果、原始样本和必要的环境/参数。删除旧源码快照、构建目录、镜像、预检、调试与进度文件。Linux 入口使用临时目录保存执行快照和过程文件，并在退出时清理；数据完整性通过保留文件的历史 SHA-256 核对，551 份主要结果和原始样本保持不变。入口修改后在 dash 与实际 BusyBox ash 下通过路径映射、失败传播和清理回归；两种 shell 语法检查与 ShellCheck 通过。 + +用户已授权验收后本地提交，不推送。首次单次全量运行因阻塞未覆盖后续 18 项；后续将这些用例组成白名单继续执行。若 VM 再次无法推进，自动排除已尝试项并以剩余白名单续跑，直到 48 项全部有实际尝试记录；各轮结果分开保存，不声称一次启动跑完。现已完成覆盖和结果核对。 + +## 最终覆盖 + +补跑 18 项全部尝试：3 项成功、13 项失败、2 项挂起。原全量调用与补跑共 4 次启动，合计 14 项成功、31 项失败、3 项挂起，未执行项为 0。各次启动均运行当时剩余的完整白名单，仅在 VM 无法继续时重新启动剩余项；没有把这些结果描述成单次启动跑完。 + +[完整覆盖表](../../references/lmbench-dragonos/20260919-remaining-18/coverage.csv) 保留逐项运行来源。最后一批完成 guest sync，TCP connect 与两项 UDP 的原始退出码为 124、1、1，与失败结果一致；TCP 64K 的原始退出码仍未导出。前面被宿主终止的运行也存在原始数据未落盘缺口，均已明确标注，不能仅凭 sync 标记认定数据完整。计时断言、ENOSYS、无效输出和超时仍是现存问题，本轮不构成稳定性能 baseline。 + +实际安装的 106 个文件与当前工作区逐个比对一致。已有主机回归、语法检查、安装布局验证通过；本轮补齐全 48 项实际尝试，不表示内核问题全部修复。按用户授权提交 wrapper 修复和参考数据精简，不推送。 diff --git a/harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md b/harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md new file mode 100644 index 0000000000..70b4152fad --- /dev/null +++ b/harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md @@ -0,0 +1,35 @@ +# LMbench 剩余用例恢复 + +## 目标与约束 + +在当前已提交 c453800b 基础上，不新增 DragonOS 本体改动，尽可能提高一次完整48项运行中的有效成功数量。此前用户授权的timeval ABI修复保持，不回退；本阶段只处理harness、基准程序和隔离测试环境中有证据的根因。 +- debug仅小白名单，最终验收才跑全部48；不隐藏失败、不改测量含义、不拼接多轮冒充单轮。 +- 所有子代理 sol-medium（gpt-5.6-sol/medium），不派生代理。 +- 保留用户config和dragonos-block-case；临时环境新建/tmp/lmbench-env-recovery-20260920，过程证据result/lmbench-env-recovery-20260920，refs仅最终精简数据。 +- 沿用串口存8MiB/收32MiB停机、stderr存1MiB/收8MiB停机、行64KiB、guest180s/controller270s及总超时；不启savevm、不加高频日志。 +- 不与Linux性能测试并发，不修改宿主全局网络；仅私有QEMU来宾网络。验收后本地提交，不推送。 + +## 阶段开始时的证据 + +上一轮单次full29ok/2failed/1panic无result/16未执行；跨3VM覆盖39wrapper-valid ok/8failed/1panic，非baseline。signal_prot先RCU panic，随后ramfs0k allocator OOM。5个virtio-labelled cases Network is unreachable，启动DHCP未验证就绪，目标10.0.2.15实际是guest自身地址，不代表外部virtio链路。unix_connect timeout而Linux同参数通过；tcp_loopback_select EAGAIN。ramfs10k独立VM通过。Linuxdash和BusyBox各48/48，当前无测试进程。上一轮临时文件按用户授权已清理，正式结果与有界失败现场均保留。 + +## 进展与结果 + +1. 网络前置条件已定位并在私有 guest 中恢复：按实际 MAC 选择 `eth2`，配置 `10.0.2.15/24` connected route，并补足 `127.0.0.1 localhost` 解析。独立 network-six 诊断 6/6 成功；该结果只覆盖 guest 自身非 loopback 地址上的协议路径，不代表 host 或外部 virtio 吞吐。 +2. 首次 full48 尝试 dispatch/result 29/29，均为 ok，未观察到 panic；第29项结果产生后的私有全局 `sync` 卡住，19项未调度。该轮协议不完整，不具备 baseline 资格。 +3. 将采集改为逐文件 fsync 并通过两项 checkpoint probe 后，独立 full48 尝试 dispatch 31、result 30：28 ok、2 failed、1 no-result、17未调度。第12项 `pipe_lat` 首次出现 `invalid RCU context transition` 和 cannot-unwind panic；仅前11条结果位于首次可见 panic 前，从第12项 `pipe_lat` 起共19条 result（含首次 panic 用例，17 ok、2 failed）均标记污染，其中严格位于 panic 用例之后的是18条。 +4. RAMFS10k 在前述 panic 后出现 allocator OOM 且无 result；本轮不能证明它在无前序 panic 的 full 序列中独立 OOM。`vfs_fcntl_lat` failure 同样发生在 panic 后，不能作为健康环境下的独立结论。 +5. 三轮结果已分别精简归档到 `harness/references/lmbench-dragonos/20260920-environment-recovery/`，没有拼接为 full48。过程日志、磁盘、脚本、源码和完整 readback 树仍位于 references 之外。 + +## 收尾状态 + +用户于 2026-09-20 选择按当前“不新增 DragonOS 本体改动”的范围收尾，RCU 缺陷转交独立 issue；本阶段结束，不再继续运行或修复内核。 + +- 本阶段文档、测试数据已提交为 `5b88d88d`，未推送；17 个本轮临时目标已清理，原始小型诊断证据、用户配置和历史材料保留。 +- RCU bug 已公开记录为 [#2292](https://github.com/DragonOS-Community/DragonOS/issues/2292)，直接包含崩溃栈、测试条件、公开源码链接及测试内核差异，不依赖读者访问本地文件。 +- 已加入 lmbench 主 issue [#2286](https://github.com/DragonOS-Community/DragonOS/issues/2286) 的问题清单，并从 #2292 回链主 issue。原生 sub-issue POST 返回 HTTP 404，当前账号没有 triage 权限；本次建立的是双向普通链接，不宣称原生父子关系已经建立。 +- 本次收尾不表示完整 48 项健康验收通过。RCU panic、后续数据污染及未执行项保持原有记录；精简数据中的 `baseline_eligible=false` 不变。后续内核修复与重新验收由 #2292 继续跟踪。 + +## 用户分工约束 + +用户已在另一个分支处理 tcp_reuseport。若剩余失败确实归因该机制，本阶段仅记录证据与外部分支归属，不重复修复、不绕过、不自行引入其他分支改动。EADDRINUSE 与 ENETUNREACH 不能直接视为 SO_REUSEPORT 问题。 diff --git a/harness/exec-plans/completed/2026-09-20-lmbench-recovery.md b/harness/exec-plans/completed/2026-09-20-lmbench-recovery.md new file mode 100644 index 0000000000..bdde6b6d38 --- /dev/null +++ b/harness/exec-plans/completed/2026-09-20-lmbench-recovery.md @@ -0,0 +1,70 @@ +# LMbench 有效结果恢复 + +## 完成结论 + +计时 ABI 修复、本轮覆盖与验收完成，但 DragonOS 稳定 baseline 尚未成立。 + +本轮修复 DragonOS 原生 64 位 `timeval` ABI，完善 LMbench FS、runner、Linux 校准透传、结果协议校验与 fixture 清理，并分别完成 DragonOS 和 Linux 验收。代码、DragonOS 数据及 Linux 数据均通过独立复核。Linux 的 dash 与真实 BusyBox sh 各自完整通过 48/48；DragonOS 的 single full attempt 因 panic/OOM 未完成，后续仅用两个独立 VM 补足 dispatch 覆盖，因此归档明确标记 `baseline_eligible=false`。 + +本计划基于 `origin/master=141bc52f` rebase 后的 `HEAD=cd59d7ba5c9833db60a88f69be89712c6d8152b4` 完成。用户自有 `config/app-blocklist.toml` 与 `dragonos-block-case/` 全程排除在暂存和清理范围之外。 + +## 实现结果 + +### timeval ABI + +- 将 `PosixSusecondsT` 从 32 位 `c_int` 修正为原生 64 位目标使用的 `c_long`，同步修正 gettimeofday、itimer 与 select copyback 的显式构造点。 +- 新增 `user/apps/c_unitest/test_timeval_abi.c`，主要以被污染内存下的 libc/raw gettimeofday 前后对照证明根因，并覆盖 itimer/select 路径。 +- rebase 后内核 SHA256 为 `1f3a5a60a3f182f7f6f171da76d2b2c18f439ae25af186b5577c5d7cc0a7ac6b`；内核构建、timeval ABI 回归 1/1 与原生校准 3/3 通过。 + +### LMbench harness + +- `lat_fs` 补丁区分显式 `-s 0k` 并按 `av[optind]` 选择目录；固定来源校验、构建脚本和真实二进制回归已覆盖。 +- 四个 FS wrapper 同时固定参数目录与 `TMPDIR`；pagefault fixture 使用 runner 自有临时目录并在退出时清理。 +- runner 保留包括 `ENOUGH=1000000` 在内的有效自动校准输出，仅在失败、空值、非法值或零值时有界回退。 +- Linux 入口独立校准、验证、记录并穿透 `ENOUGH`、`TIMING_O`、`LOOP_O` 到 `bwrap --clearenv` 后的环境。 +- collector 在持久写入前完成 schema 校验，要求完整 frame，并核对 summary 总数与状态计数。 +- 两个缩小工作集的元数据、README 与回归测试同步修正。 +- 最终 scoped re-review 判定全部阻断项已解决；保存的 dash/BusyBox focused tests、collector 5 项测试与 scoped diff check 均通过。 + +## Linux 最终验收 + +正式归档位于 `harness/references/lmbench-linux/20260920-timeval-rebased/`，包含 213 个文件。顶层 `manifest.json` SHA256 为 `0219c10d69530fcf2b115d8d54a040450df3806ee1636aab6a9f45e52bbb3674`。 + +| shell | 结果 | 本轮原生校准 | 数据校验 | 清理 | +|---|---|---|---|---| +| dash | 48 ok / 0 failed / 0 missing | `ENOUGH=5000`, `TIMING_O=0`, `LOOP_O=0.00000000` | raw 重抽取、统计、schema、176 项输入哈希全部通过 | fixture、FIFO、进程零残留 | +| BusyBox sh | 48 ok / 0 failed / 0 missing | `ENOUGH=1000000`, `TIMING_O=0`, `LOOP_O=0.00000309` | raw 重抽取、统计、schema、176 项输入哈希全部通过 | fixture、FIFO、进程零残留 | + +两轮分别执行本轮原生校准，是不同 shell 和不同校准参数下的独立运行，不能合并成同一配置的重复样本。BusyBox 的 `ENOUGH=1000000` 只证明为工具成功输出、经入口验证并透传；本轮没有额外证明其精度已经收敛。 + +## DragonOS 最终验收 + +正式归档位于 `harness/references/lmbench-dragonos/20260920-timeval-rebased-coverage/`，包含 45 个文件、53,558 字节。`artifact-sha256.json` SHA256 为 `cec6135535ce0c77b770b8dd90d858327ac02c7636ec13c7799e72e537d77ac0`。 + +- single full attempt：29 ok、2 failed、1 panic/no-result、16 未执行。该运行不完整，不能作为 baseline。 +- 三个独立 VM 合并覆盖：48/48 均 dispatch，39 个 wrapper-valid ok、8 failed、1 panic/no-result。该数字仅表示覆盖，不是单次完整或稳定 baseline。 +- 首次 panic 出现在 `signal_prot_lat` 的 RCU context transition；同一 VM 后续在 `ramfs_create_delete_files_0k_ops` 触发 allocator OOM。执行顺序不能证明 OOM 是独立根因。 +- 五个 virtio 网络用例当时报告 `Network is unreachable`，但没有 DHCP lease、地址或路由就绪证据，不能直接归因为 DragonOS 内核缺陷。 + +## 验证边界 + +- 源码实现与最终修复 scoped review：APPROVED。 +- DragonOS 三 VM coverage 数据复核：通过；归档保持 `baseline_eligible=false`。 +- Linux 两轮数据 gate：通过；每轮各 48 个唯一 case，48 个 `.out` 和 48 个 `.rc` 完整且 rc 全为 0，canonical summary 与 schema 有效。 +- 本轮没有为了收尾重复运行已完成的内核构建、性能测试或 validator；收尾只复核保存的报告、manifest、Git 范围与保护指纹。 +- 旧 patch 中为 plain `patch -p1` 精确匹配保留的 tab/context，以及旧 reference CSV 的 CRLF，属于已评审的既有数据格式，不作为本轮新增错误。 + +## 延后裁决与后续工作 + +- timeval C 回归中 gettimeofday 提供主要根因证据；itimer/select 对旧 ABI 的独立失败判别较弱，后续可增强逐接口回归。 +- collector 已做到校验后写入，但多个输出文件仍不是完全事务写；后续可采用临时文件加 rename，并处理 history 去重。 +- 保留 `lat_fifo-cleanup.patch` 的精确补丁上下文和旧 CSV 的 CRLF；若未来统一格式，必须单独验证补丁可应用性与历史数据语义。 +- 单独定位 `signal_prot_lat` 的 RCU panic，并在干净 VM 中确认 ramfs0k OOM 是否可独立复现。 +- virtio 网络性能项运行前应等待并记录地址、路由和 DHCP 状态；环境未就绪时明确报告前置失败。 +- 获得不受 panic/OOM 污染的一次完整 DragonOS 48 项运行后，才能建立稳定 baseline。 + +## 恢复与证据保留 + +pre-rebase stash `aae56652648a26f575a47b76617e33a0cb006781` 已 apply 并完成恢复完整性核对；提交前用户 dirty 指纹与准备阶段一致。恢复 ref `refs/codex/recovery/feat-lmbench-pre-rebase-20260920` 保留，以提供额外安全恢复点。 + +正式 references 只包含精简、已复核的数据。首次 RCU panic、后续 OOM 以及有界 serial、GDB、controller、run、validation 与 focused test 日志保留在忽略的 `result/lmbench-recovery-20260920/`，不复制到 references。重型 qcow2、readback、构建副本、候选副本和 `/tmp/lmbench-recovery-20260920/` 在确认无进程引用后清理；实际清理范围与本地提交 SHA 记录在 ignored closeout receipt 中。 From 54ba3ebc9a4a134da2410165c182bbcef77718a9 Mon Sep 17 00:00:00 2001 From: yuming Date: Tue, 22 Sep 2026 17:24:49 +0800 Subject: [PATCH 08/22] fix(dadk/ubuntu2404): append lmbench install config for ubuntu2404 rootfs --- user/dadk/config/sets/ubuntu2404/glibc_bin_ubuntu2404.toml | 1 + .../config/sets/ubuntu2404/lmbench_benchmark_tests-1.0.0.toml | 1 + user/dadk/config/sets/ubuntu2404/lmbench_bin_ubuntu2404.toml | 1 + 3 files changed, 3 insertions(+) create mode 120000 user/dadk/config/sets/ubuntu2404/glibc_bin_ubuntu2404.toml create mode 120000 user/dadk/config/sets/ubuntu2404/lmbench_benchmark_tests-1.0.0.toml create mode 120000 user/dadk/config/sets/ubuntu2404/lmbench_bin_ubuntu2404.toml diff --git a/user/dadk/config/sets/ubuntu2404/glibc_bin_ubuntu2404.toml b/user/dadk/config/sets/ubuntu2404/glibc_bin_ubuntu2404.toml new file mode 120000 index 0000000000..044e356b57 --- /dev/null +++ b/user/dadk/config/sets/ubuntu2404/glibc_bin_ubuntu2404.toml @@ -0,0 +1 @@ +../../all/glibc_bin_ubuntu2404.toml \ No newline at end of file diff --git a/user/dadk/config/sets/ubuntu2404/lmbench_benchmark_tests-1.0.0.toml b/user/dadk/config/sets/ubuntu2404/lmbench_benchmark_tests-1.0.0.toml new file mode 120000 index 0000000000..f54c0d1565 --- /dev/null +++ b/user/dadk/config/sets/ubuntu2404/lmbench_benchmark_tests-1.0.0.toml @@ -0,0 +1 @@ +../../all/lmbench_benchmark_tests-1.0.0.toml \ No newline at end of file diff --git a/user/dadk/config/sets/ubuntu2404/lmbench_bin_ubuntu2404.toml b/user/dadk/config/sets/ubuntu2404/lmbench_bin_ubuntu2404.toml new file mode 120000 index 0000000000..d7a37811e3 --- /dev/null +++ b/user/dadk/config/sets/ubuntu2404/lmbench_bin_ubuntu2404.toml @@ -0,0 +1 @@ +../../all/lmbench_bin_ubuntu2404.toml \ No newline at end of file From 5ae35c9f77d850127d003d08dcd7fff8419f7bab Mon Sep 17 00:00:00 2001 From: yuming Date: Wed, 23 Sep 2026 01:09:20 +0800 Subject: [PATCH 09/22] fix(lmbench): restore terminal output newline handling --- .../benchmark/lmbench/orchestrator/monitor_test_results.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh b/user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh index 908bad177b..e715cbc415 100644 --- a/user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh +++ b/user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh @@ -21,6 +21,13 @@ RUN_START_TIMEOUT="${BENCH_RUN_START_TIMEOUT:-600}" # until run begins TOTAL_TIMEOUT="${BENCH_TOTAL_TIMEOUT:-3600}" # whole run (benchmarks are slow) IDLE_TIMEOUT="${BENCH_IDLE_TIMEOUT:-600}" # no serial output +# QEMU's stdio backend leaves the shared terminal in raw mode. Restore only +# output newline handling so host messages start at column zero; retain raw +# input for the guest console. Non-interactive runs have no terminal to adjust. +if [ -t 1 ]; then + stty opost onlcr < /dev/tty 2>/dev/null || true +fi + get_qemu_pid() { [ -f "${VMSTATE_DIR}/pid" ] && cat "${VMSTATE_DIR}/pid" || echo ""; } clean_up() { From b75ad306511947a15a65833f2fbc53c3d0be2e8f Mon Sep 17 00:00:00 2001 From: yuming Date: Wed, 23 Sep 2026 15:50:27 +0800 Subject: [PATCH 10/22] =?UTF-8?q?chore(gvisor):=20=E5=90=AF=E7=94=A8=20ala?= =?UTF-8?q?rm=5Ftest=20=E6=B5=8B=E8=AF=95=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- user/apps/tests/syscall/gvisor/whitelist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/user/apps/tests/syscall/gvisor/whitelist.txt b/user/apps/tests/syscall/gvisor/whitelist.txt index 58e71858b6..6a5bf6ffd0 100644 --- a/user/apps/tests/syscall/gvisor/whitelist.txt +++ b/user/apps/tests/syscall/gvisor/whitelist.txt @@ -153,3 +153,4 @@ uidgid_test partitial_bad_buffer_test sysinfo_test ptrace_test +alarm_test \ No newline at end of file From 50e74208443436f5ab5d498d944f1869c7c77de8 Mon Sep 17 00:00:00 2001 From: yuming Date: Fri, 25 Sep 2026 17:01:14 +0800 Subject: [PATCH 11/22] feat(app-blocklist): enable lmbench app installation --- config/app-blocklist.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/app-blocklist.toml b/config/app-blocklist.toml index 21c1bddf38..1580ee0f01 100644 --- a/config/app-blocklist.toml +++ b/config/app-blocklist.toml @@ -64,7 +64,7 @@ log_skipped = true # 屏蔽gvisor系统调用测试 blocked_apps = [ { name = "gvisor syscall tests", reason = "由于文件较大，因此屏蔽。如果要允许系统调用测试，则把这行取消注释即可" }, - # { name = "lmbench benchmark tests", reason = "性能基准套件，默认屏蔽以免拖慢常规构建；make test-benchmark 会临时启用此行（注释即可启用）" }, + { name = "lmbench benchmark tests", reason = "性能基准套件，默认屏蔽以免拖慢常规构建；make test-benchmark 会临时启用此行（注释即可启用）" }, { name = "test_ebpf_new", reason = "2025.11.17,aya上游发版有问题，导致ci过不了，暂时禁用" }, { name = "test_ebpf_tp", reason = "2025.11.17,aya上游发版有问题，导致ci过不了，暂时禁用" }, { name = "runcell", reason = "依赖项依赖github，可能网络问题导致编不过" }, From afef59afe5ba321711609fb8f11b1ace67474cc3 Mon Sep 17 00:00:00 2001 From: yuming Date: Fri, 25 Sep 2026 17:05:54 +0800 Subject: [PATCH 12/22] fix(init): include system administration directories in path Add /usr/sbin and /sbin to the PATH exported by rcS so startup tasks and benchmark preparation can resolve tools such as mkfs.ext4. --- user/sysconfig/etc/init.d/rcS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user/sysconfig/etc/init.d/rcS b/user/sysconfig/etc/init.d/rcS index 3703417a1c..f714d7c0e1 100755 --- a/user/sysconfig/etc/init.d/rcS +++ b/user/sysconfig/etc/init.d/rcS @@ -1,6 +1,6 @@ #!/bin/sh -export PATH=/bin:/usr/bin:/usr/local/bin +export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin echo "[rcS] Running system init script..." /bin/about.elf From 8db728cb8cba8ab4a0791d63732d31ded91eb68d Mon Sep 17 00:00:00 2001 From: yuming Date: Sun, 27 Sep 2026 12:54:38 +0800 Subject: [PATCH 13/22] docs(harness): authorize atomic commits in agent workflow --- AGENTS.md | 3 ++- FORCE_CONSTRAIN.md | 2 +- harness/constraints/WORKFLOW.md | 24 ++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 harness/constraints/WORKFLOW.md diff --git a/AGENTS.md b/AGENTS.md index b6034e236f..d4ec54fe36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,8 @@ FORCE_CONSTRAIN.md ← 交互、兼容性、安全与开发 KEY_INFO_REMINDER.md ← 构建、Nix 环境与 QEMU 关键命令 harness/ ← 深层 Harness 文档 ├── constraints/ ← 局部操作约束入口 -│ └── COMMIT.md ← 提交上下文与权威规范入口 +│ ├── COMMIT.md ← 提交上下文与权威规范入口 +│ └── WORKFLOW.md ← 开发工作流、原子化提交时机与授权边界 ├── product-specs/ ← 产品需求文档 │ └── index.md ← 问题、价值、范围与需求索引 ├── design-docs/ ← 设计意图文档 diff --git a/FORCE_CONSTRAIN.md b/FORCE_CONSTRAIN.md index 50cff84015..42a3283f90 100644 --- a/FORCE_CONSTRAIN.md +++ b/FORCE_CONSTRAIN.md @@ -28,4 +28,4 @@ 需求的权威入口是 [harness/product-specs/index.md](harness/product-specs/index.md)，设计意图和权衡的权威入口是 [harness/design-docs/index.md](harness/design-docs/index.md)，执行步骤位于 `harness/exec-plans/`。新增内容应进入对应职责的文档，避免在多个位置形成竞争定义。 -提交消息规则不在本文件重复定义。需要提交相关上下文时，先读 [harness/constraints/COMMIT.md](harness/constraints/COMMIT.md)，再进入其中链接的权威贡献规范。 +开始任何会修改仓库的任务前，必须阅读 [harness/constraints/WORKFLOW.md](harness/constraints/WORKFLOW.md)，并按其中的原子化提交条件在工作单元完成时直接本地提交，无需再次请求确认，避免变更积压。提交边界和例外以该工作流文档为准。提交消息格式由 [harness/constraints/COMMIT.md](harness/constraints/COMMIT.md) 路由至权威贡献规范。 diff --git a/harness/constraints/WORKFLOW.md b/harness/constraints/WORKFLOW.md new file mode 100644 index 0000000000..8e56b98e4d --- /dev/null +++ b/harness/constraints/WORKFLOW.md @@ -0,0 +1,24 @@ +# 开发工作流 + +## 软件生命周期 + +### 原子化提交与持续授权 + +用户已授权 Agent：在已获授权的任务范围内，每当自己的工作变更满足以下原子化条件，就必须直接创建本地提交，无需询问“是否提交”，也不得等到整个大任务结束才集中提交。 + +- 提交只解决一个明确问题或完成一个独立工作单元，能够单独说明目的、审查和回退。 +- 该单元所需的实现、相关测试及文档已齐备，不依赖尚未提交的变更才能成立。 +- 已完成与变更风险和类型相称的验证，所需检查通过；不得把未运行或失败的检查宣称为通过。纯文档变更检查内容、引用和 diff 即可。 +- 已审查最终暂存区 diff，确认只包含本工作单元的完整变更，没有无关改动、调试残留或敏感信息。 + +执行时，先查看工作区和暂存区状态，识别任务开始前已有的变更；只按明确路径或代码块暂存本单元内容。不得把用户或其他 Agent 的未完成工作顺手纳入提交，也不得为提交而覆盖、丢弃或重置这些变更。同一文件存在混合改动时，只提交能够可靠分离的部分；不能安全分离或归属不明时，保留现场并说明具体阻塞，必要时仅询问该边界问题。 + +尚不满足条件时继续完成该单元；确有阻塞时报告未提交原因，不以积压变更或创建不完整提交代替处理。用户明确要求暂不提交、只读分析或另有提交安排时，遵从该任务的明确要求。 + +本授权仅覆盖本地新建提交，不自动授权 push、合并或重写已有历史。每次交付应报告提交哈希、提交范围、验证结果、剩余工作区变更和是否已推送。 + +提交消息的规范入口见 [COMMIT.md](COMMIT.md)。 + +## 测试约束 + +提交前按上述原子化条件完成与变更相称的验证；开发和修复方法遵循 [DragonOS 强约束](../../FORCE_CONSTRAIN.md)。 From d231341242e786550a7806a0d3dae00c480092ee Mon Sep 17 00:00:00 2001 From: yuming Date: Sun, 27 Sep 2026 12:55:29 +0800 Subject: [PATCH 14/22] docs(harness): simplify workflow map description --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index d4ec54fe36..613178286f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ KEY_INFO_REMINDER.md ← 构建、Nix 环境与 QEMU 关键命 harness/ ← 深层 Harness 文档 ├── constraints/ ← 局部操作约束入口 │ ├── COMMIT.md ← 提交上下文与权威规范入口 -│ └── WORKFLOW.md ← 开发工作流、原子化提交时机与授权边界 +│ └── WORKFLOW.md ← 开发工作流 ├── product-specs/ ← 产品需求文档 │ └── index.md ← 问题、价值、范围与需求索引 ├── design-docs/ ← 设计意图文档 From b0a1758b863b92e45231ee7dd2b25edeea4fbb13 Mon Sep 17 00:00:00 2001 From: yuming Date: Wed, 30 Sep 2026 14:06:34 +0800 Subject: [PATCH 15/22] test(lmbench): exclude observed guest runtime blockers Comment out eight cases that panicked or returned invalid measurements during sequential DragonOS runs. Record the observed failure and guest configuration beside each exclusion. Validated the working-tree whitelist: 39/39 successful, no failures or skips, with 512 MiB, 2 vCPUs, one sample and a 120-second timeout. The pre-existing fifo_lat exclusion and sample-count override remain unstaged. --- .../tests/benchmark/lmbench/whitelist.txt | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/user/apps/tests/benchmark/lmbench/whitelist.txt b/user/apps/tests/benchmark/lmbench/whitelist.txt index 2896466a96..4e8b368abd 100644 --- a/user/apps/tests/benchmark/lmbench/whitelist.txt +++ b/user/apps/tests/benchmark/lmbench/whitelist.txt @@ -2,22 +2,27 @@ # 每行一个测例名(对应 runner/test_cases/.sh,不带路径/后缀) # 以 # 开头为注释,空行忽略。runner 只跑此列表中的测例。 # -# 全量启用:48 个测例覆盖各类别。.meta 在 host 侧用 guest 同款 lmbench 二进制 +# 完整列表:48 个测例覆盖各类别，实际运行未注释的条目。 +# Blocked 注释来自 2026-09-30 的顺序运行（512 MiB、2 vCPU、1 次采样、120s 超时）。 +# .meta 在 host 侧用 guest 同款 lmbench 二进制 # 提取(见 docs/superpowers/specs/2026-07-30-lmbench-fullsuite-meta-extraction-design.md); # 部分工具在 host 跑不通(lat_fifo/bw_unix/lat_sem/lat_syscall null/bw_tcp 等), # 其 .meta 基于 strings 提取的输出格式 + lmbench 标准格式推断,标注待 guest 校准。 # memory mem_copy_bw -mem_read_bw -mem_write_bw +# Blocked: 512 MiB guest, round 01: global_alloc_error in PageReclaimer::shrink_list; kernel halted. +# mem_read_bw +# Blocked: 512 MiB guest, round 02: global_alloc_error in PageReclaimer::shrink_list; kernel panicked. +# mem_write_bw mem_mmap_bw mem_mmap_lat mem_pagefault_lat # process process_fork_lat -process_exec_lat +# Blocked: round 06: lat_proc exec returned 1 without a valid numeric result (previous rounds passed). +# process_exec_lat process_ctx_lat process_getppid_lat process_shell_lat @@ -42,15 +47,20 @@ vfs_read_lat vfs_write_lat vfs_fstat_lat vfs_stat_lat -vfs_fcntl_lat +# Blocked: round 04: lat_fcntl returned 1, lock of fd1 failed: Resource deadlock avoided. +# vfs_fcntl_lat vfs_select_lat vfs_read_pagecache_bw # filesystem -ext4_create_delete_files_0k_ops -ext4_create_delete_files_10k_ops -ramfs_create_delete_files_0k_ops -ramfs_create_delete_files_10k_ops +# Blocked: 512 MiB guest, round 03: global_alloc_error during lat_fs; kernel halted. +# ext4_create_delete_files_0k_ops +# Blocked: 512 MiB guest, round 04: global_alloc_error during lat_fs; kernel halted. +# ext4_create_delete_files_10k_ops +# Blocked: 512 MiB guest, round 06: global_alloc_error during lat_fs; kernel halted. +# ramfs_create_delete_files_0k_ops +# Blocked: 512 MiB guest, round 07: global_alloc_error during lat_fs; kernel halted. +# ramfs_create_delete_files_10k_ops ext4_copy_files_bw ramfs_copy_files_bw @@ -67,4 +77,4 @@ tcp_virtio_bw_128 tcp_virtio_bw_64k tcp_virtio_connect_lat udp_loopback_lat -udp_virtio_lat \ No newline at end of file +udp_virtio_lat From 68b5c0ff901254f494535d6b203fb7725d9a1da6 Mon Sep 17 00:00:00 2001 From: yuming Date: Wed, 30 Sep 2026 15:41:57 +0800 Subject: [PATCH 16/22] =?UTF-8?q?chore(lmbench/whitelist):=20=E5=B1=8F?= =?UTF-8?q?=E8=94=BD=20block=20=E7=9A=84=E6=B5=8B=E8=AF=95=E7=94=A8?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- user/apps/tests/benchmark/lmbench/whitelist.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user/apps/tests/benchmark/lmbench/whitelist.txt b/user/apps/tests/benchmark/lmbench/whitelist.txt index 4e8b368abd..06d0ecc873 100644 --- a/user/apps/tests/benchmark/lmbench/whitelist.txt +++ b/user/apps/tests/benchmark/lmbench/whitelist.txt @@ -30,7 +30,7 @@ process_shell_lat # ipc pipe_lat pipe_bw -fifo_lat +# fifo_lat unix_lat unix_bw unix_connect_lat From 1183640223384abadce0e5e05c0861cc727fd498 Mon Sep 17 00:00:00 2001 From: yuming Date: Wed, 30 Sep 2026 15:43:31 +0800 Subject: [PATCH 17/22] =?UTF-8?q?chore(config/app-blocklist):=20=E5=B0=86?= =?UTF-8?q?=20lmbench=20app=20=E7=A7=BB=E9=99=A4=20DADK=20=E5=85=A8?= =?UTF-8?q?=E5=B1=80=E5=AE=89=E8=A3=85=E9=BB=91=E5=90=8D=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/app-blocklist.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/app-blocklist.toml b/config/app-blocklist.toml index 1580ee0f01..21c1bddf38 100644 --- a/config/app-blocklist.toml +++ b/config/app-blocklist.toml @@ -64,7 +64,7 @@ log_skipped = true # 屏蔽gvisor系统调用测试 blocked_apps = [ { name = "gvisor syscall tests", reason = "由于文件较大，因此屏蔽。如果要允许系统调用测试，则把这行取消注释即可" }, - { name = "lmbench benchmark tests", reason = "性能基准套件，默认屏蔽以免拖慢常规构建；make test-benchmark 会临时启用此行（注释即可启用）" }, + # { name = "lmbench benchmark tests", reason = "性能基准套件，默认屏蔽以免拖慢常规构建；make test-benchmark 会临时启用此行（注释即可启用）" }, { name = "test_ebpf_new", reason = "2025.11.17,aya上游发版有问题，导致ci过不了，暂时禁用" }, { name = "test_ebpf_tp", reason = "2025.11.17,aya上游发版有问题，导致ci过不了，暂时禁用" }, { name = "runcell", reason = "依赖项依赖github，可能网络问题导致编不过" }, From 186e92e11bb3379344a74f4c36918d17b8f57171 Mon Sep 17 00:00:00 2001 From: yuming Date: Wed, 30 Sep 2026 15:44:15 +0800 Subject: [PATCH 18/22] =?UTF-8?q?chore(/):=20=E6=B7=BB=E5=8A=A0=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E7=9B=AE=E5=BD=95=E8=BD=AF=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests | 1 + 1 file changed, 1 insertion(+) create mode 120000 tests diff --git a/tests b/tests new file mode 120000 index 0000000000..0c272b6153 --- /dev/null +++ b/tests @@ -0,0 +1 @@ +user/apps/tests \ No newline at end of file From 3c154a1e3c36f80ebfeb1dae67869ac6830057d2 Mon Sep 17 00:00:00 2001 From: yuming Date: Wed, 30 Sep 2026 17:34:11 +0800 Subject: [PATCH 19/22] fix(qemu): align nix guest memory with 2g default --- tools/qemu/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/qemu/default.nix b/tools/qemu/default.nix index 390aa2e82a..8da4a75c03 100644 --- a/tools/qemu/default.nix +++ b/tools/qemu/default.nix @@ -18,7 +18,7 @@ let baseConfig = { nographic = true; - memory = "512M"; + memory = "2G"; cores = "2"; shmId = "dragonos-qemu-shm.ram"; }; From 8b8a0f9625dd7cccc743f12fe531f0034a0df1b0 Mon Sep 17 00:00:00 2001 From: yuming Date: Wed, 30 Sep 2026 17:38:24 +0800 Subject: [PATCH 20/22] fix(lmbench): use temporary disk writes for benchmark runs --- Makefile | 2 +- tools/qemu/default.nix | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index ef1afced6f..bbcd8e3873 100644 --- a/Makefile +++ b/Makefile @@ -325,7 +325,7 @@ test-benchmark: prepare_rootfs_manifest $(DADK) -f $(ROOT_PATH)/dadk-manifest.generated.toml user clean --level in-src -w $(ROOT_PATH); \ fi; \ SKIP_GRUB=1 $(MAKE) write_diskimage; \ - $(MAKE) qemu-nographic AUTO_TEST=benchmark BENCHMARK_TEST_DIR=/opt/tests/benchmark/lmbench & \ + $(MAKE) qemu-nographic DRAGONOS_QEMU_SNAPSHOT=1 AUTO_TEST=benchmark BENCHMARK_TEST_DIR=/opt/tests/benchmark/lmbench & \ sleep 5; \ status=0; \ bash user/apps/tests/benchmark/lmbench/orchestrator/monitor_test_results.sh || status=$$?; \ diff --git a/tools/qemu/default.nix b/tools/qemu/default.nix index 8da4a75c03..99c80099da 100644 --- a/tools/qemu/default.nix +++ b/tools/qemu/default.nix @@ -123,7 +123,7 @@ let kernelPath = if arch == "x86_64" then kernel else "${riscv-uboot}/u-boot.bin"; - diskArgs = + diskDeviceArgs = if arch == "x86_64" then [ "-device" @@ -132,15 +132,11 @@ let "pci-bridge,chassis_nr=1,id=pci.1" "-device" "pcie-root-port" - "-drive" - "id=disk,file=${diskPath},if=none" ] else [ "-device" "virtio-blk-device,drive=disk" - "-drive" - "id=disk,file=${diskPath},if=none" ]; # Generate bash code for dynamic parts @@ -172,6 +168,12 @@ let if [ ! -d "bin" ]; then echo "Error: Please run from project root (bin/ missing)."; exit 1; fi + # Match tools/run-qemu.sh: guest writes are temporary only when enabled. + case "''${DRAGONOS_QEMU_SNAPSHOT:-0}" in + 0|1) ;; + *) echo "Error: DRAGONOS_QEMU_SNAPSHOT must be 0 or 1"; exit 1 ;; + esac + if [ "${preferSystemQemuStr}" = "true" ]; then if ! command -v "${qemuBin}" >/dev/null 2>&1; then echo "Error: 系统未找到 ${qemuBin}，请安装 QEMU 或改用 nix 提供的 start 目标。" @@ -375,7 +377,11 @@ let BOOT_ARGS=( "-kernel" "${kernelPath}" "-append" "$FINAL_CMDLINE" ) - DISK_ARGS=( ${lib.escapeShellArgs diskArgs} ) + QEMU_DRIVE=${lib.escapeShellArg "id=disk,file=${diskPath},if=none"} + if [ "''${DRAGONOS_QEMU_SNAPSHOT:-0}" = "1" ]; then + QEMU_DRIVE+=",snapshot=on" + fi + DISK_ARGS=( ${lib.escapeShellArgs diskDeviceArgs} "-drive" "$QEMU_DRIVE" ) # 动态网络配置（使用动态分配的端口） NET_ARGS=( "-netdev" "user,id=hostnet0,hostfwd=tcp::$HOST_PORT-:12580" "-device" "virtio-net-pci,vectors=5,netdev=hostnet0,id=net0" ) From 6993b597a26107569dc6455beac2f395264c8b4c Mon Sep 17 00:00:00 2001 From: yuming Date: Fri, 2 Oct 2026 00:05:20 +0800 Subject: [PATCH 21/22] docs(lmbench): move harness plans into benchmark directory --- AGENTS.md | 1 + .../completed/2026-09-19-lmbench-wrapper-results.md | 6 ++++-- .../completed/2026-09-20-lmbench-environment-recovery.md | 2 ++ .../exec-plans/completed/2026-09-20-lmbench-recovery.md | 2 ++ 4 files changed, 9 insertions(+), 2 deletions(-) rename {harness => user/apps/tests/benchmark/lmbench/harness}/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md (78%) rename {harness => user/apps/tests/benchmark/lmbench/harness}/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md (97%) rename {harness => user/apps/tests/benchmark/lmbench/harness}/exec-plans/completed/2026-09-20-lmbench-recovery.md (98%) diff --git a/AGENTS.md b/AGENTS.md index 613178286f..f67e0fa000 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,7 @@ harness/ ← 深层 Harness 文档 ├── references/ ← 可复用参考资料 │ └── index.md ← 参考资料索引 └── generated/ ← 生成型文档预留目录 +tests/benchmark/lmbench/harness/ ← LMbench 专属 Harness 文档 ``` 以上路径均相对仓库根目录，按任务读取对应权威文档。 diff --git a/harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md b/user/apps/tests/benchmark/lmbench/harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md similarity index 78% rename from harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md rename to user/apps/tests/benchmark/lmbench/harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md index c4f23899d6..7f104eed8f 100644 --- a/harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md +++ b/user/apps/tests/benchmark/lmbench/harness/exec-plans/completed/2026-09-19-lmbench-wrapper-results.md @@ -1,5 +1,7 @@ # LMbench wrapper 结果判定修复 +下文测试归档路径均为历史记录，相对仓库根目录；当前工作树未保留这些归档。 + 所有 48 个 wrapper 与 runner 共享结果校验：命令成功退出、提取到有限正数且没有断言或致命错误，才记为成功。测量输出有大小上限，实际非零退出码保留；测量进程组和临时 FIFO 在退出时清理。内核计时与协议问题不在此次修复范围。 ## 验证 @@ -8,7 +10,7 @@ - 63 个脚本在两种 shell 下通过语法检查，安装后的布局与 help 验证通过。 - 最后一次错误分类修正后，两种 shell 的 runner/wrapper 覆盖回归通过。 - 只读检查历史 48 项输出，分类符合预期（13 项有效）；该检查不代表本轮实际运行。 -- 2026-09-19 在单台 DragonOS VM 中调用完整 suite 一次，29 项返回结果（11 成功、18 失败）。第 30 项 ext4_create_delete_files_0k_ops 超过 600 秒客体时限后仍未返回，690 秒宿主保护终止运行，后续 18 项未执行。该次运行未完成全部 48 项，随后对白名单剩余项连续补跑。结果见 [本轮数据](../../references/lmbench-dragonos/20260919T144815Z-single-suite/README.md)。 +- 2026-09-19 在单台 DragonOS VM 中调用完整 suite 一次，29 项返回结果（11 成功、18 失败）。第 30 项 ext4_create_delete_files_0k_ops 超过 600 秒客体时限后仍未返回，690 秒宿主保护终止运行，后续 18 项未执行。该次运行未完成全部 48 项，随后对白名单剩余项连续补跑。本轮数据的历史路径为 `harness/references/lmbench-dragonos/20260919T144815Z-single-suite/README.md`。 - 客体没有执行最终 sync，磁盘导出的部分原始输出和退出码缺失；收到的 29 项 runner 结果完整保留，不用缺失数据推断成功或失败。 ## 结果存储 @@ -21,6 +23,6 @@ 补跑 18 项全部尝试：3 项成功、13 项失败、2 项挂起。原全量调用与补跑共 4 次启动，合计 14 项成功、31 项失败、3 项挂起，未执行项为 0。各次启动均运行当时剩余的完整白名单，仅在 VM 无法继续时重新启动剩余项；没有把这些结果描述成单次启动跑完。 -[完整覆盖表](../../references/lmbench-dragonos/20260919-remaining-18/coverage.csv) 保留逐项运行来源。最后一批完成 guest sync，TCP connect 与两项 UDP 的原始退出码为 124、1、1，与失败结果一致；TCP 64K 的原始退出码仍未导出。前面被宿主终止的运行也存在原始数据未落盘缺口，均已明确标注，不能仅凭 sync 标记认定数据完整。计时断言、ENOSYS、无效输出和超时仍是现存问题，本轮不构成稳定性能 baseline。 +完整覆盖表的历史路径为 `harness/references/lmbench-dragonos/20260919-remaining-18/coverage.csv`，其中记录了逐项运行来源。最后一批完成 guest sync，TCP connect 与两项 UDP 的原始退出码为 124、1、1，与失败结果一致；TCP 64K 的原始退出码仍未导出。前面被宿主终止的运行也存在原始数据未落盘缺口，均已明确标注，不能仅凭 sync 标记认定数据完整。计时断言、ENOSYS、无效输出和超时仍是现存问题，本轮不构成稳定性能 baseline。 实际安装的 106 个文件与当前工作区逐个比对一致。已有主机回归、语法检查、安装布局验证通过；本轮补齐全 48 项实际尝试，不表示内核问题全部修复。按用户授权提交 wrapper 修复和参考数据精简，不推送。 diff --git a/harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md b/user/apps/tests/benchmark/lmbench/harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md similarity index 97% rename from harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md rename to user/apps/tests/benchmark/lmbench/harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md index 70b4152fad..984b11cf48 100644 --- a/harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md +++ b/user/apps/tests/benchmark/lmbench/harness/exec-plans/completed/2026-09-20-lmbench-environment-recovery.md @@ -1,5 +1,7 @@ # LMbench 剩余用例恢复 +下文测试归档路径均为历史记录，相对仓库根目录；当前工作树未保留这些归档。 + ## 目标与约束 在当前已提交 c453800b 基础上，不新增 DragonOS 本体改动，尽可能提高一次完整48项运行中的有效成功数量。此前用户授权的timeval ABI修复保持，不回退；本阶段只处理harness、基准程序和隔离测试环境中有证据的根因。 diff --git a/harness/exec-plans/completed/2026-09-20-lmbench-recovery.md b/user/apps/tests/benchmark/lmbench/harness/exec-plans/completed/2026-09-20-lmbench-recovery.md similarity index 98% rename from harness/exec-plans/completed/2026-09-20-lmbench-recovery.md rename to user/apps/tests/benchmark/lmbench/harness/exec-plans/completed/2026-09-20-lmbench-recovery.md index bdde6b6d38..77ff08ddd4 100644 --- a/harness/exec-plans/completed/2026-09-20-lmbench-recovery.md +++ b/user/apps/tests/benchmark/lmbench/harness/exec-plans/completed/2026-09-20-lmbench-recovery.md @@ -1,5 +1,7 @@ # LMbench 有效结果恢复 +下文测试归档路径均为历史记录，相对仓库根目录；当前工作树未保留这些归档。 + ## 完成结论 计时 ABI 修复、本轮覆盖与验收完成，但 DragonOS 稳定 baseline 尚未成立。 From 208e59e3c024804df1ca6e34310d7eef6506a1df Mon Sep 17 00:00:00 2001 From: yuming Date: Tue, 6 Oct 2026 21:48:16 +0800 Subject: [PATCH 22/22] fix(lmbench): install and enable fifo cleanup in the suite runner --- user/apps/tests/benchmark/lmbench/Makefile | 2 +- user/apps/tests/benchmark/lmbench/README.md | 6 ++-- .../benchmark/lmbench/runner/fifo_cleanup.sh | 6 ++-- .../tests/benchmark/lmbench/runner/run.sh | 2 ++ .../lmbench/tests/test_fifo_install.sh | 29 +++++++++++++++++++ 5 files changed, 39 insertions(+), 6 deletions(-) create mode 100755 user/apps/tests/benchmark/lmbench/tests/test_fifo_install.sh diff --git a/user/apps/tests/benchmark/lmbench/Makefile b/user/apps/tests/benchmark/lmbench/Makefile index 25ef27fcb0..51d0bede6f 100644 --- a/user/apps/tests/benchmark/lmbench/Makefile +++ b/user/apps/tests/benchmark/lmbench/Makefile @@ -11,7 +11,7 @@ endif # Files needed inside the guest at runtime. Sources live under runner/; # `install` flattens them into INSTALL_DIR (guest runtime layout is flat). RUNNER_DIR = runner -GUEST_SCRIPTS = $(RUNNER_DIR)/run.sh $(RUNNER_DIR)/init.sh $(RUNNER_DIR)/env.sh $(RUNNER_DIR)/clean_up.sh $(RUNNER_DIR)/result.sh +GUEST_SCRIPTS = $(RUNNER_DIR)/run.sh $(RUNNER_DIR)/init.sh $(RUNNER_DIR)/env.sh $(RUNNER_DIR)/clean_up.sh $(RUNNER_DIR)/result.sh $(RUNNER_DIR)/fifo_cleanup.sh GUEST_DATA = whitelist.txt config .PHONY: all build install clean help diff --git a/user/apps/tests/benchmark/lmbench/README.md b/user/apps/tests/benchmark/lmbench/README.md index 7e071d23a1..4f7c7f3d92 100644 --- a/user/apps/tests/benchmark/lmbench/README.md +++ b/user/apps/tests/benchmark/lmbench/README.md @@ -73,12 +73,14 @@ GNU `timeout` 默认使用 `/usr/bin/timeout`，可用 `LMBENCH_TIMEOUT` 指定 所有用例都使用 `LMBENCH_BIN_DIR` 下同一套预编译二进制，运行时不下载源码、打补丁或编译。 DADK 与 Nix 的包来源保持一致，输入哈希随结果保存。 -Linux 入口为原版 `lat_fifo` 启用 `runner/fifo_cleanup.sh`：收到 writer 的 EOF 诊断后， +套件 `runner/run.sh` 为原版 `lat_fifo` 显式启用 `runner/fifo_cleanup.sh`：收到 writer 的 EOF 诊断后， 通过 `/proc` 确认它属于本次 benchmark 的子进程，且其两条 FIFO 都已被 unlink，才终止 无法响应 SIGTERM 的 writer。该阶段测量数据已交回 controller；驱动不介入计时循环， 由原 controller 正常输出结果并退出。已确认清理后的重复 EOF 诊断会合并计数；其他输出保留。 仍要求真实退出码为 0 且 latency 有效，超时、测量中失败不会算成功。 -该辅助仅由 Linux 入口启用，依赖 Linux `/proc` 和 `readlink`。 +该辅助依赖 Linux 兼容的 `/proc//task//children` 及 FD `readlink` 语义； +Make 与 Nix 安装均携带该脚本。直接调用 wrapper 时可设置 `LMBENCH_FIFO_CLEANUP=0` +保留无辅助清理的故障对照；套件 runner 始终启用辅助清理。 原版 `lat_fs` 的零大小用例使用默认大小扫描，并由 metadata 只抽取 `0k` 行； 10k 用例使用 `-s 10k`。四个 wrapper 均设置 `TMPDIR`，确保测量落在指定文件系统。 diff --git a/user/apps/tests/benchmark/lmbench/runner/fifo_cleanup.sh b/user/apps/tests/benchmark/lmbench/runner/fifo_cleanup.sh index 0c8772bbcc..8351b34013 100644 --- a/user/apps/tests/benchmark/lmbench/runner/fifo_cleanup.sh +++ b/user/apps/tests/benchmark/lmbench/runner/fifo_cleanup.sh @@ -1,12 +1,12 @@ #!/bin/sh -# Linux driver for the unmodified packaged lat_fifo (-P 1). +# Driver for the unmodified packaged lat_fifo (-P 1). # Its writer inherits a SIGTERM handler and loops on EOF during cleanup. # Only reap that writer AFTER both FIFO paths are unlinked. The benchmark # has already sent its measured samples to its controller at that point. # The caller supervises this entire process group (runner/result.sh). set -eu -# Private Linux tmpfs, not persistent raw results: namespace teardown also -# reclaims this pipe if the supervisor must escalate to SIGKILL. +# This invocation owns its output pipe. Normal exits remove it below; the +# Linux entry additionally contains it in a private tmpfs namespace. work=$(mktemp -d "${LMBENCH_TMP_DIR:-/tmp}/fifo-cleanup.XXXXXXXX") trap 'rm -rf "$work"' 0 trap 'exit 129' HUP diff --git a/user/apps/tests/benchmark/lmbench/runner/run.sh b/user/apps/tests/benchmark/lmbench/runner/run.sh index 7b4d01fc3e..1010caafda 100644 --- a/user/apps/tests/benchmark/lmbench/runner/run.sh +++ b/user/apps/tests/benchmark/lmbench/runner/run.sh @@ -316,6 +316,8 @@ run_main() { . "$SCRIPT_DIR/env.sh" export LMBENCH_RUN_TMP="$WORK_TMP" + # Keep the packaged lat_fifo unchanged; its writer needs supervised cleanup. + export LMBENCH_FIFO_CLEANUP=1 command -v "$LMBENCH_TIMEOUT" >/dev/null 2>&1 || { log "ERROR: timeout command not found: $LMBENCH_TIMEOUT"; return 1; } diff --git a/user/apps/tests/benchmark/lmbench/tests/test_fifo_install.sh b/user/apps/tests/benchmark/lmbench/tests/test_fifo_install.sh new file mode 100755 index 0000000000..b74c04f9f5 --- /dev/null +++ b/user/apps/tests/benchmark/lmbench/tests/test_fifo_install.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# The installed suite must carry its cleanup driver and export its selection. +set -eu +SUITE=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +TEST_SHELL=${TEST_SHELL:-/bin/sh} +work=$(mktemp -d) +trap 'rm -rf "$work"' 0 +make -s -C "$SUITE" install DADK_CURRENT_BUILD_DIR="$work/install" +[ -x "$work/install/fifo_cleanup.sh" ] || { + echo 'FAIL: installed suite has no FIFO cleanup driver' >&2 + exit 1 +} +# Run the installed runner without filesystem fixture preparation. A tiny +# testcase observes what a dispatched child actually inherits, even when the +# caller disabled cleanup. Direct wrapper invocations remain useful controls. +rm "$work/install/init.sh" "$work/install/clean_up.sh" +cat > "$work/install/test_cases/cleanup_selection.sh" <<'CASE' +#!/bin/sh +[ "${LMBENCH_FIFO_CLEANUP:-0}" = 1 ] || exit 17 +printf 'value 1\n' +CASE +printf 'SEARCH_PATTERN=^value\nRESULT_INDEX=2\n' > "$work/install/test_cases/cleanup_selection.meta" +mkdir "$work/tmp" +LMBENCH_FIFO_CLEANUP=0 ENOUGH=50000 LMBENCH_SH="$TEST_SHELL" \ + LMBENCH_TMP_DIR="$work/tmp" LMBENCH_RUN_TMP="$work/raw" \ + "$TEST_SHELL" "$work/install/run.sh" --only cleanup_selection --samples 1 --timeout 2 \ + > "$work/runner.log" 2>&1 || { cat "$work/runner.log"; exit 1; } +grep -q '"status":"ok"' "$work/runner.log" +echo "FIFO installed runner regression: PASS ($TEST_SHELL)"