Usability improvements

.gitignore

@@ -45,6 +45,7 @@ examples/aws/credentials.json
 # Project-specific config (generated by setup configure)
 run-config.json
 demo-config.json
+pgbench-config.json
 
 # Test kernel RPMs (large binary files)
 setup/test-kernel-rpms/

README.md

@@ -167,6 +167,13 @@ Using an explicit configuration file (recommended)
 kernel-ci-cloud-runner aws run --config my-config.json
 ```
 
+To also download this run's result files (benchmark CSVs, `result.txt`, console
+logs) from S3 to a local directory, pass `--results-dir`; files land under
+`DIR/<run_prefix>/` mirroring the bucket layout:
+```
+kernel-ci-cloud-runner aws run --config my-config.json --results-dir ./results
+```
+
 - Check status by pipeline log message: "VMs: X/X spawned, Y successful, 0 failed, 0 missing"
 - **Logs:** `logs/`
 
@@ -568,7 +575,7 @@ Test: unixbench-kernel-regression
       [t-test p=0.0000, U-test p=0.0001, Cohen's d=8.13]
 
 ------------------------------------------------------------
-Tests with benchmarks: 1 | Regressions found: 1
+Tests with benchmarks: 1 | Regressions found: 1 | Improvements found: 0
 Tests with regressions: unixbench-kernel-regression
 ============================================================
 ```
@@ -584,17 +591,20 @@ The `BenchmarkAnalyzer` returns a `PipelineBenchmarkSummary` dataclass with stru
 The `PipelineBenchmarkSummary` contains:
 - `test_results` — list of `TestBenchmarkResult`, one per test
 - `tests_with_regression` / `regression_test_names` — quick summary of which tests regressed
+- `tests_with_improvement` / `improvement_test_names` — quick summary of which tests improved
 
 Each `TestBenchmarkResult` contains:
 - `base_kernel` / `tip_kernel` — kernel version strings
 - `comparisons` — list of `MetricComparison` (one per benchmark metric)
 - `regressions` — property that filters to only regressed metrics
+- `improvements` — property that filters to only improved metrics
 
 Each `MetricComparison` contains:
 - `metric`, `unit`, `more_is_better` — metric identity
 - `base` / `tip` — `MetricStats` with `mean`, `median`, `stddev`, `cv`, `values`
 - `pct_change`, `t_pvalue`, `u_pvalue`, `cohens_d` — statistical results
 - `is_regression` — boolean flag
+- `is_improvement` — boolean flag (significant + meaningful change in the better direction)
 
 Example integration at the `NOTIFICATION HOOK` in `pipeline.py`:
 

src/kernel_ci_cloud_labs/auth/aws_auth.py

@@ -285,6 +285,15 @@ def _make_client(service):
                 first_role_arn = next(iter(role_arns.values()), None)
                 task_config["execution_role_arn"] = first_role_arn
                 task_config["task_role_arn"] = first_role_arn
+
+                # Derive a per-run awslogs stream prefix so concurrent runs are
+                # distinguishable in the shared /ecs/<family> log group. Prefer
+                # an explicit config value, else the run's test_id, else "ecs".
+                if "log_stream_prefix" not in task_config:
+                    test_id = (self.config.get("test_config") or {}).get("test_id")
+                    if test_id:
+                        task_config["log_stream_prefix"] = test_id
+
                 logger.debug("Task definition family: %s", task_config.get("family"))
                 logger.debug("Execution role ARN: %s", task_config.get("execution_role_arn"))
 

src/kernel_ci_cloud_labs/auth/aws_task_definition_manager.py

@@ -39,12 +39,19 @@ def create(self, resource_name: str, resource_config: Dict[str, Any]) -> str:
 
         # Add CloudWatch logs configuration
         log_group = f"/ecs/{resource_name}"
+        # The awslogs stream name is "<prefix>/<container-name>/<task-id>".
+        # A per-run prefix (e.g. the run/test id) makes concurrent runs — which
+        # all log into the same /ecs/<family> group — easy to tell apart in
+        # CloudWatch, instead of every task sharing the generic "ecs" prefix.
+        # The <task-id> suffix already guarantees stream uniqueness; the prefix
+        # is purely for human/tool separability. Defaults to "ecs".
+        stream_prefix = resource_config.get("log_stream_prefix", "ecs")
         container_def["logConfiguration"] = {
             "logDriver": "awslogs",
             "options": {
                 "awslogs-group": log_group,
                 "awslogs-region": resource_config.get("region", "us-west-2"),
-                "awslogs-stream-prefix": "ecs",
+                "awslogs-stream-prefix": stream_prefix,
                 "awslogs-create-group": "true",
             },
         }

src/kernel_ci_cloud_labs/cli.py

@@ -1,7 +1,7 @@
 """CLI entry point for kernel-ci-cloud-runner.
 
 Usage:
-    kernel-ci-cloud-runner aws run [--config CONFIG] [--config-s3 S3_URI]
+    kernel-ci-cloud-runner aws run [--config CONFIG] [--config-s3 S3_URI] [--results-dir DIR]
     kernel-ci-cloud-runner aws analyze --bucket BUCKET --run-prefix PREFIX [--output-dir DIR]
     kernel-ci-cloud-runner aws setup configure [--prefix PREFIX] [--region REGION] [--output FILE]
     kernel-ci-cloud-runner aws setup upload-rpms --bucket BUCKET --local-rpms DIR [--region REGION]
@@ -79,6 +79,48 @@ def cmd_run(args):
 
     run_pipeline(provider, storage, run_dir=run_dir)
 
+    # Optionally persist all of this run's S3 result objects to a local
+    # directory (benchmark CSVs, result.txt, console logs, etc.). aws run
+    # otherwise only keeps the orchestrator logs under logs/run_*/.
+    if getattr(args, "results_dir", None):
+        _download_run_results(storage, args.results_dir, logger)
+
+
+def _download_run_results(storage, results_dir, logger):
+    """Download every S3 object under this run's prefix into results_dir.
+
+    Preserves the S3 key structure under results_dir/<run_prefix>/ so the
+    layout matches the bucket. Best-effort: logs and continues on error.
+    """
+    import os as _os
+
+    bucket = getattr(storage, "bucket", None)
+    run_prefix = getattr(storage, "run_prefix", None)
+    s3 = getattr(storage, "s3", None)
+    if not (bucket and run_prefix and s3 is not None):
+        logger.warning("Cannot download results: bucket/run_prefix/s3 client unavailable")
+        return
+
+    dest_root = _os.path.join(results_dir, run_prefix)
+    logger.info("Downloading run results from s3://%s/%s/ to %s", bucket, run_prefix, dest_root)
+    count = 0
+    try:
+        paginator = s3.get_paginator("list_objects_v2")
+        for page in paginator.paginate(Bucket=bucket, Prefix=f"{run_prefix}/"):
+            for obj in page.get("Contents", []):
+                key = obj["Key"]
+                if key.endswith("/"):
+                    continue
+                # Strip the run_prefix so files land under dest_root/<rest>.
+                rel = key[len(run_prefix) + 1:] if key.startswith(run_prefix + "/") else key
+                local_path = _os.path.join(dest_root, rel)
+                _os.makedirs(_os.path.dirname(local_path), exist_ok=True)
+                s3.download_file(bucket, key, local_path)
+                count += 1
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logger.error("Error downloading run results: %s", e)
+    logger.info("✓ Downloaded %d result file(s) to %s", count, dest_root)
+
 
 def cmd_setup_configure(args):
     """Configure project resources."""
@@ -229,6 +271,11 @@ def main():
         "Takes precedence over --config. Designed for EventBridge triggers.",
     )
     run_parser.add_argument("--region", help="AWS region (for S3 config download)")
+    run_parser.add_argument(
+        "--results-dir",
+        help="Download all of this run's S3 result files (benchmark CSVs, "
+        "result.txt, console logs) into DIR/<run_prefix>/ after the run",
+    )
     run_parser.set_defaults(func=cmd_run)
 
     # aws analyze

src/kernel_ci_cloud_labs/core/benchmark_analyzer.py

@@ -62,12 +62,13 @@ class MetricComparison:
     u_pvalue: float = 1.0
     cohens_d: float = 0.0
     is_regression: bool = False
+    is_improvement: bool = False
 
     def __post_init__(self):
         if self.base.mean != 0:
             self.pct_change = ((self.tip.mean - self.base.mean) / abs(self.base.mean)) * 100.0
         self._compute_tests()
-        self._detect_regression()
+        self._classify_change()
 
     def _compute_tests(self):
         """Compute t-test, Mann-Whitney U, and Cohen's d."""
@@ -82,18 +83,27 @@ def _compute_tests(self):
         # Cohen's d (pooled)
         self.cohens_d = _cohens_d(base_v, tip_v)
 
-    def _detect_regression(self):
-        """A regression requires significant p-value AND meaningful effect size."""
+    def _classify_change(self):
+        """Classify the change as a regression or an improvement.
+
+        Both require a significant p-value AND a meaningful effect size; they
+        differ only in direction. A metric moving the "worse" way (down when
+        more_is_better, up when less_is_better) is a regression; moving the
+        "better" way is an improvement. Changes that are not both significant
+        and meaningful are neither (treated as noise).
+        """
+        self.is_regression = False
+        self.is_improvement = False
         significant = self.t_pvalue < P_VALUE_THRESHOLD or self.u_pvalue < P_VALUE_THRESHOLD
         meaningful = abs(self.cohens_d) >= COHENS_D_THRESHOLD
         if not (significant and meaningful):
-            self.is_regression = False
             return
-        # Direction check: regression means performance got worse
-        if self.more_is_better:
-            self.is_regression = self.pct_change < 0
+        # pct_change > 0 means tip is larger than base.
+        got_better = (self.pct_change > 0) if self.more_is_better else (self.pct_change < 0)
+        if got_better:
+            self.is_improvement = True
         else:
-            self.is_regression = self.pct_change > 0
+            self.is_regression = True
 
 
 @dataclass
@@ -113,6 +123,14 @@ def regressions(self) -> List[MetricComparison]:
     def has_regression(self) -> bool:
         return len(self.regressions) > 0
 
+    @property
+    def improvements(self) -> List[MetricComparison]:
+        return [c for c in self.comparisons if c.is_improvement]
+
+    @property
+    def has_improvement(self) -> bool:
+        return len(self.improvements) > 0
+
 
 @dataclass
 class PipelineBenchmarkSummary:
@@ -125,6 +143,8 @@ class PipelineBenchmarkSummary:
     failed_test_names: List[str] = field(default_factory=list)
     tests_with_regression: int = 0
     regression_test_names: List[str] = field(default_factory=list)
+    tests_with_improvement: int = 0
+    improvement_test_names: List[str] = field(default_factory=list)
 
 
 class BenchmarkAnalyzer:
@@ -159,6 +179,9 @@ def analyze(
                 if result.has_regression:
                     summary.tests_with_regression += 1
                     summary.regression_test_names.append(test_name)
+                if result.has_improvement:
+                    summary.tests_with_improvement += 1
+                    summary.improvement_test_names.append(test_name)
 
         return summary
 
@@ -442,15 +465,38 @@ def log_benchmark_summary(summary: PipelineBenchmarkSummary):
         else:
             logger.info("  ✓ No regressions detected")
 
+        if result.improvements:
+            logger.info("  ✓ IMPROVEMENTS DETECTED: %d", len(result.improvements))
+            for c in result.improvements:
+                logger.info(
+                    "    %s: base=%.2f±%.2f (cv: %.2f) → tip=%.2f±%.2f (cv: %.2f) %s (%+.1f%%) "
+                    "[t-test p=%.4f, U-test p=%.4f, Cohen's d=%.2f]",
+                    c.metric,
+                    c.base.mean,
+                    c.base.stddev,
+                    c.base.cv,
+                    c.tip.mean,
+                    c.tip.stddev,
+                    c.tip.cv,
+                    c.unit,
+                    c.pct_change,
+                    c.t_pvalue,
+                    c.u_pvalue,
+                    c.cohens_d,
+                )
+
     logger.info("")
     logger.info("-" * 60)
     logger.info(
-        "Tests with benchmarks: %d | Regressions found: %d",
+        "Tests with benchmarks: %d | Regressions found: %d | Improvements found: %d",
         len(summary.test_results),
         summary.tests_with_regression,
+        summary.tests_with_improvement,
     )
     if summary.regression_test_names:
         logger.info("Tests with regressions: %s", ", ".join(summary.regression_test_names))
+    if summary.improvement_test_names:
+        logger.info("Tests with improvements: %s", ", ".join(summary.improvement_test_names))
     logger.info("=" * 60)
 
     # NOTIFICATION HOOK: Add downstream notifications here, e.g.: