feat(cli): add step-summary blocks for rendered prompt and verdict (#739)

* feat(cli): add step-summary blocks for rendered prompt and verdict

Adds `--step-summary` to `threat-detect` (writes the prompt actually
rendered, plus resolved engine/model/retries, as a collapsible block)
and to `threat-detect conclude` (writes a verdict block with per-field
booleans, reasons, conclusion, and reason code for every terminal
outcome including skipped). Both default to $GITHUB_STEP_SUMMARY, are
best-effort (a write failure never changes the exit code), and bound
the prompt size to stay within the shared per-job step summary budget.

Closes #696

Co-authored-by: David Slater <12449447+davidslater@users.noreply.github.com>

* fix(cli): escape untrusted content and reject step-summary path collisions

Addresses review feedback on #739:
- HTML-escape and render the rendered prompt inside <pre><code>
  instead of a Markdown fence, since the embedded artifact content can
  contain a closing fence and spoof the job summary.
- HTML-escape and render verdict reasons inside <pre><code> for the
  same reason, since reasons are engine-generated from untrusted
  artifacts.
- Add rejectPathCollisions and use it to reject --step-summary
  aliasing --log-file/--output in `threat-detect`, and --result-file/
  $GITHUB_OUTPUT/$GITHUB_ENV in `threat-detect conclude`, so a
  misconfigured alias fails closed instead of corrupting another
  destination.

Co-authored-by: David Slater <12449447+davidslater@users.noreply.github.com>

---------

Co-authored-by: GitHub Ace <githubnext@users.noreply.github.com>
Co-authored-by: David Slater <12449447+davidslater@users.noreply.github.com>
Co-authored-by: Copilot <copilot@github.com>

Author: 3 people

README.md

@@ -63,6 +63,7 @@ threat-detect [flags] <artifacts-dir>
 - `--custom-prompt-file` — Path to a file with additional detection instructions. Takes precedence over `--custom-prompt` and `CUSTOM_PROMPT`
 - `--output` — Path to write JSON result (defaults to stdout)
 - `--log-file` — Path to write structured JSONL run logs (one JSON object per line). Env: `THREAT_DETECTION_LOG_FILE`; defaults to `detection-runlog.jsonl` beside `--output`
+- `--step-summary` — Path to append the rendered prompt (engine/model/retries plus the prompt actually sent, including the resolved prompt-analysis section) as a collapsible block in the job step summary. Defaults to `GITHUB_STEP_SUMMARY`
 - `--retries` — Retries for malformed detection outputs. Default: `1` (env: `THREAT_DETECTION_RETRIES`)
 - `--version` — Print version and exit
 
@@ -171,6 +172,12 @@ host-side reason:
 | `config_error` | `agent_failure` |
 | absent / unrecognized / log unreadable | `agent_failure` ("Detection result file not found at: <path>") |
 
+`conclude` also accepts `--step-summary <path>` (defaulting to
+`GITHUB_STEP_SUMMARY`) to append a collapsible verdict block to the job step
+summary: per-field booleans (`prompt_injection`, `secret_leak`,
+`malicious_patch`), the reasons list, the resolved `conclusion`
+(`success`/`warning`/`failure`/`skipped`), and the reason code.
+
 `conclude` writes a verbose, self-contained diagnostic section to the job log:
 banners framing the section, the environment inputs and resolved paths, and the
 per-field verdict breakdown (`prompt_injection`/`secret_leak`/`malicious_patch`)

cmd/threat-detect/conclude.go

@@ -87,10 +87,12 @@ func runConclude(args []string) int {
 	fs.SetOutput(os.Stderr)
 	var (
 		resultFile   string
+		stepSummary  string
 		detectionLog string
 		logFile      string
 	)
 	fs.StringVar(&resultFile, "result-file", defaultConcludeResultFile, "Path to the structured detection_result.json verdict file")
+	fs.StringVar(&stepSummary, "step-summary", os.Getenv("GITHUB_STEP_SUMMARY"), "Path to append the verdict to the job step summary (defaults to env GITHUB_STEP_SUMMARY)")
 	fs.StringVar(&detectionLog, "detection-log", "", "Path to the detection run's captured log, consulted to refine agent_failure/parse_error and to render diagnostics when the result file is missing (default: <result-file dir>/detection.log)")
 	fs.StringVar(&logFile, "log-file", os.Getenv("THREAT_DETECTION_LOG_FILE"), "Path to write JSONL run logs (env: THREAT_DETECTION_LOG_FILE)")
 	if err := fs.Parse(args); err != nil {
@@ -103,6 +105,23 @@ func runConclude(args []string) int {
 		detectionLog = filepath.Join(filepath.Dir(resultFile), defaultDetectionLogName)
 	}
 
+	githubOutput := os.Getenv("GITHUB_OUTPUT")
+	githubEnv := os.Getenv("GITHUB_ENV")
+
+	// Reject collisions among the run's independently-written destinations:
+	// --step-summary must not alias the structured result file (which would be
+	// overwritten with Markdown after being read) or the GitHub Actions command
+	// files (which the runner may reject outright if polluted with Markdown).
+	if err := rejectPathCollisions(
+		namedPath{"--result-file", resultFile},
+		namedPath{"--step-summary", stepSummary},
+		namedPath{"$GITHUB_OUTPUT", githubOutput},
+		namedPath{"$GITHUB_ENV", githubEnv},
+	); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		return concludeExitFail
+	}
+
 	// The JSONL log is opened with O_TRUNC, so it must not alias an input this
 	// command reads. Aliasing --result-file would erase the verdict (yielding a
 	// bogus parse_error) and aliasing the detection log would erase the failure
@@ -153,8 +172,9 @@ func runConclude(args []string) int {
 		warnMode:         os.Getenv("GH_AW_DETECTION_CONTINUE_ON_ERROR") != "false",
 		executionFailed:  os.Getenv("DETECTION_AGENTIC_EXECUTION_OUTCOME") == "failure",
 		executionOutcome: os.Getenv("DETECTION_AGENTIC_EXECUTION_OUTCOME"),
-		githubOutput:     os.Getenv("GITHUB_OUTPUT"),
-		githubEnv:        os.Getenv("GITHUB_ENV"),
+		githubOutput:     githubOutput,
+		githubEnv:        githubEnv,
+		stepSummary:      stepSummary,
 		detectionLog:     detectionLog,
 		logger:           logger,
 		stdout:           os.Stdout,
@@ -173,6 +193,7 @@ type concluder struct {
 
 	githubOutput string // path of $GITHUB_OUTPUT (may be empty)
 	githubEnv    string // path of $GITHUB_ENV (may be empty)
+	stepSummary  string // path of $GITHUB_STEP_SUMMARY (may be empty)
 	detectionLog string // detection run log path; empty derives the sibling default
 	logger       *runlog.Logger
 	stdout       io.Writer
@@ -221,6 +242,7 @@ func (c *concluder) conclude(resultFile string) int {
 		c.setOutput("success", "true")
 		c.setOutput("reason", "")
 		c.exportVariable("GH_AW_DETECTION_REASON", "")
+		c.writeVerdictSummary(nil, "skipped", "")
 		c.info("✅ Detection skipped — no threats to evaluate.")
 		c.logger.Info("conclude_outcome", map[string]any{
 			"conclusion": "skipped",
@@ -251,16 +273,16 @@ func (c *concluder) conclude(resultFile string) int {
 
 		if errors.Is(err, fs.ErrNotExist) {
 			reason, code := c.detectionFailureReason()
-			return c.fail(reason, fmt.Sprintf("%s: ❌ Detection result file not found at: %s", code, resultFile))
+			return c.fail(nil, reason, fmt.Sprintf("%s: ❌ Detection result file not found at: %s", code, resultFile))
 		}
 		var pathErr *fs.PathError
 		if errors.As(err, &pathErr) {
 			reason, code := c.detectionFailureReason()
-			return c.fail(reason, fmt.Sprintf("%s: ❌ Detection result file unreadable at %s: %v", code, resultFile, err))
+			return c.fail(nil, reason, fmt.Sprintf("%s: ❌ Detection result file unreadable at %s: %v", code, resultFile, err))
 		}
 		c.info("💡 This usually means the AI engine did not record a verdict in the expected format.")
 		c.info(`   Expected content: {"prompt_injection":bool,"secret_leak":bool,"malicious_patch":bool,"reasons":[...]}`)
-		return c.fail("parse_error", fmt.Sprintf("%s: ❌ Failed to parse detection result file %s: %v", errCodeParse, resultFile, err))
+		return c.fail(nil, "parse_error", fmt.Sprintf("%s: ❌ Failed to parse detection result file %s: %v", errCodeParse, resultFile, err))
 	}
 
 	c.info("✔️  Structured result file found and parsed successfully.")
@@ -282,7 +304,7 @@ func (c *concluder) conclude(resultFile string) int {
 		if len(result.Reasons) > 0 {
 			message += "\nReasons: " + strings.Join(result.Reasons, "; ")
 		}
-		return c.fail("threat_detected", message)
+		return c.fail(result, "threat_detected", message)
 	}
 
 	c.info("✅ No security threats detected. Safe outputs may proceed.")
@@ -291,6 +313,7 @@ func (c *concluder) conclude(resultFile string) int {
 	c.setOutput("success", "true")
 	c.setOutput("reason", "")
 	c.exportVariable("GH_AW_DETECTION_REASON", "")
+	c.writeVerdictSummary(result, "success", "")
 	c.logger.Info("conclude_outcome", map[string]any{
 		"conclusion": "success",
 		"reason":     "",
@@ -599,7 +622,7 @@ func lastDetectionStatusReason(path string) string {
 //   - In warn mode and not mustFail, emit a warning, set conclusion=warning, and
 //     let the job proceed (exit 0).
 //   - Otherwise set conclusion=failure, emit an error, and fail closed (exit 1).
-func (c *concluder) fail(reason, message string) int {
+func (c *concluder) fail(result *detector.Result, reason, message string) int {
 	mustFail := c.executionFailed && (reason == "agent_failure" || reason == "parse_error")
 	c.setOutput("reason", reason)
 	c.exportVariable("GH_AW_DETECTION_REASON", reason)
@@ -608,6 +631,7 @@ func (c *concluder) fail(reason, message string) int {
 		c.setOutput("conclusion", "warning")
 		c.exportVariable("GH_AW_DETECTION_CONCLUSION", "warning")
 		c.setOutput("success", "false")
+		c.writeVerdictSummary(result, "warning", reason)
 		c.logger.Error("conclude_outcome", map[string]any{
 			"conclusion": "warning",
 			"reason":     reason,
@@ -621,6 +645,7 @@ func (c *concluder) fail(reason, message string) int {
 	c.setOutput("conclusion", "failure")
 	c.exportVariable("GH_AW_DETECTION_CONCLUSION", "failure")
 	c.setOutput("success", "false")
+	c.writeVerdictSummary(result, "failure", reason)
 	c.logger.Error("conclude_outcome", map[string]any{
 		"conclusion": "failure",
 		"reason":     reason,
@@ -631,6 +656,15 @@ func (c *concluder) fail(reason, message string) int {
 	return concludeExitFail
 }
 
+// writeVerdictSummary appends the verdict block to the job step summary,
+// logging (but not failing on) any write error since the summary is a
+// best-effort diagnostic aid, not part of the conclude contract.
+func (c *concluder) writeVerdictSummary(result *detector.Result, conclusion, reasonCode string) {
+	if err := detector.AppendStepSummary(c.stepSummary, detector.FormatVerdictSummary(result, conclusion, reasonCode)); err != nil {
+		fmt.Fprintf(os.Stderr, "conclude: failed to write step summary: %v\n", err)
+	}
+}
+
 // setOutput appends a step output to $GITHUB_OUTPUT. Values are single-line
 // tokens, so the simple name=value form is sufficient and unambiguous.
 func (c *concluder) setOutput(name, value string) {

cmd/threat-detect/conclude_test.go

@@ -366,6 +366,67 @@ func TestConcludeUnreadableFileIsAgentFailure(t *testing.T) {
 	}
 }
 
+func TestConcludeWritesVerdictStepSummary(t *testing.T) {
+	dir := t.TempDir()
+	stepSummaryPath := filepath.Join(dir, "step_summary.md")
+	resultFile := writeResultFixture(t, threatVerdict)
+
+	var stdout bytes.Buffer
+	c := &concluder{
+		runDetection: "true",
+		warnMode:     false,
+		githubOutput: filepath.Join(dir, "out"),
+		githubEnv:    filepath.Join(dir, "env"),
+		stepSummary:  stepSummaryPath,
+		stdout:       &stdout,
+	}
+	if code := c.run(resultFile); code != concludeExitFail {
+		t.Fatalf("exit code = %d, want %d (stdout: %s)", code, concludeExitFail, stdout.String())
+	}
+
+	data, err := os.ReadFile(stepSummaryPath)
+	if err != nil {
+		t.Fatalf("ReadFile(stepSummaryPath) error = %v", err)
+	}
+	summary := string(data)
+	for _, want := range []string{
+		"<summary>Threat Detection Verdict</summary>",
+		"| Prompt Injection | true |",
+		"| Conclusion | failure |",
+		"| Reason Code | threat_detected |",
+		"jailbreak attempt",
+	} {
+		if !strings.Contains(summary, want) {
+			t.Errorf("step summary missing %q; got:\n%s", want, summary)
+		}
+	}
+}
+
+func TestConcludeSkippedWritesVerdictStepSummary(t *testing.T) {
+	dir := t.TempDir()
+	stepSummaryPath := filepath.Join(dir, "step_summary.md")
+
+	var stdout bytes.Buffer
+	c := &concluder{
+		runDetection: "false",
+		githubOutput: filepath.Join(dir, "out"),
+		githubEnv:    filepath.Join(dir, "env"),
+		stepSummary:  stepSummaryPath,
+		stdout:       &stdout,
+	}
+	if code := c.run(filepath.Join(dir, "detection_result.json")); code != concludeExitProceed {
+		t.Fatalf("exit code = %d, want %d (stdout: %s)", code, concludeExitProceed, stdout.String())
+	}
+
+	data, err := os.ReadFile(stepSummaryPath)
+	if err != nil {
+		t.Fatalf("ReadFile(stepSummaryPath) error = %v", err)
+	}
+	if !strings.Contains(string(data), "| Conclusion | skipped |") {
+		t.Errorf("step summary missing skipped conclusion; got:\n%s", string(data))
+	}
+}
+
 func TestConcludeThreatMessageEscaped(t *testing.T) {
 	dir := t.TempDir()
 	resultFile := writeResultFixture(t, threatVerdict)
@@ -1042,3 +1103,41 @@ func TestConcludeUnopenableLogFileIsConfigError(t *testing.T) {
 		t.Fatalf("runConclude() = %d, want %d", code, concludeExitFail)
 	}
 }
+
+func TestRunConcludeRejectsStepSummaryCollidingWithResultFile(t *testing.T) {
+	dir := t.TempDir()
+	resultFile := writeResultFixture(t, safeVerdict)
+
+	t.Setenv("RUN_DETECTION", "true")
+	t.Setenv("GITHUB_OUTPUT", filepath.Join(dir, "out"))
+	t.Setenv("GITHUB_ENV", filepath.Join(dir, "env"))
+
+	code := runConclude([]string{"--result-file", resultFile, "--step-summary", resultFile})
+	if code != concludeExitFail {
+		t.Fatalf("runConclude() = %d, want %d (fail closed on collision)", code, concludeExitFail)
+	}
+	// The result file must survive untouched — a collision must be rejected
+	// before anything is written, so the structured verdict is never clobbered.
+	data, err := os.ReadFile(resultFile)
+	if err != nil {
+		t.Fatalf("ReadFile(resultFile) error = %v", err)
+	}
+	if string(data) != safeVerdict {
+		t.Fatalf("result file was modified: got %q, want %q", string(data), safeVerdict)
+	}
+}
+
+func TestRunConcludeRejectsStepSummaryCollidingWithGithubOutput(t *testing.T) {
+	dir := t.TempDir()
+	resultFile := writeResultFixture(t, safeVerdict)
+	shared := filepath.Join(dir, "shared")
+
+	t.Setenv("RUN_DETECTION", "true")
+	t.Setenv("GITHUB_OUTPUT", shared)
+	t.Setenv("GITHUB_ENV", filepath.Join(dir, "env"))
+
+	code := runConclude([]string{"--result-file", resultFile, "--step-summary", shared})
+	if code != concludeExitFail {
+		t.Fatalf("runConclude() = %d, want %d (fail closed on collision)", code, concludeExitFail)
+	}
+}

cmd/threat-detect/logfile_test.go

@@ -296,6 +296,50 @@ func TestRunRejectsLogFileCollidingWithOutput(t *testing.T) {
 	}
 }
 
+func TestRunRejectsStepSummaryCollidingWithOutput(t *testing.T) {
+	artifactsDir := t.TempDir()
+	shared := filepath.Join(t.TempDir(), "same.json")
+
+	code, stderr := runWithTestArgsCapture(t, []string{
+		"threat-detect",
+		"-output", shared,
+		"-step-summary", shared,
+		artifactsDir,
+	}, nil)
+
+	if code != exitError {
+		t.Fatalf("run() exit code = %d, want %d", code, exitError)
+	}
+	if !strings.Contains(stderr, "--step-summary") || !strings.Contains(stderr, "must not point to the same file") {
+		t.Fatalf("stderr missing collision error, got:\n%s", stderr)
+	}
+	if _, err := os.Stat(shared); !os.IsNotExist(err) {
+		t.Fatalf("expected no file to be written, stat err = %v", err)
+	}
+}
+
+func TestRunRejectsStepSummaryCollidingWithLogFile(t *testing.T) {
+	artifactsDir := t.TempDir()
+	shared := filepath.Join(t.TempDir(), "same.jsonl")
+
+	code, stderr := runWithTestArgsCapture(t, []string{
+		"threat-detect",
+		"-log-file", shared,
+		"-step-summary", shared,
+		artifactsDir,
+	}, nil)
+
+	if code != exitError {
+		t.Fatalf("run() exit code = %d, want %d", code, exitError)
+	}
+	if !strings.Contains(stderr, "--step-summary") || !strings.Contains(stderr, "--log-file") {
+		t.Fatalf("stderr missing collision error mentioning both flags, got:\n%s", stderr)
+	}
+	if _, err := os.Stat(shared); !os.IsNotExist(err) {
+		t.Fatalf("expected no file to be written, stat err = %v", err)
+	}
+}
+
 func TestRunRejectsDefaultLogCollidingThroughDanglingOutputSymlink(t *testing.T) {
 	artifactsDir := t.TempDir()
 	outputDir := t.TempDir()

cmd/threat-detect/main.go

@@ -107,6 +107,7 @@ func run() (code int) {
 		promptFile          string
 		outputJSON          string
 		logFile             string
+		stepSummary         string
 		workflowName        string
 		workflowDescription string
 		customPrompt        string
@@ -125,6 +126,7 @@ func run() (code int) {
 	flag.StringVar(&promptFile, "prompt-template", "", "Path to custom prompt template (defaults to built-in)")
 	flag.StringVar(&outputJSON, "output", "", "Path to write JSON result (defaults to stdout)")
 	flag.StringVar(&logFile, "log-file", os.Getenv("THREAT_DETECTION_LOG_FILE"), "Path to write JSONL run logs (env: THREAT_DETECTION_LOG_FILE)")
+	flag.StringVar(&stepSummary, "step-summary", os.Getenv("GITHUB_STEP_SUMMARY"), "Path to append the rendered prompt to the job step summary (defaults to env GITHUB_STEP_SUMMARY)")
 	flag.StringVar(&workflowName, "workflow-name", "", "Workflow name for the prompt (overrides WORKFLOW_NAME)")
 	flag.StringVar(&workflowDescription, "workflow-description", "", "Workflow description for the prompt (overrides WORKFLOW_DESCRIPTION)")
 	flag.StringVar(&customPrompt, "custom-prompt", "", "Additional detection instructions appended to the prompt (overrides CUSTOM_PROMPT)")
@@ -158,19 +160,17 @@ func run() (code int) {
 		logFile = dir + "detection-runlog.jsonl"
 	}
 
-	// Reject a --log-file that collides with --output: they are opened and
-	// truncated independently, so sharing an inode would interleave the JSONL
-	// trace and the result JSON and corrupt both while still reporting success.
-	if logFile != "" && outputJSON != "" {
-		if same, err := samePath(logFile, outputJSON); err != nil {
-			fmt.Fprintf(os.Stderr, "Error resolving output paths: %v\n", err)
-			reason = reasonConfigError
-			return exitError
-		} else if same {
-			fmt.Fprintf(os.Stderr, "Error: --log-file and --output must not point to the same file (%q)\n", logFile)
-			reason = reasonConfigError
-			return exitError
-		}
+	// Reject collisions among the run's independently-written destinations
+	// (--log-file, --output, --step-summary): each is opened and written on
+	// its own, so aliasing any two would interleave or clobber their content.
+	if err := rejectPathCollisions(
+		namedPath{"--log-file", logFile},
+		namedPath{"--output", outputJSON},
+		namedPath{"--step-summary", stepSummary},
+	); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		reason = reasonConfigError
+		return exitError
 	}
 
 	// Open the JSONL run log when configured or derived. A failure here is a
@@ -307,6 +307,16 @@ func run() (code int) {
 		"custom_prompt_bytes":            len(arts.CustomPrompt),
 	})
 
+	// Surface the prompt actually rendered by threat-detect (including the
+	// resolved prompt-analysis section and engine/model/retries) to the job
+	// step summary. gh-aw's own prompt-rendering step summary reflects a
+	// template threat-detect never receives, so this is the only summary that
+	// reflects what was actually sent to the engine.
+	if err := detector.AppendStepSummary(stepSummary, detector.FormatPromptSummary(engine.Canonical(engineID), model, retries, prompt)); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: failed to write prompt step summary: %v\n", err)
+		logger.Error("step_summary_write_failed", map[string]any{"stage": "prompt", "error": err.Error()})
+	}
+
 	// Create engine
 	eng, err := engine.New(engineID, model)
 	if err != nil {