feat(fixtures): add repeated metric comparisons

README.md

@@ -243,6 +243,31 @@ flags: "-s"
 | get health | /health | json.status == "ok"      |
 ```
 
+Rows can be sampled repeatedly without increasing the fixture count. `Repeat`
+overrides the file or command-block default, and metrics extract one finite
+number from every command-successful sample using the same CEL context as
+assertions:
+
+```yaml
+repeat: 5
+metrics:
+  - name: duration_ms
+    extract: json.metrics.durationMs.value
+    unit: ms
+    aggregate: median       # mean (default), median, min, max, or p95
+    direction: lower        # lower, higher, or none
+    baseline: API baseline
+    threshold:
+      max: 120000
+      regressionPercent: 25
+```
+
+`threshold.min` and `threshold.max` apply to the selected aggregate. Relative
+regressions compare a row to the uniquely named baseline row using matching
+metric name, unit, aggregate, and direction. Repeats run serially within one
+shared row timeout; JSON results retain every sample and report command, CEL,
+and metric outcomes separately.
+
 **Command blocks** — use when commands are multi-line or need per-test setup:
 
 ````markdown
@@ -289,6 +314,8 @@ terminal: pty                      # † pseudo-terminal mode (merges stdout/std
 files: "**/*.go"                   # glob: replicate tests per matching file
 codeBlocks: [bash, python]         # languages to execute (default: [bash])
 timeout: 30s                       # † total timeout
+repeat: 3                          # † serial samples per logical row (default: 1)
+metrics: []                        # † numeric CEL extraction and threshold policy
 os: linux                          # † skip on other OSes (prefix ! to negate: !darwin)
 arch: amd64                        # † skip on other architectures
 skip: "! command -v docker"        # † skip if command exits 0

cmd/gavel/fixtures.go

@@ -61,6 +61,8 @@ func fixturesHelp(cmd *cobra.Command) api.Text {
 		Add(code("  files: \"**/*.go\"")).Add(dim("                   # Glob pattern: replicate tests per matching file")).NewLine().
 		Add(code("  codeBlocks: [bash, python]")).Add(dim("         # Languages to execute (default: [bash])")).NewLine().
 		Add(code("  timeout: 30s")).Add(dim("                       # Total timeout for test execution")).NewLine().
+		Add(code("  repeat: 3")).Add(dim("                          # Serial samples per logical row (default: 1)")).NewLine().
+		Add(code("  metrics: []")).Add(dim("                        # Numeric CEL extraction and threshold policy")).NewLine().
 		Add(code("  os: linux")).Add(dim("                          # Skip on other OSes (prefix ! to negate: !darwin)")).NewLine().
 		Add(code("  arch: amd64")).Add(dim("                        # Skip on other architectures")).NewLine().
 		Add(code("  skip: \"! command -v docker\"")).Add(dim("        # Skip if command exits 0")).NewLine().
@@ -78,6 +80,8 @@ func fixturesHelp(cmd *cobra.Command) api.Text {
 		Add(kv("cli args, args", "Arguments (space-separated)")).
 		Add(kv("cwd, working directory", "Working directory")).
 		Add(kv("terminal, term", "Terminal mode (\"pty\" for pseudo-terminal)")).
+		Add(kv("repeat", "Override the file-level sample count for this logical row")).
+		Add(kv("timeout", "Total timeout shared by all repeated samples")).
 		Add(kv("os", "OS constraint (e.g. \"linux\", \"!darwin\")")).
 		Add(kv("arch", "Architecture constraint (e.g. \"amd64\")")).
 		Add(kv("skip", "Bash command; exit 0 = skip test")).
@@ -94,7 +98,7 @@ func fixturesHelp(cmd *cobra.Command) api.Text {
 	t = t.Add(h("FORMAT 2: COMMAND BLOCKS")).
 		Append("  Use heading ").Add(code("### command: <test name>")).Append(" followed by code blocks:").NewLine().NewLine().
 		Add(code("  ### command: my test\n  ```yaml\n  cwd: ./testdir\n  exitCode: 0\n  terminal: pty\n  os: linux\n  env:\n    KEY: value\n  ```\n  ```bash\n  echo \"hello world\"\n  ```")).NewLine().NewLine().
-		Append("  YAML fields: ", "text-muted").Add(code("cwd, exitCode, env, timeout, terminal, os, arch, skip")).NewLine().NewLine().
+		Append("  YAML fields: ", "text-muted").Add(code("cwd, exitCode, env, timeout, repeat, metrics, terminal, os, arch, skip")).NewLine().NewLine().
 		Add(sh("Validations")).
 		Append("    ").Add(code("* cel: stdout.contains(\"hello\")")).NewLine().
 		Append("    ").Add(code("* contains: hello")).NewLine().
@@ -128,6 +132,14 @@ func fixturesHelp(cmd *cobra.Command) api.Text {
 		Add(kv("not: contains: <text>", "!stdout.contains(\"<text>\")")).
 		Add(kv("not: <expr>", "!(<expr>)")).NewLine()
 
+	// Repeated samples and metrics
+	t = t.Add(h("REPEATED SAMPLES AND METRICS")).
+		Append("  Set ").Add(code("repeat")).Append(" in file/command YAML or use a table ").Add(code("Repeat")).Append(" override.").NewLine().
+		Append("  Repeats run serially inside one logical-row timeout and retain per-sample evidence.").NewLine().NewLine().
+		Add(code("  metrics:\n    - name: duration_ms\n      extract: json.metrics.durationMs.value\n      unit: ms\n      aggregate: median\n      direction: lower\n      baseline: API baseline\n      threshold:\n        max: 120000\n        regressionPercent: 25")).NewLine().NewLine().
+		Append("  Aggregates: ").Add(code("mean, median, min, max, p95")).Append(". Directions: ").Add(code("lower, higher, none")).Append(".").NewLine().
+		Append("  Baselines must uniquely name another row with a matching metric specification.").NewLine()
+
 	// CEL Validation
 	t = t.Add(h("CEL VALIDATION")).
 		Append("  Expressions must evaluate to ").Add(code("true")).Append(".").NewLine().NewLine().

fixtures/expectations.go

@@ -45,6 +45,17 @@ type EvaluateOptions struct {
 }
 
 func (e Expectations) Evaluate(fixture FixtureResult, p exec.ExecResult, opts EvaluateOptions) FixtureResult {
+	fixture = e.EvaluateCommand(fixture, p, opts)
+	if fixture.Status != task.StatusPASS || e.CEL == "" {
+		return fixture
+	}
+
+	return EvaluateCEL(fixture, e.CEL, EvaluationContext(&fixture))
+}
+
+// EvaluateCommand captures process evidence and evaluates only exit and stream
+// expectations, leaving CEL and metrics available as independent outcomes.
+func (e Expectations) EvaluateCommand(fixture FixtureResult, p exec.ExecResult, opts EvaluateOptions) FixtureResult {
 
 	fixture.Stdout = p.Stdout
 	fixture.Stderr = p.Stderr
@@ -57,6 +68,9 @@ func (e Expectations) Evaluate(fixture FixtureResult, p exec.ExecResult, opts Ev
 			fixture.Command = p.Command
 		}
 	}
+	if p.Error != nil && p.ExitCode < 0 {
+		return fixture.Errorf(p.Error, "command execution failed")
+	}
 	// Default exit code expectation to 0 if not specified
 	expectedExitCode := 0
 	if e.ExitCode != nil {
@@ -80,66 +94,78 @@ func (e Expectations) Evaluate(fixture FixtureResult, p exec.ExecResult, opts Ev
 	} else if updated {
 		fixture.Metadata["golden_updated_stderr"] = true
 	}
-	if e.CEL != "" {
-		// Use RunExpression for CEL expressions, not RunTemplate
-		t := fixture.Test.AsMap()
-		t["output"] = p.Stdout
-		t["stdout"] = p.Stdout
-		t["stderr"] = p.Stderr
-		t["exitCode"] = p.ExitCode
-		combined := p.Stdout + p.Stderr
-		dups := duplicateLines(combined)
-		dupList := make([]map[string]any, 0, len(dups))
-		for _, d := range dups {
-			dupList = append(dupList, map[string]any{"text": d.Text, "count": d.Count})
-		}
-		t["ansi"] = map[string]any{
-			"has_any":         hasAnyANSI(combined),
-			"has_color":       hasColorCodes(combined),
-			"has_updates":     hasCursorUpdates(combined),
-			"has_cursor_hide": hasCursorHide(combined),
-			"has_cursor_show": hasCursorShow(combined),
-			"has_reset":       hasSGRReset(combined),
-			"stray_controls":  hasStrayControls(combined),
-			"final_text":      finalText(combined),
-			"duplicate_lines": dupList,
-			"has_duplicates":  len(dups) > 0,
-		}
-		// Try to parse JSON output if it looks like JSON
-		if strings.HasPrefix(strings.TrimSpace(p.Stdout), "{") || strings.HasPrefix(strings.TrimSpace(p.Stdout), "[") {
-			var jsonData interface{}
-			if err := json.Unmarshal([]byte(p.Stdout), &jsonData); err == nil {
-				t["json"] = jsonData
-				fixture.Metadata["json"] = jsonData
+	fixture.Status = task.StatusPASS
+	return fixture
+}
+
+// EvaluationContext builds the shared CEL environment for assertions and
+// metric extraction, parsing JSON regardless of whether an assertion exists.
+func EvaluationContext(fixture *FixtureResult) map[string]any {
+	t := fixture.Test.AsMap()
+	t["output"] = fixture.Stdout
+	t["stdout"] = fixture.Stdout
+	t["stderr"] = fixture.Stderr
+	t["exitCode"] = fixture.ExitCode
+	combined := fixture.Stdout + fixture.Stderr
+	dups := duplicateLines(combined)
+	dupList := make([]map[string]any, 0, len(dups))
+	for _, d := range dups {
+		dupList = append(dupList, map[string]any{"text": d.Text, "count": d.Count})
+	}
+	t["ansi"] = map[string]any{
+		"has_any":         hasAnyANSI(combined),
+		"has_color":       hasColorCodes(combined),
+		"has_updates":     hasCursorUpdates(combined),
+		"has_cursor_hide": hasCursorHide(combined),
+		"has_cursor_show": hasCursorShow(combined),
+		"has_reset":       hasSGRReset(combined),
+		"stray_controls":  hasStrayControls(combined),
+		"final_text":      finalText(combined),
+		"duplicate_lines": dupList,
+		"has_duplicates":  len(dups) > 0,
+	}
+
+	trimmed := strings.TrimSpace(fixture.Stdout)
+	if strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "[") {
+		var jsonData interface{}
+		if err := json.Unmarshal([]byte(fixture.Stdout), &jsonData); err == nil {
+			t["json"] = jsonData
+			if fixture.Metadata == nil {
+				fixture.Metadata = map[string]interface{}{}
 			}
+			fixture.Metadata["json"] = jsonData
 		}
+	}
 
-		// Add temp file data to CEL context
-		for name, tempFile := range fixture.Test.TempFiles {
-			t[name] = tempFile.GetCELData()
-		}
-		output, err := gomplate.RunExpression(t, gomplate.Template{
-			Expression: e.CEL,
-			CelEnvs:    ANSICelFunctions(),
-		})
-		if err != nil {
-			return fixture.Errorf(err, "failed to evaluate CEL expression with gomplate")
-		}
+	for name, tempFile := range fixture.Test.TempFiles {
+		t[name] = tempFile.GetCELData()
+	}
+	return t
+}
 
-		switch v := output.(type) {
-		case bool:
-			if !v {
-				fixture.CELExpression = e.CEL
-				fixture.CELVars = t
-				return fixture.Failf("CEL expression evaluated to false")
-			}
-		case string:
-			if strings.ToLower(strings.TrimSpace(v)) != "true" {
-				return fixture.Failf("%s != true", v)
-			}
-		default:
-			return fixture.Failf("CEL expression did not return a boolean: got %T(%v)", output, output)
+// EvaluateCEL evaluates one assertion against a prepared sample context.
+func EvaluateCEL(fixture FixtureResult, expression string, variables map[string]any) FixtureResult {
+	output, err := gomplate.RunExpression(variables, gomplate.Template{
+		Expression: expression,
+		CelEnvs:    ANSICelFunctions(),
+	})
+	if err != nil {
+		return fixture.Errorf(err, "failed to evaluate CEL expression with gomplate")
+	}
+
+	switch v := output.(type) {
+	case bool:
+		if !v {
+			fixture.CELExpression = expression
+			fixture.CELVars = variables
+			return fixture.Failf("CEL expression evaluated to false")
+		}
+	case string:
+		if strings.ToLower(strings.TrimSpace(v)) != "true" {
+			return fixture.Failf("%s != true", v)
 		}
+	default:
+		return fixture.Failf("CEL expression did not return a boolean: got %T(%v)", output, output)
 	}
 	fixture.Status = task.StatusPASS
 	return fixture