diff --git a/README.md b/README.md index ae748c68f..a9ef41d8e 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,31 @@ flags: "-s" | get health | /health | json.status == "ok" | ``` +Rows can be sampled repeatedly without increasing the fixture count. `Repeat` +overrides the file or command-block default, and metrics extract one finite +number from every command-successful sample using the same CEL context as +assertions: + +```yaml +repeat: 5 +metrics: + - name: duration_ms + extract: json.metrics.durationMs.value + unit: ms + aggregate: median # mean (default), median, min, max, or p95 + direction: lower # lower, higher, or none + baseline: API baseline + threshold: + max: 120000 + regressionPercent: 25 +``` + +`threshold.min` and `threshold.max` apply to the selected aggregate. Relative +regressions compare a row to the uniquely named baseline row using matching +metric name, unit, aggregate, and direction. Repeats run serially within one +shared row timeout; JSON results retain every sample and report command, CEL, +and metric outcomes separately. + **Command blocks** — use when commands are multi-line or need per-test setup: ````markdown @@ -289,6 +314,8 @@ terminal: pty # † pseudo-terminal mode (merges stdout/std files: "**/*.go" # glob: replicate tests per matching file codeBlocks: [bash, python] # languages to execute (default: [bash]) timeout: 30s # † total timeout +repeat: 3 # † serial samples per logical row (default: 1) +metrics: [] # † numeric CEL extraction and threshold policy os: linux # † skip on other OSes (prefix ! to negate: !darwin) arch: amd64 # † skip on other architectures skip: "! command -v docker" # † skip if command exits 0 diff --git a/cmd/gavel/fixtures.go b/cmd/gavel/fixtures.go index 92b91df3c..ee98b979c 100644 --- a/cmd/gavel/fixtures.go +++ b/cmd/gavel/fixtures.go @@ -61,6 +61,8 @@ func fixturesHelp(cmd *cobra.Command) api.Text { Add(code(" files: \"**/*.go\"")).Add(dim(" # Glob pattern: replicate tests per matching file")).NewLine(). Add(code(" codeBlocks: [bash, python]")).Add(dim(" # Languages to execute (default: [bash])")).NewLine(). Add(code(" timeout: 30s")).Add(dim(" # Total timeout for test execution")).NewLine(). + Add(code(" repeat: 3")).Add(dim(" # Serial samples per logical row (default: 1)")).NewLine(). + Add(code(" metrics: []")).Add(dim(" # Numeric CEL extraction and threshold policy")).NewLine(). Add(code(" os: linux")).Add(dim(" # Skip on other OSes (prefix ! to negate: !darwin)")).NewLine(). Add(code(" arch: amd64")).Add(dim(" # Skip on other architectures")).NewLine(). Add(code(" skip: \"! command -v docker\"")).Add(dim(" # Skip if command exits 0")).NewLine(). @@ -78,6 +80,8 @@ func fixturesHelp(cmd *cobra.Command) api.Text { Add(kv("cli args, args", "Arguments (space-separated)")). Add(kv("cwd, working directory", "Working directory")). Add(kv("terminal, term", "Terminal mode (\"pty\" for pseudo-terminal)")). + Add(kv("repeat", "Override the file-level sample count for this logical row")). + Add(kv("timeout", "Total timeout shared by all repeated samples")). Add(kv("os", "OS constraint (e.g. \"linux\", \"!darwin\")")). Add(kv("arch", "Architecture constraint (e.g. \"amd64\")")). Add(kv("skip", "Bash command; exit 0 = skip test")). @@ -94,7 +98,7 @@ func fixturesHelp(cmd *cobra.Command) api.Text { t = t.Add(h("FORMAT 2: COMMAND BLOCKS")). Append(" Use heading ").Add(code("### command: ")).Append(" followed by code blocks:").NewLine().NewLine(). Add(code(" ### command: my test\n ```yaml\n cwd: ./testdir\n exitCode: 0\n terminal: pty\n os: linux\n env:\n KEY: value\n ```\n ```bash\n echo \"hello world\"\n ```")).NewLine().NewLine(). - Append(" YAML fields: ", "text-muted").Add(code("cwd, exitCode, env, timeout, terminal, os, arch, skip")).NewLine().NewLine(). + Append(" YAML fields: ", "text-muted").Add(code("cwd, exitCode, env, timeout, repeat, metrics, terminal, os, arch, skip")).NewLine().NewLine(). Add(sh("Validations")). Append(" ").Add(code("* cel: stdout.contains(\"hello\")")).NewLine(). Append(" ").Add(code("* contains: hello")).NewLine(). @@ -128,6 +132,14 @@ func fixturesHelp(cmd *cobra.Command) api.Text { Add(kv("not: contains: ", "!stdout.contains(\"\")")). Add(kv("not: ", "!()")).NewLine() + // Repeated samples and metrics + t = t.Add(h("REPEATED SAMPLES AND METRICS")). + Append(" Set ").Add(code("repeat")).Append(" in file/command YAML or use a table ").Add(code("Repeat")).Append(" override.").NewLine(). + Append(" Repeats run serially inside one logical-row timeout and retain per-sample evidence.").NewLine().NewLine(). + Add(code(" metrics:\n - name: duration_ms\n extract: json.metrics.durationMs.value\n unit: ms\n aggregate: median\n direction: lower\n baseline: API baseline\n threshold:\n max: 120000\n regressionPercent: 25")).NewLine().NewLine(). + Append(" Aggregates: ").Add(code("mean, median, min, max, p95")).Append(". Directions: ").Add(code("lower, higher, none")).Append(".").NewLine(). + Append(" Baselines must uniquely name another row with a matching metric specification.").NewLine() + // CEL Validation t = t.Add(h("CEL VALIDATION")). Append(" Expressions must evaluate to ").Add(code("true")).Append(".").NewLine().NewLine(). diff --git a/fixtures/expectations.go b/fixtures/expectations.go index 7f5b23c3d..de0c18081 100644 --- a/fixtures/expectations.go +++ b/fixtures/expectations.go @@ -45,6 +45,17 @@ type EvaluateOptions struct { } func (e Expectations) Evaluate(fixture FixtureResult, p exec.ExecResult, opts EvaluateOptions) FixtureResult { + fixture = e.EvaluateCommand(fixture, p, opts) + if fixture.Status != task.StatusPASS || e.CEL == "" { + return fixture + } + + return EvaluateCEL(fixture, e.CEL, EvaluationContext(&fixture)) +} + +// EvaluateCommand captures process evidence and evaluates only exit and stream +// expectations, leaving CEL and metrics available as independent outcomes. +func (e Expectations) EvaluateCommand(fixture FixtureResult, p exec.ExecResult, opts EvaluateOptions) FixtureResult { fixture.Stdout = p.Stdout fixture.Stderr = p.Stderr @@ -57,6 +68,9 @@ func (e Expectations) Evaluate(fixture FixtureResult, p exec.ExecResult, opts Ev fixture.Command = p.Command } } + if p.Error != nil && p.ExitCode < 0 { + return fixture.Errorf(p.Error, "command execution failed") + } // Default exit code expectation to 0 if not specified expectedExitCode := 0 if e.ExitCode != nil { @@ -80,66 +94,78 @@ func (e Expectations) Evaluate(fixture FixtureResult, p exec.ExecResult, opts Ev } else if updated { fixture.Metadata["golden_updated_stderr"] = true } - if e.CEL != "" { - // Use RunExpression for CEL expressions, not RunTemplate - t := fixture.Test.AsMap() - t["output"] = p.Stdout - t["stdout"] = p.Stdout - t["stderr"] = p.Stderr - t["exitCode"] = p.ExitCode - combined := p.Stdout + p.Stderr - dups := duplicateLines(combined) - dupList := make([]map[string]any, 0, len(dups)) - for _, d := range dups { - dupList = append(dupList, map[string]any{"text": d.Text, "count": d.Count}) - } - t["ansi"] = map[string]any{ - "has_any": hasAnyANSI(combined), - "has_color": hasColorCodes(combined), - "has_updates": hasCursorUpdates(combined), - "has_cursor_hide": hasCursorHide(combined), - "has_cursor_show": hasCursorShow(combined), - "has_reset": hasSGRReset(combined), - "stray_controls": hasStrayControls(combined), - "final_text": finalText(combined), - "duplicate_lines": dupList, - "has_duplicates": len(dups) > 0, - } - // Try to parse JSON output if it looks like JSON - if strings.HasPrefix(strings.TrimSpace(p.Stdout), "{") || strings.HasPrefix(strings.TrimSpace(p.Stdout), "[") { - var jsonData interface{} - if err := json.Unmarshal([]byte(p.Stdout), &jsonData); err == nil { - t["json"] = jsonData - fixture.Metadata["json"] = jsonData + fixture.Status = task.StatusPASS + return fixture +} + +// EvaluationContext builds the shared CEL environment for assertions and +// metric extraction, parsing JSON regardless of whether an assertion exists. +func EvaluationContext(fixture *FixtureResult) map[string]any { + t := fixture.Test.AsMap() + t["output"] = fixture.Stdout + t["stdout"] = fixture.Stdout + t["stderr"] = fixture.Stderr + t["exitCode"] = fixture.ExitCode + combined := fixture.Stdout + fixture.Stderr + dups := duplicateLines(combined) + dupList := make([]map[string]any, 0, len(dups)) + for _, d := range dups { + dupList = append(dupList, map[string]any{"text": d.Text, "count": d.Count}) + } + t["ansi"] = map[string]any{ + "has_any": hasAnyANSI(combined), + "has_color": hasColorCodes(combined), + "has_updates": hasCursorUpdates(combined), + "has_cursor_hide": hasCursorHide(combined), + "has_cursor_show": hasCursorShow(combined), + "has_reset": hasSGRReset(combined), + "stray_controls": hasStrayControls(combined), + "final_text": finalText(combined), + "duplicate_lines": dupList, + "has_duplicates": len(dups) > 0, + } + + trimmed := strings.TrimSpace(fixture.Stdout) + if strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "[") { + var jsonData interface{} + if err := json.Unmarshal([]byte(fixture.Stdout), &jsonData); err == nil { + t["json"] = jsonData + if fixture.Metadata == nil { + fixture.Metadata = map[string]interface{}{} } + fixture.Metadata["json"] = jsonData } + } - // Add temp file data to CEL context - for name, tempFile := range fixture.Test.TempFiles { - t[name] = tempFile.GetCELData() - } - output, err := gomplate.RunExpression(t, gomplate.Template{ - Expression: e.CEL, - CelEnvs: ANSICelFunctions(), - }) - if err != nil { - return fixture.Errorf(err, "failed to evaluate CEL expression with gomplate") - } + for name, tempFile := range fixture.Test.TempFiles { + t[name] = tempFile.GetCELData() + } + return t +} - switch v := output.(type) { - case bool: - if !v { - fixture.CELExpression = e.CEL - fixture.CELVars = t - return fixture.Failf("CEL expression evaluated to false") - } - case string: - if strings.ToLower(strings.TrimSpace(v)) != "true" { - return fixture.Failf("%s != true", v) - } - default: - return fixture.Failf("CEL expression did not return a boolean: got %T(%v)", output, output) +// EvaluateCEL evaluates one assertion against a prepared sample context. +func EvaluateCEL(fixture FixtureResult, expression string, variables map[string]any) FixtureResult { + output, err := gomplate.RunExpression(variables, gomplate.Template{ + Expression: expression, + CelEnvs: ANSICelFunctions(), + }) + if err != nil { + return fixture.Errorf(err, "failed to evaluate CEL expression with gomplate") + } + + switch v := output.(type) { + case bool: + if !v { + fixture.CELExpression = expression + fixture.CELVars = variables + return fixture.Failf("CEL expression evaluated to false") + } + case string: + if strings.ToLower(strings.TrimSpace(v)) != "true" { + return fixture.Failf("%s != true", v) } + default: + return fixture.Failf("CEL expression did not return a boolean: got %T(%v)", output, output) } fixture.Status = task.StatusPASS return fixture diff --git a/fixtures/metrics.go b/fixtures/metrics.go new file mode 100644 index 000000000..1edcaad90 --- /dev/null +++ b/fixtures/metrics.go @@ -0,0 +1,485 @@ +package fixtures + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strings" + + "github.com/flanksource/clicky/task" + "github.com/flanksource/gomplate/v3" +) + +const ( + metricMean = "mean" + metricMedian = "median" + metricMin = "min" + metricMax = "max" + metricP95 = "p95" + + directionLower = "lower" + directionHigher = "higher" + directionNone = "none" +) + +// MetricThreshold applies absolute limits and an optional relative baseline limit. +type MetricThreshold struct { + Min *float64 `yaml:"min,omitempty" json:"min,omitempty"` + Max *float64 `yaml:"max,omitempty" json:"max,omitempty"` + RegressionPercent *float64 `yaml:"regressionPercent,omitempty" json:"regressionPercent,omitempty"` +} + +// MetricSpec describes a numeric CEL extraction and its logical-row policy. +type MetricSpec struct { + Name string `yaml:"name" json:"name"` + Extract string `yaml:"extract" json:"extract"` + Unit string `yaml:"unit" json:"unit"` + Aggregate string `yaml:"aggregate,omitempty" json:"aggregate,omitempty"` + Direction string `yaml:"direction,omitempty" json:"direction,omitempty"` + Baseline string `yaml:"baseline,omitempty" json:"baseline,omitempty"` + Threshold *MetricThreshold `yaml:"threshold,omitempty" json:"threshold,omitempty"` +} + +func (f FixtureTest) repeatCount() int { + if f.Repeat != nil { + return *f.Repeat + } + if f.FrontMatter.Repeat != nil { + return *f.FrontMatter.Repeat + } + return 1 +} + +func (f FixtureTest) metricSpecs() []MetricSpec { + if len(f.Metrics) > 0 { + return f.Metrics + } + return f.FrontMatter.Metrics +} + +func (f FixtureTest) hasSampleConfiguration() bool { + return f.Repeat != nil || f.FrontMatter.Repeat != nil || len(f.metricSpecs()) > 0 +} + +func (m MetricSpec) normalizedAggregate() string { + if m.Aggregate == "" { + return metricMean + } + return strings.ToLower(m.Aggregate) +} + +func (m MetricSpec) normalizedDirection() string { + if m.Direction == "" { + return directionNone + } + return strings.ToLower(m.Direction) +} + +// validateFixtureConfiguration rejects metric policy errors before any command runs. +func validateFixtureConfiguration(fixtures []FixtureTest) error { + rowsByName := make(map[string][]FixtureTest, len(fixtures)) + for _, fixture := range fixtures { + rowsByName[fixture.Name] = append(rowsByName[fixture.Name], fixture) + if fixture.Repeat != nil && *fixture.Repeat < 1 { + return fmt.Errorf("fixture %q: repeat must be at least 1", fixture.Name) + } + if fixture.FrontMatter.Repeat != nil && *fixture.FrontMatter.Repeat < 1 { + return fmt.Errorf("fixture %q: frontmatter repeat must be at least 1", fixture.Name) + } + if fixture.Expected.Timeout != nil && *fixture.Expected.Timeout <= 0 { + return fmt.Errorf("fixture %q: timeout must be greater than zero", fixture.Name) + } + if fixture.Timeout != nil && *fixture.Timeout <= 0 { + return fmt.Errorf("fixture %q: frontmatter timeout must be greater than zero", fixture.Name) + } + if err := validateMetricSpecs(fixture); err != nil { + return err + } + } + + for _, fixture := range fixtures { + for _, metric := range fixture.metricSpecs() { + if metric.Baseline == "" { + continue + } + baselines := rowsByName[metric.Baseline] + switch len(baselines) { + case 0: + return fmt.Errorf("fixture %q metric %q: baseline row %q was not found", fixture.Name, metric.Name, metric.Baseline) + case 1: + default: + return fmt.Errorf("fixture %q metric %q: baseline row name %q is ambiguous (%d rows)", fixture.Name, metric.Name, metric.Baseline, len(baselines)) + } + + baselineMetric, ok := metricSpecByName(baselines[0].metricSpecs(), metric.Name) + if !ok { + return fmt.Errorf("fixture %q metric %q: baseline row %q does not configure that metric", fixture.Name, metric.Name, metric.Baseline) + } + if metric.Unit != baselineMetric.Unit || metric.normalizedAggregate() != baselineMetric.normalizedAggregate() || metric.normalizedDirection() != baselineMetric.normalizedDirection() { + return fmt.Errorf("fixture %q metric %q: baseline row %q must use matching unit, aggregate, and direction", fixture.Name, metric.Name, metric.Baseline) + } + } + } + return nil +} + +func validateMetricSpecs(fixture FixtureTest) error { + seen := make(map[string]struct{}) + for _, metric := range fixture.metricSpecs() { + prefix := fmt.Sprintf("fixture %q metric %q", fixture.Name, metric.Name) + if metric.Name == "" { + return fmt.Errorf("fixture %q: metric name is required", fixture.Name) + } + if _, ok := seen[metric.Name]; ok { + return fmt.Errorf("%s: duplicate metric name", prefix) + } + seen[metric.Name] = struct{}{} + if metric.Extract == "" { + return fmt.Errorf("%s: extract is required", prefix) + } + if metric.Unit == "" { + return fmt.Errorf("%s: unit is required", prefix) + } + switch metric.normalizedAggregate() { + case metricMean, metricMedian, metricMin, metricMax, metricP95: + default: + return fmt.Errorf("%s: unsupported aggregate %q (want mean, median, min, max, or p95)", prefix, metric.Aggregate) + } + switch metric.normalizedDirection() { + case directionLower, directionHigher, directionNone: + default: + return fmt.Errorf("%s: unsupported direction %q (want lower, higher, or none)", prefix, metric.Direction) + } + if metric.Baseline != "" && metric.normalizedDirection() == directionNone { + return fmt.Errorf("%s: baseline comparison requires direction lower or higher", prefix) + } + if metric.Threshold == nil { + continue + } + threshold := metric.Threshold + if threshold.Min != nil && !finite(*threshold.Min) { + return fmt.Errorf("%s: threshold.min must be finite", prefix) + } + if threshold.Max != nil && !finite(*threshold.Max) { + return fmt.Errorf("%s: threshold.max must be finite", prefix) + } + if threshold.Min != nil && threshold.Max != nil && *threshold.Min > *threshold.Max { + return fmt.Errorf("%s: threshold.min cannot exceed threshold.max", prefix) + } + if threshold.RegressionPercent != nil { + if !finite(*threshold.RegressionPercent) || *threshold.RegressionPercent < 0 { + return fmt.Errorf("%s: threshold.regressionPercent must be a finite non-negative number", prefix) + } + if metric.Baseline == "" { + return fmt.Errorf("%s: threshold.regressionPercent requires a baseline row", prefix) + } + if metric.normalizedDirection() == directionNone { + return fmt.Errorf("%s: threshold.regressionPercent requires direction lower or higher", prefix) + } + } + } + return nil +} + +func metricSpecByName(metrics []MetricSpec, name string) (MetricSpec, bool) { + for _, metric := range metrics { + if metric.Name == name { + return metric, true + } + } + return MetricSpec{}, false +} + +func extractMetric(metric MetricSpec, variables map[string]any) (*float64, error) { + output, err := gomplate.RunExpression(variables, gomplate.Template{ + Expression: metric.Extract, + CelEnvs: ANSICelFunctions(), + }) + if err != nil { + return nil, fmt.Errorf("evaluate extract expression %q: %w", metric.Extract, err) + } + value, ok := numericValue(output) + if !ok { + return nil, fmt.Errorf("extract expression %q returned %T; expected a number", metric.Extract, output) + } + if !finite(value) { + return nil, fmt.Errorf("extract expression %q returned a non-finite number", metric.Extract) + } + return &value, nil +} + +func numericValue(value any) (float64, bool) { + switch v := value.(type) { + case int: + return float64(v), true + case int8: + return float64(v), true + case int16: + return float64(v), true + case int32: + return float64(v), true + case int64: + return float64(v), true + case uint: + return float64(v), true + case uint8: + return float64(v), true + case uint16: + return float64(v), true + case uint32: + return float64(v), true + case uint64: + return float64(v), true + case float32: + return float64(v), true + case float64: + return v, true + case json.Number: + f, err := v.Float64() + return f, err == nil + default: + return 0, false + } +} + +func finite(value float64) bool { + return !math.IsNaN(value) && !math.IsInf(value, 0) +} + +func meanMetric(values []float64) float64 { + var scale float64 + for _, value := range values { + if absolute := math.Abs(value); absolute > scale { + scale = absolute + } + } + if scale == 0 { + return 0 + } + + var total float64 + for _, value := range values { + total += value / scale + } + return scale * (total / float64(len(values))) +} + +func aggregateMetric(values []float64, aggregate string) float64 { + sorted := append([]float64(nil), values...) + sort.Float64s(sorted) + switch aggregate { + case metricMedian: + middle := len(sorted) / 2 + if len(sorted)%2 == 0 { + return meanMetric(sorted[middle-1 : middle+1]) + } + return sorted[middle] + case metricMin: + return sorted[0] + case metricMax: + return sorted[len(sorted)-1] + case metricP95: + return sorted[int(math.Ceil(float64(len(sorted))*0.95))-1] + default: + return meanMetric(values) + } +} + +func summarizeMetrics(result *FixtureResult) { + metrics := result.Test.metricSpecs() + if len(metrics) == 0 { + return + } + result.Metrics = make(map[string]MetricSummary, len(metrics)) + for _, metric := range metrics { + summary := MetricSummary{ + Unit: metric.Unit, + Aggregate: metric.normalizedAggregate(), + Direction: metric.normalizedDirection(), + Samples: make([]float64, 0, result.Test.repeatCount()), + Threshold: metric.Threshold, + Status: OutcomeNotEvaluated, + } + var extractionErrors []string + for _, sample := range result.Samples { + extracted := sample.Metrics[metric.Name] + if extracted.Value != nil && extracted.Status == OutcomePASS { + summary.Samples = append(summary.Samples, *extracted.Value) + } + if extracted.Status == OutcomeERR { + extractionErrors = append(extractionErrors, fmt.Sprintf("sample %d: %s", sample.Index, extracted.Error)) + } + } + if len(extractionErrors) > 0 { + summary.Status = OutcomeERR + summary.Error = strings.Join(extractionErrors, "; ") + result.Metrics[metric.Name] = summary + continue + } + if len(summary.Samples) != result.Test.repeatCount() { + result.Metrics[metric.Name] = summary + continue + } + + value := aggregateMetric(summary.Samples, summary.Aggregate) + if !finite(value) { + summary.Status = OutcomeERR + summary.Error = fmt.Sprintf("%s aggregate produced a non-finite number", summary.Aggregate) + result.Metrics[metric.Name] = summary + continue + } + summary.Value = &value + summary.Status = OutcomePASS + if metric.Threshold != nil { + if metric.Threshold.Min != nil && value < *metric.Threshold.Min { + summary.Status = OutcomeFAIL + summary.Error = fmt.Sprintf("%s aggregate %.6g %s is below minimum %.6g %s", summary.Aggregate, value, metric.Unit, *metric.Threshold.Min, metric.Unit) + } + if metric.Threshold.Max != nil && value > *metric.Threshold.Max { + summary.Status = OutcomeFAIL + errorText := fmt.Sprintf("%s aggregate %.6g %s exceeds maximum %.6g %s", summary.Aggregate, value, metric.Unit, *metric.Threshold.Max, metric.Unit) + summary.Error = joinErrors(summary.Error, errorText) + } + } + result.Metrics[metric.Name] = summary + } +} + +// finalizeMetricComparisons runs only after every logical row has completed so +// callbacks and tree statistics see comparison-aware final verdicts. +func finalizeMetricComparisons(results []*FixtureResult) { + byName := make(map[string][]*FixtureResult, len(results)) + for _, result := range results { + byName[result.Name] = append(byName[result.Name], result) + } + + for _, result := range results { + for _, metric := range result.Test.metricSpecs() { + if metric.Baseline == "" || metric.Baseline == result.Name { + continue + } + summary := result.Metrics[metric.Name] + if summary.Value == nil { + continue + } + baselineResult := byName[metric.Baseline][0] + baseline, ok := baselineResult.Metrics[metric.Name] + if !ok || baseline.Value == nil { + summary.Status = OutcomeERR + summary.Error = joinErrors(summary.Error, fmt.Sprintf("baseline row %q did not produce a complete value", metric.Baseline)) + result.Metrics[metric.Name] = summary + continue + } + comparison := &MetricComparison{ + Baseline: metric.Baseline, + BaselineValue: *baseline.Value, + Status: OutcomePASS, + } + if *baseline.Value == 0 { + comparison.Status = OutcomeERR + comparison.Error = "relative comparison to a zero baseline is undefined; use an absolute threshold" + summary.Status = OutcomeERR + summary.Error = joinErrors(summary.Error, comparison.Error) + summary.Comparison = comparison + result.Metrics[metric.Name] = summary + continue + } + + currentRatio := *summary.Value / math.Abs(*baseline.Value) + baselineRatio := *baseline.Value / math.Abs(*baseline.Value) + var regressionPercent float64 + if metric.normalizedDirection() == directionLower { + regressionPercent = (currentRatio - baselineRatio) * 100 + } else { + regressionPercent = (baselineRatio - currentRatio) * 100 + } + if !finite(regressionPercent) { + comparison.Status = OutcomeERR + comparison.Error = "relative comparison produced a non-finite regression percentage" + summary.Status = OutcomeERR + summary.Error = joinErrors(summary.Error, comparison.Error) + summary.Comparison = comparison + result.Metrics[metric.Name] = summary + continue + } + comparison.RegressionPercent = regressionPercent + if metric.Threshold != nil && metric.Threshold.RegressionPercent != nil && comparison.RegressionPercent > *metric.Threshold.RegressionPercent { + comparison.Status = OutcomeFAIL + comparison.Error = fmt.Sprintf("regression %.2f%% exceeds maximum %.2f%%", comparison.RegressionPercent, *metric.Threshold.RegressionPercent) + if summary.Status != OutcomeERR { + summary.Status = OutcomeFAIL + } + summary.Error = joinErrors(summary.Error, comparison.Error) + } + summary.Comparison = comparison + result.Metrics[metric.Name] = summary + } + finalizeLogicalResult(result) + } +} + +func metricOutcome(metrics map[string]MetricSummary) *FixtureOutcome { + if len(metrics) == 0 { + return nil + } + outcome := &FixtureOutcome{Status: OutcomePASS} + var failures []string + for name, metric := range metrics { + switch metric.Status { + case OutcomeERR: + outcome.Status = OutcomeERR + case OutcomeFAIL: + if outcome.Status != OutcomeERR { + outcome.Status = OutcomeFAIL + } + case OutcomeNotEvaluated: + if outcome.Status == OutcomePASS { + outcome.Status = OutcomeNotEvaluated + } + } + if metric.Error != "" { + failures = append(failures, fmt.Sprintf("%s: %s", name, metric.Error)) + } + } + sort.Strings(failures) + outcome.Error = strings.Join(failures, "; ") + return outcome +} + +func finalizeLogicalResult(result *FixtureResult) { + if result.Outcomes == nil { + return + } + result.Outcomes.Metrics = metricOutcome(result.Metrics) + result.Status = task.StatusPASS + var errors []string + for _, outcome := range []struct { + name string + value *FixtureOutcome + }{ + {name: "command", value: &result.Outcomes.Command}, + {name: "assertions", value: result.Outcomes.Assertions}, + {name: "metrics", value: result.Outcomes.Metrics}, + } { + if outcome.value == nil { + continue + } + if outcome.value.Status == OutcomeERR { + result.Status = task.StatusERR + } else if outcome.value.Status == OutcomeFAIL && result.Status != task.StatusERR { + result.Status = task.StatusFAIL + } + if outcome.value.Error != "" { + errors = append(errors, outcome.name+": "+outcome.value.Error) + } + } + result.Error = strings.Join(errors, "; ") +} + +func joinErrors(current, next string) string { + if current == "" { + return next + } + return current + "; " + next +} diff --git a/fixtures/parser.go b/fixtures/parser.go index 2331d22a0..08b0be8b2 100644 --- a/fixtures/parser.go +++ b/fixtures/parser.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/bmatcuk/doublestar/v4" "github.com/goccy/go-yaml" @@ -70,10 +71,10 @@ func ParseMarkdownFixtures(fixtureFilePath string) ([]FixtureNode, error) { return nodes, nil } -// parseTableRow converts a table row into a FixtureTest -func parseTableRow(headers, values []string) *FixtureNode { +// parseTableRow converts a table row into a FixtureTest. +func parseTableRow(headers, values []string) (*FixtureNode, error) { if len(headers) != len(values) { - return nil + return nil, nil } fixture := FixtureTest{ @@ -87,6 +88,14 @@ func parseTableRow(headers, values []string) *FixtureNode { switch header { case "test name", "name": fixture.Name = value + case "repeat": + if value != "" && value != "-" { + repeat, err := strconv.Atoi(value) + if err != nil { + return nil, fmt.Errorf("fixture %q: invalid Repeat value %q: %w", fixture.Name, value, err) + } + fixture.Repeat = &repeat + } case "cwd", "working directory", "dir": fixture.CWD = value case "query": @@ -135,6 +144,14 @@ func parseTableRow(headers, values []string) *FixtureNode { fixture.Expected.Output = value case "cel validation", "cel", "validation", "expr": fixture.Expected.CEL = value + case "timeout": + if value != "" && value != "-" { + timeout, err := parseFixtureDuration(value) + if err != nil { + return nil, fmt.Errorf("fixture %q: invalid timeout %q: %w", fixture.Name, value, err) + } + fixture.Expected.Timeout = &timeout + } default: if value != "" { if fixture.Expected.Properties == nil { @@ -147,13 +164,20 @@ func parseTableRow(headers, values []string) *FixtureNode { // Don't return fixtures without names if fixture.Name == "" { - return nil + return nil, nil } return &FixtureNode{ Type: TestNode, Test: &fixture, + }, nil +} + +func parseFixtureDuration(value string) (time.Duration, error) { + if seconds, err := strconv.Atoi(value); err == nil { + return time.Duration(seconds) * time.Second, nil } + return time.ParseDuration(value) } // parseFrontMatter extracts YAML front-matter from a markdown file diff --git a/fixtures/parser_ast.go b/fixtures/parser_ast.go index a8d982681..e77e42013 100644 --- a/fixtures/parser_ast.go +++ b/fixtures/parser_ast.go @@ -68,7 +68,11 @@ func parseMarkdownWithGoldmarkTree(content string, frontMatter *FrontMatter, sou // Complete any pending standalone code block if standaloneCodeBlock != nil && !standaloneCodeBlock.isComplete { - if fixture := buildFixtureFromCommand(standaloneCodeBlock, frontMatter, sourceDir); fixture != nil { + fixture, err := buildFixtureFromCommand(standaloneCodeBlock, frontMatter, sourceDir) + if err != nil { + return ast.WalkStop, err + } + if fixture != nil { currentSection.AddChild(&FixtureNode{ Name: fixture.Test.Name, Type: TestNode, @@ -85,7 +89,11 @@ func parseMarkdownWithGoldmarkTree(content string, frontMatter *FrontMatter, sou // Complete previous command block if exists if currentCommand != nil && !currentCommand.isComplete { - if fixture := buildFixtureFromCommand(currentCommand, frontMatter, sourceDir); fixture != nil { + fixture, err := buildFixtureFromCommand(currentCommand, frontMatter, sourceDir) + if err != nil { + return ast.WalkStop, err + } + if fixture != nil { // Add test to the current section currentSection.AddChild(&FixtureNode{ Name: fixture.Test.Name, @@ -181,7 +189,11 @@ func parseMarkdownWithGoldmarkTree(content string, frontMatter *FrontMatter, sou // Handle standalone code blocks (new behavior) // Complete any pending standalone code block first if standaloneCodeBlock != nil && !standaloneCodeBlock.isComplete { - if fixture := buildFixtureFromCommand(standaloneCodeBlock, frontMatter, sourceDir); fixture != nil { + fixture, err := buildFixtureFromCommand(standaloneCodeBlock, frontMatter, sourceDir) + if err != nil { + return ast.WalkStop, err + } + if fixture != nil { currentSection.AddChild(&FixtureNode{ Name: fixture.Test.Name, Type: TestNode, @@ -237,7 +249,11 @@ func parseMarkdownWithGoldmarkTree(content string, frontMatter *FrontMatter, sou standaloneCodeBlock.validations = append(standaloneCodeBlock.validations, validations...) // Complete the standalone code block now that we have validations - if fixture := buildFixtureFromCommand(standaloneCodeBlock, frontMatter, sourceDir); fixture != nil { + fixture, err := buildFixtureFromCommand(standaloneCodeBlock, frontMatter, sourceDir) + if err != nil { + return ast.WalkStop, err + } + if fixture != nil { currentSection.AddChild(&FixtureNode{ Name: fixture.Test.Name, Type: TestNode, @@ -272,7 +288,11 @@ func parseMarkdownWithGoldmarkTree(content string, frontMatter *FrontMatter, sou // Complete final standalone code block if exists if standaloneCodeBlock != nil && !standaloneCodeBlock.isComplete { - if fixture := buildFixtureFromCommand(standaloneCodeBlock, frontMatter, sourceDir); fixture != nil { + fixture, err := buildFixtureFromCommand(standaloneCodeBlock, frontMatter, sourceDir) + if err != nil { + return nil, err + } + if fixture != nil { currentSection.AddChild(&FixtureNode{ Name: fixture.Test.Name, Type: TestNode, @@ -285,7 +305,11 @@ func parseMarkdownWithGoldmarkTree(content string, frontMatter *FrontMatter, sou // Complete final command block if exists if currentCommand != nil && !currentCommand.isComplete { - if fixture := buildFixtureFromCommand(currentCommand, frontMatter, sourceDir); fixture != nil { + fixture, err := buildFixtureFromCommand(currentCommand, frontMatter, sourceDir) + if err != nil { + return nil, err + } + if fixture != nil { // Add test to the current command section currentSection.AddChild(&FixtureNode{ Name: fixture.Test.Name, @@ -398,10 +422,10 @@ func extractValidationsFromList(listNode *ast.List, source []byte) []string { return validations } -// buildFixtureFromCommand converts a commandBlockBuilder to a FixtureTest -func buildFixtureFromCommand(cmd *commandBlockBuilder, frontMatter *FrontMatter, sourceDir string) *FixtureNode { +// buildFixtureFromCommand converts a commandBlockBuilder to a FixtureTest. +func buildFixtureFromCommand(cmd *commandBlockBuilder, frontMatter *FrontMatter, sourceDir string) (*FixtureNode, error) { if cmd.name == "" || cmd.content == "" { - return nil + return nil, nil } exec := ExecFixtureBase{ Exec: cmd.language, @@ -453,31 +477,43 @@ func buildFixtureFromCommand(cmd *commandBlockBuilder, frontMatter *FrontMatter, OS string `yaml:"os"` Arch string `yaml:"arch"` Skip string `yaml:"skip"` + Repeat *int `yaml:"repeat"` + Metrics []MetricSpec `yaml:"metrics"` } - if err := yaml.Unmarshal([]byte(cmd.frontmatter), &cmdFrontMatter); err == nil { - if cmdFrontMatter.CWD != "" { - fixture.CWD = cmdFrontMatter.CWD - } - if cmdFrontMatter.ExitCode != nil { - fixture.Expected.ExitCode = cmdFrontMatter.ExitCode - } - if cmdFrontMatter.Env != nil { - fixture.Env = cmdFrontMatter.Env - } - if cmdFrontMatter.Terminal != "" { - fixture.Terminal = cmdFrontMatter.Terminal - } - if cmdFrontMatter.OS != "" { - fixture.TestOS = cmdFrontMatter.OS - } - if cmdFrontMatter.Arch != "" { - fixture.TestArch = cmdFrontMatter.Arch - } - if cmdFrontMatter.Skip != "" { - fixture.TestSkip = cmdFrontMatter.Skip + if err := yaml.Unmarshal([]byte(cmd.frontmatter), &cmdFrontMatter); err != nil { + return nil, fmt.Errorf("command %q: invalid frontmatter: %w", cmd.name, err) + } + if cmdFrontMatter.CWD != "" { + fixture.CWD = cmdFrontMatter.CWD + } + if cmdFrontMatter.ExitCode != nil { + fixture.Expected.ExitCode = cmdFrontMatter.ExitCode + } + if cmdFrontMatter.Env != nil { + fixture.Env = cmdFrontMatter.Env + } + if cmdFrontMatter.Timeout != "" { + timeout, err := parseFixtureDuration(cmdFrontMatter.Timeout) + if err != nil { + return nil, fmt.Errorf("command %q: invalid timeout %q: %w", cmd.name, cmdFrontMatter.Timeout, err) } + fixture.Expected.Timeout = &timeout } + if cmdFrontMatter.Terminal != "" { + fixture.Terminal = cmdFrontMatter.Terminal + } + if cmdFrontMatter.OS != "" { + fixture.TestOS = cmdFrontMatter.OS + } + if cmdFrontMatter.Arch != "" { + fixture.TestArch = cmdFrontMatter.Arch + } + if cmdFrontMatter.Skip != "" { + fixture.TestSkip = cmdFrontMatter.Skip + } + fixture.Repeat = cmdFrontMatter.Repeat + fixture.Metrics = cmdFrontMatter.Metrics } // Apply file-level frontmatter if present @@ -511,7 +547,7 @@ func buildFixtureFromCommand(cmd *commandBlockBuilder, frontMatter *FrontMatter, Type: TestNode, Test: &fixture, Origin: cmd.origin, - } + }, nil } // parseTableFromAST parses table-based fixtures from AST (existing functionality) @@ -553,7 +589,11 @@ func parseTableFromAST(tableAST *extast.Table, source []byte, frontMatter *Front // Create fixture from row if len(headers) > 0 && len(values) == len(headers) { - if fixtureNode := parseTableRow(headers, values); fixtureNode != nil { + fixtureNode, err := parseTableRow(headers, values) + if err != nil { + return nil, err + } + if fixtureNode != nil { // Apply frontmatter and source directory if fixtureNode.Test != nil { applyFrontMatterToFixture(fixtureNode.Test, frontMatter) diff --git a/fixtures/parser_ast_test.go b/fixtures/parser_ast_test.go index 6117d023f..9af4767bc 100644 --- a/fixtures/parser_ast_test.go +++ b/fixtures/parser_ast_test.go @@ -210,7 +210,8 @@ Validations: Context("when building fixture from command", func() { DescribeTable("should build correct fixture structure", func(cmd *commandBlockBuilder, expectedTest FixtureTest) { - fixtureNode := buildFixtureFromCommand(cmd, nil, "/tmp/test") + fixtureNode, err := buildFixtureFromCommand(cmd, nil, "/tmp/test") + Expect(err).NotTo(HaveOccurred()) Expect(fixtureNode).NotTo(BeNil()) fixture := *fixtureNode.Test diff --git a/fixtures/runner.go b/fixtures/runner.go index 1e04b2420..248beb4a0 100644 --- a/fixtures/runner.go +++ b/fixtures/runner.go @@ -112,6 +112,9 @@ func (r *Runner) prepareFixtureTree() (*FixtureNode, error) { if len(r.fixtures) == 0 { return nil, fmt.Errorf("no fixtures found") } + if err := validateFixtureConfiguration(r.fixtures); err != nil { + return nil, fmt.Errorf("invalid fixture configuration: %w", err) + } return r.tree, nil } @@ -245,12 +248,8 @@ func (r *Runner) executeFixtures() (*FixtureGroup, error) { r.tree.Walk(func(node *FixtureNode) { if node.Test != nil { typedTask := fixtureGroup.Add(node.Test.String(), func(ctx flanksourceContext.Context, t *task.Task) (FixtureResult, error) { - result, err := r.executeFixture(ctx, *node.Test) - if r.options.OnResult != nil { - r.options.OnResult(result) - } - return result, err - }, clicky.WithTaskTimeout(2*time.Minute)) + return r.executeLogicalFixture(ctx, *node.Test), nil + }, clicky.WithTaskTimeout(fixtureTimeout(*node.Test))) taskToNodeMap[typedTask] = node } }) @@ -267,21 +266,37 @@ func (r *Runner) executeFixtures() (*FixtureGroup, error) { return nil, fmt.Errorf("failed to get fixture results: %w", err) } + finalized := make(map[task.TypedTask[FixtureResult]]*FixtureResult, len(fixtureResults)) + allResults := make([]*FixtureResult, 0, len(fixtureResults)) for typedTask, result := range fixtureResults { + result := result + finalized[typedTask] = &result + allResults = append(allResults, &result) + } + finalizeMetricComparisons(allResults) + + for typedTask, result := range finalized { + // The task initially completes before cross-row comparisons are possible. + // Replace its stored value so subsequent Clicky renders use the finalized + // baseline-aware verdict rather than the pre-comparison result. + typedTask.SetResult(*result) // Create a FixtureNode for the result resultNode := FixtureNode{ Name: result.Name, Type: TestNode, - Results: &result, + Results: result, } results.Tests = append(results.Tests, resultNode) // Update the corresponding tree node with results if testNode, exists := taskToNodeMap[typedTask]; exists { - testNode.Results = &result + testNode.Results = result } else { logger.Warnf("No tree node found for task: %s", typedTask.Name()) } + if r.options.OnResult != nil { + r.options.OnResult(*result) + } } r.tree.UpdateStatsRecursive() @@ -297,6 +312,206 @@ func (r *Runner) executeFixtures() (*FixtureGroup, error) { return results, nil } +func fixtureTimeout(fixture FixtureTest) time.Duration { + if fixture.Expected.Timeout != nil { + return *fixture.Expected.Timeout + } + if fixture.Timeout != nil { + return *fixture.Timeout + } + return 2 * time.Minute +} + +// executeLogicalFixture preserves one task/result per Markdown row while +// collecting serial samples within the row's shared task deadline. +func (r *Runner) executeLogicalFixture(ctx flanksourceContext.Context, fixture FixtureTest) FixtureResult { + if !fixture.hasSampleConfiguration() { + result, _ := r.executeFixture(ctx, fixture) + return result + } + if reason := fixture.ShouldSkip(); reason != "" { + return FixtureResult{ + Name: fixture.Name, + Status: task.StatusSKIP, + Test: fixture, + Error: reason, + } + } + + started := time.Now() + result := FixtureResult{ + Name: fixture.Name, + Test: fixture, + Samples: make([]FixtureSample, 0, fixture.repeatCount()), + Outcomes: &FixtureOutcomes{ + Command: FixtureOutcome{Status: OutcomePASS}, + }, + } + if fixture.Expected.CEL != "" { + result.Outcomes.Assertions = &FixtureOutcome{Status: OutcomeNotEvaluated} + } + + var rowError string + for index := 1; index <= fixture.repeatCount(); index++ { + if err := ctx.Err(); err != nil { + rowError = fmt.Sprintf("fixture timeout exhausted before sample %d: %v", index, err) + break + } + + sampleFixture := fixture + sampleFixture.Expected.CEL = "" + sampleResult, _ := r.executeFixture(ctx, sampleFixture) + if sampleResult.Status == task.StatusSKIP { + return sampleResult + } + sampleResult.Test = fixture + sample := FixtureSample{ + Index: index, + Duration: sampleResult.Duration, + Command: sampleResult.Command, + CWD: sampleResult.CWD, + ExitCode: sampleResult.ExitCode, + Stdout: sampleResult.Stdout, + Stderr: sampleResult.Stderr, + Outcome: outcomeFromTaskStatus(sampleResult.Status, sampleResult.Error), + } + if sample.Outcome.Status != OutcomePASS { + sample.Error = sampleResult.Error + } + + result.Type = sampleResult.Type + result.Command = sampleResult.Command + result.CWD = sampleResult.CWD + result.Stdout = sampleResult.Stdout + result.Stderr = sampleResult.Stderr + result.ExitCode = sampleResult.ExitCode + + metrics := fixture.metricSpecs() + if len(metrics) > 0 { + sample.Metrics = make(map[string]MetricSample, len(metrics)) + } + if sample.Outcome.Status == OutcomePASS { + variables := EvaluationContext(&sampleResult) + if fixture.Expected.CEL != "" || len(metrics) > 0 { + result.Metadata = sampleResult.Metadata + } + if fixture.Expected.CEL != "" { + assertionResult := EvaluateCEL(sampleResult, fixture.Expected.CEL, variables) + celOutcome := AssertionOutcome{FixtureOutcome: outcomeFromTaskStatus(assertionResult.Status, assertionResult.Error)} + celOutcome.Expression = fixture.Expected.CEL + passed := assertionResult.Status == task.StatusPASS + celOutcome.Result = &passed + sample.CEL = &celOutcome + if assertionResult.Status != task.StatusPASS && result.CELExpression == "" { + result.CELExpression = fixture.Expected.CEL + result.CELVars = variables + } + } + for _, metric := range metrics { + value, err := extractMetric(metric, variables) + if err != nil { + sample.Metrics[metric.Name] = MetricSample{Status: OutcomeERR, Error: err.Error()} + } else { + sample.Metrics[metric.Name] = MetricSample{Status: OutcomePASS, Value: value} + } + } + } else { + if fixture.Expected.CEL != "" { + sample.CEL = &AssertionOutcome{ + FixtureOutcome: FixtureOutcome{Status: OutcomeNotEvaluated}, + Expression: fixture.Expected.CEL, + } + } + for _, metric := range metrics { + sample.Metrics[metric.Name] = MetricSample{Status: OutcomeNotEvaluated} + } + } + + result.Samples = append(result.Samples, sample) + if sampleHasError(sample) { + break + } + } + + result.Duration = time.Since(started) + result.Outcomes.Command = aggregateCommandOutcome(result.Samples) + if rowError != "" { + result.Outcomes.Command.Status = OutcomeERR + result.Outcomes.Command.Error = joinErrors(result.Outcomes.Command.Error, rowError) + } + if fixture.Expected.CEL != "" { + result.Outcomes.Assertions = aggregateAssertionOutcome(result.Samples) + } + summarizeMetrics(&result) + finalizeLogicalResult(&result) + return result +} + +func outcomeFromTaskStatus(status task.Status, err string) FixtureOutcome { + outcome := FixtureOutcome{Error: err} + switch status { + case task.StatusPASS, task.StatusSuccess: + outcome.Status = OutcomePASS + case task.StatusFAIL, task.StatusFailed: + outcome.Status = OutcomeFAIL + default: + outcome.Status = OutcomeERR + } + return outcome +} + +func sampleHasError(sample FixtureSample) bool { + if sample.Outcome.Status == OutcomeERR || (sample.CEL != nil && sample.CEL.Status == OutcomeERR) { + return true + } + for _, metric := range sample.Metrics { + if metric.Status == OutcomeERR { + return true + } + } + return false +} + +func aggregateCommandOutcome(samples []FixtureSample) FixtureOutcome { + outcome := FixtureOutcome{Status: OutcomePASS} + for _, sample := range samples { + if sample.Outcome.Status == OutcomeERR { + outcome.Status = OutcomeERR + } else if sample.Outcome.Status == OutcomeFAIL && outcome.Status != OutcomeERR { + outcome.Status = OutcomeFAIL + } + if sample.Outcome.Error != "" { + outcome.Error = joinErrors(outcome.Error, fmt.Sprintf("sample %d: %s", sample.Index, sample.Outcome.Error)) + } + } + return outcome +} + +func aggregateAssertionOutcome(samples []FixtureSample) *FixtureOutcome { + outcome := &FixtureOutcome{Status: OutcomeNotEvaluated} + evaluated := false + for _, sample := range samples { + if sample.CEL == nil || sample.CEL.Status == OutcomeNotEvaluated { + continue + } + evaluated = true + if sample.CEL.Status == OutcomeERR { + outcome.Status = OutcomeERR + } else if sample.CEL.Status == OutcomeFAIL && outcome.Status != OutcomeERR { + outcome.Status = OutcomeFAIL + } else if outcome.Status == OutcomeNotEvaluated { + outcome.Status = OutcomePASS + } + if sample.CEL.Error != "" { + outcome.Error = joinErrors(outcome.Error, fmt.Sprintf("sample %d: %s", sample.Index, sample.CEL.Error)) + } + } + if !evaluated { + outcome.Status = OutcomeNotEvaluated + } + return outcome +} + // getBuildCommand extracts build command from first fixture that has one func (r *Runner) getBuildCommand() string { for _, fixture := range r.fixtures { diff --git a/fixtures/testdata/repeated-metrics.md b/fixtures/testdata/repeated-metrics.md new file mode 100644 index 000000000..1ca8244c4 --- /dev/null +++ b/fixtures/testdata/repeated-metrics.md @@ -0,0 +1,78 @@ +--- +exec: bash +args: + - -c + - | + state="/tmp/gavel-repeated-metrics-$PPID-{{ .key }}" + sample=$(( $(cat "$state" 2>/dev/null || echo 0) + 1 )) + printf '{"schemaVersion":"captain.observation/v1","execution":{"state":"completed"},"metrics":{"durationMs":{"state":"known","value":%s,"unit":"ms"}}}\n' "$sample" + if [ "$sample" -ge "{{ .samplecount }}" ]; then rm -f "$state"; else printf '%s\n' "$sample" > "$state"; fi +repeat: 3 +timeout: 30s +metrics: + - name: mean_value + extract: json.metrics.durationMs.value + unit: ms + aggregate: mean + direction: none + threshold: + min: 1 + max: 10 + - name: median_value + extract: json.metrics.durationMs.value + unit: ms + aggregate: median + direction: none + - name: min_value + extract: json.metrics.durationMs.value + unit: ms + aggregate: min + direction: none + - name: max_value + extract: json.metrics.durationMs.value + unit: ms + aggregate: max + direction: none + - name: p95_value + extract: json.metrics.durationMs.value + unit: ms + aggregate: p95 + direction: none + - name: lower_regression + extract: json.metrics.durationMs.value + unit: ms + aggregate: median + direction: lower + baseline: API baseline + threshold: + regressionPercent: 60 + - name: higher_regression + extract: json.metrics.durationMs.value + unit: ms + aggregate: median + direction: higher + baseline: API baseline + threshold: + regressionPercent: 60 +--- + +# Generic repeated metrics + +| Name | key | sampleCount | Repeat | CEL Validation | +|---|---|---:|---:|---| +| API baseline | baseline | 3 | | json.schemaVersion == "captain.observation/v1" && json.execution.state == "completed" | +| CLI candidate | candidate | 5 | 5 | | + +### command: Command frontmatter repeat + +```yaml +repeat: 2 +metrics: + - name: command_value + extract: json.metrics.durationMs.value + unit: ms +``` + +```bash +printf '{"schemaVersion":"captain.observation/v1","execution":{"state":"completed"},"metrics":{"durationMs":{"state":"known","value":7,"unit":"ms"}}}\n' +``` diff --git a/fixtures/types.go b/fixtures/types.go index 0542294e3..44ae3790f 100644 --- a/fixtures/types.go +++ b/fixtures/types.go @@ -27,6 +27,10 @@ type FixtureTest struct { // Name of the test to be displayed in reports Name string `json:"name,omitempty"` + // Repeat overrides the file-level repeat for this logical row. + Repeat *int `json:"repeat,omitempty"` + // Metrics overrides file-level metric extraction for command blocks. + Metrics []MetricSpec `json:"metrics,omitempty"` // The working directory for executing the test SourceDir string `json:"source_dir,omitempty"` Query string `json:"query,omitempty"` @@ -325,6 +329,10 @@ type FrontMatter struct { ExecFixtureBase `yaml:",inline" json:",inline"` Files string `yaml:"files,omitempty" json:"files,omitempty"` // Glob pattern to match files + // Repeat is the default number of samples for each logical row. + Repeat *int `yaml:"repeat,omitempty" json:"repeat,omitempty"` + // Metrics are CEL expressions extracted independently from each sample. + Metrics []MetricSpec `yaml:"metrics,omitempty" json:"metrics,omitempty"` // CodeBlocks specifies which code block languages to execute (defaults to ["bash"]) CodeBlocks []string `yaml:"codeBlocks,omitempty" json:"codeBlocks,omitempty"` @@ -361,6 +369,8 @@ func (f *FrontMatter) CleanMetadata() { delete(f.Metadata, "terminal") // Keys from FrontMatter itself delete(f.Metadata, "files") + delete(f.Metadata, "repeat") + delete(f.Metadata, "metrics") delete(f.Metadata, "codeBlocks") delete(f.Metadata, "timeout") delete(f.Metadata, "os") @@ -421,6 +431,84 @@ func (nt NodeType) Pretty() api.Text { return clicky.Text(nt.String(), "text-gray-500") } +// OutcomeStatus separates the command, assertion, and metric verdicts that +// contribute to a logical fixture row's final task status. +type OutcomeStatus string + +const ( + OutcomePASS OutcomeStatus = "pass" + OutcomeFAIL OutcomeStatus = "fail" + OutcomeERR OutcomeStatus = "error" + OutcomeNotEvaluated OutcomeStatus = "not_evaluated" +) + +// FixtureOutcome is one independently reported part of a fixture verdict. +type FixtureOutcome struct { + Status OutcomeStatus `json:"status"` + Error string `json:"error,omitempty"` +} + +// AssertionOutcome records one sample's CEL evaluation without discarding +// command or metric evidence from the same process execution. +type AssertionOutcome struct { + FixtureOutcome + Expression string `json:"expression,omitempty"` + Result *bool `json:"result,omitempty"` +} + +// MetricSample records the extraction result for one configured metric. +type MetricSample struct { + Status OutcomeStatus `json:"status"` + Value *float64 `json:"value,omitempty"` + Error string `json:"error,omitempty"` +} + +// FixtureSample retains process evidence and independent evaluation outcomes +// for one execution of a logical Markdown row. +type FixtureSample struct { + Index int `json:"index"` + Duration time.Duration `json:"duration,omitempty"` + Command string `json:"command,omitempty"` + CWD string `json:"cwd,omitempty"` + ExitCode int `json:"exit_code"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + Error string `json:"error,omitempty"` + Outcome FixtureOutcome `json:"command_outcome"` + CEL *AssertionOutcome `json:"cel,omitempty"` + Metrics map[string]MetricSample `json:"metrics,omitempty"` +} + +// FixtureOutcomes summarizes the independently evaluated parts of a logical row. +type FixtureOutcomes struct { + Command FixtureOutcome `json:"command"` + Assertions *FixtureOutcome `json:"assertions,omitempty"` + Metrics *FixtureOutcome `json:"metrics,omitempty"` +} + +// MetricComparison records the finalized comparison to another logical row. +type MetricComparison struct { + Baseline string `json:"baseline"` + BaselineValue float64 `json:"baseline_value"` + RegressionPercent float64 `json:"regression_percent"` + Status OutcomeStatus `json:"status"` + Error string `json:"error,omitempty"` +} + +// MetricSummary contains valid raw samples, the selected aggregate, and any +// absolute or baseline threshold verdict for one metric. +type MetricSummary struct { + Unit string `json:"unit"` + Aggregate string `json:"aggregate"` + Direction string `json:"direction"` + Samples []float64 `json:"samples"` + Value *float64 `json:"value,omitempty"` + Threshold *MetricThreshold `json:"threshold,omitempty"` + Comparison *MetricComparison `json:"comparison,omitempty"` + Status OutcomeStatus `json:"status"` + Error string `json:"error,omitempty"` +} + // FixtureResult represents the outcome of executing a single fixture test. // It contains core information, execution results, and metadata about the test run. type FixtureResult struct { @@ -441,6 +529,11 @@ type FixtureResult struct { CELExpression string `json:"cel_expression,omitempty"` CELVars map[string]any `json:"cel_vars,omitempty"` + // Additive logical-row evidence. These fields remain nil for legacy fixtures. + Samples []FixtureSample `json:"samples,omitempty"` + Metrics map[string]MetricSummary `json:"metrics,omitempty"` + Outcomes *FixtureOutcomes `json:"outcomes,omitempty"` + // Execution metadata Command string `json:"command,omitempty" pretty:"label=Command,style=text-cyan-600,omitempty"` CWD string `json:"cwd,omitempty" pretty:"label=Working Dir,style=text-purple-500,omitempty"` @@ -508,12 +601,31 @@ func (f FixtureResult) Pretty() api.Text { t = t.Space().Append(fmt.Sprintf("(%s)", f.Duration), "text-muted") } - if f.CELExpression != "" { + if f.Outcomes != nil && f.Error != "" { + t = t.Space().Append(f.Error, "text-red-600") + } else if f.CELExpression != "" { t = t.Space().Append(f.CELExpression, "font-mono text-red-500") } else if f.Error != "" { t = t.Space().Append(f.Error, "text-red-600") } + if len(f.Metrics) > 0 { + names := make([]string, 0, len(f.Metrics)) + for name := range f.Metrics { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + t = t.NewLine().Append(formatMetricSummary(name, f.Metrics[name]), "font-mono text-xs") + } + } + if len(f.Samples) > 0 && isFailureStatus(f.Status) { + t = t.NewLine().Add(api.Collapsed{ + Label: fmt.Sprintf("samples (%d)", len(f.Samples)), + Content: clicky.Text(formatSampleSummaries(f.Samples), "font-mono text-xs whitespace-pre-wrap"), + }) + } + if len(f.CELVars) > 0 && f.showCELVars() { t = t.NewLine().Add(api.Collapsed{ Label: "variables", @@ -551,6 +663,47 @@ func (f FixtureResult) Pretty() api.Text { return t } +func formatMetricSummary(name string, metric MetricSummary) string { + value := "incomplete" + if metric.Value != nil { + value = fmt.Sprintf("%.6g %s", *metric.Value, metric.Unit) + } + text := fmt.Sprintf("metric %s: %s (%s, %s)", name, value, metric.Aggregate, metric.Status) + if metric.Comparison != nil { + text += fmt.Sprintf(", baseline %s %.6g %s, regression %.2f%%", metric.Comparison.Baseline, metric.Comparison.BaselineValue, metric.Unit, metric.Comparison.RegressionPercent) + } + return text +} + +func formatSampleSummaries(samples []FixtureSample) string { + lines := make([]string, 0, len(samples)) + for _, sample := range samples { + line := fmt.Sprintf("sample %d: command=%s", sample.Index, sample.Outcome.Status) + if sample.CEL != nil { + line += fmt.Sprintf(" assertions=%s", sample.CEL.Status) + } + if len(sample.Metrics) > 0 { + metricStatus := OutcomePASS + for _, metric := range sample.Metrics { + if metric.Status == OutcomeERR { + metricStatus = OutcomeERR + break + } + if metric.Status == OutcomeNotEvaluated { + metricStatus = OutcomeNotEvaluated + } + } + line += fmt.Sprintf(" metrics=%s", metricStatus) + } + line += fmt.Sprintf(" exit=%d duration=%s", sample.ExitCode, sample.Duration) + if sample.Error != "" { + line += " error=" + firstNonBlankFixtureLine(sample.Error) + } + lines = append(lines, line) + } + return strings.Join(lines, "\n") +} + func (f FixtureResult) showCommand() bool { return f.Display == nil || f.Display.ShowCommand } diff --git a/fixtures/types/exec.go b/fixtures/types/exec.go index 3233600b0..1c4eddc8c 100644 --- a/fixtures/types/exec.go +++ b/fixtures/types/exec.go @@ -12,7 +12,6 @@ import ( "time" "github.com/creack/pty" - "github.com/flanksource/clicky" clickyExec "github.com/flanksource/clicky/exec" "github.com/flanksource/commons/logger" "github.com/flanksource/gavel/fixtures" @@ -113,17 +112,9 @@ func (e *ExecFixture) Run(ctx context.Context, fixture fixtures.FixtureTest, opt var p *clickyExec.ExecResult if exec.Terminal == "pty" { - p = runWithPTY(exec, workDir) + p = runWithPTY(ctx, exec, workDir) } else { - cmd := clicky.Exec(exec.Exec, exec.Args...).WithCwd(workDir) - if len(exec.Env) > 0 { - envMap := make(map[string]string, len(exec.Env)) - for k, v := range exec.Env { - envMap[k] = fmt.Sprintf("%v", v) - } - cmd = cmd.WithEnv(envMap) - } - p = cmd.Run().Result() + p = runPiped(ctx, exec, workDir) } result.Actual = p @@ -133,18 +124,59 @@ func (e *ExecFixture) Run(ctx context.Context, fixture fixtures.FixtureTest, opt }) } -func runWithPTY(execBase fixtures.ExecFixtureBase, workDir string) *clickyExec.ExecResult { +func runPiped(ctx context.Context, execBase fixtures.ExecFixtureBase, workDir string) *clickyExec.ExecResult { + process := clickyExec.NewExec(execBase.Exec, execBase.Args...).WithCwd(workDir).WithProcessGroup() + if len(execBase.Env) > 0 { + env := make(map[string]string, len(execBase.Env)) + for key, value := range execBase.Env { + env[key] = fmt.Sprintf("%v", value) + } + process.WithEnv(env) + } + + done := make(chan *clickyExec.Process, 1) + go func() { + done <- process.Run() + }() + + select { + case completed := <-done: + return completed.Result() + case <-ctx.Done(): + } + + // Cancellation can race command startup. Wait until clicky publishes the + // PID or the command exits, then kill the whole process group so the row's + // timeout remains a hard budget even when the command forks children. + for process.Pid() == 0 { + select { + case completed := <-done: + result := completed.Result() + result.Error = ctx.Err() + return result + default: + runtime.Gosched() + } + } + _ = process.KillTree() + result := (<-done).Result() + result.Error = ctx.Err() + return result +} + +func runWithPTY(ctx context.Context, execBase fixtures.ExecFixtureBase, workDir string) *clickyExec.ExecResult { // Invoke the configured executable directly so shells like bash/sh don't // get double-wrapped (`bash -c "bash -c '