From 554decc8eb18478807c5186af8a1af511a01ebb6 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sat, 5 Sep 2026 20:31:10 +0300 Subject: [PATCH 01/22] feat(ai): add Claude Fable 5.1 and GPT-6 Astra models Register the latest Claude Fable and GPT-6 Astra models with runtime availability, aliases, capabilities, pricing, and generation settings. Update family defaults so Fable and Claude resolve to Claude Fable 5.1. Claude-Session-Id: 01a07294-e289-7182-bf64-b4fdc5ab78d6 --- .../internal/gen-model-registry/patches.json | 9 +++ pkg/ai/model_parse_conformance_test.go | 10 ++- pkg/ai/runtime_selector_test.go | 2 +- .../registry/anthropic_latest_ginkgo_test.go | 54 +++++++++++++++ pkg/api/registry/astra_ginkgo_test.go | 45 +++++++++++++ pkg/api/registry/models.json | 67 +++++++++++++++++++ 6 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 pkg/api/registry/anthropic_latest_ginkgo_test.go create mode 100644 pkg/api/registry/astra_ginkgo_test.go diff --git a/pkg/ai/internal/gen-model-registry/patches.json b/pkg/ai/internal/gen-model-registry/patches.json index cafcce88..0e23cd07 100644 --- a/pkg/ai/internal/gen-model-registry/patches.json +++ b/pkg/ai/internal/gen-model-registry/patches.json @@ -1,4 +1,8 @@ { + "claude-fable-5-1": { + "preferred": true, + "defaultEffort": "high" + }, "claude-opus-5": { "preferred": true }, @@ -17,6 +21,11 @@ "claude-haiku-4-5": { "preferred": true }, + "gpt-6-astra": { + "preferred": true, + "availability": ["api", "cli", "agent", "cmux"], + "aliases": ["astra"] + }, "gpt-5.6": { "preferred": true, "availability": [ diff --git a/pkg/ai/model_parse_conformance_test.go b/pkg/ai/model_parse_conformance_test.go index 6c3856f4..a043188f 100644 --- a/pkg/ai/model_parse_conformance_test.go +++ b/pkg/ai/model_parse_conformance_test.go @@ -58,7 +58,7 @@ var agreedCases = []parseCase{ // Family aliases resolve to the exact current registry model. {in: "sonnet", want: model("claude-sonnet-5", api.Anthropic, api.ModeAgent, "")}, {in: "opus", want: model("claude-opus-5", api.Anthropic, api.ModeAgent, "")}, - {in: "fable", want: model("claude-fable-5", api.Anthropic, api.ModeAgent, "")}, + {in: "fable", want: model("claude-fable-5-1", api.Anthropic, api.ModeAgent, "")}, // A superseded exact id is rewritten to its successor. {in: "claude-sonnet-4-5", want: model("claude-sonnet-4-6", api.Anthropic, api.ModeAgent, "")}, @@ -85,13 +85,17 @@ var agreedCases = []parseCase{ {in: "sol", want: model("gpt-5.6-sol", api.OpenAI, api.ModeAgent, "")}, {in: "terra", want: model("gpt-5.6-terra", api.OpenAI, api.ModeAgent, "")}, {in: "luna", want: model("gpt-5.6-luna", api.OpenAI, api.ModeAgent, "")}, + {in: "astra", want: model("gpt-6-astra", api.OpenAI, api.ModeAgent, "")}, + {in: "api:astra:high", want: model("gpt-6-astra", api.OpenAI, api.ModeAPI, api.EffortHigh)}, + {in: "cli:astra", want: model("gpt-6-astra", api.OpenAI, api.ModeCLI, "")}, + {in: "cmux:astra", want: model("gpt-6-astra", api.OpenAI, api.ModeCmux, "")}, // A bare family sentinel resolves to that family's latest model on the mode // the provider defaults to. It used to be asymmetric — "codex" forced the CLI // and "claude" stayed a literal sentinel — because the name itself carried a // mode. It no longer does, so both now read the same way. {in: "codex", want: model("gpt-5.6-sol", api.OpenAI, api.ModeAgent, "")}, - {in: "claude", want: model("claude-opus-5", api.Anthropic, api.ModeAgent, "")}, + {in: "claude", want: model("claude-fable-5-1", api.Anthropic, api.ModeAgent, "")}, // A sentinel with an explicit mode resolves too. This does NOT go through the // agent-sentinel shortcut (that one only fires off the API mode), so it lands @@ -183,7 +187,7 @@ var _ = Describe("model parse conformance", func() { Expect(err).NotTo(HaveOccurred()) modes := make([]api.RuntimeMode, 0, len(models)) for _, m := range models { - Expect(m.Name).To(Equal("claude-fable-5")) + Expect(m.Name).To(Equal("claude-fable-5-1")) Expect(m.Provider).To(Equal(api.Anthropic)) modes = append(modes, m.Mode) } diff --git a/pkg/ai/runtime_selector_test.go b/pkg/ai/runtime_selector_test.go index d8b5d2fc..ee6765a0 100644 --- a/pkg/ai/runtime_selector_test.go +++ b/pkg/ai/runtime_selector_test.go @@ -72,7 +72,7 @@ func TestResolveRendersTheDriverModelID(t *testing.T) { }{ {"agent:opus-4-8", "claude-opus-4-8"}, {"cmux:claude-opus-4-8", "claude-opus-4-8"}, - {"cli:fable-5", "claude-fable-5"}, + {"cli:fable-5", "claude-fable-5-1"}, {"api:opus-4-8", "claude-opus-4-8"}, {"agent:sonnet", "claude-sonnet-5"}, {"agent:sonnet-4", "claude-sonnet-4-6"}, diff --git a/pkg/api/registry/anthropic_latest_ginkgo_test.go b/pkg/api/registry/anthropic_latest_ginkgo_test.go new file mode 100644 index 00000000..18f1874e --- /dev/null +++ b/pkg/api/registry/anthropic_latest_ginkgo_test.go @@ -0,0 +1,54 @@ +package registry + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Claude Fable 5.1", func() { + const model = "claude-fable-5-1" + + DescribeTable("resolves the latest family on every Claude runtime", func(mode RuntimeMode) { + for _, token := range []string{"fable", model} { + resolved, ok := Anthropic.ResolveExact(mode, token) + Expect(ok).To(BeTrue()) + Expect(resolved).To(Equal(model)) + } + known, available := Anthropic.Availability(mode, model) + Expect(known).To(BeTrue()) + Expect(available).To(BeTrue()) + }, + Entry("API", ModeAPI), + Entry("CLI", ModeCLI), + Entry("agent", ModeAgent), + Entry("cmux", ModeCmux), + ) + + It("publishes the current capabilities and cache price", func() { + entry, ok := Anthropic.Lookup(model) + Expect(ok).To(BeTrue()) + Expect(entry.Preferred).To(BeTrue()) + Expect(entry.ContextWindow).To(Equal(1_000_000)) + Expect(entry.ReleaseDate).To(Equal("2026-09-01")) + Expect(entry.Temperature).To(BeFalse()) + Expect(entry.AdaptiveThinking).To(BeTrue()) + Expect(entry.SupportedEfforts).To(Equal([]Effort{EffortLow, EffortMedium, EffortHigh, EffortXHigh, EffortMax})) + Expect(entry.DefaultEffort).To(Equal(EffortHigh)) + price, ok := CostFor(model) + Expect(ok).To(BeTrue()) + Expect(price).To(Equal(ModelCost{Input: 10, Output: 50, CacheRead: 0.25, CacheWrite: 12.5})) + previousPrice, ok := CostFor("claude-fable-5") + Expect(ok).To(BeTrue()) + Expect(previousPrice).To(Equal(ModelCost{Input: 10, Output: 50, CacheRead: 1, CacheWrite: 12.5})) + }) + + It("uses adaptive thinking and omits unsupported temperature", func() { + temperature := 0.7 + Expect(Anthropic.GenerationConfig(ModeAPI, model, EffortHigh, 4096, &temperature)).To(Equal(map[string]any{ + "max_tokens": 28672, + "thinking": map[string]any{"type": "adaptive"}, + "output_config": map[string]any{"effort": "high"}, + })) + Expect(Anthropic.GenerationConfig(ModeAPI, model, EffortNone, 4096, nil)).To(Equal(map[string]any{"max_tokens": 4096})) + }) +}) diff --git a/pkg/api/registry/astra_ginkgo_test.go b/pkg/api/registry/astra_ginkgo_test.go new file mode 100644 index 00000000..95146951 --- /dev/null +++ b/pkg/api/registry/astra_ginkgo_test.go @@ -0,0 +1,45 @@ +package registry + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("GPT-6 Astra", func() { + const model = "gpt-6-astra" + + DescribeTable("resolves the exact ID and alias on every OpenAI runtime", func(mode RuntimeMode) { + for _, token := range []string{model, "astra", "openai/" + model} { + resolved, ok := OpenAI.ResolveExact(mode, token) + Expect(ok).To(BeTrue()) + Expect(resolved).To(Equal(model)) + } + known, available := OpenAI.Availability(mode, model) + Expect(known).To(BeTrue()) + Expect(available).To(BeTrue()) + }, + Entry("API", ModeAPI), + Entry("CLI", ModeCLI), + Entry("agent", ModeAgent), + Entry("cmux", ModeCmux), + ) + + It("publishes its capabilities and base token prices", func() { + entry, ok := OpenAI.Lookup(model) + Expect(ok).To(BeTrue()) + Expect(entry.Preferred).To(BeTrue()) + Expect(entry.ContextWindow).To(Equal(1_050_000)) + Expect(entry.Temperature).To(BeFalse()) + Expect(entry.SupportedEfforts).To(Equal([]Effort{EffortLow, EffortMedium, EffortHigh, EffortXHigh, EffortMax})) + price, ok := CostFor(model) + Expect(ok).To(BeTrue()) + Expect(price).To(Equal(ModelCost{Input: 10, Output: 50, CacheRead: 1, CacheWrite: 12.5})) + }) + + It("sends reasoning effort without unsupported temperature", func() { + temperature := 0.7 + Expect(OpenAI.GenerationConfig(ModeAPI, model, EffortHigh, 0, &temperature)).To(Equal(map[string]any{ + "reasoning_effort": "high", + })) + }) +}) diff --git a/pkg/api/registry/models.json b/pkg/api/registry/models.json index adbd1297..915be053 100644 --- a/pkg/api/registry/models.json +++ b/pkg/api/registry/models.json @@ -1,4 +1,34 @@ [ + { + "id": "claude-fable-5-1", + "provider": "anthropic", + "family": "fable", + "version": "5.1", + "label": "Claude Fable 5.1", + "releaseDate": "2026-09-01", + "reasoning": true, + "contextWindow": 1000000, + "cost": { + "input": 10, + "output": 50, + "cacheRead": 0.25, + "cacheWrite": 12.5 + }, + "inputMediaTypes": [ + "image/*", + "application/pdf" + ], + "preferred": true, + "adaptiveThinking": true, + "supportedEfforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "defaultEffort": "high" + }, { "id": "claude-opus-5", "provider": "anthropic", @@ -314,6 +344,43 @@ ], "supersededBy": "claude-sonnet-4-6" }, + { + "id": "gpt-6-astra", + "provider": "openai", + "family": "gpt", + "version": "6-astra", + "label": "GPT-6 Astra", + "releaseDate": "2026-09-04", + "reasoning": true, + "contextWindow": 1050000, + "cost": { + "input": 10, + "output": 50, + "cacheRead": 1, + "cacheWrite": 12.5 + }, + "inputMediaTypes": [ + "image/*", + "application/pdf" + ], + "preferred": true, + "availability": [ + "api", + "cli", + "agent", + "cmux" + ], + "supportedEfforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "aliases": [ + "astra" + ] + }, { "id": "gpt-5.6", "provider": "openai", From c08507a5b9121025094d6af1a51d44143d963df3 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sat, 5 Sep 2026 23:25:20 +0300 Subject: [PATCH 02/22] feat(runtime): share profile catalog and raw layers --- pkg/ai/prompt/template_document.go | 17 ++ .../prompt/template_document_ginkgo_test.go | 60 +++++++ pkg/api/runtime_profile_layers_ginkgo_test.go | 46 +++++ pkg/api/runtime_profiles.go | 29 +-- pkg/api/spec_layers.go | 9 +- pkg/cli/runtime_profile_catalog.go | 98 +--------- .../runtime_profile_catalog_ginkgo_test.go | 51 ++++++ pkg/runtimeprofiles/default_catalog.go | 169 ++++++++++++++++++ .../default_catalog_ginkgo_test.go | 155 ++++++++++++++++ pkg/runtimeprofiles/layers_ginkgo_test.go | 34 ++++ pkg/runtimeprofiles/resolve.go | 33 +++- pkg/runtimeprofiles/resolver.go | 37 ++-- pkg/runtimeprofiles/resolver_ginkgo_test.go | 92 ++++++++++ pkg/runtimeprofiles/types.go | 5 +- 14 files changed, 704 insertions(+), 131 deletions(-) create mode 100644 pkg/ai/prompt/template_document.go create mode 100644 pkg/ai/prompt/template_document_ginkgo_test.go create mode 100644 pkg/api/runtime_profile_layers_ginkgo_test.go create mode 100644 pkg/cli/runtime_profile_catalog_ginkgo_test.go create mode 100644 pkg/runtimeprofiles/default_catalog.go create mode 100644 pkg/runtimeprofiles/default_catalog_ginkgo_test.go create mode 100644 pkg/runtimeprofiles/layers_ginkgo_test.go create mode 100644 pkg/runtimeprofiles/resolver_ginkgo_test.go diff --git a/pkg/ai/prompt/template_document.go b/pkg/ai/prompt/template_document.go new file mode 100644 index 00000000..21d4b143 --- /dev/null +++ b/pkg/ai/prompt/template_document.go @@ -0,0 +1,17 @@ +package prompt + +import "fmt" + +// Document renders frontmatter and returns its authored metadata and raw body. +// Model selectors remain as declared; provider and mode defaults are not applied. +func (t *Template) Document(data map[string]any) (*Document, error) { + source, err := renderFrontmatter(t.source, data) + if err != nil { + return nil, fmt.Errorf("render prompt %s frontmatter: %w", t.name, err) + } + document, err := Parse(source) + if err != nil { + return nil, fmt.Errorf("parse prompt %s: %w", t.name, err) + } + return document, nil +} diff --git a/pkg/ai/prompt/template_document_ginkgo_test.go b/pkg/ai/prompt/template_document_ginkgo_test.go new file mode 100644 index 00000000..9e251c6b --- /dev/null +++ b/pkg/ai/prompt/template_document_ginkgo_test.go @@ -0,0 +1,60 @@ +package prompt + +import ( + "testing" + "testing/fstest" + + "github.com/flanksource/captain/pkg/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPromptDocuments(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Prompt documents") +} + +var _ = Describe("Template document", func() { + DescribeTable("preserves rendered authored metadata without resolving driver defaults", + func(frontmatter string, data map[string]any, model api.Model) { + const body = "{{role \"user\"}}\nReview {{name}}\n" + document, err := Load("---\n" + frontmatter + "---\n" + body).Document(data) + Expect(err).NotTo(HaveOccurred()) + Expect(document.RuntimeProfile).To(Equal("reviewer")) + Expect(document.Spec).To(Equal(api.Spec{Model: model})) + Expect(document.Body).To(Equal(body)) + Expect(document.Frontmatter).To(HaveKeyWithValue("runtimeProfile", "reviewer")) + Expect(document.Frontmatter).To(HaveKeyWithValue("model", model.Name)) + }, + Entry("static fields with no authored mode", "runtimeProfile: reviewer\nmodel: sonnet\n", nil, api.Model{Name: "sonnet"}), + Entry("static compact selector", "runtimeProfile: reviewer\nmodel: agent:sonnet\n", nil, api.Model{Name: "agent:sonnet"}), + Entry("templated profile, model and mode", "runtimeProfile: {{profile}}\nmodel: {{model}}\nmode: {{mode}}\n", + map[string]any{"profile": "reviewer", "model": "sonnet", "mode": "cli", "name": "a change"}, + api.Model{Name: "sonnet", Mode: api.ModeCLI}), + Entry("unknown catalog model remains authored", "runtimeProfile: reviewer\nmodel: tenant-review-model\nmode: agent\n", + nil, api.Model{Name: "tenant-review-model", Mode: api.ModeAgent}), + ) + + It("returns a body-only prompt without introducing metadata", func() { + const body = "Review {{name}}\n" + document, err := Load(body).Document(map[string]any{"name": "a change"}) + Expect(err).NotTo(HaveOccurred()) + Expect(document).To(Equal(&Document{Body: body})) + }) + + DescribeTable("reports malformed frontmatter with its source name", + func(frontmatter string, data map[string]any, message string) { + const name = "prompts/review.prompt" + template, err := LoadFS(fstest.MapFS{name: {Data: []byte("---\n" + frontmatter + "---\nbody\n")}}, name) + Expect(err).NotTo(HaveOccurred()) + document, err := template.Document(data) + Expect(document).To(BeNil()) + Expect(err).To(MatchError(ContainSubstring(name))) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("invalid YAML", "model: [unfinished\n", nil, "parse prompt frontmatter"), + Entry("invalid template", "{{#if enabled}}\nmodel: sonnet\n", nil, "frontmatter template"), + Entry("invalid rendered YAML", "model: {{model}}\n", map[string]any{"model": "[unfinished"}, "parse prompt frontmatter"), + Entry("invalid rendered profile", "runtimeProfile: {{profile}}\n", map[string]any{"profile": 42}, "runtimeProfile must be a string"), + ) +}) diff --git a/pkg/api/runtime_profile_layers_ginkgo_test.go b/pkg/api/runtime_profile_layers_ginkgo_test.go new file mode 100644 index 00000000..5b1674d3 --- /dev/null +++ b/pkg/api/runtime_profile_layers_ginkgo_test.go @@ -0,0 +1,46 @@ +package api_test + +import ( + "github.com/flanksource/captain/pkg/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Runtime profile layers", func() { + It("preserves authored models and reference order without resolving or merging", func() { + layers, err := api.RuntimeProfileLayers(api.RuntimeProfileResolveRequest{ + Profile: api.RuntimeProfile{ + ID: "review", Name: "Review", Presets: []string{"Personal", "organization"}, + Spec: api.Spec{Budget: api.Budget{MaxTurns: 5}, Permissions: api.Permissions{Mode: api.PermissionDontAsk}}, + }, + Presets: []api.RuntimePreset{ + {ID: "organization", Name: "Organization", Scope: api.SpecLayerGlobal, Spec: api.RuntimePresetSpec{ + Model: api.Model{Name: "gpt-5", Mode: api.ModeAgent}, Budget: api.Budget{MaxTurns: 20}, + }}, + {ID: "personal", Name: "Personal", Scope: api.SpecLayerUser, Spec: api.RuntimePresetSpec{ + Budget: api.Budget{MaxTurns: 3}, + }}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(layers).To(Equal([]api.SpecLayer{ + {ID: "personal", Name: "Personal", Scope: api.SpecLayerUser, Source: api.SpecLayerSourcePreset, + Spec: api.Spec{Budget: api.Budget{MaxTurns: 3}}}, + {ID: "organization", Name: "Organization", Scope: api.SpecLayerGlobal, Source: api.SpecLayerSourcePreset, + Spec: api.Spec{Model: api.Model{Name: "gpt-5", Mode: api.ModeAgent}, Budget: api.Budget{MaxTurns: 20}}}, + {ID: "review:spec", Name: "Review run spec", Scope: api.SpecLayerSurface, Source: api.SpecLayerSourceProfile, + Spec: api.Spec{Budget: api.Budget{MaxTurns: 5}, Permissions: api.Permissions{Mode: api.PermissionDontAsk}}}, + })) + }) + + It("retains a permission-only profile without guessing a model", func() { + layers, err := api.RuntimeProfileLayers(api.RuntimeProfileResolveRequest{Profile: api.RuntimeProfile{ + ID: "plan", Name: "Plan", Spec: api.Spec{Permissions: api.Permissions{Mode: api.PermissionPlan}}, + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(layers).To(Equal([]api.SpecLayer{{ + ID: "plan:spec", Name: "Plan run spec", Scope: api.SpecLayerSurface, Source: api.SpecLayerSourceProfile, + Spec: api.Spec{Permissions: api.Permissions{Mode: api.PermissionPlan}}, + }})) + }) +}) diff --git a/pkg/api/runtime_profiles.go b/pkg/api/runtime_profiles.go index 6688ade6..0d1c6287 100644 --- a/pkg/api/runtime_profiles.go +++ b/pkg/api/runtime_profiles.go @@ -71,17 +71,15 @@ type RuntimeProfileResolveResponse struct { EffectivePolicy PermissionPolicy `json:"effectivePolicy"` } -// ResolveRuntimeProfile materializes selected presets, adds the task-specific -// profile spec, and delegates ordering and structural merge semantics to the -// canonical Spec layer resolver. A profile references its presets by id or by -// a name that matches exactly one preset, the runtime catalog's convention. -func ResolveRuntimeProfile(request RuntimeProfileResolveRequest) (ResolvedSpec, error) { +// RuntimeProfileLayers materializes selected presets and the profile spec in +// reference order. Runtime resolution waits until the host adds its other layers. +func RuntimeProfileLayers(request RuntimeProfileResolveRequest) ([]SpecLayer, error) { if err := validateRuntimeProfile(request.Profile); err != nil { - return ResolvedSpec{}, err + return nil, err } index, err := indexRuntimePresets(request.Presets) if err != nil { - return ResolvedSpec{}, err + return nil, err } layers := make([]SpecLayer, 0, len(request.Profile.Presets)+1) @@ -89,10 +87,10 @@ func ResolveRuntimeProfile(request RuntimeProfileResolveRequest) (ResolvedSpec, for _, ref := range request.Profile.Presets { preset, err := index.lookup(request.Profile.Name, ref) if err != nil { - return ResolvedSpec{}, err + return nil, err } if _, repeated := selected[preset.ID]; repeated { - return ResolvedSpec{}, fmt.Errorf("runtime profile %q repeats preset %q", request.Profile.Name, ref) + return nil, fmt.Errorf("runtime profile %q repeats preset %q", request.Profile.Name, ref) } selected[preset.ID] = struct{}{} layers = append(layers, SpecLayer{ @@ -100,10 +98,19 @@ func ResolveRuntimeProfile(request RuntimeProfileResolveRequest) (ResolvedSpec, Name: preset.Name, Scope: preset.Scope, Spec: preset.Spec.ToSpec(), }) } - layers = append(layers, SpecLayer{ + return append(layers, SpecLayer{ ID: request.Profile.ID + ":spec", Source: SpecLayerSourceProfile, Name: request.Profile.Name + " run spec", Scope: SpecLayerSurface, Spec: request.Profile.Spec, - }) + }), nil +} + +// ResolveRuntimeProfile resolves and validates a profile in isolation for preview. +// Hosts composing a run use RuntimeProfileLayers before adding their other layers. +func ResolveRuntimeProfile(request RuntimeProfileResolveRequest) (ResolvedSpec, error) { + layers, err := RuntimeProfileLayers(request) + if err != nil { + return ResolvedSpec{}, err + } resolved, err := ResolveSpecLayers(layers...) if err != nil { return ResolvedSpec{}, fmt.Errorf("resolve runtime profile %q: %w", request.Profile.Name, err) diff --git a/pkg/api/spec_layers.go b/pkg/api/spec_layers.go index af34063b..431c5a82 100644 --- a/pkg/api/spec_layers.go +++ b/pkg/api/spec_layers.go @@ -81,13 +81,18 @@ func RequestSpecLayer(name string, spec Spec) SpecLayer { return SpecLayer{Name: name, Source: SpecLayerSourceRequest, Scope: SpecLayerUser, Spec: spec} } -// ResolveSpecLayers deterministically overlays defaults and intersects constraints. -func ResolveSpecLayers(input ...SpecLayer) (ResolvedSpec, error) { +// OrderSpecLayers copies the stack into effective scope order, preserving ties. +func OrderSpecLayers(input ...SpecLayer) []SpecLayer { layers := append([]SpecLayer(nil), input...) slices.SortStableFunc(layers, func(left, right SpecLayer) int { return scopeRank(left.Scope) - scopeRank(right.Scope) }) + return layers +} +// ResolveSpecLayers deterministically overlays defaults and intersects constraints. +func ResolveSpecLayers(input ...SpecLayer) (ResolvedSpec, error) { + layers := OrderSpecLayers(input...) resolved := ResolvedSpec{Trace: make([]SpecLayer, 0, len(layers))} for _, layer := range layers { if err := validateSpecLayer(layer); err != nil { diff --git a/pkg/cli/runtime_profile_catalog.go b/pkg/cli/runtime_profile_catalog.go index fe6a67fe..f4d53995 100644 --- a/pkg/cli/runtime_profile_catalog.go +++ b/pkg/cli/runtime_profile_catalog.go @@ -3,12 +3,8 @@ package cli import ( "context" "errors" - "fmt" "net/http" - "os" - "path/filepath" - "github.com/flanksource/captain/pkg/captainconfig" "github.com/flanksource/captain/pkg/runtimeprofiles" "github.com/flanksource/clicky/entity" ) @@ -28,101 +24,9 @@ func buildRuntimeCatalog(ctx context.Context) (*runtimeprofiles.Catalog, error) if catalog, ok := ctx.Value(runtimeCatalogContextKey{}).(*runtimeprofiles.Catalog); ok && catalog != nil { return catalog, nil } - dbSource, err := runtimeprofiles.NewDBSource(runtimeprofiles.DBSourceOptions{ + return runtimeprofiles.NewDefaultCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{ Read: captainDB, Write: captainDefaultDB, }) - if err != nil { - return nil, err - } - dirs, err := runtimeRecordDirs() - if err != nil { - return nil, err - } - sources := []runtimeprofiles.Source{dbSource} - for _, dir := range dirs { - source, err := runtimeprofiles.NewFileSource(runtimeprofiles.FileSourceOptions{ - Kind: dir.kind, Dir: dir.path, Label: dir.path, Implicit: dir.implicit, - }) - if err != nil { - return nil, err - } - sources = append(sources, source) - } - return runtimeprofiles.NewCatalog(sources...) -} - -type runtimeRecordDir struct { - kind runtimeprofiles.Kind - path string - implicit bool -} - -func runtimeRecordDirs() ([]runtimeRecordDir, error) { - var dirs []runtimeRecordDir - seen := map[string]bool{} - add := func(kind runtimeprofiles.Kind, path string, implicit bool) { - key := string(kind) + ":" + path - if seen[key] { - return - } - seen[key] = true - dirs = append(dirs, runtimeRecordDir{kind: kind, path: path, implicit: implicit}) - } - configHome, err := captainConfigHome() - if err != nil { - return nil, err - } - add(runtimeprofiles.KindPreset, filepath.Join(configHome, "presets"), true) - add(runtimeprofiles.KindProfile, filepath.Join(configHome, "profiles"), true) - if err := addConfiguredRuntimeDirs(add); err != nil { - return nil, err - } - cwd, err := os.Getwd() - if err != nil { - return nil, err - } - add(runtimeprofiles.KindPreset, filepath.Join(cwd, ".captain", "presets"), true) - add(runtimeprofiles.KindProfile, filepath.Join(cwd, ".captain", "profiles"), true) - return dirs, nil -} - -func addConfiguredRuntimeDirs(add func(runtimeprofiles.Kind, string, bool)) error { - cfg, exists, err := captainconfig.Load() - if err != nil || !exists { - return err - } - configPath, err := captainconfig.Path() - if err != nil { - return err - } - base := filepath.Dir(configPath) - for _, raw := range cfg.Runtime.PresetDirs { - dir, err := resolvePromptDir(raw, base) - if err != nil { - return fmt.Errorf("runtime.presetDirs: %w", err) - } - add(runtimeprofiles.KindPreset, dir, false) - } - for _, raw := range cfg.Runtime.ProfileDirs { - dir, err := resolvePromptDir(raw, base) - if err != nil { - return fmt.Errorf("runtime.profileDirs: %w", err) - } - add(runtimeprofiles.KindProfile, dir, false) - } - return nil -} - -// captainConfigHome is $XDG_CONFIG_HOME/captain, defaulting to ~/.config/captain. -func captainConfigHome() (string, error) { - if base := os.Getenv("XDG_CONFIG_HOME"); base != "" { - return filepath.Join(base, "captain"), nil - } - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - return filepath.Join(home, ".config", "captain"), nil } // runtimeCatalogError maps catalog failures onto HTTP statuses for the entity diff --git a/pkg/cli/runtime_profile_catalog_ginkgo_test.go b/pkg/cli/runtime_profile_catalog_ginkgo_test.go new file mode 100644 index 00000000..26079e72 --- /dev/null +++ b/pkg/cli/runtime_profile_catalog_ginkgo_test.go @@ -0,0 +1,51 @@ +package cli + +import ( + "os" + "path/filepath" + + "github.com/flanksource/captain/pkg/captainconfig" + "github.com/flanksource/captain/pkg/runtimeprofiles" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CLI runtime catalog discovery", func() { + var configPath string + + BeforeEach(func() { + home := GinkgoT().TempDir() + GinkgoT().Setenv("HOME", home) + GinkgoT().Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + GinkgoT().Chdir(GinkgoT().TempDir()) + configPath = filepath.Join(home, ".captain.yaml") + captainconfig.SetPathForTesting(configPath) + DeferCleanup(captainconfig.SetPathForTesting, "") + }) + + It("preserves the supplied catalog without reading user config", func() { + fixture := newRuntimeEntityFixture() + Expect(os.WriteFile(configPath, []byte("runtime: [malformed\n"), 0o600)).To(Succeed()) + Expect(buildRuntimeCatalog(fixture.ctx)).To(BeIdenticalTo(fixture.catalog)) + }) + + It("uses the shared discovery sources with the CLI database registered first", func(ctx SpecContext) { + expected, err := runtimeprofiles.NewDefaultCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{ + Read: captainDB, Write: captainDefaultDB, + }) + Expect(err).NotTo(HaveOccurred()) + actual, err := buildRuntimeCatalog(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(actual.Sources()).To(Equal(expected.Sources())) + Expect(actual.Sources()).To(HaveLen(5)) + Expect(actual.Sources()[0].Kind).To(Equal(runtimeprofiles.SourceDB)) + }) + + It("reports invalid configured discovery directories", func(ctx SpecContext) { + Expect(os.WriteFile(configPath, []byte("runtime:\n profileDirs: [missing]\n"), 0o600)).To(Succeed()) + _, err := buildRuntimeCatalog(ctx) + Expect(err).To(MatchError(ContainSubstring("runtime.profileDirs"))) + Expect(err).To(MatchError(ContainSubstring("missing"))) + }) +}) diff --git a/pkg/runtimeprofiles/default_catalog.go b/pkg/runtimeprofiles/default_catalog.go new file mode 100644 index 00000000..2541208d --- /dev/null +++ b/pkg/runtimeprofiles/default_catalog.go @@ -0,0 +1,169 @@ +package runtimeprofiles + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/flanksource/captain/pkg/captainconfig" + "github.com/flanksource/captain/pkg/database" +) + +// DefaultCatalogOptions supplies a host's database, working directory and config. +type DefaultCatalogOptions struct { + // Read and Write are lazy database openers. Omit both for file-only catalogs; + // omitting only Write keeps the database read-only. + Read func(context.Context) (*database.DB, error) + Write func(context.Context) (*database.DB, error) + // Cwd is the absolute repository directory; empty uses the process directory. + Cwd string + // Config avoids reloading ~/.captain.yaml when the host already loaded it. + // Relative runtime directories still resolve against captainconfig.Path(). + Config *captainconfig.Config +} + +// NewDefaultCatalog discovers the database, user, configured and repo sources. +// Database openers run only when records are read or written. +func NewDefaultCatalog(ctx context.Context, options DefaultCatalogOptions) (*Catalog, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var sources []Source + if options.Read != nil || options.Write != nil { + source, err := NewDBSource(DBSourceOptions{Read: options.Read, Write: options.Write}) + if err != nil { + return nil, err + } + sources = append(sources, source) + } + dirs, err := runtimeRecordDirs(options) + if err != nil { + return nil, err + } + for _, dir := range dirs { + source, err := NewFileSource(dir) + if err != nil { + return nil, err + } + sources = append(sources, source) + } + return NewCatalog(sources...) +} + +func runtimeRecordDirs(options DefaultCatalogOptions) ([]FileSourceOptions, error) { + var dirs []FileSourceOptions + seen := map[string]bool{} + add := func(kind Kind, path string, implicit bool) { + key := string(kind) + ":" + path + if seen[key] { + return + } + seen[key] = true + dirs = append(dirs, FileSourceOptions{Kind: kind, Dir: path, Label: path, Implicit: implicit}) + } + configHome, err := captainConfigHome() + if err != nil { + return nil, err + } + add(KindPreset, filepath.Join(configHome, "presets"), true) + add(KindProfile, filepath.Join(configHome, "profiles"), true) + if err := addConfiguredRuntimeDirs(options.Config, add); err != nil { + return nil, err + } + cwd := options.Cwd + if cwd == "" { + cwd, err = os.Getwd() + if err != nil { + return nil, err + } + } + add(KindPreset, filepath.Join(cwd, ".captain", "presets"), true) + add(KindProfile, filepath.Join(cwd, ".captain", "profiles"), true) + return dirs, nil +} + +func addConfiguredRuntimeDirs(cfg *captainconfig.Config, add func(Kind, string, bool)) error { + if cfg == nil { + loaded, _, err := captainconfig.Load() + if err != nil { + return err + } + cfg = &loaded + } + if cfg.Runtime.IsZero() { + return nil + } + configPath, err := captainconfig.Path() + if err != nil { + return err + } + for _, entry := range []struct { + kind Kind + dirs []string + }{ + {KindPreset, cfg.Runtime.PresetDirs}, + {KindProfile, cfg.Runtime.ProfileDirs}, + } { + for _, raw := range entry.dirs { + dir, err := resolveRuntimeDir(raw, filepath.Dir(configPath)) + if err != nil { + return fmt.Errorf("runtime.%sDirs: %w", entry.kind, err) + } + add(entry.kind, dir, false) + } + } + return nil +} + +func resolveRuntimeDir(raw, base string) (string, error) { + dir := strings.TrimSpace(raw) + if dir == "" { + return "", fmt.Errorf("runtime dir cannot be empty") + } + if strings.HasPrefix(dir, "~") { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + switch { + case dir == "~": + dir = home + case strings.HasPrefix(dir, "~/"): + dir = filepath.Join(home, dir[2:]) + default: + return "", fmt.Errorf("unsupported home-relative runtime dir %q", raw) + } + } + if !filepath.IsAbs(dir) { + dir = filepath.Join(base, dir) + } + abs, err := filepath.Abs(dir) + if err != nil { + return "", err + } + info, err := os.Stat(abs) + if err != nil { + return "", fmt.Errorf("runtime dir %s: %w", abs, err) + } + if !info.IsDir() { + return "", fmt.Errorf("runtime dir %s is not a directory", abs) + } + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + return "", fmt.Errorf("resolve runtime dir %s: %w", abs, err) + } + return filepath.Clean(resolved), nil +} + +func captainConfigHome() (string, error) { + if base := os.Getenv("XDG_CONFIG_HOME"); base != "" { + return filepath.Join(base, "captain"), nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, ".config", "captain"), nil +} diff --git a/pkg/runtimeprofiles/default_catalog_ginkgo_test.go b/pkg/runtimeprofiles/default_catalog_ginkgo_test.go new file mode 100644 index 00000000..1f3cab11 --- /dev/null +++ b/pkg/runtimeprofiles/default_catalog_ginkgo_test.go @@ -0,0 +1,155 @@ +package runtimeprofiles + +import ( + "context" + "errors" + "os" + "path/filepath" + + "github.com/flanksource/captain/pkg/captainconfig" + "github.com/flanksource/captain/pkg/database" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Default runtime catalog", func() { + var home, cwd, configHome string + + BeforeEach(func() { + home = GinkgoT().TempDir() + cwd = GinkgoT().TempDir() + configHome = filepath.Join(home, "xdg", "captain") + GinkgoT().Setenv("HOME", home) + GinkgoT().Setenv("XDG_CONFIG_HOME", filepath.Dir(configHome)) + captainconfig.SetPathForTesting(filepath.Join(home, ".captain.yaml")) + DeferCleanup(captainconfig.SetPathForTesting, "") + }) + + It("discovers user and explicit working-directory records without opening a database", func(ctx SpecContext) { + writeRecordFile(filepath.Join(configHome, "presets"), "personal.yaml", "name: Personal\nscope: user\n") + writeRecordFile(filepath.Join(cwd, ".captain", "profiles"), "review.yaml", "name: Review\npresets: [Personal]\n") + catalog, err := NewDefaultCatalog(ctx, DefaultCatalogOptions{Cwd: cwd}) + Expect(err).NotTo(HaveOccurred()) + Expect(catalog.Sources()).To(Equal([]SourceInfo{ + defaultFileInfo(KindPreset, filepath.Join(configHome, "presets"), true), + defaultFileInfo(KindProfile, filepath.Join(configHome, "profiles"), true), + defaultFileInfo(KindPreset, filepath.Join(cwd, ".captain", "presets"), true), + defaultFileInfo(KindProfile, filepath.Join(cwd, ".captain", "profiles"), true), + })) + resolution, err := catalog.Resolve(ctx, "Review") + Expect(err).NotTo(HaveOccurred()) + Expect(resolution.Profile.Source.Root).To(Equal(filepath.Join(cwd, ".captain", "profiles"))) + Expect(resolution.Presets).To(HaveLen(1)) + Expect(resolution.Presets[0].Name).To(Equal("Personal")) + }) + + It("uses the process directory and standard user config directory when omitted", func(ctx SpecContext) { + GinkgoT().Setenv("XDG_CONFIG_HOME", "") + GinkgoT().Chdir(cwd) + actualCwd, err := os.Getwd() + Expect(err).NotTo(HaveOccurred()) + catalog, err := NewDefaultCatalog(ctx, DefaultCatalogOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(catalog.Sources()).To(Equal([]SourceInfo{ + defaultFileInfo(KindPreset, filepath.Join(home, ".config", "captain", "presets"), true), + defaultFileInfo(KindProfile, filepath.Join(home, ".config", "captain", "profiles"), true), + defaultFileInfo(KindPreset, filepath.Join(actualCwd, ".captain", "presets"), true), + defaultFileInfo(KindProfile, filepath.Join(actualCwd, ".captain", "profiles"), true), + })) + }) + + It("loads configured directories relative to the config file and expands home paths", func(ctx SpecContext) { + configDir := filepath.Join(home, "settings") + captainconfig.SetPathForTesting(filepath.Join(configDir, "captain.yaml")) + writeRecordFile(configDir, "captain.yaml", "runtime:\n presetDirs: [presets]\n profileDirs: [~/profiles]\n") + writeRecordFile(filepath.Join(configDir, "presets"), "shared.yaml", "name: Shared\nscope: global\n") + writeRecordFile(filepath.Join(home, "profiles"), "review.yaml", "name: Review\npresets: [Shared]\n") + presets, err := filepath.EvalSymlinks(filepath.Join(configDir, "presets")) + Expect(err).NotTo(HaveOccurred()) + profiles, err := filepath.EvalSymlinks(filepath.Join(home, "profiles")) + Expect(err).NotTo(HaveOccurred()) + catalog, err := NewDefaultCatalog(ctx, DefaultCatalogOptions{Cwd: cwd}) + Expect(err).NotTo(HaveOccurred()) + Expect(catalog.Sources()[2:4]).To(Equal([]SourceInfo{ + defaultFileInfo(KindPreset, presets, false), + defaultFileInfo(KindProfile, profiles, false), + })) + Expect(catalog.Resolve(ctx, "Review")).Error().NotTo(HaveOccurred()) + }) + + It("uses supplied config and deduplicates configured directory aliases", func(ctx SpecContext) { + writeRecordFile(home, ".captain.yaml", "runtime: [malformed\n") + presets := filepath.Join(home, "presets") + writeRecordFile(presets, "shared.yaml", "name: Shared\nscope: global\n") + alias := filepath.Join(home, "alias") + Expect(os.Symlink(presets, alias)).To(Succeed()) + cfg := captainconfig.Config{Runtime: captainconfig.RuntimeDefaults{PresetDirs: []string{presets, alias}}} + catalog, err := NewDefaultCatalog(ctx, DefaultCatalogOptions{Cwd: cwd, Config: &cfg}) + Expect(err).NotTo(HaveOccurred()) + Expect(catalog.Sources()).To(HaveLen(5)) + Expect(catalog.ListPresets(ctx)).To(HaveLen(1)) + }) + + It("registers the database first and defers opener failures until records are read", func(ctx SpecContext) { + openerErr := errors.New("database unavailable") + calls := 0 + opener := func(context.Context) (*database.DB, error) { + calls++ + return nil, openerErr + } + catalog, err := NewDefaultCatalog(ctx, DefaultCatalogOptions{Cwd: cwd, Read: opener, Write: opener}) + Expect(err).NotTo(HaveOccurred()) + Expect(calls).To(BeZero()) + Expect(catalog.Sources()[0]).To(Equal(SourceInfo{ + Kind: SourceDB, ID: DBSourceID, Label: "Database", Writable: true, Records: []Kind{KindPreset, KindProfile}, + })) + _, err = catalog.ListProfiles(ctx) + Expect(err).To(MatchError(openerErr)) + Expect(calls).To(Equal(1)) + }) + + It("rejects a write opener without a read opener", func(ctx SpecContext) { + _, err := NewDefaultCatalog(ctx, DefaultCatalogOptions{Cwd: cwd, Write: sameDB(nil)}) + Expect(err).To(MatchError(ContainSubstring("Read opener"))) + }) + + It("reports malformed loaded configuration", func(ctx SpecContext) { + writeRecordFile(home, ".captain.yaml", "runtime: [malformed\n") + _, err := NewDefaultCatalog(ctx, DefaultCatalogOptions{Cwd: cwd}) + Expect(err).To(MatchError(ContainSubstring("parse " + filepath.Join(home, ".captain.yaml")))) + }) + + DescribeTable("rejects invalid configured directories", + func(ctx SpecContext, raw, message string) { + cfg := captainconfig.Config{Runtime: captainconfig.RuntimeDefaults{ProfileDirs: []string{raw}}} + _, err := NewDefaultCatalog(ctx, DefaultCatalogOptions{Cwd: cwd, Config: &cfg}) + Expect(err).To(MatchError(ContainSubstring("runtime.profileDirs"))) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("empty", " ", "cannot be empty"), + Entry("missing", "missing", "no such file or directory"), + Entry("unsupported home alias", "~someone/profiles", "unsupported home-relative"), + ) + + It("rejects a configured path that is a file", func(ctx SpecContext) { + writeRecordFile(home, "profiles", "not a directory") + cfg := captainconfig.Config{Runtime: captainconfig.RuntimeDefaults{ProfileDirs: []string{"profiles"}}} + _, err := NewDefaultCatalog(ctx, DefaultCatalogOptions{Cwd: cwd, Config: &cfg}) + Expect(err).To(MatchError(ContainSubstring("is not a directory"))) + }) + + It("refuses canceled catalog construction", func(ctx SpecContext) { + canceled, cancel := context.WithCancel(ctx) + cancel() + _, err := NewDefaultCatalog(canceled, DefaultCatalogOptions{Cwd: cwd}) + Expect(err).To(MatchError(context.Canceled)) + }) +}) + +func defaultFileInfo(kind Kind, dir string, implicit bool) SourceInfo { + return SourceInfo{ + Kind: SourceFile, ID: hashDir(dir), Label: dir, Root: dir, Writable: true, + Implicit: implicit, Records: []Kind{kind}, + } +} diff --git a/pkg/runtimeprofiles/layers_ginkgo_test.go b/pkg/runtimeprofiles/layers_ginkgo_test.go new file mode 100644 index 00000000..9b2bcc4f --- /dev/null +++ b/pkg/runtimeprofiles/layers_ginkgo_test.go @@ -0,0 +1,34 @@ +package runtimeprofiles + +import ( + "github.com/flanksource/captain/pkg/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Catalog layers", func() { + It("canonicalises preset names without validating the profile's isolated runtime", func(ctx SpecContext) { + source := newMemSource("db", SourceDB, true) + preset := source.presets.put("model", globalPreset("Model")) + profile := source.profiles.put("review", ProfileInput{ + Name: "Review", Presets: []string{"model"}, + Spec: api.Spec{Permissions: api.Permissions{Mode: api.PermissionDontAsk}}, + }) + catalog, err := NewCatalog(source) + Expect(err).NotTo(HaveOccurred()) + + resolution, err := catalog.Layers(ctx, profile.ID) + Expect(err).NotTo(HaveOccurred()) + profile.Presets = []string{preset.ID} + Expect(resolution).To(Equal(Resolution{ + Profile: profile, Presets: []Preset{preset}, Layers: []api.SpecLayer{ + {ID: preset.ID, Name: "Model", Scope: api.SpecLayerGlobal, Source: api.SpecLayerSourcePreset, + Spec: api.Spec{Model: api.Model{Name: "gpt-5", Mode: api.ModeAgent}}}, + {ID: profile.ID + ":spec", Name: "Review run spec", Scope: api.SpecLayerSurface, Source: api.SpecLayerSourceProfile, + Spec: api.Spec{Permissions: api.Permissions{Mode: api.PermissionDontAsk}}}, + }, + })) + _, err = catalog.Resolve(ctx, profile.ID) + Expect(err).To(MatchError(ContainSubstring(`permissions.mode "dontAsk" is not available for openai agent`))) + }) +}) diff --git a/pkg/runtimeprofiles/resolve.go b/pkg/runtimeprofiles/resolve.go index aa0d2e43..1a07b800 100644 --- a/pkg/runtimeprofiles/resolve.go +++ b/pkg/runtimeprofiles/resolve.go @@ -7,12 +7,9 @@ import ( "github.com/flanksource/captain/pkg/api" ) -// Resolve loads the profile, every preset it references (by id or name, from -// any source), canonicalises the references to encoded ids and materialises the -// spec through api.ResolveRuntimeProfile. A reference that resolves nowhere is -// an error naming the profile and the reference, never a silently skipped -// layer. -func (c *Catalog) Resolve(ctx context.Context, ref string) (Resolution, error) { +// Layers loads the profile and its presets, canonicalises references to ids, +// and returns authored layers for a host to combine with the rest of a run. +func (c *Catalog) Layers(ctx context.Context, ref string) (Resolution, error) { profile, err := c.GetProfile(ctx, ref) if err != nil { return Resolution{}, err @@ -30,11 +27,31 @@ func (c *Catalog) Resolve(ctx context.Context, ref string) (Resolution, error) { ids = append(ids, preset.ID) } profile.Presets = ids - resolved, err := api.ResolveRuntimeProfile(api.RuntimeProfileResolveRequest{ + layers, err := api.RuntimeProfileLayers(api.RuntimeProfileResolveRequest{ Profile: profile.API(), Presets: apiPresets, }) if err != nil { return Resolution{}, err } - return Resolution{Profile: profile, Presets: presets, Resolved: resolved}, nil + return Resolution{Profile: profile, Presets: presets, Layers: layers}, nil +} + +// Resolve resolves and validates a profile in isolation for preview. +func (c *Catalog) Resolve(ctx context.Context, ref string) (Resolution, error) { + resolution, err := c.Layers(ctx, ref) + if err != nil { + return Resolution{}, err + } + presets := make([]api.RuntimePreset, 0, len(resolution.Presets)) + for _, preset := range resolution.Presets { + presets = append(presets, preset.API()) + } + resolved, err := api.ResolveRuntimeProfile(api.RuntimeProfileResolveRequest{ + Profile: resolution.Profile.API(), Presets: presets, + }) + if err != nil { + return Resolution{}, err + } + resolution.Resolved = resolved + return resolution, nil } diff --git a/pkg/runtimeprofiles/resolver.go b/pkg/runtimeprofiles/resolver.go index 09c5c777..3d7773ff 100644 --- a/pkg/runtimeprofiles/resolver.go +++ b/pkg/runtimeprofiles/resolver.go @@ -63,39 +63,54 @@ type ResolveResult struct { Resolved api.ResolvedSpec } -// Resolve selects one profile and resolves every layer exactly once. -func (r *Resolver) Resolve(ctx context.Context, options ResolveOptions) (ResolveResult, error) { +// LayerResult retains the selected catalog records and unresolved layer stack. +type LayerResult struct { + Profile *Resolution + Layers []api.SpecLayer +} + +// Layers selects one profile and assembles its layers without resolving a model. +func (r *Resolver) Layers(ctx context.Context, options ResolveOptions) (LayerResult, error) { if r == nil { - return ResolveResult{}, fmt.Errorf("runtime profile resolver is required") + return LayerResult{}, fmt.Errorf("runtime profile resolver is required") } ref, origin := selectProfile(options) layers := append([]api.SpecLayer(nil), options.BaseLayers...) var profile *Resolution if ref != "" { if r.catalog == nil { - return ResolveResult{}, &SelectionError{Origin: origin, Ref: ref, Err: ErrCatalogUnavailable} + return LayerResult{}, &SelectionError{Origin: origin, Ref: ref, Err: ErrCatalogUnavailable} } catalog, err := r.catalog(ctx) if err != nil { - return ResolveResult{}, &SelectionError{Origin: origin, Ref: ref, Err: err} + return LayerResult{}, &SelectionError{Origin: origin, Ref: ref, Err: err} } if catalog == nil { - return ResolveResult{}, &SelectionError{Origin: origin, Ref: ref, Err: ErrCatalogUnavailable} + return LayerResult{}, &SelectionError{Origin: origin, Ref: ref, Err: ErrCatalogUnavailable} } - resolution, err := catalog.Resolve(ctx, ref) + resolution, err := catalog.Layers(ctx, ref) if err != nil { - return ResolveResult{}, &SelectionError{Origin: origin, Ref: ref, Err: err} + return LayerResult{}, &SelectionError{Origin: origin, Ref: ref, Err: err} } profile = &resolution - layers = append(layers, resolution.Resolved.Trace...) + layers = append(layers, resolution.Layers...) } layers = append(layers, options.SurfaceLayers...) layers = append(layers, options.RequestLayers...) - resolved, err := api.ResolveSpecLayers(layers...) + return LayerResult{Profile: profile, Layers: api.OrderSpecLayers(layers...)}, nil +} + +// Resolve selects one profile and resolves every layer exactly once. +func (r *Resolver) Resolve(ctx context.Context, options ResolveOptions) (ResolveResult, error) { + layers, err := r.Layers(ctx, options) + if err != nil { + return ResolveResult{}, err + } + resolved, err := api.ResolveSpecLayers(layers.Layers...) if err != nil { return ResolveResult{}, fmt.Errorf("resolve runtime profile layers: %w", err) } - return ResolveResult{Profile: profile, Resolved: resolved}, nil + return ResolveResult{Profile: layers.Profile, Resolved: resolved}, nil } func selectProfile(options ResolveOptions) (string, SelectionOrigin) { diff --git a/pkg/runtimeprofiles/resolver_ginkgo_test.go b/pkg/runtimeprofiles/resolver_ginkgo_test.go new file mode 100644 index 00000000..f868b0a7 --- /dev/null +++ b/pkg/runtimeprofiles/resolver_ginkgo_test.go @@ -0,0 +1,92 @@ +package runtimeprofiles + +import ( + "context" + + "github.com/flanksource/captain/pkg/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Runtime profile resolver", func() { + It("returns a permission-only profile's authored stack without resolving a model", func(ctx SpecContext) { + source := newMemSource("db", SourceDB, true) + profile := source.profiles.put("review", ProfileInput{ + Name: "Review", Spec: api.Spec{Permissions: api.Permissions{Mode: api.PermissionDontAsk}}, + }) + catalog, err := NewCatalog(source) + Expect(err).NotTo(HaveOccurred()) + resolver := NewResolver(func(context.Context) (*Catalog, error) { return catalog, nil }) + base := api.SpecLayer{Name: "defaults", Scope: api.SpecLayerGlobal, Spec: api.Spec{Budget: api.Budget{MaxTurns: 5}}} + surface := api.PromptSpecLayer("prompt", api.Spec{Budget: api.Budget{Cost: 2}}) + request := api.RequestSpecLayer("request", api.Spec{Budget: api.Budget{Timeout: "1m"}}) + result, err := resolver.Layers(ctx, ResolveOptions{ + BaseLayers: []api.SpecLayer{base}, RequestedProfile: profile.ID, + SurfaceLayers: []api.SpecLayer{surface}, RequestLayers: []api.SpecLayer{request}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Profile).NotTo(BeNil()) + Expect(result.Profile.Profile).To(Equal(profile)) + Expect(result.Profile.Resolved).To(BeZero()) + Expect(result.Layers).To(Equal([]api.SpecLayer{ + base, {ID: profile.ID + ":spec", Name: "Review run spec", Scope: api.SpecLayerSurface, + Source: api.SpecLayerSourceProfile, Spec: profile.Spec}, surface, request, + })) + }) + + It("retains the selected origin when the raw-layer catalog is unavailable", func(ctx SpecContext) { + _, err := NewResolver(nil).Layers(ctx, ResolveOptions{RequestedProfile: " requested ", PinnedProfile: "pin", DefaultProfile: "default"}) + Expect(err).To(MatchError(&SelectionError{Origin: SelectionRequested, Ref: "requested", Err: ErrCatalogUnavailable})) + }) + + It("places user-scope presets after surface layers and before the request", func(ctx SpecContext) { + source := newMemSource("db", SourceDB, true) + preset := globalPreset("User model") + preset.Scope = api.SpecLayerUser + record := source.presets.put("user-model", preset) + profile := source.profiles.put("review", ProfileInput{Name: "Review", Presets: []string{record.ID}}) + catalog, err := NewCatalog(source) + Expect(err).NotTo(HaveOccurred()) + resolver := NewResolver(func(context.Context) (*Catalog, error) { return catalog, nil }) + surface := api.PromptSpecLayer("prompt", api.Spec{Model: api.Model{Name: "haiku"}}) + request := api.RequestSpecLayer("request", api.Spec{Model: api.Model{Name: "sonnet"}}) + result, err := resolver.Layers(ctx, ResolveOptions{ + RequestedProfile: profile.ID, SurfaceLayers: []api.SpecLayer{surface}, RequestLayers: []api.SpecLayer{request}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Layers).To(HaveLen(4)) + Expect(result.Layers[0].Name).To(Equal("Review run spec")) + Expect(result.Layers[1]).To(Equal(surface)) + Expect(result.Layers[2].Name).To(Equal("User model")) + Expect(result.Layers[3]).To(Equal(request)) + }) + + DescribeTable("assembles the full stack before resolving the profile's runtime", + func(ctx SpecContext, profileModel api.Model) { + source := newMemSource("db", SourceDB, true) + profile := source.profiles.put("review", ProfileInput{ + Name: "Review", Spec: api.Spec{ + Model: profileModel, Permissions: api.Permissions{Mode: api.PermissionDontAsk}, + }, + }) + catalog, err := NewCatalog(source) + Expect(err).NotTo(HaveOccurred()) + resolver := NewResolver(func(context.Context) (*Catalog, error) { return catalog, nil }) + request := api.RequestSpecLayer("request", api.Spec{ + Model: api.Model{Name: "claude-sonnet-4-6", Mode: api.ModeCLI}, + }) + + result, err := resolver.Resolve(ctx, ResolveOptions{ + RequestedProfile: profile.ID, RequestLayers: []api.SpecLayer{request}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Resolved.Spec.Model).To(Equal(request.Spec.Model)) + Expect(result.Resolved.Spec.Permissions.Mode).To(Equal(api.PermissionDontAsk)) + Expect(result.Resolved.Trace).To(HaveLen(2)) + Expect(result.Resolved.Trace[0].Spec.Model).To(Equal(profileModel)) + Expect(result.Resolved.Trace[1]).To(Equal(request)) + }, + Entry("without a profile model", api.Model{}), + Entry("when the request changes an incompatible profile runtime", api.Model{Name: "gpt-5", Mode: api.ModeAgent}), + ) +}) diff --git a/pkg/runtimeprofiles/types.go b/pkg/runtimeprofiles/types.go index d2490052..31cddc39 100644 --- a/pkg/runtimeprofiles/types.go +++ b/pkg/runtimeprofiles/types.go @@ -97,11 +97,12 @@ type ProfileInput struct { } // Resolution is a profile materialised through the catalog: the profile with -// its preset references canonicalised to ids, the presets in reference order, -// and the resolved spec with its layer trace. +// its preset references canonicalised to ids, the presets and layers in reference +// order, and the effective spec populated only by Catalog.Resolve for preview. type Resolution struct { Profile Profile `json:"profile"` Presets []Preset `json:"presets"` + Layers []api.SpecLayer `json:"-"` Resolved api.ResolvedSpec `json:"resolved"` } From 1fd278271cf6a720273bd2b47c4d3beec0fde139 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sat, 5 Sep 2026 23:26:22 +0300 Subject: [PATCH 03/22] fix(prompts): preserve explicit runtime overrides --- pkg/cli/prompt_entity_write_ginkgo_test.go | 15 +++++- pkg/cli/prompt_render.go | 19 ------- pkg/cli/webapp/src/PromptDetailPane.tsx | 10 ++-- pkg/cli/webapp/src/PromptWorkbench.tsx | 11 ++-- pkg/cli/webapp/src/promptDetailState.test.ts | 56 ++++++++++++++++++-- pkg/cli/webapp/src/promptDetailState.ts | 23 ++++++-- 6 files changed, 95 insertions(+), 39 deletions(-) diff --git a/pkg/cli/prompt_entity_write_ginkgo_test.go b/pkg/cli/prompt_entity_write_ginkgo_test.go index ce933a53..90f2906e 100644 --- a/pkg/cli/prompt_entity_write_ginkgo_test.go +++ b/pkg/cli/prompt_entity_write_ginkgo_test.go @@ -160,13 +160,13 @@ var _ = Describe("prompt entity writes", func() { }) Describe("render", func() { - It("renders the draft and lets the draft's frontmatter model win over the saved seed", func() { + It("renders the draft model when the sparse request has no model override", func() { detail := createPrompt("summary", validPromptSource) result, err := renderPrompt(ctx, detail.ID, PromptRenderRequest{ Content: draftPromptSource, Variables: map[string]any{"patch": "abc"}, - Spec: detail.Run.Spec, + Spec: &api.Spec{}, }) Expect(err).NotTo(HaveOccurred()) @@ -174,6 +174,17 @@ var _ = Describe("prompt entity writes", func() { Expect(result.Model).To(Equal("gpt-5")) }) + It("preserves an explicit model equal to the saved prompt when rendering a draft", func() { + detail := createPrompt("summary", validPromptSource) + result, err := renderPrompt(ctx, detail.ID, PromptRenderRequest{ + Content: draftPromptSource, + Variables: map[string]any{"patch": "abc"}, + Spec: &api.Spec{Model: api.Model{Name: detail.Model, Mode: api.RuntimeMode(detail.Mode)}}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model).To(Equal("claude-sonnet-4-6")) + }) + It("keeps an explicit runtime override when rendering a draft", func() { detail := createPrompt("summary", validPromptSource) diff --git a/pkg/cli/prompt_render.go b/pkg/cli/prompt_render.go index 056e3826..cb97f473 100644 --- a/pkg/cli/prompt_render.go +++ b/pkg/cli/prompt_render.go @@ -11,24 +11,6 @@ import ( "github.com/flanksource/captain/pkg/api" ) -// withoutSavedModelSeed drops a model override that merely echoes the saved -// prompt's own frontmatter (the seed PromptDetail.Run hands the editor), so a -// draft that changes `model:` renders with the draft's model. An override that -// differs from the saved seed is an explicit choice and is kept. -func withoutSavedModelSeed(spec *api.Spec, record promptRecord, savedContent string) *api.Spec { - if spec == nil { - return nil - } - saved, err := promptSummaryFromContent(record, savedContent) - if err != nil || spec.Name != saved.Model || string(spec.Mode) != saved.Mode { - return spec - } - stripped := *spec - stripped.Name, stripped.ID, stripped.Mode = "", "", "" - stripped.Provider = nil - return &stripped -} - // renderPrompt is the HTTP/Spec render path: the caller's structured api.Spec // (the web UI's runtime overrides) is the last layer over the selected runtime // profile and the rendered frontmatter. A non-empty Content renders that draft @@ -46,7 +28,6 @@ func renderPrompt(ctx context.Context, id string, renderReq PromptRenderRequest) return PromptRenderResult{}, err } if strings.TrimSpace(renderReq.Content) != "" { - renderReq.Spec = withoutSavedModelSeed(renderReq.Spec, record, content) content = renderReq.Content } vars := renderReq.Variables diff --git a/pkg/cli/webapp/src/PromptDetailPane.tsx b/pkg/cli/webapp/src/PromptDetailPane.tsx index 49fc2355..18af2097 100644 --- a/pkg/cli/webapp/src/PromptDetailPane.tsx +++ b/pkg/cli/webapp/src/PromptDetailPane.tsx @@ -1,5 +1,5 @@ import type { ReactNode } from "react"; -import { Button, Tabs } from "@flanksource/clicky-ui/components"; +import { Button, Field, Tabs } from "@flanksource/clicky-ui/components"; import { CodeBlock, Icon, @@ -30,7 +30,7 @@ import { PromptSourceMarkdownEditor } from "./PromptWriteModal"; import type { PromptBatchHandle } from "./hooks/usePromptRunStream"; import type { PromptDetail, PromptPreviewResult } from "./promptData"; import type { PromptSchemaKind } from "./promptSchemaSource"; -import { isScratchPrompt } from "./promptDetailState"; +import { isScratchPrompt, promptRuntimeForDisplay } from "./promptDetailState"; import { CAPTAIN_SECRET_SELECTOR, errorMessage, @@ -162,14 +162,15 @@ export function PromptDetailPane({ const schema = scratch ? undefined : normalizeObjectSchema(detail.inputSchema); + const effectiveRuntime = promptRuntimeForDisplay(runRequest, previewResult); const selectedProvider = resolveProvider( models, - runRequest.spec?.model, + effectiveRuntime.model, ); const runtimeCliArgs = promptSchema?.runtimeAdapters?.find( (runtime) => runtime.provider === selectedProvider && - runtime.mode === runRequest.spec?.mode, + runtime.mode === effectiveRuntime.mode, )?.args; const runtimeFamilies = familiesFromRuntimeCatalog(runtimeCatalog); const promptReady = @@ -226,6 +227,7 @@ export function PromptDetailPane({ ) : (
+ {previewResult && {[effectiveRuntime.model, effectiveRuntime.mode].filter(Boolean).join(' · ')}} { + it("keeps saved runtime defaults out of the request and preserves explicit profile overrides", () => { + const seeded: PromptDetail = { + ...alpha, + run: { variables: { topic: 'example' }, chat: true, spec: { model: 'saved-model', mode: 'agent', budget: { timeout: '2h' } }, runtimes: [{ model: 'saved-comparison' }] }, + }; + const initial = promptDetailStateFor({}, seeded); + expect(initial.runRequest).toEqual({ variables: { topic: 'example' }, chat: true, spec: {} }); + const selected = { ...initial.runRequest, runtimeProfile: 'review-profile' }; + let states = promptDetailReducer({}, { type: 'run-request', detail: seeded, value: selected }); + expect(promptDetailStateFor(states, seeded).runRequest).toEqual(selected); + const explicit = { ...selected, spec: { model: 'saved-model' } }; + states = promptDetailReducer(states, { type: 'run-request', detail: seeded, value: explicit }); + expect(promptDetailStateFor(states, seeded).runRequest).toEqual(explicit); + }); + + it.each(['draft', 'run-request'] as const)('clears stale resolution after a %s edit', type => { + let states = promptDetailReducer({}, { type: 'preview-result', detail: alpha, key: promptPreviewKey(promptDetailStateFor({}, alpha)), value: { id: alpha.id, name: alpha.name, model: 'previous-model' } }); + states = promptDetailReducer(states, type === 'draft' + ? { type, detail: alpha, value: 'updated source' } + : { type, detail: alpha, value: { runtimeProfile: 'new-profile', spec: {} } }); + expect(promptDetailStateFor(states, alpha).previewResult).toBeUndefined(); + }); + + it.each(['draft', 'run-request'] as const)('ignores an old preview received after a %s edit', type => { + const key = promptPreviewKey(promptDetailStateFor({}, alpha)); + let states = promptDetailReducer({}, type === 'draft' + ? { type, detail: alpha, value: 'updated source' } + : { type, detail: alpha, value: { runtimeProfile: 'new-profile', spec: {} } }); + states = promptDetailReducer(states, { type: 'preview-result', detail: alpha, key, value: { id: alpha.id, name: alpha.name, model: 'previous-model' } }); + expect(promptDetailStateFor(states, alpha).previewResult).toBeUndefined(); + const latest = { id: alpha.id, name: alpha.name, model: 'updated-model' }; + states = promptDetailReducer(states, { type: 'preview-result', detail: alpha, key: promptPreviewKey(promptDetailStateFor(states, alpha)), value: latest }); + expect(promptDetailStateFor(states, alpha).previewResult).toEqual(latest); + }); + + it('uses resolved profile runtime only for display', () => { + const request = { runtimeProfile: 'review-profile', spec: {} }; + const runtime = promptRuntimeForDisplay(request, { + id: alpha.id, name: alpha.name, + resolution: { spec: { model: 'preset-model', mode: 'cmux' }, constraints: {}, trace: [] }, + }); + expect(runtime).toEqual({ model: 'preset-model', mode: 'cmux' }); + expect(request).toEqual({ runtimeProfile: 'review-profile', spec: {} }); + expect(promptRuntimeForDisplay({ spec: { model: 'operator-model', mode: 'api' } })).toEqual({ model: 'operator-model', mode: 'api' }); + }); + it("keeps a draft when the user switches to another prompt and back", () => { let states: PromptDetailStates = {}; states = promptDetailReducer(states, { type: "draft", detail: alpha, value: "alpha edited" }); @@ -78,17 +126,15 @@ describe("promptDetailReducer", () => { expect(isPromptDirty(reloaded, alpha)).toBe(true); }); - it("seeds the run request with the runtime profile the prompt pins, and only then", () => { + it("keeps a frontmatter profile pin out of explicit request overrides when the draft changes", () => { const pinned: PromptDetail = { ...detail("alpha", "alpha v1"), runtimeProfile: "Review", run: { ...EMPTY_RUN_REQUEST, runtimeProfile: "Review" }, }; - expect(promptDetailStateFor({}, pinned).runRequest).toMatchObject({ - runtimeProfile: "Review", - spec: EMPTY_RUN_REQUEST.spec, - }); + const states = promptDetailReducer({}, { type: 'draft', detail: pinned, value: 'runtimeProfile: Plan' }); + expect(promptDetailStateFor(states, pinned).runRequest).toEqual(EMPTY_RUN_REQUEST); expect(promptDetailStateFor({}, alpha).runRequest).not.toHaveProperty("runtimeProfile"); }); diff --git a/pkg/cli/webapp/src/promptDetailState.ts b/pkg/cli/webapp/src/promptDetailState.ts index 1c9eebc1..c0cbb2bc 100644 --- a/pkg/cli/webapp/src/promptDetailState.ts +++ b/pkg/cli/webapp/src/promptDetailState.ts @@ -5,10 +5,17 @@ import type { PromptSchemaKind } from "./promptSchemaSource"; export const EMPTY_RUN_REQUEST: AIPromptRunValue = { variables: {}, - spec: { budget: { timeout: "2h" } }, + spec: {}, chat: true, }; +export function promptRuntimeForDisplay(request: AIPromptRunValue, preview?: PromptPreviewResult): { model?: string; mode?: string } { + return { + model: preview?.resolution?.spec.model ?? preview?.model ?? request.spec?.model, + mode: preview?.resolution?.spec.mode ?? preview?.mode ?? request.spec?.mode, + }; +} + export const SCRATCH_PROMPT_ID = "__scratch__"; export const SCRATCH_PROMPT: PromptDetail = { @@ -48,6 +55,10 @@ export type PromptDetailState = { /** One editing slot per prompt id, so switching prompts never discards a draft. */ export type PromptDetailStates = Record; +export function promptPreviewKey(state: Pick): string { + return JSON.stringify([state.draft, state.runRequest]); +} + export type PromptDetailStateAction = | { type: "draft"; detail?: PromptDetail; value: string } | { type: "run-request"; detail?: PromptDetail; value: AIPromptRunValue } @@ -61,6 +72,7 @@ export type PromptDetailStateAction = | { type: "preview-result"; detail?: PromptDetail; + key: string; value?: PromptPreviewResult; } | { type: "active-run"; detail?: PromptDetail; value?: string } @@ -87,9 +99,9 @@ function applyAction( ): PromptDetailState { switch (action.type) { case "draft": - return { ...current, draft: action.value }; + return { ...current, draft: action.value, previewResult: undefined }; case "run-request": - return { ...current, runRequest: action.value }; + return { ...current, runRequest: action.value, previewResult: undefined }; case "variables-validity": return { ...current, variablesValid: action.value }; case "schema-validity": @@ -101,6 +113,7 @@ function applyAction( }, }; case "preview-result": + if (action.value && action.key !== promptPreviewKey(current)) return current; return { ...current, previewResult: action.value }; case "active-run": return { ...current, activeRunID: action.value }; @@ -118,7 +131,7 @@ function applyAction( export function initialPromptDetailState( detail?: PromptDetail, ): PromptDetailState { - const runRequest = detail?.run ?? EMPTY_RUN_REQUEST; + const { spec: _spec, runtimes: _runtimes, runtimeProfile: _runtimeProfile, ...runRequest } = detail?.run ?? EMPTY_RUN_REQUEST; const content = detail?.content ?? ""; return { draft: content, @@ -126,7 +139,7 @@ export function initialPromptDetailState( detailContent: content, runRequest: { ...runRequest, - spec: { ...EMPTY_RUN_REQUEST.spec, ...runRequest.spec }, + spec: {}, }, variablesValid: true, schemaValidity: { input: true, output: true }, From 1e871eb2f6347d8ba70cb46b346e0867236e3a7b Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sat, 5 Sep 2026 23:26:23 +0300 Subject: [PATCH 04/22] feat(webapp): render live and stored verification --- pkg/cli/webapp/package.json | 2 +- pkg/cli/webapp/pnpm-lock.yaml | 10 +- pkg/cli/webapp/src/PromptRunStream.test.tsx | 54 ++++++++--- pkg/cli/webapp/src/PromptRunStream.tsx | 45 +-------- pkg/cli/webapp/src/RunVerification.tsx | 31 ++++++ pkg/cli/webapp/src/SessionDetail.test.tsx | 81 ++++++++++++++++ pkg/cli/webapp/src/SessionDetail.tsx | 22 +++-- pkg/cli/webapp/src/types/verifyReport.ts | 100 +++----------------- 8 files changed, 190 insertions(+), 155 deletions(-) create mode 100644 pkg/cli/webapp/src/RunVerification.tsx create mode 100644 pkg/cli/webapp/src/SessionDetail.test.tsx diff --git a/pkg/cli/webapp/package.json b/pkg/cli/webapp/package.json index ce60576e..d8b5bcfc 100644 --- a/pkg/cli/webapp/package.json +++ b/pkg/cli/webapp/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "@ai-sdk/react": "^3.0.201", - "@flanksource/clicky-ui": "0.3.29", + "@flanksource/clicky-ui": "0.3.34", "@shikijs/langs": "^1.24.0", "@shikijs/themes": "^1.24.0", "@shikijs/transformers": "^1.24.0", diff --git a/pkg/cli/webapp/pnpm-lock.yaml b/pkg/cli/webapp/pnpm-lock.yaml index b6c9ca22..2099e3d8 100644 --- a/pkg/cli/webapp/pnpm-lock.yaml +++ b/pkg/cli/webapp/pnpm-lock.yaml @@ -16,8 +16,8 @@ importers: specifier: ^3.0.201 version: 3.0.216(react@18.3.1)(zod@4.4.3) '@flanksource/clicky-ui': - specifier: 0.3.29 - version: 0.3.29(@ai-sdk/react@3.0.216(react@18.3.1)(zod@4.4.3))(@babel/core@7.29.7)(@babel/template@7.29.7)(@shikijs/langs@1.29.2)(@shikijs/themes@1.29.2)(@shikijs/transformers@1.29.2)(@types/react@19.2.17)(ai@6.0.214(zod@4.4.3))(marked@15.0.12)(monaco-editor@0.48.0)(react-dom@18.3.1(react@18.3.1))(react-rnd@10.5.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)(recharts@3.10.1(@types/react@19.2.17)(react-dom@18.3.1(react@18.3.1))(react-is@17.0.2)(react@18.3.1)(redux@5.0.1))(shiki@1.29.2)(streamdown@2.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(tailwindcss@4.3.1) + specifier: 0.3.34 + version: 0.3.34(@ai-sdk/react@3.0.216(react@18.3.1)(zod@4.4.3))(@babel/core@7.29.7)(@babel/template@7.29.7)(@shikijs/langs@1.29.2)(@shikijs/themes@1.29.2)(@shikijs/transformers@1.29.2)(@types/react@19.2.17)(ai@6.0.214(zod@4.4.3))(marked@15.0.12)(monaco-editor@0.48.0)(react-dom@18.3.1(react@18.3.1))(react-rnd@10.5.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)(recharts@3.10.1(@types/react@19.2.17)(react-dom@18.3.1(react@18.3.1))(react-is@17.0.2)(react@18.3.1)(redux@5.0.1))(shiki@1.29.2)(streamdown@2.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(tailwindcss@4.3.1) '@shikijs/langs': specifier: ^1.24.0 version: 1.29.2 @@ -426,8 +426,8 @@ packages: cpu: [x64] os: [win32] - '@flanksource/clicky-ui@0.3.29': - resolution: {integrity: sha512-iQqRadiTqqfUuhcxAASbZoV3bgHl2UoMvL5NiVO5lQVfUNZlPQYYW3svz2xUWDQdz9mqgNCCtggQClsh3Pdi5A==} + '@flanksource/clicky-ui@0.3.34': + resolution: {integrity: sha512-5o1e8K2ZWbS9bWjsE0sd/rm0UkxZMbQ4/d0PRNQMHwGh3a55XG3oAkBhlBYxv2uUfhBj1zR1F6f195rDnKLvRg==} peerDependencies: '@ai-sdk/react': ^3.0.0 '@mdxeditor/editor': ^4.0.4 @@ -2771,7 +2771,7 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true - '@flanksource/clicky-ui@0.3.29(@ai-sdk/react@3.0.216(react@18.3.1)(zod@4.4.3))(@babel/core@7.29.7)(@babel/template@7.29.7)(@shikijs/langs@1.29.2)(@shikijs/themes@1.29.2)(@shikijs/transformers@1.29.2)(@types/react@19.2.17)(ai@6.0.214(zod@4.4.3))(marked@15.0.12)(monaco-editor@0.48.0)(react-dom@18.3.1(react@18.3.1))(react-rnd@10.5.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)(recharts@3.10.1(@types/react@19.2.17)(react-dom@18.3.1(react@18.3.1))(react-is@17.0.2)(react@18.3.1)(redux@5.0.1))(shiki@1.29.2)(streamdown@2.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(tailwindcss@4.3.1)': + '@flanksource/clicky-ui@0.3.34(@ai-sdk/react@3.0.216(react@18.3.1)(zod@4.4.3))(@babel/core@7.29.7)(@babel/template@7.29.7)(@shikijs/langs@1.29.2)(@shikijs/themes@1.29.2)(@shikijs/transformers@1.29.2)(@types/react@19.2.17)(ai@6.0.214(zod@4.4.3))(marked@15.0.12)(monaco-editor@0.48.0)(react-dom@18.3.1(react@18.3.1))(react-rnd@10.5.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)(recharts@3.10.1(@types/react@19.2.17)(react-dom@18.3.1(react@18.3.1))(react-is@17.0.2)(react@18.3.1)(redux@5.0.1))(shiki@1.29.2)(streamdown@2.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(tailwindcss@4.3.1)': dependencies: '@codemirror/autocomplete': 6.20.3 '@codemirror/commands': 6.11.0 diff --git a/pkg/cli/webapp/src/PromptRunStream.test.tsx b/pkg/cli/webapp/src/PromptRunStream.test.tsx index 07e9144f..c25c0c46 100644 --- a/pkg/cli/webapp/src/PromptRunStream.test.tsx +++ b/pkg/cli/webapp/src/PromptRunStream.test.tsx @@ -1,4 +1,4 @@ -import { cleanup, render, screen } from "@testing-library/react"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { PromptRunStream } from "./PromptRunStream"; import type { VerifyReport } from "./types/verifyReport"; @@ -28,6 +28,10 @@ function verifyReport(overrides: Partial = {}): VerifyReport { running: 0, timedout: 0, }, + tests: [ + { name: "Compile packages", framework: "fixture", passed: true }, + { name: "Check policy", framework: "fixture", running: true }, + ], state: "running", ...overrides, }; @@ -82,10 +86,10 @@ describe("PromptRunStream", () => { it("shows nothing when there is no verify frame yet", () => { render(); - expect(screen.queryByTestId("verify-status")).not.toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "Verification" })).not.toBeInTheDocument(); }); - it("shows a live progress line while a check is still running", () => { + it("shows live verification checks beside the transcript", () => { useSessionChatMock.mockReturnValue({ messages: [], status: "streaming", @@ -95,12 +99,13 @@ describe("PromptRunStream", () => { render(); - expect(screen.getByTestId("verify-status")).toHaveTextContent( - "verifying · 3/5 passed", - ); + expect(screen.getByText("Running verification…")).toBeInTheDocument(); + expect(screen.getByText("Compile packages")).toBeInTheDocument(); + expect(screen.getByText("Check policy")).toBeInTheDocument(); + expect(screen.getByText("Starting run…")).toBeInTheDocument(); }); - it("shows a plain verdict once the check has passed", () => { + it("keeps the verification checks visible once the run has passed", () => { useSessionChatMock.mockReturnValue({ messages: [], status: "done", @@ -113,7 +118,8 @@ describe("PromptRunStream", () => { render(); - expect(screen.getByTestId("verify-status")).toHaveTextContent("verified"); + expect(screen.getByText("Compile packages")).toBeInTheDocument(); + expect(screen.queryByText("Running verification…")).not.toBeInTheDocument(); }); it("renders a malformed verify frame's error without dropping the transcript", () => { @@ -128,7 +134,7 @@ describe("PromptRunStream", () => { render(); expect(screen.getByRole("alert")).toHaveTextContent(/invalid verify frame/); - expect(screen.queryByTestId("verify-status")).not.toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "Verification" })).not.toBeInTheDocument(); }); it("shows the failure reason once the check has failed", () => { @@ -147,8 +153,32 @@ describe("PromptRunStream", () => { render(); - expect(screen.getByTestId("verify-status")).toHaveTextContent( - "verification failed · 2 of 5 checks failed", - ); + expect(screen.getByText("2 of 5 checks failed")).toBeInTheDocument(); + }); + + it("opens failed fixture evidence including command output", () => { + useSessionChatMock.mockReturnValue({ + messages: [], + status: "error", + run: { runId: "run-verify", status: "error" }, + verify: { + report: verifyReport({ + state: "failed", + tests: [{ + name: "Policy assertion", + framework: "fixture", + failed: true, + command: "check-policy", + stdout: "Observed retry limit: 2", + context: { exit_code: 1, cel_expression: "retry_limit == 3" }, + }], + }), + done: true, + }, + }); + + render(); + fireEvent.click(screen.getByText("Policy assertion")); + expect(screen.getByText("Observed retry limit: 2")).toBeInTheDocument(); }); }); diff --git a/pkg/cli/webapp/src/PromptRunStream.tsx b/pkg/cli/webapp/src/PromptRunStream.tsx index f5b3534e..76627d9b 100644 --- a/pkg/cli/webapp/src/PromptRunStream.tsx +++ b/pkg/cli/webapp/src/PromptRunStream.tsx @@ -5,7 +5,7 @@ import { type PromptRunSummary, } from "./hooks/usePromptRunStream"; import { useSessionChat } from "./hooks/useSessionChat"; -import type { VerifyFrame, VerifyState } from "./types/verifyReport"; +import { RunVerification } from "./RunVerification"; /** * PromptRunStream renders a prompt run's session history live: it subscribes to @@ -16,7 +16,6 @@ export function PromptRunStream({ runID }: { runID: string }) { const chat = useSessionChat({ initialRunID: runID }); const { messages, summary, status, error, run, chatState, verify } = chat; const empty = messages.length === 0; - const verifyLine = verifyStatusLine(verify); return (
@@ -28,14 +27,7 @@ export function PromptRunStream({ runID }: { runID: string }) { ) : null}
- {verifyLine && ( -
- {verifyLine} -
- )} + {error && ( )} -
+
{empty ? (
{status === "done" @@ -136,37 +128,6 @@ const STATUS_DOT: Record = { error: "bg-destructive", }; -const VERIFY_FAILURE_STATES: readonly VerifyState[] = [ - "failed", - "errored", - "timed_out", - "cancelled", -]; - -/** - * verifyStatusLine renders the run's latest `verify` frame as a single line: - * a live progress count while a check is still running, and a terse verdict - * once it is done. Returns null when there is nothing to show yet. - */ -function verifyStatusLine(verify: VerifyFrame | null): string | null { - const report = verify?.report; - if (!report) return null; - if (report.state === "passed") return "verified"; - if (VERIFY_FAILURE_STATES.includes(report.state)) { - return report.reason - ? `verification failed · ${report.reason}` - : "verification failed"; - } - if (report.state === "warned") { - return report.reason - ? `verification warned · ${report.reason}` - : "verification warned"; - } - if (report.state === "skipped") return "verification skipped"; - const { passed, total } = report.summary; - return `verifying · ${passed}/${total} passed`; -} - function StatusPill({ status }: { status: PromptRunStreamStatus }) { return ( diff --git a/pkg/cli/webapp/src/RunVerification.tsx b/pkg/cli/webapp/src/RunVerification.tsx new file mode 100644 index 00000000..ca924b21 --- /dev/null +++ b/pkg/cli/webapp/src/RunVerification.tsx @@ -0,0 +1,31 @@ +import { VerificationResults } from "@flanksource/clicky-ui/data"; +import { parseVerifyFrame, type VerifyFrame } from "./types/verifyReport"; + +export function RunVerification({ + frame, + storedReport, + title = "Verification", +}: { + frame?: VerifyFrame | null; + storedReport?: unknown; + title?: string; +}) { + let verification = frame; + if (!verification && storedReport != null) { + try { + verification = parseVerifyFrame({ report: storedReport, done: true }); + } catch (error) { + return ( +
+ Invalid stored verification report: {error instanceof Error ? error.message : String(error)} +
+ ); + } + } + if (!verification?.report) return null; + return ( +
+ +
+ ); +} diff --git a/pkg/cli/webapp/src/SessionDetail.test.tsx b/pkg/cli/webapp/src/SessionDetail.test.tsx new file mode 100644 index 00000000..5afcba91 --- /dev/null +++ b/pkg/cli/webapp/src/SessionDetail.test.tsx @@ -0,0 +1,81 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { SessionDetail } from "./SessionDetail"; +import type { SessionGetResult } from "./sessionData"; +import type { VerifyFrame, VerifyReport } from "./types/verifyReport"; + +const useSessionChatMock = vi.hoisted(() => vi.fn(() => ({ messages: [], verify: null as VerifyFrame | null }))); + +vi.mock("./hooks/useSessionChat", async (importOriginal) => ({ + ...await importOriginal(), + useSessionChat: useSessionChatMock, +})); + +vi.mock("@flanksource/clicky-ui/ai", async (importOriginal) => ({ + ...await importOriginal(), + SessionInspector: () =>
Stored transcript
, +})); + +afterEach(cleanup); + +const report: VerifyReport = { + kind: "fixture", ran: true, passed: false, state: "failed", iteration: 1, + summary: { total: 1, passed: 0, failed: 1, warned: 0, skipped: 0, pending: 0, running: 0, timedout: 0 }, + tests: [{ name: "Persisted acceptance check", framework: "fixture", failed: true }], + reason: "Expected three retries", +}; + +function storedSession(verify: unknown): SessionGetResult { + return { + total: 1, + sessions: [{ + captainId: "stored-session", + detailAvailable: true, + summary: { key: "stored-session", id: "stored-session", source: "captain", messages: 1, toolCalls: 0 }, + detail: { id: "stored-session", source: "captain", messages: [], structuredOutput: { verify } }, + }], + }; +} + +describe("SessionDetail verification", () => { + it.each([false, true])("renders a persisted report with collection=%s", (collection) => { + render(); + expect(screen.getByText("Persisted acceptance check")).toBeInTheDocument(); + expect(screen.getByText("Expected three retries")).toBeInTheDocument(); + expect(screen.getByText("Stored transcript")).toBeInTheDocument(); + expect(screen.queryByText("Running verification…")).not.toBeInTheDocument(); + }); + + it("surfaces malformed persisted verification without losing the transcript", () => { + render(); + expect(screen.getByRole("alert")).toHaveTextContent("Invalid stored verification report"); + expect(screen.getByText("Stored transcript")).toBeInTheDocument(); + }); + + it("shows the live retry report instead of the stored prior verdict", () => { + useSessionChatMock.mockReturnValueOnce({ + messages: [], + verify: { + done: false, + report: { + ...report, + state: "running", + reason: undefined, + tests: [{ name: "Retrying acceptance check", framework: "fixture", running: true }], + summary: { ...report.summary, failed: 0, running: 1 }, + }, + }, + }); + render(); + expect(screen.getByText("Retrying acceptance check")).toBeInTheDocument(); + expect(screen.getByText("Running verification…")).toBeInTheDocument(); + expect(screen.queryByText("Persisted acceptance check")).not.toBeInTheDocument(); + }); +}); diff --git a/pkg/cli/webapp/src/SessionDetail.tsx b/pkg/cli/webapp/src/SessionDetail.tsx index 27a6b9b4..3ab35c90 100644 --- a/pkg/cli/webapp/src/SessionDetail.tsx +++ b/pkg/cli/webapp/src/SessionDetail.tsx @@ -13,6 +13,7 @@ import { errorMessage, } from "./sessionData"; import { sessionResultCollection } from "./sessionCollection"; +import { RunVerification } from "./RunVerification"; import { mergeSessionMessages, useSessionChat, @@ -53,11 +54,19 @@ export function SessionDetail({ const collection = sessionResultCollection(result); if (collection) { return ( -
- +
+
+ +
+ {result.sessions.map((item) => ( + + ))}
); } @@ -154,7 +163,7 @@ function SessionGetItemDetail({
) : null} {detail ? ( -
+
)} + ); } diff --git a/pkg/cli/webapp/src/types/verifyReport.ts b/pkg/cli/webapp/src/types/verifyReport.ts index f71ff27d..2e98c127 100644 --- a/pkg/cli/webapp/src/types/verifyReport.ts +++ b/pkg/cli/webapp/src/types/verifyReport.ts @@ -1,12 +1,14 @@ -/** - * Local mirror of captain's wire-level verification types (Go source of truth: - * `pkg/api/verify_report.go`). Kept as a plain type module — no clicky-ui - * import — so it can later be swapped for that package's own export (once the - * release carrying `VerificationResults` lands) without touching consumers. - * - * Field names stay snake_case to match the JSON the server actually sends; - * this module does not camelCase or otherwise reshape the wire. - */ +import type { + VerifyChecklistItem, + VerifyNode, + VerifyNodeContext, + VerifyNodeProgress, + VerifyReport, + VerifyState, + VerifySummary, +} from "@flanksource/clicky-ui/data"; + +export type { VerifyReport } from "@flanksource/clicky-ui/data"; /** Canonical order mirrors Go's `AllVerifyStates()`. */ export const VERIFY_STATES = [ @@ -21,86 +23,6 @@ export const VERIFY_STATES = [ "timed_out", ] as const; -export type VerifyState = (typeof VERIFY_STATES)[number]; - -export interface VerifySummary { - total: number; - passed: number; - failed: number; - warned: number; - skipped: number; - pending: number; - running: number; - timedout: number; -} - -export interface VerifyNodeProgress { - phase?: string; - done: number; - total: number; -} - -export interface VerifyNodeContext { - command?: string; - exit_code: number; - cwd?: string; - cel_expression?: string; - cel_vars?: Record; - expected?: unknown; - actual?: unknown; -} - -export interface VerifyNode { - name: string; - framework?: string; - task_id?: string; - file?: string; - line?: number; - message?: string; - command?: string; - work_dir?: string; - stdout?: string; - stderr?: string; - duration?: number; - passed?: boolean; - failed?: boolean; - warned?: boolean; - skipped?: boolean; - pending?: boolean; - running?: boolean; - timed_out?: boolean; - progress?: VerifyNodeProgress; - context?: VerifyNodeContext; - /** Rolled-up counts for this node's subtree, same shape as the report summary. */ - summary?: VerifySummary; - detail?: unknown; - children?: VerifyNode[]; -} - -export interface VerifyChecklistItem { - item: string; - passed: boolean | null; - message?: string; -} - -export interface VerifyReport { - kind: string; - name?: string; - ran: boolean; - passed: boolean; - reason?: string; - feedback?: string; - /** 1-based loop turn ("turn 1 of 3"); always on the wire. */ - iteration: number; - summary: VerifySummary; - tests?: VerifyNode[]; - checklist?: VerifyChecklistItem[]; - state: VerifyState; - started_at?: string; - finished_at?: string; - duration?: number; -} - /** The `verify` SSE event payload: the newest report, and whether it is the * verdict (`done: true`) or a still-running snapshot (`done: false`). */ export interface VerifyFrame { From fba381df4c559698d40b9284ed65d0dff9b46fdb Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 00:00:30 +0300 Subject: [PATCH 05/22] feat(promptrun): add execution preflight Validate complete run inputs before provider construction, setup or admission. Share actual model candidate, verifier, sandbox and constraint checks with execution while returning new capability diagnostics as warnings. Preserve model-free verification and authored requests. Cover preview side effects, fallback selection, budget ceilings and isolation fidelity with focused regressions. --- pkg/ai/agent/verify/declarations.go | 46 ++++++ pkg/ai/agent/verify/registry.go | 32 +--- pkg/ai/client.go | 14 +- pkg/ai/runtime_candidates.go | 26 +++ pkg/api/permission_capabilities.go | 63 +------- .../permission_capabilities_ginkgo_test.go | 4 +- pkg/api/permission_options.go | 47 ++++++ pkg/api/registry/model.go | 6 + pkg/api/runtime_constraints.go | 68 +++++++- pkg/api/runtime_profiles.go | 80 +++++---- pkg/api/spec.go | 6 +- pkg/api/spec_verification_ginkgo_test.go | 22 +++ pkg/promptrun/README.md | 36 +++++ pkg/promptrun/hooks.go | 11 +- pkg/promptrun/preflight.go | 153 ++++++++++++++++++ pkg/promptrun/preflight_config.go | 65 ++++++++ .../preflight_constraints_ginkgo_test.go | 109 +++++++++++++ pkg/promptrun/preflight_ginkgo_test.go | 151 +++++++++++++++++ .../preflight_runtime_config_ginkgo_test.go | 136 ++++++++++++++++ pkg/promptrun/promptrun_ginkgo_test.go | 4 +- pkg/promptrun/run.go | 67 +++----- 21 files changed, 969 insertions(+), 177 deletions(-) create mode 100644 pkg/ai/agent/verify/declarations.go create mode 100644 pkg/ai/runtime_candidates.go create mode 100644 pkg/api/permission_options.go create mode 100644 pkg/api/spec_verification_ginkgo_test.go create mode 100644 pkg/promptrun/README.md create mode 100644 pkg/promptrun/preflight.go create mode 100644 pkg/promptrun/preflight_config.go create mode 100644 pkg/promptrun/preflight_constraints_ginkgo_test.go create mode 100644 pkg/promptrun/preflight_ginkgo_test.go create mode 100644 pkg/promptrun/preflight_runtime_config_ginkgo_test.go diff --git a/pkg/ai/agent/verify/declarations.go b/pkg/ai/agent/verify/declarations.go new file mode 100644 index 00000000..93fc115f --- /dev/null +++ b/pkg/ai/agent/verify/declarations.go @@ -0,0 +1,46 @@ +package verify + +import ( + "fmt" + "strings" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/prompt" + "github.com/flanksource/captain/pkg/api" +) + +type DeclarationOptions struct { + // Provider is authoritative when supplied; Model names a provider not built yet. + Provider ai.Provider + Model string +} + +// ValidateDeclarations inspects verifier wiring and judge files without invoking +// registered factories, executing commands, or constructing a provider. +func ValidateDeclarations(wf *api.Workflow, opts DeclarationOptions) error { + if wf == nil || wf.Verify == nil { + return nil + } + if strings.TrimSpace(wf.Verify.Fixture) != "" && !Registered(KindFixture) { + return fmt.Errorf("workflow.verify.fixture declared but no fixture verifier is registered " + + "(link a fixture runner in-process or set verify.fixtureRunner in ~/.captain.yaml)") + } + model := opts.Model + if opts.Provider != nil { + model = opts.Provider.GetModel() + } + for i, path := range wf.Verify.Prompts { + path = strings.TrimSpace(path) + if path == "" { + return fmt.Errorf("workflow.verify.prompts[%d] is empty", i) + } + tmpl, err := prompt.LoadFile(path) + if err != nil { + return fmt.Errorf("verify prompt %q: %w", path, err) + } + if err := rejectJudgeOverrides(path, tmpl, model); err != nil { + return err + } + } + return nil +} diff --git a/pkg/ai/agent/verify/registry.go b/pkg/ai/agent/verify/registry.go index 50367c8f..43000934 100644 --- a/pkg/ai/agent/verify/registry.go +++ b/pkg/ai/agent/verify/registry.go @@ -127,9 +127,8 @@ func HooksFor(ctx context.Context, wf *api.Workflow, opts Options) ([]any, error if wf == nil || wf.Verify == nil { return nil, nil } - if strings.TrimSpace(wf.Verify.Fixture) != "" && !Registered(KindFixture) { - return nil, fmt.Errorf("workflow.verify.fixture declared but no fixture verifier is registered " + - "(link a fixture runner in-process or set verify.fixtureRunner in ~/.captain.yaml)") + if err := ValidateDeclarations(wf, DeclarationOptions{Provider: opts.Provider}); err != nil { + return nil, err } var hooks []any for _, kind := range kindOrder { @@ -149,25 +148,6 @@ func HooksFor(ctx context.Context, wf *api.Workflow, opts Options) ([]any, error return hooks, nil } -// ValidatePromptDeclarations loads every declared judge prompt before a run -// constructs its provider. This keeps a broken workflow attributable to the -// prompt declaration even when the selected provider is unavailable. -func ValidatePromptDeclarations(wf *api.Workflow) error { - if wf == nil || wf.Verify == nil { - return nil - } - for i, path := range wf.Verify.Prompts { - path = strings.TrimSpace(path) - if path == "" { - return fmt.Errorf("workflow.verify.prompts[%d] is empty", i) - } - if _, err := prompt.LoadFile(path); err != nil { - return fmt.Errorf("verify prompt %q: %w", path, err) - } - } - return nil -} - // DeclaresExec reports whether the workflow declares a check that starts a // process — a shell command or a fixture handed to an external runner. A // receive path asks before it has any hooks, because the confinement wrapper is @@ -231,7 +211,7 @@ func promptFactory(_ context.Context, spec api.Verify, opts Options) ([]*Plugin, if err != nil { return nil, fmt.Errorf("verify prompt %q: %w", path, err) } - if err := rejectJudgeOverrides(path, tmpl, opts.Provider); err != nil { + if err := rejectJudgeOverrides(path, tmpl, opts.Provider.GetModel()); err != nil { return nil, err } plugins = append(plugins, New("judge:"+path, &LLMJudgeVerifier{Provider: opts.Provider, Prompt: tmpl})) @@ -245,7 +225,7 @@ func promptFactory(_ context.Context, spec api.Verify, opts Options) ([]*Plugin, // ignored, which is exactly the downgrade issue #39 forbids (R5.4: a hook // prompt declaring a relocating sandbox is a validation error, never a silent // fallback). -func rejectJudgeOverrides(path string, tmpl *prompt.Template, provider ai.Provider) error { +func rejectJudgeOverrides(path string, tmpl *prompt.Template, model string) error { probe, _, err := tmpl.Render(map[string]any{"cwd": "", "changed": []string{}}, nil) if err != nil { return fmt.Errorf("verify prompt %q: %w", path, err) @@ -253,9 +233,9 @@ func rejectJudgeOverrides(path string, tmpl *prompt.Template, provider ai.Provid if probe.Sandbox != nil { return fmt.Errorf("verify prompt %q declares a sandbox; judge hooks run on the run's provider and cannot relocate", path) } - if declared := strings.TrimSpace(probe.Name); declared != "" && declared != provider.GetModel() { + if declared := strings.TrimSpace(probe.Name); declared != "" && declared != model { return fmt.Errorf("verify prompt %q declares model %q but judge hooks run on the run's provider (%s); remove the model or match it", - path, declared, provider.GetModel()) + path, declared, model) } return nil } diff --git a/pkg/ai/client.go b/pkg/ai/client.go index 98266d62..eeee1a18 100644 --- a/pkg/ai/client.go +++ b/pkg/ai/client.go @@ -28,22 +28,10 @@ func RegisterProvider(runtime Runtime, factory ProviderFactory) { // comma-separated Name or a Fallbacks list), a fallback provider is returned that // tries each in order on a fallback-eligible failure. func NewProvider(cfg Config) (Provider, error) { - resolved, err := Resolve(cfg.Model) + candidates, err := ResolveCandidates(cfg.Model) if err != nil { return nil, err } - cfg.Model = resolved - // Each candidate is resolved through the same entry point rather than a - // bespoke normalizer: a fallback chain and a substitute picked from the - // catalog must land on exactly the ids and modes the primary would. - candidates := cfg.Model.Candidates() - for i := range candidates { - candidate, err := Resolve(candidates[i]) - if err != nil { - return nil, suggestModelName(err, candidates[i].Name) - } - candidates[i] = candidate - } cfg.Model = candidates[0] if len(candidates) > 1 { return newFallbackProvider(cfg, candidates), nil diff --git a/pkg/ai/runtime_candidates.go b/pkg/ai/runtime_candidates.go new file mode 100644 index 00000000..fa118dd8 --- /dev/null +++ b/pkg/ai/runtime_candidates.go @@ -0,0 +1,26 @@ +package ai + +import ( + "fmt" + + "github.com/flanksource/captain/pkg/api" +) + +// ResolveCandidates returns the exact ordered models NewProvider will try, +// including disabled-model filtering and the catalog's enabled replacement. +// It performs no provider construction or credential lookup. +func ResolveCandidates(model api.Model) ([]api.Model, error) { + resolved, err := Resolve(model) + if err != nil { + return nil, err + } + candidates := resolved.Candidates() + for i := range candidates { + candidate, err := Resolve(candidates[i]) + if err != nil { + return nil, fmt.Errorf("candidate[%d] %q: %w", i, candidates[i].Name, err) + } + candidates[i] = candidate + } + return candidates, nil +} diff --git a/pkg/api/permission_capabilities.go b/pkg/api/permission_capabilities.go index a0c16829..8e1128f3 100644 --- a/pkg/api/permission_capabilities.go +++ b/pkg/api/permission_capabilities.go @@ -127,9 +127,8 @@ type PermissionCapabilities struct { ToolPolicies map[ToolProvenance]map[ToolPolicy]Support `json:"toolPolicies"` // Resources is the availability axis, keyed by kind and then by the value // requested. Both keys matter because the two directions are independent and - // today they are opposites: MCP is only switchable *off* (there is no - // per-server enable), while skills are only switchable *on* (a disabled entry - // is dropped before it reaches any provider). One Support per kind would + // MCP is only switchable *off* (there is no per-server enable), while disabled + // skills are enforced by omission before reaching any provider. One Support per kind would // report "supported" for a request that is silently ignored. Resources map[ResourceKind]map[ResourceMode]Support `json:"resources"` // Tools is the runtime's built-in tool vocabulary — the names a per-tool @@ -280,9 +279,8 @@ func callerTools() map[ToolPolicy]Support { // - mcpOff: does `permissions.mcp.disabled` actually silence ambient servers. // - skillsOn: does `permissions.skills: {dir: enabled}` actually load them. // -// Everything else is unsupported on every runtime today, and says why: -// `mcp.servers` and `mcp.modes` have no reader outside Validate, a disabled skill -// is dropped by ResourcePolicies.Enabled before any provider sees it, and +// Disabled skills are enforced by omission before any provider sees them. +// Other cells are unsupported: `mcp.servers` and `mcp.modes` have no reader, and // `permissions.plugins` reaches req.Permissions.Plugins and stops there. func resources(mcpOff, skillsOn bool) map[ResourceKind]map[ResourceMode]Support { mcpDisabled := unsupported("permissions.mcp.disabled is accepted and then dropped on this runtime") @@ -300,7 +298,7 @@ func resources(mcpOff, skillsOn bool) map[ResourceKind]map[ResourceMode]Support }, ResourceKindSkills: { ResourceEnabled: skillsEnabled, - ResourceDisabled: unsupported("ResourcePolicies.Enabled drops disabled skills before any provider sees them"), + ResourceDisabled: native("omitted by ResourcePolicies.Enabled before any provider sees it"), }, ResourceKindPlugins: { ResourceEnabled: unsupported("permissions.plugins reaches req.Permissions.Plugins and has no reader beyond it"), @@ -476,54 +474,3 @@ var permissionCapabilities = map[Runtime]PermissionCapabilities{ Tools: geminiBuiltinTools, }, } - -// SupportedPermissionModes lists the postures a runtime honours natively or by -// approximation, in canonical order. It is what a picker should offer. -func SupportedPermissionModes(p *ModelProvider, mode RuntimeMode) []PermissionMode { - caps := PermissionCapabilitiesFor(RuntimeOf(p, mode)) - out := make([]PermissionMode, 0, len(caps.Modes)) - for _, posture := range AllPermissionModes() { - if caps.ModeSupport(posture).Honoured() { - out = append(out, posture) - } - } - return out -} - -// SupportedToolPolicies lists the policy values a runtime can enforce for one -// provenance, in canonical order. A requires-broker value is included because -// whether it is usable depends on runtime state this table cannot see; the -// caller decides. -func SupportedToolPolicies(p *ModelProvider, mode RuntimeMode, provenance ToolProvenance) []ToolPolicy { - caps := PermissionCapabilitiesFor(RuntimeOf(p, mode)) - out := make([]ToolPolicy, 0, 4) - for _, policy := range AllToolPolicies() { - if s := caps.ToolPolicySupport(provenance, policy); s.Kind != SupportUnsupported { - out = append(out, policy) - } - } - return out -} - -// ToolPolicyProvenances lists the provenances that can carry a constraining -// policy on a runtime, in canonical order. Empty means no per-tool policy is -// enforceable there from any source. -func ToolPolicyProvenances(p *ModelProvider, mode RuntimeMode) []ToolProvenance { - caps := PermissionCapabilitiesFor(RuntimeOf(p, mode)) - var out []ToolProvenance - for _, provenance := range AllToolProvenances() { - for _, policy := range []ToolPolicy{ToolPolicyAllow, ToolPolicyDeny} { - if caps.ToolPolicySupport(provenance, policy).Honoured() { - out = append(out, provenance) - break - } - } - } - return out -} - -// AllToolPolicies lists the policy values in canonical order. ToolPolicy had no -// All* helper of its own because, until now, nothing enumerated it. -func AllToolPolicies() []ToolPolicy { - return []ToolPolicy{ToolPolicyAuto, ToolPolicyAsk, ToolPolicyAllow, ToolPolicyDeny} -} diff --git a/pkg/api/permission_capabilities_ginkgo_test.go b/pkg/api/permission_capabilities_ginkgo_test.go index e99e9923..20581dbe 100644 --- a/pkg/api/permission_capabilities_ginkgo_test.go +++ b/pkg/api/permission_capabilities_ginkgo_test.go @@ -236,7 +236,7 @@ var _ = Describe("PermissionCapabilities", func() { } }) - It("loads skills only on claude-cli, and never unloads them anywhere", func() { + It("loads skills only on claude-cli and omits disabled skills on every runtime", func() { for _, runtime := range api.AllRuntimes() { caps := api.PermissionCapabilitiesFor(runtime) want := api.SupportUnsupported @@ -246,7 +246,7 @@ var _ = Describe("PermissionCapabilities", func() { Expect(caps.ResourceSupport(api.ResourceKindSkills, api.ResourceEnabled).Kind). To(Equal(want), runtime.String()) Expect(caps.ResourceSupport(api.ResourceKindSkills, api.ResourceDisabled).Kind). - To(Equal(api.SupportUnsupported), runtime.String()) + To(Equal(api.SupportNative), runtime.String()) } }) diff --git a/pkg/api/permission_options.go b/pkg/api/permission_options.go new file mode 100644 index 00000000..390bfdfe --- /dev/null +++ b/pkg/api/permission_options.go @@ -0,0 +1,47 @@ +package api + +// SupportedPermissionModes lists the postures a runtime honours natively or by +// approximation, in canonical order. It is what a picker should offer. +func SupportedPermissionModes(p *ModelProvider, mode RuntimeMode) []PermissionMode { + caps := PermissionCapabilitiesFor(RuntimeOf(p, mode)) + out := make([]PermissionMode, 0, len(caps.Modes)) + for _, posture := range AllPermissionModes() { + if caps.ModeSupport(posture).Honoured() { + out = append(out, posture) + } + } + return out +} + +// SupportedToolPolicies includes broker-dependent policies; callers decide +// whether a broker is available for the run. +func SupportedToolPolicies(p *ModelProvider, mode RuntimeMode, provenance ToolProvenance) []ToolPolicy { + caps := PermissionCapabilitiesFor(RuntimeOf(p, mode)) + out := make([]ToolPolicy, 0, 4) + for _, policy := range AllToolPolicies() { + if s := caps.ToolPolicySupport(provenance, policy); s.Kind != SupportUnsupported { + out = append(out, policy) + } + } + return out +} + +// ToolPolicyProvenances lists the sources with enforceable policies. +func ToolPolicyProvenances(p *ModelProvider, mode RuntimeMode) []ToolProvenance { + caps := PermissionCapabilitiesFor(RuntimeOf(p, mode)) + var out []ToolProvenance + for _, provenance := range AllToolProvenances() { + for _, policy := range []ToolPolicy{ToolPolicyAllow, ToolPolicyDeny} { + if caps.ToolPolicySupport(provenance, policy).Honoured() { + out = append(out, provenance) + break + } + } + } + return out +} + +// AllToolPolicies lists the policy values in canonical order. +func AllToolPolicies() []ToolPolicy { + return []ToolPolicy{ToolPolicyAuto, ToolPolicyAsk, ToolPolicyAllow, ToolPolicyDeny} +} diff --git a/pkg/api/registry/model.go b/pkg/api/registry/model.go index 426bb7b6..754984c3 100644 --- a/pkg/api/registry/model.go +++ b/pkg/api/registry/model.go @@ -185,6 +185,12 @@ func (m Model) Validate() error { if m.Name == "" { return fmt.Errorf("model name is required") } + return m.ValidateOptions() +} + +// ValidateOptions checks inference knobs and fallback declarations without +// requiring a primary model for operations that do not call one. +func (m Model) ValidateOptions() error { if err := m.validateKnobs(); err != nil { return err } diff --git a/pkg/api/runtime_constraints.go b/pkg/api/runtime_constraints.go index d8cbcb80..6bd6f2fa 100644 --- a/pkg/api/runtime_constraints.go +++ b/pkg/api/runtime_constraints.go @@ -1,6 +1,9 @@ package api -import "fmt" +import ( + "fmt" + "strings" +) // RuntimeConstraintViolation identifies the constraint that rejected a run. type RuntimeConstraintViolation string @@ -42,8 +45,15 @@ func (e *RuntimeConstraintError) Error() string { } } -// ValidateRuntimeConstraints checks model selection, quotas, and input limits. +// ValidateRuntimeConstraints checks the actual run against model, budget, quota +// and input ceilings. It never clamps a copy while leaving execution unbounded. func ValidateRuntimeConstraints(resolved ResolvedSpec, model Model, estimatedInputTokens int) error { + if err := resolved.Constraints.Validate(); err != nil { + return err + } + if err := validateBudgetLimits(resolved.Spec.Budget, resolved.Constraints.Limits.Budget); err != nil { + return err + } if estimatedInputTokens < 0 { return &RuntimeConstraintError{ Violation: RuntimeConstraintInvalidInput, EstimatedInputTokens: estimatedInputTokens, @@ -73,3 +83,57 @@ func ValidateRuntimeConstraints(resolved ResolvedSpec, model Model, estimatedInp } return nil } + +// Validate checks effective constraints without resolving models or merging layers. +func (constraints RuntimeConstraints) Validate() error { + if _, err := strictRunLimits(RunLimits{}, constraints.Limits); err != nil { + return fmt.Errorf("runtime constraints: %w", err) + } + for _, selector := range constraints.Models { + if strings.TrimSpace(selector) == "" { + return fmt.Errorf("runtime constraints model catalog contains an empty selector") + } + } + for _, quota := range constraints.Quotas { + if strings.TrimSpace(quota.Name) == "" { + return fmt.Errorf("runtime constraints quota name is required") + } + if quota.Scope != SpecLayerGlobal && quota.Scope != SpecLayerContext { + return fmt.Errorf("runtime constraints quota %q requires global or context scope", quota.Name) + } + if quota.TokenLimit < 0 || quota.TokensUsed < 0 || quota.CostLimitUSD < 0 || quota.CostUsedUSD < 0 { + return fmt.Errorf("runtime constraints quota %q cannot contain negative usage or limits", quota.Name) + } + } + return nil +} + +func validateBudgetLimits(actual, limit Budget) error { + if err := actual.Validate(); err != nil { + return err + } + for _, ceiling := range []struct { + name string + actual, limit float64 + }{ + {"cost", actual.Cost, limit.Cost}, + {"maxTokens", float64(actual.MaxTokens), float64(limit.MaxTokens)}, + {"maxTurns", float64(actual.MaxTurns), float64(limit.MaxTurns)}, + } { + if ceiling.limit > 0 && (ceiling.actual == 0 || ceiling.actual > ceiling.limit) { + return fmt.Errorf("budget.%s %v exceeds the effective limit %v (zero is unbounded)", ceiling.name, ceiling.actual, ceiling.limit) + } + } + actualTimeout, err := actual.ParseTimeout() + if err != nil { + return err + } + limitTimeout, err := limit.ParseTimeout() + if err != nil { + return err + } + if limitTimeout > 0 && (actualTimeout == 0 || actualTimeout > limitTimeout) { + return fmt.Errorf("budget.timeout %s exceeds the effective limit %s", actualTimeout, limitTimeout) + } + return nil +} diff --git a/pkg/api/runtime_profiles.go b/pkg/api/runtime_profiles.go index 0d1c6287..c1bbb2d6 100644 --- a/pkg/api/runtime_profiles.go +++ b/pkg/api/runtime_profiles.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "slices" "strings" "github.com/flanksource/commons-db/shell" @@ -125,7 +126,7 @@ func ResolveRuntimeProfile(request RuntimeProfileResolveRequest) (ResolvedSpec, } resolved.Spec.Model = model } - if err := validateResolvedSandbox(resolved.Spec); err != nil { + if err := ValidateResolvedSandbox(resolved.Spec); err != nil { return ResolvedSpec{}, err } if err := validateResolvedPermissions(resolved.Spec); err != nil { @@ -254,17 +255,33 @@ func validateRuntimePreset(preset RuntimePreset) error { } func validateResolvedPermissions(spec Spec) error { + warnings := UnsupportedPermissions(spec) + if len(warnings) > 0 { + return fmt.Errorf("%s", warnings[0]) + } + return nil +} + +// UnsupportedPermissions reports settings the selected runtime cannot honour. +// Callers choose whether these capability diagnostics refuse a run or warn. +func UnsupportedPermissions(spec Spec) []string { if !hasResolvedPermissionSettings(spec) { return nil } provider, mode, err := spec.Runtime() if err != nil { - return fmt.Errorf("permission settings require a resolved runtime: %w", err) + return []string{fmt.Sprintf("permission settings require a resolved runtime: %v", err)} } runtime := RuntimeOf(provider, mode) caps := PermissionCapabilitiesFor(runtime) + var warnings []string + add := func(err error) { + if err != nil { + warnings = append(warnings, err.Error()) + } + } if posture := spec.Permissions.Mode; posture != "" && !caps.ModeSupport(posture).Honoured() { - return fmt.Errorf("permissions.mode %q is not available for %s", posture, runtime) + warnings = append(warnings, fmt.Sprintf("permissions.mode %q is not available for %s", posture, runtime)) } for tool, policy := range spec.Permissions.Tools { // Same rule as RequireToolPolicySupport: only an allow for another @@ -276,42 +293,32 @@ func validateResolvedPermissions(spec Spec) error { return err } } - for _, policy := range spec.ToolPreferences { - if err := requireResolvedToolPolicy(caps, runtime, ProvenanceCaller, policy); err != nil { - return err - } + for _, name := range sortedKeys(spec.ToolPreferences) { + add(requireResolvedToolPolicy(caps, runtime, ProvenanceCaller, spec.ToolPreferences[name])) } for _, rule := range spec.ToolPolicy { - if err := requireResolvedToolPolicy(caps, runtime, ProvenanceCaller, rule.Policy); err != nil { - return err - } + add(requireResolvedToolPolicy(caps, runtime, ProvenanceCaller, rule.Policy)) } if spec.Permissions.MCP.Disabled { - if err := requireResolvedResource(caps, runtime, ResourceKindMCP, ResourceDisabled); err != nil { - return err - } + add(requireResolvedResource(caps, runtime, ResourceKindMCP, ResourceDisabled)) } if len(spec.Permissions.MCP.Servers) > 0 { - if err := requireResolvedResource(caps, runtime, ResourceKindMCP, ResourceEnabled); err != nil { - return err - } + add(requireResolvedResource(caps, runtime, ResourceKindMCP, ResourceEnabled)) } - for _, mode := range spec.Permissions.MCP.Modes { - if err := requireResolvedResource(caps, runtime, ResourceKindMCP, mode); err != nil { - return err - } + for _, name := range sortedKeys(spec.Permissions.MCP.Modes) { + add(requireResolvedResource(caps, runtime, ResourceKindMCP, spec.Permissions.MCP.Modes[name])) } - for _, mode := range spec.Permissions.Skills { - if err := requireResolvedResource(caps, runtime, ResourceKindSkills, mode); err != nil { - return err + for _, name := range sortedKeys(spec.Permissions.Skills) { + add(requireResolvedResource(caps, runtime, ResourceKindSkills, spec.Permissions.Skills[name])) + if spec.Permissions.Skills[name] == ResourceDisabled && slices.Contains(spec.Memory.Skills, name) && + caps.ResourceSupport(ResourceKindSkills, ResourceEnabled).Honoured() { + warnings = append(warnings, fmt.Sprintf("permissions.skills %q is disabled but memory.skills still loads it for %s", name, runtime)) } } - for _, mode := range spec.Permissions.Plugins { - if err := requireResolvedResource(caps, runtime, ResourceKindPlugins, mode); err != nil { - return err - } + for _, name := range sortedKeys(spec.Permissions.Plugins) { + add(requireResolvedResource(caps, runtime, ResourceKindPlugins, spec.Permissions.Plugins[name])) } - return nil + return warnings } func hasResolvedPermissionSettings(spec Spec) bool { @@ -322,10 +329,14 @@ func hasResolvedPermissionSettings(spec Spec) bool { len(spec.ToolPreferences) > 0 || len(spec.ToolPolicy) > 0 } -func validateResolvedSandbox(spec Spec) error { +// ValidateResolvedSandbox refuses sandbox isolation unsupported by the runtime. +func ValidateResolvedSandbox(spec Spec) error { if spec.Sandbox == nil { return nil } + if err := spec.Sandbox.Validate(); err != nil { + return err + } provider, mode, err := spec.Runtime() if err != nil { return fmt.Errorf("sandbox settings require a resolved runtime: %w", err) @@ -334,6 +345,17 @@ func validateResolvedSandbox(spec Spec) error { if !containsSandboxMode(capabilities.Modes, spec.Sandbox.Mode) { return fmt.Errorf("sandbox mode %q is not available for %s", spec.Sandbox.Mode, RuntimeOf(provider, mode)) } + if mode != ModeAPI { + switch provider { + case Anthropic: + _, err = TranslateClaudeSandbox(RuntimeOf(provider, mode), *spec.Sandbox) + case OpenAI: + _, err = TranslateCodexSandbox(RuntimeOf(provider, mode), spec.Sandbox, spec.Permissions.Mode) + } + } + if err != nil { + return err + } return nil } diff --git a/pkg/api/spec.go b/pkg/api/spec.go index a1a2757e..89a250bf 100644 --- a/pkg/api/spec.go +++ b/pkg/api/spec.go @@ -172,7 +172,11 @@ func (s Spec) MarshalYAML() (any, error) { // Validate runs each component's validation, failing loud on the first error. func (s Spec) Validate() error { - if err := s.Model.Validate(); err != nil { + validateModel := s.Model.Validate + if s.IsVerifyOnly() && len(s.Workflow.Verify.Prompts) == 0 { + validateModel = s.ValidateOptions + } + if err := validateModel(); err != nil { return fmt.Errorf("model: %w", err) } if s.ToolApproval != nil { diff --git a/pkg/api/spec_verification_ginkgo_test.go b/pkg/api/spec_verification_ginkgo_test.go new file mode 100644 index 00000000..b021f8da --- /dev/null +++ b/pkg/api/spec_verification_ginkgo_test.go @@ -0,0 +1,22 @@ +package api_test + +import ( + "github.com/flanksource/captain/pkg/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("model-free verification spec validation", func() { + It("accepts inherited runtime options without requiring an unused model", func() { + spec := api.Spec{Model: api.Model{Mode: api.ModeAgent, Effort: api.EffortHigh}, Workflow: &api.Workflow{Verify: &api.Verify{Fixture: "acceptance"}}} + Expect(spec.Validate()).To(Succeed()) + }) + It("still rejects malformed inherited tuning", func() { + spec := api.Spec{Model: api.Model{Mode: "invalid"}, Workflow: &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}}}} + Expect(spec.Validate()).To(MatchError(ContainSubstring("mode"))) + }) + It("requires a model for a judge prompt", func() { + spec := api.Spec{Model: api.Model{Mode: api.ModeAgent}, Workflow: &api.Workflow{Verify: &api.Verify{Prompts: []string{"judge.prompt"}}}} + Expect(spec.Validate()).To(MatchError(ContainSubstring("model name"))) + }) +}) diff --git a/pkg/promptrun/README.md b/pkg/promptrun/README.md new file mode 100644 index 00000000..8f383a12 --- /dev/null +++ b/pkg/promptrun/README.md @@ -0,0 +1,36 @@ +# Prompt run admission + +Call `Preflight(Input) ([]string, error)` with the same complete input that will be passed to `Run`. It reads judge prompt files and validates declarations, attachments, deadlines, model selection, runtime policies, verifier registration, commit ownership, and effective constraints. It does not construct a provider, invoke a verifier factory or approval callback, run hooks or setup, emit events, or persist a run. `Run` uses the same admission check before dispatch. + +```go +input := promptrun.Input{ + Request: resolved.Spec, + Config: config, + Constraints: resolved.Constraints, + Timeout: 10 * time.Minute, + Hooks: hostHooks, + CallerOwnsCommits: true, +} +warnings, err := promptrun.Preflight(input) +if err != nil { + return err +} +for _, warning := range warnings { + logger.Warnf("%s", warning) +} +result, err := promptrun.Run(ctx, input) +``` + +The host owns resolution and passes the final `ResolvedSpec.Constraints` intact. Admission rejects a budget or deadline above those ceilings; it never clamps a temporary copy. A declared deadline outranks `Input.Timeout`. `Request.Budget.Cost` bounds the whole run; a provider's legacy `Config.Budget.Cost` fallback only bounds its individual calls and cannot replace that whole-run limit. Input-size admission uses the existing four-bytes-per-token approximation over prompt text and message text, including the appended system prompt. It is not an exact tokenizer or attachment-token guarantee. + +A supplied `Provider` owns its runtime and workspace; construction `Config` is ignored. Otherwise, `Config.Model` selects the constructed provider when present, with `Request.Model` used when absent. Request tuning is still validated because the runner dispatches it. Command/fixture-only verification needs no model; declared judge prompts need either the run provider or `Verify.Provider`. Fixture verification requires a registered fixture factory, which preflight checks without invoking. Providerless verification refuses a non-off sandbox declaration because no run provider would apply that isolation. + +Invalid input, existing tool-policy refusals, unsupported sandbox isolation or native policy fields, missing fixture wiring, broken judge declarations, and constraint violations are errors. Newly diagnosed unsupported permission/resource settings and missing approval brokers produce warnings for this compatibility release. A disabled skill is omitted; a contradictory skill still explicitly loaded through `memory.skills` is diagnosed. These warnings are also logged by `Run`. + +Preflight validates the runtime identity exposed by a supplied provider. Its private fallback chain, credentials, adapter-specific configuration, external service availability, and runtime launch failures remain the provider's responsibility. This API is execution admission; structural runtime-profile layer validation and saved-model default resolution remain separate contracts. + +Run the executable examples and focused admission regressions without making AI calls: + +```sh +go test ./pkg/promptrun -run TestPromptRun -ginkgo.focus=promptrun.Preflight -ginkgo.succinct -ginkgo.no-color -count=1 +``` diff --git a/pkg/promptrun/hooks.go b/pkg/promptrun/hooks.go index 6b3f5551..3ed5a338 100644 --- a/pkg/promptrun/hooks.go +++ b/pkg/promptrun/hooks.go @@ -29,8 +29,8 @@ import ( // Input.CallerOwnsCommits drops the leading commit hooks: the host commits, and // its own hooks keep their position between the checks and setup. func Hooks(ctx context.Context, in Input, provider ai.Provider) ([]any, error) { - if in.CallerOwnsCommits && len(in.Hooks) == 0 { - return nil, fmt.Errorf("promptrun: CallerOwnsCommits is set but Input.Hooks is empty: nothing would commit") + if err := validateCommitOwnership(in); err != nil { + return nil, err } opts := in.Verify if opts.Provider == nil { @@ -57,3 +57,10 @@ func Hooks(ctx context.Context, in Input, provider ai.Provider) ([]any, error) { } return hooks, nil } + +func validateCommitOwnership(in Input) error { + if in.CallerOwnsCommits && len(in.Hooks) == 0 { + return fmt.Errorf("promptrun: CallerOwnsCommits is set but Input.Hooks is empty: nothing would commit") + } + return nil +} diff --git a/pkg/promptrun/preflight.go b/pkg/promptrun/preflight.go new file mode 100644 index 00000000..a2d9d8a0 --- /dev/null +++ b/pkg/promptrun/preflight.go @@ -0,0 +1,153 @@ +package promptrun + +import ( + "fmt" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/api" +) + +// Preflight validates one run without constructing providers or invoking hooks. +// Capability warnings are separate from invalid-state errors. Input estimates +// use four bytes per token; they are a guardrail, not an exact tokenizer count. +func Preflight(in Input) ([]string, error) { + admission, err := preflight(in) + return admission.warnings, err +} + +type admission struct { + model api.Model + timeout time.Duration + warnings []string +} + +func preflight(in Input) (admission, error) { + var out admission + if err := in.Request.ValidateRunnable(); err != nil { + return out, fmt.Errorf("promptrun: %w", err) + } + var err error + if out.timeout, err = runTimeout(in); err != nil { + return out, err + } + if err := validateCommitOwnership(in); err != nil { + return out, err + } + if in.MaxIterations < 0 { + return out, fmt.Errorf("promptrun: Input.MaxIterations must be non-negative") + } + if in.Scope != "" && !in.Scope.Valid() { + return out, fmt.Errorf("promptrun: invalid scope %q", in.Scope) + } + if err := validateVerificationIsolation(in); err != nil { + return out, err + } + out.model = executingModel(in) + if out.model.Provider != nil { + if err := api.RequireToolPolicySupport(out.model.Provider, out.model.Mode, in.Request.Permissions); err != nil { + return out, err + } + } + needsModel := !in.Request.IsVerifyOnly() || declaresPrompts(in.Request.Workflow) + if constructsProvider(in) { + candidates, err := ai.ResolveCandidates(out.model) + if err != nil { + return out, fmt.Errorf("promptrun model: %w", err) + } + out.model = candidates[0] + out.model.Fallbacks = candidates[1:] + } + if err := verify.ValidateDeclarations(in.Request.Workflow, verify.DeclarationOptions{Provider: in.Verify.Provider, Model: out.model.Name}); err != nil { + return out, err + } + spec := in.Request + if spec.Name == "" { + spec.Name = out.model.Name + } + if err := spec.Validate(); err != nil { + return out, fmt.Errorf("promptrun: %w", err) + } + spec.Model = out.model + if err := validateAttachments(spec.Prompt.Attachments, out.model); err != nil { + return out, err + } + if needsModel { + if err := out.model.Validate(); err != nil { + return out, fmt.Errorf("promptrun model: %w", err) + } + out.warnings, err = validateRuntime(in, spec) + if err != nil { + return out, err + } + } + spec.Budget.Timeout = out.timeout.String() + return out, api.ValidateRuntimeConstraints(api.ResolvedSpec{Spec: spec, Constraints: in.Constraints}, out.model, estimatedInputTokens(in.Request)) +} + +func validateRuntime(in Input, spec api.Spec) ([]string, error) { + var warnings []string + if constructsProvider(in) { + if err := validateConstructionConfig(in); err != nil { + return nil, err + } + } + for index, model := range append([]api.Model{spec.Model}, spec.Fallbacks...) { + if _, _, err := model.Runtime(); err != nil { + return warnings, fmt.Errorf("promptrun model %q: %w", model.Name, err) + } + if err := api.RequireToolPolicySupport(model.Provider, model.Mode, spec.Permissions); err != nil { + return warnings, err + } + candidate := spec + candidate.Model = model + if err := api.ValidateResolvedSandbox(candidate); err != nil { + return warnings, fmt.Errorf("promptrun model %q: %w", model.Name, err) + } + if constructsProvider(in) && in.Config.SandboxSelection != nil { + descriptor, _ := api.SandboxFor(in.Config.SandboxSelection.Kind) + if err := descriptor.ValidateMode(model.Mode); err != nil { + return warnings, err + } + } + for _, warning := range api.UnsupportedPermissions(candidate) { + if index > 0 { + warning = fmt.Sprintf("fallback[%d] %q: %s", index-1, model.Name, warning) + } + warnings = append(warnings, warning) + } + caps := api.PermissionCapabilitiesFor(api.RuntimeOf(model.Provider, model.Mode)) + if constructsProvider(in) && in.Config.CanUseTool == nil && requiresBroker(candidate, caps) { + warnings = append(warnings, fmt.Sprintf("caller-tool policy ask requires Config.CanUseTool for %s", api.RuntimeOf(model.Provider, model.Mode))) + } + } + return warnings, nil +} + +func requiresBroker(spec api.Spec, caps api.PermissionCapabilities) bool { + if caps.ToolPolicySupport(api.ProvenanceCaller, api.ToolPolicyAsk).Kind != api.SupportRequiresBroker { + return false + } + for _, policy := range spec.ToolPreferences { + if policy == api.ToolPolicyAsk { + return true + } + } + for _, rule := range spec.ToolPolicy { + if rule.Policy == api.ToolPolicyAsk { + return true + } + } + return false +} + +func estimatedInputTokens(request api.Spec) int { + size := len(request.Prompt.System) + len(request.Prompt.AppendSystem) + len(request.Prompt.User) + for _, message := range request.Messages { + for _, part := range message.Parts { + size += len(part.Text) + } + } + return (size + 3) / 4 +} diff --git a/pkg/promptrun/preflight_config.go b/pkg/promptrun/preflight_config.go new file mode 100644 index 00000000..7a05f59a --- /dev/null +++ b/pkg/promptrun/preflight_config.go @@ -0,0 +1,65 @@ +package promptrun + +import ( + "fmt" + "reflect" + + "github.com/flanksource/captain/pkg/api" +) + +func constructsProvider(in Input) bool { + return in.Provider == nil && (!in.Request.IsVerifyOnly() || declaresPrompts(in.Request.Workflow) && in.Verify.Provider == nil) +} + +func validateVerificationIsolation(in Input) error { + if !in.Request.IsVerifyOnly() || in.Provider != nil || constructsProvider(in) { + return nil + } + if sandbox := in.Request.Sandbox; sandbox != nil && sandbox.Mode != api.SandboxOff { + return fmt.Errorf("promptrun: verify-only execution without a run provider cannot apply sandbox mode %q", sandbox.Mode) + } + if selection := in.Config.SandboxSelection; selection != nil && selection.Kind != api.SandboxOff { + return fmt.Errorf("promptrun: verify-only execution without a run provider cannot apply Config.SandboxSelection %q", selection.Kind) + } + return nil +} + +func validateConstructionConfig(in Input) error { + if err := in.Config.Budget.Validate(); err != nil { + return fmt.Errorf("promptrun Config budget: %w", err) + } + if in.Config.CallerTools != nil { + if err := in.Config.CallerTools.Validate(); err != nil { + return err + } + } + selection := in.Config.SandboxSelection + if selection != nil { + if _, ok := api.SandboxFor(selection.Kind); !ok { + return fmt.Errorf("unknown sandbox kind %q; want one of: %s", selection.Kind, api.SandboxKindList()) + } + if err := selection.Dispatch.Validate(); err != nil { + return err + } + } + if declared := in.Request.Sandbox; declared != nil { + if selection == nil && (declared.Mode == api.SandboxDocker || declared.Mode == api.SandboxGitAgent) { + return fmt.Errorf("sandbox mode %q requires a resolved Config.SandboxSelection before running", declared.Mode) + } + if selection != nil && declared.Mode != selection.Kind { + return fmt.Errorf("sandbox mode %q does not match Config.SandboxSelection kind %q", declared.Mode, selection.Kind) + } + if selection != nil { + if declared.Backend != "" && declared.Backend != selection.Name { + return fmt.Errorf("sandbox backend %q does not match Config.SandboxSelection name %q", declared.Backend, selection.Name) + } + if declared.Agent != "" && declared.Agent != selection.Agent { + return fmt.Errorf("sandbox agent %q does not match Config.SandboxSelection agent %q", declared.Agent, selection.Agent) + } + if declared.Dispatch != nil && !reflect.DeepEqual(declared.Dispatch, selection.Dispatch) { + return fmt.Errorf("sandbox dispatch policy does not match Config.SandboxSelection dispatch policy") + } + } + } + return nil +} diff --git a/pkg/promptrun/preflight_constraints_ginkgo_test.go b/pkg/promptrun/preflight_constraints_ginkgo_test.go new file mode 100644 index 00000000..e985783f --- /dev/null +++ b/pkg/promptrun/preflight_constraints_ginkgo_test.go @@ -0,0 +1,109 @@ +package promptrun_test + +import ( + "context" + "strings" + "time" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/promptrun" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("promptrun.Preflight constraints and runtimes", func() { + var in promptrun.Input + BeforeEach(func() { + in = promptrun.Input{ + Request: api.Spec{Model: api.Model{Name: "claude-sonnet-5", Mode: api.ModeAgent}, Prompt: api.Prompt{User: "review"}}, + Timeout: time.Minute, + } + }) + + DescribeTable("rejects a budget that the dispatched input would exceed", + func(budget, limit api.Budget, callerTimeout time.Duration, message string) { + in.Request.Budget = budget + in.Constraints.Limits.Budget = limit + in.Timeout = callerTimeout + _, err := promptrun.Preflight(in) + Expect(err).To(MatchError(ContainSubstring(message))) + _, runErr := promptrun.Run(context.Background(), in) + Expect(runErr).To(MatchError(err.Error())) + Expect(in.Request.Budget).To(Equal(budget)) + }, + Entry("unbounded cost", api.Budget{}, api.Budget{Cost: 1}, time.Minute, "cost"), + Entry("excess cost", api.Budget{Cost: 2}, api.Budget{Cost: 1}, time.Minute, "cost"), + Entry("excess output tokens", api.Budget{MaxTokens: 200}, api.Budget{MaxTokens: 100}, time.Minute, "maxTokens"), + Entry("unbounded turns", api.Budget{}, api.Budget{MaxTurns: 2}, time.Minute, "maxTurns"), + Entry("declared deadline exceeds cap", api.Budget{Timeout: "2m"}, api.Budget{Timeout: "1m"}, time.Second, "timeout"), + Entry("caller deadline exceeds cap", api.Budget{}, api.Budget{Timeout: "1m"}, 2*time.Minute, "timeout"), + ) + + It("uses the declared deadline and actual tighter budget within limits", func() { + in.Request.Budget = api.Budget{Cost: 1, MaxTokens: 100, MaxTurns: 2, Timeout: "30s"} + in.Constraints.Limits.Budget = api.Budget{Cost: 2, MaxTokens: 200, MaxTurns: 3, Timeout: "1m"} + in.Timeout = time.Hour + _, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(in.Request.Budget.Timeout).To(Equal("30s")) + }) + + DescribeTable("checks constraints against the actual run", + func(mutate func(*promptrun.Input), message string) { + mutate(&in) + _, err := promptrun.Preflight(in) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("model catalog", func(in *promptrun.Input) { in.Constraints.Models = []string{"gpt-5"} }, "effective model catalog"), + Entry("fallback model catalog", func(in *promptrun.Input) { + in.Constraints.Models = []string{"claude-sonnet-5"} + in.Request.Fallbacks = api.ModelList{{Name: "gpt-5", Mode: api.ModeAgent}} + }, "fallback model"), + Entry("input ceiling counts append system", func(in *promptrun.Input) { + in.Constraints.Limits.MaxInputTokens = 5 + in.Request.Prompt.AppendSystem = strings.Repeat("context ", 10) + }, "input is about"), + Entry("token quota", func(in *promptrun.Input) { + in.Constraints.Quotas = []api.UsageQuota{{Name: "daily", Scope: api.SpecLayerGlobal, Layer: "workspace", TokenLimit: 10, TokensUsed: 10}} + }, "quota"), + Entry("negative ceiling", func(in *promptrun.Input) { in.Constraints.Limits.MaxInputTokens = -1 }, "non-negative"), + ) + + DescribeTable("rejects incompatible sandbox declarations", + func(model api.Model, sandbox api.SandboxRef, message string) { + in.Request.Model = model + in.Request.Sandbox = &sandbox + _, err := promptrun.Preflight(in) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("unsupported primary", api.Model{Name: "gpt-5", Mode: api.ModeAPI}, api.SandboxRef{Mode: api.SandboxNative}, "sandbox mode"), + Entry("invalid mode", api.Model{Name: "claude-sonnet-5", Mode: api.ModeAgent}, api.SandboxRef{Mode: "invalid"}, "sandbox"), + Entry("unsupported fallback", api.Model{Name: "claude-sonnet-5", Mode: api.ModeAgent, Fallbacks: api.ModelList{{Name: "gpt-5", Mode: api.ModeAPI}}}, api.SandboxRef{Mode: api.SandboxNative}, "gpt-5"), + ) + + It("accepts a supported sandbox without starting setup", func() { + in.Request.Sandbox = &api.SandboxRef{Mode: api.SandboxNative} + warnings, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + }) + + It("reports fallback capability warnings using the fallback runtime", func() { + in.Request.Model.Mode = api.ModeCLI + in.Request.Fallbacks = api.ModelList{{Name: "gpt-5", Mode: api.ModeAgent}} + in.Request.Permissions.Skills = api.ResourcePolicies{"review-tools": api.ResourceEnabled} + warnings, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(ConsistOf(And(ContainSubstring("fallback[0]"), ContainSubstring("skills=enabled"), ContainSubstring("openai agent")))) + }) + + It("applies constraints to the enabled replacement construction would select", func() { + previous := registry.Disabled() + DeferCleanup(func() { registry.SetDisabled(previous) }) + registry.SetDisabled(registry.NewDisabledSet(nil, []string{"anthropic"}, nil, nil, nil)) + in.Constraints.Models = []string{"claude-sonnet-5"} + _, err := promptrun.Preflight(in) + Expect(err).To(MatchError(ContainSubstring("effective model catalog"))) + }) +}) diff --git a/pkg/promptrun/preflight_ginkgo_test.go b/pkg/promptrun/preflight_ginkgo_test.go new file mode 100644 index 00000000..a54dc41c --- /dev/null +++ b/pkg/promptrun/preflight_ginkgo_test.go @@ -0,0 +1,151 @@ +package promptrun_test + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/promptrun" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("promptrun.Preflight", func() { + var in promptrun.Input + var provider *scriptedProvider + BeforeEach(func() { + provider = &scriptedProvider{model: "claude-sonnet-5"} + in = promptrun.Input{ + Request: api.Spec{Model: api.Model{Name: "claude-sonnet-5", Provider: api.Anthropic, Mode: api.ModeAgent}, Prompt: api.Prompt{User: "review the change"}}, + Provider: provider, Timeout: time.Minute, + } + }) + AfterEach(func() { verify.Unregister(verify.KindFixture) }) + + DescribeTable("refuses invalid input before dispatch", + func(mutate func(*promptrun.Input), message string) { + mutate(&in) + _, err := promptrun.Preflight(in) + Expect(err).To(MatchError(ContainSubstring(message))) + _, runErr := promptrun.Run(context.Background(), in) + Expect(runErr).To(MatchError(err.Error())) + Expect(provider.Calls()).To(BeZero()) + }, + Entry("no work", func(in *promptrun.Input) { in.Request.Prompt = api.Prompt{} }, "workflow.verify"), + Entry("no deadline", func(in *promptrun.Input) { in.Timeout = 0 }, "no timeout"), + Entry("malformed declared deadline despite caller timeout", func(in *promptrun.Input) { in.Request.Budget.Timeout = "later" }, "timeout"), + Entry("unresolved attachment", func(in *promptrun.Input) { in.Request.Prompt.Attachments = []api.AttachmentRef{{Path: "notes.txt"}} }, "not resolved"), + Entry("missing fixture runner", func(in *promptrun.Input) { + in.Request.Workflow = &api.Workflow{Verify: &api.Verify{Fixture: "acceptance"}} + }, "no fixture verifier is registered"), + Entry("missing commit hook", func(in *promptrun.Input) { in.CallerOwnsCommits = true }, "nothing would commit"), + Entry("empty judge declaration", func(in *promptrun.Input) { + in.Request.Workflow = &api.Workflow{Verify: &api.Verify{Prompts: []string{" "}}} + }, "prompts[0]"), + Entry("missing judge file", func(in *promptrun.Input) { + in.Request.Workflow = &api.Workflow{Verify: &api.Verify{Prompts: []string{filepath.Join(GinkgoT().TempDir(), "absent.prompt")}}} + }, "absent.prompt"), + Entry("invalid permission value", func(in *promptrun.Input) { in.Request.Permissions.Mode = "invalid" }, "invalid permission mode"), + Entry("unsupported existing tool denial", func(in *promptrun.Input) { + in.Provider = nil + in.Config.Model = api.Model{Name: "gpt-5", Provider: api.OpenAI, Mode: api.ModeAPI} + in.Request.Permissions.Tools = api.Tools{"Bash": api.ToolPolicyDeny} + }, "cannot enforce a per-tool policy"), + Entry("missing generating model", func(in *promptrun.Input) { in.Provider = nil; in.Request.Model = api.Model{} }, "model"), + Entry("unknown generating model", func(in *promptrun.Input) { + in.Provider = nil + in.Request.Model = api.Model{Name: "no-such-model-at-all"} + }, "no-such-model-at-all"), + Entry("invalid request tuning with supplied provider", func(in *promptrun.Input) { + invalid := 3.0 + in.Request.Temperature = &invalid + }, "temperature"), + Entry("incompatible configured attachment", func(in *promptrun.Input) { + in.Provider = nil + in.Config.Model = api.Model{Name: "claude-sonnet-5", Mode: api.ModeCLI} + in.Request.Prompt.Attachments = []api.AttachmentRef{preparedAttachment("image/png")} + }, "image/png"), + ) + + It("has no provider, verifier, caller hook or event side effects", func() { + factoryCalls := 0 + verify.Register(verify.KindFixture, func(context.Context, api.Verify, verify.Options) ([]*verify.Plugin, error) { + factoryCalls++ + return nil, nil + }) + var hookLog []string + in.Request.Workflow = &api.Workflow{Verify: &api.Verify{Fixture: "acceptance"}} + in.Hooks = []any{&recordingHook{name: "caller", log: &hookLog}} + in.CallerOwnsCommits = true + events := 0 + in.OnEvent = func(int, ai.Event) { events++ } + warnings, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + Expect([]int{provider.Calls(), factoryCalls, len(hookLog), events}).To(Equal([]int{0, 0, 0, 0})) + }) + + It("allows command and registered fixture verification without any model", func() { + verify.Register(verify.KindFixture, func(context.Context, api.Verify, verify.Options) ([]*verify.Plugin, error) { return nil, nil }) + in.Provider = nil + in.Request = api.Spec{Workflow: &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}, Fixture: "acceptance"}}} + warnings, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + }) + + It("uses the supplied provider instead of an ignored conflicting configuration", func() { + in.Config.Model = api.Model{Name: "gpt-5", Provider: api.OpenAI, Mode: api.ModeAPI} + in.Request.Permissions.Tools = api.Tools{"Bash": api.ToolPolicyDeny} + warnings, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + result, err := promptrun.Run(context.Background(), in) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Passed).To(BeTrue()) + }) + + DescribeTable("rejects unsupported judge overrides before constructing providers", + func(frontmatter, message string) { + path := filepath.Join(GinkgoT().TempDir(), "judge.prompt") + Expect(os.WriteFile(path, []byte("---\n"+frontmatter+"\n---\n{{role \"user\"}}\nReview."), 0o600)).To(Succeed()) + in.Request.Workflow = &api.Workflow{Verify: &api.Verify{Prompts: []string{path}}} + _, err := promptrun.Preflight(in) + Expect(err).To(MatchError(ContainSubstring(message))) + Expect(provider.Calls()).To(BeZero()) + }, + Entry("another model", "model: gpt-5", "declares model"), + Entry("sandbox", "sandbox:\n mode: off", "declares a sandbox"), + ) + + It("uses the explicit judge provider for model-free verification", func() { + in.Provider = nil + in.Verify.Provider = provider + in.Request = api.Spec{Workflow: &api.Workflow{Verify: &api.Verify{Prompts: []string{writeJudgePrompt(GinkgoT().TempDir())}}}} + _, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(provider.Calls()).To(BeZero()) + }) + + It("returns unsupported permissions as warnings without hiding invalid values", func() { + in.Request.Permissions.Plugins = api.ResourcePolicies{"review-tools": api.ResourceEnabled} + warnings, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(ContainElement(ContainSubstring("plugins"))) + Expect(provider.Calls()).To(BeZero()) + result, err := promptrun.Run(context.Background(), in) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Passed).To(BeTrue()) + }) + + It("does not warn for skills disabled by omission", func() { + in.Request.Permissions.Skills = api.ResourcePolicies{"review-tools": api.ResourceDisabled} + warnings, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + }) +}) diff --git a/pkg/promptrun/preflight_runtime_config_ginkgo_test.go b/pkg/promptrun/preflight_runtime_config_ginkgo_test.go new file mode 100644 index 00000000..579e7548 --- /dev/null +++ b/pkg/promptrun/preflight_runtime_config_ginkgo_test.go @@ -0,0 +1,136 @@ +package promptrun_test + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/promptrun" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("promptrun.Preflight runtime configuration", func() { + var in promptrun.Input + BeforeEach(func() { + in = promptrun.Input{ + Request: api.Spec{Model: api.Model{Name: "sonnet", Mode: api.ModeAgent}, Prompt: api.Prompt{User: "review"}}, + Timeout: time.Minute, + } + }) + + It("resolves a construction alias before comparing policies and judge models", func() { + path := filepath.Join(GinkgoT().TempDir(), "judge.prompt") + Expect(os.WriteFile(path, []byte("---\nmodel: claude-sonnet-5\n---\n{{role \"user\"}}\nReview."), 0o600)).To(Succeed()) + in.Request.Permissions.Tools = api.Tools{"Bash": api.ToolPolicyDeny} + in.Request.Workflow = &api.Workflow{Verify: &api.Verify{Prompts: []string{path}}} + _, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(in.Request.Name).To(Equal("sonnet")) + }) + + DescribeTable("refuses invalid construction configuration", + func(mutate func(*promptrun.Input), message string) { + mutate(&in) + _, err := promptrun.Preflight(in) + Expect(err).To(MatchError(ContainSubstring(message))) + _, runErr := promptrun.Run(context.Background(), in) + Expect(runErr).To(MatchError(err.Error())) + }, + Entry("unknown sandbox selection", func(in *promptrun.Input) { in.Config.SandboxSelection = &api.SandboxConfig{Kind: "invalid"} }, "sandbox"), + Entry("missing external selection", func(in *promptrun.Input) { in.Request.Sandbox = &api.SandboxRef{Mode: api.SandboxDocker} }, "SandboxSelection"), + Entry("mismatched external selection", func(in *promptrun.Input) { + in.Request.Sandbox = &api.SandboxRef{Mode: api.SandboxGitAgent} + in.Config.SandboxSelection = &api.SandboxConfig{Kind: api.SandboxOff} + }, "SandboxSelection"), + Entry("unsupported sandbox selection", func(in *promptrun.Input) { + in.Config.Model = api.Model{Name: "gpt-5", Mode: api.ModeAPI} + in.Config.SandboxSelection = &api.SandboxConfig{Kind: api.SandboxDocker} + }, "docker"), + Entry("invalid provider budget", func(in *promptrun.Input) { in.Config.Budget.Cost = -1 }, "budget"), + Entry("invalid caller endpoint", func(in *promptrun.Input) { in.Config.CallerTools = &api.CallerToolEndpoint{Name: "review"} }, "caller-tool"), + Entry("invalid scope", func(in *promptrun.Input) { in.Scope = "invalid" }, "scope"), + Entry("invalid loop bound", func(in *promptrun.Input) { in.MaxIterations = -1 }, "MaxIterations"), + Entry("unsupported Codex policy field", func(in *promptrun.Input) { + in.Request.Model = api.Model{Name: "gpt-5", Mode: api.ModeAgent} + in.Request.Sandbox = &api.SandboxRef{Mode: api.SandboxNative, Policy: &api.NativeSandboxPolicy{Network: &api.SandboxNetworkPolicy{AllowedDomains: []string{"example.com"}}}} + }, "allowedDomains"), + Entry("unsupported Claude policy field", func(in *promptrun.Input) { + include := false + in.Request.Sandbox = &api.SandboxRef{Mode: api.SandboxNative, Policy: &api.NativeSandboxPolicy{Filesystem: &api.SandboxFilesystemPolicy{IncludeSystemTemp: &include}}} + }, "includeSystemTemp"), + ) + + It("ignores construction configuration for a supplied provider", func() { + in.Provider = &scriptedProvider{model: "claude-sonnet-5"} + in.Config.SandboxSelection = &api.SandboxConfig{Kind: "invalid"} + in.Config.Budget.Cost = -1 + in.Config.CallerTools = &api.CallerToolEndpoint{} + _, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + }) + + It("warns for an absent approval broker and never calls an attached broker", func() { + in.Request.ToolPreferences = api.ToolPreferences{"review": api.ToolPolicyAsk} + warnings, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(ContainElement(ContainSubstring("CanUseTool"))) + calls := 0 + in.Config.CanUseTool = func(context.Context, api.PermissionRequest) (api.PermissionDecision, error) { + calls++ + return api.PermissionDecision{}, nil + } + warnings, err = promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + Expect(calls).To(BeZero()) + }) + + It("reports a disabled skill still explicitly loaded through memory", func() { + in.Request.Mode = api.ModeCLI + in.Request.Permissions.Skills = api.ResourcePolicies{"review-tools": api.ResourceDisabled} + in.Request.Memory.Skills = []string{"review-tools"} + warnings, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(ContainElement(ContainSubstring("memory.skills"))) + }) + + DescribeTable("refuses providerless verification whose isolation would never be applied", + func(sandbox *api.SandboxRef, selection *api.SandboxConfig) { + in.Request = api.Spec{Sandbox: sandbox, Workflow: &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}}}} + in.Config.SandboxSelection = selection + var hookLog []string + in.Hooks = []any{&recordingHook{name: "setup", log: &hookLog}} + _, err := promptrun.Preflight(in) + Expect(err).To(MatchError(ContainSubstring("verify-only"))) + _, runErr := promptrun.Run(context.Background(), in) + Expect(runErr).To(MatchError(err.Error())) + Expect(hookLog).To(BeEmpty()) + }, + Entry("native policy", &api.SandboxRef{Mode: api.SandboxNative}, nil), + Entry("Docker", &api.SandboxRef{Mode: api.SandboxDocker}, nil), + Entry("Git Agent", &api.SandboxRef{Mode: api.SandboxGitAgent}, nil), + Entry("configured boundary", nil, &api.SandboxConfig{Kind: api.SandboxDocker}), + ) + + DescribeTable("requires external selection to preserve authored restrictions", + func(selection api.SandboxConfig, message string) { + in.Request.Sandbox = &api.SandboxRef{Mode: api.SandboxGitAgent, Backend: "review-pool", Agent: "review-worker", Dispatch: &api.SandboxDispatchPolicy{MaxAttempts: 1, Paths: []string{"allowed/**"}}} + in.Config.SandboxSelection = &selection + _, err := promptrun.Preflight(in) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("backend", api.SandboxConfig{Kind: api.SandboxGitAgent, Name: "other-pool"}, "backend"), + Entry("agent", api.SandboxConfig{Kind: api.SandboxGitAgent, Name: "review-pool", Agent: "other-worker"}, "agent"), + Entry("dispatch policy", api.SandboxConfig{Kind: api.SandboxGitAgent, Name: "review-pool", Agent: "review-worker"}, "dispatch"), + ) + + It("accepts an external selection with the exact authored restrictions", func() { + in.Request.Sandbox = &api.SandboxRef{Mode: api.SandboxGitAgent, Backend: "review-pool", Agent: "review-worker", Dispatch: &api.SandboxDispatchPolicy{MaxAttempts: 1, Paths: []string{"allowed/**"}}} + in.Config.SandboxSelection = &api.SandboxConfig{Kind: api.SandboxGitAgent, Name: "review-pool", Agent: "review-worker", Dispatch: &api.SandboxDispatchPolicy{MaxAttempts: 1, Paths: []string{"allowed/**"}}} + _, err := promptrun.Preflight(in) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/pkg/promptrun/promptrun_ginkgo_test.go b/pkg/promptrun/promptrun_ginkgo_test.go index 5858fb06..d9e17734 100644 --- a/pkg/promptrun/promptrun_ginkgo_test.go +++ b/pkg/promptrun/promptrun_ginkgo_test.go @@ -322,7 +322,7 @@ var _ = Describe("promptrun.Run", func() { request.Permissions.Tools = api.Tools{"Bash": api.ToolPolicyDeny} _, err := promptrun.Run(context.Background(), promptrun.Input{ - Request: request, Provider: provider, Timeout: testTimeout, + Request: request, Timeout: testTimeout, Config: ai.Config{Model: api.Model{Name: "gpt-5", Provider: api.OpenAI, Mode: api.ModeAPI}}, }) Expect(err).To(MatchError(ContainSubstring(api.RuntimeOf(api.OpenAI, api.ModeAPI).String()))) @@ -334,7 +334,7 @@ var _ = Describe("promptrun.Run", func() { request.Prompt.Attachments = []api.AttachmentRef{preparedAttachment("image/png")} _, err := promptrun.Run(context.Background(), promptrun.Input{ - Request: request, Provider: provider, Timeout: testTimeout, + Request: request, Timeout: testTimeout, Config: ai.Config{Model: api.Model{Name: "claude-sonnet-5", Provider: api.Anthropic, Mode: api.ModeCLI}}, }) Expect(err).To(MatchError(ContainSubstring("image/png"))) diff --git a/pkg/promptrun/run.go b/pkg/promptrun/run.go index db532d61..eb1c9353 100644 --- a/pkg/promptrun/run.go +++ b/pkg/promptrun/run.go @@ -24,6 +24,7 @@ import ( "github.com/flanksource/captain/pkg/ai/agent/verify" "github.com/flanksource/captain/pkg/ai/middleware" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/commons/logger" ) // Input is everything one run needs. @@ -78,6 +79,8 @@ type Input struct { // Repo is the root of the tree the run's changed files are recorded relative // to; empty means the request's cwd. Repo string + // Constraints is the restrictive channel from the final ResolvedSpec. + Constraints api.RuntimeConstraints } // Run executes one prompt run and returns its outcome. A failing verdict is a @@ -85,29 +88,16 @@ type Input struct { // not complete — a hook failed, the provider failed, the policy is unenforceable. func Run(ctx context.Context, in Input) (Result, error) { start := time.Now() - // One classification, the same one the runner makes: a run generates, or it - // verifies what is already there. Anything else — attachments or a message - // history with no prompt and nothing declared to verify — used to build a - // provider and then quietly report a pass having done neither. - if err := in.Request.ValidateRunnable(); err != nil { - return Result{}, fmt.Errorf("promptrun: %w", err) - } - if err := validateAttachments(in); err != nil { - return Result{}, err - } - timeout, err := runTimeout(in) + admission, err := preflight(in) if err != nil { return Result{}, err } - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - if err := requireToolPolicy(in); err != nil { - return Result{}, err - } - if err := verify.ValidatePromptDeclarations(in.Request.Workflow); err != nil { - return Result{}, err + for _, warning := range admission.warnings { + logger.Warnf("promptrun: %s", warning) } + in.Config.Model = admission.model + ctx, cancel := context.WithTimeout(ctx, admission.timeout) + defer cancel() provider, release, err := buildProvider(in) if err != nil { return Result{}, err @@ -164,8 +154,7 @@ func scopeOf(in Input) agent.Scope { // validateAttachments refuses a request whose attachments were never resolved // — a path or URL the provider would have to fetch itself — and one whose // resolved attachments the selected models cannot accept. -func validateAttachments(in Input) error { - refs := in.Request.Prompt.Attachments +func validateAttachments(refs []api.AttachmentRef, model api.Model) error { if len(refs) == 0 { return nil } @@ -174,28 +163,30 @@ func validateAttachments(in Input) error { return fmt.Errorf("promptrun: attachment %d (%s) is not resolved; resolve attachments against a store before running", i, ref.Path+ref.URL+ref.ID) } } - model := executingModel(in) models := append([]api.Model{model}, model.Fallbacks...) return ai.ValidateAttachmentCompatibility(models, refs) } -// executingModel is the model this run will actually be answered by: the one -// middleware.NewProvider is handed, which is Config.Model whenever the config -// names one, and the request's own model otherwise (a caller that supplied its -// provider, or a test). -// -// It exists because the two pre-flight checks resolved it differently — the -// attachment check preferred the request's model and the tool-policy check the -// config's — so a prompt whose frontmatter named a different model than the -// config had its attachments validated against a runtime that would never see -// them, and its policy against one that could not enforce it. +// executingModel follows provider construction, including caller-owned providers. func executingModel(in Input) api.Model { + if in.Provider != nil { + return suppliedModel(in.Provider) + } + if in.Request.IsVerifyOnly() && in.Verify.Provider != nil { + return suppliedModel(in.Verify.Provider) + } if in.Config.Model.Name != "" || in.Config.Model.Provider != nil { return in.Config.Model } return in.Request.Model } +func suppliedModel(provider ai.Provider) api.Model { + runtime := provider.GetRuntime() + descriptor, _ := runtime.ModelProvider() + return api.Model{Name: provider.GetModel(), Provider: descriptor, Mode: runtime.Mode} +} + // runTimeout is the spec's budget.timeout when declared, else the caller's. The // spec wins because it is what the run's author declared; the caller's value is // a host default. @@ -217,15 +208,6 @@ func runTimeout(in Input) (time.Duration, error) { return 0, fmt.Errorf("promptrun: no timeout: declare budget.timeout on the spec or set Input.Timeout") } -// requireToolPolicy refuses a per-tool policy the selected runtime cannot -// enforce before anything runs. Every provider repeats the check at execution -// time, but by then setup has materialised a checkout and a host has recorded -// a run that was never going to start. -func requireToolPolicy(in Input) error { - model := executingModel(in) - return api.RequireToolPolicySupport(model.Provider, model.Mode, in.Request.Permissions) -} - // buildProvider returns the caller's provider, or constructs one from Config // when the run will call a model: a generating run always does, and a // verify-only run does when it declares judge prompts. A verify-only run of @@ -236,10 +218,11 @@ func buildProvider(in Input) (ai.Provider, func(), error) { if in.Provider != nil { return in.Provider, release, nil } - if in.Request.IsVerifyOnly() && !declaresPrompts(in.Request.Workflow) { + if !constructsProvider(in) { return nil, release, nil } cfg := in.Config + cfg.Model = executingModel(in) if in.Request.NoCache { cfg.NoCache = true } From 333d373ffcef481ed149e0b253a9758aaa42a8b6 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 06:49:45 +0300 Subject: [PATCH 06/22] feat(api)!: validate runtime layers Validate authored layers before composition and check the complete declared runtime after merging. Keep raw profile and chat defaults available until request resolution, with typed configuration errors and separate capability warnings. BREAKING CHANGE: aichat.RuntimeProfile now carries Composed (api.ComposedSpec). Profile providers must use api.ComposeSpecLayers and preserve its raw Trace. --- pkg/aichat/README.md | 46 ++++++ pkg/aichat/approval_execution.go | 4 +- pkg/aichat/approval_execution_test.go | 8 +- .../layered_profile_ownership_ginkgo_test.go | 54 +++++++ pkg/aichat/layered_runtime_profile.go | 14 +- pkg/aichat/messages.go | 41 +++--- pkg/aichat/messages_model_ginkgo_test.go | 28 ++-- .../profile_composition_http_ginkgo_test.go | 79 ++++++++++ pkg/aichat/provider_config.go | 12 +- pkg/aichat/request_layers_ginkgo_test.go | 59 ++++++++ pkg/aichat/runtime_profile.go | 64 +++++++-- pkg/aichat/runtime_profile_ginkgo_test.go | 14 +- pkg/aichat/service.go | 136 +----------------- pkg/aichat/service_persistence.go | 98 +++++++++++++ pkg/api/runtime_profiles.go | 61 ++------ pkg/api/runtime_profiles_ginkgo_test.go | 21 +-- pkg/api/spec.go | 51 +------ pkg/api/spec_composition.go | 42 ++++++ pkg/api/spec_layer_validation.go | 28 ++++ pkg/api/spec_layer_validation_ginkgo_test.go | 59 ++++++++ pkg/api/spec_layers.go | 39 ++--- pkg/api/spec_runtime.go | 30 ++++ pkg/api/spec_runtime_ginkgo_test.go | 110 ++++++++++++++ pkg/api/spec_validation.go | 70 +++++++++ pkg/cli/prompt_layers.go | 19 ++- pkg/cli/prompt_profile_layers_ginkgo_test.go | 72 ++++++++++ pkg/cli/serve_chat_profile.go | 16 ++- pkg/cli/serve_chat_profile_ginkgo_test.go | 18 +-- pkg/promptrun/README.md | 2 +- pkg/promptrun/preflight.go | 22 +-- pkg/runtimeprofiles/README.md | 41 ++++++ pkg/runtimeprofiles/doc.go | 3 +- pkg/runtimeprofiles/layer_errors.go | 16 +++ .../layer_validation_ginkgo_test.go | 67 +++++++++ pkg/runtimeprofiles/layers_ginkgo_test.go | 5 +- pkg/runtimeprofiles/resolve.go | 16 +-- pkg/runtimeprofiles/resolver.go | 3 + pkg/runtimeprofiles/resolver_ginkgo_test.go | 4 +- pkg/runtimeprofiles/types.go | 8 +- 39 files changed, 1103 insertions(+), 377 deletions(-) create mode 100644 pkg/aichat/README.md create mode 100644 pkg/aichat/layered_profile_ownership_ginkgo_test.go create mode 100644 pkg/aichat/profile_composition_http_ginkgo_test.go create mode 100644 pkg/aichat/request_layers_ginkgo_test.go create mode 100644 pkg/aichat/service_persistence.go create mode 100644 pkg/api/spec_composition.go create mode 100644 pkg/api/spec_layer_validation.go create mode 100644 pkg/api/spec_layer_validation_ginkgo_test.go create mode 100644 pkg/api/spec_runtime.go create mode 100644 pkg/api/spec_runtime_ginkgo_test.go create mode 100644 pkg/api/spec_validation.go create mode 100644 pkg/cli/prompt_profile_layers_ginkgo_test.go create mode 100644 pkg/runtimeprofiles/README.md create mode 100644 pkg/runtimeprofiles/layer_errors.go create mode 100644 pkg/runtimeprofiles/layer_validation_ginkgo_test.go diff --git a/pkg/aichat/README.md b/pkg/aichat/README.md new file mode 100644 index 00000000..f3084cd7 --- /dev/null +++ b/pkg/aichat/README.md @@ -0,0 +1,46 @@ +# Chat runtime profiles + +`RuntimeProfileProvider` supplies application-owned defaults and restrictions before the chat request selects its final runtime. Profiles may contain only permissions, budgets, or model options. Catalog endpoints can use this partial configuration without requiring an executable model. + +## Migrating `RuntimeProfile.Resolved` to `RuntimeProfile.Composed` + +This is a Go API change for applications implementing `RuntimeProfileProvider`. Replace the `Resolved: api.ResolvedSpec` field with `Composed: api.ComposedSpec`, and replace profile-time `api.ResolveSpecLayers` calls with `api.ComposeSpecLayers`. Keep the original layers: converting an already resolved `Spec` into a new layer loses authored provenance and may carry normalized runtime defaults into a later request. + +For example, a provider can construct its result from application-owned layers: + +```go +func applicationProfile(system string, layers []api.SpecLayer) (aichat.RuntimeProfile, error) { + composed, err := api.ComposeSpecLayers(layers...) + if err != nil { + return aichat.RuntimeProfile{}, err + } + return aichat.RuntimeProfile{ + System: system, + Composed: composed, + }, nil +} +``` + +`ComposeSpecLayers` validates authored structures and merges their values and constraints. Its `Trace` retains the raw layers; its `Spec` is a partial projection with no promise of runtime capability validity. Supply the composition result intact, including `Trace`. A nonempty profile projection without its composition trace is rejected. `System` and `ProviderConfig` retain their existing meanings. + +The chat service adds the explicit request to that raw trace and performs final resolution. Applications that own a different request pipeline use the same boundary: + +```go +func resolveRequest(profile aichat.RuntimeProfile, request api.Spec) (api.ResolvedSpec, error) { + layers := append([]api.SpecLayer(nil), profile.Composed.Trace...) + layers = append(layers, api.RequestSpecLayer("request", request)) + return api.ResolveSpecLayers(layers...) +} +``` + +Call `ResolveSpecLayers` after the complete request is available. It resolves the effective model and fallbacks and applies runtime capability checks. Inspect its separate `Warnings` when implementing a custom pipeline; the chat service logs them before provider admission. Saved model defaults are not loaded by either composition API. + +Malformed application-owned layers remain server errors even if a request would overwrite them. Invalid explicit request fields are client errors. A structurally valid partial profile can be completed or repaired by the final request. Missing nested preset references are owned configuration failures; only an absent or ambiguous requested top-level profile should be classified as an invalid caller selection. + +See [runtime profile composition](../runtimeprofiles/README.md) for raw catalog loading, layer ordering, compact selector precedence, and typed ownership errors. Downstream providers using `Resolved`, including OIPA's settings-backed chat provider, must migrate before adopting the release containing this API change. + +Run the chat composition and ownership coverage without making AI calls: + +```sh +go test ./pkg/aichat -run 'TestAIChat|TestEnforceApprovalRuntimeProfile' -count=1 -ginkgo.no-color +``` diff --git a/pkg/aichat/approval_execution.go b/pkg/aichat/approval_execution.go index 057392d3..481a8166 100644 --- a/pkg/aichat/approval_execution.go +++ b/pkg/aichat/approval_execution.go @@ -57,7 +57,7 @@ func (s *Service) resumeToolApproval(ctx context.Context, threadID string, conti if err != nil { return false, fmt.Errorf("load chat runtime profile: %w", err) } - if err := enforceApprovalRuntimeProfile(continuation.Spec, profile.Resolved); err != nil { + if err := enforceApprovalRuntimeProfile(continuation.Spec, profile.Composed); err != nil { if interruptErr := execution.Interrupt(ctx, err.Error()); interruptErr != nil { return false, fmt.Errorf("%w (interrupt rejected approval continuation: %v)", err, interruptErr) } @@ -140,7 +140,7 @@ func (s *Service) resumeToolApproval(ctx context.Context, threadID string, conti return true, nil } -func enforceApprovalRuntimeProfile(spec api.Spec, resolved api.ResolvedSpec) error { +func enforceApprovalRuntimeProfile(spec api.Spec, resolved api.ComposedSpec) error { if err := enforceRuntimeQuotas(resolved); err != nil { return err } diff --git a/pkg/aichat/approval_execution_test.go b/pkg/aichat/approval_execution_test.go index 35346577..03968615 100644 --- a/pkg/aichat/approval_execution_test.go +++ b/pkg/aichat/approval_execution_test.go @@ -13,19 +13,19 @@ func TestEnforceApprovalRuntimeProfile(t *testing.T) { tests := []struct { name string spec api.Spec - resolved api.ResolvedSpec + resolved api.ComposedSpec wantErr string }{ { name: "persisted model is no longer allowed", spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}}, - resolved: api.ResolvedSpec{Constraints: api.RuntimeConstraints{Models: []string{"claude-sonnet-5"}}}, + resolved: api.ComposedSpec{Constraints: api.RuntimeConstraints{Models: []string{"claude-sonnet-5"}}}, wantErr: `model "gpt-5.6-sol" is outside the current effective model catalog`, }, { name: "current quota is exhausted", spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}}, - resolved: api.ResolvedSpec{Constraints: api.RuntimeConstraints{Quotas: []api.UsageQuota{{ + resolved: api.ComposedSpec{Constraints: api.RuntimeConstraints{Quotas: []api.UsageQuota{{ Name: "monthly", Scope: api.SpecLayerUser, Layer: "claims", TokenLimit: 100, TokensUsed: 100, }}}}, wantErr: `quota "monthly" from layer "claims" exhausted`, @@ -33,7 +33,7 @@ func TestEnforceApprovalRuntimeProfile(t *testing.T) { { name: "changed default does not replace an allowed persisted model", spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}}, - resolved: api.ResolvedSpec{ + resolved: api.ComposedSpec{ Spec: api.Spec{Model: api.Model{Name: "claude-sonnet-5"}}, Constraints: api.RuntimeConstraints{Models: []string{"gpt-5.6-sol", "claude-sonnet-5"}}, }, diff --git a/pkg/aichat/layered_profile_ownership_ginkgo_test.go b/pkg/aichat/layered_profile_ownership_ginkgo_test.go new file mode 100644 index 00000000..63b1a4de --- /dev/null +++ b/pkg/aichat/layered_profile_ownership_ginkgo_test.go @@ -0,0 +1,54 @@ +package aichat + +import ( + "context" + "net/http" + "path/filepath" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/runtimeprofiles" + g "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = g.Describe("Layered chat profile ownership", func() { + g.It("validates malformed server base layers before selecting a missing request profile", func() { + catalogCalls := 0 + provider, err := NewLayeredRuntimeProfileProvider(LayeredRuntimeProfileProviderOptions{ + Resolver: runtimeprofiles.NewResolver(func(context.Context) (*runtimeprofiles.Catalog, error) { + catalogCalls++ + return nil, runtimeprofiles.ErrNotFound + }), + Base: func(context.Context) (RuntimeProfileBase, error) { + return RuntimeProfileBase{Layers: []api.SpecLayer{{Name: "application", Scope: api.SpecLayerGlobal, + Spec: api.Spec{Model: api.Model{Effort: "invalid"}}, + }}}, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + _, err = provider.RuntimeProfile(context.Background(), WithRuntimeProfileRef("missing")) + Expect(runtimeProfileStatus(err)).To(Equal(http.StatusInternalServerError)) + Expect(err).To(MatchError(ContainSubstring("application"))) + Expect(catalogCalls).To(BeZero()) + }) + + g.It("keeps a selected profile's missing preset as a server defect", func() { + ctx := context.Background() + source, err := runtimeprofiles.NewFileSource(runtimeprofiles.FileSourceOptions{ + Kind: runtimeprofiles.KindProfile, Dir: filepath.Join(g.GinkgoT().TempDir(), "profiles"), Label: "test profiles", Implicit: true, + }) + Expect(err).NotTo(HaveOccurred()) + catalog, err := runtimeprofiles.NewCatalog(source) + Expect(err).NotTo(HaveOccurred()) + _, err = catalog.CreateProfile(ctx, source.Info().ID, runtimeprofiles.ProfileInput{Name: "Broken", Presets: []string{"missing"}}) + Expect(err).NotTo(HaveOccurred()) + provider, err := NewLayeredRuntimeProfileProvider(LayeredRuntimeProfileProviderOptions{ + Resolver: runtimeprofiles.NewResolver(func(context.Context) (*runtimeprofiles.Catalog, error) { return catalog, nil }), + Base: func(context.Context) (RuntimeProfileBase, error) { return RuntimeProfileBase{}, nil }, + }) + Expect(err).NotTo(HaveOccurred()) + _, err = provider.RuntimeProfile(ctx, WithRuntimeProfileRef("broken")) + Expect(runtimeProfileStatus(err)).To(Equal(http.StatusInternalServerError)) + Expect(err).To(MatchError(ContainSubstring("missing"))) + }) +}) diff --git a/pkg/aichat/layered_runtime_profile.go b/pkg/aichat/layered_runtime_profile.go index f648e90f..8ba2012d 100644 --- a/pkg/aichat/layered_runtime_profile.go +++ b/pkg/aichat/layered_runtime_profile.go @@ -45,6 +45,9 @@ func NewLayeredRuntimeProfileProvider(options LayeredRuntimeProfileProviderOptio if err != nil { return RuntimeProfile{}, fmt.Errorf("load runtime profile base: %w", err) } + if err := api.ValidateSpecLayers(base.Layers...); err != nil { + return RuntimeProfile{}, fmt.Errorf("chat runtime profile base: %w", err) + } var defaultProfile string if request.Ref == "" && options.DefaultProfile != nil { defaultProfile, err = options.DefaultProfile(ctx) @@ -52,12 +55,13 @@ func NewLayeredRuntimeProfileProvider(options LayeredRuntimeProfileProviderOptio return RuntimeProfile{}, fmt.Errorf("load default runtime profile: %w", err) } } - result, err := options.Resolver.Resolve(ctx, runtimeprofiles.ResolveOptions{ + result, err := options.Resolver.Layers(ctx, runtimeprofiles.ResolveOptions{ BaseLayers: base.Layers, RequestedProfile: request.Ref, DefaultProfile: strings.TrimSpace(defaultProfile), }) if err != nil { var selection *runtimeprofiles.SelectionError - if errors.As(err, &selection) && selection.Origin == runtimeprofiles.SelectionRequested && + var owned *runtimeprofiles.OwnedLayersError + if !errors.As(err, &owned) && errors.As(err, &selection) && selection.Origin == runtimeprofiles.SelectionRequested && (errors.Is(err, runtimeprofiles.ErrNotFound) || errors.Is(err, runtimeprofiles.ErrAmbiguous) || errors.Is(err, runtimeprofiles.ErrCatalogUnavailable)) { @@ -65,8 +69,12 @@ func NewLayeredRuntimeProfileProvider(options LayeredRuntimeProfileProviderOptio } return RuntimeProfile{}, err } + composed, err := api.ComposeSpecLayers(result.Layers...) + if err != nil { + return RuntimeProfile{}, fmt.Errorf("compose chat runtime profile: %w", err) + } return RuntimeProfile{ - System: base.System, Resolved: result.Resolved, ProviderConfig: base.ProviderConfig, + System: base.System, Composed: composed, ProviderConfig: base.ProviderConfig, }, nil }), nil } diff --git a/pkg/aichat/messages.go b/pkg/aichat/messages.go index d1126ad9..44ca553b 100644 --- a/pkg/aichat/messages.go +++ b/pkg/aichat/messages.go @@ -70,7 +70,7 @@ func (s *Service) resolveAttachments(ctx context.Context, messages []UIMessage) } func requestSpec(request ChatRequest, profile RuntimeProfile, attachments map[partLocation]api.AttachmentRef) (api.ResolvedSpec, error) { - override, err := chatModel(request, profile.Resolved.Spec.Model) + override, err := chatModel(request) if err != nil { return api.ResolvedSpec{}, err } @@ -84,12 +84,15 @@ func requestSpec(request ChatRequest, profile RuntimeProfile, attachments map[pa Permissions: api.Permissions{Mode: request.PermissionMode}, SessionID: request.ProviderSessionID, }} - layers := append([]api.SpecLayer(nil), profile.Resolved.Trace...) + layers := append([]api.SpecLayer(nil), profile.Composed.Trace...) resolved, err := api.ResolveSpecLayers(append(layers, user)...) if err != nil { return api.ResolvedSpec{}, fmt.Errorf("resolve chat runtime profile: %w", err) } spec := resolved.Spec + if err := validateChatModel(request, spec.Model); err != nil { + return api.ResolvedSpec{}, err + } baseSystem := strings.TrimSpace(strings.Join([]string{ profile.System, spec.Prompt.System, spec.Prompt.AppendSystem, }, "\n\n")) @@ -131,19 +134,23 @@ func requestSpec(request ChatRequest, profile RuntimeProfile, attachments map[pa return resolved, nil } -func chatModel(request ChatRequest, fallback api.Model) (api.Model, error) { - selected := fallback +func chatModel(request ChatRequest) (api.Model, error) { + var selected api.Model if request.Runtime != nil { selected = *request.Runtime } else if model := strings.TrimSpace(request.Model); model != "" { selected = api.Model{Name: model} } - if strings.TrimSpace(selected.Name) == "" { - return api.Model{}, fmt.Errorf("chat model is required") + if err := api.ValidateSpecLayers(api.RequestSpecLayer("chat request", api.Spec{Model: selected})); err != nil { + return api.Model{}, err + } + expanded, err := selected.Expand() + if err != nil { + return api.Model{}, fmt.Errorf("invalid chat runtime: %w", err) } if request.ReasoningEffort != "" { - if selected.Effort != "" && selected.Effort != request.ReasoningEffort { - return api.Model{}, fmt.Errorf("chat runtime effort %q conflicts with reasoning effort %q", selected.Effort, request.ReasoningEffort) + if expanded.Effort != "" && expanded.Effort != request.ReasoningEffort { + return api.Model{}, fmt.Errorf("chat runtime effort %q conflicts with reasoning effort %q", expanded.Effort, request.ReasoningEffort) } selected.Effort = request.ReasoningEffort } @@ -153,14 +160,10 @@ func chatModel(request ChatRequest, fallback api.Model) (api.Model, error) { } selected.Temperature = request.Temperature } - // Resolve, not just Expand: the browser sends the chat catalog's id form - // ("anthropic/claude-opus-5"), which Expand leaves untouched because it - // carries no compact-form ":" separator — keeping the provider glued to the - // name and the mode empty for the configured default to claim. - resolved, err := ai.Resolve(selected) - if err != nil { - return api.Model{}, fmt.Errorf("invalid chat runtime: %w", err) - } + return selected, nil +} + +func validateChatModel(request ChatRequest, resolved api.Model) error { if request.Runtime != nil && strings.TrimSpace(request.Model) != "" { // The catalog id names a model, never a mode, so it is resolved under the // structured runtime's mode. Resolving it bare would take the provider's @@ -168,13 +171,13 @@ func chatModel(request ChatRequest, fallback api.Model) (api.Model, error) { // authored — the two fields would contradict each other by construction. bare, err := ai.Resolve(api.Model{Name: strings.TrimSpace(request.Model), Mode: resolved.Mode}) if err != nil { - return api.Model{}, fmt.Errorf("invalid chat model %q: %w", request.Model, err) + return fmt.Errorf("invalid chat model %q: %w", request.Model, err) } if bare.Name != resolved.Name || bare.Provider != resolved.Provider { - return api.Model{}, fmt.Errorf("chat model %q conflicts with structured runtime %s/%s", request.Model, api.RuntimeOf(resolved.Provider, resolved.Mode), resolved.Name) + return fmt.Errorf("chat model %q conflicts with structured runtime %s/%s", request.Model, api.RuntimeOf(resolved.Provider, resolved.Mode), resolved.Name) } } - return resolved, nil + return nil } func canonicalMessages(messages []UIMessage, attachments map[partLocation]api.AttachmentRef) ([]api.Message, error) { diff --git a/pkg/aichat/messages_model_ginkgo_test.go b/pkg/aichat/messages_model_ginkgo_test.go index 0b486561..30da778e 100644 --- a/pkg/aichat/messages_model_ginkgo_test.go +++ b/pkg/aichat/messages_model_ginkgo_test.go @@ -7,32 +7,40 @@ import ( "github.com/flanksource/captain/pkg/api" ) -var _ = ginkgo.Describe("chatModel", func() { +var _ = ginkgo.Describe("chatModel composition", func() { // The chat catalog (/api/chat/models) publishes ids as "provider/model", and // the browser sends that id back in ChatRequest.Model. Resolution must yield // the provider the id names rather than leaving it empty for the configured // default to claim — that gap is how an agent-backed session came back as api. - // A catalog id carries no mode, so the mode is the provider's own default. - fallback := api.Model{Name: "gpt-5.6-luna", Mode: api.ModeAgent} + // A catalog id carries no mode, so the authored profile mode survives. + resolve := func(request ChatRequest) (api.Model, error) { + composed, err := api.ComposeSpecLayers(api.SpecLayer{Name: "application", Scope: api.SpecLayerGlobal, + Spec: api.Spec{Model: api.Model{Name: "gpt-5.6-luna", Mode: api.ModeAPI}}, + }) + Expect(err).NotTo(HaveOccurred()) + request.Messages = []UIMessage{{Role: "user", Parts: []UIPart{{Type: "text", Text: "Hello"}}}} + resolved, err := requestSpec(request, RuntimeProfile{Composed: composed}, nil) + return resolved.Spec.Model, err + } ginkgo.DescribeTable("resolves a catalog id to its concrete runtime", func(id string, wantName string, wantProvider *api.ModelProvider, wantMode api.RuntimeMode) { - resolved, err := chatModel(ChatRequest{Model: id}, fallback) + resolved, err := resolve(ChatRequest{Model: id}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Name).To(Equal(wantName)) Expect(resolved.Provider).To(Equal(wantProvider)) Expect(resolved.Mode).To(Equal(wantMode)) }, - ginkgo.Entry("anthropic catalog id", "anthropic/claude-opus-5", "claude-opus-5", api.Anthropic, api.Anthropic.DefaultMode), - ginkgo.Entry("openai catalog id", "openai/gpt-5.6-sol", "gpt-5.6-sol", api.OpenAI, api.OpenAI.DefaultMode), + ginkgo.Entry("anthropic catalog id", "anthropic/claude-opus-5", "claude-opus-5", api.Anthropic, api.ModeAPI), + ginkgo.Entry("openai catalog id", "openai/gpt-5.6-sol", "gpt-5.6-sol", api.OpenAI, api.ModeAPI), ) ginkgo.It("does not treat a catalog id and its own structured runtime as a conflict", func() { runtime := api.Model{Name: "claude-opus-5", Mode: api.ModeAPI} - resolved, err := chatModel(ChatRequest{ + resolved, err := resolve(ChatRequest{ Model: "anthropic/claude-opus-5", Runtime: &runtime, - }, fallback) + }) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Provider).To(Equal(api.Anthropic)) Expect(resolved.Mode).To(Equal(api.ModeAPI)) @@ -40,10 +48,10 @@ var _ = ginkgo.Describe("chatModel", func() { ginkgo.It("still rejects a model that disagrees with the structured runtime", func() { runtime := api.Model{Name: "claude-opus-5", Mode: api.ModeAPI} - _, err := chatModel(ChatRequest{ + _, err := resolve(ChatRequest{ Model: "openai/gpt-5.6-sol", Runtime: &runtime, - }, fallback) + }) Expect(err).To(MatchError(ContainSubstring("conflicts with structured runtime"))) }) }) diff --git a/pkg/aichat/profile_composition_http_ginkgo_test.go b/pkg/aichat/profile_composition_http_ginkgo_test.go new file mode 100644 index 00000000..0968cede --- /dev/null +++ b/pkg/aichat/profile_composition_http_ginkgo_test.go @@ -0,0 +1,79 @@ +package aichat_test + +import ( + "context" + "net/http" + "net/http/httptest" + + "github.com/flanksource/captain/pkg/aichat" + "github.com/flanksource/captain/pkg/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Chat profile composition HTTP", func() { + serviceFor := func(spec api.Spec, resolver *fakeResolver) *aichat.Service { + return aichat.NewService(aichat.ServiceOptions{Resolver: resolver, + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context, ...aichat.RuntimeProfileOption) (aichat.RuntimeProfile, error) { + return aichat.RuntimeProfile{Composed: api.ComposedSpec{Trace: []api.SpecLayer{ + {Name: "application", Scope: api.SpecLayerGlobal, Spec: spec}, + }}}, nil + }), + }) + } + request := aichat.ChatRequest{Messages: []aichat.UIMessage{{Role: "user", Parts: []aichat.UIPart{{Type: "text", Text: "Hello"}}}}} + + It("allows a request runtime to repair structurally valid server defaults", func() { + provider := &fakeStreamingProvider{} + resolver := &fakeResolver{provider: provider} + service := serviceFor(api.Spec{Model: api.Model{Name: "gpt-5.6-sol", Mode: api.ModeCLI}, + Permissions: api.Permissions{Tools: api.Tools{"Bash": api.ToolPolicyDeny}}, + }, resolver) + selected := request + selected.Model = "agent:claude-sonnet-5" + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, requestJSON(http.MethodPost, "/api/chat", selected)) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(provider.specs).To(HaveLen(1)) + Expect(provider.specs[0].Mode).To(Equal(api.ModeAgent)) + Expect(provider.specs[0].Permissions.Tools).To(Equal(api.Tools{"Bash": api.ToolPolicyDeny})) + }) + + DescribeTable("retains server ownership for malformed defaults", func(spec api.Spec, fragment string) { + resolver := &fakeResolver{provider: &fakeStreamingProvider{}} + service := serviceFor(spec, resolver) + selected := request + selected.Runtime = &api.Model{Name: "agent:claude-sonnet-5:high"} + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, requestJSON(http.MethodPost, "/api/chat", selected)) + Expect(response.Code).To(Equal(http.StatusInternalServerError), response.Body.String()) + Expect(response.Body.String()).To(And(ContainSubstring("application"), ContainSubstring(fragment))) + Expect(resolver.configs).To(BeEmpty()) + }, + Entry("invalid authored mode", api.Spec{Model: api.Model{Name: "sonnet", Mode: "invalid"}}, "mode"), + Entry("invalid authored effort", api.Spec{Model: api.Model{Name: "sonnet", Effort: "invalid"}}, "effort"), + ) + + It("serves catalogs for a model-free partial profile without invoking a provider", func() { + resolver := &fakeResolver{} + service := serviceFor(api.Spec{Permissions: api.Permissions{Tools: api.Tools{"Bash": api.ToolPolicyDeny}}}, resolver) + for _, path := range []string{"/api/chat/models", "/api/chat/runtimes"} { + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil)) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + } + Expect(resolver.configs).To(BeEmpty()) + }) + + It("rejects an invalid explicit runtime as a bad request before provider creation", func() { + resolver := &fakeResolver{} + service := serviceFor(api.Spec{Model: api.Model{Name: "sonnet", Mode: api.ModeAPI}}, resolver) + selected := request + selected.Runtime = &api.Model{Name: "agent:sonnet:high", Mode: "invalid"} + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, requestJSON(http.MethodPost, "/api/chat", selected)) + Expect(response.Code).To(Equal(http.StatusBadRequest), response.Body.String()) + Expect(response.Body.String()).To(ContainSubstring("mode")) + Expect(resolver.configs).To(BeEmpty()) + }) +}) diff --git a/pkg/aichat/provider_config.go b/pkg/aichat/provider_config.go index cf7c23af..251d2c51 100644 --- a/pkg/aichat/provider_config.go +++ b/pkg/aichat/provider_config.go @@ -11,7 +11,7 @@ import ( "github.com/flanksource/captain/pkg/api/registry" ) -func annotateProfileModels(resolved api.ResolvedSpec, models ModelCatalogResponse) { +func annotateProfileModels(resolved api.ComposedSpec, models ModelCatalogResponse) { if len(resolved.Constraints.Models) == 0 { return } @@ -26,7 +26,7 @@ func annotateProfileModels(resolved api.ResolvedSpec, models ModelCatalogRespons } } -func annotateProfileRuntimes(resolved api.ResolvedSpec, runtimes []api.RuntimeFamily) { +func annotateProfileRuntimes(resolved api.ComposedSpec, runtimes []api.RuntimeFamily) { if len(resolved.Constraints.Models) == 0 { return } @@ -64,17 +64,17 @@ func restrictedAvailability(layer *api.SpecLayer, allowed []string) api.Availabi } } -func modelRestrictionLayer(resolved api.ResolvedSpec, model api.Model) *api.SpecLayer { +func modelRestrictionLayer(resolved api.ComposedSpec, model api.Model) *api.SpecLayer { for index := len(resolved.Trace) - 1; index >= 0; index-- { layer := &resolved.Trace[index] - if len(layer.Constraints.Models) > 0 && !(api.ResolvedSpec{Constraints: layer.Constraints}).AllowsModel(model) { + if len(layer.Constraints.Models) > 0 && !(api.ComposedSpec{Constraints: layer.Constraints}).AllowsModel(model) { return layer } } return nil } -func runtimeRestrictionLayer(resolved api.ResolvedSpec, provider *api.ModelProvider, mode api.RuntimeMode) *api.SpecLayer { +func runtimeRestrictionLayer(resolved api.ComposedSpec, provider *api.ModelProvider, mode api.RuntimeMode) *api.SpecLayer { for index := len(resolved.Trace) - 1; index >= 0; index-- { layer := &resolved.Trace[index] if len(layer.Constraints.Models) > 0 && !runtimeAllowed(layer.Constraints.Models, provider, mode) { @@ -96,7 +96,7 @@ func runtimeAllowed(models []string, provider *api.ModelProvider, mode api.Runti return true } } - resolved := api.ResolvedSpec{Constraints: api.RuntimeConstraints{Models: models}} + resolved := api.ComposedSpec{Constraints: api.RuntimeConstraints{Models: models}} for _, model := range provider.Models() { if !model.Preferred { continue diff --git a/pkg/aichat/request_layers_ginkgo_test.go b/pkg/aichat/request_layers_ginkgo_test.go new file mode 100644 index 00000000..74cdffef --- /dev/null +++ b/pkg/aichat/request_layers_ginkgo_test.go @@ -0,0 +1,59 @@ +package aichat + +import ( + "context" + + "github.com/flanksource/captain/pkg/api" + g "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = g.Describe("Chat request layers", func() { + profile := func(spec api.Spec) RuntimeProfile { + return RuntimeProfile{Composed: api.ComposedSpec{Trace: []api.SpecLayer{ + {Name: "application", Scope: api.SpecLayerGlobal, Spec: spec}, + }}} + } + request := ChatRequest{Messages: []UIMessage{{Role: "user", Parts: []UIPart{{Type: "text", Text: "Hello"}}}}} + + g.It("keeps inherited model fields out of the explicit request trace", func() { + base := profile(api.Spec{Model: api.Model{Name: "gpt-5.6-sol", Mode: api.ModeAPI, Effort: api.EffortHigh}}) + service := NewService(ServiceOptions{Profile: RuntimeProfileProviderFunc(func(context.Context, ...RuntimeProfileOption) (RuntimeProfile, error) { + return base, nil + })}) + loaded, err := service.runtimeProfile(context.Background()) + Expect(err).NotTo(HaveOccurred()) + resolved, err := requestSpec(request, loaded, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Trace[1].Spec.Model).To(Equal(api.Model{})) + Expect(resolved.Spec.Mode).To(Equal(api.ModeAPI)) + Expect(resolved.Spec.Effort).To(Equal(api.EffortHigh)) + }) + + g.It("merges a bare catalog model only when composing the final request", func() { + base := profile(api.Spec{Model: api.Model{Name: "gpt-5.6-luna", Mode: api.ModeAPI, Effort: api.EffortHigh}}) + base.Composed.Spec = base.Composed.Trace[0].Spec + selected := request + selected.Model = "openai/gpt-5.6-sol" + resolved, err := requestSpec(selected, base, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Trace[1].Spec.Model).To(Equal(api.Model{Name: "openai/gpt-5.6-sol"})) + Expect(resolved.Spec.Mode).To(Equal(api.ModeAPI)) + Expect(resolved.Spec.Effort).To(Equal(api.EffortHigh)) + }) + + g.DescribeTable("rejects authored model defects before expanding compact selectors", func(model api.Model) { + _, err := chatModel(ChatRequest{Runtime: &model}) + Expect(err).To(HaveOccurred()) + }, + g.Entry("invalid mode", api.Model{Name: "agent:sonnet:high", Mode: "invalid"}), + g.Entry("invalid effort", api.Model{Name: "agent:sonnet:high", Effort: "invalid"}), + ) + + g.It("preserves the caller's compact selector in the raw request layer", func() { + model := api.Model{Name: "agent:sonnet:high"} + selected, err := chatModel(ChatRequest{Runtime: &model}) + Expect(err).NotTo(HaveOccurred()) + Expect(selected).To(Equal(model)) + }) +}) diff --git a/pkg/aichat/runtime_profile.go b/pkg/aichat/runtime_profile.go index acdaa4d7..545bd033 100644 --- a/pkg/aichat/runtime_profile.go +++ b/pkg/aichat/runtime_profile.go @@ -11,12 +11,12 @@ import ( "github.com/flanksource/captain/pkg/api" ) -// RuntimeProfile is the request-scoped, hierarchically resolved application -// configuration for a chat. Resolved carries the effective Spec, constraints, -// and ordered provenance; provider credentials remain runtime-only. +// RuntimeProfile is the request-scoped application configuration for a chat. +// Composed carries structurally validated defaults, constraints and raw layers; +// runtime capability validation waits for the complete chat request. type RuntimeProfile struct { System string - Resolved api.ResolvedSpec + Composed api.ComposedSpec ProviderConfig api.Config } @@ -77,17 +77,17 @@ func (s *Service) runtimeProfile(ctx context.Context, options ...RuntimeProfileO if err != nil { return RuntimeProfile{}, err } - if len(profile.Resolved.Trace) == 0 { - if !api.IsEmpty(profile.Resolved.Spec) || !api.IsEmpty(profile.Resolved.Constraints) { - return RuntimeProfile{}, fmt.Errorf("chat runtime profile must include its resolution trace") + if len(profile.Composed.Trace) == 0 { + if !api.IsEmpty(profile.Composed.Spec) || !api.IsEmpty(profile.Composed.Constraints) { + return RuntimeProfile{}, fmt.Errorf("chat runtime profile must include its composition trace") } return profile, nil } - resolved, err := api.ResolveSpecLayers(profile.Resolved.Trace...) + composed, err := api.ComposeSpecLayers(profile.Composed.Trace...) if err != nil { return RuntimeProfile{}, fmt.Errorf("resolve chat runtime profile: %w", err) } - profile.Resolved = resolved + profile.Composed = composed return profile, nil } @@ -127,7 +127,7 @@ func requestErrorStatus(err error) int { return http.StatusBadRequest } -func enforceRuntimeProfile(request ChatRequest, resolved api.ResolvedSpec) error { +func enforceRuntimeProfile(request ChatRequest, resolved api.ComposedSpec) error { if err := enforceRuntimeQuotas(resolved); err != nil { return err } @@ -153,7 +153,7 @@ func enforceRuntimeProfile(request ChatRequest, resolved api.ResolvedSpec) error return nil } -func enforceRuntimeQuotas(resolved api.ResolvedSpec) error { +func enforceRuntimeQuotas(resolved api.ComposedSpec) error { for _, quota := range resolved.Constraints.Quotas { if quota.CostLimitUSD > 0 && quota.CostUsedUSD >= quota.CostLimitUSD { return requestError{status: http.StatusPaymentRequired, text: fmt.Sprintf( @@ -170,3 +170,45 @@ func enforceRuntimeQuotas(resolved api.ResolvedSpec) error { } return nil } + +func (s *Service) handleRuntimes(w http.ResponseWriter, request *http.Request) { + profile, err := s.runtimeProfile(request.Context(), WithRuntimeProfileRef(request.URL.Query().Get("runtimeProfile"))) + if err != nil { + http.Error(w, fmt.Sprintf("load chat runtime profile: %v", err), runtimeProfileStatus(err)) + return + } + runtimes, err := s.resolver.Runtimes(request.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + if err := s.annotateConfiguredRuntimes(request.Context(), runtimes); err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + annotateProfileRuntimes(profile.Composed, runtimes) + if err := writeJSON(w, http.StatusOK, runtimes); err != nil { + serviceLog.Errorf("write chat runtimes response: %v", err) + } +} + +func (s *Service) handleModels(w http.ResponseWriter, request *http.Request) { + profile, err := s.runtimeProfile(request.Context(), WithRuntimeProfileRef(request.URL.Query().Get("runtimeProfile"))) + if err != nil { + http.Error(w, fmt.Sprintf("load chat runtime profile: %v", err), runtimeProfileStatus(err)) + return + } + models, err := s.resolver.Models(request.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + if err := s.annotateConfiguredModels(request.Context(), models); err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + annotateProfileModels(profile.Composed, models) + if err := writeJSON(w, http.StatusOK, models); err != nil { + serviceLog.Errorf("write chat models response: %v", err) + } +} diff --git a/pkg/aichat/runtime_profile_ginkgo_test.go b/pkg/aichat/runtime_profile_ginkgo_test.go index cab8877b..eb7266f6 100644 --- a/pkg/aichat/runtime_profile_ginkgo_test.go +++ b/pkg/aichat/runtime_profile_ginkgo_test.go @@ -79,19 +79,19 @@ var _ = Describe("Resolved runtime profiles", func() { It("reports malformed server profiles as internal errors", func() { cases := []struct { name string - resolved api.ResolvedSpec + composed api.ComposedSpec message string }{ { name: "missing trace", - resolved: api.ResolvedSpec{Spec: api.Spec{ + composed: api.ComposedSpec{Spec: api.Spec{ Model: api.Model{Name: "gpt-5.4"}, }}, - message: "must include its resolution trace", + message: "must include its composition trace", }, { name: "invalid trace", - resolved: api.ResolvedSpec{Trace: []api.SpecLayer{{ + composed: api.ComposedSpec{Trace: []api.SpecLayer{{ Name: "broken", Scope: api.SpecLayerScope("invalid"), }}}, message: "invalid scope", @@ -101,7 +101,7 @@ var _ = Describe("Resolved runtime profiles", func() { for _, test := range cases { service := aichat.NewService(aichat.ServiceOptions{ Profile: aichat.RuntimeProfileProviderFunc(func(context.Context, ...aichat.RuntimeProfileOption) (aichat.RuntimeProfile, error) { - return aichat.RuntimeProfile{Resolved: test.resolved}, nil + return aichat.RuntimeProfile{Composed: test.composed}, nil }), }) response := httptest.NewRecorder() @@ -168,7 +168,7 @@ var _ = Describe("Resolved runtime profiles", func() { }) func mustRuntimeProfile(layers ...api.SpecLayer) aichat.RuntimeProfile { - resolved, err := api.ResolveSpecLayers(layers...) + composed, err := api.ComposeSpecLayers(layers...) Expect(err).NotTo(HaveOccurred()) - return aichat.RuntimeProfile{Resolved: resolved} + return aichat.RuntimeProfile{Composed: composed} } diff --git a/pkg/aichat/service.go b/pkg/aichat/service.go index a223a0f1..0436cac2 100644 --- a/pkg/aichat/service.go +++ b/pkg/aichat/service.go @@ -8,9 +8,7 @@ import ( "net/http" "strings" "sync" - "time" - "github.com/flanksource/captain/pkg/ai" aitools "github.com/flanksource/captain/pkg/ai/tools" "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/database" @@ -106,48 +104,6 @@ func (s *Service) Handler() http.Handler { return mux } -func (s *Service) handleRuntimes(w http.ResponseWriter, request *http.Request) { - profile, err := s.runtimeProfile(request.Context(), WithRuntimeProfileRef(request.URL.Query().Get("runtimeProfile"))) - if err != nil { - http.Error(w, fmt.Sprintf("load chat runtime profile: %v", err), runtimeProfileStatus(err)) - return - } - runtimes, err := s.resolver.Runtimes(request.Context()) - if err != nil { - http.Error(w, err.Error(), http.StatusServiceUnavailable) - return - } - if err := s.annotateConfiguredRuntimes(request.Context(), runtimes); err != nil { - http.Error(w, err.Error(), http.StatusServiceUnavailable) - return - } - annotateProfileRuntimes(profile.Resolved, runtimes) - if err := writeJSON(w, http.StatusOK, runtimes); err != nil { - serviceLog.Errorf("write chat runtimes response: %v", err) - } -} - -func (s *Service) handleModels(w http.ResponseWriter, request *http.Request) { - profile, err := s.runtimeProfile(request.Context(), WithRuntimeProfileRef(request.URL.Query().Get("runtimeProfile"))) - if err != nil { - http.Error(w, fmt.Sprintf("load chat runtime profile: %v", err), runtimeProfileStatus(err)) - return - } - models, err := s.resolver.Models(request.Context()) - if err != nil { - http.Error(w, err.Error(), http.StatusServiceUnavailable) - return - } - if err := s.annotateConfiguredModels(request.Context(), models); err != nil { - http.Error(w, err.Error(), http.StatusServiceUnavailable) - return - } - annotateProfileModels(profile.Resolved, models) - if err := writeJSON(w, http.StatusOK, models); err != nil { - serviceLog.Errorf("write chat models response: %v", err) - } -} - func (s *Service) handleChat(w http.ResponseWriter, request *http.Request) { var chat ChatRequest if err := json.NewDecoder(request.Body).Decode(&chat); err != nil { @@ -190,7 +146,7 @@ func (s *Service) handleChat(w http.ResponseWriter, request *http.Request) { http.Error(w, err.Error(), requestErrorStatus(err)) return } - if err := enforceRuntimeProfile(chat, profile.Resolved); err != nil { + if err := enforceRuntimeProfile(chat, profile.Composed); err != nil { http.Error(w, err.Error(), requestErrorStatus(err)) return } @@ -207,6 +163,9 @@ func (s *Service) handleChat(w http.ResponseWriter, request *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } + for _, warning := range resolved.Warnings { + serviceLog.Warnf("%s", warning) + } spec := resolved.Spec set, err := s.loadTools(request.Context()) if err != nil { @@ -537,90 +496,3 @@ func (s *Service) bindThreadRuntime( }() return out } - -func closeExecution(execution Execution) { - if execution == nil { - return - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := execution.Close(ctx); err != nil { - serviceLog.Errorf("close authoritative chat execution: %v", err) - } -} - -func (s *Service) persistIncoming(ctx context.Context, request ChatRequest, runtime api.Model) error { - if request.ThreadID == "" || request.Trigger != "submit-message" || request.MessageID != "" || len(request.Messages) == 0 { - return nil - } - last := request.Messages[len(request.Messages)-1] - if !strings.EqualFold(last.Role, string(api.RoleUser)) { - return nil - } - store, err := s.threads(ctx) - if err != nil { - return err - } - if candidates := runtime.Candidates(); len(candidates) == 1 { - if err := store.SetRuntime(ctx, request.ThreadID, candidates[0]); err != nil { - return err - } - } - if err := store.AppendMessage(ctx, request.ThreadID, last); err != nil { - return err - } - // Names an as-yet-unnamed thread after the message that opened it. The store - // keeps this from displacing a title the agent or the user already chose. - s.setThreadTitle(ctx, request.ThreadID, TitleUpdate{ - Title: derivedTitle(request.Messages), Source: TitleSourceDerived, - }) - return nil -} - -// persistEvent accrues a completed turn against its thread. The thread returned -// by AddUsage carries the conversation's running total, which is recorded on -// costs so the finish part can report cumulative rather than per-turn spend. -func (s *Service) persistEvent(ctx context.Context, threadID string, event api.Event, model api.Model, costs *TurnCosts) error { - store, err := s.threads(ctx) - if err != nil { - return err - } - if event.SessionID != "" { - if err := store.SetProviderSession(ctx, threadID, event.SessionID); err != nil { - return fmt.Errorf("persist provider session: %w", err) - } - } - if event.Kind != api.EventResult || event.Usage == nil { - return nil - } - thread, err := store.AddUsage(ctx, threadID, TurnUsage{ - InputTokens: event.Usage.InputTokens, OutputTokens: event.Usage.OutputTokens, - ReasoningTokens: event.Usage.ReasoningTokens, CacheReadTokens: event.Usage.CacheReadTokens, - CacheWriteTokens: event.Usage.CacheWriteTokens, CostUSD: event.CostUSD, - }) - if err != nil { - return fmt.Errorf("persist thread usage: %w", err) - } - if costs != nil { - costs.Breakdown = costBreakdownMetadata(model, *event.Usage, event.CostUSD) - if thread != nil { - costs.ThreadCostUSD = thread.TotalCostUSD - } - } - return nil -} - -func costBreakdownMetadata(model api.Model, usage api.Usage, providerCostUSD float64) *CostBreakdownMetadata { - cost := ai.PriceUsage(model.Provider, model.Name, usage, providerCostUSD) - return &CostBreakdownMetadata{ - Model: cost.Model, - InputUSD: cost.InputCost, - OutputUSD: cost.OutputCost, - ReasoningUSD: cost.ReasoningCost, - CacheReadUSD: cost.CacheReadCost, - // genkit reports no cache-write tokens on the API backends, so this - // stays zero there rather than being silently omitted. - CacheWriteUSD: cost.CacheWriteCost, - TotalUSD: cost.Total(), - } -} diff --git a/pkg/aichat/service_persistence.go b/pkg/aichat/service_persistence.go new file mode 100644 index 00000000..d4948901 --- /dev/null +++ b/pkg/aichat/service_persistence.go @@ -0,0 +1,98 @@ +package aichat + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" +) + +func closeExecution(execution Execution) { + if execution == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := execution.Close(ctx); err != nil { + serviceLog.Errorf("close authoritative chat execution: %v", err) + } +} + +func (s *Service) persistIncoming(ctx context.Context, request ChatRequest, runtime api.Model) error { + if request.ThreadID == "" || request.Trigger != "submit-message" || request.MessageID != "" || len(request.Messages) == 0 { + return nil + } + last := request.Messages[len(request.Messages)-1] + if !strings.EqualFold(last.Role, string(api.RoleUser)) { + return nil + } + store, err := s.threads(ctx) + if err != nil { + return err + } + if candidates := runtime.Candidates(); len(candidates) == 1 { + if err := store.SetRuntime(ctx, request.ThreadID, candidates[0]); err != nil { + return err + } + } + if err := store.AppendMessage(ctx, request.ThreadID, last); err != nil { + return err + } + // Names an as-yet-unnamed thread after the message that opened it. The store + // keeps this from displacing a title the agent or the user already chose. + s.setThreadTitle(ctx, request.ThreadID, TitleUpdate{ + Title: derivedTitle(request.Messages), Source: TitleSourceDerived, + }) + return nil +} + +// persistEvent accrues a completed turn against its thread. The thread returned +// by AddUsage carries the conversation's running total, which is recorded on +// costs so the finish part can report cumulative rather than per-turn spend. +func (s *Service) persistEvent(ctx context.Context, threadID string, event api.Event, model api.Model, costs *TurnCosts) error { + store, err := s.threads(ctx) + if err != nil { + return err + } + if event.SessionID != "" { + if err := store.SetProviderSession(ctx, threadID, event.SessionID); err != nil { + return fmt.Errorf("persist provider session: %w", err) + } + } + if event.Kind != api.EventResult || event.Usage == nil { + return nil + } + thread, err := store.AddUsage(ctx, threadID, TurnUsage{ + InputTokens: event.Usage.InputTokens, OutputTokens: event.Usage.OutputTokens, + ReasoningTokens: event.Usage.ReasoningTokens, CacheReadTokens: event.Usage.CacheReadTokens, + CacheWriteTokens: event.Usage.CacheWriteTokens, CostUSD: event.CostUSD, + }) + if err != nil { + return fmt.Errorf("persist thread usage: %w", err) + } + if costs != nil { + costs.Breakdown = costBreakdownMetadata(model, *event.Usage, event.CostUSD) + if thread != nil { + costs.ThreadCostUSD = thread.TotalCostUSD + } + } + return nil +} + +func costBreakdownMetadata(model api.Model, usage api.Usage, providerCostUSD float64) *CostBreakdownMetadata { + cost := ai.PriceUsage(model.Provider, model.Name, usage, providerCostUSD) + return &CostBreakdownMetadata{ + Model: cost.Model, + InputUSD: cost.InputCost, + OutputUSD: cost.OutputCost, + ReasoningUSD: cost.ReasoningCost, + CacheReadUSD: cost.CacheReadCost, + // genkit reports no cache-write tokens on the API backends, so this + // stays zero there rather than being silently omitted. + CacheWriteUSD: cost.CacheWriteCost, + TotalUSD: cost.Total(), + } +} diff --git a/pkg/api/runtime_profiles.go b/pkg/api/runtime_profiles.go index c1bbb2d6..466d9824 100644 --- a/pkg/api/runtime_profiles.go +++ b/pkg/api/runtime_profiles.go @@ -99,10 +99,14 @@ func RuntimeProfileLayers(request RuntimeProfileResolveRequest) ([]SpecLayer, er Name: preset.Name, Scope: preset.Scope, Spec: preset.Spec.ToSpec(), }) } - return append(layers, SpecLayer{ + layers = append(layers, SpecLayer{ ID: request.Profile.ID + ":spec", Source: SpecLayerSourceProfile, Name: request.Profile.Name + " run spec", Scope: SpecLayerSurface, Spec: request.Profile.Spec, - }), nil + }) + if err := ValidateSpecLayers(layers...); err != nil { + return nil, err + } + return layers, nil } // ResolveRuntimeProfile resolves and validates a profile in isolation for preview. @@ -116,22 +120,6 @@ func ResolveRuntimeProfile(request RuntimeProfileResolveRequest) (ResolvedSpec, if err != nil { return ResolvedSpec{}, fmt.Errorf("resolve runtime profile %q: %w", request.Profile.Name, err) } - // The layers carry an authored model (name plus mode); the validators below - // need the resolved adapter, which no longer survives serialization. Derive it - // here rather than letting them reject their own input. - if strings.TrimSpace(resolved.Spec.Name) != "" { - model, modelErr := ResolveModel(resolved.Spec.Model) - if modelErr != nil { - return ResolvedSpec{}, fmt.Errorf("resolve runtime profile %q model: %w", request.Profile.Name, modelErr) - } - resolved.Spec.Model = model - } - if err := ValidateResolvedSandbox(resolved.Spec); err != nil { - return ResolvedSpec{}, err - } - if err := validateResolvedPermissions(resolved.Spec); err != nil { - return ResolvedSpec{}, err - } return resolved, nil } @@ -225,41 +213,8 @@ func validateRuntimePreset(preset RuntimePreset) error { if strings.TrimSpace(preset.Name) == "" { return fmt.Errorf("runtime preset %q name is required", preset.ID) } - if scopeRank(preset.Scope) < 0 { - return fmt.Errorf("runtime preset %q has invalid scope %q", preset.ID, preset.Scope) - } - spec := preset.Spec.ToSpec() - if !IsEmpty(spec.Model) { - if err := spec.Model.Validate(); err != nil { - return fmt.Errorf("runtime preset %q model: %w", preset.ID, err) - } - } - if err := spec.Budget.Validate(); err != nil { - return fmt.Errorf("runtime preset %q budget: %w", preset.ID, err) - } - if err := spec.Permissions.Validate(); err != nil { - return fmt.Errorf("runtime preset %q permissions: %w", preset.ID, err) - } - if err := spec.ToolPreferences.Validate(); err != nil { - return fmt.Errorf("runtime preset %q: %w", preset.ID, err) - } - if err := spec.ToolPolicy.Validate(); err != nil { - return fmt.Errorf("runtime preset %q: %w", preset.ID, err) - } - if spec.Sandbox != nil { - if err := spec.Sandbox.Validate(); err != nil { - return fmt.Errorf("runtime preset %q sandbox: %w", preset.ID, err) - } - } - return nil -} - -func validateResolvedPermissions(spec Spec) error { - warnings := UnsupportedPermissions(spec) - if len(warnings) > 0 { - return fmt.Errorf("%s", warnings[0]) - } - return nil + return ValidateSpecLayers(SpecLayer{ID: preset.ID, Name: preset.Name, Scope: preset.Scope, + Source: SpecLayerSourcePreset, Spec: preset.Spec.ToSpec()}) } // UnsupportedPermissions reports settings the selected runtime cannot honour. diff --git a/pkg/api/runtime_profiles_ginkgo_test.go b/pkg/api/runtime_profiles_ginkgo_test.go index fceb3c52..9655935f 100644 --- a/pkg/api/runtime_profiles_ginkgo_test.go +++ b/pkg/api/runtime_profiles_ginkgo_test.go @@ -111,8 +111,8 @@ var _ = Describe("Runtime profiles", func() { Expect(err).To(MatchError(ContainSubstring(`repeats preset "Org defaults"`))) }) - It("rejects a permission mode the resolved runtime cannot honour", func() { - _, err := api.ResolveRuntimeProfile(api.RuntimeProfileResolveRequest{ + It("warns about a permission mode the resolved runtime cannot honour", func() { + resolved, err := api.ResolveRuntimeProfile(api.RuntimeProfileResolveRequest{ Profile: api.RuntimeProfile{ ID: "codex", Name: "Codex", Spec: api.Spec{ Model: api.Model{Name: "gpt-5", Mode: api.ModeAgent}, @@ -121,7 +121,8 @@ var _ = Describe("Runtime profiles", func() { }, }) - Expect(err).To(MatchError(ContainSubstring("is not available for openai agent"))) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Warnings).To(Equal([]string{`permissions.mode "dontAsk" is not available for openai agent`})) }) // Profile validation shares RequireToolPolicySupport's one relaxation: an @@ -171,8 +172,8 @@ var _ = Describe("Runtime profiles", func() { Expect(resolved.Spec.Permissions.Mode).To(Equal(api.PermissionPlan)) }) - It("rejects caller-tool policy on a runtime that cannot serve caller tools", func() { - _, err := api.ResolveRuntimeProfile(api.RuntimeProfileResolveRequest{ + It("warns about caller-tool policy on a runtime that cannot serve caller tools", func() { + resolved, err := api.ResolveRuntimeProfile(api.RuntimeProfileResolveRequest{ Profile: api.RuntimeProfile{ ID: "cli", Name: "CLI", Spec: api.Spec{ Model: api.Model{Name: "claude", Mode: api.ModeCLI}, @@ -184,7 +185,8 @@ var _ = Describe("Runtime profiles", func() { }, }) - Expect(err).To(MatchError(ContainSubstring(`caller-tool policy "deny" is not available for anthropic cli`))) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Warnings).To(Equal([]string{`caller-tool policy "deny" is not available for anthropic cli`})) }) It("accepts brokered caller-tool policy when the backend supports caller tools", func() { @@ -203,8 +205,8 @@ var _ = Describe("Runtime profiles", func() { Expect(err).NotTo(HaveOccurred()) }) - It("rejects resource controls the resolved runtime drops", func() { - _, err := api.ResolveRuntimeProfile(api.RuntimeProfileResolveRequest{ + It("warns about resource controls the resolved runtime drops", func() { + resolved, err := api.ResolveRuntimeProfile(api.RuntimeProfileResolveRequest{ Profile: api.RuntimeProfile{ ID: "codex", Name: "Codex", Spec: api.Spec{ Model: api.Model{Name: "codex", Mode: api.ModeCLI}, @@ -213,7 +215,8 @@ var _ = Describe("Runtime profiles", func() { }, }) - Expect(err).To(MatchError(ContainSubstring(`resource policy mcp=disabled is not available for openai cli`))) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Warnings).To(Equal([]string{`resource policy mcp=disabled is not available for openai cli`})) }) }) diff --git a/pkg/api/spec.go b/pkg/api/spec.go index 89a250bf..910fe819 100644 --- a/pkg/api/spec.go +++ b/pkg/api/spec.go @@ -172,6 +172,9 @@ func (s Spec) MarshalYAML() (any, error) { // Validate runs each component's validation, failing loud on the first error. func (s Spec) Validate() error { + if err := s.ValidateStructure(); err != nil { + return err + } validateModel := s.Model.Validate if s.IsVerifyOnly() && len(s.Workflow.Verify.Prompts) == 0 { validateModel = s.ValidateOptions @@ -179,54 +182,10 @@ func (s Spec) Validate() error { if err := validateModel(); err != nil { return fmt.Errorf("model: %w", err) } - if s.ToolApproval != nil { - if s.hasPromptBody() || len(s.Messages) > 0 { - return fmt.Errorf("tool approval resume state, prompt body, and messages are mutually exclusive request modes") - } - if err := s.ToolApproval.Validate(); err != nil { - return fmt.Errorf("tool approval: %w", err) - } - if err := s.Prompt.SchemaStrictness.Validate(); err != nil { - return fmt.Errorf("prompt: %w", err) - } - } else if len(s.Messages) > 0 { - if err := s.ValidateRequestMode(); err != nil { - return err - } - if err := ValidateMessages(s.Messages); err != nil { - return fmt.Errorf("messages: %w", err) - } - if err := s.Prompt.SchemaStrictness.Validate(); err != nil { - return fmt.Errorf("prompt: %w", err) - } - // A verify-only spec (no body, workflow.verify present) legitimately has an - // empty prompt; only its strictness setting is checked. - } else if s.IsVerifyOnly() { - if err := s.Prompt.SchemaStrictness.Validate(); err != nil { + if s.ToolApproval == nil && len(s.Messages) == 0 && !s.IsVerifyOnly() { + if err := s.Prompt.Validate(); err != nil { return fmt.Errorf("prompt: %w", err) } - } else if err := s.Prompt.Validate(); err != nil { - return fmt.Errorf("prompt: %w", err) - } - if err := s.Budget.Validate(); err != nil { - return fmt.Errorf("budget: %w", err) - } - if err := s.Permissions.Validate(); err != nil { - return fmt.Errorf("permissions: %w", err) - } - if err := s.ToolPreferences.Validate(); err != nil { - return err - } - if err := s.ToolPolicy.Validate(); err != nil { - return err - } - if err := s.Workflow.Validate(); err != nil { - return fmt.Errorf("workflow: %w", err) - } - if s.Sandbox != nil { - if err := s.Sandbox.Validate(); err != nil { - return fmt.Errorf("sandbox: %w", err) - } } return nil } diff --git a/pkg/api/spec_composition.go b/pkg/api/spec_composition.go new file mode 100644 index 00000000..63819019 --- /dev/null +++ b/pkg/api/spec_composition.go @@ -0,0 +1,42 @@ +package api + +import "fmt" + +// ComposedSpec is an ordered structural projection, not a validated runtime. +type ComposedSpec struct { + Spec Spec `json:"spec" yaml:"spec"` + Constraints RuntimeConstraints `json:"constraints" yaml:"constraints"` + Trace []SpecLayer `json:"trace" yaml:"trace"` +} + +// ResolveSpecLayers validates the effective runtime after composing every layer. +func ResolveSpecLayers(input ...SpecLayer) (ResolvedSpec, error) { + composed, err := ComposeSpecLayers(input...) + if err != nil { + return ResolvedSpec{}, err + } + resolved := ResolvedSpec{Spec: composed.Spec, Constraints: composed.Constraints, Trace: composed.Trace} + if err := resolved.Spec.ValidateStructure(); err != nil { + return ResolvedSpec{}, fmt.Errorf("effective spec: %w", err) + } + if resolved.Spec.Name == "" { + return resolved, nil + } + resolved.Spec.Model, err = ResolveModel(resolved.Spec.Model) + if err != nil { + return ResolvedSpec{}, err + } + if err := validateResolvedModels(resolved); err != nil { + return ResolvedSpec{}, err + } + resolved.Warnings, err = ValidateRuntimeSpec(resolved.Spec) + if err != nil { + return ResolvedSpec{}, err + } + return resolved, nil +} + +// AllowsModel reports membership in the composed restrictive catalog. +func (composed ComposedSpec) AllowsModel(model Model) bool { + return allowsModel(composed.Constraints.Models, model) +} diff --git a/pkg/api/spec_layer_validation.go b/pkg/api/spec_layer_validation.go new file mode 100644 index 00000000..a57da9bb --- /dev/null +++ b/pkg/api/spec_layer_validation.go @@ -0,0 +1,28 @@ +package api + +import "fmt" + +// LayerValidationError identifies malformed configuration before composition. +type LayerValidationError struct { + Layer string + Err error +} + +func (e *LayerValidationError) Error() string { + return fmt.Sprintf("spec layer %q: %v", e.Layer, e.Err) +} + +func (e *LayerValidationError) Unwrap() error { return e.Err } + +// ValidateSpecLayers checks authored structure without requiring a model or prompt. +func ValidateSpecLayers(layers ...SpecLayer) error { + for _, layer := range layers { + if err := validateSpecLayer(layer); err != nil { + return &LayerValidationError{Layer: layer.Name, Err: err} + } + if err := layer.Spec.ValidateStructure(); err != nil { + return &LayerValidationError{Layer: layer.Name, Err: err} + } + } + return nil +} diff --git a/pkg/api/spec_layer_validation_ginkgo_test.go b/pkg/api/spec_layer_validation_ginkgo_test.go new file mode 100644 index 00000000..0d86dfc0 --- /dev/null +++ b/pkg/api/spec_layer_validation_ginkgo_test.go @@ -0,0 +1,59 @@ +package api + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Structural spec layer validation", func() { + invalidTemperature := 3.0 + DescribeTable("rejects malformed authored fields even if a request replaces them", func(invalid Spec, fragment string) { + layer := PromptSpecLayer("project.prompt", invalid) + err := ValidateSpecLayers(layer) + Expect(err).To(HaveOccurred()) + var structural *LayerValidationError + Expect(errors.As(err, &structural)).To(BeTrue()) + Expect(structural.Layer).To(Equal("project.prompt")) + Expect(err.Error()).To(ContainSubstring(fragment)) + _, err = ResolveSpecLayers(layer, RequestSpecLayer("request", Spec{Model: Model{Name: "agent:sonnet"}, Budget: Budget{Timeout: "1m"}})) + Expect(errors.As(err, &structural)).To(BeTrue()) + }, + Entry("budget", Spec{Budget: Budget{Timeout: "tomorrow"}}, "timeout"), + Entry("mode hidden by compact name", Spec{Model: Model{Name: "agent:sonnet", Mode: "invalid"}}, "mode"), + Entry("effort hidden by compact name", Spec{Model: Model{Name: "agent:sonnet:high", Effort: "invalid"}}, "effort"), + Entry("temperature", Spec{Model: Model{Temperature: &invalidTemperature}}, "temperature"), + Entry("fallback options", Spec{Model: Model{Fallbacks: []Model{{Name: "sol", Effort: "invalid"}}}}, "effort"), + Entry("fallback compact mode", Spec{Model: Model{Fallbacks: []Model{{Name: "invalid:sol"}}}}, "mode"), + Entry("permissions", Spec{Permissions: Permissions{Mode: "invalid"}}, "permissions"), + Entry("strictness", Spec{Prompt: Prompt{SchemaStrictness: "invalid"}}, "schemaStrictness"), + Entry("attachment", Spec{Prompt: Prompt{Attachments: []AttachmentRef{{URL: "file:///private/data"}}}}, "attachment"), + Entry("workflow", Spec{Workflow: &Workflow{Verify: &Verify{MaxIterations: -1}}}, "maxIterations"), + Entry("sandbox", Spec{Sandbox: &SandboxRef{Mode: "invalid"}}, "sandbox"), + ) + + It("attributes malformed layer metadata and constraints", func() { + for _, layer := range []SpecLayer{ + {Name: "scope", Scope: "invalid"}, + {Name: "limits", Scope: SpecLayerGlobal, Constraints: RuntimeConstraints{Limits: RunLimits{MaxInputTokens: -1}}}, + } { + err := ValidateSpecLayers(layer) + var structural *LayerValidationError + Expect(errors.As(err, &structural)).To(BeTrue()) + Expect(structural.Layer).To(Equal(layer.Name)) + } + }) + + It("accepts incomplete models and named models awaiting a request runtime", func() { + layers := []SpecLayer{ + {Name: "defaults", Scope: SpecLayerGlobal, Spec: Spec{Model: Model{Mode: ModeAgent, Effort: EffortHigh}, Permissions: Permissions{Mode: PermissionPlan}}}, + PromptSpecLayer("unknown.prompt", Spec{Model: Model{Name: "unregistered-model"}}), + } + Expect(ValidateSpecLayers(layers...)).To(Succeed()) + composed, err := ComposeSpecLayers(layers...) + Expect(err).NotTo(HaveOccurred()) + Expect(composed.Spec.Model).To(Equal(Model{Name: "unregistered-model", Mode: ModeAgent, Effort: EffortHigh})) + Expect(composed.Trace).To(Equal(layers)) + }) +}) diff --git a/pkg/api/spec_layers.go b/pkg/api/spec_layers.go index 431c5a82..1471c5ab 100644 --- a/pkg/api/spec_layers.go +++ b/pkg/api/spec_layers.go @@ -68,6 +68,7 @@ type ResolvedSpec struct { Spec Spec `json:"spec" yaml:"spec"` Constraints RuntimeConstraints `json:"constraints" yaml:"constraints"` Trace []SpecLayer `json:"trace" yaml:"trace"` + Warnings []string `json:"warnings,omitempty" yaml:"warnings,omitempty"` } // PromptSpecLayer adapts parsed .prompt frontmatter into the normal surface layer. @@ -90,24 +91,24 @@ func OrderSpecLayers(input ...SpecLayer) []SpecLayer { return layers } -// ResolveSpecLayers deterministically overlays defaults and intersects constraints. -func ResolveSpecLayers(input ...SpecLayer) (ResolvedSpec, error) { +// ComposeSpecLayers overlays raw defaults and constraints without resolving a runtime. +func ComposeSpecLayers(input ...SpecLayer) (ComposedSpec, error) { + if err := ValidateSpecLayers(input...); err != nil { + return ComposedSpec{}, err + } layers := OrderSpecLayers(input...) - resolved := ResolvedSpec{Trace: make([]SpecLayer, 0, len(layers))} + resolved := ComposedSpec{Trace: make([]SpecLayer, 0, len(layers))} for _, layer := range layers { - if err := validateSpecLayer(layer); err != nil { - return ResolvedSpec{}, err - } resolved.Spec = resolved.Spec.Merge(layer.Spec) if len(layer.Constraints.Models) > 0 { resolved.Constraints.Models = intersectModels(resolved.Constraints.Models, layer.Constraints.Models) if len(resolved.Constraints.Models) == 0 { - return ResolvedSpec{}, fmt.Errorf("spec layer %q leaves the effective model catalog empty", layer.Name) + return ComposedSpec{}, fmt.Errorf("spec layer %q leaves the effective model catalog empty", layer.Name) } } limits, err := strictRunLimits(resolved.Constraints.Limits, layer.Constraints.Limits) if err != nil { - return ResolvedSpec{}, fmt.Errorf("spec layer %q limits: %w", layer.Name, err) + return ComposedSpec{}, fmt.Errorf("spec layer %q limits: %w", layer.Name, err) } resolved.Constraints.Limits = limits for _, quota := range layer.Constraints.Quotas { @@ -121,21 +122,22 @@ func ResolveSpecLayers(input ...SpecLayer) (ResolvedSpec, error) { budget, err := strictBudget(resolved.Spec.Budget, resolved.Constraints.Limits.Budget) if err != nil { - return ResolvedSpec{}, fmt.Errorf("effective run budget: %w", err) + return ComposedSpec{}, fmt.Errorf("effective run budget: %w", err) } resolved.Spec.Budget = budget - if err := validateResolvedModels(resolved); err != nil { - return ResolvedSpec{}, err - } return resolved, nil } // AllowsModel reports whether a model belongs to the effective restrictive catalog. func (resolved ResolvedSpec) AllowsModel(model Model) bool { - if len(resolved.Constraints.Models) == 0 { + return allowsModel(resolved.Constraints.Models, model) +} + +func allowsModel(models []string, model Model) bool { + if len(models) == 0 { return true } - for _, allowed := range resolved.Constraints.Models { + for _, allowed := range models { if modelSelectorMatches(allowed, model) { return true } @@ -316,9 +318,12 @@ func modelSelectorMatches(selector string, model Model) bool { if selector == model.Name || selector == model.ID { return true } - allowed, allowedErr := (Model{Name: selector}).Expand() - actual, actualErr := model.Expand() - return allowedErr == nil && actualErr == nil && allowed.Name == actual.Name && allowed.Mode == actual.Mode + actual, actualErr := ResolveModel(model) + if actualErr != nil { + return false + } + allowed, allowedErr := ResolveModel(Model{Name: selector, Mode: actual.Mode}) + return allowedErr == nil && allowed.Name == actual.Name && allowed.Mode == actual.Mode && allowed.Provider == actual.Provider } func cloneSpecLayer(layer SpecLayer) SpecLayer { diff --git a/pkg/api/spec_runtime.go b/pkg/api/spec_runtime.go new file mode 100644 index 00000000..115da702 --- /dev/null +++ b/pkg/api/spec_runtime.go @@ -0,0 +1,30 @@ +package api + +import "fmt" + +// ValidateRuntimeSpec checks an already-resolved primary and its fallbacks. +// Unsupported permission/resource capabilities warn; malformed states, agent +// tool-policy refusals, and unsupported sandbox isolation remain hard errors. +func ValidateRuntimeSpec(spec Spec) ([]string, error) { + var warnings []string + for index, model := range append([]Model{spec.Model}, spec.Fallbacks...) { + if _, _, err := model.Runtime(); err != nil { + return warnings, fmt.Errorf("model %q: %w", model.Name, err) + } + if err := RequireToolPolicySupport(model.Provider, model.Mode, spec.Permissions); err != nil { + return warnings, fmt.Errorf("model %q: %w", model.Name, err) + } + candidate := spec + candidate.Model = model + if err := ValidateResolvedSandbox(candidate); err != nil { + return warnings, fmt.Errorf("model %q: %w", model.Name, err) + } + for _, warning := range UnsupportedPermissions(candidate) { + if index > 0 { + warning = fmt.Sprintf("fallback[%d] %q: %s", index-1, model.Name, warning) + } + warnings = append(warnings, warning) + } + } + return warnings, nil +} diff --git a/pkg/api/spec_runtime_ginkgo_test.go b/pkg/api/spec_runtime_ginkgo_test.go new file mode 100644 index 00000000..cd41723a --- /dev/null +++ b/pkg/api/spec_runtime_ginkgo_test.go @@ -0,0 +1,110 @@ +package api + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Effective layered runtime validation", func() { + It("resolves after a request repairs a profile runtime and preserves authored trace", func() { + profile := PromptSpecLayer("profile", Spec{Model: Model{Name: "agent:sol"}, Permissions: Permissions{Tools: Tools{"Bash": ToolPolicyDeny}}}) + request := RequestSpecLayer("request", Spec{Model: Model{Name: "sonnet", Mode: ModeCLI}}) + Expect(ValidateSpecLayers(profile)).To(Succeed()) + _, err := ResolveSpecLayers(profile) + Expect(err).To(HaveOccurred()) + resolved, err := ResolveSpecLayers(profile, request) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Name).To(Equal("claude-sonnet-5")) + Expect(resolved.Spec.Provider).To(Equal(Anthropic)) + Expect(resolved.Spec.Mode).To(Equal(ModeCLI)) + Expect(resolved.Trace).To(Equal([]SpecLayer{profile, request})) + Expect(resolved.Warnings).To(BeEmpty()) + }) + + It("keeps an explicit API mode after subsequent provider model resolution", func() { + resolved, err := ResolveSpecLayers(PromptSpecLayer("profile", Spec{Model: Model{Name: "api:sonnet"}})) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Mode).To(Equal(ModeAPI)) + again, err := ResolveModel(resolved.Spec.Model) + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(Equal(resolved.Spec.Model)) + }) + + It("returns unsupported primary and fallback resource policies as warnings", func() { + layer := PromptSpecLayer("profile", Spec{ + Model: Model{Name: "agent:sonnet", Fallbacks: []Model{{Name: "cli:sol"}}}, + Permissions: Permissions{Plugins: ResourcePolicies{"example": ResourceEnabled}}, + }) + resolved, err := ResolveSpecLayers(layer) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Warnings).To(ConsistOf( + "resource policy plugins=enabled is not available for anthropic agent", + `fallback[0] "gpt-5.6-sol": resource policy plugins=enabled is not available for openai cli`, + )) + Expect(resolved.Trace).To(Equal([]SpecLayer{layer})) + }) + + DescribeTable("refuses unsupported isolation on every candidate", func(model Model) { + _, err := ResolveSpecLayers(PromptSpecLayer("profile", Spec{Model: model, Sandbox: &SandboxRef{Mode: SandboxNative}})) + Expect(err).To(MatchError(ContainSubstring(`sandbox mode "native" is not available`))) + }, + Entry("primary", Model{Name: "api:sonnet"}), + Entry("fallback", Model{Name: "agent:sonnet", Fallbacks: []Model{{Name: "api:sol"}}}), + ) + + It("retains hard agent-tool policy refusal for fallback runtimes", func() { + _, err := ResolveSpecLayers(PromptSpecLayer("profile", Spec{ + Model: Model{Name: "cli:sonnet", Fallbacks: []Model{{Name: "cli:sol"}}}, + Permissions: Permissions{Tools: Tools{"Bash": ToolPolicyDeny}}, + })) + Expect(err).To(MatchError(ContainSubstring("tool policy"))) + }) + + It("uses native translators for unsupported policy fields on fallbacks", func() { + _, err := ResolveSpecLayers(PromptSpecLayer("profile", Spec{ + Model: Model{Name: "agent:sonnet", Fallbacks: []Model{{Name: "agent:sol"}}}, + Sandbox: &SandboxRef{Mode: SandboxNative, Policy: &NativeSandboxPolicy{ + Network: &SandboxNetworkPolicy{AllowedDomains: []string{"example.com"}}, + }}, + })) + Expect(err).To(MatchError(ContainSubstring("allowedDomains"))) + }) + + It("preserves supported native sandbox policy on primary and fallback", func() { + layer := PromptSpecLayer("profile", Spec{ + Model: Model{Name: "agent:sonnet", Fallbacks: []Model{{Name: "agent:sol"}}}, + Sandbox: &SandboxRef{Mode: SandboxNative, Policy: &NativeSandboxPolicy{ + Filesystem: &SandboxFilesystemPolicy{Access: SandboxFilesystemWorkspaceWrite}, + }}, + }) + resolved, err := ResolveSpecLayers(layer) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Sandbox).To(Equal(layer.Spec.Sandbox)) + Expect(resolved.Trace).To(Equal([]SpecLayer{layer})) + }) + + It("keeps incomplete model-free verification valid without inventing runtime warnings", func() { + layer := PromptSpecLayer("fixture", Spec{Model: Model{Mode: ModeAgent}, + Workflow: &Workflow{Verify: &Verify{Commands: []string{"true"}}}, + Permissions: Permissions{Plugins: ResourcePolicies{"example": ResourceEnabled}}, + }) + resolved, err := ResolveSpecLayers(layer) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Model).To(Equal(layer.Spec.Model)) + Expect(resolved.Warnings).To(BeEmpty()) + Expect(resolved.Trace).To(Equal([]SpecLayer{layer})) + }) + + It("matches an authored alias constraint against the canonical primary and fallback", func() { + layer := SpecLayer{Name: "catalog", Scope: SpecLayerGlobal, + Constraints: RuntimeConstraints{Models: []string{"sol", "sonnet"}}, + Spec: Spec{Model: Model{Name: "sol", Fallbacks: []Model{{Name: "sonnet"}}}}, + } + resolved, err := ResolveSpecLayers(layer) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Name).To(Equal("gpt-5.6-sol")) + Expect(resolved.Spec.Fallbacks[0].Name).To(Equal("claude-sonnet-5")) + Expect(resolved.Constraints).To(Equal(layer.Constraints)) + Expect(ValidateRuntimeConstraints(resolved, resolved.Spec.Model, 0)).To(Succeed()) + }) +}) diff --git a/pkg/api/spec_validation.go b/pkg/api/spec_validation.go new file mode 100644 index 00000000..537cfe5d --- /dev/null +++ b/pkg/api/spec_validation.go @@ -0,0 +1,70 @@ +package api + +import "fmt" + +// ValidateStructure checks supplied fields without requiring a model or prompt, +// resolving aliases, inspecting files, or testing runtime capabilities. +func (s Spec) ValidateStructure() error { + if err := s.ValidateOptions(); err != nil { + return fmt.Errorf("model: %w", err) + } + expanded, err := s.Expand() + if err != nil { + return fmt.Errorf("model: %w", err) + } + for index, model := range append([]Model{expanded}, expanded.Fallbacks...) { + model, err = model.Expand() + if err == nil { + err = model.ValidateOptions() + } + if err != nil { + return fmt.Errorf("model candidate %d: %w", index, err) + } + } + if err := s.ValidateRequestMode(); err != nil { + return err + } + if s.ToolApproval != nil { + if err := s.ToolApproval.Validate(); err != nil { + return fmt.Errorf("tool approval: %w", err) + } + } + if len(s.Messages) > 0 { + if err := ValidateMessages(s.Messages); err != nil { + return fmt.Errorf("messages: %w", err) + } + } + if err := s.Prompt.SchemaStrictness.Validate(); err != nil { + return fmt.Errorf("prompt: %w", err) + } + for i, attachment := range s.Prompt.Attachments { + if err := attachment.Validate(); err != nil { + return fmt.Errorf("prompt attachment %d: %w", i+1, err) + } + } + return s.validateRuntimeFields() +} + +func (s Spec) validateRuntimeFields() error { + if err := s.Budget.Validate(); err != nil { + return fmt.Errorf("budget: %w", err) + } + if err := s.Permissions.Validate(); err != nil { + return fmt.Errorf("permissions: %w", err) + } + if err := s.ToolPreferences.Validate(); err != nil { + return err + } + if err := s.ToolPolicy.Validate(); err != nil { + return err + } + if err := s.Workflow.Validate(); err != nil { + return fmt.Errorf("workflow: %w", err) + } + if s.Sandbox != nil { + if err := s.Sandbox.Validate(); err != nil { + return fmt.Errorf("sandbox: %w", err) + } + } + return nil +} diff --git a/pkg/cli/prompt_layers.go b/pkg/cli/prompt_layers.go index 7fbf29e3..62f3a2e2 100644 --- a/pkg/cli/prompt_layers.go +++ b/pkg/cli/prompt_layers.go @@ -29,32 +29,31 @@ func selectRuntimeProfile(ctx context.Context, requested, pin string) (*runtimep if err != nil { return nil, fmt.Errorf("runtime profile %q: %w", ref, err) } - resolution, err := catalog.Resolve(ctx, ref) + resolution, err := catalog.Layers(ctx, ref) if err != nil { return nil, fmt.Errorf("runtime profile %q: %w", ref, err) } return &resolution, nil } -// promptLayers orders a render's layers: the profile's presets and spec, the -// prompt frontmatter, then the caller's request. The request model is expanded -// first (the aiflags invariant) so a mode-prefixed selector such as api:opus -// overrides the frontmatter mode while a bare name inherits it. +// promptLayers assembles authored profile, prompt and request layers. Captain's +// final resolution expands the effective model selector while retaining the raw trace. func promptLayers(profile *runtimeprofiles.Resolution, source string, frontmatter ai.Request, user *api.Spec) ([]api.SpecLayer, error) { var layers []api.SpecLayer if profile != nil { - layers = append(layers, profile.Resolved.Trace...) + layers = append(layers, profile.Layers...) } layers = append(layers, api.PromptSpecLayer(source, frontmatter)) + if err := api.ValidateSpecLayers(layers...); err != nil { + return nil, fmt.Errorf("prompt configuration: %w", err) + } if user == nil { return layers, nil } request := *user - model, err := request.Expand() - if err != nil { - return nil, fmt.Errorf("render request model: %w", err) + if err := api.ValidateSpecLayers(api.RequestSpecLayer(renderRequestLayer, request)); err != nil { + return nil, err } - request.Model = model return append(layers, api.RequestSpecLayer(renderRequestLayer, request)), nil } diff --git a/pkg/cli/prompt_profile_layers_ginkgo_test.go b/pkg/cli/prompt_profile_layers_ginkgo_test.go new file mode 100644 index 00000000..8e708b23 --- /dev/null +++ b/pkg/cli/prompt_profile_layers_ginkgo_test.go @@ -0,0 +1,72 @@ +package cli + +import ( + "github.com/flanksource/captain/pkg/aichat" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/runtimeprofiles" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "net/http" + "net/http/httptest" +) + +var _ = Describe("Composed prompt profile layers", func() { + It("allows the complete render request to repair a profile runtime", func() { + f, _, _ := newRuntimeCatalogFixture() + f.profile(runtimeprofiles.ProfileInput{Name: "Restricted", Spec: api.Spec{ + Model: api.Model{Name: "gpt-5.6-sol", Mode: api.ModeCLI}, + Permissions: api.Permissions{Tools: api.Tools{"Bash": api.ToolPolicyDeny}}, + }}) + profile, err := selectRuntimeProfile(f.ctx, "restricted", "") + Expect(err).NotTo(HaveOccurred()) + Expect(profile.Resolved).To(Equal(api.ResolvedSpec{})) + Expect(profile.Layers).To(HaveLen(1)) + layers, err := promptLayers(profile, "review.prompt", api.Spec{}, &api.Spec{ + Model: api.Model{Name: "agent:claude-sonnet-5"}, + }) + Expect(err).NotTo(HaveOccurred()) + resolved, err := resolvePromptLayers(layers) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Mode).To(Equal(api.ModeAgent)) + Expect(resolved.Spec.Permissions.Tools).To(Equal(api.Tools{"Bash": api.ToolPolicyDeny})) + Expect(traceNames(resolved)).To(Equal([]string{"Restricted run spec", "review.prompt", "render request"})) + }) + + It("keeps an incomplete chat profile available for a later request", func() { + f, _, _ := newRuntimeCatalogFixture() + f.profile(runtimeprofiles.ProfileInput{Name: "Restricted", Spec: api.Spec{ + Model: api.Model{Name: "gpt-5.6-sol", Mode: api.ModeCLI}, + Permissions: api.Permissions{Tools: api.Tools{"Bash": api.ToolPolicyDeny}}, + }}) + profile, err := captainChatProfileProvider(GinkgoT().TempDir()).RuntimeProfile(f.ctx, aichat.WithRuntimeProfileRef("restricted")) + Expect(err).NotTo(HaveOccurred()) + Expect(profile.Composed.Spec.Name).To(Equal("gpt-5.6-sol")) + Expect(profile.Composed.Spec.Provider).To(BeNil()) + Expect(profile.Composed.Trace).To(HaveExactElements(HaveField("Name", "captain serve"), HaveField("Name", "Restricted run spec"))) + }) + + DescribeTable("rejects malformed request fields hidden by a compact model", func(model api.Model) { + _, err := promptLayers(nil, "review.prompt", api.Spec{}, &api.Spec{Model: model}) + Expect(err).To(HaveOccurred()) + }, + Entry("invalid mode", api.Model{Name: "agent:sonnet:high", Mode: "invalid"}), + Entry("invalid effort", api.Model{Name: "agent:sonnet:high", Effort: "invalid"}), + ) + + It("preserves a compact render request until the shared final fold", func() { + request := api.Spec{Model: api.Model{Name: "agent:sonnet:high"}} + layers, err := promptLayers(nil, "review.prompt", api.Spec{}, &request) + Expect(err).NotTo(HaveOccurred()) + Expect(layers[1].Spec).To(Equal(request)) + }) + + It("keeps a selected profile's missing preset as a server error", func() { + f, _, _ := newRuntimeCatalogFixture() + f.profile(runtimeprofiles.ProfileInput{Name: "Broken", Presets: []string{"missing"}}) + service := aichat.NewService(aichat.ServiceOptions{Profile: captainChatProfileProvider(GinkgoT().TempDir())}) + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/chat/models?runtimeProfile=broken", nil).WithContext(f.ctx)) + Expect(response.Code).To(Equal(http.StatusInternalServerError), response.Body.String()) + Expect(response.Body.String()).To(ContainSubstring("missing")) + }) +}) diff --git a/pkg/cli/serve_chat_profile.go b/pkg/cli/serve_chat_profile.go index 38823a5d..93ca3bd4 100644 --- a/pkg/cli/serve_chat_profile.go +++ b/pkg/cli/serve_chat_profile.go @@ -35,18 +35,21 @@ func captainChatProfileProvider(cwd string) aichat.RuntimeProfileProvider { if err != nil { return aichat.RuntimeProfile{}, err } - resolved, err := api.ResolveSpecLayers(layers...) + composed, err := api.ComposeSpecLayers(layers...) if err != nil { return aichat.RuntimeProfile{}, fmt.Errorf("resolve chat runtime profile: %w", err) } - return aichat.RuntimeProfile{System: captainChatSystemPrompt, Resolved: resolved}, nil + return aichat.RuntimeProfile{System: captainChatSystemPrompt, Composed: composed}, nil }) } -// chatProfileLayers appends the selected profile's trace to the base layer. A +// chatProfileLayers appends the selected profile's raw layers to the base. A // reference the caller supplied that resolves nowhere is the caller's error; a // configured default that fails stays a server error. func chatProfileLayers(ctx context.Context, base api.SpecLayer, selection aichat.RuntimeProfileOptions) ([]api.SpecLayer, error) { + if err := api.ValidateSpecLayers(base); err != nil { + return nil, fmt.Errorf("chat runtime profile base: %w", err) + } ref := strings.TrimSpace(selection.Ref) requested := ref != "" if !requested { @@ -63,12 +66,13 @@ func chatProfileLayers(ctx context.Context, base api.SpecLayer, selection aichat if err != nil { return nil, fmt.Errorf("chat runtime profile %q: %w", ref, err) } - resolution, err := catalog.Resolve(ctx, ref) + resolution, err := catalog.Layers(ctx, ref) if err != nil { - if requested && (errors.Is(err, runtimeprofiles.ErrNotFound) || errors.Is(err, runtimeprofiles.ErrAmbiguous)) { + var owned *runtimeprofiles.OwnedLayersError + if requested && !errors.As(err, &owned) && (errors.Is(err, runtimeprofiles.ErrNotFound) || errors.Is(err, runtimeprofiles.ErrAmbiguous)) { return nil, aichat.RequestError(http.StatusBadRequest, fmt.Sprintf("runtime profile %q: %v", ref, err)) } return nil, fmt.Errorf("chat runtime profile %q: %w", ref, err) } - return append([]api.SpecLayer{base}, resolution.Resolved.Trace...), nil + return append([]api.SpecLayer{base}, resolution.Layers...), nil } diff --git a/pkg/cli/serve_chat_profile_ginkgo_test.go b/pkg/cli/serve_chat_profile_ginkgo_test.go index cfb71f34..8fff788d 100644 --- a/pkg/cli/serve_chat_profile_ginkgo_test.go +++ b/pkg/cli/serve_chat_profile_ginkgo_test.go @@ -45,15 +45,15 @@ var _ = Describe("chat runtime profile provider", func() { Expect(err).NotTo(HaveOccurred()) Expect(profile.System).To(Equal(captainChatSystemPrompt)) - Expect(traceNames(profile.Resolved)).To(Equal([]string{"captain serve", "Team", "Review run spec"})) - Expect(profile.Resolved.Trace).To(HaveExactElements( + Expect(profile.Composed.Trace).To(HaveExactElements(HaveField("Name", "captain serve"), HaveField("Name", "Team"), HaveField("Name", "Review run spec"))) + Expect(profile.Composed.Trace).To(HaveExactElements( HaveField("Scope", api.SpecLayerGlobal), HaveField("Scope", api.SpecLayerContext), HaveField("Scope", api.SpecLayerSurface), )) - Expect(profile.Resolved.Spec.Model.Name).To(Equal("claude-sonnet-4-6"), "the preset overrides the base model") - Expect(profile.Resolved.Spec.Budget.MaxTurns).To(Equal(5), "the profile spec overrides the preset") - Expect(profile.Resolved.Spec.Cwd()).To(Equal(cwd), "the base layer survives") + Expect(profile.Composed.Spec.Model.Name).To(Equal("claude-sonnet-4-6"), "the preset overrides the base model") + Expect(profile.Composed.Spec.Budget.MaxTurns).To(Equal(5), "the profile spec overrides the preset") + Expect(profile.Composed.Spec.Cwd()).To(Equal(cwd), "the base layer survives") }) It("serves the base layer alone when nothing selects a profile", func() { @@ -62,8 +62,8 @@ var _ = Describe("chat runtime profile provider", func() { profile, err := captainChatProfileProvider(GinkgoT().TempDir()).RuntimeProfile(f.ctx) Expect(err).NotTo(HaveOccurred()) - Expect(traceNames(profile.Resolved)).To(Equal([]string{"captain serve"})) - Expect(profile.Resolved.Spec.Model.Name).To(Equal("sol")) + Expect(profile.Composed.Trace).To(HaveExactElements(HaveField("Name", "captain serve"))) + Expect(profile.Composed.Spec.Model.Name).To(Equal("sol")) }) It("applies the configured chat default when the request names no profile", func() { @@ -73,7 +73,7 @@ var _ = Describe("chat runtime profile provider", func() { profile, err := captainChatProfileProvider(GinkgoT().TempDir()).RuntimeProfile(f.ctx) Expect(err).NotTo(HaveOccurred()) - Expect(traceNames(profile.Resolved)).To(Equal([]string{"captain serve", "Team", "Review run spec"})) + Expect(profile.Composed.Trace).To(HaveExactElements(HaveField("Name", "captain serve"), HaveField("Name", "Team"), HaveField("Name", "Review run spec"))) }) It("lets the request's profile override the configured default", func() { @@ -83,7 +83,7 @@ var _ = Describe("chat runtime profile provider", func() { profile, err := captainChatProfileProvider(GinkgoT().TempDir()).RuntimeProfile(f.ctx, aichat.WithRuntimeProfileRef("plan")) Expect(err).NotTo(HaveOccurred()) - Expect(traceNames(profile.Resolved)).To(Equal([]string{"captain serve", "Plan run spec"})) + Expect(profile.Composed.Trace).To(HaveExactElements(HaveField("Name", "captain serve"), HaveField("Name", "Plan run spec"))) }) It("rejects an unknown request profile as a 400 and a broken default as a server error", func() { diff --git a/pkg/promptrun/README.md b/pkg/promptrun/README.md index 8f383a12..693a6c64 100644 --- a/pkg/promptrun/README.md +++ b/pkg/promptrun/README.md @@ -27,7 +27,7 @@ A supplied `Provider` owns its runtime and workspace; construction `Config` is i Invalid input, existing tool-policy refusals, unsupported sandbox isolation or native policy fields, missing fixture wiring, broken judge declarations, and constraint violations are errors. Newly diagnosed unsupported permission/resource settings and missing approval brokers produce warnings for this compatibility release. A disabled skill is omitted; a contradictory skill still explicitly loaded through `memory.skills` is diagnosed. These warnings are also logged by `Run`. -Preflight validates the runtime identity exposed by a supplied provider. Its private fallback chain, credentials, adapter-specific configuration, external service availability, and runtime launch failures remain the provider's responsibility. This API is execution admission; structural runtime-profile layer validation and saved-model default resolution remain separate contracts. +Preflight validates the runtime identity exposed by a supplied provider. Its private fallback chain, credentials, adapter-specific configuration, external service availability, and runtime launch failures remain the provider's responsibility. This API is execution admission; [runtime-profile composition](../runtimeprofiles/README.md) and saved-model default resolution remain separate contracts. The final layer resolver and Preflight share the same pure runtime capability checker; Preflight additionally checks the actual supplied or constructed execution runtime and its input-specific configuration. Run the executable examples and focused admission regressions without making AI calls: diff --git a/pkg/promptrun/preflight.go b/pkg/promptrun/preflight.go index a2d9d8a0..48215aa1 100644 --- a/pkg/promptrun/preflight.go +++ b/pkg/promptrun/preflight.go @@ -87,36 +87,24 @@ func preflight(in Input) (admission, error) { } func validateRuntime(in Input, spec api.Spec) ([]string, error) { - var warnings []string if constructsProvider(in) { if err := validateConstructionConfig(in); err != nil { return nil, err } } - for index, model := range append([]api.Model{spec.Model}, spec.Fallbacks...) { - if _, _, err := model.Runtime(); err != nil { - return warnings, fmt.Errorf("promptrun model %q: %w", model.Name, err) - } - if err := api.RequireToolPolicySupport(model.Provider, model.Mode, spec.Permissions); err != nil { - return warnings, err - } + warnings, err := api.ValidateRuntimeSpec(spec) + if err != nil { + return warnings, fmt.Errorf("promptrun: %w", err) + } + for _, model := range append([]api.Model{spec.Model}, spec.Fallbacks...) { candidate := spec candidate.Model = model - if err := api.ValidateResolvedSandbox(candidate); err != nil { - return warnings, fmt.Errorf("promptrun model %q: %w", model.Name, err) - } if constructsProvider(in) && in.Config.SandboxSelection != nil { descriptor, _ := api.SandboxFor(in.Config.SandboxSelection.Kind) if err := descriptor.ValidateMode(model.Mode); err != nil { return warnings, err } } - for _, warning := range api.UnsupportedPermissions(candidate) { - if index > 0 { - warning = fmt.Sprintf("fallback[%d] %q: %s", index-1, model.Name, warning) - } - warnings = append(warnings, warning) - } caps := api.PermissionCapabilitiesFor(api.RuntimeOf(model.Provider, model.Mode)) if constructsProvider(in) && in.Config.CanUseTool == nil && requiresBroker(candidate, caps) { warnings = append(warnings, fmt.Sprintf("caller-tool policy ask requires Config.CanUseTool for %s", api.RuntimeOf(model.Provider, model.Mode))) diff --git a/pkg/runtimeprofiles/README.md b/pkg/runtimeprofiles/README.md new file mode 100644 index 00000000..3795a8af --- /dev/null +++ b/pkg/runtimeprofiles/README.md @@ -0,0 +1,41 @@ +# Runtime profile composition + +Use `Catalog.Layers` or `Resolver.Layers` to load reusable configuration before adding a prompt and request. They validate authored structures without choosing a model or checking a guessed runtime. A profile can contain only permissions, mode, effort, or budget; a later request can supply or replace its runtime. + +Validate each owner's raw layers with `api.ValidateSpecLayers` before combining them. It returns `*api.LayerValidationError`, whose `Layer` and wrapped `Err` identify malformed metadata, constraints, model options, prompt attachments, permissions, workflow, or sandbox declarations. Invalid lower-priority values remain errors even when another layer would overwrite them. Missing model names and prompt bodies are valid in reusable fragments. Structural validation neither reads prompt/attachment files nor runs setup. + +```go +layers, err := resolver.Layers(ctx, runtimeprofiles.ResolveOptions{ + RequestedProfile: "review", + SurfaceLayers: []api.SpecLayer{ + api.PromptSpecLayer("review.prompt", document.Spec), + }, + RequestLayers: []api.SpecLayer{ + api.RequestSpecLayer("request", request), + }, +}) +if err != nil { + return err +} +resolved, err := api.ResolveSpecLayers(layers.Layers...) +if err != nil { + return err +} +for _, warning := range resolved.Warnings { + logger.Warnf("%s", warning) +} +``` + +`ResolveSpecLayers` composes in global → context → surface → user scope order, preserving order within a scope. It intersects restrictive catalogs, applies the strictest nonzero budget limits, and preserves every quota and raw layer in `Trace`. Compact model selectors retain their existing pin semantics: a prefix inside the effective model name wins over its sibling mode field. Model aliases and fallback names resolve only after composition. Final model, sandbox, and tool-policy refusals are errors; unsupported permission/resource capabilities produce separate `Warnings` for this compatibility release. No saved model defaults are loaded. Model-free compositions remain valid; execution requirements belong to `promptrun.Preflight`. + +Use `api.ComposeSpecLayers` for forms and defaults before a complete request exists. Its distinct `ComposedSpec` exposes the structural `Spec`, `Constraints`, `Trace`, and `AllowsModel` query without claiming runtime validity. It shares the same fold as final resolution. Keep its raw `Trace` when adding a request, then call `ResolveSpecLayers` once for the final stack. + +Layer resolution checks every declared primary and fallback runtime. Execution preflight separately checks the enabled candidates selected for the actual provider configuration. Disabling a candidate for one execution does not make an unsupported sandbox or tool policy valid in the declared profile. + +Catalog failures preserve ownership: `*runtimeprofiles.OwnedLayersError` wraps invalid stored data and missing nested preset references. An absent or ambiguous top-level profile remains `ErrNotFound` or `ErrAmbiguous`. `Resolver` additionally wraps selection failures in `*SelectionError`, recording whether the request, prompt pin, or configured default selected the profile. Use `errors.As` and `errors.Is` instead of parsing error text. + +Run the composition, ownership, and execution-admission examples without making AI calls: + +```sh +go test ./pkg/api ./pkg/runtimeprofiles ./pkg/promptrun -ginkgo.no-color -ginkgo.succinct -count=1 +``` diff --git a/pkg/runtimeprofiles/doc.go b/pkg/runtimeprofiles/doc.go index dd5bf584..202d82b4 100644 --- a/pkg/runtimeprofiles/doc.go +++ b/pkg/runtimeprofiles/doc.go @@ -6,5 +6,6 @@ // that must resolve to exactly one record across every source. Names are unique // case-insensitively across sources. A profile's preset references are an // ordered list of ids or names; deleting a preset a profile still names is -// refused, and Resolve materialises a profile through api.ResolveRuntimeProfile. +// refused. Layers returns structurally validated raw configuration; Resolve +// materialises a complete profile through api.ResolveSpecLayers for preview. package runtimeprofiles diff --git a/pkg/runtimeprofiles/layer_errors.go b/pkg/runtimeprofiles/layer_errors.go new file mode 100644 index 00000000..5c3033e1 --- /dev/null +++ b/pkg/runtimeprofiles/layer_errors.go @@ -0,0 +1,16 @@ +package runtimeprofiles + +import "fmt" + +// OwnedLayersError marks invalid stored profile data or its referenced presets, +// distinct from an absent or ambiguous top-level selection supplied by a caller. +type OwnedLayersError struct { + Ref string + Err error +} + +func (e *OwnedLayersError) Error() string { + return fmt.Sprintf("runtime profile %q configuration: %v", e.Ref, e.Err) +} + +func (e *OwnedLayersError) Unwrap() error { return e.Err } diff --git a/pkg/runtimeprofiles/layer_validation_ginkgo_test.go b/pkg/runtimeprofiles/layer_validation_ginkgo_test.go new file mode 100644 index 00000000..0006e34c --- /dev/null +++ b/pkg/runtimeprofiles/layer_validation_ginkgo_test.go @@ -0,0 +1,67 @@ +package runtimeprofiles + +import ( + "context" + "errors" + + "github.com/flanksource/captain/pkg/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Owned runtime layer validation", func() { + It("retains catalog ownership when an existing profile references a missing preset", func(ctx SpecContext) { + source := newMemSource("db", SourceDB, true) + profile := source.profiles.put("review", ProfileInput{Name: "Review", Presets: []string{"missing"}}) + catalog, err := NewCatalog(source) + Expect(err).NotTo(HaveOccurred()) + _, err = catalog.Layers(ctx, profile.ID) + var owned *OwnedLayersError + Expect(errors.As(err, &owned)).To(BeTrue()) + Expect(owned.Ref).To(Equal(profile.ID)) + Expect(errors.Is(err, ErrNotFound)).To(BeTrue()) + _, err = catalog.Layers(ctx, "missing-profile") + Expect(errors.As(err, &owned)).To(BeFalse()) + Expect(errors.Is(err, ErrNotFound)).To(BeTrue()) + }) + + It("rejects malformed catalog fields before a request can overwrite them", func(ctx SpecContext) { + source := newMemSource("db", SourceDB, true) + profile := source.profiles.put("review", ProfileInput{Name: "Review", Spec: api.Spec{Budget: api.Budget{Timeout: "invalid"}}}) + catalog, err := NewCatalog(source) + Expect(err).NotTo(HaveOccurred()) + resolver := NewResolver(func(context.Context) (*Catalog, error) { return catalog, nil }) + _, err = resolver.Layers(ctx, ResolveOptions{RequestedProfile: profile.ID, + RequestLayers: []api.SpecLayer{api.RequestSpecLayer("request", api.Spec{Budget: api.Budget{Timeout: "1m"}})}, + }) + var owned *OwnedLayersError + var structural *api.LayerValidationError + var selected *SelectionError + Expect(errors.As(err, &owned)).To(BeTrue()) + Expect(errors.As(err, &structural)).To(BeTrue()) + Expect(errors.As(err, &selected)).To(BeTrue()) + Expect(selected.Origin).To(Equal(SelectionRequested)) + Expect(structural.Layer).To(Equal("Review run spec")) + }) + + It("rejects invalid caller layers without claiming catalog ownership", func(ctx SpecContext) { + _, err := NewResolver(nil).Layers(ctx, ResolveOptions{RequestLayers: []api.SpecLayer{ + api.RequestSpecLayer("request", api.Spec{Budget: api.Budget{Cost: -1}}), + }}) + var owned *OwnedLayersError + var structural *api.LayerValidationError + Expect(errors.As(err, &owned)).To(BeFalse()) + Expect(errors.As(err, &structural)).To(BeTrue()) + Expect(structural.Layer).To(Equal("request")) + }) + + It("allows partial reusable presets and rejects malformed profile writes", func() { + Expect((PresetInput{Name: "Effort", Scope: api.SpecLayerGlobal, + Spec: api.RuntimePresetSpec{Model: api.Model{Mode: api.ModeAgent, Effort: api.EffortHigh}}, + }).validate()).To(Succeed()) + err := (ProfileInput{Name: "Review", Spec: api.Spec{Model: api.Model{Effort: "invalid"}}}).validate() + var structural *api.LayerValidationError + Expect(errors.As(err, &structural)).To(BeTrue()) + Expect(errors.Is(err, ErrInvalid)).To(BeTrue()) + }) +}) diff --git a/pkg/runtimeprofiles/layers_ginkgo_test.go b/pkg/runtimeprofiles/layers_ginkgo_test.go index 9b2bcc4f..74980740 100644 --- a/pkg/runtimeprofiles/layers_ginkgo_test.go +++ b/pkg/runtimeprofiles/layers_ginkgo_test.go @@ -28,7 +28,8 @@ var _ = Describe("Catalog layers", func() { Spec: api.Spec{Permissions: api.Permissions{Mode: api.PermissionDontAsk}}}, }, })) - _, err = catalog.Resolve(ctx, profile.ID) - Expect(err).To(MatchError(ContainSubstring(`permissions.mode "dontAsk" is not available for openai agent`))) + resolution, err = catalog.Resolve(ctx, profile.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(resolution.Resolved.Warnings).To(Equal([]string{`permissions.mode "dontAsk" is not available for openai agent`})) }) }) diff --git a/pkg/runtimeprofiles/resolve.go b/pkg/runtimeprofiles/resolve.go index 1a07b800..d3808451 100644 --- a/pkg/runtimeprofiles/resolve.go +++ b/pkg/runtimeprofiles/resolve.go @@ -2,6 +2,7 @@ package runtimeprofiles import ( "context" + "errors" "fmt" "github.com/flanksource/captain/pkg/api" @@ -12,6 +13,9 @@ import ( func (c *Catalog) Layers(ctx context.Context, ref string) (Resolution, error) { profile, err := c.GetProfile(ctx, ref) if err != nil { + if !errors.Is(err, ErrNotFound) && !errors.Is(err, ErrAmbiguous) { + return Resolution{}, &OwnedLayersError{Ref: ref, Err: err} + } return Resolution{}, err } presets := make([]Preset, 0, len(profile.Presets)) @@ -20,7 +24,7 @@ func (c *Catalog) Layers(ctx context.Context, ref string) (Resolution, error) { for _, presetRef := range profile.Presets { preset, err := c.GetPreset(ctx, presetRef) if err != nil { - return Resolution{}, fmt.Errorf("runtime profile %q references preset %q: %w", profile.Name, presetRef, err) + return Resolution{}, &OwnedLayersError{Ref: ref, Err: fmt.Errorf("runtime profile %q references preset %q: %w", profile.Name, presetRef, err)} } presets = append(presets, preset) apiPresets = append(apiPresets, preset.API()) @@ -31,7 +35,7 @@ func (c *Catalog) Layers(ctx context.Context, ref string) (Resolution, error) { Profile: profile.API(), Presets: apiPresets, }) if err != nil { - return Resolution{}, err + return Resolution{}, &OwnedLayersError{Ref: ref, Err: err} } return Resolution{Profile: profile, Presets: presets, Layers: layers}, nil } @@ -42,13 +46,7 @@ func (c *Catalog) Resolve(ctx context.Context, ref string) (Resolution, error) { if err != nil { return Resolution{}, err } - presets := make([]api.RuntimePreset, 0, len(resolution.Presets)) - for _, preset := range resolution.Presets { - presets = append(presets, preset.API()) - } - resolved, err := api.ResolveRuntimeProfile(api.RuntimeProfileResolveRequest{ - Profile: resolution.Profile.API(), Presets: presets, - }) + resolved, err := api.ResolveSpecLayers(resolution.Layers...) if err != nil { return Resolution{}, err } diff --git a/pkg/runtimeprofiles/resolver.go b/pkg/runtimeprofiles/resolver.go index 3d7773ff..761d439f 100644 --- a/pkg/runtimeprofiles/resolver.go +++ b/pkg/runtimeprofiles/resolver.go @@ -97,6 +97,9 @@ func (r *Resolver) Layers(ctx context.Context, options ResolveOptions) (LayerRes } layers = append(layers, options.SurfaceLayers...) layers = append(layers, options.RequestLayers...) + if err := api.ValidateSpecLayers(layers...); err != nil { + return LayerResult{}, err + } return LayerResult{Profile: profile, Layers: api.OrderSpecLayers(layers...)}, nil } diff --git a/pkg/runtimeprofiles/resolver_ginkgo_test.go b/pkg/runtimeprofiles/resolver_ginkgo_test.go index f868b0a7..0e9342f8 100644 --- a/pkg/runtimeprofiles/resolver_ginkgo_test.go +++ b/pkg/runtimeprofiles/resolver_ginkgo_test.go @@ -80,7 +80,9 @@ var _ = Describe("Runtime profile resolver", func() { RequestedProfile: profile.ID, RequestLayers: []api.SpecLayer{request}, }) Expect(err).NotTo(HaveOccurred()) - Expect(result.Resolved.Spec.Model).To(Equal(request.Spec.Model)) + expected, err := api.ResolveModel(request.Spec.Model) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Resolved.Spec.Model).To(Equal(expected)) Expect(result.Resolved.Spec.Permissions.Mode).To(Equal(api.PermissionDontAsk)) Expect(result.Resolved.Trace).To(HaveLen(2)) Expect(result.Resolved.Trace[0].Spec.Model).To(Equal(profileModel)) diff --git a/pkg/runtimeprofiles/types.go b/pkg/runtimeprofiles/types.go index 31cddc39..be8ff0b7 100644 --- a/pkg/runtimeprofiles/types.go +++ b/pkg/runtimeprofiles/types.go @@ -102,7 +102,7 @@ type ProfileInput struct { type Resolution struct { Profile Profile `json:"profile"` Presets []Preset `json:"presets"` - Layers []api.SpecLayer `json:"-"` + Layers []api.SpecLayer `json:"-"` Resolved api.ResolvedSpec `json:"resolved"` } @@ -175,7 +175,7 @@ func (in PresetInput) validate() error { } preset := api.RuntimePreset{ID: name, Name: name, Scope: in.Scope, Spec: in.Spec} if err := api.ValidateRuntimePreset(preset); err != nil { - return fmt.Errorf("%w: %v", ErrInvalid, err) + return fmt.Errorf("%w: %w", ErrInvalid, err) } return nil } @@ -210,6 +210,10 @@ func (in ProfileInput) validate() error { return fmt.Errorf("%w: profile %q preset reference %d is blank", ErrInvalid, name, index) } } + if err := api.ValidateSpecLayers(api.SpecLayer{Name: name + " run spec", Scope: api.SpecLayerSurface, + Source: api.SpecLayerSourceProfile, Spec: in.Spec}); err != nil { + return fmt.Errorf("%w: %w", ErrInvalid, err) + } return nil } From a2ba920e14df9a5cb8d123878abf071931e06ae4 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 12:12:36 +0300 Subject: [PATCH 07/22] fix(cmux): resolve cmux CLI from GUI installations Resolve cmux CLI paths reliably across PATH entries, GUI bundles, symlinks, and explicit overrides, while returning actionable errors when unavailable. --- pkg/cmux/binary.go | 40 ++++++++++++++++++ pkg/cmux/binary_ginkgo_test.go | 77 ++++++++++++++++++++++++++++++++++ pkg/cmux/client.go | 16 +++---- 3 files changed, 122 insertions(+), 11 deletions(-) create mode 100644 pkg/cmux/binary.go create mode 100644 pkg/cmux/binary_ginkgo_test.go diff --git a/pkg/cmux/binary.go b/pkg/cmux/binary.go new file mode 100644 index 00000000..0b637710 --- /dev/null +++ b/pkg/cmux/binary.go @@ -0,0 +1,40 @@ +package cmux + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +func CmuxBin() (string, error) { + binary := os.Getenv("CMUX_BIN") + if binary == "" { + path, err := exec.LookPath("cmux") + if err != nil && !errors.Is(err, exec.ErrNotFound) { + return "", fmt.Errorf("resolve cmux CLI: %w", err) + } + binary = path + if errors.Is(err, exec.ErrNotFound) { + binary = "/Applications/cmux.app/Contents/Resources/bin/cmux" + } + } + path, err := exec.LookPath(binary) + if err != nil { + return "", fmt.Errorf("resolve cmux CLI %q: %w", binary, err) + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", fmt.Errorf("resolve cmux CLI symlinks %q: %w", path, err) + } + if strings.HasSuffix(filepath.ToSlash(resolved), ".app/Contents/MacOS/cmux") { + path = filepath.Join(filepath.Dir(filepath.Dir(resolved)), "Resources", "bin", "cmux") + path, err = exec.LookPath(path) + if err != nil { + return "", fmt.Errorf("resolve bundled cmux CLI for %q: %w", resolved, err) + } + } + return path, nil +} diff --git a/pkg/cmux/binary_ginkgo_test.go b/pkg/cmux/binary_ginkgo_test.go new file mode 100644 index 00000000..6e431299 --- /dev/null +++ b/pkg/cmux/binary_ginkgo_test.go @@ -0,0 +1,77 @@ +package cmux + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CmuxBin", func() { + var gui, cli, bin string + + BeforeEach(func() { + root, err := filepath.EvalSymlinks(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + gui = filepath.Join(root, "cmux.app", "Contents", "MacOS", "cmux") + cli = filepath.Join(root, "cmux.app", "Contents", "Resources", "bin", "cmux") + bin = filepath.Join(root, "bin") + Expect(os.MkdirAll(bin, 0o755)).To(Succeed()) + for _, path := range []string{gui, cli} { + Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) + Expect(os.WriteFile(path, nil, 0o755)).To(Succeed()) + } + GinkgoT().Setenv("CMUX_BIN", "") + }) + + It("resolves the bundled CLI when PATH contains the GUI executable", func() { + GinkgoT().Setenv("PATH", filepath.Dir(gui)) + Expect(CmuxBin()).To(Equal(cli)) + }) + + It("resolves the bundled CLI through a PATH symlink to the GUI", func() { + Expect(os.Symlink(gui, filepath.Join(bin, "cmux"))).To(Succeed()) + GinkgoT().Setenv("PATH", bin) + Expect(CmuxBin()).To(Equal(cli)) + }) + + It("resolves a GUI override to its bundled CLI", func() { + GinkgoT().Setenv("CMUX_BIN", gui) + Expect(CmuxBin()).To(Equal(cli)) + }) + + It("preserves an explicit CLI override ahead of PATH", func() { + GinkgoT().Setenv("CMUX_BIN", cli) + GinkgoT().Setenv("PATH", filepath.Dir(gui)) + Expect(CmuxBin()).To(Equal(cli)) + }) + + It("preserves a standalone CLI from PATH", func() { + path := filepath.Join(bin, "cmux") + Expect(os.WriteFile(path, nil, 0o755)).To(Succeed()) + GinkgoT().Setenv("PATH", bin) + Expect(CmuxBin()).To(Equal(path)) + }) + + It("rejects a missing bundled CLI instead of returning the GUI executable", func() { + Expect(os.Remove(cli)).To(Succeed()) + GinkgoT().Setenv("PATH", filepath.Dir(gui)) + path, err := CmuxBin() + Expect(path).To(BeEmpty()) + Expect(err).To(MatchError(ContainSubstring("resolve bundled cmux CLI"))) + }) + + It("rejects a non-executable bundled CLI", func() { + Expect(os.Chmod(cli, 0o644)).To(Succeed()) + GinkgoT().Setenv("PATH", filepath.Dir(gui)) + _, err := CmuxBin() + Expect(err).To(MatchError(ContainSubstring("resolve bundled cmux CLI"))) + }) + + It("rejects a missing explicit override", func() { + GinkgoT().Setenv("CMUX_BIN", filepath.Join(bin, "missing")) + _, err := CmuxBin() + Expect(err).To(MatchError(ContainSubstring("resolve cmux CLI"))) + }) +}) diff --git a/pkg/cmux/client.go b/pkg/cmux/client.go index 0d66833d..ad3b4bdd 100644 --- a/pkg/cmux/client.go +++ b/pkg/cmux/client.go @@ -10,18 +10,12 @@ import ( "unicode/utf8" ) -func CmuxBin() string { - if p := os.Getenv("CMUX_BIN"); p != "" { - return p - } - if p, err := exec.LookPath("cmux"); err == nil { - return p - } - return "/Applications/cmux.app/Contents/Resources/bin/cmux" -} - func run(args ...string) (string, error) { - cmd := exec.Command(CmuxBin(), args...) + binary, err := CmuxBin() + if err != nil { + return "", err + } + cmd := exec.Command(binary, args...) out, err := cmd.Output() if err != nil { if ee, ok := err.(*exec.ExitError); ok { From e476f753c81099e001ab5c38c4d5a48ed44d037b Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 12:12:58 +0300 Subject: [PATCH 08/22] refactor(aiflags): Centralize model default resolution and preserve authored fields Centralize saved, provider, catalog, and fallback default handling at the final resolution boundary. Preserve authored zero values, compact selectors, provider aliases, and provenance while reporting unresolved modes explicitly. BREAKING CHANGE: Replace the old ApplyDefaults(model, saved), ResolveForRun, and ResolveForRunWith APIs with DefaultOptions-based ApplyDefaults. --- .../default_normalization_ginkgo_test.go | 127 ++++++++++ pkg/aiflags/default_resolution.go | 195 +++++++++++++++ pkg/aiflags/default_resolution_ginkgo_test.go | 226 ++++++++++++++++++ pkg/aiflags/default_selection.go | 108 +++++++++ pkg/aiflags/default_sources.go | 74 ++++++ pkg/aiflags/defaults.go | 132 ++-------- pkg/aiflags/flags.go | 85 +++---- pkg/aiflags/unconfigured.go | 57 +---- pkg/aiflags/unconfigured_test.go | 43 ++-- pkg/api/registry/disabled.go | 19 +- pkg/api/registry/disabled_ginkgo_test.go | 7 + pkg/api/registry/model.go | 11 +- pkg/api/registry/model_compact.go | 16 +- pkg/api/registry/model_compact_test.go | 4 +- pkg/api/registry/model_presence.go | 127 ++++++++++ .../registry/model_presence_ginkgo_test.go | 89 +++++++ pkg/api/registry/model_test.go | 7 +- pkg/api/registry/parse.go | 5 + pkg/api/registry/providers.go | 6 +- 19 files changed, 1089 insertions(+), 249 deletions(-) create mode 100644 pkg/aiflags/default_normalization_ginkgo_test.go create mode 100644 pkg/aiflags/default_resolution.go create mode 100644 pkg/aiflags/default_resolution_ginkgo_test.go create mode 100644 pkg/aiflags/default_selection.go create mode 100644 pkg/aiflags/default_sources.go create mode 100644 pkg/api/registry/model_presence.go create mode 100644 pkg/api/registry/model_presence_ginkgo_test.go diff --git a/pkg/aiflags/default_normalization_ginkgo_test.go b/pkg/aiflags/default_normalization_ginkgo_test.go new file mode 100644 index 00000000..49db29aa --- /dev/null +++ b/pkg/aiflags/default_normalization_ginkgo_test.go @@ -0,0 +1,127 @@ +package aiflags + +import ( + "errors" + + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/captainconfig" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Saved model selection normalization", func() { + DescribeTable("inherits authored primary effort before saved fallback effort", func(fallbacks registry.ModelList, effort registry.Effort, inherited bool) { + result, err := ApplyDefaults(DefaultOptions{ + Model: registry.Model{Name: "sonnet", Effort: registry.EffortHigh, Fallbacks: fallbacks}, + Saved: captainconfig.AIDefaults{DefaultModel: "api:sonnet,api:haiku:low"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Fallbacks).To(HaveLen(1)) + Expect(result.Model.Fallbacks[0].Effort).To(Equal(effort)) + if inherited { + Expect(result.Sources).To(HaveKeyWithValue("/fallbacks/0/effort", "primary.effort")) + } else { + Expect(result.Sources).NotTo(HaveKey("/fallbacks/0/effort")) + } + }, + Entry("saved fallback knob remains a gap", nil, registry.EffortHigh, true), + Entry("authored fallback effort wins", registry.ModelList{{Name: "api:haiku:low"}}, registry.EffortLow, false), + Entry("authored empty fallback effort wins", registry.ModelList{(registry.Model{Name: "api:haiku"}).WithExplicit("/effort")}, registry.EffortNone, false), + ) + + It("normalizes the complete saved candidate chain once before provider defaults", func() { + saved := captainconfig.AIDefaults{ + DefaultModel: "sonnet,sol", Temperature: 0.3, + Providers: map[string]captainconfig.ProviderDefaults{ + "anthropic": {Mode: "api", ReasoningEffort: "high"}, + "openai": {Mode: "api", ReasoningEffort: "low"}, + }, + } + calls := 0 + result, err := ApplyDefaults(DefaultOptions{Saved: saved, Normalize: func(model registry.Model) (registry.Model, error) { + calls++ + Expect(model.Name).To(Equal("sonnet")) + Expect(model.Mode).To(BeEmpty()) + Expect(model.Effort).To(BeEmpty()) + Expect(model.Temperature).To(BeNil()) + Expect(model.Fallbacks).To(HaveLen(1)) + Expect(model.Fallbacks[0].Name).To(Equal("sol")) + Expect(model.Fallbacks[0].Mode).To(BeEmpty()) + Expect(model.Fallbacks[0].Effort).To(BeEmpty()) + model.Mode = registry.ModeCLI + model.Fallbacks[0].Mode = registry.ModeCLI + return model, nil + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(calls).To(Equal(1)) + Expect(result.Model.Mode).To(Equal(registry.ModeCLI)) + Expect(result.Model.Fallbacks[0].Mode).To(Equal(registry.ModeCLI)) + Expect(result.Model.Effort).To(Equal(registry.EffortHigh)) + Expect(result.Model.Fallbacks[0].Effort).To(Equal(registry.EffortLow)) + Expect(result.Sources).To(HaveKeyWithValue("/model", "ai.defaultModel")) + Expect(result.Sources).To(HaveKeyWithValue("/fallbacks/0/model", "ai.defaultModel")) + Expect(result.Sources).NotTo(HaveKey("/mode")) + Expect(result.Sources).NotTo(HaveKey("/fallbacks/0/mode")) + }) + + It("keeps explicit empty fallbacks and passes authored compact pins to normalization", func() { + model := (registry.Model{Name: "api:sonnet:high", Fallbacks: registry.ModelList{}}).WithExplicit("/fallbacks") + calls := 0 + result, err := ApplyDefaults(DefaultOptions{ + Model: model, Saved: captainconfig.AIDefaults{DefaultModel: "sonnet,sol"}, + Normalize: func(selected registry.Model) (registry.Model, error) { + calls++ + Expect(selected.Name).To(Equal("sonnet")) + Expect(selected.Mode).To(Equal(registry.ModeAPI)) + Expect(selected.Effort).To(Equal(registry.EffortHigh)) + Expect(selected.Fallbacks).To(BeEmpty()) + return selected, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(calls).To(Equal(1)) + Expect(result.Model.Fallbacks).To(BeEmpty()) + Expect(model.Name).To(Equal("api:sonnet:high")) + }) + + It("treats saved compact modes as gaps on both primary and fallback", func() { + result, err := ApplyDefaults(DefaultOptions{ + Saved: captainconfig.AIDefaults{DefaultModel: "api:sonnet:high,api:sol:medium"}, + Normalize: func(model registry.Model) (registry.Model, error) { + Expect(model.Mode).To(BeEmpty()) + Expect(model.Fallbacks).To(HaveLen(1)) + Expect(model.Fallbacks[0].Mode).To(BeEmpty()) + model.Mode = registry.ModeCLI + model.Fallbacks[0].Mode = registry.ModeCLI + return model, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Mode).To(Equal(registry.ModeCLI)) + Expect(result.Model.Fallbacks[0].Mode).To(Equal(registry.ModeCLI)) + Expect(result.Model.Effort).To(Equal(registry.EffortHigh)) + Expect(result.Model.Fallbacks[0].Effort).To(Equal(registry.EffortMedium)) + Expect(result.Sources).NotTo(HaveKey("/mode")) + Expect(result.Sources).NotTo(HaveKey("/fallbacks/0/mode")) + Expect(result.Sources).To(HaveKeyWithValue("/fallbacks/0/effort", "ai.defaultModel")) + }) + + It("propagates normalization errors before adding candidate defaults", func() { + failure := errors.New("sandbox rejects selected runtime") + _, err := ApplyDefaults(DefaultOptions{ + Saved: captainconfig.AIDefaults{DefaultModel: "sonnet,sol"}, + Normalize: func(registry.Model) (registry.Model, error) { return registry.Model{}, failure }, + }) + Expect(err).To(MatchError(failure)) + }) + + It("rejects malformed saved values before invoking normalization", func() { + calls := 0 + _, err := ApplyDefaults(DefaultOptions{ + Saved: captainconfig.AIDefaults{DefaultModel: "sonnet,sol", Temperature: 3}, + Normalize: func(model registry.Model) (registry.Model, error) { calls++; return model, nil }, + }) + Expect(err).To(MatchError(ContainSubstring("ai.temperature"))) + Expect(calls).To(BeZero()) + }) +}) diff --git a/pkg/aiflags/default_resolution.go b/pkg/aiflags/default_resolution.go new file mode 100644 index 00000000..dd3c8903 --- /dev/null +++ b/pkg/aiflags/default_resolution.go @@ -0,0 +1,195 @@ +package aiflags + +import ( + "fmt" + + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/captainconfig" +) + +type DefaultOptions struct { + Model registry.Model + Saved captainconfig.AIDefaults + CatalogDefaults bool + AllowUnknownModel bool + // Normalize derives execution context after model selection and before knob defaults. + // It must preserve selected model names and fallback order. + Normalize func(registry.Model) (registry.Model, error) +} + +type DefaultedModel struct { + Model registry.Model + Sources map[string]string + Unconfigured []UnconfiguredCandidate +} + +type UnconfiguredCandidate struct { + Path string + Model string + Provider *registry.Provider +} + +func ApplyDefaults(options DefaultOptions) (DefaultedModel, error) { + if err := options.Saved.Validate(); err != nil { + return DefaultedModel{}, err + } + model := (registry.Model{}).Merge(options.Model) + if err := model.ValidateOptions(); err != nil { + return DefaultedModel{}, err + } + global, err := globalDefaultModel(options.Saved) + if err != nil { + return DefaultedModel{}, err + } + model, err = model.Expand() + if err != nil { + return DefaultedModel{}, err + } + result := DefaultedModel{Sources: map[string]string{}} + selection, err := result.selectModels(modelSelectionOptions{DefaultOptions: options, Model: model, Global: global}) + if err != nil { + return DefaultedModel{}, err + } + model = selection.Model + if options.Normalize != nil { + model, err = options.Normalize((registry.Model{}).Merge(model)) + if err != nil { + return DefaultedModel{}, err + } + if err := selection.validateNormalized(model); err != nil { + return DefaultedModel{}, err + } + } + authored := model + primary := selection.Candidates[0] + primary.Model = model + model, err = result.applyCandidate(primary) + if err != nil { + return DefaultedModel{}, err + } + for i, fallback := range model.Fallbacks { + candidate := selection.Candidates[i+1] + fallback = result.inheritPrimary(fallback, authored, model, fmt.Sprintf("/fallbacks/%d", i)) + fallback = result.fillCandidate(fallback, candidate.Selected, candidate) + candidate.Model = fallback + model.Fallbacks[i], err = result.applyCandidate(candidate) + if err != nil { + return DefaultedModel{}, err + } + } + result.Model = model + return result, nil +} + +func (result *DefaultedModel) fillCatalogEffort(options candidateDefaults) (registry.Model, error) { + model := options.Model + if !options.CatalogDefaults || model.Name == "" || model.Fields().Has("/effort") { + return model, nil + } + provider, err := registry.ProviderFor(model.Name) + if err != nil { + if options.AllowUnknownModel && registry.IsUnknownModel(err) { + return model, nil + } + return registry.Model{}, err + } + mode := model.Mode + if mode == "" { + mode = provider.DefaultMode + } + id, _ := provider.ResolveExact(mode, model.Name) + _, effort, known := registry.ModelEfforts(provider, mode, id) + if !known { + return model, nil + } + if effort == "" { + if raw, ok := provider.Lookup(id); ok && raw.DefaultEffort != "" { + if _, err := registry.ResolveEffort(provider, mode, id, raw.DefaultEffort); err != nil { + return registry.Model{}, err + } + } + return model, nil + } + model.Effort = effort + result.Sources[options.Path+"/effort"] = "registry.models." + id + ".defaultEffort" + return model, nil +} + +func (result *DefaultedModel) inheritPrimary(fallback, authored, primary registry.Model, path string) registry.Model { + present, inherited := fallback.Fields(), authored.Fields() + if !present.Has("/temperature") && inherited.Has("/temperature") { + fallback.Temperature = authored.Temperature + fallback = fallback.WithExplicit("/temperature") + result.Sources[path+"/temperature"] = "primary.temperature" + } + if !present.Has("/noCache") && inherited.Has("/noCache") { + fallback.NoCache = authored.NoCache + fallback = fallback.WithExplicit("/noCache") + result.Sources[path+"/noCache"] = "primary.noCache" + } + primaryProvider, primaryErr := registry.ProviderFor(primary.Name) + fallbackProvider, fallbackErr := registry.ProviderFor(fallback.Name) + if !present.Has("/effort") && inherited.Has("/effort") && primaryErr == nil && fallbackErr == nil && primaryProvider == fallbackProvider { + fallback.Effort = authored.Effort + fallback = fallback.WithExplicit("/effort") + result.Sources[path+"/effort"] = "primary.effort" + } + return fallback +} + +type candidateDefaults struct { + Model registry.Model + Saved captainconfig.AIDefaults + Path string + CatalogDefaults bool + AllowUnknownModel bool + Provider *registry.Provider + Defaults DefaultedModel + Selected DefaultedModel +} + +func (result *DefaultedModel) applyCandidate(options candidateDefaults) (registry.Model, error) { + model := options.Model + provider := options.Provider + if provider != nil { + model = result.fillCandidate(model, options.Defaults, options) + if model.Mode == "" { + if modes := provider.Modes(); len(modes) == 1 && !model.Fields().Has("/mode") { + model.Mode = modes[0] + result.Sources[options.Path+"/mode"] = "registry.providers." + provider.Name + ".modes" + } else if model.Name != "" { + result.Unconfigured = append(result.Unconfigured, UnconfiguredCandidate{Path: options.Path + "/mode", Model: model.Name, Provider: provider}) + } + } + } + options.Model = result.fillGeneration(model, options) + return result.fillCatalogEffort(options) +} + +func (result *DefaultedModel) fillCandidate(model registry.Model, defaults DefaultedModel, options candidateDefaults) registry.Model { + present := model.Fields() + if !present.Has("/mode") && defaults.Model.Mode != "" { + model.Mode = defaults.Model.Mode + result.Sources[options.Path+"/mode"] = defaults.Sources["/mode"] + } + if !present.Has("/effort") && defaults.Model.Effort != "" { + model.Effort = defaults.Model.Effort + result.Sources[options.Path+"/effort"] = defaults.Sources["/effort"] + } + return model +} + +func (result *DefaultedModel) fillGeneration(model registry.Model, options candidateDefaults) registry.Model { + present, saved := model.Fields(), options.Saved.Fields() + if !present.Has("/temperature") && saved.Has("/temperature") { + temperature := options.Saved.Temperature + model.Temperature = &temperature + result.Sources[options.Path+"/temperature"] = "ai.temperature" + } + if !present.Has("/noCache") && saved.Has("/noCache") { + model.NoCache = options.Saved.NoCache + model = model.WithExplicit("/noCache") + result.Sources[options.Path+"/noCache"] = "ai.noCache" + } + return model +} diff --git a/pkg/aiflags/default_resolution_ginkgo_test.go b/pkg/aiflags/default_resolution_ginkgo_test.go new file mode 100644 index 00000000..4122887a --- /dev/null +++ b/pkg/aiflags/default_resolution_ginkgo_test.go @@ -0,0 +1,226 @@ +package aiflags + +import ( + "fmt" + "testing" + + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/captainconfig" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDefaultResolution(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Final model defaults") +} + +var _ = Describe("final model saved defaults", func() { + DescribeTable("keeps unknown authored candidates repairable only for partial composition", func(allow bool) { + input := registry.Model{Name: "unknown-primary-example", Fallbacks: registry.ModelList{{Name: "unknown-fallback-example"}}} + result, err := ApplyDefaults(DefaultOptions{Model: input, Saved: captainconfig.AIDefaults{Temperature: 0.4, NoCache: true}, CatalogDefaults: true, AllowUnknownModel: allow}) + if !allow { + Expect(registry.IsUnknownModel(err)).To(BeTrue()) + return + } + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Name).To(Equal(input.Name)) + Expect(result.Model.Fallbacks[0].Name).To(Equal(input.Fallbacks[0].Name)) + for _, candidate := range []registry.Model{result.Model, result.Model.Fallbacks[0]} { + Expect(candidate.Mode).To(BeEmpty()) + Expect(candidate.Effort).To(BeEmpty()) + Expect(*candidate.Temperature).To(Equal(0.4)) + Expect(candidate.NoCache).To(BeTrue()) + } + }, Entry("partial composition", true), Entry("direct run resolution", false)) + + It("still rejects malformed saved configuration while unknown authored candidates are allowed", func() { + _, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "unknown-example"}, Saved: captainconfig.AIDefaults{DefaultModel: "unknown-saved-example"}, AllowUnknownModel: true}) + Expect(err).To(MatchError(ContainSubstring("ai.defaultModel"))) + }) + + It("applies each candidate's own provider settings and records their exact keys", func() { + saved := captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{ + "anthropic": {Mode: "agent", ReasoningEffort: "high"}, + "openai": {Mode: "api", ReasoningEffort: "medium"}, + }} + input := registry.Model{Name: "sonnet", Fallbacks: registry.ModelList{{Name: "sol"}}} + result, err := ApplyDefaults(DefaultOptions{Model: input, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Mode).To(Equal(registry.ModeAgent)) + Expect(result.Model.Effort).To(Equal(registry.EffortHigh)) + Expect(result.Model.Fallbacks[0].Mode).To(Equal(registry.ModeAPI)) + Expect(result.Model.Fallbacks[0].Effort).To(Equal(registry.EffortMedium)) + Expect(result.Sources).To(Equal(map[string]string{ + "/mode": "ai.providers.anthropic.mode", "/effort": "ai.providers.anthropic.reasoningEffort", + "/fallbacks/0/mode": "ai.providers.openai.mode", "/fallbacks/0/effort": "ai.providers.openai.reasoningEffort", + })) + Expect(input.Fallbacks[0].Mode).To(BeEmpty()) + Expect(result.Unconfigured).To(BeEmpty()) + }) + + It("resolves an agent-named provider block and retains its authored source key", func() { + saved := captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{ + "gemini": {Mode: "cli", Model: "gemini-3.5-flash"}, + }} + result, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "gemini-3.5-flash"}, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Mode).To(Equal(registry.ModeCLI)) + Expect(result.Sources).To(HaveKeyWithValue("/mode", "ai.providers.gemini.mode")) + }) + + It("rejects two saved keys that name the same provider", func() { + saved := captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{ + "google": {Mode: "api"}, + "gemini": {Mode: "cli"}, + }} + _, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "gemini-3.5-flash"}, Saved: saved}) + Expect(err).To(MatchError(And(ContainSubstring("ai.providers.gemini"), ContainSubstring("ai.providers.google"), ContainSubstring("same provider")))) + }) + + It("preserves the global compact selector's effort and fallback chain", func() { + saved := captainconfig.AIDefaults{DefaultModel: "agent:sonnet:high, api:sol:medium"} + result, err := ApplyDefaults(DefaultOptions{Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Name).To(Equal("sonnet")) + Expect(result.Model.Effort).To(Equal(registry.EffortHigh)) + Expect(result.Model.Fallbacks).To(HaveLen(1)) + Expect(result.Model.Fallbacks[0].Mode).To(Equal(registry.ModeAPI)) + Expect(result.Model.Fallbacks[0].Effort).To(Equal(registry.EffortMedium)) + Expect(result.Sources).To(HaveKeyWithValue("/fallbacks", "ai.defaultModel")) + Expect(result.Sources).To(HaveKeyWithValue("/fallbacks/0/effort", "ai.defaultModel")) + }) + + It("reports every missing mode without leaking a global selector across providers", func() { + saved := captainconfig.AIDefaults{DefaultModel: "api:sonnet:high"} + result, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "sol", Fallbacks: registry.ModelList{{Name: "gemini-3.5-flash"}}}, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Mode).To(BeEmpty()) + Expect(result.Model.Effort).To(BeEmpty()) + Expect(result.Model.Fallbacks[0].Mode).To(BeEmpty()) + Expect(result.Unconfigured).To(Equal([]UnconfiguredCandidate{ + {Path: "/mode", Model: "sol", Provider: registry.OpenAI}, + {Path: "/fallbacks/0/mode", Model: "gemini-3.5-flash", Provider: registry.Google}, + })) + }) + + It("does not load any model when no selection or saved configuration exists", func() { + result, err := ApplyDefaults(DefaultOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Name).To(BeEmpty()) + Expect(result.Sources).To(BeEmpty()) + }) + + It("preserves explicit false, zero and empty fallbacks over saved settings", func() { + zero := 0.0 + model := (registry.Model{Name: "sonnet", Temperature: &zero, Fallbacks: registry.ModelList{}}).WithExplicit("/noCache", "/fallbacks") + saved := captainconfig.AIDefaults{DefaultModel: "agent:sonnet:high,api:sol", Temperature: 0.8, NoCache: true} + result, err := ApplyDefaults(DefaultOptions{Model: model, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.NoCache).To(BeFalse()) + Expect(*result.Model.Temperature).To(BeZero()) + Expect(result.Model.Fallbacks).To(BeEmpty()) + Expect(result.Sources).NotTo(HaveKey("/noCache")) + Expect(result.Sources).NotTo(HaveKey("/temperature")) + Expect(result.Sources).NotTo(HaveKey("/fallbacks")) + }) + + It("applies an explicitly saved zero temperature and false cache policy", func() { + saved := (captainconfig.AIDefaults{}).WithExplicit("/temperature", "/noCache") + result, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "agent:sonnet"}, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Temperature).NotTo(BeNil()) + Expect(*result.Model.Temperature).To(BeZero()) + Expect(result.Sources).To(HaveKeyWithValue("/temperature", "ai.temperature")) + Expect(result.Sources).To(HaveKeyWithValue("/noCache", "ai.noCache")) + }) + + It("keeps same-valued explicit choices out of saved provenance", func() { + saved := captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{"anthropic": {Mode: "agent", ReasoningEffort: "high"}}} + result, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "agent:sonnet:high"}, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Mode).To(Equal(registry.ModeAgent)) + Expect(result.Model.Effort).To(Equal(registry.EffortHigh)) + Expect(result.Sources).To(BeEmpty()) + }) + + DescribeTable("applies only declared catalog effort defaults when enabled", func(enabled bool, expected registry.Effort) { + result, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "agent:fable", Fallbacks: registry.ModelList{{Name: "api:fable"}}}, CatalogDefaults: enabled}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Name).To(Equal("fable")) + Expect(result.Model.Effort).To(Equal(expected)) + Expect(result.Model.Fallbacks[0].Effort).To(Equal(expected)) + if enabled { + Expect(result.Sources).To(HaveKeyWithValue("/effort", "registry.models.claude-fable-5-1.defaultEffort")) + Expect(result.Sources).To(HaveKeyWithValue("/fallbacks/0/effort", "registry.models.claude-fable-5-1.defaultEffort")) + } + }, Entry("saved defaults enabled", true, registry.EffortHigh), Entry("catalog/replay projection", false, registry.EffortNone)) + + It("keeps an explicit empty effort over the catalog default", func() { + result, err := ApplyDefaults(DefaultOptions{Model: (registry.Model{Name: "agent:fable"}).WithExplicit("/effort"), CatalogDefaults: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Effort).To(BeEmpty()) + Expect(result.Sources).NotTo(HaveKey("/effort")) + }) + + It("keeps explicit zero temperature at the flag projection boundary", func() { + model, err := (ModelFlags{Temperature: "0"}).ToModel() + Expect(err).NotTo(HaveOccurred()) + Expect(model.Temperature).NotTo(BeNil()) + Expect(*model.Temperature).To(BeZero()) + }) + + DescribeTable("retains raw selectors when an explicit mode is validated across flag fallbacks", func(primary string) { + model, err := (ModelFlags{Model: primary, Mode: "api", Fallback: []string{"api:sol:medium"}}).ToModel() + Expect(err).NotTo(HaveOccurred()) + Expect(model.Name).To(Equal(primary)) + Expect(model.Fallbacks[0].Name).To(Equal("api:sol:medium")) + }, Entry("compact primary", "api:sonnet:high"), Entry("bare primary", "sonnet")) + + It("does not replace an explicitly false cache flag with a saved true", func() { + flags := (ModelFlags{Model: "agent:sonnet"}).WithExplicit("/noCache") + model, err := flags.ResolveWith(captainconfig.AIDefaults{NoCache: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(model.NoCache).To(BeFalse()) + }) + + It("rejects a separate mode that contradicts the compact selector", func() { + _, err := (ModelFlags{Model: "agent:sonnet", Mode: "api"}).ToModel() + Expect(err).To(MatchError(ContainSubstring("contradicts requested mode"))) + }) + + It("inherits authored primary knobs before provider defaults without inheriting its mode", func() { + temperature := 0.2 + model := (registry.Model{Name: "api:sonnet:high", Temperature: &temperature, Fallbacks: registry.ModelList{{Name: "haiku"}, {Name: "sol"}}}).WithExplicit("/noCache") + saved := captainconfig.AIDefaults{Temperature: 0.8, NoCache: true, Providers: map[string]captainconfig.ProviderDefaults{ + "anthropic": {Mode: "agent", ReasoningEffort: "low"}, "openai": {Mode: "api", ReasoningEffort: "medium"}, + }} + result, err := ApplyDefaults(DefaultOptions{Model: model, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Model.Fallbacks[0].Effort).To(Equal(registry.EffortHigh)) + Expect(result.Model.Fallbacks[0].Mode).To(Equal(registry.ModeAgent)) + Expect(result.Model.Fallbacks[1].Effort).To(Equal(registry.EffortMedium)) + for i, fallback := range result.Model.Fallbacks { + Expect(*fallback.Temperature).To(Equal(temperature)) + Expect(fallback.NoCache).To(BeFalse()) + Expect(result.Sources).To(HaveKeyWithValue(fmt.Sprintf("/fallbacks/%d/temperature", i), "primary.temperature")) + Expect(result.Sources).To(HaveKeyWithValue(fmt.Sprintf("/fallbacks/%d/noCache", i), "primary.noCache")) + } + Expect(result.Sources).To(HaveKeyWithValue("/fallbacks/0/effort", "primary.effort")) + }) + + DescribeTable("rejects malformed saved values even when authored fields would replace them", func(saved captainconfig.AIDefaults, source string) { + temperature := 0.2 + _, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "agent:sonnet:high", Temperature: &temperature}, Saved: saved}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(source)) + }, + Entry("temperature", captainconfig.AIDefaults{Temperature: 3}, "ai.temperature"), + Entry("cost", captainconfig.AIDefaults{BudgetUSD: -1}, "ai.budgetUSD"), + Entry("token ceiling", captainconfig.AIDefaults{MaxTokens: -1}, "ai.maxTokens"), + Entry("timeout", captainconfig.AIDefaults{Timeout: "tomorrow"}, "ai.timeout"), + Entry("unknown global model", captainconfig.AIDefaults{DefaultModel: "unknown-example"}, "ai.defaultModel"), + Entry("provider mode", captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{"openai": {Mode: "invalid"}}}, "ai.providers.openai.mode"), + Entry("provider effort", captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{"openai": {ReasoningEffort: "invalid"}}}, "ai.providers.openai.reasoningEffort"), + ) +}) diff --git a/pkg/aiflags/default_selection.go b/pkg/aiflags/default_selection.go new file mode 100644 index 00000000..480d8800 --- /dev/null +++ b/pkg/aiflags/default_selection.go @@ -0,0 +1,108 @@ +package aiflags + +import ( + "fmt" + "strings" + + "github.com/flanksource/captain/pkg/api/registry" +) + +type selectedModels struct { + Model registry.Model + Candidates []candidateDefaults +} + +type modelSelectionOptions struct { + DefaultOptions + Model registry.Model + Global registry.Model +} + +func (result *DefaultedModel) selectModels(options modelSelectionOptions) (selectedModels, error) { + model := options.Model + primary, err := prepareCandidate(options, "") + if err != nil { + return selectedModels{}, err + } + present := model.Fields() + if !present.Has("/model") && primary.Defaults.Model.Name != "" { + model.Name = primary.Defaults.Model.Name + result.Sources["/model"] = primary.Defaults.Sources["/model"] + } + var selected registry.ModelList + if !present.Has("/fallbacks") && primary.Defaults.Model.Fields().Has("/fallbacks") { + selected = primary.Defaults.Model.Fallbacks + model.Fallbacks = make(registry.ModelList, len(selected)) + result.Sources["/fallbacks"] = primary.Defaults.Sources["/fallbacks"] + for i, fallback := range selected { + model.Fallbacks[i] = registry.Model{Name: fallback.Name} + path := fmt.Sprintf("/fallbacks/%d/model", i) + result.Sources[path] = primary.Defaults.Sources[path] + } + } + selection := selectedModels{Candidates: []candidateDefaults{primary}} + for i, fallback := range model.Fallbacks { + fallback, err = fallback.Expand() + if err != nil { + return selectedModels{}, fmt.Errorf("fallback[%d]: %w", i, err) + } + path := fmt.Sprintf("/fallbacks/%d", i) + fallbackOptions := options + fallbackOptions.Model = fallback + candidate, err := prepareCandidate(fallbackOptions, path) + if err != nil { + return selectedModels{}, err + } + if selected != nil { + candidate.Selected = DefaultedModel{Model: selected[i], Sources: map[string]string{ + "/mode": primary.Defaults.Sources[path+"/mode"], "/effort": primary.Defaults.Sources[path+"/effort"], + }} + } + model.Fallbacks[i] = fallback + selection.Candidates = append(selection.Candidates, candidate) + } + selection.Model = model + return selection, nil +} + +func prepareCandidate(options modelSelectionOptions, path string) (candidateDefaults, error) { + model := options.Model + candidate := candidateDefaults{Model: model, Saved: options.Saved, Path: path, + CatalogDefaults: options.CatalogDefaults, AllowUnknownModel: options.AllowUnknownModel, Provider: model.Provider} + if model.Name != "" { + var err error + candidate.Provider, err = registry.ProviderFor(model.Name) + if err != nil && (!options.AllowUnknownModel || !registry.IsUnknownModel(err)) { + return candidateDefaults{}, err + } + } else if path == "" && !model.Fields().Has("/model") { + candidate.Provider = options.Global.Provider + if candidate.Provider == nil && strings.TrimSpace(options.Saved.DefaultProvider) != "" { + var ok bool + candidate.Provider, ok = registry.ProviderByName(options.Saved.DefaultProvider) + if !ok { + return candidateDefaults{}, fmt.Errorf("ai.defaultProvider %q is unknown", options.Saved.DefaultProvider) + } + } + } + if candidate.Provider != nil { + var err error + candidate.Defaults, err = savedProviderModel(options.Saved, candidate.Provider, options.Global) + if err != nil { + return candidateDefaults{}, err + } + } + return candidate, nil +} + +func (selection selectedModels) validateNormalized(model registry.Model) error { + if model.Name != selection.Model.Name || len(model.Fallbacks) != len(selection.Model.Fallbacks) { + return fmt.Errorf("model normalization must preserve selected model names and fallback order") + } + for i, fallback := range model.Fallbacks { + if fallback.Name != selection.Model.Fallbacks[i].Name { + return fmt.Errorf("model normalization must preserve fallback[%d] selection %q", i, selection.Model.Fallbacks[i].Name) + } + } + return model.ValidateOptions() +} diff --git a/pkg/aiflags/default_sources.go b/pkg/aiflags/default_sources.go new file mode 100644 index 00000000..aab32190 --- /dev/null +++ b/pkg/aiflags/default_sources.go @@ -0,0 +1,74 @@ +package aiflags + +import ( + "fmt" + "strings" + + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/captainconfig" +) + +func savedProviderModel(saved captainconfig.AIDefaults, provider *registry.Provider, global registry.Model) (DefaultedModel, error) { + result := DefaultedModel{Sources: map[string]string{}} + if global.Provider == provider { + result.Model = (registry.Model{}).Merge(global) + attributeFields(result.Sources, result.Model, "", "ai.defaultModel") + } + configured, providerKey, err := saved.Provider(provider) + if err != nil { + return DefaultedModel{}, err + } + if providerKey == "" { + providerKey = provider.Name + } + model := registry.Model{Name: strings.TrimSpace(configured.Model), Mode: registry.RuntimeMode(strings.TrimSpace(configured.Mode)), Effort: registry.Effort(strings.TrimSpace(configured.ReasoningEffort))} + if err := model.ValidateOptions(); err != nil { + return DefaultedModel{}, fmt.Errorf("ai.providers.%s: %w", providerKey, err) + } + compact, err := (registry.Model{Name: model.Name}).Expand() + if err != nil { + return DefaultedModel{}, fmt.Errorf("ai.providers.%s.model: %w", providerKey, err) + } + model, err = model.Expand() + if err != nil { + return DefaultedModel{}, err + } + if model.Name != "" { + actual, err := registry.ProviderFor(model.Name) + if err != nil || actual != provider { + return DefaultedModel{}, fmt.Errorf("ai.providers.%s.model %q does not name a model from %s", providerKey, configured.Model, provider.Name) + } + } + result.Model = result.Model.Merge(model) + for path := range model.Fields() { + key := strings.TrimPrefix(path, "/") + if compact.Fields().Has(path) { + key = "model" + } else if path == "/effort" { + key = "reasoningEffort" + } + result.Sources[path] = "ai.providers." + providerKey + "." + key + } + if len(model.Fallbacks) > 0 { + attributeFallbacks(result.Sources, model.Fallbacks, "", "ai.providers."+providerKey+".model") + } + if result.Model.Mode != "" { + if _, err := provider.RequireMode(result.Model.Mode); err != nil { + return DefaultedModel{}, fmt.Errorf("%s: %w", result.Sources["/mode"], err) + } + } + return result, nil +} + +func attributeFields(sources map[string]string, model registry.Model, prefix, source string) { + for path := range model.Fields() { + sources[prefix+path] = source + } + attributeFallbacks(sources, model.Fallbacks, prefix, source) +} + +func attributeFallbacks(sources map[string]string, models registry.ModelList, prefix, source string) { + for i, model := range models { + attributeFields(sources, model, fmt.Sprintf("%s/fallbacks/%d", prefix, i), source) + } +} diff --git a/pkg/aiflags/defaults.go b/pkg/aiflags/defaults.go index bfef7731..a775d363 100644 --- a/pkg/aiflags/defaults.go +++ b/pkg/aiflags/defaults.go @@ -19,10 +19,8 @@ type ProviderDefaultView struct { // LoadDefaults reads the saved AI defaults from ~/.captain.yaml. // -// It reports a broken config rather than swallowing it: whether to degrade to zero -// defaults and carry on is a CLI policy, not a library's call. captain's own -// commands make that choice in pkg/cli (loadSavedAI warns and continues); callers -// wanting the same behaviour pass their own AIDefaults to ResolveWith. +// Broken configuration is returned to the caller. Request pipelines capture one +// settings snapshot and pass its AIDefaults to the shared specification resolver. // // Deliberately no logger here — commons/logger pulls ~55 packages (prometheus, // fsnotify, …) and would cost this leaf its entire reason for existing. @@ -70,45 +68,29 @@ func SavedDefaults(saved captainconfig.AIDefaults, provider *registry.Provider) if provider == nil { return ProviderDefaultView{}, fmt.Errorf("provider is required") } - configured, exists := saved.Providers[provider.Name] - mode := registry.RuntimeMode(strings.TrimSpace(configured.Mode)) - model := strings.TrimSpace(configured.Model) - effort := registry.Effort(strings.TrimSpace(configured.ReasoningEffort)) - - if model == "" || mode == "" { - global, err := globalDefaultModel(saved) - if err != nil { - return ProviderDefaultView{}, err - } - // Only adopt the global model when it belongs to this provider; its mode - // is a mechanism and travels regardless. - if model == "" && global.Provider == provider { - model = global.Name - } - if mode == "" { - mode = global.Mode - } + if err := saved.Validate(); err != nil { + return ProviderDefaultView{}, err } - - // A provider that serves exactly one mode leaves nothing to guess: naming it - // is arithmetic, not a default. Without this, configuring one provider and - // then naming a model from another would demand a second `captain configure` - // for a mechanism that was never ambiguous. - if mode == "" { - if modes := provider.Modes(); len(modes) == 1 { - mode = modes[0] - } + global, err := globalDefaultModel(saved) + if err != nil { + return ProviderDefaultView{}, err } - if mode != "" { - if _, err := provider.RequireMode(mode); err != nil { - return ProviderDefaultView{}, err + defaults, err := savedProviderModel(saved, provider, global) + if err != nil { + return ProviderDefaultView{}, err + } + model := defaults.Model + if model.Mode == "" { + if modes := provider.Modes(); len(modes) == 1 { + model.Mode = modes[0] } } - if err := effort.Validate(); err != nil { + _, providerKey, err := saved.Provider(provider) + if err != nil { return ProviderDefaultView{}, err } return ProviderDefaultView{ - Mode: string(mode), Model: model, Effort: string(effort), Configured: exists, + Mode: string(model.Mode), Model: model.Name, Effort: string(model.Effort), Configured: providerKey != "", }, nil } @@ -129,84 +111,6 @@ func globalDefaultModel(saved captainconfig.AIDefaults) (registry.Model, error) return model, nil } -// ApplyDefaults fills a model's unset fields from the saved per-provider defaults, -// primary and fallbacks alike. It expects an already-expanded model (see the -// package doc) and does not resolve — the caller resolves once, afterwards. -func ApplyDefaults(model registry.Model, saved captainconfig.AIDefaults) (registry.Model, error) { - var err error - if strings.TrimSpace(model.Name) != "" { - if model, err = model.Expand(); err != nil { - return registry.Model{}, err - } - } - if model, err = applyCandidateDefaults(model, saved, true); err != nil { - return registry.Model{}, err - } - for i := range model.Fallbacks { - fallback := model.Fallbacks[i] - if fallback, err = fallback.Expand(); err != nil { - return registry.Model{}, fmt.Errorf("fallback[%d]: %w", i, err) - } - // allowActive=false: a fallback must not silently become the active - // provider's model — that would make the fallback chain a no-op. - if fallback, err = applyCandidateDefaults(fallback, saved, false); err != nil { - return registry.Model{}, fmt.Errorf("fallback[%d]: %w", i, err) - } - model.Fallbacks[i] = fallback - } - return model, nil -} - -func applyCandidateDefaults(model registry.Model, saved captainconfig.AIDefaults, allowActive bool) (registry.Model, error) { - provider := model.Provider - if provider == nil && strings.TrimSpace(model.Name) != "" { - p, err := registry.ProviderFor(model.Name) - if err != nil { - return registry.Model{}, err - } - provider = p - } - // A nameless model takes the configured provider, preferring the one - // ai.defaultModel names over the coarser defaultProvider key. - if provider == nil && allowActive { - if global, err := globalDefaultModel(saved); err != nil { - return registry.Model{}, err - } else if global.Provider != nil { - provider = global.Provider - } - } - if provider == nil && allowActive { - provider, _ = registry.ProviderByName(saved.ActiveProvider()) - } - if provider == nil { - return registry.Model{}, fmt.Errorf("provider cannot be resolved for model %q", model.Name) - } - // SavedDefaults, not EffectiveDefaults: a run inherits only what the user - // configured. EffectiveDefaults would fill the gaps from the registry's - // built-in tables, which is right for seeding a form and wrong here — it is - // exactly how an unconfigured `--ai-model haiku` silently acquired agent mode. - defaults, err := SavedDefaults(saved, provider) - if err != nil { - return registry.Model{}, err - } - // An explicit --mode owns the mechanism; the saved default only fills a gap. - if model.Mode == "" { - model.Mode = registry.RuntimeMode(defaults.Mode) - } - if strings.TrimSpace(model.Name) == "" { - model.Name = defaults.Model - } - if model.Effort == registry.EffortNone { - model.Effort = registry.Effort(defaults.Effort) - } - // Preserve valid requested tiers here; execution resolves them against the - // exact provider model after configuration and selector overlays are complete. - if err := model.Effort.Validate(); err != nil { - return registry.Model{}, err - } - return model, nil -} - // AllProviderDefaults resolves every provider's effective defaults. func AllProviderDefaults(saved captainconfig.AIDefaults) (map[string]ProviderDefaultView, error) { out := make(map[string]ProviderDefaultView, len(registry.Providers())) diff --git a/pkg/aiflags/flags.go b/pkg/aiflags/flags.go index 05a867ae..f0dc3ce7 100644 --- a/pkg/aiflags/flags.go +++ b/pkg/aiflags/flags.go @@ -13,15 +13,9 @@ // // # The invariant // -// Expand, then Merge, then Resolve — once, at the end. -// -// Merging an unexpanded override onto a base silently keeps the base's mode: -// base{Mode: agent}.Merge(Model{Name: "api:opus"}) still says agent, and -// resolution then runs the wrong runtime. Expand sets Mode only when the string -// carries a prefix, so a bare `--model opus` correctly inherits the base's mode -// while `api:opus` correctly overrides it. Every ladder in this package and its -// callers follows that order; departing from it is how a model string loses its -// mode again. +// Preserve authored selectors while layering, then apply saved defaults and +// resolve once at the complete request boundary. Compact selector pins retain +// precedence over sibling fields when the final model is expanded. package aiflags import ( @@ -59,24 +53,24 @@ import ( // // No field carries a `default:` tag: defaults only materialize through cobra // binding, so a directly-constructed ModelFlags would disagree with a bound one. -// Zero means unset, everywhere. +// Explicit records changed flags whose zero values must override saved values. type ModelFlags struct { - Model string `flag:"model" help:"Model name(s), e.g. claude-sonnet-5, a compact selector like agent:opus:high, or a comma-separated primary,fallback list (defaults to the value saved by 'captain configure')"` - Fallback []string `flag:"fallback" help:"Model to try if the primary is unavailable (repeatable; comma-separated allowed)"` - Mode string `flag:"mode" help:"Runtime mechanism: api|cli|agent|cmux (sdk aliases agent). The provider comes from the model name; a mode prefix on --model wins, and contradicting it fails loud"` - Effort string `flag:"effort" help:"Reasoning effort: low|medium|high|xhigh|max|ultra (model-dependent)"` - Temperature string `flag:"temperature" help:"Sampling temperature (0.0-2.0)"` - NoCache bool `flag:"no-cache" help:"Disable response caching"` + Explicit registry.FieldPresence `flag:"-" json:"-" yaml:"-"` + Model string `flag:"model" help:"Model name(s), e.g. claude-sonnet-5, a compact selector like agent:opus:high, or a comma-separated primary,fallback list (defaults to the value saved by 'captain configure')"` + Fallback []string `flag:"fallback" help:"Model to try if the primary is unavailable (repeatable; comma-separated allowed)"` + Mode string `flag:"mode" help:"Runtime mechanism: api|cli|agent|cmux (sdk aliases agent). The provider comes from the model name; a mode prefix on --model wins, and contradicting it fails loud"` + Effort string `flag:"effort" help:"Reasoning effort: low|medium|high|xhigh|max|ultra (model-dependent)"` + Temperature string `flag:"temperature" help:"Sampling temperature (0.0-2.0)"` + NoCache bool `flag:"no-cache" help:"Disable response caching"` } -// ToModel projects the flags onto a Model and expands any compact selector, but -// does NOT resolve it against the catalog. That is the merge-safe form: callers -// layering flags over a spec must Merge before resolving (see the package doc). +// ToModel projects authored flags without normalizing compact selectors. func (f ModelFlags) ToModel() (registry.Model, error) { m := registry.Model{ - Name: strings.TrimSpace(f.Model), - Effort: registry.Effort(strings.TrimSpace(f.Effort)), - NoCache: f.NoCache, + Explicit: f.Explicit.Clone(), + Name: strings.TrimSpace(f.Model), + Effort: registry.Effort(strings.TrimSpace(f.Effort)), + NoCache: f.NoCache, } if err := m.Effort.Validate(); err != nil { return registry.Model{}, fmt.Errorf("invalid --effort %q: %w", f.Effort, err) @@ -97,20 +91,31 @@ func (f ModelFlags) ToModel() (registry.Model, error) { m.Temperature = temp m.Fallbacks = FallbackModels(f.Fallback) - if m.Name != "" { - if m, err = m.Expand(); err != nil { - return registry.Model{}, err - } + if _, err = m.Expand(); err != nil { + return registry.Model{}, err } if strings.TrimSpace(f.Mode) != "" { - return m.WithMode(m.Mode) + if _, err := m.WithMode(m.Mode); err != nil { + return registry.Model{}, err + } + m = m.ExpandCSV() + for i := range m.Fallbacks { + m.Fallbacks[i].Mode = m.Mode + } } return m, nil } +// WithExplicit records the JSON-pointer fields set by a flag binder, including +// --no-cache=false and an explicitly empty --fallback list. +func (f ModelFlags) WithExplicit(paths ...string) ModelFlags { + f.Explicit = (registry.Model{Explicit: f.Explicit}).WithExplicit(paths...).Explicit + return f +} + // Resolve is the one-call path: flags + ~/.captain.yaml → a fully resolved Model. -// A broken config surfaces as an error; callers that prefer to warn and carry on -// with zero defaults load their own and use ResolveWith. +// A broken config surfaces as an error; callers with a captured snapshot use +// ResolveWith. func (f ModelFlags) Resolve() (registry.Model, error) { saved, err := LoadDefaults() if err != nil { @@ -126,21 +131,18 @@ func (f ModelFlags) ResolveWith(saved captainconfig.AIDefaults) (registry.Model, if err != nil { return registry.Model{}, err } - if !f.NoCache && saved.NoCache { - m.NoCache = true - } - if m, err = ApplyDefaults(m, saved); err != nil { + applied, err := ApplyDefaults(DefaultOptions{Model: m, Saved: saved}) + if err != nil { return registry.Model{}, err } - return registry.ResolveModel(m) + return registry.ResolveModel(applied.Model) } // Overlay layers the flags over an already-structured base (a spec's model), // flags winning field by field, and resolves the result once. // -// This is the entry point for callers that have both a config/spec and flags — -// which is most of them. Doing it by hand is how a caller ends up merging an -// unexpanded name onto a populated backend and losing the mode. +// Model-only consumers use this helper. Full request pipelines preserve these +// authored flags as a layer and resolve their complete specification together. func (f ModelFlags) Overlay(base registry.Model) (registry.Model, error) { saved, err := LoadDefaults() if err != nil { @@ -155,11 +157,11 @@ func (f ModelFlags) OverlayWith(base registry.Model, saved captainconfig.AIDefau if err != nil { return registry.Model{}, err } - merged, err := ApplyDefaults(base.Merge(over), saved) + merged, err := ApplyDefaults(DefaultOptions{Model: base.Merge(over), Saved: saved}) if err != nil { return registry.Model{}, err } - return registry.ResolveModel(merged) + return registry.ResolveModel(merged.Model) } // Temp parses --temperature. A nil result means unset: an explicit 0 and "unset" @@ -167,7 +169,7 @@ func (f ModelFlags) OverlayWith(base registry.Model, saved captainconfig.AIDefau // from "use the model's default". func (f ModelFlags) Temp() (*float64, error) { v, err := parseFloat("temperature", f.Temperature) - if err != nil || v == 0 { + if err != nil || strings.TrimSpace(f.Temperature) == "" { return nil, err } if v < 0 || v > 2 { @@ -179,6 +181,9 @@ func (f ModelFlags) Temp() (*float64, error) { // FallbackModels turns repeatable/comma-separated --fallback values into models. func FallbackModels(flags []string) registry.ModelList { var out registry.ModelList + if flags != nil { + out = registry.ModelList{} + } for _, flag := range flags { for _, name := range strings.Split(flag, ",") { if name = strings.TrimSpace(name); name != "" { diff --git a/pkg/aiflags/unconfigured.go b/pkg/aiflags/unconfigured.go index 12a4d5cf..38804d56 100644 --- a/pkg/aiflags/unconfigured.go +++ b/pkg/aiflags/unconfigured.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/flanksource/captain/pkg/api/registry" - "github.com/flanksource/captain/pkg/captainconfig" ) // ErrUnconfigured marks the "nothing chose a model or a mode" failure so callers @@ -16,11 +15,8 @@ var ErrUnconfigured = errors.New("no model configured") // UnconfiguredError reports that a run reached the execution boundary without a // model or a mechanism, and names the commands that fix it. // -// It exists because the alternative is guessing. captain used to fill the gap -// from a compiled-in table — Provider.DefaultMode and the DefaultModelFor list — -// so an unconfigured `--ai-model haiku` silently became `agent:claude-haiku-4-5` -// and no configuration file said so. Those tables now seed `captain configure` -// only; a run that nothing configured stops here. +// The shared request resolver decides whether a missing mode is a migration +// warning or a strict error. A missing model remains an execution error. type UnconfiguredError struct { // Field is "model" or "mode": which half is missing. Field string @@ -71,51 +67,8 @@ func modeList(modes []registry.RuntimeMode) string { // IsUnconfigured reports whether err is the "nothing configured a model" failure. func IsUnconfigured(err error) bool { return errors.Is(err, ErrUnconfigured) } -// ResolveForRun is the execution boundary: it applies the saved ~/.captain.yaml -// defaults, refuses to proceed when the selection is still incomplete, and then -// resolves against the catalog. -// -// The refusal happens BEFORE registry.ResolveModel deliberately. The registry is -// the grammar — it parses "api:haiku" into a concrete triple and is used to -// render catalogs, validate prompts and replay recorded history, all of which -// legitimately resolve names they never intend to run. Leaving its provider -// fallback intact and gating here means only runs are strict. -// -// Every path that is about to execute goes through this rather than bare -// ResolveModel; that is what makes configuration the single source of a model. -func ResolveForRun(model registry.Model) (registry.Model, error) { - saved, err := LoadDefaults() - if err != nil { - return registry.Model{}, err - } - return ResolveForRunWith(model, saved) -} - -// ResolveForRunWith is ResolveForRun against an already-loaded config, for -// callers that read ~/.captain.yaml once and resolve many models. -func ResolveForRunWith(model registry.Model, saved captainconfig.AIDefaults) (registry.Model, error) { - applied, err := ApplyDefaults(model, saved) - if err != nil { - return registry.Model{}, err - } - if err := requireConfigured(applied); err != nil { - return registry.Model{}, err - } - return registry.ResolveModel(applied) +func (c UnconfiguredCandidate) Error() string { + return c.Path + ": " + (&UnconfiguredError{Field: "mode", Model: c.Model, Provider: c.Provider}).Error() } -func requireConfigured(model registry.Model) error { - if strings.TrimSpace(model.Name) == "" { - return &UnconfiguredError{Field: "model", Provider: model.Provider} - } - if model.Mode == "" { - provider := model.Provider - if provider == nil { - if p, _, ok := registry.ProviderForToken(model.Name); ok { - provider = p - } - } - return &UnconfiguredError{Field: "mode", Provider: provider, Model: model.Name} - } - return nil -} +func (c UnconfiguredCandidate) Unwrap() error { return ErrUnconfigured } diff --git a/pkg/aiflags/unconfigured_test.go b/pkg/aiflags/unconfigured_test.go index 4153c221..09093e19 100644 --- a/pkg/aiflags/unconfigured_test.go +++ b/pkg/aiflags/unconfigured_test.go @@ -8,11 +8,12 @@ import ( "github.com/flanksource/captain/pkg/captainconfig" ) -// The reported bug in its most reduced form: an unconfigured captain filled the -// mode from Provider.DefaultMode, so a bare model silently became an agent run. -// The registry still does that for parsing and display; a run must not. -func TestResolveForRunRefusesAModeNobodyConfigured(t *testing.T) { - _, err := ResolveForRunWith(registry.Model{Name: "haiku"}, captainconfig.AIDefaults{}) +func TestDefaultsReportAModeNobodyConfigured(t *testing.T) { + result, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "haiku"}}) + if err != nil || len(result.Unconfigured) != 1 { + t.Fatalf("want one unconfigured candidate, got %v: %v", result.Unconfigured, err) + } + err = result.Unconfigured[0] if !IsUnconfigured(err) { t.Fatalf("want an unconfigured error, got %v", err) @@ -24,8 +25,8 @@ func TestResolveForRunRefusesAModeNobodyConfigured(t *testing.T) { } } -func TestResolveForRunRefusesAModelNobodyConfigured(t *testing.T) { - _, err := ResolveForRunWith(registry.Model{}, captainconfig.AIDefaults{}) +func TestUnconfiguredModelNamesTheRemediation(t *testing.T) { + err := &UnconfiguredError{Field: "model"} if !IsUnconfigured(err) { t.Fatalf("want an unconfigured error, got %v", err) @@ -36,27 +37,29 @@ func TestResolveForRunRefusesAModelNobodyConfigured(t *testing.T) { } // An explicit compact selector is complete on its own and must not need config. -func TestResolveForRunAcceptsAnExplicitSelector(t *testing.T) { - resolved, err := ResolveForRunWith(registry.Model{Name: "api:haiku"}, captainconfig.AIDefaults{}) +func TestDefaultsAcceptAnExplicitSelector(t *testing.T) { + result, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "api:haiku"}}) + resolved := result.Model if err != nil { t.Fatalf("explicit selector must resolve without config: %v", err) } if resolved.Mode != registry.ModeAPI { t.Errorf("mode = %q, want api", resolved.Mode) } - if resolved.Name != "claude-haiku-4-5" { - t.Errorf("name = %q, want claude-haiku-4-5", resolved.Name) + if resolved.Name != "haiku" { + t.Errorf("name = %q, want unresolved alias haiku", resolved.Name) } } // The per-provider block is the primary source, and it supplies the mode a bare // name is missing. -func TestResolveForRunTakesTheProviderBlock(t *testing.T) { +func TestDefaultsTakeTheProviderBlock(t *testing.T) { saved := captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{ registry.Anthropic.Name: {Mode: "api"}, }} - resolved, err := ResolveForRunWith(registry.Model{Name: "haiku"}, saved) + result, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "haiku"}, Saved: saved}) + resolved := result.Model if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -70,7 +73,8 @@ func TestResolveForRunTakesTheProviderBlock(t *testing.T) { func TestGlobalDefaultModelSuppliesBothHalves(t *testing.T) { saved := captainconfig.AIDefaults{DefaultModel: "api:claude-haiku-4-5"} - resolved, err := ResolveForRunWith(registry.Model{}, saved) + result, err := ApplyDefaults(DefaultOptions{Saved: saved}) + resolved := result.Model if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -79,17 +83,16 @@ func TestGlobalDefaultModelSuppliesBothHalves(t *testing.T) { } } -// A selector naming another provider still needs a mechanism, and the global -// default's mode is a mechanism, so it travels even across families. -func TestGlobalDefaultModeAppliesToAnotherProvidersModel(t *testing.T) { +func TestGlobalDefaultModeDoesNotLeakToAnotherProvider(t *testing.T) { saved := captainconfig.AIDefaults{DefaultModel: "api:claude-haiku-4-5"} - resolved, err := ResolveForRunWith(registry.Model{Name: "gemini-3.5-flash"}, saved) + result, err := ApplyDefaults(DefaultOptions{Model: registry.Model{Name: "gemini-3.5-flash"}, Saved: saved}) + resolved := result.Model if err != nil { t.Fatalf("unexpected error: %v", err) } - if resolved.Mode != registry.ModeAPI { - t.Errorf("mode = %q, want api", resolved.Mode) + if resolved.Mode != "" || len(result.Unconfigured) != 1 { + t.Errorf("mode = %q, want one reported unconfigured mode: %v", resolved.Mode, result.Unconfigured) } if resolved.Name != "gemini-3.5-flash" { t.Errorf("the global default must not replace an explicitly named model, got %q", resolved.Name) diff --git a/pkg/api/registry/disabled.go b/pkg/api/registry/disabled.go index b0badc46..ec7ac7a9 100644 --- a/pkg/api/registry/disabled.go +++ b/pkg/api/registry/disabled.go @@ -37,7 +37,7 @@ type DisabledSet struct { func NewDisabledSet(modes, providers []string, runtimes []Runtime, models, efforts []string) DisabledSet { return DisabledSet{ modes: tokenSet(modes), - providers: tokenSet(providers), + providers: providerSet(providers), runtimes: runtimeSet(runtimes), models: tokenSet(models), efforts: tokenSet(efforts), @@ -50,8 +50,12 @@ func runtimeSet(values []Runtime) map[Runtime]struct{} { } out := make(map[Runtime]struct{}, len(values)) for _, v := range values { + provider := strings.ToLower(strings.TrimSpace(v.Provider)) + if known, ok := ProviderByName(provider); ok { + provider = known.Name + } normalized := Runtime{ - Provider: strings.ToLower(strings.TrimSpace(v.Provider)), + Provider: provider, Mode: RuntimeMode(strings.ToLower(strings.TrimSpace(string(v.Mode)))), } if normalized.Provider == "" || normalized.Mode == "" { @@ -65,6 +69,17 @@ func runtimeSet(values []Runtime) map[Runtime]struct{} { return out } +func providerSet(values []string) map[string]struct{} { + out := tokenSet(values) + for token := range out { + if provider, ok := ProviderByName(token); ok && token != provider.Name { + delete(out, token) + out[provider.Name] = struct{}{} + } + } + return out +} + func tokenSet(values []string) map[string]struct{} { if len(values) == 0 { return nil diff --git a/pkg/api/registry/disabled_ginkgo_test.go b/pkg/api/registry/disabled_ginkgo_test.go index a98a238f..f3dcb86b 100644 --- a/pkg/api/registry/disabled_ginkgo_test.go +++ b/pkg/api/registry/disabled_ginkgo_test.go @@ -38,6 +38,13 @@ var _ = Describe("DisabledSet", func() { Expect(set.Effort(EffortUltra)).To(BeTrue()) }) + It("canonicalizes provider aliases for provider and runtime lookups", func() { + set := NewDisabledSet(nil, []string{"gemini"}, []Runtime{{Provider: "codex", Mode: ModeAgent}}, nil, nil) + + Expect(set.Provider(Google)).To(BeTrue()) + Expect(set.Runtime(OpenAI, ModeAgent)).To(BeTrue()) + }) + It("drops blank tokens rather than disabling the empty string", func() { set := NewDisabledSet([]string{" ", ""}, nil, nil, nil, nil) diff --git a/pkg/api/registry/model.go b/pkg/api/registry/model.go index 754984c3..e5ea701d 100644 --- a/pkg/api/registry/model.go +++ b/pkg/api/registry/model.go @@ -17,6 +17,9 @@ const CodexAutoReviewModel = "codex-auto-review" // knobs. Maps onto the legacy ai.Config.Model + ai.Request.{Temperature, // ReasoningEffort}. type Model struct { + // Explicit records authored fields whose zero values must survive merging. + Explicit FieldPresence `json:"-" yaml:"-" pretty:"-"` + // Name is the catalog model slug, e.g. "claude-sonnet-4-6"; it drives provider // inference and pricing lookup. Name string `json:"model,omitempty" yaml:"model,omitempty" jsonschema:"required" pretty:"label=Model"` @@ -117,6 +120,7 @@ func (m Model) WithMode(mode RuntimeMode) (Model, error) { return Model{}, invalidRuntimeMode(mode) } + m = (Model{}).Merge(m) var err error if m, err = m.Expand(); err != nil { return Model{}, err @@ -259,6 +263,7 @@ func MergePolicy() merge.Policy { // with `--model codex` came out as an anthropic runtime running "codex". func (m Model) Merge(o Model) Model { merged := merge.Apply(m, o, MergePolicy()) + merged = mergeExplicitModel(merged, o) if name := strings.TrimSpace(o.Name); name != "" && !strings.EqualFold(name, strings.TrimSpace(m.Name)) { merged.Provider = o.Provider merged.Streaming, merged.MediaTypes = o.Streaming, o.MediaTypes @@ -332,13 +337,13 @@ func (m Model) candidates() []Model { for _, fb := range m.Fallbacks { fb.Fallbacks = nil fb.ID = "" - if fb.Temperature == nil { + if fb.Temperature == nil && !fb.Explicit.Has("/temperature") { fb.Temperature = m.Temperature } - if fb.Effort == "" && modelProvider(fb) == modelProvider(m) { + if fb.Effort == "" && !fb.Explicit.Has("/effort") && modelProvider(fb) == modelProvider(m) { fb.Effort = m.Effort } - if !fb.NoCache { + if !fb.NoCache && !fb.Explicit["/noCache"] { fb.NoCache = m.NoCache } out = append(out, fb) diff --git a/pkg/api/registry/model_compact.go b/pkg/api/registry/model_compact.go index 95bd472c..f84ac193 100644 --- a/pkg/api/registry/model_compact.go +++ b/pkg/api/registry/model_compact.go @@ -98,6 +98,7 @@ func (m Model) Expand() (Model, error) { parsed.Mode = m.Mode } parsed.ID = m.ID + parsed.Explicit = m.Explicit parsed.Temperature = m.Temperature if !parsed.NoCache { parsed.NoCache = m.NoCache @@ -108,10 +109,7 @@ func (m Model) Expand() (Model, error) { // ModelList is the type of Model.Fallbacks. Each entry may be written as a // compact string ("agent:opus:high") or the object form. It is a named slice -// rather than a method on Model itself because Model is inline-embedded in Spec: -// a value-level Unmarshaler on Model would hijack the whole Spec object and break -// field promotion. A Spec-level `model:` scalar lands in Model.Name (a string) — -// Model.Expand parses that. +// so compact strings retain their authored spelling until final resolution. type ModelList []Model func (l *ModelList) UnmarshalJSON(data []byte) error { @@ -132,11 +130,10 @@ func (l *ModelList) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(e, &s); err != nil { return err } - m, err := parseCompactElement(s) - if err != nil { + if _, err := parseCompactElement(s); err != nil { return err } - out = append(out, m) + out = append(out, Model{Name: s}) continue } var m Model @@ -156,11 +153,10 @@ func (l *ModelList) UnmarshalYAML(value *yaml.Node) error { out := make(ModelList, 0, len(value.Content)) for _, node := range value.Content { if node.Kind == yaml.ScalarNode && node.Tag != "!!null" { - m, err := parseCompactElement(node.Value) - if err != nil { + if _, err := parseCompactElement(node.Value); err != nil { return err } - out = append(out, m) + out = append(out, Model{Name: node.Value}) continue } var m Model diff --git a/pkg/api/registry/model_compact_test.go b/pkg/api/registry/model_compact_test.go index e4b1164e..ff2d9831 100644 --- a/pkg/api/registry/model_compact_test.go +++ b/pkg/api/registry/model_compact_test.go @@ -111,7 +111,7 @@ func TestModelList_Unmarshal(t *testing.T) { if len(l) != 2 { t.Fatalf("len = %d", len(l)) } - if l[0].Name != "opus" || l[0].Mode != ModeAgent || l[0].Effort != EffortHigh { + if l[0].Name != "agent:opus:high" || l[0].Mode != "" || l[0].Effort != "" { t.Errorf("l0 = %+v", l[0]) } if l[1].Name != "sonnet" || l[1].Effort != EffortMedium { @@ -123,7 +123,7 @@ func TestModelList_Unmarshal(t *testing.T) { if err := yaml.Unmarshal([]byte("- api:opus\n- model: sonnet\n effort: high\n"), &l); err != nil { t.Fatal(err) } - if len(l) != 2 || l[0].Name != "opus" || l[0].Mode != ModeAPI || l[1].Name != "sonnet" { + if len(l) != 2 || l[0].Name != "api:opus" || l[0].Mode != "" || l[1].Name != "sonnet" { t.Errorf("got %+v", l) } }) diff --git a/pkg/api/registry/model_presence.go b/pkg/api/registry/model_presence.go new file mode 100644 index 00000000..77047375 --- /dev/null +++ b/pkg/api/registry/model_presence.go @@ -0,0 +1,127 @@ +package registry + +import ( + "encoding/json" + "reflect" + "strings" + + "gopkg.in/yaml.v3" +) + +// FieldPresence records explicitly authored JSON-pointer paths. It is runtime +// metadata; serializers preserve the corresponding values, not this map. +type FieldPresence map[string]bool + +func (f FieldPresence) Has(path string) bool { return f[path] } + +func (f FieldPresence) Clone() FieldPresence { + if f == nil { + return nil + } + out := make(FieldPresence, len(f)) + for path, present := range f { + out[path] = present + } + return out +} + +// WithExplicit marks authored zero values in programmatically constructed models. +func (m Model) WithExplicit(paths ...string) Model { + m.Explicit = m.Explicit.Clone() + if m.Explicit == nil { + m.Explicit = FieldPresence{} + } + for _, path := range paths { + m.Explicit[path] = true + } + return m +} + +// Fields includes both explicit zero values and nonzero Go-authored fields. +func (m Model) Fields() FieldPresence { + fields := FieldPresence{} + for key := range m.fieldValues() { + fields["/"+key] = true + } + return fields +} + +func (m Model) fieldValues() map[string]any { + out := map[string]any{} + value := reflect.ValueOf(m) + for i := range value.NumField() { + name := strings.Split(value.Type().Field(i).Tag.Get("json"), ",")[0] + if name == "-" || name == "" { + continue + } + field := value.Field(i) + if !field.IsZero() || m.Explicit["/"+name] { + out[name] = field.Interface() + } + } + return out +} + +func (m Model) MarshalJSON() ([]byte, error) { return json.Marshal(m.fieldValues()) } + +func (m Model) MarshalYAML() (any, error) { return m.fieldValues(), nil } + +type modelWire Model + +func (Model) DecodeFields() any { return modelWire{} } + +func (ModelList) DecodeFields() any { return []modelWire{} } + +func (m *Model) UnmarshalJSON(data []byte) error { + var value modelWire + if err := json.Unmarshal(data, &value); err != nil { + return err + } + var fields map[string]any + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + *m = Model(value) + m.captureFields(fields) + return nil +} + +func (m *Model) UnmarshalYAML(node *yaml.Node) error { + var value modelWire + if err := node.Decode(&value); err != nil { + return err + } + var fields map[string]any + if err := node.Decode(&fields); err != nil { + return err + } + *m = Model(value) + m.captureFields(fields) + return nil +} + +func (m *Model) captureFields(fields map[string]any) { + m.Explicit = FieldPresence{} + typeOf := reflect.TypeOf(*m) + for i := range typeOf.NumField() { + name := strings.Split(typeOf.Field(i).Tag.Get("json"), ",")[0] + if _, present := fields[name]; present && name != "-" { + m.Explicit["/"+name] = true + } + } +} + +func mergeExplicitModel(merged, override Model) Model { + out := reflect.ValueOf(&merged).Elem() + value := reflect.ValueOf(override) + for i := range value.NumField() { + name := strings.Split(value.Type().Field(i).Tag.Get("json"), ",")[0] + if override.Explicit["/"+name] && value.Field(i).IsZero() { + out.Field(i).Set(value.Field(i)) + } + } + if override.Fallbacks != nil && len(override.Fallbacks) == 0 { + merged.Fallbacks = ModelList{} + } + return merged +} diff --git a/pkg/api/registry/model_presence_ginkgo_test.go b/pkg/api/registry/model_presence_ginkgo_test.go new file mode 100644 index 00000000..18b9d220 --- /dev/null +++ b/pkg/api/registry/model_presence_ginkgo_test.go @@ -0,0 +1,89 @@ +package registry + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +var _ = Describe("authored model field presence", func() { + DescribeTable("preserves explicit zero values across decoding and encoding", func(input string, decode func([]byte, any) error, encode func(any) ([]byte, error)) { + var model Model + Expect(decode([]byte(input), &model)).To(Succeed()) + Expect(model.Explicit).To(HaveKeyWithValue("/noCache", true)) + Expect(model.Explicit).To(HaveKeyWithValue("/fallbacks", true)) + Expect(model.Temperature).NotTo(BeNil()) + Expect(*model.Temperature).To(BeZero()) + encoded, err := encode(model) + Expect(err).NotTo(HaveOccurred()) + var roundtrip Model + Expect(decode(encoded, &roundtrip)).To(Succeed()) + Expect(roundtrip).To(Equal(model)) + }, + Entry("JSON", `{"noCache":false,"temperature":0,"fallbacks":[]}`, json.Unmarshal, json.Marshal), + Entry("YAML", "noCache: false\ntemperature: 0\nfallbacks: []\n", yaml.Unmarshal, yaml.Marshal), + ) + + It("lets explicit false and an empty list replace inherited values without mutating inputs", func() { + base := Model{NoCache: true, Fallbacks: ModelList{{Name: "agent:sonnet"}}} + override := Model{Explicit: FieldPresence{"/noCache": true, "/fallbacks": true}, Fallbacks: ModelList{}} + merged := base.Merge(override) + Expect(merged.NoCache).To(BeFalse()) + Expect(merged.Fallbacks).To(BeEmpty()) + Expect(base.NoCache).To(BeTrue()) + Expect(base.Fallbacks).To(HaveLen(1)) + merged.Explicit["/mode"] = true + Expect(override.Explicit).NotTo(HaveKey("/mode")) + }) + + It("keeps raw compact fallback selectors and nested explicit false on the wire", func() { + var models ModelList + Expect(json.Unmarshal([]byte(`["agent:sonnet:high",{"model":"api:haiku","noCache":false}]`), &models)).To(Succeed()) + Expect(models[0].Name).To(Equal("agent:sonnet:high")) + Expect(models[0].Mode).To(BeEmpty()) + Expect(models[1].Explicit).To(HaveKeyWithValue("/noCache", true)) + encoded, err := json.Marshal(models) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).To(ContainSubstring(`"noCache":false`)) + }) + + It("preserves an explicitly cacheable fallback when generating execution candidates", func() { + model := Model{Name: "agent:sonnet", NoCache: true, Fallbacks: ModelList{{Name: "api:haiku", Explicit: FieldPresence{"/noCache": true}}}} + Expect(model.Candidates()[1].NoCache).To(BeFalse()) + }) + + It("preserves explicit empty fallback knobs when generating execution candidates", func() { + temperature := 0.6 + model := Model{Name: "sonnet", Mode: ModeAgent, Effort: EffortHigh, Temperature: &temperature, + Fallbacks: ModelList{(Model{Name: "haiku", Mode: ModeAgent}).WithExplicit("/effort", "/temperature")}, + } + Expect(model.Candidates()[1].Effort).To(BeEmpty()) + Expect(model.Candidates()[1].Temperature).To(BeNil()) + }) + + It("preserves zero values inherited through YAML merge keys", func() { + var model Model + Expect(yaml.Unmarshal([]byte("<<: &defaults {noCache: false, temperature: 0}\nmodel: sonnet\n"), &model)).To(Succeed()) + Expect(model.Explicit).To(HaveKeyWithValue("/noCache", true)) + Expect(model.Explicit).To(HaveKeyWithValue("/temperature", true)) + Expect(model.Explicit).NotTo(HaveKey("/<<")) + }) + + It("tracks only model fields when decoding unknown or outer object fields", func() { + var model Model + Expect(json.Unmarshal([]byte(`{"model":"sonnet","backend":"cli","budget":{"cost":0}}`), &model)).To(Succeed()) + Expect(model.Explicit).To(Equal(FieldPresence{"/model": true})) + }) + + It("normalizes authored effort only at final resolution for primary and fallback", func() { + model := Model{Name: "agent:sonnet:ultra", Fallbacks: ModelList{{Name: "api:sonnet:ultra"}}} + resolved, err := ResolveModel(model) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Effort).To(Equal(EffortMax)) + Expect(resolved.Fallbacks[0].Effort).To(Equal(EffortMax)) + Expect(model.Name).To(Equal("agent:sonnet:ultra")) + Expect(model.Fallbacks[0].Name).To(Equal("api:sonnet:ultra")) + }) +}) diff --git a/pkg/api/registry/model_test.go b/pkg/api/registry/model_test.go index ced4108e..3f1ebff0 100644 --- a/pkg/api/registry/model_test.go +++ b/pkg/api/registry/model_test.go @@ -139,10 +139,11 @@ func TestModel_Validate_Fallbacks(t *testing.T) { // YAML round-trips with their per-model knobs intact. func TestModel_FallbacksRoundTrip(t *testing.T) { in := Model{ - Name: "claude-sonnet-5", + Explicit: FieldPresence{"/model": true, "/fallbacks": true}, + Name: "claude-sonnet-5", Fallbacks: []Model{ - {Name: "gpt-4o", Effort: EffortHigh}, - {Name: "gemini-2.0-flash", Temperature: floatPtr(0.2)}, + {Name: "gpt-4o", Effort: EffortHigh, Explicit: FieldPresence{"/model": true, "/effort": true}}, + {Name: "gemini-2.0-flash", Temperature: floatPtr(0.2), Explicit: FieldPresence{"/model": true, "/temperature": true}}, }, } for _, tc := range []struct { diff --git a/pkg/api/registry/parse.go b/pkg/api/registry/parse.go index 0cfd2e42..96ab595a 100644 --- a/pkg/api/registry/parse.go +++ b/pkg/api/registry/parse.go @@ -345,6 +345,11 @@ func mergeResolved(original, resolved Model) (Model, error) { if resolved.Effort != EffortNone { out.Effort = resolved.Effort } + effort, err := ResolveEffort(out.Provider, out.Mode, out.Name, out.Effort) + if err != nil { + return Model{}, err + } + out.Effort = effort // Capabilities are derived, never carried over from the request: whatever the // caller wrote for them is replaced by what the resolved adapter can do. return out.WithCapabilities() diff --git a/pkg/api/registry/providers.go b/pkg/api/registry/providers.go index 1450791a..a29b5e61 100644 --- a/pkg/api/registry/providers.go +++ b/pkg/api/registry/providers.go @@ -107,12 +107,12 @@ func Providers() []*Provider { return []*Provider{Anthropic, OpenAI, Google, DeepSeek} } -// ProviderByName resolves a provider by Name, CatalogPrefix, or PricingPrefix, -// so both "google" and "googleai" find Google. +// ProviderByName resolves a provider by Name, AgentName, CatalogPrefix, or +// PricingPrefix, so "google", "gemini", and "googleai" all find Google. func ProviderByName(name string) (*Provider, bool) { name = strings.ToLower(strings.TrimSpace(name)) for _, p := range Providers() { - if name == p.Name || name == p.CatalogPrefix || name == p.PricingPrefix { + if name == p.Name || name == p.AgentName || name == p.CatalogPrefix || name == p.PricingPrefix { return p, true } } From 44cbaf988db2f4fb2fd9b620005e03068b761fdc Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 12:13:09 +0300 Subject: [PATCH 09/22] feat(config): Preserve explicit AI defaults and validate provider configuration Preserve explicitly authored zero-valued AI settings during YAML and JSON round trips, and validate saved providers, selectors, modes, and generation limits before use. Update provider access to support aliases and reject ambiguous configurations. BREAKING CHANGE: Replace AIDefaults.Provider(string) with Provider(*registry.Provider) (ProviderDefaults, string, error). --- pkg/captainconfig/config.go | 10 +- pkg/captainconfig/config_test.go | 4 +- pkg/captainconfig/defaults_presence.go | 91 +++++++++++ .../defaults_presence_ginkgo_test.go | 48 ++++++ pkg/captainconfig/defaults_validation.go | 148 ++++++++++++++++++ 5 files changed, 293 insertions(+), 8 deletions(-) create mode 100644 pkg/captainconfig/defaults_presence.go create mode 100644 pkg/captainconfig/defaults_presence_ginkgo_test.go create mode 100644 pkg/captainconfig/defaults_validation.go diff --git a/pkg/captainconfig/config.go b/pkg/captainconfig/config.go index f122f830..013347e6 100644 --- a/pkg/captainconfig/config.go +++ b/pkg/captainconfig/config.go @@ -163,6 +163,8 @@ func (a AttachmentDefaults) WithDefaults() AttachmentDefaults { } type AIDefaults struct { + Explicit registry.FieldPresence `yaml:"-" json:"-"` + DefaultProvider string `yaml:"defaultProvider,omitempty"` // DefaultModel is the global fallback when no provider block, prompt, spec or // flag names a model. It is a COMPACT SELECTOR ("agent:claude-sonnet-5"), not @@ -170,7 +172,7 @@ type AIDefaults struct { // the caller is missing when it falls through to here. // // It is the single value that makes a one-line ~/.captain.yaml sufficient, and - // the last stop before ResolveForRun refuses to guess. + // supplies the model before generating requests enforce their required-model gate. DefaultModel string `yaml:"defaultModel,omitempty"` Providers map[string]ProviderDefaults `yaml:"providers,omitempty"` Disabled DisabledSelections `yaml:"disabled,omitempty"` @@ -255,12 +257,6 @@ func (a AIDefaults) ActiveProvider() string { return registry.Anthropic.Name } -// Provider returns one provider's saved defaults. A provider with nothing saved -// returns the zero value, which the resolution path fills from the registry. -func (a AIDefaults) Provider(provider string) ProviderDefaults { - return a.Providers[strings.TrimSpace(provider)] -} - type PromptDefaults struct { Dirs []string `yaml:"dirs,omitempty"` SchemaRepair SchemaRepairDefaults `yaml:"schemaRepair,omitempty"` diff --git a/pkg/captainconfig/config_test.go b/pkg/captainconfig/config_test.go index 32caa703..7668ca47 100644 --- a/pkg/captainconfig/config_test.go +++ b/pkg/captainconfig/config_test.go @@ -64,6 +64,7 @@ func TestSaveLoad_RoundTrip(t *testing.T) { }, Chat: ChatDefaults{RuntimeProfile: "Review"}, } + want.AI = want.AI.WithExplicit("/defaultProvider", "/providers", "/budgetUSD", "/maxTokens", "/temperature", "/timeout", "/noCache", "/noMCP", "/noMemory") if err := Save(want); err != nil { t.Fatalf("Save() err = %v", err) } @@ -297,7 +298,8 @@ func TestCurrentKeysLoadCleanly(t *testing.T) { if err != nil || !exists { t.Fatalf("Load() = %v, %v", exists, err) } - if got := cfg.AI.Provider("anthropic"); got.Mode != "agent" || got.Model != "claude-opus-5" { + got, _, err := cfg.AI.Provider(registry.Anthropic) + if err != nil || got.Mode != "agent" || got.Model != "claude-opus-5" { t.Fatalf("anthropic defaults = %+v", got) } if got := cfg.AI.Disabled.Runtimes; len(got) != 1 || got[0].Provider != "anthropic" || got[0].Mode != "cmux" { diff --git a/pkg/captainconfig/defaults_presence.go b/pkg/captainconfig/defaults_presence.go new file mode 100644 index 00000000..dcb7a264 --- /dev/null +++ b/pkg/captainconfig/defaults_presence.go @@ -0,0 +1,91 @@ +package captainconfig + +import ( + "encoding/json" + "reflect" + "strings" + + "github.com/flanksource/captain/pkg/api/registry" + "gopkg.in/yaml.v3" +) + +func (a AIDefaults) WithExplicit(paths ...string) AIDefaults { + a.Explicit = a.Explicit.Clone() + if a.Explicit == nil { + a.Explicit = registry.FieldPresence{} + } + for _, path := range paths { + a.Explicit[path] = true + } + return a +} + +// Fields distinguishes an omitted global setting from an authored zero value. +func (a AIDefaults) Fields() registry.FieldPresence { + fields := registry.FieldPresence{} + for key := range a.fieldValues() { + fields["/"+key] = true + } + return fields +} + +func (a AIDefaults) fieldValues() map[string]any { + out := map[string]any{} + value := reflect.ValueOf(a) + for i := range value.NumField() { + name := strings.Split(value.Type().Field(i).Tag.Get("yaml"), ",")[0] + if name == "-" || name == "" { + continue + } + field := value.Field(i) + if !field.IsZero() || a.Explicit["/"+name] { + out[name] = field.Interface() + } + } + return out +} + +func (a AIDefaults) MarshalJSON() ([]byte, error) { return json.Marshal(a.fieldValues()) } + +func (a AIDefaults) MarshalYAML() (any, error) { return a.fieldValues(), nil } + +type defaultsWire AIDefaults + +func (a *AIDefaults) UnmarshalJSON(data []byte) error { + var value defaultsWire + if err := json.Unmarshal(data, &value); err != nil { + return err + } + var fields map[string]any + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + *a = AIDefaults(value) + a.captureFields(fields) + return nil +} + +func (a *AIDefaults) UnmarshalYAML(node *yaml.Node) error { + var value defaultsWire + if err := node.Decode(&value); err != nil { + return err + } + var fields map[string]any + if err := node.Decode(&fields); err != nil { + return err + } + *a = AIDefaults(value) + a.captureFields(fields) + return nil +} + +func (a *AIDefaults) captureFields(fields map[string]any) { + a.Explicit = registry.FieldPresence{} + typeOf := reflect.TypeOf(*a) + for i := range typeOf.NumField() { + name := strings.Split(typeOf.Field(i).Tag.Get("yaml"), ",")[0] + if _, present := fields[name]; present && name != "-" { + a.Explicit["/"+name] = true + } + } +} diff --git a/pkg/captainconfig/defaults_presence_ginkgo_test.go b/pkg/captainconfig/defaults_presence_ginkgo_test.go new file mode 100644 index 00000000..f2fc18a9 --- /dev/null +++ b/pkg/captainconfig/defaults_presence_ginkgo_test.go @@ -0,0 +1,48 @@ +package captainconfig + +import ( + "encoding/json" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +func TestSavedDefaultsPresence(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Saved defaults presence") +} + +var _ = Describe("saved defaults field presence", func() { + DescribeTable("roundtrips explicit global zero values", func(input string, decode func([]byte, any) error, encode func(any) ([]byte, error)) { + var saved AIDefaults + Expect(decode([]byte(input), &saved)).To(Succeed()) + for _, path := range []string{"/temperature", "/budgetUSD", "/maxTokens", "/noCache", "/noMCP"} { + Expect(saved.Explicit).To(HaveKeyWithValue(path, true)) + } + encoded, err := encode(saved) + Expect(err).NotTo(HaveOccurred()) + var roundtrip AIDefaults + Expect(decode(encoded, &roundtrip)).To(Succeed()) + Expect(roundtrip).To(Equal(saved)) + }, + Entry("JSON", `{"temperature":0,"budgetUSD":0,"maxTokens":0,"noCache":false,"noMCP":false}`, json.Unmarshal, json.Marshal), + Entry("YAML", "temperature: 0\nbudgetUSD: 0\nmaxTokens: 0\nnoCache: false\nnoMCP: false\n", yaml.Unmarshal, yaml.Marshal), + ) + + It("leaves absent generation settings unset", func() { + var saved AIDefaults + Expect(yaml.Unmarshal([]byte("defaultModel: agent:sonnet:high\n"), &saved)).To(Succeed()) + Expect(saved.Explicit).NotTo(HaveKey("/temperature")) + Expect(saved.Explicit).NotTo(HaveKey("/noCache")) + }) + + It("tracks zero values from YAML merged defaults", func() { + var saved AIDefaults + Expect(yaml.Unmarshal([]byte("<<: &defaults {noCache: false, temperature: 0}\ndefaultModel: agent:sonnet\n"), &saved)).To(Succeed()) + Expect(saved.Explicit).To(HaveKeyWithValue("/noCache", true)) + Expect(saved.Explicit).To(HaveKeyWithValue("/temperature", true)) + Expect(saved.Explicit).NotTo(HaveKey("/<<")) + }) +}) diff --git a/pkg/captainconfig/defaults_validation.go b/pkg/captainconfig/defaults_validation.go new file mode 100644 index 00000000..482ca0e2 --- /dev/null +++ b/pkg/captainconfig/defaults_validation.go @@ -0,0 +1,148 @@ +package captainconfig + +import ( + "fmt" + "math" + "sort" + "strings" + "time" + + "github.com/flanksource/captain/pkg/api/registry" +) + +// Validate checks saved declarations before a request can hide invalid defaults. +func (a AIDefaults) Validate() error { + if math.IsNaN(a.Temperature) || math.IsInf(a.Temperature, 0) || a.Temperature < 0 || a.Temperature > 2 { + return fmt.Errorf("ai.temperature must be between 0 and 2, got %v", a.Temperature) + } + if math.IsNaN(a.BudgetUSD) || math.IsInf(a.BudgetUSD, 0) || a.BudgetUSD < 0 { + return fmt.Errorf("ai.budgetUSD must be nonnegative, got %v", a.BudgetUSD) + } + if a.MaxTokens < 0 { + return fmt.Errorf("ai.maxTokens must be nonnegative, got %d", a.MaxTokens) + } + if a.Timeout != "" { + if timeout, err := time.ParseDuration(a.Timeout); err != nil || timeout <= 0 { + return fmt.Errorf("ai.timeout must be a positive duration, got %q", a.Timeout) + } + } + if a.DefaultProvider != "" { + if _, ok := registry.ProviderByName(strings.TrimSpace(a.DefaultProvider)); !ok { + return fmt.Errorf("ai.defaultProvider %q is unknown", a.DefaultProvider) + } + } + if err := validateSavedSelector(a.DefaultModel, "ai.defaultModel", nil); err != nil { + return err + } + providers := make([]string, 0, len(a.Providers)) + for name := range a.Providers { + providers = append(providers, name) + } + sort.Strings(providers) + configured := map[string]string{} + for _, name := range providers { + provider, ok := registry.ProviderByName(name) + if !ok { + return fmt.Errorf("ai.providers.%s is unknown", name) + } + if previous := configured[provider.Name]; previous != "" { + return fmt.Errorf("ai.providers.%s and ai.providers.%s configure the same provider %s", previous, name, provider.Name) + } + configured[provider.Name] = name + if err := validateProviderDefaults(name, a.Providers[name]); err != nil { + return err + } + } + return nil +} + +// Provider returns one provider's saved defaults and the exact key that +// authored them. Aliases are accepted, while duplicate keys for one provider +// fail because neither value has a well-defined precedence. +func (a AIDefaults) Provider(provider *registry.Provider) (ProviderDefaults, string, error) { + if provider == nil { + return ProviderDefaults{}, "", fmt.Errorf("provider is required") + } + keys := make([]string, 0, len(a.Providers)) + for key := range a.Providers { + keys = append(keys, key) + } + sort.Strings(keys) + var defaults ProviderDefaults + var source string + for _, key := range keys { + configured, ok := registry.ProviderByName(key) + if !ok { + return ProviderDefaults{}, "", fmt.Errorf("ai.providers.%s is unknown", key) + } + if configured != provider { + continue + } + if source != "" { + return ProviderDefaults{}, "", fmt.Errorf("ai.providers.%s and ai.providers.%s configure the same provider %s", source, key, provider.Name) + } + defaults, source = a.Providers[key], key + } + return defaults, source, nil +} + +// SetProvider updates a provider through the exact key already authored in the +// saved file. A provider without an existing entry is written by canonical name. +func (a *AIDefaults) SetProvider(provider *registry.Provider, defaults ProviderDefaults) error { + if a == nil { + return fmt.Errorf("AI defaults are required") + } + _, key, err := a.Provider(provider) + if err != nil { + return err + } + if key == "" { + key = provider.Name + } + if a.Providers == nil { + a.Providers = map[string]ProviderDefaults{} + } + a.Providers[key] = defaults + return nil +} + +func validateProviderDefaults(name string, defaults ProviderDefaults) error { + provider, ok := registry.ProviderByName(name) + if !ok { + return fmt.Errorf("ai.providers.%s is unknown", name) + } + if defaults.Mode != "" { + if _, err := provider.RequireMode(registry.RuntimeMode(strings.TrimSpace(defaults.Mode))); err != nil { + return fmt.Errorf("ai.providers.%s.mode: %w", name, err) + } + } + if err := registry.Effort(strings.TrimSpace(defaults.ReasoningEffort)).Validate(); err != nil { + return fmt.Errorf("ai.providers.%s.reasoningEffort: %w", name, err) + } + return validateSavedSelector(defaults.Model, "ai.providers."+name+".model", provider) +} + +func validateSavedSelector(value, key string, expected *registry.Provider) error { + if strings.TrimSpace(value) == "" { + return nil + } + model, err := (registry.Model{Name: value}).Expand() + if err != nil { + return fmt.Errorf("%s: %w", key, err) + } + for i, candidate := range append([]registry.Model{model}, model.Fallbacks...) { + provider, err := registry.ProviderFor(candidate.Name) + if err != nil { + return fmt.Errorf("%s: %w", key, err) + } + if i == 0 && expected != nil && provider != expected { + return fmt.Errorf("%s %q must select a model from %s", key, value, expected.Name) + } + if candidate.Mode != "" { + if _, err := provider.RequireMode(candidate.Mode); err != nil { + return fmt.Errorf("%s: %w", key, err) + } + } + } + return nil +} From 59cf6397e8892bb80998414d68cac96dbfaed192 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 12:13:58 +0300 Subject: [PATCH 10/22] feat(api): support authored runtime composition and prompt declaration preservation Add explicit field presence, saved-default composition, normalization, and provenance tracking across runtime specs. Preserve authored prompt declarations and structured-output schemas through rendering and transport, while validating nested declaration fields. BREAKING CHANGE: update Render and ResolveSpecLayers callers to use the new options-based APIs. --- pkg/ai/prompt/README.md | 28 ++ pkg/ai/prompt/decode_fields.go | 36 ++ pkg/ai/prompt/decode_fields_ginkgo_test.go | 26 ++ pkg/ai/prompt/document.go | 8 +- pkg/ai/prompt/document_test.go | 2 +- pkg/ai/prompt/frontmatter_test.go | 6 +- pkg/ai/prompt/prompt.go | 43 ++- pkg/ai/prompt/prompt_test.go | 24 +- pkg/ai/prompt/render_declared_ginkgo_test.go | 77 +++++ pkg/api/runtime_preset_codec.go | 80 +++++ pkg/api/runtime_profiles.go | 5 +- pkg/api/sandbox_ref.go | 18 +- pkg/api/sandbox_ref_ginkgo_test.go | 15 +- pkg/api/spec.go | 37 ++- pkg/api/spec_codec.go | 308 ++++++++++++++++++ pkg/api/spec_composition.go | 78 ++++- pkg/api/spec_defaults.go | 155 +++++++++ pkg/api/spec_defaults_ginkgo_test.go | 167 ++++++++++ pkg/api/spec_layer_validation_ginkgo_test.go | 4 +- pkg/api/spec_layers.go | 29 +- pkg/api/spec_layers_ginkgo_test.go | 14 +- pkg/api/spec_merge.go | 58 +++- pkg/api/spec_merge_differential_test.go | 4 + pkg/api/spec_normalization.go | 53 +++ pkg/api/spec_normalization_ginkgo_test.go | 110 +++++++ pkg/api/spec_presence.go | 184 +++++++++++ pkg/api/spec_presence_ginkgo_test.go | 113 +++++++ pkg/api/spec_provenance.go | 160 +++++++++ pkg/api/spec_runtime_ginkgo_test.go | 24 +- pkg/api/spec_test.go | 6 +- pkg/api/spec_validation.go | 2 +- pkg/cli/ai_runtime_flags.go | 109 +++++++ pkg/cli/ai_runtime_helpers_test.go | 45 +++ pkg/cli/ai_runtime_normalize.go | 111 +++++++ pkg/cli/ai_runtime_resolve_ginkgo_test.go | 194 +++++++++++ pkg/cli/runtime_preset_entity.go | 10 +- pkg/cli/runtime_profile_catalog.go | 7 +- .../runtime_profile_catalog_ginkgo_test.go | 6 +- pkg/cli/runtime_profile_entity.go | 12 +- pkg/runtimeprofiles/README.md | 20 +- pkg/runtimeprofiles/resolve.go | 2 +- pkg/runtimeprofiles/resolver.go | 6 +- 42 files changed, 2265 insertions(+), 131 deletions(-) create mode 100644 pkg/ai/prompt/README.md create mode 100644 pkg/ai/prompt/decode_fields.go create mode 100644 pkg/ai/prompt/decode_fields_ginkgo_test.go create mode 100644 pkg/ai/prompt/render_declared_ginkgo_test.go create mode 100644 pkg/api/runtime_preset_codec.go create mode 100644 pkg/api/spec_codec.go create mode 100644 pkg/api/spec_defaults.go create mode 100644 pkg/api/spec_defaults_ginkgo_test.go create mode 100644 pkg/api/spec_normalization.go create mode 100644 pkg/api/spec_normalization_ginkgo_test.go create mode 100644 pkg/api/spec_presence.go create mode 100644 pkg/api/spec_presence_ginkgo_test.go create mode 100644 pkg/api/spec_provenance.go create mode 100644 pkg/cli/ai_runtime_flags.go create mode 100644 pkg/cli/ai_runtime_helpers_test.go create mode 100644 pkg/cli/ai_runtime_normalize.go create mode 100644 pkg/cli/ai_runtime_resolve_ginkgo_test.go diff --git a/pkg/ai/prompt/README.md b/pkg/ai/prompt/README.md new file mode 100644 index 00000000..bbc86b1e --- /dev/null +++ b/pkg/ai/prompt/README.md @@ -0,0 +1,28 @@ +# Rendering prompt specifications + +`Template.Render` takes one `RenderOptions` value. `Data` supplies template variables. `Output` supplies a Go structured-output target; when omitted, the template's output schema is retained as `Spec.Prompt.SchemaJSON`. + +Set `Declared: true` when the rendered prompt will participate in configuration composition. This preserves the authored model selector and runtime axes while rendering the body, source, native frontmatter, dotprompt configuration, and output schema. It does not choose a provider or add model defaults. Resolve the resulting specification with the other configuration layers before execution. + +```go +template := prompt.Load("---\nmodel: agent:sonnet\nconfig:\n temperature: 0\n---\nReview {{target}}.\n") +spec, config, err := template.Render(prompt.RenderOptions{ + Data: map[string]any{"target": "parser.go"}, + Declared: true, +}) +``` + +`spec` and `config` carry the same projected model and budget values. Dotprompt `config.temperature`, `config.reasoning`, and `config.maxOutputTokens` override their native frontmatter equivalents. Explicit zero values keep their presence metadata for later composition. Omit `Declared` when the caller wants the existing immediate model resolution behavior. + +`Library.Render` accepts a name and the same options: `library.Render("review.prompt", options)`. + +Named model calls now use `ai.PromptRequest{Name: "review", Spec: resolvedSpec}`. `Agent.ExecutePrompt` forwards that complete specification directly to the provider. Put user/system text, source, JSON Schema, schema strictness, and native Go schema targets under `Spec.Prompt`. Other runtime groups, session state, and explicit-value metadata stay on `Spec`. Batch responses remain keyed by `Name`; cost accounting and terminal outcomes are unchanged. + +Migration: replace `template.Render(data, output)` with `template.Render(prompt.RenderOptions{Data: data, Output: output})`, and replace flat named-request prompt/schema fields with the corresponding `Spec.Prompt` fields. Callers composing layers should also set `Declared: true` and must not overwrite the rendered model from a second frontmatter parse. + +Run the focused rendering and transport examples/tests from the Captain checkout: + +```sh +go test ./pkg/ai/prompt -run '^TestPromptDocuments$' -ginkgo.focus='Declared prompt rendering' -count=1 +go test ./pkg/ai -run '^TestAttachmentCapabilities$' -ginkgo.focus='Named prompt Spec transport' -count=1 +``` diff --git a/pkg/ai/prompt/decode_fields.go b/pkg/ai/prompt/decode_fields.go new file mode 100644 index 00000000..f1af38f8 --- /dev/null +++ b/pkg/ai/prompt/decode_fields.go @@ -0,0 +1,36 @@ +package prompt + +import ( + "bytes" + "fmt" + "reflect" + + "github.com/flanksource/captain/pkg/api" + "gopkg.in/yaml.v3" +) + +func validateDeclarationFields(data []byte, fields any) error { + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(reflect.New(reflect.TypeOf(fields)).Interface()); err != nil { + return err + } + var raw map[string]any + if err := yaml.Unmarshal(data, &raw); err != nil { + return err + } + fallbacks, _ := raw["fallbacks"].([]any) + for i, fallback := range fallbacks { + if _, object := fallback.(map[string]any); !object { + continue + } + encoded, err := yaml.Marshal(fallback) + if err != nil { + return err + } + if err := validateDeclarationFields(encoded, (api.Model{}).DecodeFields()); err != nil { + return fmt.Errorf("fallback %d: %w", i+1, err) + } + } + return nil +} diff --git a/pkg/ai/prompt/decode_fields_ginkgo_test.go b/pkg/ai/prompt/decode_fields_ginkgo_test.go new file mode 100644 index 00000000..97d9a046 --- /dev/null +++ b/pkg/ai/prompt/decode_fields_ginkgo_test.go @@ -0,0 +1,26 @@ +package prompt + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Prompt declaration field policy", func() { + DescribeTable("rejects unknown native declarations before rendering or execution", func(frontmatter string) { + _, err := Parse("---\n" + frontmatter + "\n---\nReview the change\n") + Expect(err).To(MatchError(ContainSubstring("unexpected"))) + }, + Entry("top level", "unexpected: false"), + Entry("budget", "budget: {unexpected: 0}"), + Entry("fallback", "fallbacks: [{model: sonnet, unexpected: false}]"), + Entry("runtime", "runtimes: [{model: sonnet, unexpected: false}, api:sol]"), + Entry("runtime fallback", "runtimes: [{model: sonnet, fallbacks: [{model: sol, unexpected: 0}]}, api:sol]"), + ) + + It("retains zero declarations after strict field inspection", func() { + doc, err := Parse("---\nmodel: sonnet\nnoCache: false\nbudget: {cost: 0}\n---\nReview\n") + Expect(err).NotTo(HaveOccurred()) + Expect(doc.Spec.Fields()).To(HaveKey("/noCache")) + Expect(doc.Spec.Fields()).To(HaveKey("/budget/cost")) + }) +}) diff --git a/pkg/ai/prompt/document.go b/pkg/ai/prompt/document.go index 8feae585..68085980 100644 --- a/pkg/ai/prompt/document.go +++ b/pkg/ai/prompt/document.go @@ -1,7 +1,6 @@ package prompt import ( - "bytes" "fmt" "strings" @@ -117,9 +116,10 @@ func decodePromptRuntimes(raw map[string]any) ([]api.Model, error) { continue } var runtime api.Model - dec := yaml.NewDecoder(bytes.NewReader(encoded)) - dec.KnownFields(true) - if err := dec.Decode(&runtime); err != nil { + if err := validateDeclarationFields(encoded, runtime.DecodeFields()); err != nil { + return nil, fmt.Errorf("runtime %d: %w", i+1, err) + } + if err := yaml.Unmarshal(encoded, &runtime); err != nil { return nil, fmt.Errorf("runtime %d: %w", i+1, err) } runtimes = append(runtimes, runtime) diff --git a/pkg/ai/prompt/document_test.go b/pkg/ai/prompt/document_test.go index 02bdd1c8..ba1bed78 100644 --- a/pkg/ai/prompt/document_test.go +++ b/pkg/ai/prompt/document_test.go @@ -126,7 +126,7 @@ func TestParse_RuntimeProfileMustBeNonEmptyString(t *testing.T) { // Render's strict spec decode must accept the pin the same way it accepts the // other prompt-only keys (name, runtimes, ...). func TestRender_RuntimeProfilePin(t *testing.T) { - req, _, err := Load("---\nruntimeProfile: review\nmodel: claude-sonnet-4-6\n---\n{{role \"user\"}}\nhello\n").Render(nil, nil) + req, _, err := Load("---\nruntimeProfile: review\nmodel: claude-sonnet-4-6\n---\n{{role \"user\"}}\nhello\n").Render(RenderOptions{}) require.NoError(t, err) assert.Equal(t, "claude-sonnet-4-6", req.Model.Name) assert.Equal(t, "hello", req.Prompt.User) diff --git a/pkg/ai/prompt/frontmatter_test.go b/pkg/ai/prompt/frontmatter_test.go index 645274ef..9ca82bc6 100644 --- a/pkg/ai/prompt/frontmatter_test.go +++ b/pkg/ai/prompt/frontmatter_test.go @@ -42,7 +42,7 @@ func groupsNode(t *testing.T, schemaJSON json.RawMessage) map[string]any { } func TestRenderFrontmatter_InterpolatesOutputSchemaCap(t *testing.T) { - req, _, err := Load(templatedSchemaPrompt).Render(map[string]any{"limit": 3}, nil) + req, _, err := Load(templatedSchemaPrompt).Render(RenderOptions{Data: map[string]any{"limit": 3}}) require.NoError(t, err) assert.Contains(t, req.Prompt.User, "limit 3", "body is still templated normally") @@ -52,7 +52,7 @@ func TestRenderFrontmatter_InterpolatesOutputSchemaCap(t *testing.T) { } func TestRenderFrontmatter_GuardOmitsCapWhenFalsy(t *testing.T) { - req, _, err := Load(templatedSchemaPrompt).Render(map[string]any{"limit": 0}, nil) + req, _, err := Load(templatedSchemaPrompt).Render(RenderOptions{Data: map[string]any{"limit": 0}}) require.NoError(t, err) groups := groupsNode(t, req.Prompt.SchemaJSON) @@ -66,7 +66,7 @@ func TestRenderFrontmatter_NoTemplateIsInert(t *testing.T) { // literal {{limit}} in the BODY still renders — proving only the frontmatter is // pre-rendered. const static = "---\nmodel: claude-sonnet-4-6\n---\nEcho {{limit}}.\n" - req, _, err := Load(static).Render(map[string]any{"limit": 5}, nil) + req, _, err := Load(static).Render(RenderOptions{Data: map[string]any{"limit": 5}}) require.NoError(t, err) assert.Contains(t, req.Prompt.User, "Echo 5.") } diff --git a/pkg/ai/prompt/prompt.go b/pkg/ai/prompt/prompt.go index 42460e35..9555ee96 100644 --- a/pkg/ai/prompt/prompt.go +++ b/pkg/ai/prompt/prompt.go @@ -20,7 +20,6 @@ package prompt import ( - "bytes" "encoding/json" "fmt" "io/fs" @@ -101,10 +100,16 @@ func LoadFS(fsys fs.FS, path string) (*Template, error) { return t, nil } -// Render executes the template body with data and folds the frontmatter into an -// ai.Request and ai.Config. When out is non-nil it becomes -// Request.Prompt.Schema (the structured-output target). -func (t *Template) Render(data map[string]any, out any) (ai.Request, ai.Config, error) { +type RenderOptions struct { + Data map[string]any + Output any + Declared bool +} + +// Render executes the body and preserves frontmatter and output schemas. +// Declared leaves model selection authored for subsequent layer resolution. +func (t *Template) Render(options RenderOptions) (ai.Request, ai.Config, error) { + data, out := options.Data, options.Output src, err := renderFrontmatter(t.source, data) if err != nil { return ai.Request{}, ai.Config{}, fmt.Errorf("render prompt %s frontmatter: %w", t.name, err) @@ -144,6 +149,7 @@ func (t *Template) Render(data map[string]any, out any) (ai.Request, ai.Config, if cfg.Model.Name == "" { cfg.Model.Name = rendered.Model } + req.Model = cfg.Model // Resolve name+mode together: the adapter follows from both, so inferring it // from the name alone would send `model: opus` + `backend: agent` to the // Anthropic API. Both copies are assigned so they cannot disagree downstream. @@ -151,7 +157,7 @@ func (t *Template) Render(data map[string]any, out any) (ai.Request, ai.Config, // A name the catalog does not know is not fatal here — a caller may supply its // own provider — so it stays as authored and the runtime decides. An authored // mode we cannot honour is fatal: dropping it would pick an adapter silently. - if cfg.Model.Name != "" { + if cfg.Model.Name != "" && !options.Declared { resolved, rerr := ai.Resolve(cfg.Model) switch { case rerr == nil: @@ -163,7 +169,8 @@ func (t *Template) Render(data map[string]any, out any) (ai.Request, ai.Config, } // The dotprompt config: block stays canonical for maxOutputTokens/temperature/ // reasoning when a file mixes both frontmatter dialects, so apply it last. - applyModelConfig(rendered.Config, &req, &cfg) + applyModelConfig(rendered.Config, &req) + cfg.Model = req.Model cfg.Budget = req.Budget if out != nil { req.Prompt.Schema = out @@ -203,9 +210,10 @@ func decodeSpecFrontmatter(raw map[string]any, req *ai.Request) error { if err != nil { return fmt.Errorf("re-encode frontmatter: %w", err) } - dec := yaml.NewDecoder(bytes.NewReader(b)) - dec.KnownFields(true) - return dec.Decode(req) + if err := validateDeclarationFields(b, req.DecodeFields()); err != nil { + return err + } + return yaml.Unmarshal(b, req) } // Library renders named .prompt files from an fs.FS (typically an embed.FS), the @@ -216,28 +224,29 @@ type Library struct{ fsys fs.FS } func NewLibrary(fsys fs.FS) *Library { return &Library{fsys: fsys} } // Render loads name from the library and renders it. -func (l *Library) Render(name string, data map[string]any, out any) (ai.Request, ai.Config, error) { +func (l *Library) Render(name string, options RenderOptions) (ai.Request, ai.Config, error) { t, err := LoadFS(l.fsys, name) if err != nil { return ai.Request{}, ai.Config{}, err } - return t.Render(data, out) + return t.Render(options) } // applyModelConfig maps the dotprompt config block (model-agnostic keys) onto // the captain request/config. -func applyModelConfig(c dp.ModelConfig, req *ai.Request, cfg *ai.Config) { +func applyModelConfig(c dp.ModelConfig, req *ai.Request) { if v, ok := floatOf(c["maxOutputTokens"]); ok { req.Budget.MaxTokens = int(v) - cfg.Budget.MaxTokens = int(v) + *req = req.WithExplicit("/budget/maxTokens") } if v, ok := floatOf(c["temperature"]); ok { - temp := v - req.Temperature = &temp - cfg.Model.Temperature = &temp + req.Temperature = &v + *req = req.WithExplicit("/temperature") } if s, ok := c["reasoning"].(string); ok { req.Effort = api.Effort(s) + req.Model = req.Model.WithExplicit("/effort") + *req = req.WithExplicit("/effort") } } diff --git a/pkg/ai/prompt/prompt_test.go b/pkg/ai/prompt/prompt_test.go index b517bbef..d22956b9 100644 --- a/pkg/ai/prompt/prompt_test.go +++ b/pkg/ai/prompt/prompt_test.go @@ -21,10 +21,10 @@ func TestRender_FrontmatterAndMessages(t *testing.T) { tmpl, err := LoadFS(library, "testdata/commit.prompt") require.NoError(t, err) - req, cfg, err := tmpl.Render(map[string]any{ + req, cfg, err := tmpl.Render(RenderOptions{Data: map[string]any{ "patch": "+func Login() bool { return a < b && c > d }", "maxBodyLines": 3, - }, nil) + }}) require.NoError(t, err) assert.Contains(t, req.Prompt.System, "commit message generator") @@ -56,10 +56,10 @@ func TestRender_FrontmatterAndMessages(t *testing.T) { } }`, string(req.Prompt.SchemaJSON)) - withoutCap, _, err := tmpl.Render(map[string]any{ + withoutCap, _, err := tmpl.Render(RenderOptions{Data: map[string]any{ "patch": "+trivial change", "maxBodyLines": 0, - }, nil) + }}) require.NoError(t, err) assert.Contains(t, withoutCap.Prompt.User, "body: omit unless the change is non-trivial") assert.NotContains(t, withoutCap.Prompt.User, "body: at most") @@ -79,7 +79,7 @@ func TestRender_SpecFrontmatter(t *testing.T) { tmpl, err := LoadFS(library, "testdata/options.prompt") require.NoError(t, err) - req, _, err := tmpl.Render(map[string]any{"target": "parser.go"}, nil) + req, _, err := tmpl.Render(RenderOptions{Data: map[string]any{"target": "parser.go"}}) require.NoError(t, err) // Spec-native keys from the second parse. @@ -122,7 +122,7 @@ func TestRender_StructuredOutputTarget(t *testing.T) { } out := &commitMsg{} - req, _, err := Load("{{role \"user\"}}\nhi").Render(nil, out) + req, _, err := Load("{{role \"user\"}}\nhi").Render(RenderOptions{Output: out}) require.NoError(t, err) assert.Same(t, out, req.Prompt.Schema) assert.Empty(t, req.Prompt.SchemaJSON, "a Go target takes precedence over any frontmatter schema") @@ -144,7 +144,7 @@ func TestRender_FrontmatterOutputSchema(t *testing.T) { "---\n" + "{{role \"user\"}}\nname a PR" - req, _, err := Load(src).Render(nil, nil) + req, _, err := Load(src).Render(RenderOptions{}) require.NoError(t, err) require.Nil(t, req.Prompt.Schema, "no Go target was passed") require.NotEmpty(t, req.Prompt.SchemaJSON, "frontmatter output.schema must reach SchemaJSON") @@ -159,10 +159,10 @@ func TestRender_FrontmatterOutputSchema(t *testing.T) { func TestLibrary_Render(t *testing.T) { lib := NewLibrary(library) - req, cfg, err := lib.Render("testdata/commit.prompt", map[string]any{ + req, cfg, err := lib.Render("testdata/commit.prompt", RenderOptions{Data: map[string]any{ "patch": "x", "maxBodyLines": 0, - }, nil) + }}) require.NoError(t, err) assert.Equal(t, "claude-sonnet-4-6", cfg.Model.Name) assert.Contains(t, req.Prompt.User, "x") @@ -185,8 +185,8 @@ func TestRender_AuthoredModeSelectsRuntime(t *testing.T) { {name: "api mode", mode: "api", model: "opus", runtime: api.RuntimeOf(api.Anthropic, api.ModeAPI)}, } { t.Run(tc.name, func(t *testing.T) { - req, cfg, err := Load("---\nmodel: "+tc.model+"\nmode: "+tc.mode+"\n---\n{{role \"user\"}}\nGo.\n"). - Render(nil, nil) + req, cfg, err := Load("---\nmodel: " + tc.model + "\nmode: " + tc.mode + "\n---\n{{role \"user\"}}\nGo.\n"). + Render(RenderOptions{}) require.NoError(t, err) assert.Equal(t, tc.runtime, api.RuntimeOf(cfg.Model.Provider, cfg.Model.Mode), "the config runtime must follow the authored mode") assert.Equal(t, tc.runtime, api.RuntimeOf(req.Model.Provider, req.Model.Mode), "the request runtime must follow the authored mode") @@ -240,7 +240,7 @@ func TestRender_RuntimeFixtureExamples(t *testing.T) { tmpl, err := LoadFS(library, path) require.NoError(t, err) - req, cfg, err := tmpl.Render(map[string]any{"task": "summarize the change and propose next steps"}, nil) + req, cfg, err := tmpl.Render(RenderOptions{Data: map[string]any{"task": "summarize the change and propose next steps"}}) require.NoError(t, err) require.NoError(t, req.Validate()) diff --git a/pkg/ai/prompt/render_declared_ginkgo_test.go b/pkg/ai/prompt/render_declared_ginkgo_test.go new file mode 100644 index 00000000..96bc7900 --- /dev/null +++ b/pkg/ai/prompt/render_declared_ginkgo_test.go @@ -0,0 +1,77 @@ +package prompt + +import ( + "testing/fstest" + + "github.com/flanksource/captain/pkg/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Declared prompt rendering", func() { + const source = `--- +model: agent:sonnet +temperature: 0.8 +effort: high +budget: + maxTokens: 512 + maxTurns: 3 +memory: + skipUser: false +config: + temperature: 0 + reasoning: medium + maxOutputTokens: 128 +output: + schema: + type: object + properties: + summary: + type: string +--- +{{role "system"}} +Inspect {{target}}. +{{role "user"}} +Review {{target}}. +` + It("preserves authored selection and rendered config, body and output schema", func() { + spec, cfg, err := Load(source).Render(RenderOptions{Data: map[string]any{"target": "parser.go"}, Declared: true}) + + Expect(err).NotTo(HaveOccurred()) + Expect(spec.Model.Name).To(Equal("agent:sonnet")) + Expect(spec.Model.Provider).To(BeNil()) + Expect(spec.Model.Mode).To(BeEmpty()) + Expect(spec.Temperature).To(HaveValue(Equal(0.0))) + Expect(spec.Effort).To(Equal(api.EffortMedium)) + Expect(spec.Budget).To(Equal(api.Budget{MaxTokens: 128, MaxTurns: 3})) + Expect(cfg.Model).To(Equal(spec.Model)) + Expect(cfg.Budget).To(Equal(spec.Budget)) + Expect(spec.Prompt.System).To(Equal("Inspect parser.go.")) + Expect(spec.Prompt.User).To(Equal("Review parser.go.")) + Expect(spec.Prompt.Source).To(Equal("")) + Expect(spec.Prompt.SchemaJSON).To(MatchJSON(`{"type":"object","properties":{"summary":{"type":"string"}}}`)) + }) + + It("passes the declared option and Go schema target through library rendering", func() { + target := &struct{ Summary string }{} + library := NewLibrary(fstest.MapFS{"review.prompt": {Data: []byte(source)}}) + + spec, cfg, err := library.Render("review.prompt", RenderOptions{Data: map[string]any{"target": "parser.go"}, Output: target, Declared: true}) + + Expect(err).NotTo(HaveOccurred()) + Expect(spec.Model.Name).To(Equal("agent:sonnet")) + Expect(cfg.Model).To(Equal(spec.Model)) + Expect(spec.Prompt.Schema).To(BeIdenticalTo(target)) + Expect(spec.Prompt.SchemaJSON).To(BeEmpty()) + Expect(spec.Prompt.Source).To(Equal("review.prompt")) + }) + + It("retains explicit zero config values for later composition", func() { + spec, cfg, err := Load("---\nconfig:\n maxOutputTokens: 0\n reasoning: \"\"\n---\nReview\n").Render(RenderOptions{Declared: true}) + + Expect(err).NotTo(HaveOccurred()) + Expect(spec.Fields().Has("/budget/maxTokens")).To(BeTrue()) + Expect(spec.Fields().Has("/effort")).To(BeTrue()) + Expect(cfg.Model).To(Equal(spec.Model)) + }) +}) diff --git a/pkg/api/runtime_preset_codec.go b/pkg/api/runtime_preset_codec.go new file mode 100644 index 00000000..55e11c53 --- /dev/null +++ b/pkg/api/runtime_preset_codec.go @@ -0,0 +1,80 @@ +package api + +import ( + "bytes" + "encoding/json" + + "gopkg.in/yaml.v3" +) + +type runtimePresetWire struct { + ModelFields `json:",inline" yaml:",inline"` + Budget Budget `json:"budget,omitempty" yaml:"budget,omitempty"` + Memory Memory `json:"memory,omitempty" yaml:"memory,omitempty"` + Permissions Permissions `json:"permissions,omitempty" yaml:"permissions,omitempty"` + ToolPreferences ToolPreferences `json:"toolPreferences,omitempty" yaml:"toolPreferences,omitempty"` + ToolPolicy PermissionPolicy `json:"toolPolicy,omitempty" yaml:"toolPolicy,omitempty"` + Setup *RuntimePresetSetup `json:"setup,omitempty" yaml:"setup,omitempty"` + Sandbox *SandboxRef `json:"sandbox,omitempty" yaml:"sandbox,omitempty"` +} + +func (RuntimePresetSpec) DecodeFields() any { return runtimePresetWire{} } + +func (s RuntimePresetSpec) MarshalJSON() ([]byte, error) { return json.Marshal(s.ToSpec()) } + +func (s RuntimePresetSpec) MarshalYAML() (any, error) { return s.ToSpec().MarshalYAML() } + +func (s *RuntimePresetSpec) UnmarshalJSON(data []byte) error { + if err := validateFallbackJSON(data); err != nil { + return err + } + var wire runtimePresetWire + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&wire); err != nil { + return err + } + *s = wire.toPreset() + spec := s.ToSpec() + if err := spec.capturePresence(data); err != nil { + return err + } + s.Explicit = spec.Explicit + return nil +} + +func (s *RuntimePresetSpec) UnmarshalYAML(node *yaml.Node) error { + data, err := yaml.Marshal(node) + if err != nil { + return err + } + var wire runtimePresetWire + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(&wire); err != nil { + return err + } + *s = wire.toPreset() + var fields any + if err := node.Decode(&fields); err != nil { + return err + } + if err := validateFallbackValues(fields); err != nil { + return err + } + explicit := FieldPresence{} + capturePresence(fields, "", explicit) + if len(explicit) > 0 { + spec := s.ToSpec() + spec.Explicit = explicit + spec.pruneOpaquePresence() + s.Explicit = spec.Explicit + } + return nil +} + +func (wire runtimePresetWire) toPreset() RuntimePresetSpec { + return RuntimePresetSpec{Model: Model(wire.ModelFields), Budget: wire.Budget, Memory: wire.Memory, + Permissions: wire.Permissions, ToolPreferences: wire.ToolPreferences, ToolPolicy: wire.ToolPolicy, + Setup: wire.Setup, Sandbox: wire.Sandbox} +} diff --git a/pkg/api/runtime_profiles.go b/pkg/api/runtime_profiles.go index 466d9824..d846ca89 100644 --- a/pkg/api/runtime_profiles.go +++ b/pkg/api/runtime_profiles.go @@ -14,6 +14,7 @@ import ( // task-specific profile fields and cannot be represented here. type RuntimePresetSpec struct { Model `json:",inline" yaml:",inline"` + Explicit FieldPresence `json:"-" yaml:"-"` Budget Budget `json:"budget,omitempty" yaml:"budget,omitempty"` Memory Memory `json:"memory,omitempty" yaml:"memory,omitempty"` Permissions Permissions `json:"permissions,omitempty" yaml:"permissions,omitempty"` @@ -116,7 +117,7 @@ func ResolveRuntimeProfile(request RuntimeProfileResolveRequest) (ResolvedSpec, if err != nil { return ResolvedSpec{}, err } - resolved, err := ResolveSpecLayers(layers...) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: layers}) if err != nil { return ResolvedSpec{}, fmt.Errorf("resolve runtime profile %q: %w", request.Profile.Name, err) } @@ -167,7 +168,7 @@ func (i runtimePresetIndex) lookup(profile, ref string) (RuntimePreset, error) { func (s RuntimePresetSpec) ToSpec() Spec { return Spec{ - Model: s.Model, Budget: s.Budget, Memory: s.Memory, + Model: s.Model, Explicit: s.Explicit.Clone(), Budget: s.Budget, Memory: s.Memory, Permissions: s.Permissions, ToolPreferences: s.ToolPreferences, ToolPolicy: s.ToolPolicy, Setup: s.Setup.toSetup(), Sandbox: s.Sandbox, } diff --git a/pkg/api/sandbox_ref.go b/pkg/api/sandbox_ref.go index 95a189d4..b3fca70d 100644 --- a/pkg/api/sandbox_ref.go +++ b/pkg/api/sandbox_ref.go @@ -18,7 +18,7 @@ import ( // two are independent, and a run with the sandbox off may still want a // restrictive posture. type SandboxRef struct { - Mode SandboxKind `json:"mode" yaml:"mode"` + Mode SandboxKind `json:"mode,omitempty" yaml:"mode,omitempty"` // Backend names a configured Docker or Git Agent backend. Backend string `json:"backend,omitempty" yaml:"backend,omitempty"` // Policy is translated into the active provider's native sandbox settings. @@ -115,7 +115,7 @@ func (SandboxRef) JSONSchema() *jsonschema.Schema { { Type: "object", Properties: properties, - Required: []string{"mode"}, + AnyOf: []*jsonschema.Schema{{Required: []string{"mode"}}, {Required: []string{"backend"}}}, AdditionalProperties: jsonschema.FalseSchema, }, }, @@ -134,13 +134,23 @@ func (r SandboxRef) Validate() error { if err := r.Mode.Validate(); err != nil { return err } + return r.ValidateStructure() +} + +// ValidateStructure permits a named backend pending captured-context selection. +func (r SandboxRef) ValidateStructure() error { + if r.Mode != "" || r.Backend == "" { + if err := r.Mode.Validate(); err != nil { + return err + } + } if r.Policy != nil && r.Mode != SandboxNative { return fmt.Errorf("native policy requires sandbox mode native, got %q", r.Mode) } - if r.Backend != "" && r.Mode != SandboxDocker && r.Mode != SandboxGitAgent { + if r.Backend != "" && r.Mode != "" && r.Mode != SandboxDocker && r.Mode != SandboxGitAgent { return fmt.Errorf("sandbox backend is only valid for docker or git-agent mode, got %q", r.Mode) } - if (r.Agent != "" || r.Dispatch != nil) && r.Mode != SandboxGitAgent { + if (r.Agent != "" || r.Dispatch != nil) && r.Mode != "" && r.Mode != SandboxGitAgent { return fmt.Errorf("sandbox agent/dispatch settings require git-agent mode, got %q", r.Mode) } if err := r.Policy.Validate(); err != nil { diff --git a/pkg/api/sandbox_ref_ginkgo_test.go b/pkg/api/sandbox_ref_ginkgo_test.go index 866cecdc..6593aa4c 100644 --- a/pkg/api/sandbox_ref_ginkgo_test.go +++ b/pkg/api/sandbox_ref_ginkgo_test.go @@ -70,11 +70,14 @@ var _ = Describe("SandboxRef", func() { Expect(ref).To(Equal(SandboxRef{Mode: SandboxNative})) }) - It("declares only the four public modes in its JSON schema", func() { + It("declares the public modes and permits a named backend awaiting context resolution", func() { schema := SandboxRef{}.JSONSchema() Expect(schema.OneOf).To(HaveLen(2)) Expect(schema.OneOf[0].Enum).To(Equal(enumValues(AllSandboxModes()))) - Expect(schema.OneOf[1].Required).To(Equal([]string{"mode"})) + Expect(schema.OneOf[1].Required).To(BeEmpty()) + Expect(schema.OneOf[1].AnyOf).To(HaveLen(2)) + Expect(schema.OneOf[1].AnyOf[0].Required).To(Equal([]string{"mode"})) + Expect(schema.OneOf[1].AnyOf[1].Required).To(Equal([]string{"backend"})) }) It("rejects a negative dispatch attempt bound", func() { @@ -113,12 +116,16 @@ func jsonTagSet(t reflect.Type) []string { var tags []string for i := range t.NumField() { field := t.Field(i) - if !field.IsExported() { + if !field.IsExported() && !field.Anonymous { continue } tag, _, _ := strings.Cut(field.Tag.Get("json"), ",") + if tag == "-" { + continue + } if tag == "" && field.Anonymous { - tag = "(inline)" + field.Type.Name() + tags = append(tags, jsonTagSet(field.Type)...) + continue } tags = append(tags, tag) } diff --git a/pkg/api/spec.go b/pkg/api/spec.go index 910fe819..d1e1884d 100644 --- a/pkg/api/spec.go +++ b/pkg/api/spec.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "fmt" "reflect" "strings" @@ -19,11 +18,12 @@ import ( // domain object. type Spec struct { Model `json:",inline" yaml:",inline"` - Prompt Prompt `json:"prompt" yaml:"prompt"` - Messages []Message `json:"messages,omitempty" yaml:"messages,omitempty" pretty:"-"` - Budget Budget `json:"budget,omitempty" yaml:"budget,omitempty"` - Memory Memory `json:"memory,omitempty" yaml:"memory,omitempty"` - Permissions Permissions `json:"permissions,omitempty" yaml:"permissions,omitempty"` + Explicit FieldPresence `json:"-" yaml:"-"` + Prompt Prompt `json:"prompt" yaml:"prompt"` + Messages []Message `json:"messages,omitempty" yaml:"messages,omitempty" pretty:"-"` + Budget Budget `json:"budget,omitempty" yaml:"budget,omitempty"` + Memory Memory `json:"memory,omitempty" yaml:"memory,omitempty"` + Permissions Permissions `json:"permissions,omitempty" yaml:"permissions,omitempty"` // ToolPreferences is the serializable per-turn tool/group selection policy. // Executable tool handlers remain in Config.Tools. ToolPreferences ToolPreferences `json:"toolPreferences,omitempty" yaml:"toolPreferences,omitempty" pretty:"-"` @@ -53,7 +53,7 @@ type Spec struct { } type specMarshal struct { - Model `json:",inline" yaml:",inline"` + ModelFields `json:",inline" yaml:",inline"` Prompt *Prompt `json:"prompt,omitempty" yaml:"prompt,omitempty"` Messages []Message `json:"messages,omitempty" yaml:"messages,omitempty"` Budget *Budget `json:"budget,omitempty" yaml:"budget,omitempty"` @@ -95,6 +95,14 @@ func isEmpty(value reflect.Value) bool { } if value.CanInterface() { switch typed := value.Interface().(type) { + case Spec: + if len(typed.Explicit) > 0 || len(typed.Model.Explicit) > 0 { + return false + } + case Model: + if len(typed.Explicit) > 0 { + return false + } case Tools: return len(typed.Policies()) == 0 case MCP: @@ -145,7 +153,7 @@ func omitEmptyPointer[T any](value *T) *T { func (s Spec) marshalValue() specMarshal { return specMarshal{ - Model: s.Model, + ModelFields: specModelWire(s.Model), Prompt: omitEmptyValue(s.Prompt), Messages: s.Messages, Budget: omitEmptyValue(s.Budget), @@ -162,19 +170,16 @@ func (s Spec) marshalValue() specMarshal { } } -func (s Spec) MarshalJSON() ([]byte, error) { - return json.Marshal(s.marshalValue()) -} - -func (s Spec) MarshalYAML() (any, error) { - return s.marshalValue(), nil -} - // Validate runs each component's validation, failing loud on the first error. func (s Spec) Validate() error { if err := s.ValidateStructure(); err != nil { return err } + if s.Sandbox != nil { + if err := s.Sandbox.Validate(); err != nil { + return fmt.Errorf("sandbox: %w", err) + } + } validateModel := s.Model.Validate if s.IsVerifyOnly() && len(s.Workflow.Verify.Prompts) == 0 { validateModel = s.ValidateOptions diff --git a/pkg/api/spec_codec.go b/pkg/api/spec_codec.go new file mode 100644 index 00000000..bed1392b --- /dev/null +++ b/pkg/api/spec_codec.go @@ -0,0 +1,308 @@ +package api + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "sort" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func (s Spec) MarshalJSON() ([]byte, error) { + value, err := s.wireFields() + if err != nil { + return nil, err + } + return json.Marshal(value) +} + +func (s Spec) MarshalYAML() (any, error) { + fields, err := s.wireFields() + if err != nil { + return nil, err + } + for path := range s.Fields() { + tokens := strings.Split(strings.TrimPrefix(path, "/"), "/") + value := serializedField(reflect.ValueOf(s), tokens) + if value.IsValid() && value.Type() == reflect.TypeOf(json.RawMessage(nil)) { + putWireField(fields, tokens, []byte(value.Interface().(json.RawMessage))) + } + } + return yamlNumbers(fields) +} + +func yamlNumbers(value any) (any, error) { + switch typed := value.(type) { + case json.Number: + if integer, err := typed.Int64(); err == nil { + return integer, nil + } + if integer, err := strconv.ParseUint(typed.String(), 10, 64); err == nil { + return integer, nil + } + return typed.Float64() + case map[string]any: + for key, child := range typed { + converted, err := yamlNumbers(child) + if err != nil { + return nil, err + } + typed[key] = converted + } + case []any: + for i, child := range typed { + converted, err := yamlNumbers(child) + if err != nil { + return nil, err + } + typed[i] = converted + } + } + return value, nil +} + +func (s Spec) wireFields() (map[string]any, error) { + data, err := json.Marshal(s.marshalValue()) + if err != nil { + return nil, err + } + var fields map[string]any + if err := decodeWireJSON(data, &fields); err != nil { + return nil, err + } + pruneUnownedFields(fields, "", s.Fields()) + paths := make([]string, 0, len(s.explicitFields())) + for path := range s.explicitFields() { + paths = append(paths, path) + } + sort.Strings(paths) + for _, path := range paths { + tokens := strings.Split(strings.TrimPrefix(path, "/"), "/") + value := serializedField(reflect.ValueOf(s), tokens) + if !value.IsValid() { + return nil, fmt.Errorf("unknown explicit spec field %q", path) + } + data, err := json.Marshal(value.Interface()) + if err != nil { + return nil, err + } + var raw any + if err := decodeWireJSON(data, &raw); err != nil { + return nil, err + } + putWireField(fields, tokens, raw) + } + return fields, nil +} + +// DecodeFields lets the enclosing decoder choose its unknown-field policy. +func (Spec) DecodeFields() any { return specMarshal{} } + +func (s *Spec) UnmarshalJSON(data []byte) error { + var wire specMarshal + decoder := json.NewDecoder(bytes.NewReader(data)) + if err := decoder.Decode(&wire); err != nil { + return err + } + *s = wire.toSpec() + return s.capturePresence(data) +} + +func (s *Spec) UnmarshalYAML(node *yaml.Node) error { + data, err := yaml.Marshal(node) + if err != nil { + return err + } + var wire specMarshal + decoder := yaml.NewDecoder(bytes.NewReader(data)) + if err := decoder.Decode(&wire); err != nil { + return err + } + *s = wire.toSpec() + var fields any + if err := node.Decode(&fields); err != nil { + return err + } + s.Explicit = FieldPresence{} + capturePresence(fields, "", s.Explicit) + s.pruneOpaquePresence() + if len(s.Explicit) == 0 { + s.Explicit = nil + } + return nil +} + +func (wire specMarshal) toSpec() Spec { + s := Spec{Model: Model(wire.ModelFields), Messages: wire.Messages, + ToolPolicy: wire.ToolPolicy, ToolApproval: wire.Approval, Setup: wire.Setup, + Sandbox: wire.Sandbox, Workflow: wire.Workflow, SessionID: wire.SessionID, CLIArgs: wire.CLIArgs} + if wire.Prompt != nil { + s.Prompt = *wire.Prompt + } + if wire.Budget != nil { + s.Budget = *wire.Budget + } + if wire.Memory != nil { + s.Memory = *wire.Memory + } + if wire.Permissions != nil { + s.Permissions = *wire.Permissions + } + if wire.Preferences != nil { + s.ToolPreferences = *wire.Preferences + } + return s +} + +func serializedField(value reflect.Value, tokens []string) reflect.Value { + if len(tokens) == 0 { + return value + } + for value.IsValid() && (value.Kind() == reflect.Pointer || value.Kind() == reflect.Interface) { + if value.IsNil() { + return reflect.Value{} + } + value = value.Elem() + } + if !value.IsValid() { + return value + } + token := unescapeField(tokens[0]) + if value.CanInterface() { + if _, ok := value.Interface().(MCP); ok { + switch token { + case "disabled": + return serializedField(value.FieldByName("Disabled"), tokens[1:]) + case "servers": + return serializedField(value.FieldByName("Servers"), tokens[1:]) + default: + return serializedField(value.FieldByName("Modes"), tokens) + } + } + } + switch value.Kind() { + case reflect.Struct: + for i := range value.NumField() { + field := value.Type().Field(i) + name := strings.Split(field.Tag.Get("json"), ",")[0] + if name == "-" || !field.IsExported() { + continue + } + if field.Anonymous && name == "" { + if found := serializedField(value.Field(i), tokens); found.IsValid() { + return found + } + } else if name == token { + return serializedField(value.Field(i), tokens[1:]) + } + } + case reflect.Map: + if value.Type().Key().Kind() == reflect.String { + return serializedField(value.MapIndex(reflect.ValueOf(token).Convert(value.Type().Key())), tokens[1:]) + } + case reflect.Slice, reflect.Array: + if index, err := strconv.Atoi(token); err == nil && index >= 0 && index < value.Len() { + return serializedField(value.Index(index), tokens[1:]) + } + } + return reflect.Value{} +} + +func putWireField(value any, tokens []string, field any) { + key := unescapeField(tokens[0]) + switch typed := value.(type) { + case map[string]any: + if len(tokens) == 1 { + typed[key] = field + return + } + if typed[key] == nil { + typed[key] = map[string]any{} + } + putWireField(typed[key], tokens[1:], field) + case []any: + index, err := strconv.Atoi(key) + if err != nil || index < 0 || index >= len(typed) { + return + } + if len(tokens) == 1 { + typed[index] = field + } else { + putWireField(typed[index], tokens[1:], field) + } + } +} + +func fmtFieldIndex(path string, index int) string { return path + "/" + strconv.Itoa(index) } + +func pruneUnownedFields(value any, path string, present FieldPresence) { + switch fields := value.(type) { + case map[string]any: + for key, child := range fields { + childPath := path + "/" + escapeField(key) + pruneUnownedFields(child, childPath, present) + empty := child == nil + if child != nil { + v := reflect.ValueOf(child) + empty = v.IsZero() || (v.Kind() == reflect.Map || v.Kind() == reflect.Slice) && v.Len() == 0 + if number, ok := child.(json.Number); ok { + parsed, err := number.Float64() + empty = err == nil && parsed == 0 + } + } + if empty && !fieldCovered(present, childPath) { + delete(fields, key) + } + } + case []any: + for i, child := range fields { + pruneUnownedFields(child, fmtFieldIndex(path, i), present) + } + } +} + +func decodeWireJSON(data []byte, value any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + return decoder.Decode(value) +} + +func validateFallbackJSON(data []byte) error { + var fields any + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + return validateFallbackValues(fields) +} + +func validateFallbackValues(value any) error { + fields, ok := value.(map[string]any) + if !ok { + return nil + } + fallbacks, _ := fields["fallbacks"].([]any) + for i, fallback := range fallbacks { + object, ok := fallback.(map[string]any) + if !ok { + continue + } + data, err := json.Marshal(object) + if err != nil { + return err + } + var wire specModelWire + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&wire); err != nil { + return fmt.Errorf("fallback[%d]: %w", i, err) + } + if err := validateFallbackValues(object); err != nil { + return fmt.Errorf("fallback[%d]: %w", i, err) + } + } + return nil +} diff --git a/pkg/api/spec_composition.go b/pkg/api/spec_composition.go index 63819019..b50c965e 100644 --- a/pkg/api/spec_composition.go +++ b/pkg/api/spec_composition.go @@ -1,27 +1,87 @@ package api -import "fmt" +import ( + "fmt" + + "github.com/flanksource/captain/pkg/captainconfig" +) + +// ResolveSpecOptions supplies a complete authored stack and optional immutable +// defaults. Nil Saved performs no saved or built-in default injection. +type ResolveSpecOptions struct { + Layers []SpecLayer + Saved *captainconfig.AIDefaults + RequireModel bool + Normalize func(Spec) (SpecNormalization, error) +} + +// SpecNormalization declares context-derived values without adding an authored +// trace layer. Fields identifies only the paths derived by the callback. +type SpecNormalization struct { + Spec Spec + Fields FieldPresence + Source FieldSource +} + +// FieldSourceKind distinguishes authored values, saved settings, and catalog facts. +type FieldSourceKind string + +const ( + FieldSourceLayer FieldSourceKind = "layer" + FieldSourceSaved FieldSourceKind = "saved" + FieldSourceCatalog FieldSourceKind = "catalog" + FieldSourceContext FieldSourceKind = "context" +) + +// FieldSource names a serialized field's owner and its original source key. +type FieldSource struct { + Kind FieldSourceKind `json:"kind" yaml:"kind"` + Name string `json:"name" yaml:"name"` + Key string `json:"key" yaml:"key"` + LayerID string `json:"layerId,omitempty" yaml:"layerId,omitempty"` +} + +// FieldProvenance retains authorship when a runtime or constraint normalizes it. +type FieldProvenance struct { + Source FieldSource `json:"source" yaml:"source"` + NormalizedBy *FieldSource `json:"normalizedBy,omitempty" yaml:"normalizedBy,omitempty"` +} // ComposedSpec is an ordered structural projection, not a validated runtime. type ComposedSpec struct { - Spec Spec `json:"spec" yaml:"spec"` - Constraints RuntimeConstraints `json:"constraints" yaml:"constraints"` - Trace []SpecLayer `json:"trace" yaml:"trace"` + fieldLayers map[string]int + Spec Spec `json:"spec" yaml:"spec"` + Constraints RuntimeConstraints `json:"constraints" yaml:"constraints"` + Trace []SpecLayer `json:"trace" yaml:"trace"` + Provenance map[string]FieldProvenance `json:"provenance,omitempty" yaml:"provenance,omitempty"` + Warnings []string `json:"warnings,omitempty" yaml:"warnings,omitempty"` } // ResolveSpecLayers validates the effective runtime after composing every layer. -func ResolveSpecLayers(input ...SpecLayer) (ResolvedSpec, error) { - composed, err := ComposeSpecLayers(input...) +func ResolveSpecLayers(options ResolveSpecOptions) (ResolvedSpec, error) { + composed, err := ComposeSpecLayers(options) if err != nil { return ResolvedSpec{}, err } - resolved := ResolvedSpec{Spec: composed.Spec, Constraints: composed.Constraints, Trace: composed.Trace} + if options.Saved == nil && !options.RequireModel && options.Normalize == nil { + if err := composed.expandModel(); err != nil { + return ResolvedSpec{}, err + } + } + resolved := ResolvedSpec{Spec: composed.Spec, Constraints: composed.Constraints, Trace: composed.Trace, + Provenance: composed.Provenance, Warnings: composed.Warnings} if err := resolved.Spec.ValidateStructure(); err != nil { return ResolvedSpec{}, fmt.Errorf("effective spec: %w", err) } + if resolved.Spec.Sandbox != nil { + if err := resolved.Spec.Sandbox.Validate(); err != nil { + return ResolvedSpec{}, fmt.Errorf("effective sandbox: %w", err) + } + } if resolved.Spec.Name == "" { return resolved, nil } + before := resolved.Spec.Model resolved.Spec.Model, err = ResolveModel(resolved.Spec.Model) if err != nil { return ResolvedSpec{}, err @@ -29,10 +89,12 @@ func ResolveSpecLayers(input ...SpecLayer) (ResolvedSpec, error) { if err := validateResolvedModels(resolved); err != nil { return ResolvedSpec{}, err } - resolved.Warnings, err = ValidateRuntimeSpec(resolved.Spec) + resolved.recordNormalization(before) + warnings, err := ValidateRuntimeSpec(resolved.Spec) if err != nil { return ResolvedSpec{}, err } + resolved.Warnings = append(resolved.Warnings, warnings...) return resolved, nil } diff --git a/pkg/api/spec_defaults.go b/pkg/api/spec_defaults.go new file mode 100644 index 00000000..b786a76c --- /dev/null +++ b/pkg/api/spec_defaults.go @@ -0,0 +1,155 @@ +package api + +import ( + "fmt" + "reflect" + "strings" + + "github.com/flanksource/captain/pkg/aiflags" + "github.com/flanksource/captain/pkg/captainconfig" +) + +// SavedDefaultsError identifies malformed injected configuration independently +// from authored request errors, so hosts can attribute it to their config owner. +type SavedDefaultsError struct { + Source string + Err error +} + +func (e *SavedDefaultsError) Error() string { + return fmt.Sprintf("saved defaults %s: %v", e.Source, e.Err) +} + +func (e *SavedDefaultsError) Unwrap() error { return e.Err } + +func (composed *ComposedSpec) applyDefaults(options ResolveSpecOptions) error { + if options.Saved == nil && !options.RequireModel { + normalized, err := composed.normalize(options.Normalize) + composed.recordContext(normalized) + return err + } + saved := captainconfig.AIDefaults{} + if options.Saved != nil { + saved = *options.Saved + } + model := composed.Spec.modelWithPresence() + defaults := aiflags.DefaultOptions{Model: model, Saved: saved, CatalogDefaults: options.Saved != nil, AllowUnknownModel: true} + var normalized *SpecNormalization + if options.Normalize != nil { + defaults.Normalize = func(model Model) (Model, error) { + composed.Spec.Model = model + var err error + normalized, err = composed.normalize(options.Normalize) + return composed.Spec.modelWithPresence(), err + } + } + defaulted, err := aiflags.ApplyDefaults(defaults) + if err != nil { + if invalid := saved.Validate(); invalid != nil { + return &SavedDefaultsError{Source: "~/.captain.yaml ai", Err: invalid} + } + return err + } + composed.Spec.Model = defaulted.Model + composed.recordDefaults(defaulted.Sources, normalized) + for _, missing := range defaulted.Unconfigured { + diagnostic := &aiflags.UnconfiguredError{Field: "mode", Model: missing.Model, Provider: missing.Provider} + composed.Warnings = append(composed.Warnings, fmt.Sprintf("%s: %s; using the registry runtime default during the compatibility window", missing.Path, diagnostic.Error())) + } + composed.fillSavedSpec(saved) + if options.Saved != nil && !fieldCovered(composed.Spec.Fields(), "/budget/maxTokens") { + composed.Spec.Budget.MaxTokens = 4096 + composed.Provenance["/budget/maxTokens"] = FieldProvenance{Source: FieldSource{Kind: FieldSourceCatalog, Name: "Captain defaults", Key: "captain.defaults.maxTokens"}} + } + if options.RequireModel && strings.TrimSpace(composed.Spec.Name) == "" { + return &aiflags.UnconfiguredError{Field: "model", Model: composed.Spec.Name, Provider: composed.Spec.Provider} + } + return nil +} + +func (composed *ComposedSpec) recordDefaults(sources map[string]string, normalized *SpecNormalization) { + for path, key := range sources { + if strings.HasPrefix(key, "primary.") { + continue + } + kind, name := FieldSourceSaved, "~/.captain.yaml" + if strings.HasPrefix(key, "registry.") { + kind, name = FieldSourceCatalog, "model registry" + } + composed.Provenance[path] = FieldProvenance{Source: FieldSource{Kind: kind, Name: name, Key: key}} + } + composed.recordContext(normalized) + for path, key := range sources { + if primary, inherited := strings.CutPrefix(key, "primary."); inherited { + composed.Provenance[path] = composed.Provenance["/"+primary] + } + } +} + +func (s Spec) modelWithPresence() Model { + return applyModelPresence(s.Model, s.explicitFields(), "") +} + +func applyModelPresence(model Model, fields FieldPresence, prefix string) Model { + for path := range fields { + local, belongs := strings.CutPrefix(path, prefix) + if belongs && strings.Count(local, "/") == 1 && serializedField(reflect.ValueOf(model), []string{local[1:]}).IsValid() { + model = model.WithExplicit(local) + } + } + for i, fallback := range model.Fallbacks { + model.Fallbacks[i] = applyModelPresence(fallback, fields, fmtFieldIndex(prefix+"/fallbacks", i)) + } + return model +} + +func (composed *ComposedSpec) fillSavedSpec(saved captainconfig.AIDefaults) { + present := composed.Spec.Fields() + fields := []struct { + Path string + Key string + Value any + }{ + {"/budget/cost", "budgetUSD", saved.BudgetUSD}, + {"/budget/maxTokens", "maxTokens", saved.MaxTokens}, + {"/budget/timeout", "timeout", saved.Timeout}, + {"/permissions/mcp/disabled", "noMCP", saved.NoMCP}, + {"/memory/skipHooks", "noHooks", saved.NoHooks}, + {"/memory/skipSkills", "noSkills", saved.NoSkills}, + {"/memory/skipUser", "noUser", saved.NoUser}, + {"/memory/skipProject", "noProject", saved.NoProject}, + {"/memory/skipMemory", "noMemory", saved.NoMemory}, + } + for _, field := range fields { + if !saved.Fields().Has("/"+field.Key) || fieldCovered(present, field.Path) || hasDescendant(present, field.Path) { + continue + } + target := serializedField(reflect.ValueOf(&composed.Spec).Elem(), strings.Split(field.Path[1:], "/")) + target.Set(reflect.ValueOf(field.Value)) + composed.Spec = composed.Spec.WithExplicit(field.Path) + composed.Provenance[field.Path] = FieldProvenance{Source: FieldSource{Kind: FieldSourceSaved, Name: "~/.captain.yaml", Key: "ai." + field.Key}} + } +} + +func hasDescendant(fields FieldPresence, path string) bool { + for field, present := range fields { + if present && strings.HasPrefix(field, path+"/") { + return true + } + } + return false +} + +func fieldCovered(fields FieldPresence, path string) bool { + for path != "" { + if fields.Has(path) { + return true + } + index := strings.LastIndex(path, "/") + if index < 0 { + return false + } + path = path[:index] + } + return false +} diff --git a/pkg/api/spec_defaults_ginkgo_test.go b/pkg/api/spec_defaults_ginkgo_test.go new file mode 100644 index 00000000..a38ff24b --- /dev/null +++ b/pkg/api/spec_defaults_ginkgo_test.go @@ -0,0 +1,167 @@ +package api + +import ( + "encoding/json" + "errors" + + "github.com/flanksource/captain/pkg/captainconfig" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Saved spec defaults", func() { + It("fills the complete spec after authored layers and records equal-valued ownership", func() { + saved := captainconfig.AIDefaults{DefaultModel: "api:sonnet:high", BudgetUSD: 2, MaxTokens: 1200, NoCache: true, NoSkills: true, Timeout: "2m"} + project := PromptSpecLayer("project", Spec{Budget: Budget{Cost: 2}, Prompt: Prompt{User: "review"}}) + request := RequestSpecLayer("request", Spec{Budget: Budget{Cost: 2}}) + result, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{project, request}, Saved: &saved, RequireModel: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Spec.Name).To(Equal("claude-sonnet-5")) + Expect(result.Spec.Mode).To(Equal(ModeAPI)) + Expect(result.Spec.Effort).To(Equal(EffortHigh)) + Expect(result.Spec.Prompt.User).To(Equal("review")) + Expect(result.Spec.Budget).To(Equal(Budget{Cost: 2, MaxTokens: 1200, Timeout: "2m"})) + Expect(result.Spec.Memory.SkipSkills).To(BeTrue()) + Expect(result.Provenance["/budget/cost"].Source).To(Equal(FieldSource{Kind: FieldSourceLayer, Name: "request", Key: "/budget/cost"})) + Expect(result.Provenance["/budget/maxTokens"].Source.Key).To(Equal("ai.maxTokens")) + Expect(result.Provenance["/model"].Source.Key).To(Equal("ai.defaultModel")) + Expect(result.Provenance["/model"].NormalizedBy).NotTo(BeNil()) + Expect(result.Trace).To(Equal([]SpecLayer{project, request})) + }) + + It("keeps explicit false zero and empty fallbacks authoritative against saved defaults", func() { + var request Spec + Expect(json.Unmarshal([]byte(`{"model":"agent:sonnet","noCache":false,"fallbacks":[],"budget":{"cost":0},"memory":{"skipSkills":false}}`), &request)).To(Succeed()) + saved := captainconfig.AIDefaults{DefaultModel: "agent:sonnet,api:sol", NoCache: true, BudgetUSD: 4, NoSkills: true} + result, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{RequestSpecLayer("request", request)}, Saved: &saved, RequireModel: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Spec.NoCache).To(BeFalse()) + Expect(result.Spec.Fallbacks).To(BeEmpty()) + Expect(result.Spec.Budget.Cost).To(BeZero()) + Expect(result.Spec.Memory.SkipSkills).To(BeFalse()) + for _, path := range []string{"/noCache", "/fallbacks", "/budget/cost", "/memory/skipSkills"} { + Expect(result.Provenance[path].Source.Name).To(Equal("request"), path) + } + }) + + It("uses each final candidate provider and identifies CSV versus list authorship", func() { + saved := captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{ + "anthropic": {Mode: "agent", ReasoningEffort: "high"}, "openai": {Mode: "api", ReasoningEffort: "low"}, + }} + profile := PromptSpecLayer("profile", Spec{Model: Model{Name: "sonnet,sol"}}) + request := RequestSpecLayer("request", Spec{Model: Model{Fallbacks: []Model{{Name: "haiku"}}}}) + result, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{profile, request}, Saved: &saved, RequireModel: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Spec.Fallbacks).To(HaveLen(2)) + Expect(result.Spec.Fallbacks[0].Mode).To(Equal(ModeAPI)) + Expect(result.Spec.Fallbacks[1].Mode).To(Equal(ModeAgent)) + Expect(result.Provenance["/fallbacks/0/model"].Source).To(Equal(FieldSource{Kind: FieldSourceLayer, Name: "profile", Key: "/model"})) + Expect(result.Provenance["/fallbacks/1/model"].Source).To(Equal(FieldSource{Kind: FieldSourceLayer, Name: "request", Key: "/fallbacks/0/model"})) + Expect(result.Provenance["/fallbacks/0/mode"].Source.Key).To(Equal("ai.providers.openai.mode")) + }) + + It("warns on every unconfigured candidate mode while retaining the compatibility runtime", func() { + saved := captainconfig.AIDefaults{} + result, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{RequestSpecLayer("request", Spec{Model: Model{Name: "sonnet,sol"}})}, Saved: &saved, RequireModel: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Warnings).To(HaveLen(2)) + Expect(result.Warnings[0]).To(ContainSubstring("configure")) + Expect(result.Warnings[1]).To(ContainSubstring("fallbacks/0/mode")) + Expect(result.Spec.Mode).To(Equal(ModeAgent)) + }) + + It("requires a configured model only for generating runs", func() { + saved := captainconfig.AIDefaults{} + _, err := ResolveSpecLayers(ResolveSpecOptions{Saved: &saved, RequireModel: true}) + Expect(err).To(MatchError(ContainSubstring("configure"))) + result, err := ResolveSpecLayers(ResolveSpecOptions{Saved: &saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Spec.Name).To(BeEmpty()) + Expect(result.Warnings).To(BeEmpty()) + }) + + It("keeps saved preview composition repairable until final runtime validation", func() { + saved := captainconfig.AIDefaults{DefaultModel: "api:sonnet"} + options := ResolveSpecOptions{Saved: &saved, Layers: []SpecLayer{PromptSpecLayer("profile", Spec{Sandbox: &SandboxRef{Mode: SandboxNative}})}} + composed, err := ComposeSpecLayers(options) + Expect(err).NotTo(HaveOccurred()) + Expect(composed.Spec.Mode).To(Equal(ModeAPI)) + _, err = ResolveSpecLayers(options) + Expect(err).To(MatchError(ContainSubstring("sandbox mode"))) + }) + + It("uses catalog effort and the existing token default after saved gaps, preserving explicit clears", func() { + saved := captainconfig.AIDefaults{} + layer := RequestSpecLayer("request", Spec{Model: Model{Name: "agent:fable"}}) + result, err := ResolveSpecLayers(ResolveSpecOptions{Saved: &saved, Layers: []SpecLayer{layer}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Spec.Budget.MaxTokens).To(Equal(4096)) + Expect(result.Spec.Effort).To(Equal(EffortHigh)) + Expect(result.Provenance["/budget/maxTokens"].Source.Kind).To(Equal(FieldSourceCatalog)) + Expect(result.Provenance["/effort"].Source.Kind).To(Equal(FieldSourceCatalog)) + layer.Spec = layer.Spec.WithExplicit("/budget/maxTokens", "/effort") + result, err = ResolveSpecLayers(ResolveSpecOptions{Saved: &saved, Layers: []SpecLayer{layer}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Spec.Budget.MaxTokens).To(BeZero()) + Expect(result.Spec.Effort).To(BeEmpty()) + pure, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer}}) + Expect(err).NotTo(HaveOccurred()) + Expect(pure.Spec.Budget.MaxTokens).To(BeZero()) + }) + + It("records a constraint that reduces a saved budget without rewriting its source", func() { + saved := captainconfig.AIDefaults{BudgetUSD: 4} + limit := SpecLayer{Name: "organization", Scope: SpecLayerGlobal, Constraints: RuntimeConstraints{Limits: RunLimits{Budget: Budget{Cost: 2}}}} + result, err := ComposeSpecLayers(ResolveSpecOptions{Saved: &saved, Layers: []SpecLayer{limit}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Spec.Budget.Cost).To(Equal(float64(2))) + Expect(result.Provenance["/budget/cost"]).To(Equal(FieldProvenance{ + Source: FieldSource{Kind: FieldSourceSaved, Name: "~/.captain.yaml", Key: "ai.budgetUSD"}, + NormalizedBy: &FieldSource{Kind: FieldSourceLayer, Name: "organization", Key: "/constraints/limits/budget/cost"}, + })) + }) + + It("preserves the native MCP wire form and saved toggle provenance", func() { + saved := captainconfig.AIDefaults{NoMCP: true} + result, err := ComposeSpecLayers(ResolveSpecOptions{Saved: &saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Spec.Permissions.MCP.Disabled).To(BeTrue()) + Expect(result.Provenance["/permissions/mcp/disabled"].Source.Key).To(Equal("ai.noMCP")) + encoded, err := json.Marshal(result.Spec) + Expect(err).NotTo(HaveOccurred()) + var again Spec + Expect(json.Unmarshal(encoded, &again)).To(Succeed()) + Expect(again.Permissions.MCP.Disabled).To(BeTrue()) + request := RequestSpecLayer("request", (Spec{Permissions: Permissions{MCP: MCP{Modes: ResourcePolicies{"example": ResourceEnabled}}}}).WithExplicit("/permissions/mcp/disabled")) + result, err = ComposeSpecLayers(ResolveSpecOptions{Saved: &saved, Layers: []SpecLayer{request}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Spec.Permissions.MCP.Disabled).To(BeFalse()) + Expect(result.Spec.Permissions.MCP.Modes["example"]).To(Equal(ResourceEnabled)) + }) + + It("attributes invalid saved declarations even when a request replaces them", func() { + for _, saved := range []captainconfig.AIDefaults{{Temperature: 3}, {DefaultModel: "unknown-example-model"}} { + _, err := ResolveSpecLayers(ResolveSpecOptions{Saved: &saved, Layers: []SpecLayer{RequestSpecLayer("request", Spec{Model: Model{Name: "agent:sonnet", Temperature: floatPtr(1)}})}}) + var configured *SavedDefaultsError + Expect(errors.As(err, &configured)).To(BeTrue()) + Expect(configured.Source).To(Equal("~/.captain.yaml ai")) + } + saved := captainconfig.AIDefaults{} + _, err := ResolveSpecLayers(ResolveSpecOptions{Saved: &saved, Layers: []SpecLayer{RequestSpecLayer("request", Spec{Model: Model{Name: "unknown-example-model"}})}}) + var configured *SavedDefaultsError + Expect(errors.As(err, &configured)).To(BeFalse()) + Expect(err).To(HaveOccurred()) + }) + + It("keeps unknown authored models visible in saved previews while final resolution rejects them", func() { + saved := captainconfig.AIDefaults{MaxTokens: 500} + layer := RequestSpecLayer("request", Spec{Model: Model{Name: "unknown-example-model"}}) + composed, err := ComposeSpecLayers(ResolveSpecOptions{Saved: &saved, Layers: []SpecLayer{layer}}) + Expect(err).NotTo(HaveOccurred()) + Expect(composed.Spec.Name).To(Equal("unknown-example-model")) + Expect(composed.Spec.Budget.MaxTokens).To(Equal(500)) + Expect(composed.Trace).To(Equal([]SpecLayer{layer})) + _, err = ResolveSpecLayers(ResolveSpecOptions{Saved: &saved, Layers: []SpecLayer{layer}}) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/pkg/api/spec_layer_validation_ginkgo_test.go b/pkg/api/spec_layer_validation_ginkgo_test.go index 0d86dfc0..7630666d 100644 --- a/pkg/api/spec_layer_validation_ginkgo_test.go +++ b/pkg/api/spec_layer_validation_ginkgo_test.go @@ -17,7 +17,7 @@ var _ = Describe("Structural spec layer validation", func() { Expect(errors.As(err, &structural)).To(BeTrue()) Expect(structural.Layer).To(Equal("project.prompt")) Expect(err.Error()).To(ContainSubstring(fragment)) - _, err = ResolveSpecLayers(layer, RequestSpecLayer("request", Spec{Model: Model{Name: "agent:sonnet"}, Budget: Budget{Timeout: "1m"}})) + _, err = ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer, RequestSpecLayer("request", Spec{Model: Model{Name: "agent:sonnet"}, Budget: Budget{Timeout: "1m"}})}}) Expect(errors.As(err, &structural)).To(BeTrue()) }, Entry("budget", Spec{Budget: Budget{Timeout: "tomorrow"}}, "timeout"), @@ -51,7 +51,7 @@ var _ = Describe("Structural spec layer validation", func() { PromptSpecLayer("unknown.prompt", Spec{Model: Model{Name: "unregistered-model"}}), } Expect(ValidateSpecLayers(layers...)).To(Succeed()) - composed, err := ComposeSpecLayers(layers...) + composed, err := ComposeSpecLayers(ResolveSpecOptions{Layers: layers}) Expect(err).NotTo(HaveOccurred()) Expect(composed.Spec.Model).To(Equal(Model{Name: "unregistered-model", Mode: ModeAgent, Effort: EffortHigh})) Expect(composed.Trace).To(Equal(layers)) diff --git a/pkg/api/spec_layers.go b/pkg/api/spec_layers.go index 1471c5ab..d95afa24 100644 --- a/pkg/api/spec_layers.go +++ b/pkg/api/spec_layers.go @@ -65,10 +65,11 @@ type SpecLayer struct { // ResolvedSpec is Captain's effective runtime profile plus ordered provenance. type ResolvedSpec struct { - Spec Spec `json:"spec" yaml:"spec"` - Constraints RuntimeConstraints `json:"constraints" yaml:"constraints"` - Trace []SpecLayer `json:"trace" yaml:"trace"` - Warnings []string `json:"warnings,omitempty" yaml:"warnings,omitempty"` + Spec Spec `json:"spec" yaml:"spec"` + Constraints RuntimeConstraints `json:"constraints" yaml:"constraints"` + Trace []SpecLayer `json:"trace" yaml:"trace"` + Warnings []string `json:"warnings,omitempty" yaml:"warnings,omitempty"` + Provenance map[string]FieldProvenance `json:"provenance,omitempty" yaml:"provenance,omitempty"` } // PromptSpecLayer adapts parsed .prompt frontmatter into the normal surface layer. @@ -92,13 +93,15 @@ func OrderSpecLayers(input ...SpecLayer) []SpecLayer { } // ComposeSpecLayers overlays raw defaults and constraints without resolving a runtime. -func ComposeSpecLayers(input ...SpecLayer) (ComposedSpec, error) { - if err := ValidateSpecLayers(input...); err != nil { +func ComposeSpecLayers(options ResolveSpecOptions) (ComposedSpec, error) { + if err := ValidateSpecLayers(options.Layers...); err != nil { return ComposedSpec{}, err } - layers := OrderSpecLayers(input...) - resolved := ComposedSpec{Trace: make([]SpecLayer, 0, len(layers))} + layers := OrderSpecLayers(options.Layers...) + limitSources := budgetLimitSources{} + resolved := ComposedSpec{Trace: make([]SpecLayer, 0, len(layers)), Provenance: map[string]FieldProvenance{}} for _, layer := range layers { + resolved.recordLayer(layer) resolved.Spec = resolved.Spec.Merge(layer.Spec) if len(layer.Constraints.Models) > 0 { resolved.Constraints.Models = intersectModels(resolved.Constraints.Models, layer.Constraints.Models) @@ -111,6 +114,7 @@ func ComposeSpecLayers(input ...SpecLayer) (ComposedSpec, error) { return ComposedSpec{}, fmt.Errorf("spec layer %q limits: %w", layer.Name, err) } resolved.Constraints.Limits = limits + limitSources.record(layer, limits.Budget) for _, quota := range layer.Constraints.Quotas { quota.Name = strings.TrimSpace(quota.Name) quota.Scope = layer.Scope @@ -120,11 +124,20 @@ func ComposeSpecLayers(input ...SpecLayer) (ComposedSpec, error) { resolved.Trace = append(resolved.Trace, cloneSpecLayer(layer)) } + if options.Saved != nil || options.RequireModel || options.Normalize != nil { + if err := resolved.expandModel(); err != nil { + return ComposedSpec{}, err + } + } + if err := resolved.applyDefaults(options); err != nil { + return ComposedSpec{}, err + } budget, err := strictBudget(resolved.Spec.Budget, resolved.Constraints.Limits.Budget) if err != nil { return ComposedSpec{}, fmt.Errorf("effective run budget: %w", err) } resolved.Spec.Budget = budget + resolved.recordLimits(limitSources) return resolved, nil } diff --git a/pkg/api/spec_layers_ginkgo_test.go b/pkg/api/spec_layers_ginkgo_test.go index abf35f78..aa8c78f1 100644 --- a/pkg/api/spec_layers_ginkgo_test.go +++ b/pkg/api/spec_layers_ginkgo_test.go @@ -23,7 +23,7 @@ var _ = Describe("Hierarchical spec profiles", func() { Spec: Spec{Model: Model{Effort: EffortLow}}, } - resolved, err := ResolveSpecLayers(user, surface, context, global) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{user, surface, context, global}}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Spec.Model.Name).To(Equal("claude-sonnet-5")) @@ -54,7 +54,7 @@ var _ = Describe("Hierarchical spec profiles", func() { }, } - resolved, err := ResolveSpecLayers(layers...) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: layers}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Constraints.Models).To(Equal([]string{"claude-sonnet-5", "gpt-5.6-sol"})) @@ -62,12 +62,12 @@ var _ = Describe("Hierarchical spec profiles", func() { Expect(resolved.AllowsModel(Model{Name: "gpt-5.6-sol"})).To(BeTrue()) layers[2].Spec.Model.Fallbacks = []Model{{Name: "gpt-5.4"}} - _, err = ResolveSpecLayers(layers...) + _, err = ResolveSpecLayers(ResolveSpecOptions{Layers: layers}) Expect(err).To(MatchError(ContainSubstring(`fallback model "gpt-5.4" is outside the effective model catalog`))) }) It("normalizes model selectors before intersecting catalogs", func() { - resolved, err := ResolveSpecLayers( + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{ SpecLayer{ Name: "platform", Scope: SpecLayerGlobal, Constraints: RuntimeConstraints{Models: []string{" gpt-5.4 "}}, @@ -76,14 +76,14 @@ var _ = Describe("Hierarchical spec profiles", func() { Name: "claims", Scope: SpecLayerContext, Constraints: RuntimeConstraints{Models: []string{"gpt-5.4"}}, }, - ) + }}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Constraints.Models).To(Equal([]string{"gpt-5.4"})) }) It("uses strict non-zero run ceilings and retains each named quota independently", func() { - resolved, err := ResolveSpecLayers( + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{ SpecLayer{ Name: "platform", Scope: SpecLayerGlobal, Spec: Spec{Budget: Budget{Cost: 12, MaxTokens: 9000, MaxTurns: 10, Timeout: "10m"}}, @@ -100,7 +100,7 @@ var _ = Describe("Hierarchical spec profiles", func() { Quotas: []UsageQuota{{Name: "claims-monthly", CostLimitUSD: 50, CostUsedUSD: 2}}, }, }, - ) + }}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Spec.Budget).To(Equal(Budget{Cost: 5, MaxTokens: 6000, MaxTurns: 6, Timeout: "5m"})) diff --git a/pkg/api/spec_merge.go b/pkg/api/spec_merge.go index e4d4ae4c..95d6a1b2 100644 --- a/pkg/api/spec_merge.go +++ b/pkg/api/spec_merge.go @@ -1,6 +1,10 @@ package api import ( + "reflect" + "sort" + "strings" + "github.com/flanksource/captain/pkg/api/registry" "github.com/flanksource/commons/merge" ) @@ -29,8 +33,8 @@ func MergePolicy() merge.Policy { }) } -// Merge returns a copy of s with override's set (non-zero) fields taking -// precedence. A zero-valued field in override is treated as "unset" and keeps +// Merge returns a copy of s with override's supplied fields taking precedence. +// A zero-valued field without explicit presence is "unset" and keeps // s's value, so a base spec can supply defaults that an operation-specific spec // selectively overrides: // @@ -40,21 +44,59 @@ func MergePolicy() merge.Policy { // slices are replaced wholesale when the override's is non-empty, maps merge // key-wise, and structs — including Setup, Workflow and Permissions.Tools behind // their pointers — merge field by field, so setting one sub-field does not erase -// its siblings. Boolean toggles follow zero=unset: an override can turn a flag on -// but not off, since false is indistinguishable from absent. +// its siblings. Decoded explicit zero values, or fields marked by WithExplicit, +// replace inherited values, including false booleans and empty collections. // // Neither operand is mutated and the result shares no mutable memory with // either, so a merged spec can be edited without reaching back into the config // it inherited from. func (s Spec) Merge(override Spec) Spec { + s = s.withoutReplacedPresence(override) merged := merge.Apply(s, override, MergePolicy()) if s.Sandbox != nil && override.Sandbox != nil && s.Sandbox.Mode == override.Sandbox.Mode { resolved := merge.Apply(*s.Sandbox, *override.Sandbox, MergePolicy()) merged.Sandbox = &resolved } + cloned := merge.Apply(Spec{}, override, MergePolicy()) + paths := make([]string, 0, len(override.explicitFields())) + for path := range override.explicitFields() { + paths = append(paths, path) + } + sort.Strings(paths) + for _, path := range paths { + tokens := strings.Split(strings.TrimPrefix(path, "/"), "/") + source := serializedField(reflect.ValueOf(cloned), tokens) + target := serializedField(reflect.ValueOf(&merged).Elem(), tokens) + if source.IsValid() && target.IsValid() && target.CanSet() { + target.Set(source) + } + } return merged } +func (s Spec) withoutReplacedPresence(override Spec) Spec { + s.Explicit = s.Explicit.Clone() + s.Model.Explicit = s.Model.Explicit.Clone() + for path := range override.Fields() { + value := serializedField(reflect.ValueOf(override), strings.Split(strings.TrimPrefix(path, "/"), "/")) + if !replacesField(value) && path != "/toolApproval" && (path != "/sandbox" || s.Sandbox == nil || override.Sandbox == nil || s.Sandbox.Mode == override.Sandbox.Mode) { + continue + } + for _, fields := range []FieldPresence{s.Explicit, s.Model.Explicit} { + for previous := range fields { + if previous == path || strings.HasPrefix(previous, path+"/") { + delete(fields, previous) + } + } + } + } + return s +} + +func replacesField(value reflect.Value) bool { + return value.IsValid() && (value.Kind() == reflect.Slice || value.Kind() == reflect.Map && value.Len() == 0 || value.Kind() == reflect.Pointer && value.IsNil()) +} + // WithoutSession returns s stripped of everything that binds it to a prior // conversation: the session to resume, pending tool-approval resume state, and // canonical message history. @@ -68,5 +110,13 @@ func (s Spec) WithoutSession() Spec { s.SessionID = "" s.ToolApproval = nil s.Messages = nil + s.Explicit = s.Explicit.Clone() + for path := range s.Explicit { + for _, removed := range []string{"/sessionId", "/toolApproval", "/messages"} { + if path == removed || strings.HasPrefix(path, removed+"/") { + delete(s.Explicit, path) + } + } + } return s } diff --git a/pkg/api/spec_merge_differential_test.go b/pkg/api/spec_merge_differential_test.go index feb059a6..dd2b12d0 100644 --- a/pkg/api/spec_merge_differential_test.go +++ b/pkg/api/spec_merge_differential_test.go @@ -374,6 +374,10 @@ func randomValue(rnd *rand.Rand, v reflect.Value, depth int) { if !v.CanSet() || depth > randomMaxDepth { return } + // Authorship metadata is exercised by presence tests, not arbitrary keys. + if v.Type() == reflect.TypeOf(FieldPresence(nil)) { + return + } // Raw JSON is a []byte to Go but not to anything that reads it, and the // diagnostics on failure marshal the spec — random bytes would fail there // instead of at the comparison. diff --git a/pkg/api/spec_normalization.go b/pkg/api/spec_normalization.go new file mode 100644 index 00000000..1dd9d2f5 --- /dev/null +++ b/pkg/api/spec_normalization.go @@ -0,0 +1,53 @@ +package api + +import ( + "fmt" + "reflect" + "strings" +) + +func (composed *ComposedSpec) normalize(normalize func(Spec) (SpecNormalization, error)) (*SpecNormalization, error) { + if normalize == nil { + return nil, nil + } + normalized, err := normalize(Spec{}.Merge(composed.Spec)) + if err != nil { + return nil, err + } + if err := normalized.Spec.ValidateStructure(); err != nil { + return nil, fmt.Errorf("normalized spec: %w", err) + } + for path, present := range normalized.Fields { + if !present { + continue + } + if !serializedField(reflect.ValueOf(normalized.Spec), strings.Split(strings.TrimPrefix(path, "/"), "/")).IsValid() { + return nil, fmt.Errorf("normalization declares unknown field %q", path) + } + normalized.Spec = normalized.Spec.WithExplicit(path) + } + composed.Spec = normalized.Spec + return &normalized, nil +} + +func (composed *ComposedSpec) recordContext(normalized *SpecNormalization) { + if normalized == nil { + return + } + for path, present := range normalized.Fields { + if !present { + continue + } + provenance, exists := composed.Provenance[path] + source := normalized.Source + if source.Key == "" { + source.Key = path + } + if exists { + provenance.NormalizedBy = &source + } else { + provenance.Source = source + } + composed.Provenance[path] = provenance + } +} diff --git a/pkg/api/spec_normalization_ginkgo_test.go b/pkg/api/spec_normalization_ginkgo_test.go new file mode 100644 index 00000000..6b20835b --- /dev/null +++ b/pkg/api/spec_normalization_ginkgo_test.go @@ -0,0 +1,110 @@ +package api + +import ( + "github.com/flanksource/captain/pkg/captainconfig" + "github.com/flanksource/commons-db/shell" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Spec runtime context normalization", func() { + It("attributes fallback inheritance to a primary field derived by runtime context", func() { + saved := captainconfig.AIDefaults{DefaultModel: "agent:sonnet,agent:haiku"} + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Saved: &saved, Normalize: func(spec Spec) (SpecNormalization, error) { + spec.Temperature = floatPtr(0.2) + return SpecNormalization{Spec: spec, Fields: FieldPresence{"/temperature": true}, Source: FieldSource{Kind: FieldSourceContext, Name: "runtime context"}}, nil + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Fallbacks[0].Temperature).To(Equal(floatPtr(0.2))) + Expect(resolved.Provenance["/fallbacks/0/temperature"].Source).To(Equal(FieldSource{Kind: FieldSourceContext, Name: "runtime context", Key: "/temperature"})) + }) + + It("normalizes saved primary and fallback selections once before provider mode defaults", func() { + saved := captainconfig.AIDefaults{DefaultModel: "api:sonnet,api:sol"} + calls := 0 + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Saved: &saved, RequireModel: true, Normalize: func(spec Spec) (SpecNormalization, error) { + calls++ + Expect(spec.Name).To(Equal("sonnet")) + Expect(spec.Fallbacks).To(HaveLen(1)) + Expect(spec.Mode).To(BeEmpty()) + Expect(spec.Fallbacks[0].Mode).To(BeEmpty()) + spec.Mode, spec.Fallbacks[0].Mode = ModeCLI, ModeCLI + return SpecNormalization{Spec: spec, Fields: FieldPresence{"/mode": true, "/fallbacks/0/mode": true}, Source: FieldSource{Kind: FieldSourceContext, Name: "runtime context"}}, nil + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(calls).To(Equal(1)) + Expect(resolved.Spec.Mode).To(Equal(ModeCLI)) + Expect(resolved.Spec.Fallbacks[0].Mode).To(Equal(ModeCLI)) + Expect(resolved.Provenance["/fallbacks/0/model"].Source.Key).To(Equal("ai.defaultModel")) + Expect(resolved.Provenance["/fallbacks/0/mode"].Source.Kind).To(Equal(FieldSourceContext)) + Expect(resolved.Trace).To(BeEmpty()) + }) + + It("preserves higher authored effort over a lower compact selector and its field ownership", func() { + layers := []SpecLayer{PromptSpecLayer("project", Spec{Model: Model{Name: "agent:sol:high"}}), RequestSpecLayer("request", Spec{Model: Model{Effort: EffortLow}})} + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: layers, Saved: &captainconfig.AIDefaults{}}) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Effort).To(Equal(EffortLow)) + Expect(resolved.Provenance["/effort"].Source.Name).To(Equal("request")) + Expect(resolved.Provenance["/effort"].Source.Key).To(Equal("/effort")) + Expect(resolved.Trace).To(Equal(layers)) + }) + + It("expands authored CSV candidates before applying runtime context", func() { + layer := RequestSpecLayer("request", Spec{Model: Model{Name: "sonnet,sol", Fallbacks: []Model{{Name: "haiku"}}}}) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer}, Normalize: func(spec Spec) (SpecNormalization, error) { + Expect(spec.Name).To(Equal("sonnet")) + Expect(spec.Fallbacks).To(HaveLen(2)) + spec.Mode = ModeCLI + for i := range spec.Fallbacks { + spec.Fallbacks[i].Mode = ModeCLI + } + return SpecNormalization{Spec: spec, Fields: FieldPresence{"/mode": true, "/fallbacks/0/mode": true, "/fallbacks/1/mode": true}, Source: FieldSource{Kind: FieldSourceContext, Name: "runtime context"}}, nil + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Provenance["/fallbacks/0/model"].Source.Key).To(Equal("/model")) + Expect(resolved.Provenance["/fallbacks/1/model"].Source.Key).To(Equal("/fallbacks/0/model")) + Expect(resolved.Trace).To(Equal([]SpecLayer{layer})) + }) + + It("keeps named sandbox references structurally composable until context resolves them", func() { + layer := RequestSpecLayer("request", Spec{Model: Model{Name: "sonnet", Mode: ModeCLI}, Sandbox: &SandboxRef{Backend: "review-pool"}}) + Expect(ValidateSpecLayers(layer)).To(Succeed()) + _, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer}}) + Expect(err).To(HaveOccurred()) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer}, Normalize: func(spec Spec) (SpecNormalization, error) { + spec.Sandbox.Mode = SandboxDocker + return SpecNormalization{Spec: spec, Fields: FieldPresence{"/sandbox/mode": true}, Source: FieldSource{Kind: FieldSourceContext, Name: "runtime context"}}, nil + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Sandbox).To(Equal(&SandboxRef{Mode: SandboxDocker, Backend: "review-pool"})) + Expect(resolved.Trace).To(Equal([]SpecLayer{layer})) + }) + + It("normalizes the complete authored stack before saved gaps without replacing source ownership", func() { + saved := captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{"anthropic": {Mode: "api"}}} + layer := PromptSpecLayer("project", Spec{Model: Model{Name: "sonnet"}, Setup: &shell.Setup{Cwd: ".", DotEnv: []string{".env"}, Env: []string{"EXAMPLE=value"}}}) + calls := 0 + result, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer}, Saved: &saved, + Normalize: func(spec Spec) (SpecNormalization, error) { + calls++ + Expect(spec.Mode).To(BeEmpty()) + spec.Setup.Cwd = "/project" + spec.Mode = ModeAgent + return SpecNormalization{Spec: spec, Fields: FieldPresence{"/setup/cwd": true, "/mode": true}, Source: FieldSource{Kind: FieldSourceContext, Name: "runtime context"}}, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(calls).To(Equal(1)) + Expect(result.Spec.Mode).To(Equal(ModeAgent)) + Expect(result.Spec.Setup.Cwd).To(Equal("/project")) + Expect(result.Provenance["/setup/cwd"].Source.Name).To(Equal("project")) + Expect(result.Provenance["/setup/cwd"].NormalizedBy.Name).To(Equal("runtime context")) + Expect(result.Provenance["/setup/cwd"].NormalizedBy.Key).To(Equal("/setup/cwd")) + Expect(result.Provenance["/setup/dotenv"].Source.Name).To(Equal("project")) + Expect(result.Provenance["/setup/dotenv"].NormalizedBy).To(BeNil()) + Expect(result.Spec.Setup.Env).To(Equal(layer.Spec.Setup.Env)) + Expect(result.Trace).To(Equal([]SpecLayer{layer})) + Expect(layer.Spec.Setup.Cwd).To(Equal(".")) + }) +}) diff --git a/pkg/api/spec_presence.go b/pkg/api/spec_presence.go new file mode 100644 index 00000000..cef30a1e --- /dev/null +++ b/pkg/api/spec_presence.go @@ -0,0 +1,184 @@ +package api + +import ( + "encoding/json" + "reflect" + "strings" + + "github.com/flanksource/captain/pkg/api/registry" +) + +type FieldPresence = registry.FieldPresence + +type specModelWire Model + +// ModelFields exposes the plain model shape to enclosing field inspectors. +type ModelFields = specModelWire + +// WithExplicit marks zero values authored by a programmatic caller. +func (s Spec) WithExplicit(paths ...string) Spec { + s.Explicit = s.Explicit.Clone() + if s.Explicit == nil { + s.Explicit = FieldPresence{} + } + for _, path := range paths { + s.Explicit[path] = true + } + return s +} + +// Fields reports authored values, including explicitly present zero values. +func (s Spec) Fields() FieldPresence { + fields := s.explicitFields() + collectPresentFields(reflect.ValueOf(s), "", fields) + return fields +} + +func (s Spec) explicitFields() FieldPresence { + fields := s.Explicit.Clone() + if fields == nil { + fields = FieldPresence{} + } + for path, present := range s.Model.Explicit { + fields[path] = present + } + return fields +} + +func collectPresentFields(value reflect.Value, path string, fields FieldPresence) { + for value.IsValid() && (value.Kind() == reflect.Pointer || value.Kind() == reflect.Interface) { + if value.IsNil() { + return + } + value = value.Elem() + if value.IsZero() && path != "" { + fields[path] = true + } + } + if !value.IsValid() { + return + } + if value.CanInterface() { + if raw, ok := value.Interface().(json.RawMessage); ok { + if len(raw) > 0 { + fields[path] = true + } + return + } + if mcp, ok := value.Interface().(MCP); ok { + if mcp.Disabled { + fields[path+"/disabled"] = true + } + if len(mcp.Servers) > 0 { + fields[path+"/servers"] = true + } + for name := range mcp.Modes { + fields[path+"/"+escapeField(name)] = true + } + return + } + if model, ok := value.Interface().(Model); ok { + for field := range model.Fields() { + fields[path+field] = true + } + for i, fallback := range model.Fallbacks { + collectPresentFields(reflect.ValueOf(fallback), fmtFieldIndex(path+"/fallbacks", i), fields) + } + return + } + } + switch value.Kind() { + case reflect.Struct: + for i := range value.NumField() { + field := value.Type().Field(i) + name := strings.Split(field.Tag.Get("json"), ",")[0] + if name == "-" || !field.IsExported() { + continue + } + if field.Anonymous && name == "" { + collectPresentFields(value.Field(i), path, fields) + } else if name != "" { + collectPresentFields(value.Field(i), path+"/"+escapeField(name), fields) + } + } + case reflect.Map: + for _, key := range value.MapKeys() { + fieldPath := path + "/" + escapeField(key.String()) + fields[fieldPath] = true + collectPresentFields(value.MapIndex(key), fieldPath, fields) + } + case reflect.Slice, reflect.Array: + if value.Len() > 0 || value.Kind() == reflect.Slice && !value.IsNil() { + fields[path] = true + } + for i := range value.Len() { + collectPresentFields(value.Index(i), fmtFieldIndex(path, i), fields) + } + default: + if !value.IsZero() { + fields[path] = true + } + } +} + +func capturePresence(value any, path string, fields FieldPresence) { + switch typed := value.(type) { + case map[string]any: + if len(typed) == 0 && path != "" { + fields[path] = true + } + for key, child := range typed { + capturePresence(child, path+"/"+escapeField(key), fields) + } + case []any: + if len(typed) == 0 { + fields[path] = true + } + for i, child := range typed { + capturePresence(child, fmtFieldIndex(path, i), fields) + } + default: + if value == nil || reflect.ValueOf(value).IsZero() { + fields[path] = true + } + } +} + +func (s *Spec) capturePresence(data []byte) error { + var fields map[string]any + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + s.Explicit = FieldPresence{} + capturePresence(fields, "", s.Explicit) + s.pruneOpaquePresence() + if len(s.Explicit) == 0 { + s.Explicit = nil + } + return nil +} + +func (s *Spec) pruneOpaquePresence() { + for path := range s.Explicit { + tokens := strings.Split(strings.TrimPrefix(path, "/"), "/") + if !serializedField(reflect.ValueOf(*s), tokens).IsValid() { + delete(s.Explicit, path) + continue + } + for i := 1; i < len(tokens); i++ { + parent := serializedField(reflect.ValueOf(*s), tokens[:i]) + if parent.IsValid() && parent.Type() == reflect.TypeOf(json.RawMessage(nil)) { + delete(s.Explicit, path) + break + } + } + } +} + +func escapeField(value string) string { + return strings.ReplaceAll(strings.ReplaceAll(value, "~", "~0"), "/", "~1") +} + +func unescapeField(value string) string { + return strings.ReplaceAll(strings.ReplaceAll(value, "~1", "/"), "~0", "~") +} diff --git a/pkg/api/spec_presence_ginkgo_test.go b/pkg/api/spec_presence_ginkgo_test.go new file mode 100644 index 00000000..0e1e62ad --- /dev/null +++ b/pkg/api/spec_presence_ginkgo_test.go @@ -0,0 +1,113 @@ +package api + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +var _ = Describe("Spec field presence", func() { + DescribeTable("preserves false zero and empty values through the full wire", func(decode func([]byte, any) error, encode func(any) ([]byte, error), data string) { + var spec Spec + Expect(decode([]byte(data), &spec)).To(Succeed()) + Expect(spec.Prompt.User).To(Equal("review")) + Expect(spec.Budget.MaxTurns).To(Equal(4)) + Expect(spec.Fields()).To(HaveKeyWithValue("/noCache", true)) + Expect(spec.Fields()).To(HaveKeyWithValue("/budget/cost", true)) + Expect(spec.Fields()).To(HaveKeyWithValue("/fallbacks", true)) + Expect(spec.Fields()).To(HaveKeyWithValue("/memory/skipSkills", true)) + encoded, err := encode(spec) + Expect(err).NotTo(HaveOccurred()) + var again Spec + Expect(decode(encoded, &again)).To(Succeed()) + Expect(again).To(Equal(spec)) + }, + Entry("JSON", json.Unmarshal, json.Marshal, `{"model":"agent:sonnet","noCache":false,"fallbacks":[],"budget":{"cost":0,"maxTurns":4},"memory":{"skipSkills":false},"prompt":{"user":"review"}}`), + Entry("YAML", yaml.Unmarshal, yaml.Marshal, "model: agent:sonnet\nnoCache: false\nfallbacks: []\nbudget: {cost: 0, maxTurns: 4}\nmemory: {skipSkills: false}\nprompt: review\n"), + ) + + It("merges intentional zero values without losing sibling fields or mutating input", func() { + base := Spec{Model: Model{Name: "sonnet", NoCache: true, Fallbacks: []Model{{Name: "sol"}}}, Budget: Budget{Cost: 2, MaxTurns: 4}, Memory: Memory{SkipSkills: true}} + override := (Spec{}).WithExplicit("/noCache", "/fallbacks", "/budget/cost", "/memory/skipSkills") + merged := base.Merge(override) + Expect(merged.NoCache).To(BeFalse()) + Expect(merged.Fallbacks).To(BeEmpty()) + Expect(merged.Budget).To(Equal(Budget{MaxTurns: 4})) + Expect(merged.Memory.SkipSkills).To(BeFalse()) + Expect(base.NoCache).To(BeTrue()) + Expect(base.Fallbacks).To(HaveLen(1)) + Expect(IsEmpty(override)).To(BeFalse()) + }) + + It("removes session presence when deriving an independent run", func() { + spec := (Spec{SessionID: "old-session"}).WithExplicit("/sessionId", "/messages", "/toolApproval") + independent := spec.WithoutSession() + Expect(independent.Fields()).NotTo(HaveKey("/sessionId")) + Expect(independent.Fields()).NotTo(HaveKey("/messages")) + Expect(independent.Fields()).NotTo(HaveKey("/toolApproval")) + Expect(IsEmpty(independent)).To(BeTrue()) + }) + + It("replaces fallback lists without retaining the replaced entries' presence", func() { + var base Spec + Expect(json.Unmarshal([]byte(`{"fallbacks":[{"model":"sonnet","noCache":false},{"model":"sol","noCache":false}]}`), &base)).To(Succeed()) + merged := base.Merge(Spec{Model: Model{Fallbacks: []Model{{Name: "haiku"}}}}) + encoded, err := json.Marshal(merged) + Expect(err).NotTo(HaveOccurred()) + var again Spec + Expect(json.Unmarshal(encoded, &again)).To(Succeed()) + Expect(again.Fallbacks).To(HaveLen(1)) + Expect(again.Fields()).NotTo(HaveKey("/fallbacks/1/noCache")) + Expect(again.Fallbacks[0].Fields()).NotTo(HaveKey("/noCache")) + }) + + DescribeTable("delegates unknown field policy to the enclosing decoder", func(decode func([]byte, any) error, data string) { + var spec Spec + Expect(decode([]byte(data), &spec)).To(Succeed()) + encoded, err := json.Marshal(spec) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).NotTo(ContainSubstring("unexpected")) + }, + Entry("JSON budget", json.Unmarshal, `{"budget":{"unexpected":0}}`), + Entry("JSON fallback", json.Unmarshal, `{"fallbacks":[{"model":"sonnet","unexpected":1}]}`), + Entry("YAML budget", yaml.Unmarshal, "budget: {unexpected: 0}"), + Entry("YAML fallback", yaml.Unmarshal, "fallbacks: [{model: sonnet, unexpected: 1}]"), + ) + + It("preserves reusable preset groups and zero presence through both wire formats", func() { + for _, encoding := range []struct { + Decode func([]byte, any) error + Encode func(any) ([]byte, error) + Input string + }{ + {json.Unmarshal, json.Marshal, `{"model":"sonnet","budget":{"cost":0,"maxTurns":3},"memory":{"skipSkills":false},"setup":{"checkout":{"worktree":{"keep":false}}}}`}, + {yaml.Unmarshal, yaml.Marshal, "model: sonnet\nbudget: {cost: 0, maxTurns: 3}\nmemory: {skipSkills: false}\nsetup: {checkout: {worktree: {keep: false}}}"}, + } { + var preset RuntimePresetSpec + Expect(encoding.Decode([]byte(encoding.Input), &preset)).To(Succeed()) + Expect(preset.Budget.MaxTurns).To(Equal(3)) + Expect(preset.ToSpec().Fields()).To(HaveKey("/budget/cost")) + Expect(preset.ToSpec().Fields()).To(HaveKey("/setup/checkout/worktree/keep")) + encoded, err := encoding.Encode(preset) + Expect(err).NotTo(HaveOccurred()) + var again RuntimePresetSpec + Expect(encoding.Decode(encoded, &again)).To(Succeed()) + Expect(again).To(Equal(preset)) + } + }) + + DescribeTable("preserves exact numeric values inside raw structured-output schemas", func(encode func(any) ([]byte, error), decode func([]byte, any) error) { + schema := json.RawMessage(`{"const":9007199254740993,"additionalProperties":false,"minimum":0}`) + spec := Spec{Prompt: Prompt{User: "return the constant", SchemaJSON: schema}} + encoded, err := encode(spec) + Expect(err).NotTo(HaveOccurred()) + var again Spec + Expect(decode(encoded, &again)).To(Succeed()) + Expect(again.Prompt.SchemaJSON).To(MatchJSON(schema)) + encoded, err = json.Marshal(again) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).To(ContainSubstring(`9007199254740993`)) + }, Entry("JSON", json.Marshal, json.Unmarshal), Entry("YAML", yaml.Marshal, yaml.Unmarshal)) +}) diff --git a/pkg/api/spec_provenance.go b/pkg/api/spec_provenance.go new file mode 100644 index 00000000..c6806bf7 --- /dev/null +++ b/pkg/api/spec_provenance.go @@ -0,0 +1,160 @@ +package api + +import ( + "reflect" + "strconv" + "strings" +) + +func (composed *ComposedSpec) recordLayer(layer SpecLayer) { + if composed.fieldLayers == nil { + composed.fieldLayers = map[string]int{} + } + for _, path := range sortedKeys(layer.Spec.Fields()) { + composed.fieldLayers[path] = len(composed.Trace) + 1 + value := serializedField(reflect.ValueOf(layer.Spec), strings.Split(strings.TrimPrefix(path, "/"), "/")) + if replacesField(value) { + for previous := range composed.Provenance { + if strings.HasPrefix(previous, path+"/") { + delete(composed.Provenance, previous) + } + } + } + composed.Provenance[path] = FieldProvenance{Source: FieldSource{Kind: FieldSourceLayer, Name: layer.Name, Key: path, LayerID: layer.ID}} + } +} + +func (composed *ComposedSpec) expandProvenance() error { + raw := composed.Spec.Model + nameSource := composed.Provenance["/model"] + parts := strings.Split(raw.Name, ",") + if count := len(parts) - 1; count > 0 { + shifted := map[string]FieldProvenance{} + for path, source := range composed.Provenance { + if suffix, ok := strings.CutPrefix(path, "/fallbacks/"); ok { + index, rest, _ := strings.Cut(suffix, "/") + if i, err := strconv.Atoi(index); err == nil { + shifted[fmtFieldIndex("/fallbacks", i+count)+"/"+rest] = source + delete(composed.Provenance, path) + } + } + } + for path, source := range shifted { + composed.Provenance[path] = source + } + for i, part := range parts[1:] { + model, err := (Model{Name: strings.TrimSpace(part)}).Expand() + if err != nil { + return err + } + for path := range model.Fields() { + composed.Provenance[fmtFieldIndex("/fallbacks", i)+path] = nameSource + } + } + if _, found := composed.Provenance["/fallbacks"]; !found { + composed.Provenance["/fallbacks"] = nameSource + } + } + primary, err := (Model{Name: parts[0]}).Expand() + if err != nil { + return err + } + for _, field := range []string{"/mode", "/effort"} { + if field == "/effort" && composed.fieldLayers[field] > composed.fieldLayers["/model"] { + continue + } + if primary.Fields().Has(field) { + composed.Provenance[field] = nameSource + } + } + for i, fallback := range raw.Fallbacks { + expanded, err := (Model{Name: fallback.Name}).Expand() + if err != nil { + return err + } + prefix := fmtFieldIndex("/fallbacks", i+len(parts)-1) + for _, field := range []string{"/mode", "/effort"} { + if expanded.Fields().Has(field) { + composed.Provenance[prefix+field] = composed.Provenance[prefix+"/model"] + } + } + } + return nil +} + +func (composed *ComposedSpec) expandModel() error { + if err := composed.expandProvenance(); err != nil { + return err + } + raw := composed.Spec.modelWithPresence() + expanded, err := raw.Expand() + if err != nil { + return err + } + if composed.fieldLayers["/effort"] > composed.fieldLayers["/model"] { + expanded.Effort = raw.Effort + expanded = expanded.WithExplicit("/effort") + } + for i, fallback := range expanded.Fallbacks { + expanded.Fallbacks[i], err = fallback.Expand() + if err != nil { + return err + } + } + composed.Spec.Model = expanded + for path := range composed.Spec.Explicit { + if strings.HasPrefix(path, "/fallbacks/") { + delete(composed.Spec.Explicit, path) + } + } + if len(composed.Spec.Explicit) == 0 { + composed.Spec.Explicit = nil + } + return nil +} + +func (resolved *ResolvedSpec) recordNormalization(before Model) { + for path := range (Spec{Model: resolved.Spec.Model}).Fields() { + tokens := strings.Split(strings.TrimPrefix(path, "/"), "/") + previous := serializedField(reflect.ValueOf(before), tokens) + value := serializedField(reflect.ValueOf(resolved.Spec.Model), tokens) + source, exists := resolved.Provenance[path] + catalog := FieldSource{Kind: FieldSourceCatalog, Name: "model registry", Key: "registry.ResolveModel" + path} + if !exists { + source.Source = catalog + } else if !previous.IsValid() || !value.IsValid() || !reflect.DeepEqual(previous.Interface(), value.Interface()) { + source.NormalizedBy = &catalog + } + resolved.Provenance[path] = source + } +} + +type budgetLimitSources map[string]FieldSource + +func (sources budgetLimitSources) record(layer SpecLayer, limits Budget) { + for _, key := range []string{"cost", "maxTokens", "maxTurns", "timeout"} { + value := serializedField(reflect.ValueOf(layer.Constraints.Limits.Budget), []string{key}) + limit := serializedField(reflect.ValueOf(limits), []string{key}) + if !value.IsZero() && reflect.DeepEqual(value.Interface(), limit.Interface()) { + sources[key] = FieldSource{Kind: FieldSourceLayer, Name: layer.Name, LayerID: layer.ID, Key: "/constraints/limits/budget/" + key} + } + } +} + +func (composed *ComposedSpec) recordLimits(sources budgetLimitSources) { + for key, constraint := range sources { + limit := serializedField(reflect.ValueOf(composed.Constraints.Limits.Budget), []string{key}) + effective := serializedField(reflect.ValueOf(composed.Spec.Budget), []string{key}) + if !reflect.DeepEqual(limit.Interface(), effective.Interface()) { + continue + } + path := "/budget/" + key + provenance, exists := composed.Provenance[path] + if exists { + provenance.NormalizedBy = &constraint + } else { + provenance.Source = constraint + } + composed.Provenance[path] = provenance + } +} diff --git a/pkg/api/spec_runtime_ginkgo_test.go b/pkg/api/spec_runtime_ginkgo_test.go index cd41723a..2b269f46 100644 --- a/pkg/api/spec_runtime_ginkgo_test.go +++ b/pkg/api/spec_runtime_ginkgo_test.go @@ -10,9 +10,9 @@ var _ = Describe("Effective layered runtime validation", func() { profile := PromptSpecLayer("profile", Spec{Model: Model{Name: "agent:sol"}, Permissions: Permissions{Tools: Tools{"Bash": ToolPolicyDeny}}}) request := RequestSpecLayer("request", Spec{Model: Model{Name: "sonnet", Mode: ModeCLI}}) Expect(ValidateSpecLayers(profile)).To(Succeed()) - _, err := ResolveSpecLayers(profile) + _, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{profile}}) Expect(err).To(HaveOccurred()) - resolved, err := ResolveSpecLayers(profile, request) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{profile, request}}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Spec.Name).To(Equal("claude-sonnet-5")) Expect(resolved.Spec.Provider).To(Equal(Anthropic)) @@ -22,7 +22,7 @@ var _ = Describe("Effective layered runtime validation", func() { }) It("keeps an explicit API mode after subsequent provider model resolution", func() { - resolved, err := ResolveSpecLayers(PromptSpecLayer("profile", Spec{Model: Model{Name: "api:sonnet"}})) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{PromptSpecLayer("profile", Spec{Model: Model{Name: "api:sonnet"}})}}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Spec.Mode).To(Equal(ModeAPI)) again, err := ResolveModel(resolved.Spec.Model) @@ -35,7 +35,7 @@ var _ = Describe("Effective layered runtime validation", func() { Model: Model{Name: "agent:sonnet", Fallbacks: []Model{{Name: "cli:sol"}}}, Permissions: Permissions{Plugins: ResourcePolicies{"example": ResourceEnabled}}, }) - resolved, err := ResolveSpecLayers(layer) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer}}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Warnings).To(ConsistOf( "resource policy plugins=enabled is not available for anthropic agent", @@ -45,7 +45,7 @@ var _ = Describe("Effective layered runtime validation", func() { }) DescribeTable("refuses unsupported isolation on every candidate", func(model Model) { - _, err := ResolveSpecLayers(PromptSpecLayer("profile", Spec{Model: model, Sandbox: &SandboxRef{Mode: SandboxNative}})) + _, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{PromptSpecLayer("profile", Spec{Model: model, Sandbox: &SandboxRef{Mode: SandboxNative}})}}) Expect(err).To(MatchError(ContainSubstring(`sandbox mode "native" is not available`))) }, Entry("primary", Model{Name: "api:sonnet"}), @@ -53,20 +53,20 @@ var _ = Describe("Effective layered runtime validation", func() { ) It("retains hard agent-tool policy refusal for fallback runtimes", func() { - _, err := ResolveSpecLayers(PromptSpecLayer("profile", Spec{ + _, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{PromptSpecLayer("profile", Spec{ Model: Model{Name: "cli:sonnet", Fallbacks: []Model{{Name: "cli:sol"}}}, Permissions: Permissions{Tools: Tools{"Bash": ToolPolicyDeny}}, - })) + })}}) Expect(err).To(MatchError(ContainSubstring("tool policy"))) }) It("uses native translators for unsupported policy fields on fallbacks", func() { - _, err := ResolveSpecLayers(PromptSpecLayer("profile", Spec{ + _, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{PromptSpecLayer("profile", Spec{ Model: Model{Name: "agent:sonnet", Fallbacks: []Model{{Name: "agent:sol"}}}, Sandbox: &SandboxRef{Mode: SandboxNative, Policy: &NativeSandboxPolicy{ Network: &SandboxNetworkPolicy{AllowedDomains: []string{"example.com"}}, }}, - })) + })}}) Expect(err).To(MatchError(ContainSubstring("allowedDomains"))) }) @@ -77,7 +77,7 @@ var _ = Describe("Effective layered runtime validation", func() { Filesystem: &SandboxFilesystemPolicy{Access: SandboxFilesystemWorkspaceWrite}, }}, }) - resolved, err := ResolveSpecLayers(layer) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer}}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Spec.Sandbox).To(Equal(layer.Spec.Sandbox)) Expect(resolved.Trace).To(Equal([]SpecLayer{layer})) @@ -88,7 +88,7 @@ var _ = Describe("Effective layered runtime validation", func() { Workflow: &Workflow{Verify: &Verify{Commands: []string{"true"}}}, Permissions: Permissions{Plugins: ResourcePolicies{"example": ResourceEnabled}}, }) - resolved, err := ResolveSpecLayers(layer) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer}}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Spec.Model).To(Equal(layer.Spec.Model)) Expect(resolved.Warnings).To(BeEmpty()) @@ -100,7 +100,7 @@ var _ = Describe("Effective layered runtime validation", func() { Constraints: RuntimeConstraints{Models: []string{"sol", "sonnet"}}, Spec: Spec{Model: Model{Name: "sol", Fallbacks: []Model{{Name: "sonnet"}}}}, } - resolved, err := ResolveSpecLayers(layer) + resolved, err := ResolveSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{layer}}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Spec.Name).To(Equal("gpt-5.6-sol")) Expect(resolved.Spec.Fallbacks[0].Name).To(Equal("claude-sonnet-5")) diff --git a/pkg/api/spec_test.go b/pkg/api/spec_test.go index caf39937..e20bd679 100644 --- a/pkg/api/spec_test.go +++ b/pkg/api/spec_test.go @@ -208,8 +208,8 @@ func TestSpec_FallbacksCompact(t *testing.T) { if len(spec.Model.Fallbacks) != 2 { t.Fatalf("fallbacks = %+v", spec.Model.Fallbacks) } - // The compact prefix is a runtime mode; the adapter follows from resolution. - if spec.Model.Fallbacks[0].Name != "sonnet" || spec.Model.Fallbacks[0].Mode != ModeAgent { + // Keep authored compact selectors until final composition and resolution. + if spec.Model.Fallbacks[0].Name != "agent:sonnet:medium" || spec.Model.Fallbacks[0].Mode != "" { t.Errorf("fb0 = %+v", spec.Model.Fallbacks[0]) } resolved, err := ResolveModel(spec.Model.Fallbacks[0]) @@ -218,7 +218,7 @@ func TestSpec_FallbacksCompact(t *testing.T) { } // Candidates flattens primary + fallbacks in order. cands := spec.Model.Candidates() - if len(cands) != 3 || cands[0].Name != "opus" || cands[1].Name != "sonnet" || cands[2].Name != "gpt-5.5" { + if len(cands) != 3 || cands[0].Name != "opus" || cands[1].Name != "agent:sonnet:medium" || cands[2].Name != "api:gpt-5.5" { t.Errorf("candidates = %+v", cands) } } diff --git a/pkg/api/spec_validation.go b/pkg/api/spec_validation.go index 537cfe5d..29a97031 100644 --- a/pkg/api/spec_validation.go +++ b/pkg/api/spec_validation.go @@ -62,7 +62,7 @@ func (s Spec) validateRuntimeFields() error { return fmt.Errorf("workflow: %w", err) } if s.Sandbox != nil { - if err := s.Sandbox.Validate(); err != nil { + if err := s.Sandbox.ValidateStructure(); err != nil { return fmt.Errorf("sandbox: %w", err) } } diff --git a/pkg/cli/ai_runtime_flags.go b/pkg/cli/ai_runtime_flags.go new file mode 100644 index 00000000..145451d8 --- /dev/null +++ b/pkg/cli/ai_runtime_flags.go @@ -0,0 +1,109 @@ +package cli + +import ( + "fmt" + "strings" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/api/registry" + "github.com/spf13/pflag" +) + +// WithExplicit preserves authored zero values from flag binders. +func (o AIRuntimeOptions) WithExplicit(paths ...string) AIRuntimeOptions { + o.ExplicitFields = (api.Spec{Explicit: o.ExplicitFields}).WithExplicit(paths...).Explicit + return o +} + +var runtimeFlagFields = map[string]string{ + "model": "/model", "mode": "/mode", "fallback": "/fallbacks", "effort": "/effort", "temperature": "/temperature", "no-cache": "/noCache", + "budget": "/budget/cost", "max-tokens": "/budget/maxTokens", "max-turns": "/budget/maxTurns", "resume": "/sessionId", + "no-mcp": "/permissions/mcp/disabled", "permission-mode": "/permissions/mode", + "no-hooks": "/memory/skipHooks", "no-skills": "/memory/skipSkills", "no-user": "/memory/skipUser", + "no-project": "/memory/skipProject", "no-memory": "/memory/skipMemory", "bare": "/memory/bare", "skill-dir": "/memory/skills", + "timeout": "/budget/timeout", "system": "/prompt/system", "append-system": "/prompt/appendSystem", +} + +// WithChangedFlags carries explicit false values through Cobra's typed binding. +func (o AIRuntimeOptions) WithChangedFlags(flags *pflag.FlagSet) AIRuntimeOptions { + flags.Visit(func(flag *pflag.Flag) { + if path, ok := runtimeFlagFields[flag.Name]; ok { + o = o.WithExplicit(path) + switch path { + case "/model", "/mode", "/effort", "/temperature", "/noCache", "/fallbacks": + o.ModelFlags = o.ModelFlags.WithExplicit(path) + } + } + }) + return o +} + +func (o AIRuntimeOptions) requestSpec() (api.Spec, error) { + model, err := o.ToModel() + if err != nil { + return api.Spec{}, err + } + budget, err := o.BudgetUSD() + if err != nil { + return api.Spec{}, err + } + if o.MaxTurns < 0 || o.MaxTurns > 100 { + return api.Spec{}, fmt.Errorf("invalid --max-turns %d (valid: 0-100, 0=provider default)", o.MaxTurns) + } + if err := validatePermissionMode(o.PermissionMode); err != nil { + return api.Spec{}, err + } + permissions := api.Permissions{ + Mode: api.PermissionMode(o.PermissionMode), + Tools: api.ToolsFromLists(o.AllowedTools, o.DisallowedTools), + MCP: api.MCP{Disabled: o.NoMCP}, + } + if o.Edit { + permissions.Presets = []api.Preset{api.PresetEdit} + if permissions.Mode == "" { + permissions.Mode = api.PermissionAcceptEdits + } + } + spec := api.Spec{ + Explicit: o.ExplicitFields.Clone(), + Model: model, + Budget: api.Budget{Cost: budget, MaxTokens: o.MaxTokens, MaxTurns: o.MaxTurns}, + Memory: api.Memory{Skills: o.SkillDirs, SkipHooks: o.NoHooks, SkipSkills: o.NoSkills, + SkipUser: o.NoUser, SkipProject: o.NoProject, SkipMemory: o.NoMemory, Bare: o.Bare}, + Permissions: permissions, + SessionID: o.Resume, + } + if strings.TrimSpace(o.Budget) != "" { + spec = spec.WithExplicit("/budget/cost") + } + for _, path := range []string{"/budget/timeout", "/prompt/system", "/prompt/appendSystem"} { + delete(spec.Explicit, path) + } + if selector := o.SandboxSelector(); selector != "" { + spec.Sandbox = &api.SandboxRef{Backend: selector} + if kind, ok := registry.ParseSandboxKind(selector); ok { + spec.Sandbox = &api.SandboxRef{Mode: kind} + spec = spec.WithExplicit("/sandbox/backend") + } + } + return spec, nil +} + +func (o AIPromptOptions) promptSpec() (api.Spec, error) { + var attachments []api.AttachmentRef + if len(o.Attach) > 0 { + var err error + attachments, err = attachmentRefsFromFlags(o.Attach) + if err != nil { + return api.Spec{}, err + } + } + spec := api.Spec{Prompt: api.Prompt{System: o.System, AppendSystem: o.AppendSystem, Attachments: attachments}, + Budget: api.Budget{Timeout: o.Timeout}} + for _, path := range []string{"/budget/timeout", "/prompt/system", "/prompt/appendSystem"} { + if o.ExplicitFields.Has(path) { + spec = spec.WithExplicit(path) + } + } + return spec, nil +} diff --git a/pkg/cli/ai_runtime_helpers_test.go b/pkg/cli/ai_runtime_helpers_test.go new file mode 100644 index 00000000..4b029c3f --- /dev/null +++ b/pkg/cli/ai_runtime_helpers_test.go @@ -0,0 +1,45 @@ +package cli + +import ( + "os" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" +) + +func promptRequestForTest(options AIPromptOptions) (ai.Request, error) { + spec, err := options.promptSpec() + if err != nil { + return ai.Request{}, err + } + spec.Prompt.User = options.Prompt + resolved, err := runtimeProjectionForTest(options.AIRuntimeOptions, []api.SpecLayer{api.PromptSpecLayer("prompt input", spec)}) + return resolved.Request, err +} + +func runtimeRequestForTest(options AIRuntimeOptions, prompt api.Prompt) (ai.Request, error) { + resolved, err := runtimeProjectionForTest(options, []api.SpecLayer{api.PromptSpecLayer("prompt input", api.Spec{Prompt: prompt})}) + return resolved.Request, err +} + +func providerConfigForTest(options AIProviderOptions) (ai.Config, error) { + resolved, err := runtimeProjectionForTest(AIRuntimeOptions{AIProviderOptions: options}, nil) + return resolved.Config, err +} + +func runtimeLayersForTest(base api.Spec, options AIPromptOptions) (ai.Request, ai.Config, error) { + resolved, err := runtimeProjectionForTest(options.AIRuntimeOptions, []api.SpecLayer{api.PromptSpecLayer("file", base)}) + return resolved.Request, resolved.Config, err +} + +func runtimeProjectionForTest(options AIRuntimeOptions, layers []api.SpecLayer) (AIRuntimeResolved, error) { + saved, err := loadSavedConfig() + if err != nil { + return AIRuntimeResolved{}, err + } + cwd, err := os.Getwd() + if err != nil { + return AIRuntimeResolved{}, err + } + return options.Resolve(AIRuntimeResolveOptions{Layers: layers, Saved: saved, Cwd: cwd}) +} diff --git a/pkg/cli/ai_runtime_normalize.go b/pkg/cli/ai_runtime_normalize.go new file mode 100644 index 00000000..1c15b0e0 --- /dev/null +++ b/pkg/cli/ai_runtime_normalize.go @@ -0,0 +1,111 @@ +package cli + +import ( + "fmt" + "reflect" + "slices" + "strings" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/captainconfig" + "github.com/flanksource/commons-db/shell" +) + +type AIRuntimeNormalizeOptions struct { + Spec api.Spec + Saved captainconfig.Config + Cwd string +} + +// Normalize derives execution context before saved defaults and final validation. +func (o AIRuntimeOptions) Normalize(options AIRuntimeNormalizeOptions) (api.SpecNormalization, error) { + spec := options.Spec + selection, err := resolveSandboxSelection(sandboxSelectionOptions{Selector: o.SandboxSelector(), Spec: spec, Saved: options.Saved.Sandbox}) + if err != nil { + return api.SpecNormalization{}, err + } + fields := api.FieldPresence{} + if options.Cwd != "" { + before := shell.Setup{} + if spec.Setup != nil { + before = *spec.Setup + } + if err := normalizePromptContextDir(&spec, options.Cwd); err != nil { + return api.SpecNormalization{}, err + } + for path := range normalizedSetupFields(before, *spec.Setup) { + fields[path] = true + } + } + skills := slices.Clone(spec.Memory.Skills) + foldSkillPolicies(&spec) + if !reflect.DeepEqual(skills, spec.Memory.Skills) { + fields["/memory/skills"] = true + } + if spec.Sandbox != nil || o.SandboxSelector() != "" || selection.Kind != registry.SandboxOff { + ref := sandboxRefFromSelection(selection) + if spec.Sandbox != nil { + ref.Policy, ref.Agent, ref.Dispatch = spec.Sandbox.Policy, spec.Sandbox.Agent, spec.Sandbox.Dispatch + } + if spec.Sandbox == nil || spec.Sandbox.Mode != ref.Mode { + fields["/sandbox/mode"] = true + } + if spec.Sandbox == nil || spec.Sandbox.Backend != ref.Backend { + fields["/sandbox/backend"] = true + } + spec.Sandbox = &ref + } + if forced := sandboxForcedMode(selection.Kind); forced != "" { + if raw := strings.TrimSpace(o.Mode); raw != "" { + mode, ok := registry.ParseRuntimeMode(raw) + if !ok || mode != forced { + return api.SpecNormalization{}, fmt.Errorf("sandbox %q requires %s mode, but --mode is %q", selection.Kind, forced, raw) + } + } + if err := normalizeSandboxMode(&spec.Model, forced, "/mode", fields); err != nil { + return api.SpecNormalization{}, err + } + for i := range spec.Fallbacks { + if err := normalizeSandboxMode(&spec.Fallbacks[i], forced, fmt.Sprintf("/fallbacks/%d/mode", i), fields); err != nil { + return api.SpecNormalization{}, err + } + } + } + return api.SpecNormalization{Spec: spec, Fields: fields, Source: api.FieldSource{Kind: api.FieldSourceContext, Name: "runtime context"}}, nil +} + +func normalizeSandboxMode(model *api.Model, mode api.RuntimeMode, path string, fields api.FieldPresence) error { + if model.Mode != "" && model.Mode != mode { + return fmt.Errorf("sandbox requires %s mode, but %s declares %q", mode, path, model.Mode) + } + if model.Mode != mode { + model.Mode = mode + fields[path] = true + } + return nil +} + +func normalizedSetupFields(before, after shell.Setup) api.FieldPresence { + fields := api.FieldPresence{} + if before.Cwd != after.Cwd { + fields["/setup/cwd"] = true + } + if before.BaseDir != after.BaseDir { + fields["/setup/baseDir"] = true + } + for i := range before.DotEnv { + if before.DotEnv[i] != after.DotEnv[i] { + fields[fmt.Sprintf("/setup/dotenv/%d", i)] = true + } + } + if before.Checkout != nil && after.Checkout != nil { + if before.Checkout.Path != after.Checkout.Path { + fields["/setup/checkout/path"] = true + } + if before.Checkout.Worktree != nil && after.Checkout.Worktree != nil && before.Checkout.Worktree.Path != after.Checkout.Worktree.Path { + fields["/setup/checkout/worktree/path"] = true + } + } + return fields +} diff --git a/pkg/cli/ai_runtime_resolve_ginkgo_test.go b/pkg/cli/ai_runtime_resolve_ginkgo_test.go new file mode 100644 index 00000000..b907b54e --- /dev/null +++ b/pkg/cli/ai_runtime_resolve_ginkgo_test.go @@ -0,0 +1,194 @@ +package cli + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" + "github.com/flanksource/commons-db/shell" + "github.com/flanksource/commons-db/types" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/pflag" +) + +var _ = Describe("captured CLI runtime projection", func() { + It("rejects a generating invocation without a configured model", func() { + captainconfig.SetPathForTesting(filepath.Join(GinkgoT().TempDir(), ".captain.yaml")) + DeferCleanup(captainconfig.SetPathForTesting, "") + _, err := resolveInvocation(AIRuntimeOptions{}, []api.SpecLayer{api.PromptSpecLayer("generate", api.Spec{Prompt: api.Prompt{User: "Review"}})}) + Expect(err).To(MatchError(ContainSubstring("no model configured"))) + }) + + It("renders command-only verification without a configured model", func() { + captainconfig.SetPathForTesting(filepath.Join(GinkgoT().TempDir(), ".captain.yaml")) + DeferCleanup(captainconfig.SetPathForTesting, "") + result, err := renderPrompt(context.Background(), "", PromptRenderRequest{Spec: &api.Spec{Workflow: &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}}}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Input.IsVerifyOnly()).To(BeTrue()) + Expect(result.ValidationError).To(BeEmpty()) + Expect(result.Config.Model.Name).To(BeEmpty()) + }) + + It("clears explicitly empty prompt flags after authored and saved defaults", func() { + opts, err := actionFlagsToOptions(map[string]string{"timeout": "", "system": "", "append-system": ""}) + Expect(err).NotTo(HaveOccurred()) + saved := captainconfig.Config{AI: captainconfig.AIDefaults{Timeout: "2m"}} + layers, err := renderLoadedLayers(context.Background(), "---\nmodel: agent:claude-sonnet-5\nprompt:\n system: Authored system\n appendSystem: Authored suffix\nbudget:\n timeout: 1m\n---\nReview", "review.prompt", nil, opts, saved) + Expect(err).NotTo(HaveOccurred()) + result, err := opts.Resolve(AIRuntimeResolveOptions{Layers: layers, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Request.Prompt.System).To(BeEmpty()) + Expect(result.Request.Prompt.AppendSystem).To(BeEmpty()) + Expect(result.Request.Budget.Timeout).To(BeEmpty()) + for _, path := range []string{"/prompt/system", "/prompt/appendSystem", "/budget/timeout"} { + Expect(result.Resolution.Provenance[path].Source.Name).To(Equal("prompt flags")) + } + }) + + It("uses injected prompt source settings without rereading a malformed config file", func() { + dir := GinkgoT().TempDir() + path := filepath.Join(dir, ".captain.yaml") + Expect(os.WriteFile(path, []byte("prompts: [invalid"), 0o600)).To(Succeed()) + captainconfig.SetPathForTesting(path) + DeferCleanup(captainconfig.SetPathForTesting, "") + saved := captainconfig.Config{Prompts: captainconfig.PromptDefaults{Dirs: []string{dir}}} + sources, err := buildPromptSources(context.Background(), promptSourceOptions{Config: &saved}) + Expect(err).NotTo(HaveOccurred()) + resolvedDir, err := filepath.EvalSymlinks(dir) + Expect(err).NotTo(HaveOccurred()) + Expect(sources).To(ContainElement(HaveField("Root", resolvedDir))) + }) + + It("preserves Cobra changed false flags for typed command options", func() { + flags := pflag.NewFlagSet("runtime", pflag.ContinueOnError) + flags.Bool("no-cache", false, "") + flags.Int("max-tokens", 0, "") + Expect(flags.Parse([]string{"--no-cache=false", "--max-tokens=0"})).To(Succeed()) + opts := (AIRuntimeOptions{}).WithChangedFlags(flags) + result, err := opts.Resolve(AIRuntimeResolveOptions{Saved: captainconfig.Config{AI: captainconfig.AIDefaults{NoCache: true, MaxTokens: 8000}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Request.NoCache).To(BeFalse()) + Expect(result.Config.NoCache).To(BeFalse()) + Expect(result.Request.Budget.MaxTokens).To(BeZero()) + }) + + It("retains authored setup origins while recording path normalization", func() { + cwd := GinkgoT().TempDir() + operation := api.Spec{Model: api.Model{Name: "agent:claude-sonnet-5"}, Setup: &shell.Setup{Cwd: "workspace", EnvVars: []types.EnvVar{{Name: "REVIEW_MODE", ValueStatic: "check"}}}} + result, err := (AIRuntimeOptions{}).Resolve(AIRuntimeResolveOptions{Layers: []api.SpecLayer{api.PromptSpecLayer("operation", operation)}, Cwd: cwd}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Request.Cwd()).To(Equal(filepath.Join(cwd, "workspace"))) + Expect(result.Request.Setup.EnvVars).To(Equal(operation.Setup.EnvVars)) + Expect(result.Resolution.Provenance["/setup/envVars/0/value"].Source.Name).To(Equal("operation")) + Expect(result.Resolution.Provenance["/setup/envVars/0/value"].NormalizedBy).To(BeNil()) + Expect(result.Resolution.Provenance["/setup/cwd"].Source.Name).To(Equal("operation")) + Expect(result.Resolution.Provenance["/setup/cwd"].NormalizedBy).To(HaveField("Name", "runtime context")) + Expect(result.Resolution.Trace[0].Spec.Setup.Cwd).To(Equal("workspace")) + }) + + It("projects an already resolved spec without reapplying flags or saved defaults", func() { + input := api.ResolvedSpec{Spec: api.Spec{Model: api.Model{Name: "claude-sonnet-5", Mode: api.ModeAgent}, Budget: api.Budget{Cost: 2}}, Warnings: []string{"declared warning"}} + opts := AIRuntimeOptions{AIProviderOptions: AIProviderOptions{Budget: "3"}} + result, err := opts.Project(AIRuntimeProjectOptions{Resolved: input, Saved: captainconfig.Config{AI: captainconfig.AIDefaults{BudgetUSD: 1, NoCache: true}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Resolution).To(Equal(input)) + Expect(result.Request).To(Equal(input.Spec)) + Expect(result.Config.Budget).To(Equal(input.Spec.Budget)) + Expect(result.Config.NoCache).To(BeFalse()) + }) + + It("lets explicit false and zero CLI flags override saved settings", func() { + opts, err := actionFlagsToOptions(map[string]string{"model": "agent:claude-sonnet-5", "no-cache": "false", "no-hooks": "false", "max-tokens": "0", "temperature": "0"}) + Expect(err).NotTo(HaveOccurred()) + result, err := opts.Resolve(AIRuntimeResolveOptions{Saved: captainconfig.Config{AI: captainconfig.AIDefaults{NoCache: true, NoHooks: true, MaxTokens: 8000}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Request.NoCache).To(BeFalse()) + Expect(result.Request.Memory.SkipHooks).To(BeFalse()) + Expect(result.Request.Budget.MaxTokens).To(BeZero()) + Expect(result.Request.Temperature).To(HaveValue(BeZero())) + }) + + It("preserves the complete operation and authored false values in matching request and config", func() { + cwd := GinkgoT().TempDir() + operation := api.Spec{ + Model: api.Model{Name: "claude-sonnet-5", Mode: api.ModeAgent}, + Budget: api.Budget{Cost: 2, MaxTokens: 512, Timeout: "1m"}, + Prompt: api.Prompt{User: "Review the change", SchemaJSON: json.RawMessage(`{"type":"object"}`)}, + Permissions: api.Permissions{Mode: api.PermissionAcceptEdits, Tools: api.Tools{"Read": api.ToolPolicyAllow}}, + Memory: api.Memory{Skills: []string{"review-skills"}}, + Setup: &shell.Setup{Env: []string{"REVIEW_MODE=check"}}, + SessionID: "review-session", + CLIArgs: map[string]any{"review": true}, + }.WithExplicit("/noCache", "/memory/skipHooks", "/permissions/mcp/disabled") + saved := captainconfig.Config{AI: captainconfig.AIDefaults{BudgetUSD: 1, NoCache: true, NoHooks: true, NoMCP: true}} + path := filepath.Join(cwd, ".captain.yaml") + Expect(os.WriteFile(path, []byte("ai: [invalid"), 0o600)).To(Succeed()) + captainconfig.SetPathForTesting(path) + DeferCleanup(captainconfig.SetPathForTesting, "") + opts := AIRuntimeOptions{AIProviderOptions: AIProviderOptions{APIKey: "example", APIURL: "http://127.0.0.1:9911"}} + + resolved, err := opts.Resolve(AIRuntimeResolveOptions{Layers: []api.SpecLayer{api.PromptSpecLayer("operation", operation)}, Saved: saved, Cwd: cwd, RequireModel: true}) + + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Request.Budget).To(Equal(operation.Budget)) + Expect(resolved.Config.Budget).To(Equal(operation.Budget)) + Expect(resolved.Config.Model).To(Equal(resolved.Request.Model)) + Expect(resolved.Request.NoCache).To(BeFalse()) + Expect(resolved.Config.NoCache).To(BeFalse()) + Expect(resolved.Request.Memory).To(Equal(operation.Memory)) + Expect(resolved.Request.Permissions).To(Equal(operation.Permissions)) + Expect(resolved.Request.Prompt).To(Equal(operation.Prompt)) + Expect(resolved.Request.CLIArgs).To(Equal(operation.CLIArgs)) + Expect(resolved.Request.SessionID).To(Equal(operation.SessionID)) + Expect(resolved.Config.SessionID).To(Equal(operation.SessionID)) + Expect(resolved.Request.Setup.Env).To(Equal(operation.Setup.Env)) + Expect(resolved.Request.Cwd()).To(Equal(cwd)) + Expect(resolved.Config.APIKey).To(Equal("example")) + Expect(resolved.Config.APIURL).To(Equal("http://127.0.0.1:9911")) + Expect(resolved.Resolution.Spec).To(Equal(resolved.Request)) + }) + + It("resolves a named sandbox from the injected full settings snapshot", func() { + operation := api.Spec{Model: api.Model{Name: "claude-sonnet-5", Mode: api.ModeCLI}, Sandbox: &api.SandboxRef{Backend: "review-pool"}} + saved := captainconfig.Config{Sandbox: captainconfig.SandboxDefaults{Backends: map[string]captainconfig.SandboxBackend{ + "review-pool": {Kind: "docker", Options: map[string]any{"image": "example/review"}}, + }}} + resolved, err := (AIRuntimeOptions{}).Resolve(AIRuntimeResolveOptions{Layers: []api.SpecLayer{api.PromptSpecLayer("operation", operation)}, Saved: saved, Cwd: GinkgoT().TempDir(), RequireModel: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Config.SandboxSelection).To(Equal(&api.SandboxConfig{Kind: api.SandboxDocker, Name: "review-pool", Options: map[string]any{"image": "example/review"}})) + Expect(resolved.Request.Sandbox.Backend).To(Equal("review-pool")) + }) + + It("retains an explicit sandbox selector in the CLI layer before backend normalization", func() { + opts := AIRuntimeOptions{AIProviderOptions: AIProviderOptions{Sandbox: "review-pool"}} + saved := captainconfig.Config{Sandbox: captainconfig.SandboxDefaults{Backends: map[string]captainconfig.SandboxBackend{ + "review-pool": {Kind: "docker"}, + }}} + result, err := opts.Resolve(AIRuntimeResolveOptions{Layers: []api.SpecLayer{api.PromptSpecLayer("operation", api.Spec{Model: api.Model{Name: "claude-sonnet-5", Mode: api.ModeCLI}})}, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Resolution.Trace[len(result.Resolution.Trace)-1].Spec.Sandbox).To(Equal(&api.SandboxRef{Backend: "review-pool"})) + Expect(result.Resolution.Provenance["/sandbox/backend"].Source.Name).To(Equal("CLI flags")) + Expect(result.Request.Sandbox.Mode).To(Equal(api.SandboxDocker)) + }) + + It("keeps an explicit sandbox null from restoring the saved default during resolution or projection", func() { + spec := api.Spec{Model: api.Model{Name: "claude-sonnet-5", Mode: api.ModeCLI}}.WithExplicit("/sandbox") + saved := captainconfig.Config{Sandbox: captainconfig.SandboxDefaults{Default: "docker"}} + result, err := (AIRuntimeOptions{}).Resolve(AIRuntimeResolveOptions{Layers: []api.SpecLayer{api.RequestSpecLayer("clear sandbox", spec)}, Saved: saved}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Request.Sandbox).To(BeNil()) + Expect(result.Config.SandboxSelection).To(BeNil()) + Expect(result.Resolution.Provenance["/sandbox"].Source.Name).To(Equal("clear sandbox")) + }) + + It("returns final capability warnings for the complete authored request", func() { + operation := api.Spec{Model: api.Model{Name: "claude-sonnet-5", Mode: api.ModeAgent}, Permissions: api.Permissions{Plugins: api.ResourcePolicies{"review-tools": api.ResourceEnabled}}} + resolved, err := (AIRuntimeOptions{}).Resolve(AIRuntimeResolveOptions{Layers: []api.SpecLayer{api.PromptSpecLayer("operation", operation)}, Cwd: GinkgoT().TempDir(), RequireModel: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Resolution.Warnings).To(HaveExactElements(ContainSubstring("plugins"))) + }) +}) diff --git a/pkg/cli/runtime_preset_entity.go b/pkg/cli/runtime_preset_entity.go index 24ec1d3e..cf10dd9f 100644 --- a/pkg/cli/runtime_preset_entity.go +++ b/pkg/cli/runtime_preset_entity.go @@ -95,7 +95,7 @@ func listRuntimePresets(ctx context.Context, opts RuntimePresetListOptions) ([]R if err != nil { return nil, err } - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return nil, err } @@ -120,7 +120,7 @@ func listRuntimePresets(ctx context.Context, opts RuntimePresetListOptions) ([]R } func getRuntimePreset(ctx context.Context, id string) (RuntimePresetRecord, error) { - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return RuntimePresetRecord{}, err } @@ -143,7 +143,7 @@ func createRuntimePreset(ctx context.Context, body map[string]any) (RuntimePrese if err != nil { return RuntimePresetRecord{}, err } - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return RuntimePresetRecord{}, err } @@ -169,7 +169,7 @@ func updateRuntimePreset(ctx context.Context, id string, body map[string]any) (R if err != nil { return RuntimePresetRecord{}, err } - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return RuntimePresetRecord{}, err } @@ -181,7 +181,7 @@ func updateRuntimePreset(ctx context.Context, id string, body map[string]any) (R } func deleteRuntimePreset(ctx context.Context, id string) error { - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return err } diff --git a/pkg/cli/runtime_profile_catalog.go b/pkg/cli/runtime_profile_catalog.go index f4d53995..947b8088 100644 --- a/pkg/cli/runtime_profile_catalog.go +++ b/pkg/cli/runtime_profile_catalog.go @@ -20,13 +20,12 @@ func ContextWithRuntimeCatalog(ctx context.Context, catalog *runtimeprofiles.Cat // buildRuntimeCatalog assembles the preset and profile sources: the monitored // database, the user's ~/.config/captain directories, the directories named in // ~/.captain.yaml, and the repository's .captain directories. -func buildRuntimeCatalog(ctx context.Context) (*runtimeprofiles.Catalog, error) { +func buildRuntimeCatalog(ctx context.Context, options runtimeprofiles.DefaultCatalogOptions) (*runtimeprofiles.Catalog, error) { if catalog, ok := ctx.Value(runtimeCatalogContextKey{}).(*runtimeprofiles.Catalog); ok && catalog != nil { return catalog, nil } - return runtimeprofiles.NewDefaultCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{ - Read: captainDB, Write: captainDefaultDB, - }) + options.Read, options.Write = captainDB, captainDefaultDB + return runtimeprofiles.NewDefaultCatalog(ctx, options) } // runtimeCatalogError maps catalog failures onto HTTP statuses for the entity diff --git a/pkg/cli/runtime_profile_catalog_ginkgo_test.go b/pkg/cli/runtime_profile_catalog_ginkgo_test.go index 26079e72..acfa58ba 100644 --- a/pkg/cli/runtime_profile_catalog_ginkgo_test.go +++ b/pkg/cli/runtime_profile_catalog_ginkgo_test.go @@ -27,7 +27,7 @@ var _ = Describe("CLI runtime catalog discovery", func() { It("preserves the supplied catalog without reading user config", func() { fixture := newRuntimeEntityFixture() Expect(os.WriteFile(configPath, []byte("runtime: [malformed\n"), 0o600)).To(Succeed()) - Expect(buildRuntimeCatalog(fixture.ctx)).To(BeIdenticalTo(fixture.catalog)) + Expect(buildRuntimeCatalog(fixture.ctx, runtimeprofiles.DefaultCatalogOptions{})).To(BeIdenticalTo(fixture.catalog)) }) It("uses the shared discovery sources with the CLI database registered first", func(ctx SpecContext) { @@ -35,7 +35,7 @@ var _ = Describe("CLI runtime catalog discovery", func() { Read: captainDB, Write: captainDefaultDB, }) Expect(err).NotTo(HaveOccurred()) - actual, err := buildRuntimeCatalog(ctx) + actual, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(actual.Sources()).To(Equal(expected.Sources())) Expect(actual.Sources()).To(HaveLen(5)) @@ -44,7 +44,7 @@ var _ = Describe("CLI runtime catalog discovery", func() { It("reports invalid configured discovery directories", func(ctx SpecContext) { Expect(os.WriteFile(configPath, []byte("runtime:\n profileDirs: [missing]\n"), 0o600)).To(Succeed()) - _, err := buildRuntimeCatalog(ctx) + _, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) Expect(err).To(MatchError(ContainSubstring("runtime.profileDirs"))) Expect(err).To(MatchError(ContainSubstring("missing"))) }) diff --git a/pkg/cli/runtime_profile_entity.go b/pkg/cli/runtime_profile_entity.go index 87ae8070..e17af87e 100644 --- a/pkg/cli/runtime_profile_entity.go +++ b/pkg/cli/runtime_profile_entity.go @@ -97,7 +97,7 @@ func registerRuntimeProfileEntity() { } func listRuntimeProfiles(ctx context.Context, opts RuntimeProfileListOptions) ([]RuntimeProfileRecord, error) { - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return nil, err } @@ -132,7 +132,7 @@ func runtimeProfileCandidates(ctx context.Context, catalog *runtimeprofiles.Cata } func getRuntimeProfile(ctx context.Context, id string) (RuntimeProfileRecord, error) { - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return RuntimeProfileRecord{}, err } @@ -155,7 +155,7 @@ func createRuntimeProfile(ctx context.Context, body map[string]any) (RuntimeProf if err != nil { return RuntimeProfileRecord{}, err } - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return RuntimeProfileRecord{}, err } @@ -181,7 +181,7 @@ func updateRuntimeProfile(ctx context.Context, id string, body map[string]any) ( if err != nil { return RuntimeProfileRecord{}, err } - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return RuntimeProfileRecord{}, err } @@ -193,7 +193,7 @@ func updateRuntimeProfile(ctx context.Context, id string, body map[string]any) ( } func deleteRuntimeProfile(ctx context.Context, id string) error { - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return err } @@ -204,7 +204,7 @@ func deleteRuntimeProfile(ctx context.Context, id string) error { // GET /api/v1/runtime-profile/{id}/resolve: the profile with its references // canonicalised, the presets in reference order, and the resolved spec. func resolveRuntimeProfileAction(ctx context.Context, id string, _ map[string]string) (runtimeprofiles.Resolution, error) { - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{}) if err != nil { return runtimeprofiles.Resolution{}, err } diff --git a/pkg/runtimeprofiles/README.md b/pkg/runtimeprofiles/README.md index 3795a8af..bca046df 100644 --- a/pkg/runtimeprofiles/README.md +++ b/pkg/runtimeprofiles/README.md @@ -17,7 +17,11 @@ layers, err := resolver.Layers(ctx, runtimeprofiles.ResolveOptions{ if err != nil { return err } -resolved, err := api.ResolveSpecLayers(layers.Layers...) +resolved, err := api.ResolveSpecLayers(api.ResolveSpecOptions{ + Layers: layers.Layers, + Saved: &config.AI, // one snapshot loaded by the application boundary + RequireModel: true, +}) if err != nil { return err } @@ -26,9 +30,19 @@ for _, warning := range resolved.Warnings { } ``` -`ResolveSpecLayers` composes in global → context → surface → user scope order, preserving order within a scope. It intersects restrictive catalogs, applies the strictest nonzero budget limits, and preserves every quota and raw layer in `Trace`. Compact model selectors retain their existing pin semantics: a prefix inside the effective model name wins over its sibling mode field. Model aliases and fallback names resolve only after composition. Final model, sandbox, and tool-policy refusals are errors; unsupported permission/resource capabilities produce separate `Warnings` for this compatibility release. No saved model defaults are loaded. Model-free compositions remain valid; execution requirements belong to `promptrun.Preflight`. +`ResolveSpecLayers` composes in global → context → surface → user scope order, preserving order within a scope. It intersects restrictive catalogs, applies the strictest nonzero budget limits, and preserves every quota and raw layer in `Trace`. Compact model selectors retain their existing mode pin semantics: a prefix inside the effective model name wins over its sibling mode field. A higher layer's explicit effort overrides a lower compact effort and retains that higher layer's ownership. Model aliases and fallback names resolve only after composition. Final model, sandbox, and tool-policy refusals are errors; unsupported permission/resource capabilities produce separate `Warnings` for this compatibility release. The library never loads saved defaults from disk. Model-free compositions remain valid unless `RequireModel` is true; full execution admission still belongs to `promptrun.Preflight`. + +Use `api.ComposeSpecLayers` with the same `ResolveSpecOptions` for forms and defaults before a complete request exists. Its distinct `ComposedSpec` exposes `Spec`, `Constraints`, `Trace`, `Provenance`, `Warnings`, and the `AllowsModel` query without claiming runtime validity. It shares the same fold and saved-default pass as final resolution. Unknown authored models remain visible in composition; provider-specific defaults wait until a provider is known. Keep the authored layers when adding a request, then call `ResolveSpecLayers` once for the final stack. + +An optional `Saved *captainconfig.AIDefaults` snapshot fills gaps after every authored layer. Provider mode and effort defaults apply independently to the final primary and each fallback. The global compact selector contributes its mode, effort, and fallback chain only within its provider family. Authored primary temperature/cache settings, and same-family effort, retain the existing fallback inheritance rules. File-wide temperature, cache, budget, timeout, and ambient-memory toggles remain run-wide. With a saved snapshot, a missing token budget uses Captain's existing 4096 default and missing effort uses the selected model's catalog default when one is declared. A nil snapshot injects neither saved nor built-in defaults, supporting catalog and authoritative snapshot consumers. + +JSON/YAML decoding retains explicitly authored `false`, zero, empty lists/maps, and null values. Go callers mark intentional zero values with `spec.WithExplicit("/noCache", "/budget/cost", "/fallbacks")`; ordinary nonzero fields and scalar pointers already count as supplied. Paths use the serialized JSON-pointer vocabulary, including `/permissions/mcp/disabled` for the native MCP toggle. Merging replaces explicit clears, preserves unrelated groups, and removes stale presence when a list is replaced. `WithoutSession` removes both conversation data and its presence metadata. `Spec.DecodeFields` exposes the complete native shape so enclosing decoders retain their unknown-field policy; Captain prompt documents reject unknown declarations, while hosts can report them as warnings. + +`Provenance` maps effective field paths to a `FieldProvenance`. Its `Source` records the actual authored layer, saved config key, or catalog default that supplied the value; equal-valued request overrides still own their fields. `NormalizedBy` records later catalog normalization or a restrictive budget limit without relabeling the original source. CSV fallback entries point to the raw `/model` field that declared them, while explicit list entries retain their original `/fallbacks/N/...` source paths. The raw `Trace` contains no synthesized saved-default or normalized-request rows. + +Applications that derive a working directory or sandbox mode can supply a pure `Normalize` callback. It receives an owned copy of the complete authored `Spec`, after compact grammar and any saved model/fallback selection, before saved mode, effort, and generation gaps are filled. It returns `SpecNormalization{Spec, Fields, Source}`. `Fields` explicitly identifies the derived paths; their existing source remains visible alongside `NormalizedBy`. Saved modes remain defaults for the primary and every fallback, so sandbox context can select their runtime; authored compact mode pins remain explicit constraints. Named sandbox references may remain partial during composition but must resolve to a concrete mode before final validation. This ordering lets provider defaults see the resulting runtime without a second layer fold. The callback must not construct providers, run setup, or persist state. -Use `api.ComposeSpecLayers` for forms and defaults before a complete request exists. Its distinct `ComposedSpec` exposes the structural `Spec`, `Constraints`, `Trace`, and `AllowsModel` query without claiming runtime validity. It shares the same fold as final resolution. Keep its raw `Trace` when adding a request, then call `ResolveSpecLayers` once for the final stack. +Malformed saved declarations fail before a request can hide them and return `*api.SavedDefaultsError` with a source and wrapped error. Missing required model selection remains an actionable configuration error. Missing modes on multi-mode providers warn separately and use the existing registry runtime default during the compatibility window; this applies to every fallback as well as the primary. Layer resolution checks every declared primary and fallback runtime. Execution preflight separately checks the enabled candidates selected for the actual provider configuration. Disabling a candidate for one execution does not make an unsupported sandbox or tool policy valid in the declared profile. diff --git a/pkg/runtimeprofiles/resolve.go b/pkg/runtimeprofiles/resolve.go index d3808451..15930b49 100644 --- a/pkg/runtimeprofiles/resolve.go +++ b/pkg/runtimeprofiles/resolve.go @@ -46,7 +46,7 @@ func (c *Catalog) Resolve(ctx context.Context, ref string) (Resolution, error) { if err != nil { return Resolution{}, err } - resolved, err := api.ResolveSpecLayers(resolution.Layers...) + resolved, err := api.ResolveSpecLayers(api.ResolveSpecOptions{Layers: resolution.Layers}) if err != nil { return Resolution{}, err } diff --git a/pkg/runtimeprofiles/resolver.go b/pkg/runtimeprofiles/resolver.go index 761d439f..8a9340aa 100644 --- a/pkg/runtimeprofiles/resolver.go +++ b/pkg/runtimeprofiles/resolver.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" ) // ErrCatalogUnavailable reports selection against a resolver with no catalog. @@ -55,6 +56,9 @@ type ResolveOptions struct { DefaultProfile string SurfaceLayers []api.SpecLayer RequestLayers []api.SpecLayer + Saved *captainconfig.AIDefaults + RequireModel bool + Normalize func(api.Spec) (api.SpecNormalization, error) } // ResolveResult retains the selected catalog records and effective spec. @@ -109,7 +113,7 @@ func (r *Resolver) Resolve(ctx context.Context, options ResolveOptions) (Resolve if err != nil { return ResolveResult{}, err } - resolved, err := api.ResolveSpecLayers(layers.Layers...) + resolved, err := api.ResolveSpecLayers(api.ResolveSpecOptions{Layers: layers.Layers, Saved: options.Saved, RequireModel: options.RequireModel, Normalize: options.Normalize}) if err != nil { return ResolveResult{}, fmt.Errorf("resolve runtime profile layers: %w", err) } From 68f4daefc3094910d03d8fddc68c05234cdef297 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 12:14:26 +0300 Subject: [PATCH 11/22] feat(aichat): Preserve runtime defaults and request field presence Carry injected saved defaults through profile composition and final request resolution, while preserving explicit zero-valued JSON settings. Record the provider candidate actually selected after fallback and exclude resolution metadata from tool-approval payloads. BREAKING CHANGE: update spec composition and resolution callers to use ResolveSpecOptions. --- pkg/aichat/README.md | 18 +++- .../database_fallback_integration_test.go | 95 +++++++++++++++++++ .../database_threads_integration_test.go | 71 -------------- pkg/aichat/execution_database_authority.go | 5 +- pkg/aichat/layered_runtime_profile.go | 11 ++- pkg/aichat/messages.go | 8 +- pkg/aichat/messages_model_ginkgo_test.go | 4 +- pkg/aichat/request_presence.go | 79 +++++++++++++++ pkg/aichat/runtime_profile.go | 7 +- pkg/aichat/runtime_profile_ginkgo_test.go | 2 +- pkg/aichat/saved_defaults_ginkgo_test.go | 94 ++++++++++++++++++ pkg/aichat/wire.go | 8 +- pkg/aichat/wire_ginkgo_test.go | 3 +- 13 files changed, 317 insertions(+), 88 deletions(-) create mode 100644 pkg/aichat/database_fallback_integration_test.go create mode 100644 pkg/aichat/request_presence.go create mode 100644 pkg/aichat/saved_defaults_ginkgo_test.go diff --git a/pkg/aichat/README.md b/pkg/aichat/README.md index f3084cd7..9253681e 100644 --- a/pkg/aichat/README.md +++ b/pkg/aichat/README.md @@ -9,14 +9,14 @@ This is a Go API change for applications implementing `RuntimeProfileProvider`. For example, a provider can construct its result from application-owned layers: ```go -func applicationProfile(system string, layers []api.SpecLayer) (aichat.RuntimeProfile, error) { - composed, err := api.ComposeSpecLayers(layers...) +func applicationProfile(layers []api.SpecLayer, saved *captainconfig.AIDefaults) (aichat.RuntimeProfile, error) { + composed, err := api.ComposeSpecLayers(api.ResolveSpecOptions{Layers: layers, Saved: saved}) if err != nil { return aichat.RuntimeProfile{}, err } return aichat.RuntimeProfile{ - System: system, Composed: composed, + Saved: saved, }, nil } ``` @@ -29,11 +29,19 @@ The chat service adds the explicit request to that raw trace and performs final func resolveRequest(profile aichat.RuntimeProfile, request api.Spec) (api.ResolvedSpec, error) { layers := append([]api.SpecLayer(nil), profile.Composed.Trace...) layers = append(layers, api.RequestSpecLayer("request", request)) - return api.ResolveSpecLayers(layers...) + return api.ResolveSpecLayers(api.ResolveSpecOptions{ + Layers: layers, + Saved: profile.Saved, + RequireModel: true, + }) } ``` -Call `ResolveSpecLayers` after the complete request is available. It resolves the effective model and fallbacks and applies runtime capability checks. Inspect its separate `Warnings` when implementing a custom pipeline; the chat service logs them before provider admission. Saved model defaults are not loaded by either composition API. +Call `ResolveSpecLayers` after the complete request is available. It resolves the effective model and fallbacks and applies runtime capability checks. Inspect its separate `Warnings` when implementing a custom pipeline; the chat service logs them before provider admission. + +Load settings once at the application boundary and pass the same `Saved` snapshot through profile composition and final resolution. Neither API reads ambient configuration: `Saved: nil` disables saved defaults. Saved values fill gaps after all authored layers, using the final primary and fallback providers. They appear in `Provenance`, never as fabricated layers in `Trace`. `RequireModel: true` rejects a generating request with no configured model and includes configuration guidance; model-free catalog previews remain valid. Missing mode origins currently produce migration warnings before the release switches to strict enforcement. + +JSON chat requests retain explicit zero budgets, false cache values, and empty fallback lists through the shared field-presence contract. Go callers use `ChatRequest.WithExplicit("/budget/cost")` or `Model.WithExplicit("/noCache", "/fallbacks")` for intentional zero values. Nonzero fields and pointer-zero temperatures are detected directly. Malformed application-owned layers remain server errors even if a request would overwrite them. Invalid explicit request fields are client errors. A structurally valid partial profile can be completed or repaired by the final request. Missing nested preset references are owned configuration failures; only an absent or ambiguous requested top-level profile should be classified as an invalid caller selection. diff --git a/pkg/aichat/database_fallback_integration_test.go b/pkg/aichat/database_fallback_integration_test.go new file mode 100644 index 00000000..86a6feab --- /dev/null +++ b/pkg/aichat/database_fallback_integration_test.go @@ -0,0 +1,95 @@ +package aichat_test + +import ( + "encoding/json" + "github.com/flanksource/captain/pkg/aichat" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/database" + "github.com/flanksource/commons-db/dbtest" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "net/http" + "net/http/httptest" +) + +var _ = Describe("Database chat sessions", func() { + It("binds and accounts for the provider candidate selected after fallback", func(ctx SpecContext) { + testDB := dbtest.ForGinkgo(dbtest.Options{Name: "captain_aichat_selected_fallback"}) + db, err := database.Open(ctx, database.WithDSN(testDB.DSN()), database.WithMigrations()) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(db.Close) + store, err := aichat.NewDatabaseThreadStore(db) + Expect(err).NotTo(HaveOccurred()) + thread, err := store.Create(ctx, "Fallback") + Expect(err).NotTo(HaveOccurred()) + authority, err := aichat.NewDatabaseExecutionAuthority(db) + Expect(err).NotTo(HaveOccurred()) + provider := &fakeStreamingProvider{ + model: "gemini-2.5-pro", runtime: api.RuntimeOf(api.Google, api.ModeAPI), + events: []api.Event{ + {Kind: api.EventSystem, SessionID: "fallback-session", Model: "gemini-2.5-pro"}, + {Kind: api.EventText, Text: "Fallback answer", Model: "gemini-2.5-pro"}, + {Kind: api.EventResult, Success: true, Model: "gemini-2.5-pro", CostUSD: 0.25, + Usage: &api.Usage{InputTokens: 12, OutputTokens: 4}}, + }, + } + resolver := &fakeResolver{provider: provider} + service := aichat.NewService(aichat.ServiceOptions{ + Resolver: resolver, Threads: aichat.FixedThreadStore(store), Authority: authority, + }) + submit := func(id string, runtime api.Model) *httptest.ResponseRecorder { + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, requestJSON(http.MethodPost, "/api/chat", aichat.ChatRequest{ + ID: thread.ID, ThreadID: thread.ID, Trigger: "submit-message", Runtime: &runtime, + Messages: []aichat.UIMessage{{ + ID: id, Role: "user", Parts: []aichat.UIPart{{Type: "text", Text: "Use the selected runtime"}}, + }}, + })) + return response + } + + primary := api.Model{ + Name: "gpt-5.6-sol", Mode: api.ModeAPI, + Fallbacks: []api.Model{{Name: "gemini-2.5-pro", Mode: api.ModeAPI, Effort: api.EffortHigh}}, + } + first := submit("fallback-user-1", primary) + Expect(first.Code).To(Equal(http.StatusOK), first.Body.String()) + stored, err := store.Get(ctx, thread.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(stored.Runtime).To(Equal(&api.RuntimeIdentity{ + Model: "gemini-2.5-pro", Provider: api.Google.Name, Mode: api.ModeAPI, + }), "the identity records the fallback candidate that actually ran — model and runtime, not effort") + Expect(stored.ProviderSessionID).To(Equal("fallback-session")) + + aggregate, err := store.GetSession(ctx, thread.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(aggregate.Model).To(Equal("gemini-2.5-pro")) + Expect(aggregate.ModelMode).To(Equal(api.ModeAPI)) + Expect(aggregate.Usage.InputTokens).To(Equal(12)) + Expect(aggregate.Cost.Total()).To(BeNumerically("~", 0.25, 0.000001)) + sessionID := uuid.MustParse(thread.ID) + runs, err := db.ListPromptRuns(ctx, database.PromptRunFilter{SessionID: &sessionID}) + Expect(err).NotTo(HaveOccurred()) + Expect(runs).To(HaveLen(1)) + Expect(runs[0].Runtime.Requested.Model).To(Equal(primary.Name)) + Expect(runs[0].Runtime.Resolved.Model).To(Equal("gemini-2.5-pro")) + Expect(runs[0].Runtime.Resolved.Effort).To(BeEmpty(), "the catalog declares no effort tiers for this model") + resolutionJSON, err := json.Marshal(runs[0].RenderedSpec["resolution"]) + Expect(err).NotTo(HaveOccurred()) + var resolution struct { + Trace []api.SpecLayer `json:"trace"` + } + Expect(json.Unmarshal(resolutionJSON, &resolution)).To(Succeed()) + Expect(resolution.Trace[0].Spec.Fallbacks[0].Effort).To(Equal(api.EffortHigh), "raw authored effort survives final catalog normalization") + + conflict := submit("fallback-user-conflict", primary) + Expect(conflict.Code).To(Equal(http.StatusConflict), conflict.Body.String()) + selected := api.Model{Name: "gemini-2.5-pro", Mode: api.ModeAPI} + second := submit("fallback-user-2", selected) + Expect(second.Code).To(Equal(http.StatusOK), second.Body.String()) + Expect(resolver.configs).To(HaveLen(2)) + Expect(resolver.configs[1].SessionID).To(Equal("fallback-session")) + }) + +}) diff --git a/pkg/aichat/database_threads_integration_test.go b/pkg/aichat/database_threads_integration_test.go index d9e1343f..5d3bba3f 100644 --- a/pkg/aichat/database_threads_integration_test.go +++ b/pkg/aichat/database_threads_integration_test.go @@ -79,77 +79,6 @@ var _ = Describe("Database chat sessions", func() { } }) - It("binds and accounts for the provider candidate selected after fallback", func(ctx SpecContext) { - testDB := dbtest.ForGinkgo(dbtest.Options{Name: "captain_aichat_selected_fallback"}) - db, err := database.Open(ctx, database.WithDSN(testDB.DSN()), database.WithMigrations()) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(db.Close) - store, err := aichat.NewDatabaseThreadStore(db) - Expect(err).NotTo(HaveOccurred()) - thread, err := store.Create(ctx, "Fallback") - Expect(err).NotTo(HaveOccurred()) - authority, err := aichat.NewDatabaseExecutionAuthority(db) - Expect(err).NotTo(HaveOccurred()) - provider := &fakeStreamingProvider{ - model: "gemini-2.5-pro", runtime: api.RuntimeOf(api.Google, api.ModeAPI), - events: []api.Event{ - {Kind: api.EventSystem, SessionID: "fallback-session", Model: "gemini-2.5-pro"}, - {Kind: api.EventText, Text: "Fallback answer", Model: "gemini-2.5-pro"}, - {Kind: api.EventResult, Success: true, Model: "gemini-2.5-pro", CostUSD: 0.25, - Usage: &api.Usage{InputTokens: 12, OutputTokens: 4}}, - }, - } - resolver := &fakeResolver{provider: provider} - service := aichat.NewService(aichat.ServiceOptions{ - Resolver: resolver, Threads: aichat.FixedThreadStore(store), Authority: authority, - }) - submit := func(id string, runtime api.Model) *httptest.ResponseRecorder { - response := httptest.NewRecorder() - service.Handler().ServeHTTP(response, requestJSON(http.MethodPost, "/api/chat", aichat.ChatRequest{ - ID: thread.ID, ThreadID: thread.ID, Trigger: "submit-message", Runtime: &runtime, - Messages: []aichat.UIMessage{{ - ID: id, Role: "user", Parts: []aichat.UIPart{{Type: "text", Text: "Use the selected runtime"}}, - }}, - })) - return response - } - - primary := api.Model{ - Name: "gpt-5.6-sol", Mode: api.ModeAPI, - Fallbacks: []api.Model{{Name: "gemini-2.5-pro", Mode: api.ModeAPI, Effort: api.EffortHigh}}, - } - first := submit("fallback-user-1", primary) - Expect(first.Code).To(Equal(http.StatusOK), first.Body.String()) - stored, err := store.Get(ctx, thread.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(stored.Runtime).To(Equal(&api.RuntimeIdentity{ - Model: "gemini-2.5-pro", Provider: api.Google.Name, Mode: api.ModeAPI, - }), "the identity records the fallback candidate that actually ran — model and runtime, not effort") - Expect(stored.ProviderSessionID).To(Equal("fallback-session")) - - aggregate, err := store.GetSession(ctx, thread.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(aggregate.Model).To(Equal("gemini-2.5-pro")) - Expect(aggregate.ModelMode).To(Equal(api.ModeAPI)) - Expect(aggregate.Usage.InputTokens).To(Equal(12)) - Expect(aggregate.Cost.Total()).To(BeNumerically("~", 0.25, 0.000001)) - sessionID := uuid.MustParse(thread.ID) - runs, err := db.ListPromptRuns(ctx, database.PromptRunFilter{SessionID: &sessionID}) - Expect(err).NotTo(HaveOccurred()) - Expect(runs).To(HaveLen(1)) - Expect(runs[0].Runtime.Requested.Model).To(Equal(primary.Name)) - Expect(runs[0].Runtime.Resolved.Model).To(Equal("gemini-2.5-pro")) - Expect(runs[0].Runtime.Resolved.Effort).To(Equal(string(api.EffortHigh))) - - conflict := submit("fallback-user-conflict", primary) - Expect(conflict.Code).To(Equal(http.StatusConflict), conflict.Body.String()) - selected := api.Model{Name: "gemini-2.5-pro", Mode: api.ModeAPI} - second := submit("fallback-user-2", selected) - Expect(second.Code).To(Equal(http.StatusOK), second.Body.String()) - Expect(resolver.configs).To(HaveLen(2)) - Expect(resolver.configs[1].SessionID).To(Equal("fallback-session")) - }) - It("rejects stale database history snapshots before turn admission or fork creation", func(ctx SpecContext) { testDB := dbtest.ForGinkgo(dbtest.Options{Name: "captain_aichat_stale_history"}) db, err := database.Open(ctx, database.WithDSN(testDB.DSN()), database.WithMigrations()) diff --git a/pkg/aichat/execution_database_authority.go b/pkg/aichat/execution_database_authority.go index bc64e680..65cb8e8a 100644 --- a/pkg/aichat/execution_database_authority.go +++ b/pkg/aichat/execution_database_authority.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "strings" "github.com/flanksource/captain/pkg/api" @@ -214,7 +215,9 @@ func (a *DatabaseExecutionAuthority) resolveToolApproval( if err != nil { return nil, err } - rendered, err := json.Marshal(run.RenderedSpec) + renderedSpec := maps.Clone(run.RenderedSpec) + delete(renderedSpec, "resolution") + rendered, err := json.Marshal(renderedSpec) if err != nil { return nil, fmt.Errorf("encode prompt run %s rendered spec: %w", run.ID, err) } diff --git a/pkg/aichat/layered_runtime_profile.go b/pkg/aichat/layered_runtime_profile.go index 8ba2012d..1c8732f6 100644 --- a/pkg/aichat/layered_runtime_profile.go +++ b/pkg/aichat/layered_runtime_profile.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" "github.com/flanksource/captain/pkg/runtimeprofiles" ) @@ -15,6 +16,7 @@ import ( type RuntimeProfileBase struct { System string Layers []api.SpecLayer + Saved *captainconfig.AIDefaults ProviderConfig api.Config } @@ -48,6 +50,11 @@ func NewLayeredRuntimeProfileProvider(options LayeredRuntimeProfileProviderOptio if err := api.ValidateSpecLayers(base.Layers...); err != nil { return RuntimeProfile{}, fmt.Errorf("chat runtime profile base: %w", err) } + if base.Saved != nil { + if err := base.Saved.Validate(); err != nil { + return RuntimeProfile{}, fmt.Errorf("chat saved defaults: %w", err) + } + } var defaultProfile string if request.Ref == "" && options.DefaultProfile != nil { defaultProfile, err = options.DefaultProfile(ctx) @@ -69,12 +76,12 @@ func NewLayeredRuntimeProfileProvider(options LayeredRuntimeProfileProviderOptio } return RuntimeProfile{}, err } - composed, err := api.ComposeSpecLayers(result.Layers...) + composed, err := api.ComposeSpecLayers(api.ResolveSpecOptions{Layers: result.Layers, Saved: base.Saved}) if err != nil { return RuntimeProfile{}, fmt.Errorf("compose chat runtime profile: %w", err) } return RuntimeProfile{ - System: base.System, Composed: composed, ProviderConfig: base.ProviderConfig, + System: base.System, Composed: composed, Saved: base.Saved, ProviderConfig: base.ProviderConfig, }, nil }), nil } diff --git a/pkg/aichat/messages.go b/pkg/aichat/messages.go index 44ca553b..968c4c2e 100644 --- a/pkg/aichat/messages.go +++ b/pkg/aichat/messages.go @@ -77,6 +77,7 @@ func requestSpec(request ChatRequest, profile RuntimeProfile, attachments map[pa // A requested posture is a permissions concern, so it no longer has to // conjure a sandbox to carry it — the profile's isolation setting stands. user := api.SpecLayer{Name: "chat request", Scope: api.SpecLayerUser, Spec: api.Spec{ + Explicit: request.Explicit.Clone(), Model: override, Budget: request.Budget, ToolPreferences: request.ToolPreferences, @@ -85,7 +86,7 @@ func requestSpec(request ChatRequest, profile RuntimeProfile, attachments map[pa SessionID: request.ProviderSessionID, }} layers := append([]api.SpecLayer(nil), profile.Composed.Trace...) - resolved, err := api.ResolveSpecLayers(append(layers, user)...) + resolved, err := api.ResolveSpecLayers(api.ResolveSpecOptions{Layers: append(layers, user), Saved: profile.Saved, RequireModel: true}) if err != nil { return api.ResolvedSpec{}, fmt.Errorf("resolve chat runtime profile: %w", err) } @@ -141,6 +142,11 @@ func chatModel(request ChatRequest) (api.Model, error) { } else if model := strings.TrimSpace(request.Model); model != "" { selected = api.Model{Name: model} } + for _, path := range []string{"/model", "/effort", "/temperature"} { + if request.Explicit.Has(path) { + selected = selected.WithExplicit(path) + } + } if err := api.ValidateSpecLayers(api.RequestSpecLayer("chat request", api.Spec{Model: selected})); err != nil { return api.Model{}, err } diff --git a/pkg/aichat/messages_model_ginkgo_test.go b/pkg/aichat/messages_model_ginkgo_test.go index 30da778e..83a650df 100644 --- a/pkg/aichat/messages_model_ginkgo_test.go +++ b/pkg/aichat/messages_model_ginkgo_test.go @@ -14,9 +14,9 @@ var _ = ginkgo.Describe("chatModel composition", func() { // default to claim — that gap is how an agent-backed session came back as api. // A catalog id carries no mode, so the authored profile mode survives. resolve := func(request ChatRequest) (api.Model, error) { - composed, err := api.ComposeSpecLayers(api.SpecLayer{Name: "application", Scope: api.SpecLayerGlobal, + composed, err := api.ComposeSpecLayers(api.ResolveSpecOptions{Layers: []api.SpecLayer{{Name: "application", Scope: api.SpecLayerGlobal, Spec: api.Spec{Model: api.Model{Name: "gpt-5.6-luna", Mode: api.ModeAPI}}, - }) + }}}) Expect(err).NotTo(HaveOccurred()) request.Messages = []UIMessage{{Role: "user", Parts: []UIPart{{Type: "text", Text: "Hello"}}}} resolved, err := requestSpec(request, RuntimeProfile{Composed: composed}, nil) diff --git a/pkg/aichat/request_presence.go b/pkg/aichat/request_presence.go new file mode 100644 index 00000000..f94d4745 --- /dev/null +++ b/pkg/aichat/request_presence.go @@ -0,0 +1,79 @@ +package aichat + +import ( + "encoding/json" + + "github.com/flanksource/captain/pkg/api" +) + +var chatSettingFields = map[string]string{ + "model": "model", "reasoningEffort": "effort", "temperature": "temperature", + "budget": "budget", "toolPreferences": "toolPreferences", +} + +func (r ChatRequest) WithExplicit(paths ...string) ChatRequest { + r.Explicit = (api.Spec{Explicit: r.Explicit}).WithExplicit(paths...).Explicit + return r +} + +func (r *ChatRequest) captureSettings(fields map[string]json.RawMessage) error { + projected := map[string]json.RawMessage{} + for wire, field := range chatSettingFields { + if value, present := fields[wire]; present { + projected[field] = value + } + } + if mode, present := fields["permissionMode"]; present { + value, err := json.Marshal(map[string]json.RawMessage{"mode": mode}) + if err != nil { + return err + } + projected["permissions"] = value + } + data, err := json.Marshal(projected) + if err != nil { + return err + } + var settings api.Spec + if err := json.Unmarshal(data, &settings); err != nil { + return err + } + r.Explicit = settings.Explicit + return nil +} + +func (r ChatRequest) MarshalJSON() ([]byte, error) { + type wireChatRequest ChatRequest + data, err := json.Marshal(wireChatRequest(r)) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return nil, err + } + settings := api.Spec{Explicit: r.Explicit, Model: api.Model{Name: r.Model, Effort: r.ReasoningEffort, Temperature: r.Temperature}, + Budget: r.Budget, ToolPreferences: r.ToolPreferences, Permissions: api.Permissions{Mode: r.PermissionMode}, + } + data, err = json.Marshal(settings) + if err != nil { + return nil, err + } + var projected map[string]json.RawMessage + if err := json.Unmarshal(data, &projected); err != nil { + return nil, err + } + for wire, field := range chatSettingFields { + delete(fields, wire) + if value, present := projected[field]; present { + fields[wire] = value + } + } + if r.Explicit.Has("/permissions/mode") { + fields["permissionMode"], err = json.Marshal(r.PermissionMode) + if err != nil { + return nil, err + } + } + return json.Marshal(fields) +} diff --git a/pkg/aichat/runtime_profile.go b/pkg/aichat/runtime_profile.go index 545bd033..2de57c5b 100644 --- a/pkg/aichat/runtime_profile.go +++ b/pkg/aichat/runtime_profile.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" ) // RuntimeProfile is the request-scoped application configuration for a chat. @@ -17,6 +18,7 @@ import ( type RuntimeProfile struct { System string Composed api.ComposedSpec + Saved *captainconfig.AIDefaults ProviderConfig api.Config } @@ -78,12 +80,11 @@ func (s *Service) runtimeProfile(ctx context.Context, options ...RuntimeProfileO return RuntimeProfile{}, err } if len(profile.Composed.Trace) == 0 { - if !api.IsEmpty(profile.Composed.Spec) || !api.IsEmpty(profile.Composed.Constraints) { + if (profile.Saved == nil && !api.IsEmpty(profile.Composed.Spec)) || !api.IsEmpty(profile.Composed.Constraints) { return RuntimeProfile{}, fmt.Errorf("chat runtime profile must include its composition trace") } - return profile, nil } - composed, err := api.ComposeSpecLayers(profile.Composed.Trace...) + composed, err := api.ComposeSpecLayers(api.ResolveSpecOptions{Layers: profile.Composed.Trace, Saved: profile.Saved}) if err != nil { return RuntimeProfile{}, fmt.Errorf("resolve chat runtime profile: %w", err) } diff --git a/pkg/aichat/runtime_profile_ginkgo_test.go b/pkg/aichat/runtime_profile_ginkgo_test.go index eb7266f6..3f8e314a 100644 --- a/pkg/aichat/runtime_profile_ginkgo_test.go +++ b/pkg/aichat/runtime_profile_ginkgo_test.go @@ -168,7 +168,7 @@ var _ = Describe("Resolved runtime profiles", func() { }) func mustRuntimeProfile(layers ...api.SpecLayer) aichat.RuntimeProfile { - composed, err := api.ComposeSpecLayers(layers...) + composed, err := api.ComposeSpecLayers(api.ResolveSpecOptions{Layers: layers}) Expect(err).NotTo(HaveOccurred()) return aichat.RuntimeProfile{Composed: composed} } diff --git a/pkg/aichat/saved_defaults_ginkgo_test.go b/pkg/aichat/saved_defaults_ginkgo_test.go new file mode 100644 index 00000000..a55cd16a --- /dev/null +++ b/pkg/aichat/saved_defaults_ginkgo_test.go @@ -0,0 +1,94 @@ +package aichat + +import ( + "context" + "encoding/json" + "errors" + "net/http" + + "github.com/flanksource/captain/pkg/aiflags" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" + "github.com/flanksource/captain/pkg/runtimeprofiles" + g "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = g.Describe("Chat saved defaults", func() { + request := ChatRequest{Messages: []UIMessage{{Role: "user", Parts: []UIPart{{Type: "text", Text: "Hello"}}}}} + profile := func(saved captainconfig.AIDefaults, spec api.Spec) RuntimeProfile { + return RuntimeProfile{Saved: &saved, Composed: api.ComposedSpec{Trace: []api.SpecLayer{ + {Name: "application", Scope: api.SpecLayerGlobal, Spec: spec}, + }}} + } + + g.It("uses the same injected defaults for partial composition and final chat resolution", func() { + saved := captainconfig.AIDefaults{DefaultModel: "api:sonnet:high,api:sol:medium", BudgetUSD: 4} + service := NewService(ServiceOptions{Profile: RuntimeProfileProviderFunc(func(context.Context, ...RuntimeProfileOption) (RuntimeProfile, error) { + return profile(saved, api.Spec{}), nil + })}) + loaded, err := service.runtimeProfile(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(loaded.Composed.Spec.Name).To(Equal("sonnet")) + Expect(loaded.Composed.Spec.Budget.Cost).To(Equal(float64(4))) + resolved, err := requestSpec(request, loaded, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Name).To(Equal("claude-sonnet-5")) + Expect(resolved.Spec.Fallbacks).To(HaveLen(1)) + Expect(resolved.Spec.Fallbacks[0].Mode).To(Equal(api.ModeAPI)) + Expect(resolved.Trace).To(HaveLen(2)) + Expect(resolved.Trace[1].Spec.Name).To(BeEmpty()) + Expect(resolved.Provenance["/model"].Source.Key).To(Equal("ai.defaultModel")) + }) + + g.It("chooses saved mode defaults for the final provider after a bare request changes families", func() { + saved := captainconfig.AIDefaults{DefaultModel: "agent:sonnet:high", Providers: map[string]captainconfig.ProviderDefaults{"openai": {Mode: "api", ReasoningEffort: "medium"}}} + selected := request + selected.Model = "sol" + resolved, err := requestSpec(selected, profile(saved, api.Spec{}), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Mode).To(Equal(api.ModeAPI)) + Expect(resolved.Spec.Effort).To(Equal(api.EffortMedium)) + Expect(resolved.Provenance["/mode"].Source.Key).To(Equal("ai.providers.openai.mode")) + }) + + g.It("preserves explicit zero budgets, false cache and empty fallbacks through chat JSON", func() { + var selected ChatRequest + Expect(json.Unmarshal([]byte(`{"runtime":{"model":"api:sonnet","noCache":false,"fallbacks":[]},"budget":{"cost":0}}`), &selected)).To(Succeed()) + encoded, err := json.Marshal(selected) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).To(And(ContainSubstring(`"cost":0`), ContainSubstring(`"noCache":false`), ContainSubstring(`"fallbacks":[]`))) + Expect(json.Unmarshal(encoded, &selected)).To(Succeed()) + selected.Messages = request.Messages + resolved, err := requestSpec(selected, profile(captainconfig.AIDefaults{DefaultModel: "api:sonnet,api:sol", BudgetUSD: 4, NoCache: true}, api.Spec{Budget: api.Budget{Cost: 8}}), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Budget.Cost).To(BeZero()) + Expect(resolved.Spec.NoCache).To(BeFalse()) + Expect(resolved.Spec.Fallbacks).To(BeEmpty()) + Expect(resolved.Provenance["/budget/cost"].Source.Name).To(Equal("chat request")) + }) + + g.It("validates malformed saved settings before a missing requested profile changes ownership", func() { + catalogCalls := 0 + provider, err := NewLayeredRuntimeProfileProvider(LayeredRuntimeProfileProviderOptions{ + Resolver: runtimeprofiles.NewResolver(func(context.Context) (*runtimeprofiles.Catalog, error) { + catalogCalls++ + return nil, runtimeprofiles.ErrNotFound + }), + Base: func(context.Context) (RuntimeProfileBase, error) { + return RuntimeProfileBase{Saved: &captainconfig.AIDefaults{Temperature: 3}}, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + _, err = provider.RuntimeProfile(context.Background(), WithRuntimeProfileRef("missing")) + Expect(runtimeProfileStatus(err)).To(Equal(http.StatusInternalServerError)) + Expect(err).To(MatchError(ContainSubstring("ai.temperature"))) + Expect(catalogCalls).To(BeZero()) + }) + + g.It("returns an actionable typed error for an unconfigured model", func() { + _, err := requestSpec(request, RuntimeProfile{}, nil) + Expect(errors.Is(err, aiflags.ErrUnconfigured)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("captain configure")) + }) +}) diff --git a/pkg/aichat/wire.go b/pkg/aichat/wire.go index 5cb97f6c..e49165b3 100644 --- a/pkg/aichat/wire.go +++ b/pkg/aichat/wire.go @@ -13,6 +13,7 @@ import ( // ChatRequest is the body posted by the AI SDK DefaultChatTransport. type ChatRequest struct { + Explicit api.FieldPresence `json:"-"` ID string `json:"id,omitempty"` Trigger string `json:"trigger,omitempty"` MessageID string `json:"messageId,omitempty"` @@ -43,7 +44,12 @@ func (r *ChatRequest) UnmarshalJSON(data []byte) error { return fmt.Errorf("toolApproval is server-owned; resolve approvals through the Captain session approval endpoint") } type wireChatRequest ChatRequest - return json.Unmarshal(data, (*wireChatRequest)(r)) + var decoded wireChatRequest + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *r = ChatRequest(decoded) + return r.captureSettings(fields) } // ChatContextItem carries app-owned structured state alongside its readable label. diff --git a/pkg/aichat/wire_ginkgo_test.go b/pkg/aichat/wire_ginkgo_test.go index 6c951e8c..e0694d04 100644 --- a/pkg/aichat/wire_ginkgo_test.go +++ b/pkg/aichat/wire_ginkgo_test.go @@ -81,7 +81,8 @@ var _ = Describe("AI SDK v6 wire types", func() { Expect(request.Runtime).NotTo(BeNil()) Expect(*request.Runtime).To(Equal(api.Model{ - Name: "sonnet", Mode: api.ModeAgent, Effort: api.EffortHigh, + Explicit: api.FieldPresence{"/model": true, "/mode": true, "/effort": true}, + Name: "sonnet", Mode: api.ModeAgent, Effort: api.EffortHigh, })) resolved, err := api.ResolveModel(*request.Runtime) Expect(err).NotTo(HaveOccurred()) From 7e9dff06d9d5e461b6b7e64de998fbacd2914cef Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 12:14:40 +0300 Subject: [PATCH 12/22] refactor(cli): Unify AI runtime resolution across CLI and prompt execution Consolidate AI runtime resolution across CLI, prompt rendering, profiles, and execution paths, preserving field provenance and explicit flag presence. Correct provider-alias configuration updates, sandbox handling, multi-runtime variant admission, structured output reporting, and transcript session linkage with expanded regression coverage. BREAKING CHANGE: Remove the legacy AIProviderOptions.ToConfig, AIRuntimeOptions.ToRequest, and prompt overlay resolution APIs in favor of unified runtime resolution. --- pkg/cli/ai.go | 334 +----------------- pkg/cli/ai_agent.go | 19 +- pkg/cli/ai_options.go | 159 +++++++++ pkg/cli/ai_output_test.go | 226 ++++++++++++ pkg/cli/ai_prompt_file.go | 227 ------------ pkg/cli/ai_prompt_file_test.go | 100 +++--- pkg/cli/ai_runtime_resolve.go | 92 +++++ pkg/cli/ai_sandbox.go | 62 +--- pkg/cli/ai_sandbox_test.go | 15 +- pkg/cli/ai_test.go | 290 ++------------- pkg/cli/attachments.go | 6 +- pkg/cli/attachments_gc.go | 6 +- pkg/cli/attachments_ginkgo_test.go | 7 +- pkg/cli/configure.go | 32 +- pkg/cli/configure_alias_ginkgo_test.go | 143 ++++++++ pkg/cli/configure_provider.go | 7 +- pkg/cli/gitagent_hook.go | 4 +- pkg/cli/gitagent_runtask.go | 11 +- pkg/cli/model_selection_ginkgo_test.go | 42 +-- pkg/cli/permissions_matrix_test.go | 4 +- pkg/cli/prompt_batch_run.go | 5 +- pkg/cli/prompt_batch_session.go | 6 +- pkg/cli/prompt_batch_session_ginkgo_test.go | 67 ++++ pkg/cli/prompt_entity.go | 11 +- pkg/cli/prompt_layers.go | 41 +-- pkg/cli/prompt_layers_ginkgo_test.go | 3 +- pkg/cli/prompt_observe.go | 78 +--- pkg/cli/prompt_observe_execute.go | 67 ++++ pkg/cli/prompt_profile_layers_ginkgo_test.go | 4 +- pkg/cli/prompt_records.go | 22 +- pkg/cli/prompt_render.go | 169 ++++----- pkg/cli/prompt_render_test.go | 34 +- pkg/cli/prompt_run.go | 32 +- pkg/cli/prompt_run_history.go | 7 - pkg/cli/prompt_run_persist.go | 82 +++-- pkg/cli/prompt_run_test.go | 43 ++- pkg/cli/prompt_runtime_variants.go | 110 ++++++ .../prompt_runtime_variants_ginkgo_test.go | 116 ++++++ pkg/cli/prompt_runtimes_ginkgo_test.go | 18 +- pkg/cli/prompt_schema.go | 8 +- pkg/cli/prompt_source.go | 55 +-- pkg/cli/prompt_source_test.go | 8 +- pkg/cli/prompt_sources.go | 20 +- pkg/cli/prompt_spec.go | 51 +-- pkg/cli/provider_defaults.go | 102 ------ pkg/cli/provider_defaults_test.go | 10 +- pkg/cli/serve_chat_profile.go | 40 ++- pkg/cli/serve_chat_profile_ginkgo_test.go | 29 +- pkg/cli/serve_disabled.go | 9 +- pkg/cli/serve_provider_defaults.go | 7 +- pkg/cli/serve_sandbox.go | 7 +- pkg/cli/verify.go | 4 +- pkg/cli/webapp/dist/index.html | 4 +- 53 files changed, 1547 insertions(+), 1508 deletions(-) create mode 100644 pkg/cli/ai_options.go create mode 100644 pkg/cli/ai_output_test.go create mode 100644 pkg/cli/ai_runtime_resolve.go create mode 100644 pkg/cli/configure_alias_ginkgo_test.go create mode 100644 pkg/cli/prompt_observe_execute.go create mode 100644 pkg/cli/prompt_runtime_variants.go create mode 100644 pkg/cli/prompt_runtime_variants_ginkgo_test.go diff --git a/pkg/cli/ai.go b/pkg/cli/ai.go index 0b1d2539..f12c6e00 100644 --- a/pkg/cli/ai.go +++ b/pkg/cli/ai.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "os" - "strconv" "strings" "time" @@ -13,332 +12,10 @@ import ( "github.com/flanksource/captain/pkg/ai/agent/setup" "github.com/flanksource/captain/pkg/ai/middleware" "github.com/flanksource/captain/pkg/ai/pricing" - "github.com/flanksource/captain/pkg/aiflags" "github.com/flanksource/captain/pkg/api" - "github.com/flanksource/captain/pkg/api/registry" - "github.com/flanksource/captain/pkg/captainconfig" "github.com/flanksource/captain/pkg/collections" ) -// loadSavedAI returns the saved AI defaults from ~/.captain.yaml. Errors are -// surfaced as zero-valued defaults rather than failing the command — a missing -// or unreadable config should never block `captain ai prompt`. -func loadSavedAI() captainconfig.AIDefaults { - return loadSavedConfig().AI -} - -func loadSavedConfig() captainconfig.Config { - cfg, _, err := captainconfig.Load() - if err != nil { - log.Warnf("captainconfig load: %v (continuing with zero defaults)", err) - return captainconfig.Config{} - } - return cfg -} - -// AIProviderOptions binds model selection plus the knobs that belong to the -// request rather than the model: the endpoint, the API key and the spend budget. -// -// The model flags themselves live in pkg/aiflags — a leaf any clicky CLI can embed -// without inheriting pkg/cli's ~1000 transitive packages. Embedding it here keeps -// captain's flag surface unchanged (clicky promotes embedded flags at any depth) -// while giving downstream repos the same parsing captain uses. -type AIProviderOptions struct { - aiflags.ModelFlags - - APIKey string `flag:"api-key" help:"API key (env: ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY, GOOGLE_API_KEY, DEEPSEEK_API_KEY)"` - APIURL string `flag:"api-url" help:"Override the provider endpoint, e.g. a 'captain ai mock' URL. Required for the openai cli mode, which ignores OPENAI_BASE_URL when a ChatGPT credential is stored"` - Budget string `flag:"budget" help:"Max spend in USD, 0=unlimited" default:"0"` - Sandbox string `flag:"sandbox" help:"Sandbox for the run: off|native|docker|git-agent, or a configured Docker/Git Agent backend"` -} - -// SandboxSelector trims the public sandbox mode or configured backend selector. -func (o AIProviderOptions) SandboxSelector() string { - return strings.TrimSpace(o.Sandbox) -} - -// BudgetUSD parses --budget, failing loud on malformed input. -func (o AIProviderOptions) BudgetUSD() (float64, error) { - return parseFloatFlag("budget", o.Budget) -} - -// parseFloatFlag parses a numeric string flag, returning a descriptive error -// instead of silently coercing malformed input to zero. -func parseFloatFlag(name, val string) (float64, error) { - if val == "" { - return 0, nil - } - f, err := strconv.ParseFloat(val, 64) - if err != nil { - return 0, fmt.Errorf("invalid --%s %q: %w", name, val, err) - } - return f, nil -} - -func (o AIProviderOptions) ToConfig() (ai.Config, error) { - savedCfg := loadSavedConfig() - saved := savedCfg.AI - budget, err := o.BudgetUSD() - if err != nil { - return ai.Config{}, err - } - if budget == 0 { - budget = saved.BudgetUSD - } - - // Sandbox precedence here is flag > global default > none: this path has no - // prompt file, so there is no frontmatter layer (that one is overlayCLI's). - sandbox, err := resolveSandboxSelection(o.SandboxSelector(), nil, savedCfg.Sandbox) - if err != nil { - return ai.Config{}, err - } - - // One resolve: flags → Model → saved per-provider defaults → catalog. The - // warn-and-continue policy for a broken config stays here (loadSavedConfig), so - // aiflags can hand the error back instead of swallowing it. - flags := o.ModelFlags - if forced := sandboxForcedMode(sandbox.Kind); forced != "" { - if value := strings.TrimSpace(flags.Mode); value != "" { - mode, ok := registry.ParseRuntimeMode(value) - if !ok { - return ai.Config{}, fmt.Errorf("invalid --mode %q (valid: %s)", value, registry.RuntimeModeList()) - } - if mode != forced { - return ai.Config{}, fmt.Errorf("sandbox %q requires %s mode, but --mode is %q", sandbox.Kind, forced, mode) - } - } - flags.Mode = string(forced) - } - m, err := flags.ResolveWith(saved) - if err != nil { - return ai.Config{}, err - } - return ai.Config{ - Model: m, - Budget: api.Budget{Cost: budget}, - APIKey: o.APIKey, - APIURL: strings.TrimSpace(o.APIURL), - SandboxSelection: sandboxSelectionConfig(sandbox, nil), - NoCache: o.NoCache || saved.NoCache, - SchemaRepair: schemaRepairConfig(savedCfg.Prompts.SchemaRepair), - }, nil -} - -// schemaRepairConfig reads the `model:`/`mode:` pair out of ~/.captain.yaml. -// `mode:` names the mechanism (api|agent|cli|cmux); the provider follows from the -// model name, and the pair is resolved where the repair provider is built. -func schemaRepairConfig(saved captainconfig.SchemaRepairDefaults) api.SchemaRepairConfig { - return api.SchemaRepairConfig{ - Model: api.Model{ - Name: strings.TrimSpace(saved.Model), - Mode: api.RuntimeMode(strings.TrimSpace(saved.Mode)), - }, - Prompt: strings.TrimSpace(saved.Prompt), - } -} - -func isZeroSchemaRepair(c api.SchemaRepairConfig) bool { - return strings.TrimSpace(c.Prompt) == "" && - c.Model.Name == "" && - c.Model.ID == "" && - c.Model.Mode == "" && - c.Model.Provider == nil && - c.Model.Temperature == nil && - c.Model.Effort == "" && - !c.Model.NoCache && - len(c.Model.Fallbacks) == 0 -} - -// AIRuntimeOptions binds the per-invocation knobs every AI command shares — -// model selection (via embedded AIProviderOptions), generation parameters -// (max tokens, temperature, timeout, reasoning), permission/sandbox toggles -// (edit, allowed/disallowed tools, permission mode), and ambient-context -// toggles (mcp/hooks/skills/user/project/memory/bare). It deliberately -// omits the user-prompt fields so non-prompt commands (e.g. gavel's lint -// --ai-fix loop) can embed it without inheriting a required --prompt flag. -// -// AIPromptOptions embeds this struct and adds Prompt/System/AppendSystem/ -// NoStream on top. -type AIRuntimeOptions struct { - AIProviderOptions - - // Effort and Temperature are NOT here: they describe the model and so live on - // the embedded aiflags.ModelFlags, promoted through AIProviderOptions. - // Redeclaring them would bind --effort twice and panic cobra at init. - MaxTokens int `flag:"max-tokens" help:"Maximum output tokens (0 = saved default or 4096)"` - MaxTurns int `flag:"max-turns" help:"Max agent turns 0-100, 0 = provider default (agent mode)"` - Resume string `flag:"resume" help:"Resume an existing session by id (agent and cli modes)"` - - Edit bool `flag:"edit" help:"Safe defaults: acceptEdits + Read/Edit/Write/Glob/Grep allowlist"` - AllowedTools []string `flag:"allowed-tools" help:"Override --edit's built-in allowlist (claude only)"` - DisallowedTools []string `flag:"disallowed-tools" help:"Tools to deny (claude only)"` - PermissionMode string `flag:"permission-mode" help:"acceptEdits|auto|bypassPermissions|default|plan"` - - NoMCP bool `flag:"no-mcp" help:"Disable all MCP servers"` - NoHooks bool `flag:"no-hooks" help:"Skip hooks"` - NoSkills bool `flag:"no-skills" help:"Disable slash commands"` - SkillDirs []string `flag:"skill-dir" help:"Additional skill/plugin directory (repeatable)"` - NoUser bool `flag:"no-user" help:"Skip user-level settings"` - NoProject bool `flag:"no-project" help:"Skip project-level settings"` - NoMemory bool `flag:"no-memory" help:"Skip auto-memory and CLAUDE.md"` - Bare bool `flag:"bare" help:"Skip hooks, skills, memory, and ambient settings"` -} - -var validPermissionModes = []string{"acceptEdits", "auto", "bypassPermissions", "default", "plan"} - -func validatePermissionMode(s string) error { - if s == "" { - return nil - } - for _, m := range validPermissionModes { - if s == m { - return nil - } - } - return fmt.Errorf("invalid --permission-mode %q (valid: %s)", s, strings.Join(validPermissionModes, "|")) -} - -type AIPromptOptions struct { - AIRuntimeOptions - - // File is a positional .prompt template path rendered through pkg/ai/prompt. - // The frontmatter sets model + any ai.Request option; the body is the prompt. - File string `args:"true" help:"Path to a .prompt template to render"` - Prompt string `flag:"prompt" clicky:"cli-file-read" help:"Prompt text, or @file to load and render a .prompt template" short:"p"` - System string `flag:"system" help:"System prompt" short:"s"` - AppendSystem string `flag:"append-system" help:"Append text to the default system prompt"` - Var []string `flag:"var" help:"Template variable key=value (repeatable)" short:"V"` - Attach []string `flag:"attach" help:"Attach a local path or URL (repeatable; RFC 4180 comma-separated values allowed)" short:"A"` - MultiModels []string `flag:"multi-models" help:"Run prompt once per runtime selector in parallel, e.g. cli:sonnet-5,cmux:opus (repeatable; comma-separated allowed)" short:"M"` - Timeout string `flag:"timeout" help:"Request timeout (default 120s; a relocating sandbox waits for the remote agent instead)"` - NoStream bool `flag:"no-stream" help:"Disable streaming; print only the final text to stdout"` - - // RuntimeProfile is the catalog profile (id or name) `captain prompt - // run|render --runtime-profile` layers beneath the frontmatter. It is not a - // flag here: the deprecated `captain ai prompt` alias does not grow it. - RuntimeProfile string -} - -type AIPromptResult struct { - Text string `json:"text" pretty:"label=Response"` - StructuredOutput map[string]any `json:"structuredOutput,omitempty" pretty:"-"` - Model string `json:"model" pretty:"label=Model"` - Provider string `json:"provider" pretty:"label=Provider"` - Mode string `json:"mode" pretty:"label=Mode"` - Dir string `json:"dir,omitempty" pretty:"label=Dir"` - SessionID string `json:"sessionId,omitempty" pretty:"label=Session"` - HistoryFile string `json:"historyFile,omitempty" pretty:"label=History"` - Input ai.Request `json:"input" pretty:"-"` - InputTokens int `json:"inputTokens" pretty:"label=Input Tokens"` - Output int `json:"outputTokens" pretty:"label=Output Tokens"` - CostUSD float64 `json:"costUSD,omitempty" pretty:"label=Cost USD"` - Duration string `json:"duration" pretty:"label=Duration"` -} - -// ToRequest translates the runtime knobs into the typed ai.Request, overlaying -// saved defaults from ~/.captain.yaml onto unset fields. Precedence is -// flag > saved > built-in: max-tokens uses the explicit flag when > 0, else the -// saved default, else 4096; --effort uses the flag when set, else saved. -// The ambient toggles are negative flags (--no-mcp, …) that compose with the -// saved No* defaults via OR, so either a flag or a saved default switches a -// feature off; re-enabling a saved-off feature is done via `captain configure`. -// -// systemPrompt / appendSystemPrompt / userPrompt are passed explicitly so -// non-prompt callers (gavel's ai-fix loop) can build them per-iteration without -// leaking those fields into the shared runtime struct. Parse/validation errors -// are returned rather than silently coerced to zero values. -func (o AIRuntimeOptions) ToRequest(systemPrompt, appendSystemPrompt, userPrompt string) (ai.Request, error) { - saved := loadSavedAI() - - temperature, err := parseFloatFlag("temperature", o.Temperature) - if err != nil { - return ai.Request{}, err - } - if temperature < 0 || temperature > 2 { - return ai.Request{}, fmt.Errorf("invalid --temperature %v (valid: 0.0-2.0)", temperature) - } - if o.MaxTurns < 0 || o.MaxTurns > 100 { - return ai.Request{}, fmt.Errorf("invalid --max-turns %d (valid: 0-100, 0=provider default)", o.MaxTurns) - } - if err := validatePermissionMode(o.PermissionMode); err != nil { - return ai.Request{}, err - } - - maxTokens := o.MaxTokens - switch { - case maxTokens > 0: // explicit flag wins - case saved.MaxTokens != 0: - maxTokens = saved.MaxTokens - default: - maxTokens = 4096 - } - - // Resolve the USD budget onto the request (flag > saved) so the runtimes that - // read req.Budget.Cost — the anthropic cli and agent — enforce it without a - // later config-side reconciliation that not every path performs (finding A4). - budget, err := parseFloatFlag("budget", o.Budget) - if err != nil { - return ai.Request{}, err - } - if budget == 0 { - budget = saved.BudgetUSD - } - - effort := o.Effort - if err := api.Effort(effort).Validate(); err != nil { - return ai.Request{}, fmt.Errorf("invalid --effort %q: %w", effort, err) - } - - // Temperature is *float64 on the model: leave it nil for the default 0 so an - // explicit 0 and "unset" hash identically (matches the prior flat behaviour); - // no captain provider sends temperature to the model, only the cache key. - var temperaturePtr *float64 - if temperature != 0 { - t := temperature - temperaturePtr = &t - } - - perms := api.Permissions{ - Tools: api.ToolsFromLists(o.AllowedTools, o.DisallowedTools), - MCP: api.MCP{Disabled: o.NoMCP || saved.NoMCP}, - } - perms.Mode = api.PermissionMode(o.PermissionMode) - if o.Edit { - perms.Presets = append(perms.Presets, api.PresetEdit) - if perms.Mode == "" { - perms.Mode = api.PermissionAcceptEdits - } - } - - return ai.Request{ - Prompt: api.Prompt{System: systemPrompt, AppendSystem: appendSystemPrompt, User: userPrompt}, - Model: api.Model{Temperature: temperaturePtr, Effort: api.Effort(effort), NoCache: o.NoCache || saved.NoCache}, - Budget: api.Budget{Cost: budget, MaxTokens: maxTokens, MaxTurns: o.MaxTurns}, - Memory: api.Memory{ - Skills: o.SkillDirs, - SkipHooks: o.NoHooks || saved.NoHooks, - SkipSkills: o.NoSkills || saved.NoSkills, - SkipUser: o.NoUser || saved.NoUser, - SkipProject: o.NoProject || saved.NoProject, - SkipMemory: o.NoMemory || saved.NoMemory, - Bare: o.Bare, - }, - Permissions: perms, - SessionID: o.Resume, - }, nil -} - -// ToRequest delegates to AIRuntimeOptions.ToRequest, lifting the prompt -// fields the prompt-shaped command owns onto the typed request. -func (o AIPromptOptions) ToRequest() (ai.Request, error) { - req, err := o.AIRuntimeOptions.ToRequest(o.System, o.AppendSystem, o.Prompt) - if err != nil { - return ai.Request{}, err - } - req.Prompt.Attachments, err = attachmentRefsFromFlags(o.Attach) - return req, err -} - func executePromptRequest(parent context.Context, req ai.Request, cfg ai.Config, timeout time.Duration, noStream bool) (any, error) { ctx, cancel, err := runContext(parent, req, remoteAwareTimeout(req, cfg, timeout)) if err != nil { @@ -680,10 +357,14 @@ type AITestResult struct { } func RunAITest(opts AITestOptions) (any, error) { - cfg, err := opts.ToConfig() + resolved, err := resolveInvocation(AIRuntimeOptions{AIProviderOptions: opts.AIProviderOptions}, []api.SpecLayer{api.PromptSpecLayer("connectivity test", api.Spec{ + Prompt: api.Prompt{User: "Respond with exactly: ok"}, Budget: api.Budget{MaxTokens: 10}, + })}) if err != nil { return nil, err } + cfg := resolved.Config + logRuntimeWarnings(resolved.Resolution.Warnings) if cfg.Model.Name == "" { return nil, fmt.Errorf("no model: pass --model or run 'captain configure' to set a default") } @@ -703,10 +384,7 @@ func RunAITest(opts AITestOptions) (any, error) { defer cancel() start := time.Now() - _, err = p.Execute(ctx, ai.Request{ - Prompt: api.Prompt{User: "Respond with exactly: ok"}, - Budget: api.Budget{MaxTokens: 10}, - }) + _, err = p.Execute(ctx, resolved.Request) result := AITestResult{ Model: p.GetModel(), diff --git a/pkg/cli/ai_agent.go b/pkg/cli/ai_agent.go index f8c755b8..2c4b3479 100644 --- a/pkg/cli/ai_agent.go +++ b/pkg/cli/ai_agent.go @@ -168,10 +168,14 @@ func RunAIAgent(opts AIAgentOptions) (any, error) { return nil, fmt.Errorf("prompt text required (use --prompt or pipe via stdin)") } - cfg, err := opts.ToConfig() + resolved, err := resolveInvocation(opts.AIRuntimeOptions, []api.SpecLayer{api.PromptSpecLayer("agent prompt", api.Spec{ + Prompt: api.Prompt{System: opts.System, AppendSystem: opts.AppendSystem, User: opts.Prompt}, + })}) if err != nil { return nil, err } + cfg := resolved.Config + logRuntimeWarnings(resolved.Resolution.Warnings) if cfg.Model.Name == "" { return nil, fmt.Errorf("no model: pass --model or run 'captain configure' to set a default") } @@ -179,12 +183,7 @@ func RunAIAgent(opts AIAgentOptions) (any, error) { if err != nil { return nil, err } - // Validate runtime knobs (temperature/permission-mode/effort) and - // snapshot the base request once; Build only varies the prompt per turn. - baseReq, err := opts.ToRequest(opts.System, opts.AppendSystem, opts.Prompt) - if err != nil { - return nil, err - } + baseReq := resolved.Request timeout, _ := time.ParseDuration(opts.Timeout) if timeout <= 0 { @@ -212,11 +211,7 @@ func RunAIAgent(opts AIAgentOptions) (any, error) { return nil, err } - cwd, err := os.Getwd() - if err != nil { - return nil, err - } - baseReq.SetCwd(cwd) + cwd := baseReq.Cwd() renderer := NewEventRenderer(os.Stderr) start := time.Now() diff --git a/pkg/cli/ai_options.go b/pkg/cli/ai_options.go new file mode 100644 index 00000000..654da31b --- /dev/null +++ b/pkg/cli/ai_options.go @@ -0,0 +1,159 @@ +package cli + +import ( + "fmt" + "strconv" + "strings" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/aiflags" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" +) + +func loadSavedConfig() (captainconfig.Config, error) { + cfg, _, err := captainconfig.Load() + if err != nil { + return captainconfig.Config{}, fmt.Errorf("load Captain configuration: %w", err) + } + return cfg, nil +} + +// AIProviderOptions binds model selection plus the knobs that belong to the +// request rather than the model: the endpoint, the API key and the spend budget. +// +// The model flags themselves live in pkg/aiflags — a leaf any clicky CLI can embed +// without inheriting pkg/cli's ~1000 transitive packages. Embedding it here keeps +// captain's flag surface unchanged (clicky promotes embedded flags at any depth) +// while giving downstream repos the same parsing captain uses. +type AIProviderOptions struct { + aiflags.ModelFlags + + APIKey string `flag:"api-key" help:"API key (env: ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY, GOOGLE_API_KEY, DEEPSEEK_API_KEY)"` + APIURL string `flag:"api-url" help:"Override the provider endpoint, e.g. a 'captain ai mock' URL. Required for the openai cli mode, which ignores OPENAI_BASE_URL when a ChatGPT credential is stored"` + Budget string `flag:"budget" help:"Max spend in USD, 0=unlimited"` + Sandbox string `flag:"sandbox" help:"Sandbox for the run: off|native|docker|git-agent, or a configured Docker/Git Agent backend"` +} + +// SandboxSelector trims the public sandbox mode or configured backend selector. +func (o AIProviderOptions) SandboxSelector() string { + return strings.TrimSpace(o.Sandbox) +} + +// BudgetUSD parses --budget, failing loud on malformed input. +func (o AIProviderOptions) BudgetUSD() (float64, error) { + return parseFloatFlag("budget", o.Budget) +} + +// parseFloatFlag parses a numeric string flag, returning a descriptive error +// instead of silently coercing malformed input to zero. +func parseFloatFlag(name, val string) (float64, error) { + if val == "" { + return 0, nil + } + f, err := strconv.ParseFloat(val, 64) + if err != nil { + return 0, fmt.Errorf("invalid --%s %q: %w", name, val, err) + } + return f, nil +} + +// schemaRepairConfig reads the `model:`/`mode:` pair out of ~/.captain.yaml. +// `mode:` names the mechanism (api|agent|cli|cmux); the provider follows from the +// model name, and the pair is resolved where the repair provider is built. +func schemaRepairConfig(saved captainconfig.SchemaRepairDefaults) api.SchemaRepairConfig { + return api.SchemaRepairConfig{ + Model: api.Model{ + Name: strings.TrimSpace(saved.Model), + Mode: api.RuntimeMode(strings.TrimSpace(saved.Mode)), + }, + Prompt: strings.TrimSpace(saved.Prompt), + } +} + +// AIRuntimeOptions binds the per-invocation knobs every AI command shares — +// model selection (via embedded AIProviderOptions), generation parameters +// (max tokens, temperature, timeout, reasoning), permission/sandbox toggles +// (edit, allowed/disallowed tools, permission mode), and ambient-context +// toggles (mcp/hooks/skills/user/project/memory/bare). It deliberately +// omits the user-prompt fields so non-prompt commands (e.g. gavel's lint +// --ai-fix loop) can embed it without inheriting a required --prompt flag. +// +// AIPromptOptions embeds this struct and adds Prompt/System/AppendSystem/ +// NoStream on top. +type AIRuntimeOptions struct { + AIProviderOptions + ExplicitFields api.FieldPresence `flag:"-" json:"-" yaml:"-"` + + // Effort and Temperature are NOT here: they describe the model and so live on + // the embedded aiflags.ModelFlags, promoted through AIProviderOptions. + // Redeclaring them would bind --effort twice and panic cobra at init. + MaxTokens int `flag:"max-tokens" help:"Maximum output tokens (unset = saved default or 4096; 0 = provider default)"` + MaxTurns int `flag:"max-turns" help:"Max agent turns 0-100, 0 = provider default (agent mode)"` + Resume string `flag:"resume" help:"Resume an existing session by id (agent and cli modes)"` + + Edit bool `flag:"edit" help:"Safe defaults: acceptEdits + Read/Edit/Write/Glob/Grep allowlist"` + AllowedTools []string `flag:"allowed-tools" help:"Override --edit's built-in allowlist (claude only)"` + DisallowedTools []string `flag:"disallowed-tools" help:"Tools to deny (claude only)"` + PermissionMode string `flag:"permission-mode" help:"acceptEdits|auto|bypassPermissions|default|plan"` + + NoMCP bool `flag:"no-mcp" help:"Disable all MCP servers"` + NoHooks bool `flag:"no-hooks" help:"Skip hooks"` + NoSkills bool `flag:"no-skills" help:"Disable slash commands"` + SkillDirs []string `flag:"skill-dir" help:"Additional skill/plugin directory (repeatable)"` + NoUser bool `flag:"no-user" help:"Skip user-level settings"` + NoProject bool `flag:"no-project" help:"Skip project-level settings"` + NoMemory bool `flag:"no-memory" help:"Skip auto-memory and CLAUDE.md"` + Bare bool `flag:"bare" help:"Skip hooks, skills, memory, and ambient settings"` +} + +var validPermissionModes = []string{"acceptEdits", "auto", "bypassPermissions", "default", "plan"} + +func validatePermissionMode(s string) error { + if s == "" { + return nil + } + for _, m := range validPermissionModes { + if s == m { + return nil + } + } + return fmt.Errorf("invalid --permission-mode %q (valid: %s)", s, strings.Join(validPermissionModes, "|")) +} + +type AIPromptOptions struct { + AIRuntimeOptions + + // File is a positional .prompt template path rendered through pkg/ai/prompt. + // The frontmatter sets model + any ai.Request option; the body is the prompt. + File string `args:"true" help:"Path to a .prompt template to render"` + Prompt string `flag:"prompt" clicky:"cli-file-read" help:"Prompt text, or @file to load and render a .prompt template" short:"p"` + System string `flag:"system" help:"System prompt" short:"s"` + AppendSystem string `flag:"append-system" help:"Append text to the default system prompt"` + Var []string `flag:"var" help:"Template variable key=value (repeatable)" short:"V"` + Attach []string `flag:"attach" help:"Attach a local path or URL (repeatable; RFC 4180 comma-separated values allowed)" short:"A"` + MultiModels []string `flag:"multi-models" help:"Run prompt once per runtime selector in parallel, e.g. cli:sonnet-5,cmux:opus (repeatable; comma-separated allowed)" short:"M"` + Timeout string `flag:"timeout" help:"Request timeout (default 120s; a relocating sandbox waits for the remote agent instead)"` + NoStream bool `flag:"no-stream" help:"Disable streaming; print only the final text to stdout"` + + // RuntimeProfile is the catalog profile (id or name) `captain prompt + // run|render --runtime-profile` layers beneath the frontmatter. It is not a + // flag here: the deprecated `captain ai prompt` alias does not grow it. + RuntimeProfile string +} + +type AIPromptResult struct { + Text string `json:"text" pretty:"label=Response"` + StructuredOutput map[string]any `json:"structuredOutput,omitempty" pretty:"-"` + Model string `json:"model" pretty:"label=Model"` + Provider string `json:"provider" pretty:"label=Provider"` + Mode string `json:"mode" pretty:"label=Mode"` + Dir string `json:"dir,omitempty" pretty:"label=Dir"` + SessionID string `json:"sessionId,omitempty" pretty:"label=Session"` + HistoryFile string `json:"historyFile,omitempty" pretty:"label=History"` + Input ai.Request `json:"input" pretty:"-"` + InputTokens int `json:"inputTokens" pretty:"label=Input Tokens"` + Output int `json:"outputTokens" pretty:"label=Output Tokens"` + CostUSD float64 `json:"costUSD,omitempty" pretty:"label=Cost USD"` + Duration string `json:"duration" pretty:"label=Duration"` +} diff --git a/pkg/cli/ai_output_test.go b/pkg/cli/ai_output_test.go new file mode 100644 index 00000000..82bb6976 --- /dev/null +++ b/pkg/cli/ai_output_test.go @@ -0,0 +1,226 @@ +package cli + +import ( + "context" + "encoding/json" + "testing" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/commons-db/shell" +) + +type promptResultProvider struct{} + +func (promptResultProvider) GetModel() string { return "claude-sonnet-4-6" } + +func (promptResultProvider) GetRuntime() ai.Runtime { return ai.RuntimeOf(ai.Anthropic, ai.ModeAPI) } + +func (promptResultProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { + return &ai.Response{ + Text: "done", + Model: "claude-sonnet-4-6", + Runtime: ai.RuntimeOf(ai.Anthropic, ai.ModeAPI), + Usage: ai.Usage{InputTokens: 12, OutputTokens: 7}, + }, nil +} + +type structuredPromptResultProvider struct{} + +func (structuredPromptResultProvider) GetModel() string { return "claude-sonnet-4-6" } + +func (structuredPromptResultProvider) GetRuntime() ai.Runtime { + return ai.RuntimeOf(ai.Anthropic, ai.ModeAPI) +} + +func (structuredPromptResultProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { + return &ai.Response{ + Text: `{"answer":"42"}`, + StructuredData: json.RawMessage(`{"answer":"42"}`), + Model: "claude-sonnet-4-6", + Runtime: ai.RuntimeOf(ai.Anthropic, ai.ModeAPI), + }, nil +} + +type promptResultStreamingProvider struct{} + +func (promptResultStreamingProvider) GetModel() string { return "gpt-5-codex" } + +func (promptResultStreamingProvider) GetRuntime() ai.Runtime { + return ai.RuntimeOf(ai.OpenAI, ai.ModeCLI) +} + +func (promptResultStreamingProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { + return nil, nil +} + +func (promptResultStreamingProvider) ExecuteStream(_ context.Context, _ ai.Request) (<-chan ai.Event, error) { + events := make(chan ai.Event, 3) + events <- ai.Event{Kind: ai.EventSystem, SessionID: "stream-session-1", Model: "gpt-5-codex"} + events <- ai.Event{Kind: ai.EventText, Text: "streamed", Model: "gpt-5-codex"} + events <- ai.Event{Kind: ai.EventResult, Model: "gpt-5-codex", Usage: &ai.Usage{InputTokens: 21, OutputTokens: 9}, CostUSD: 0.02} + close(events) + return events, nil +} + +type structuredResultStreamingProvider struct { + text string +} + +func (structuredResultStreamingProvider) GetModel() string { return "gpt-5-codex" } + +func (structuredResultStreamingProvider) GetRuntime() ai.Runtime { + return ai.RuntimeOf(ai.OpenAI, ai.ModeCLI) +} + +func (structuredResultStreamingProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { + return nil, nil +} + +func (p structuredResultStreamingProvider) ExecuteStream(_ context.Context, _ ai.Request) (<-chan ai.Event, error) { + events := make(chan ai.Event, 2) + if p.text != "" { + events <- ai.Event{Kind: ai.EventText, Text: p.text} + } + events <- ai.Event{Kind: ai.EventResult, StructuredData: json.RawMessage(`{"answer":"42"}`)} + close(events) + return events, nil +} + +func TestRunBuffered_JSONIncludesFullInputSpec(t *testing.T) { + req := ai.Request{ + Model: api.Model{Name: "claude-sonnet-4-6", Mode: api.ModeAPI, Effort: api.EffortMedium}, + Prompt: api.Prompt{System: "be precise", User: "summarize"}, + Budget: api.Budget{MaxTokens: 2048}, + Setup: &shell.Setup{Cwd: "/repo"}, + Permissions: api.Permissions{ + Presets: []api.Preset{api.PresetEdit}, + Tools: api.Tools{"Read": api.ToolPolicyAllow}, + }, + SessionID: "resume-1", + } + + got, err := runBuffered(context.Background(), promptResultProvider{}, req) + if err != nil { + t.Fatal(err) + } + result, ok := got.(AIPromptResult) + if !ok { + t.Fatalf("runBuffered returned %T, want AIPromptResult", got) + } + if result.Input.Prompt.User != "summarize" || result.Input.Model.Name != "claude-sonnet-4-6" { + t.Fatalf("result input = %+v, want original request", result.Input) + } + if result.InputTokens != 12 { + t.Fatalf("InputTokens = %d, want 12", result.InputTokens) + } + if result.Model != "claude-sonnet-4-6" || result.Provider != "anthropic" || result.Mode != "api" || result.Dir != "/repo" || result.SessionID != "resume-1" { + t.Fatalf("resolved fields = model %q provider %q mode %q dir %q session %q", result.Model, result.Provider, result.Mode, result.Dir, result.SessionID) + } + + var encoded map[string]any + data, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &encoded); err != nil { + t.Fatal(err) + } + input, ok := encoded["input"].(map[string]any) + if !ok { + t.Fatalf("json input = %T, want object: %s", encoded["input"], data) + } + prompt, ok := input["prompt"].(map[string]any) + if !ok || prompt["user"] != "summarize" || prompt["system"] != "be precise" { + t.Fatalf("json input.prompt = %#v, want rendered prompt", input["prompt"]) + } + // input is the authored spec, so it carries the model and effort; the resolved + // runtime is run history and is published on the result itself. + if input["model"] != "claude-sonnet-4-6" || input["effort"] != "medium" { + t.Fatalf("json input model fields = %#v", input) + } + if encoded["provider"] != "anthropic" || encoded["mode"] != "api" { + t.Fatalf("json runtime = %#v/%#v, want the resolved pair", encoded["provider"], encoded["mode"]) + } + setup, ok := input["setup"].(map[string]any) + if !ok || setup["cwd"] != "/repo" { + t.Fatalf("json input.setup = %#v, want cwd /repo", input["setup"]) + } + if input["sessionId"] != "resume-1" || encoded["sessionId"] != "resume-1" || encoded["dir"] != "/repo" { + t.Fatalf("json session/dir fields = top %#v input %#v", encoded, input) + } + if got := encoded["inputTokens"]; got != float64(12) { + t.Fatalf("json inputTokens = %#v, want 12", got) + } +} + +func TestRunBuffered_PreservesStructuredOutput(t *testing.T) { + got, err := runBuffered(context.Background(), structuredPromptResultProvider{}, ai.Request{}) + if err != nil { + t.Fatal(err) + } + result, ok := got.(AIPromptResult) + if !ok { + t.Fatalf("runBuffered returned %T, want AIPromptResult", got) + } + if result.Text != `{"answer":"42"}` { + t.Fatalf("Text = %q, want JSON transcript text", result.Text) + } + if result.StructuredOutput["answer"] != "42" { + t.Fatalf("StructuredOutput = %#v, want decoded answer", result.StructuredOutput) + } +} + +func TestRunStreaming_JSONIncludesFullInputSpec(t *testing.T) { + req := ai.Request{ + Model: api.Model{Name: "gpt-5-codex", Mode: api.ModeCLI, Effort: api.EffortHigh}, + Prompt: api.Prompt{User: "fix tests"}, + Setup: &shell.Setup{Cwd: "/repo"}, + Permissions: api.Permissions{MCP: api.MCP{Disabled: true}}, + } + + got, err := runStreaming(context.Background(), promptResultStreamingProvider{}, req) + if err != nil { + t.Fatal(err) + } + result, ok := got.(AIPromptResult) + if !ok { + t.Fatalf("runStreaming returned %T, want AIPromptResult", got) + } + if result.Input.Prompt.User != "fix tests" || result.Input.Model.Mode != api.ModeCLI { + t.Fatalf("result input = %+v, want original request", result.Input) + } + if result.Dir != "/repo" || result.SessionID != "stream-session-1" || result.Input.SessionID != "stream-session-1" { + t.Fatalf("dir/session = dir %q session %q input session %q", result.Dir, result.SessionID, result.Input.SessionID) + } + if result.InputTokens != 21 || result.Output != 9 || result.CostUSD != 0.02 { + t.Fatalf("usage/cost = input %d output %d cost %f", result.InputTokens, result.Output, result.CostUSD) + } +} + +func TestRunStreaming_StructuredResultIsReturnedOnce(t *testing.T) { + for _, tc := range []struct { + name string + text string + }{ + {name: "result only"}, + {name: "replaces prior text", text: "discarded narrative"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := runStreaming(context.Background(), structuredResultStreamingProvider{text: tc.text}, ai.Request{}) + if err != nil { + t.Fatal(err) + } + result, ok := got.(AIPromptResult) + if !ok { + t.Fatalf("runStreaming returned %T, want AIPromptResult", got) + } + if result.Text != `{"answer":"42"}` { + t.Fatalf("Text = %q, want authoritative structured JSON once", result.Text) + } + if result.StructuredOutput["answer"] != "42" { + t.Fatalf("StructuredOutput = %#v, want decoded answer", result.StructuredOutput) + } + }) + } +} diff --git a/pkg/cli/ai_prompt_file.go b/pkg/cli/ai_prompt_file.go index 0e87039f..9ca85066 100644 --- a/pkg/cli/ai_prompt_file.go +++ b/pkg/cli/ai_prompt_file.go @@ -7,7 +7,6 @@ import ( "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/ai/prompt" "github.com/flanksource/captain/pkg/api" - "github.com/flanksource/captain/pkg/api/registry" "github.com/flanksource/commons-db/shell" ) @@ -45,177 +44,6 @@ func parseVars(pairs []string) (map[string]any, error) { return data, nil } -// overlayCLI layers the CLI flags over the rendered file spec, implementing the -// precedence CLI flag (non-zero) > frontmatter > saved defaults > built-in. The -// user prompt always comes from the rendered template body; everything else is -// merged per nested group. Negative toggles (--no-*) OR across all three layers, -// matching the existing flag semantics. Range/enum validation is left to the -// caller via req.Validate so the rules live in one place (pkg/api). -func overlayCLI(base ai.Request, baseCfg ai.Config, o AIPromptOptions) (ai.Request, ai.Config, error) { - saved := loadSavedAI() - - temperature, err := parseFloatFlag("temperature", o.Temperature) - if err != nil { - return base, baseCfg, err - } - budget, err := parseFloatFlag("budget", o.Budget) - if err != nil { - return base, baseCfg, err - } - - req := base - - bm := base.Model - if bm.Name == "" { - bm.Name = baseCfg.Model.Name - } - if bm.ID == "" { - bm.ID = baseCfg.Model.ID - } - if bm.Mode == "" { - bm.Mode = baseCfg.Model.Mode - } - if bm.Provider == nil { - bm.Provider = baseCfg.Model.Provider - } - if bm.Temperature == nil { - bm.Temperature = baseCfg.Model.Temperature - } - if bm.Effort == "" { - bm.Effort = baseCfg.Model.Effort - } - identity := selectModelIdentity( - api.Model{Name: bm.Name, ID: bm.ID, Mode: bm.Mode, Provider: bm.Provider}, - api.Model{Name: o.Model}, - ) - m := bm - m.Name, m.ID, m.Mode, m.Provider = identity.Name, identity.ID, identity.Mode, identity.Provider - if temperature != 0 { - t := temperature - m.Temperature = &t - } - m.Effort = api.Effort(firstNonEmpty(o.Effort, string(bm.Effort))) - m.NoCache = o.NoCache || bm.NoCache || saved.NoCache - m.Fallbacks = firstFallbacks(o.Fallback, bm.Fallbacks) - - requestedMode := registry.RuntimeMode("") - if value := strings.TrimSpace(o.Mode); value != "" { - var ok bool - requestedMode, ok = registry.ParseRuntimeMode(value) - if !ok { - return base, baseCfg, fmt.Errorf("invalid --mode %q (valid: %s)", o.Mode, registry.RuntimeModeList()) - } - } - // Sandbox precedence: --sandbox > frontmatter (req.Sandbox) > global default. - // Resolved here rather than at the end because the winning kind can force the - // runtime mode below; the selection is recorded onto req/cfg once cfg exists. - sandbox, err := resolveRunSandbox(&req, o.SandboxSelector()) - if err != nil { - return base, baseCfg, err - } - if selector := o.SandboxSelector(); selector != "" || req.Sandbox == nil { - resolved := sandboxRefFromSelection(sandbox) - if req.Sandbox != nil { - resolved.Policy = req.Sandbox.Policy - resolved.Agent = req.Sandbox.Agent - resolved.Dispatch = req.Sandbox.Dispatch - } - req.Sandbox = &resolved - } else if req.Sandbox.Mode == "" { - req.Sandbox.Mode = sandbox.Kind - } - if forced := sandboxForcedMode(sandbox.Kind); forced != "" { - if requestedMode != "" && requestedMode != forced { - return base, baseCfg, fmt.Errorf("sandbox %q requires %s mode, but --mode is %q", sandbox.Kind, forced, requestedMode) - } - requestedMode = forced - } - if requestedMode != "" { - m.Mode = "" - if len(o.Fallback) == 0 { - for i := range m.Fallbacks { - m.Fallbacks[i].Mode = "" - } - } - m, err = m.WithMode(requestedMode) - if err != nil { - return base, baseCfg, err - } - } - m, err = applyProviderDefaults(m, saved) - if err != nil { - return base, baseCfg, err - } - req.Model, err = ai.Resolve(m) - if err != nil { - return base, baseCfg, err - } - - req.Budget.MaxTokens = firstPositive(o.MaxTokens, base.Budget.MaxTokens, baseCfg.Budget.MaxTokens, saved.MaxTokens, 4096) - req.Budget.Cost = firstPositiveFloat(budget, base.Budget.Cost, baseCfg.Budget.Cost, saved.BudgetUSD) - req.Budget.MaxTurns = firstPositive(o.MaxTurns, base.Budget.MaxTurns) - req.Budget.Timeout = firstNonEmpty(o.Timeout, base.Budget.Timeout) - - if o.System != "" { - req.Prompt.System = o.System - } - if o.AppendSystem != "" { - req.Prompt.AppendSystem = o.AppendSystem - } - if len(o.Attach) > 0 { - attachments, err := attachmentRefsFromFlags(o.Attach) - if err != nil { - return base, baseCfg, err - } - req.Prompt.Attachments = append(req.Prompt.Attachments, attachments...) - } - - if o.PermissionMode != "" { - req.Permissions.Mode = api.PermissionMode(o.PermissionMode) - } else if o.Edit { - req.Permissions.Mode = api.PermissionAcceptEdits - } - if o.Edit && !req.Permissions.HasPreset(api.PresetEdit) { - req.Permissions.Presets = append(req.Permissions.Presets, api.PresetEdit) - } - if o.AllowedTools != nil { - req.Permissions.Tools.SetList(api.ToolPolicyAllow, o.AllowedTools) - } - if o.DisallowedTools != nil { - req.Permissions.Tools.SetList(api.ToolPolicyDeny, o.DisallowedTools) - } - req.Permissions.MCP.Disabled = o.NoMCP || base.Permissions.MCP.Disabled || saved.NoMCP - - if o.SkillDirs != nil { - req.Memory.Skills = o.SkillDirs - } - req.Memory.SkipHooks = o.NoHooks || base.Memory.SkipHooks || saved.NoHooks - req.Memory.SkipSkills = o.NoSkills || base.Memory.SkipSkills || saved.NoSkills - req.Memory.SkipUser = o.NoUser || base.Memory.SkipUser || saved.NoUser - req.Memory.SkipProject = o.NoProject || base.Memory.SkipProject || saved.NoProject - req.Memory.SkipMemory = o.NoMemory || base.Memory.SkipMemory || saved.NoMemory - req.Memory.Bare = o.Bare || base.Memory.Bare - - if o.Resume != "" { - req.SessionID = o.Resume - } - - // Config mirrors the resolved model + budget; runtime-only knobs from CLI+saved. - cfg := baseCfg - cfg.Model = req.Model - cfg.Budget = req.Budget - cfg.APIKey = o.APIKey - cfg.APIURL = firstNonEmpty(strings.TrimSpace(o.APIURL), baseCfg.APIURL) - // The overlay's resolution saw every layer (flag > frontmatter > default), - // so it overwrites rather than ORs with baseCfg: an explicit "none" must be - // able to turn an inherited external selection off. - // req.Sandbox is the winning ref (frontmatter, or the flag's override - // recorded above), so its agent pin and dispatch policy ride along. - cfg.SandboxSelection = sandboxSelectionConfig(sandbox, req.Sandbox) - cfg.NoCache = req.NoCache - return req, cfg, nil -} - // normalizePromptContextDir resolves the complete Setup through its owning // commons-db type before providers see the request. func normalizePromptContextDir(req *ai.Request, cwd string) error { @@ -248,15 +76,6 @@ func fallbackModelsFromFlags(flags []string) []api.Model { return out } -// firstFallbacks implements the CLI-over-frontmatter precedence for the fallback -// list: the --fallback flags win when present, otherwise the frontmatter list stands. -func firstFallbacks(flags []string, frontmatter []api.Model) []api.Model { - if models := fallbackModelsFromFlags(flags); len(models) > 0 { - return models - } - return frontmatter -} - // firstNonEmpty returns the first non-empty string, or "" when all are empty. func firstNonEmpty(vals ...string) string { for _, v := range vals { @@ -266,49 +85,3 @@ func firstNonEmpty(vals ...string) string { } return "" } - -// selectModelIdentity applies precedence from lowest to highest while keeping a -// model name and its runtime coupled. A higher-priority name clears a -// lower-priority mode/provider unless that same layer explicitly supplies one. -func selectModelIdentity(layers ...api.Model) api.Model { - var selected api.Model - for _, layer := range layers { - if layer.Name != "" { - selected.Name = layer.Name - selected.ID = layer.ID - selected.Mode = layer.Mode - selected.Provider = layer.Provider - continue - } - if layer.ID != "" { - selected.ID = layer.ID - } - if layer.Mode != "" { - selected.Mode = layer.Mode - } - if layer.Provider != nil { - selected.Provider = layer.Provider - } - } - return selected -} - -// firstPositive returns the first value > 0, or 0 when none qualify. -func firstPositive(vals ...int) int { - for _, v := range vals { - if v > 0 { - return v - } - } - return 0 -} - -// firstPositiveFloat returns the first value > 0, or 0 when none qualify. -func firstPositiveFloat(vals ...float64) float64 { - for _, v := range vals { - if v > 0 { - return v - } - } - return 0 -} diff --git a/pkg/cli/ai_prompt_file_test.go b/pkg/cli/ai_prompt_file_test.go index 9e6f4ce4..e00c6bda 100644 --- a/pkg/cli/ai_prompt_file_test.go +++ b/pkg/cli/ai_prompt_file_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/prompt" "github.com/flanksource/captain/pkg/api" "github.com/flanksource/commons-db/shell" ) @@ -64,7 +65,7 @@ func TestResolvePromptTemplate(t *testing.T) { if usedStdin { t.Error("usedStdin = true, want false when a file is the source") } - req, _, err := tmpl.Render(nil, nil) + req, _, err := tmpl.Render(prompt.RenderOptions{}) if err != nil { t.Fatal(err) } @@ -83,7 +84,7 @@ func TestResolvePromptTemplate(t *testing.T) { if usedStdin { t.Error("usedStdin = true, want false when --prompt is the source") } - req, _, _ := tmpl.Render(nil, nil) + req, _, _ := tmpl.Render(prompt.RenderOptions{}) if req.Prompt.User != "literal text" { t.Errorf("User = %q, want %q", req.Prompt.User, "literal text") } @@ -97,7 +98,7 @@ func TestResolvePromptTemplate(t *testing.T) { if !usedStdin { t.Error("usedStdin = false, want true when stdin is the source") } - req, _, _ := tmpl.Render(nil, nil) + req, _, _ := tmpl.Render(prompt.RenderOptions{}) if req.Prompt.User != "from stdin" { t.Errorf("User = %q, want %q", req.Prompt.User, "from stdin") } @@ -115,7 +116,7 @@ func TestResolvePromptTemplate(t *testing.T) { func baseFileReq() ai.Request { return ai.Request{ Prompt: api.Prompt{User: "body prompt"}, - Model: api.Model{Name: "claude-file-4-6"}, + Model: api.Model{Name: "claude-sonnet-5"}, Budget: api.Budget{MaxTokens: 100}, Sandbox: &api.SandboxRef{Mode: api.SandboxNative}, Permissions: api.Permissions{Mode: api.PermissionAcceptEdits}, @@ -123,21 +124,21 @@ func baseFileReq() ai.Request { } } -func TestOverlayCLI_CLIOverridesFrontmatter(t *testing.T) { +func TestResolveCLI_CLIOverridesFrontmatter(t *testing.T) { isolateSavedAI(t) opts := AIPromptOptions{} - opts.Model = "claude-cli-4-6" + opts.Model = "claude-opus-4-8" opts.MaxTokens = 200 opts.PermissionMode = "plan" - req, cfg, err := overlayCLI(baseFileReq(), ai.Config{}, opts) + req, cfg, err := runtimeLayersForTest(baseFileReq(), opts) if err != nil { t.Fatal(err) } - if req.Model.Name != "claude-cli-4-6" { - t.Errorf("Model.Name = %q, want CLI value claude-cli-4-6", req.Model.Name) + if req.Model.Name != "claude-opus-4-8" { + t.Errorf("Model.Name = %q, want CLI value claude-opus-4-8", req.Model.Name) } - if cfg.Model.Name != "claude-cli-4-6" { + if cfg.Model.Name != "claude-opus-4-8" { t.Errorf("cfg.Model.Name = %q, want CLI value mirrored into config", cfg.Model.Name) } if req.Budget.MaxTokens != 200 { @@ -148,19 +149,20 @@ func TestOverlayCLI_CLIOverridesFrontmatter(t *testing.T) { } } -func TestOverlayCLI_Sandbox(t *testing.T) { - for _, tt := range []struct{ name, mode, wantErr string }{ +func TestResolveCLI_Sandbox(t *testing.T) { + for _, tt := range []struct{ name, mode, baseMode, wantErr string }{ {name: "selects CLI for frontmatter model"}, {name: "rejects explicit API mode", mode: "api", wantErr: "requires cli mode"}, + {name: "rejects authored API mode", baseMode: "api", wantErr: "requires cli mode"}, } { t.Run(tt.name, func(t *testing.T) { isolateSavedAI(t) base := baseFileReq() base.Model.Provider = api.Anthropic - base.Model.Mode = api.ModeAPI + base.Model.Mode = api.RuntimeMode(tt.baseMode) opts := AIPromptOptions{AIRuntimeOptions: AIRuntimeOptions{AIProviderOptions: AIProviderOptions{Sandbox: "docker"}}} opts.Mode = tt.mode - req, cfg, err := overlayCLI(base, ai.Config{}, opts) + req, cfg, err := runtimeLayersForTest(base, opts) if tt.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Fatalf("err = %v, want %q", err, tt.wantErr) @@ -180,13 +182,13 @@ func TestOverlayCLI_Sandbox(t *testing.T) { // An explicit --sandbox=off must turn off a sandbox the base config carried: // the overlay resolved every layer, so it overwrites instead of ORing. -func TestOverlayCLI_SandboxOffClearsInherited(t *testing.T) { +func TestResolveCLI_SandboxOffClearsInherited(t *testing.T) { isolateSavedAI(t) base := baseFileReq() - baseCfg := ai.Config{SandboxSelection: &api.SandboxConfig{Kind: api.SandboxDocker}} + base.Sandbox = &api.SandboxRef{Mode: api.SandboxDocker} opts := AIPromptOptions{AIRuntimeOptions: AIRuntimeOptions{AIProviderOptions: AIProviderOptions{Sandbox: "off"}}} - _, cfg, err := overlayCLI(base, baseCfg, opts) + _, cfg, err := runtimeLayersForTest(base, opts) if err != nil { t.Fatal(err) } @@ -198,7 +200,7 @@ func TestOverlayCLI_SandboxOffClearsInherited(t *testing.T) { } } -func TestOverlayCLI_SandboxFlagPreservesFrontmatterAgentAndPolicy(t *testing.T) { +func TestResolveCLI_SandboxFlagPreservesFrontmatterAgentAndPolicy(t *testing.T) { isolateSavedAI(t) base := baseFileReq() base.Sandbox = &api.SandboxRef{ @@ -209,7 +211,7 @@ func TestOverlayCLI_SandboxFlagPreservesFrontmatterAgentAndPolicy(t *testing.T) } opts := AIPromptOptions{AIRuntimeOptions: AIRuntimeOptions{AIProviderOptions: AIProviderOptions{Sandbox: "git-agent"}}} - req, cfg, err := overlayCLI(base, ai.Config{}, opts) + req, cfg, err := runtimeLayersForTest(base, opts) if err != nil { t.Fatal(err) } @@ -224,14 +226,12 @@ func TestOverlayCLI_SandboxFlagPreservesFrontmatterAgentAndPolicy(t *testing.T) } } -// --api-url is what points a run at a `captain ai mock` endpoint, so it has to -// survive the overlay; a prompt file may pin its own endpoint, and the flag wins. -func TestOverlayCLI_APIURLFlagBeatsFrontmatter(t *testing.T) { +func TestResolveCLI_APIURLTransport(t *testing.T) { isolateSavedAI(t) opts := AIPromptOptions{} opts.APIURL = "http://127.0.0.1:18096/v1" - _, cfg, err := overlayCLI(baseFileReq(), ai.Config{APIURL: "https://api.openai.com/v1"}, opts) + _, cfg, err := runtimeLayersForTest(baseFileReq(), opts) if err != nil { t.Fatal(err) } @@ -239,34 +239,34 @@ func TestOverlayCLI_APIURLFlagBeatsFrontmatter(t *testing.T) { t.Errorf("cfg.APIURL = %q, want CLI value %q", cfg.APIURL, opts.APIURL) } - _, cfg, err = overlayCLI(baseFileReq(), ai.Config{APIURL: "https://api.openai.com/v1"}, AIPromptOptions{}) + _, cfg, err = runtimeLayersForTest(baseFileReq(), AIPromptOptions{}) if err != nil { t.Fatal(err) } - if cfg.APIURL != "https://api.openai.com/v1" { - t.Errorf("cfg.APIURL = %q, want the frontmatter value to stand without a flag", cfg.APIURL) + if cfg.APIURL != "" { + t.Errorf("cfg.APIURL = %q, want no endpoint without an explicit flag", cfg.APIURL) } } -func TestOverlayCLI_FrontmatterOverridesSaved(t *testing.T) { - seedSavedAI(t, "ai:\n model: claude-saved-4-6\n maxTokens: 16000\n") - req, _, err := overlayCLI(baseFileReq(), ai.Config{}, AIPromptOptions{}) +func TestResolveCLI_FrontmatterOverridesSaved(t *testing.T) { + seedSavedAI(t, "ai:\n defaultModel: agent:claude-haiku-4-5\n maxTokens: 16000\n") + req, _, err := runtimeLayersForTest(baseFileReq(), AIPromptOptions{}) if err != nil { t.Fatal(err) } - if req.Model.Name != "claude-file-4-6" { - t.Errorf("Model.Name = %q, want frontmatter value claude-file-4-6 (beats saved)", req.Model.Name) + if req.Model.Name != "claude-sonnet-5" { + t.Errorf("Model.Name = %q, want frontmatter value claude-sonnet-5 (beats saved)", req.Model.Name) } if req.Budget.MaxTokens != 100 { t.Errorf("MaxTokens = %d, want frontmatter value 100 (beats saved 16000)", req.Budget.MaxTokens) } } -func TestOverlayCLI_BuiltinMaxTokens(t *testing.T) { +func TestResolveCLI_BuiltinMaxTokens(t *testing.T) { isolateSavedAI(t) base := baseFileReq() base.Budget.MaxTokens = 0 // frontmatter omitted it - req, _, err := overlayCLI(base, ai.Config{}, AIPromptOptions{}) + req, _, err := runtimeLayersForTest(base, AIPromptOptions{}) if err != nil { t.Fatal(err) } @@ -275,14 +275,14 @@ func TestOverlayCLI_BuiltinMaxTokens(t *testing.T) { } } -func TestOverlayCLI_BooleansOR(t *testing.T) { +func TestResolveCLI_OmittedBooleansInherit(t *testing.T) { isolateSavedAI(t) opts := AIPromptOptions{} opts.NoMCP = true // CLI sets it opts.Edit = true // CLI preset opts.NoUser = false // base already has SkipUser=true - req, _, err := overlayCLI(baseFileReq(), ai.Config{}, opts) + req, _, err := runtimeLayersForTest(baseFileReq(), opts) if err != nil { t.Fatal(err) } @@ -290,7 +290,7 @@ func TestOverlayCLI_BooleansOR(t *testing.T) { t.Error("MCP.Disabled = false, want true (CLI --no-mcp)") } if !req.Memory.SkipUser { - t.Error("SkipUser = false, want true (frontmatter set it; CLI false must not clear it)") + t.Error("SkipUser = false, want true (frontmatter set it; omitted CLI flag must not clear it)") } if !req.Permissions.HasPreset(api.PresetEdit) { t.Error("missing edit preset from --edit") @@ -298,7 +298,7 @@ func TestOverlayCLI_BooleansOR(t *testing.T) { // --edit must not duplicate a preset the frontmatter already declared. base := baseFileReq() base.Permissions.Presets = []api.Preset{api.PresetEdit} - req2, _, err := overlayCLI(base, ai.Config{}, opts) + req2, _, err := runtimeLayersForTest(base, opts) if err != nil { t.Fatal(err) } @@ -318,17 +318,18 @@ func fallbackNames(models []api.Model) []string { return out } -func TestOverlayCLI_ModelCSVExpandsToFallbacks(t *testing.T) { +func TestResolveCLI_ModelCSVExpandsToFallbacks(t *testing.T) { isolateSavedAI(t) opts := AIPromptOptions{} - opts.Model = "claude-primary-5,gpt-4o,gemini-2.0-flash" + opts.Sandbox = "off" + opts.Model = "claude-sonnet-5,gpt-4o,gemini-2.0-flash" - req, cfg, err := overlayCLI(baseFileReq(), ai.Config{}, opts) + req, cfg, err := runtimeLayersForTest(baseFileReq(), opts) if err != nil { t.Fatal(err) } - if req.Model.Name != "claude-primary-5" { - t.Errorf("Model.Name = %q, want CSV head claude-primary-5", req.Model.Name) + if req.Model.Name != "claude-sonnet-5" { + t.Errorf("Model.Name = %q, want CSV head claude-sonnet-5", req.Model.Name) } if got := fallbackNames(req.Model.Fallbacks); !reflect.DeepEqual(got, []string{"gpt-4o", "gemini-2.0-flash"}) { t.Errorf("req fallbacks = %v, want [gpt-4o gemini-2.0-flash]", got) @@ -338,14 +339,15 @@ func TestOverlayCLI_ModelCSVExpandsToFallbacks(t *testing.T) { } } -func TestOverlayCLI_FallbackFlagOverridesFrontmatter(t *testing.T) { +func TestResolveCLI_FallbackFlagOverridesFrontmatter(t *testing.T) { isolateSavedAI(t) base := baseFileReq() base.Model.Fallbacks = []api.Model{{Name: "frontmatter-fallback"}} opts := AIPromptOptions{} + opts.Sandbox = "off" opts.Fallback = []string{"gemini-3.5-flash", "gpt-5.5,claude-sonnet-5"} // repeatable + comma-split - req, _, err := overlayCLI(base, ai.Config{}, opts) + req, _, err := runtimeLayersForTest(base, opts) if err != nil { t.Fatal(err) } @@ -355,17 +357,17 @@ func TestOverlayCLI_FallbackFlagOverridesFrontmatter(t *testing.T) { } } -func TestOverlayCLI_FrontmatterFallbacksStandWithoutFlag(t *testing.T) { +func TestResolveCLI_FrontmatterFallbacksStandWithoutFlag(t *testing.T) { isolateSavedAI(t) base := baseFileReq() - base.Model.Fallbacks = []api.Model{{Name: "gpt-4o", Effort: api.EffortHigh}} + base.Model.Fallbacks = []api.Model{{Name: "gpt-5.6-sol", Effort: api.EffortHigh}} - req, _, err := overlayCLI(base, ai.Config{}, AIPromptOptions{}) + req, _, err := runtimeLayersForTest(base, AIPromptOptions{}) if err != nil { t.Fatal(err) } - if got := fallbackNames(req.Model.Fallbacks); !reflect.DeepEqual(got, []string{"gpt-4o"}) { - t.Errorf("fallbacks = %v, want frontmatter [gpt-4o] when no --fallback", got) + if got := fallbackNames(req.Model.Fallbacks); !reflect.DeepEqual(got, []string{"gpt-5.6-sol"}) { + t.Errorf("fallbacks = %v, want frontmatter [gpt-5.6-sol] when no --fallback", got) } if req.Model.Fallbacks[0].Effort != api.EffortHigh { t.Errorf("frontmatter fallback effort = %q, want preserved high", req.Model.Fallbacks[0].Effort) diff --git a/pkg/cli/ai_runtime_resolve.go b/pkg/cli/ai_runtime_resolve.go new file mode 100644 index 00000000..c9b705c6 --- /dev/null +++ b/pkg/cli/ai_runtime_resolve.go @@ -0,0 +1,92 @@ +package cli + +import ( + "fmt" + "os" + "strings" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/captainconfig" +) + +type AIRuntimeResolveOptions struct { + Layers []api.SpecLayer + Saved captainconfig.Config + Cwd string + RequireModel bool +} + +func logRuntimeWarnings(warnings []string) { + for _, warning := range warnings { + log.Warnf("preflight: %s", warning) + } +} + +type AIRuntimeResolved struct { + Request ai.Request + Config ai.Config + Resolution api.ResolvedSpec +} + +func resolveInvocation(options AIRuntimeOptions, layers []api.SpecLayer) (AIRuntimeResolved, error) { + saved, err := loadSavedConfig() + if err != nil { + return AIRuntimeResolved{}, err + } + cwd, err := os.Getwd() + if err != nil { + return AIRuntimeResolved{}, fmt.Errorf("get working directory: %w", err) + } + return options.Resolve(AIRuntimeResolveOptions{Layers: layers, Saved: saved, Cwd: cwd, RequireModel: true}) +} + +type AIRuntimeProjectOptions struct { + Resolved api.ResolvedSpec + Saved captainconfig.Config +} + +func (o AIRuntimeOptions) Project(options AIRuntimeProjectOptions) (AIRuntimeResolved, error) { + selection, err := resolveSandboxSelection(sandboxSelectionOptions{Spec: options.Resolved.Spec, Saved: options.Saved.Sandbox}) + if err != nil { + return AIRuntimeResolved{}, err + } + if descriptor, ok := registry.SandboxFor(selection.Kind); ok && options.Resolved.Spec.Mode != "" { + if err := descriptor.ValidateMode(options.Resolved.Spec.Mode); err != nil { + return AIRuntimeResolved{}, err + } + } + cfg := configFromResolved(options.Resolved.Spec) + cfg.APIKey = o.APIKey + cfg.APIURL = strings.TrimSpace(o.APIURL) + cfg.SchemaRepair = schemaRepairConfig(options.Saved.Prompts.SchemaRepair) + cfg.SandboxSelection = sandboxSelectionConfig(selection, options.Resolved.Spec.Sandbox) + return AIRuntimeResolved{Request: options.Resolved.Spec, Config: cfg, Resolution: options.Resolved}, nil +} + +func (o AIRuntimeOptions) Resolve(options AIRuntimeResolveOptions) (AIRuntimeResolved, error) { + request, err := o.requestSpec() + if err != nil { + return AIRuntimeResolved{}, err + } + layers := append([]api.SpecLayer(nil), options.Layers...) + if len(request.Fields()) > 0 { + layers = append(layers, api.RequestSpecLayer("CLI flags", request)) + } + options.Layers = layers + return o.resolveAuthored(options) +} + +func (o AIRuntimeOptions) resolveAuthored(options AIRuntimeResolveOptions) (AIRuntimeResolved, error) { + resolved, err := api.ResolveSpecLayers(api.ResolveSpecOptions{ + Layers: options.Layers, Saved: &options.Saved.AI, RequireModel: options.RequireModel, + Normalize: func(spec api.Spec) (api.SpecNormalization, error) { + return o.Normalize(AIRuntimeNormalizeOptions{Spec: spec, Saved: options.Saved, Cwd: options.Cwd}) + }, + }) + if err != nil { + return AIRuntimeResolved{}, err + } + return o.Project(AIRuntimeProjectOptions{Resolved: resolved, Saved: options.Saved}) +} diff --git a/pkg/cli/ai_sandbox.go b/pkg/cli/ai_sandbox.go index df3b336a..61468440 100644 --- a/pkg/cli/ai_sandbox.go +++ b/pkg/cli/ai_sandbox.go @@ -3,72 +3,36 @@ package cli import ( "strings" - "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/api/registry" "github.com/flanksource/captain/pkg/captainconfig" ) -// resolveSandboxSelection applies the sandbox precedence chain — CLI flag > -// prompt frontmatter > global default > off — and resolves the winner against -// the backends configured in ~/.captain.yaml. -// -// Kinds whose execution seam is not wired yet fail loud here rather than -// resolving successfully and then silently running unsandboxed; each adapter -// removes itself from the guard when its wiring lands. -func resolveSandboxSelection(flagSelector string, ref *api.SandboxRef, defaults captainconfig.SandboxDefaults) (captainconfig.SandboxSelection, error) { - selector := strings.TrimSpace(flagSelector) +type sandboxSelectionOptions struct { + Selector string + Spec api.Spec + Saved captainconfig.SandboxDefaults +} + +func resolveSandboxSelection(options sandboxSelectionOptions) (captainconfig.SandboxSelection, error) { + selector := strings.TrimSpace(options.Selector) + ref := options.Spec.Sandbox + if selector == "" && ref == nil && options.Spec.Fields().Has("/sandbox") { + return captainconfig.SandboxSelection{Kind: registry.SandboxOff}, nil + } if selector == "" && ref != nil { selector = strings.TrimSpace(ref.Backend) if selector == "" { selector = string(ref.Mode) } } - selection, err := defaults.Resolve(selector) + selection, err := options.Saved.Resolve(selector) if err != nil { return captainconfig.SandboxSelection{}, err } return selection, nil } -// resolveRunSandbox resolves the sandbox for one run from the request's own ref -// — the prompt's `sandbox:` frontmatter, or a spec override already layered onto -// it — plus an optional operator-supplied selector. -func resolveRunSandbox(req *ai.Request, flagSelector string) (captainconfig.SandboxSelection, error) { - return resolveSandboxSelection(flagSelector, req.Sandbox, loadSavedConfig().Sandbox) -} - -// recordSandboxSelection writes the winning selection onto the run: the config -// the exec seam reads, and — when an operator named one explicitly — the request -// ref, so the serialized spec carries the choice the run was actually made with. -func recordSandboxSelection(req *ai.Request, cfg *ai.Config, selection captainconfig.SandboxSelection, flagSelector string) { - if flagSelector != "" { - ref := sandboxRefFromSelection(selection) - if req.Sandbox != nil { - ref.Policy = req.Sandbox.Policy - ref.Agent = req.Sandbox.Agent - ref.Dispatch = req.Sandbox.Dispatch - } - req.Sandbox = &ref - } - cfg.SandboxSelection = sandboxSelectionConfig(selection, req.Sandbox) -} - -// applyRunSandbox is the transport-neutral seam: resolve, then record. Every -// entrypoint that builds a run must reach the sandbox through it or through its -// two halves. Resolution used to live only inside overlayCLI, so a run submitted -// over HTTP carried no selection at all — and the exec seam reads a nil -// selection as "unsandboxed", silently, because both fail-loud guards only fire -// once a selection exists. -func applyRunSandbox(req *ai.Request, cfg *ai.Config, flagSelector string) error { - selection, err := resolveRunSandbox(req, flagSelector) - if err != nil { - return err - } - recordSandboxSelection(req, cfg, selection, flagSelector) - return nil -} - // sandboxForcedMode returns the single runtime mode a sandbox kind can serve, // or "" when the kind serves several (or is off/native). Argv-wrapping adapters are // CLI-only, so selecting one forces CLI mode the way --sandbox always has. diff --git a/pkg/cli/ai_sandbox_test.go b/pkg/cli/ai_sandbox_test.go index 2d2aa6f1..620b64e1 100644 --- a/pkg/cli/ai_sandbox_test.go +++ b/pkg/cli/ai_sandbox_test.go @@ -4,7 +4,6 @@ import ( "strings" "testing" - "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/api/registry" "github.com/flanksource/captain/pkg/captainconfig" @@ -19,7 +18,7 @@ func TestResolveSandboxSelection_Precedence(t *testing.T) { } t.Run("flag beats frontmatter", func(t *testing.T) { - got, err := resolveSandboxSelection("off", &api.SandboxRef{Mode: api.SandboxDocker, Backend: "pool"}, defaults) + got, err := resolveSandboxSelection(sandboxSelectionOptions{Selector: "off", Spec: api.Spec{Sandbox: &api.SandboxRef{Mode: api.SandboxDocker, Backend: "pool"}}, Saved: defaults}) if err != nil { t.Fatal(err) } @@ -29,7 +28,7 @@ func TestResolveSandboxSelection_Precedence(t *testing.T) { }) t.Run("frontmatter beats the global default", func(t *testing.T) { - got, err := resolveSandboxSelection("", &api.SandboxRef{Mode: api.SandboxDocker, Backend: "pool"}, defaults) + got, err := resolveSandboxSelection(sandboxSelectionOptions{Spec: api.Spec{Sandbox: &api.SandboxRef{Mode: api.SandboxDocker, Backend: "pool"}}, Saved: defaults}) if err != nil { t.Fatal(err) } @@ -39,7 +38,7 @@ func TestResolveSandboxSelection_Precedence(t *testing.T) { }) t.Run("global default applies when nothing else selects", func(t *testing.T) { - got, err := resolveSandboxSelection("", nil, defaults) + got, err := resolveSandboxSelection(sandboxSelectionOptions{Saved: defaults}) if err != nil { t.Fatal(err) } @@ -49,7 +48,7 @@ func TestResolveSandboxSelection_Precedence(t *testing.T) { }) t.Run("everything empty resolves to off", func(t *testing.T) { - got, err := resolveSandboxSelection("", nil, captainconfig.SandboxDefaults{}) + got, err := resolveSandboxSelection(sandboxSelectionOptions{}) if err != nil { t.Fatal(err) } @@ -59,7 +58,7 @@ func TestResolveSandboxSelection_Precedence(t *testing.T) { }) t.Run("git-agent resolves now that remote execution is wired", func(t *testing.T) { - got, err := resolveSandboxSelection("git-agent", nil, defaults) + got, err := resolveSandboxSelection(sandboxSelectionOptions{Selector: "git-agent", Saved: defaults}) if err != nil { t.Fatal(err) } @@ -69,7 +68,7 @@ func TestResolveSandboxSelection_Precedence(t *testing.T) { }) t.Run("an unknown selector fails loud", func(t *testing.T) { - _, err := resolveSandboxSelection("nope", nil, defaults) + _, err := resolveSandboxSelection(sandboxSelectionOptions{Selector: "nope", Saved: defaults}) if err == nil || !strings.Contains(err.Error(), `unknown sandbox "nope"`) { t.Fatalf("err = %v", err) } @@ -93,7 +92,7 @@ func TestActionFlags_ModeSandboxConflictRejected(t *testing.T) { t.Fatalf("Mode = %q: --mode must survive actionFlagsToOptions", opts.Mode) } - _, _, err = overlayCLI(baseFileReq(), ai.Config{}, opts) + _, _, err = runtimeLayersForTest(baseFileReq(), opts) if err == nil || !strings.Contains(err.Error(), "requires cli mode") { t.Fatalf("err = %v, want the API-mode × container-sandbox contradiction rejected", err) } diff --git a/pkg/cli/ai_test.go b/pkg/cli/ai_test.go index 667dc110..ef6db9cb 100644 --- a/pkg/cli/ai_test.go +++ b/pkg/cli/ai_test.go @@ -1,8 +1,6 @@ package cli import ( - "context" - "encoding/json" "os" "path/filepath" "reflect" @@ -14,86 +12,8 @@ import ( "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/api/registry" "github.com/flanksource/captain/pkg/captainconfig" - "github.com/flanksource/commons-db/shell" ) -type promptResultProvider struct{} - -func (promptResultProvider) GetModel() string { return "claude-sonnet-4-6" } - -func (promptResultProvider) GetRuntime() ai.Runtime { return ai.RuntimeOf(ai.Anthropic, ai.ModeAPI) } - -func (promptResultProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { - return &ai.Response{ - Text: "done", - Model: "claude-sonnet-4-6", - Runtime: ai.RuntimeOf(ai.Anthropic, ai.ModeAPI), - Usage: ai.Usage{InputTokens: 12, OutputTokens: 7}, - }, nil -} - -type structuredPromptResultProvider struct{} - -func (structuredPromptResultProvider) GetModel() string { return "claude-sonnet-4-6" } - -func (structuredPromptResultProvider) GetRuntime() ai.Runtime { - return ai.RuntimeOf(ai.Anthropic, ai.ModeAPI) -} - -func (structuredPromptResultProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { - return &ai.Response{ - Text: `{"answer":"42"}`, - StructuredData: json.RawMessage(`{"answer":"42"}`), - Model: "claude-sonnet-4-6", - Runtime: ai.RuntimeOf(ai.Anthropic, ai.ModeAPI), - }, nil -} - -type promptResultStreamingProvider struct{} - -func (promptResultStreamingProvider) GetModel() string { return "gpt-5-codex" } - -func (promptResultStreamingProvider) GetRuntime() ai.Runtime { - return ai.RuntimeOf(ai.OpenAI, ai.ModeCLI) -} - -func (promptResultStreamingProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { - return nil, nil -} - -func (promptResultStreamingProvider) ExecuteStream(_ context.Context, _ ai.Request) (<-chan ai.Event, error) { - events := make(chan ai.Event, 3) - events <- ai.Event{Kind: ai.EventSystem, SessionID: "stream-session-1", Model: "gpt-5-codex"} - events <- ai.Event{Kind: ai.EventText, Text: "streamed", Model: "gpt-5-codex"} - events <- ai.Event{Kind: ai.EventResult, Model: "gpt-5-codex", Usage: &ai.Usage{InputTokens: 21, OutputTokens: 9}, CostUSD: 0.02} - close(events) - return events, nil -} - -type structuredResultStreamingProvider struct { - text string -} - -func (structuredResultStreamingProvider) GetModel() string { return "gpt-5-codex" } - -func (structuredResultStreamingProvider) GetRuntime() ai.Runtime { - return ai.RuntimeOf(ai.OpenAI, ai.ModeCLI) -} - -func (structuredResultStreamingProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { - return nil, nil -} - -func (p structuredResultStreamingProvider) ExecuteStream(_ context.Context, _ ai.Request) (<-chan ai.Event, error) { - events := make(chan ai.Event, 2) - if p.text != "" { - events <- ai.Event{Kind: ai.EventText, Text: p.text} - } - events <- ai.Event{Kind: ai.EventResult, StructuredData: json.RawMessage(`{"answer":"42"}`)} - close(events) - return events, nil -} - // isolateSavedAI redirects captainconfig.Path() to an empty file inside // t.TempDir() so loadSavedAI() returns zero defaults rather than leaking // the developer's real ~/.captain.yaml into table-test expectations. @@ -113,12 +33,12 @@ func seedSavedAI(t *testing.T, yaml string) { t.Cleanup(func() { captainconfig.SetPathForTesting("") }) } -func TestAIPromptOptions_ToRequest_Defaults(t *testing.T) { +func TestAIPromptOptions_Resolve_Defaults(t *testing.T) { isolateSavedAI(t) opts := defaultPromptOptions(t) - req, err := opts.ToRequest() + req, err := promptRequestForTest(opts) if err != nil { - t.Fatalf("ToRequest: %v", err) + t.Fatalf("Resolve: %v", err) } if req.Prompt.User != "hello" { @@ -138,9 +58,9 @@ func TestAIPromptOptions_ToRequest_Defaults(t *testing.T) { } } -// TestAIPromptOptions_ToRequest_NegativeFlags verifies each --no-* flag sets the +// TestAIPromptOptions_Resolve_NegativeFlags verifies each --no-* flag sets the // matching No* field on the request. -func TestAIPromptOptions_ToRequest_NegativeFlags(t *testing.T) { +func TestAIPromptOptions_Resolve_NegativeFlags(t *testing.T) { isolateSavedAI(t) cases := []struct { name string @@ -158,9 +78,9 @@ func TestAIPromptOptions_ToRequest_NegativeFlags(t *testing.T) { t.Run(tc.name, func(t *testing.T) { opts := defaultPromptOptions(t) tc.mutate(&opts) - req, err := opts.ToRequest() + req, err := promptRequestForTest(opts) if err != nil { - t.Fatalf("ToRequest: %v", err) + t.Fatalf("Resolve: %v", err) } if !tc.get(req) { t.Errorf("flag --%s did not set the mapped request field", tc.name) @@ -169,7 +89,7 @@ func TestAIPromptOptions_ToRequest_NegativeFlags(t *testing.T) { } } -func TestAIPromptOptions_ToRequest_PassesScalars(t *testing.T) { +func TestAIPromptOptions_Resolve_PassesScalars(t *testing.T) { isolateSavedAI(t) opts := defaultPromptOptions(t) opts.System = "be careful" @@ -186,9 +106,9 @@ func TestAIPromptOptions_ToRequest_PassesScalars(t *testing.T) { opts.MaxTurns = 7 opts.Resume = "sess-123" - req, err := opts.ToRequest() + req, err := promptRequestForTest(opts) if err != nil { - t.Fatalf("ToRequest: %v", err) + t.Fatalf("Resolve: %v", err) } if req.Prompt.System != "be careful" { t.Errorf("SystemPrompt = %q", req.Prompt.System) @@ -231,9 +151,9 @@ func TestAIPromptOptions_ToRequest_PassesScalars(t *testing.T) { } } -// TestAIRuntimeOptions_ToRequest_ValidationErrors verifies malformed input fails +// TestAIRuntimeOptions_Resolve_ValidationErrors verifies malformed input fails // loudly instead of being silently coerced to a zero value. -func TestAIRuntimeOptions_ToRequest_ValidationErrors(t *testing.T) { +func TestAIRuntimeOptions_Resolve_ValidationErrors(t *testing.T) { isolateSavedAI(t) cases := []struct { name string @@ -252,24 +172,24 @@ func TestAIRuntimeOptions_ToRequest_ValidationErrors(t *testing.T) { t.Run(tc.name, func(t *testing.T) { opts := defaultPromptOptions(t) tc.mutate(&opts) - if _, err := opts.ToRequest(); err == nil || !strings.Contains(err.Error(), tc.want) { + if _, err := promptRequestForTest(opts); err == nil || !strings.Contains(err.Error(), tc.want) { t.Fatalf("err = %v, want mention of %q", err, tc.want) } }) } } -func TestAIProviderOptions_ToConfig_ValidationErrors(t *testing.T) { +func TestAIProviderOptions_Resolve_ValidationErrors(t *testing.T) { isolateSavedAI(t) - if _, err := (AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "claude-x", Mode: "nope"}}).ToConfig(); err == nil || !strings.Contains(err.Error(), "mode") { + if _, err := providerConfigForTest(AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "claude-x", Mode: "nope"}}); err == nil || !strings.Contains(err.Error(), "mode") { t.Fatalf("expected invalid mode error, got %v", err) } - if _, err := (AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "claude-x"}, Budget: "free"}).ToConfig(); err == nil || !strings.Contains(err.Error(), "budget") { + if _, err := providerConfigForTest(AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "claude-x"}, Budget: "free"}); err == nil || !strings.Contains(err.Error(), "budget") { t.Fatalf("expected invalid budget error, got %v", err) } } -func TestAIProviderOptions_ToConfig_Sandbox(t *testing.T) { +func TestAIProviderOptions_Resolve_Sandbox(t *testing.T) { tests := []struct { name, saved, wantName, wantErr string flags aiflags.ModelFlags @@ -278,7 +198,7 @@ func TestAIProviderOptions_ToConfig_Sandbox(t *testing.T) { }{ {name: "selects CLI without changing model", flags: aiflags.ModelFlags{Model: "claude-sonnet-5"}, wantName: "claude-sonnet-5", providers: []*api.ModelProvider{api.Anthropic}}, {name: "rejects explicit API mode", flags: aiflags.ModelFlags{Model: "claude-sonnet-5", Mode: "api"}, wantErr: "requires cli mode"}, - {name: "overrides the saved agent mode", saved: "ai:\n providers:\n anthropic:\n model: opus\n mode: agent\n", providers: []*api.ModelProvider{api.Anthropic}}, + {name: "overrides the saved agent mode", saved: "ai:\n defaultProvider: anthropic\n providers:\n anthropic:\n model: opus\n mode: agent\n", providers: []*api.ModelProvider{api.Anthropic}}, {name: "resolves fallbacks in CLI mode", flags: aiflags.ModelFlags{Model: "claude-sonnet-5,gpt-5.5", Fallback: []string{"gemini-3.5-flash"}}, providers: []*api.ModelProvider{api.Anthropic, api.OpenAI, api.Google}}, } for _, tt := range tests { @@ -288,7 +208,7 @@ func TestAIProviderOptions_ToConfig_Sandbox(t *testing.T) { } else { seedSavedAI(t, tt.saved) } - cfg, err := (AIProviderOptions{ModelFlags: tt.flags, Sandbox: "docker"}).ToConfig() + cfg, err := providerConfigForTest(AIProviderOptions{ModelFlags: tt.flags, Sandbox: "docker"}) if tt.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Fatalf("err = %v, want %q", err, tt.wantErr) @@ -317,22 +237,22 @@ func TestAIProviderOptions_ToConfig_Sandbox(t *testing.T) { // --api-url is the only way to point a captain run at a `captain ai mock` // endpoint for the runtimes that read Config.APIURL — the genkit ones, and // openai cli, which ignores OPENAI_BASE_URL when a ChatGPT credential is stored. -func TestAIProviderOptions_ToConfig_CarriesAPIURL(t *testing.T) { +func TestAIProviderOptions_Resolve_CarriesAPIURL(t *testing.T) { isolateSavedAI(t) const endpoint = "http://127.0.0.1:18095" - cfg, err := (AIProviderOptions{ + cfg, err := providerConfigForTest(AIProviderOptions{ ModelFlags: aiflags.ModelFlags{Model: "claude-sonnet-5"}, APIURL: " " + endpoint + " ", - }).ToConfig() + }) if err != nil { - t.Fatalf("ToConfig: %v", err) + t.Fatalf("Resolve: %v", err) } if cfg.APIURL != endpoint { t.Fatalf("APIURL = %q, want %q", cfg.APIURL, endpoint) } } -func TestAIProviderOptions_ToConfig_LoadsSchemaRepairDefaults(t *testing.T) { +func TestAIProviderOptions_Resolve_LoadsSchemaRepairDefaults(t *testing.T) { seedSavedAI(t, ` ai: providers: @@ -345,9 +265,9 @@ prompts: mode: api prompt: /repo/prompts/json-repair.prompt `) - cfg, err := (AIProviderOptions{}).ToConfig() + cfg, err := providerConfigForTest(AIProviderOptions{}) if err != nil { - t.Fatalf("ToConfig: %v", err) + t.Fatalf("Resolve: %v", err) } // Saved `mode:` is the mechanism; the provider follows from the model name. if cfg.SchemaRepair.Model.Name != "gpt-5" || cfg.SchemaRepair.Model.Mode != api.ModeAPI { @@ -362,12 +282,12 @@ prompts: } } -// TestAIRuntimeOptions_ToRequest_MaxTokensPrecedence pins the flag > saved > +// TestAIRuntimeOptions_Resolve_MaxTokensPrecedence pins the flag > saved > // built-in order, replacing the old magic-4096 sentinel. -func TestAIRuntimeOptions_ToRequest_MaxTokensPrecedence(t *testing.T) { +func TestAIRuntimeOptions_Resolve_MaxTokensPrecedence(t *testing.T) { t.Run("explicit flag wins over saved", func(t *testing.T) { seedSavedAI(t, "ai:\n maxTokens: 16000\n") - req, err := AIRuntimeOptions{MaxTokens: 2048}.ToRequest("", "", "p") + req, err := runtimeRequestForTest(AIRuntimeOptions{MaxTokens: 2048}, api.Prompt{System: "", AppendSystem: "", User: "p"}) if err != nil { t.Fatal(err) } @@ -377,7 +297,7 @@ func TestAIRuntimeOptions_ToRequest_MaxTokensPrecedence(t *testing.T) { }) t.Run("unset falls back to saved", func(t *testing.T) { seedSavedAI(t, "ai:\n maxTokens: 16000\n") - req, err := AIRuntimeOptions{}.ToRequest("", "", "p") + req, err := runtimeRequestForTest(AIRuntimeOptions{}, api.Prompt{System: "", AppendSystem: "", User: "p"}) if err != nil { t.Fatal(err) } @@ -387,7 +307,7 @@ func TestAIRuntimeOptions_ToRequest_MaxTokensPrecedence(t *testing.T) { }) t.Run("unset with no saved uses built-in 4096", func(t *testing.T) { isolateSavedAI(t) - req, err := AIRuntimeOptions{}.ToRequest("", "", "p") + req, err := runtimeRequestForTest(AIRuntimeOptions{}, api.Prompt{System: "", AppendSystem: "", User: "p"}) if err != nil { t.Fatal(err) } @@ -397,17 +317,17 @@ func TestAIRuntimeOptions_ToRequest_MaxTokensPrecedence(t *testing.T) { }) } -// TestAIRuntimeOptions_ToRequest_OverlaysSaved verifies the path gavel (and any +// TestAIRuntimeOptions_Resolve_OverlaysSaved verifies the path gavel (and any // other embedder) takes: AIRuntimeOptions with zero flag values should pick up // NoMCP/.../MaxTokens/ReasoningEffort from ~/.captain.yaml, and an explicit // --effort flag should override the saved value. -func TestAIRuntimeOptions_ToRequest_OverlaysSaved(t *testing.T) { +func TestAIRuntimeOptions_Resolve_OverlaysSaved(t *testing.T) { seedSavedAI(t, "ai:\n noMCP: true\n noHooks: true\n noSkills: true\n noUser: true\n noProject: true\n noMemory: true\n maxTokens: 16000\n providers:\n anthropic:\n reasoningEffort: low\n") opts := AIRuntimeOptions{AIProviderOptions: AIProviderOptions{ModelFlags: aiflags.ModelFlags{Effort: "high"}}} // flag overrides saved low - req, err := opts.ToRequest("sys", "", "user") + req, err := runtimeRequestForTest(opts, api.Prompt{System: "sys", User: "user"}) if err != nil { - t.Fatalf("ToRequest: %v", err) + t.Fatalf("Resolve: %v", err) } if req.Prompt.System != "sys" || req.Prompt.User != "user" { @@ -463,148 +383,10 @@ func TestRuntimeFlagHelpEnumeratesEveryAxis(t *testing.T) { } } -func TestRunBuffered_JSONIncludesFullInputSpec(t *testing.T) { - req := ai.Request{ - Model: api.Model{Name: "claude-sonnet-4-6", Mode: api.ModeAPI, Effort: api.EffortMedium}, - Prompt: api.Prompt{System: "be precise", User: "summarize"}, - Budget: api.Budget{MaxTokens: 2048}, - Setup: &shell.Setup{Cwd: "/repo"}, - Permissions: api.Permissions{ - Presets: []api.Preset{api.PresetEdit}, - Tools: api.Tools{"Read": api.ToolPolicyAllow}, - }, - SessionID: "resume-1", - } - - got, err := runBuffered(context.Background(), promptResultProvider{}, req) - if err != nil { - t.Fatal(err) - } - result, ok := got.(AIPromptResult) - if !ok { - t.Fatalf("runBuffered returned %T, want AIPromptResult", got) - } - if result.Input.Prompt.User != "summarize" || result.Input.Model.Name != "claude-sonnet-4-6" { - t.Fatalf("result input = %+v, want original request", result.Input) - } - if result.InputTokens != 12 { - t.Fatalf("InputTokens = %d, want 12", result.InputTokens) - } - if result.Model != "claude-sonnet-4-6" || result.Provider != "anthropic" || result.Mode != "api" || result.Dir != "/repo" || result.SessionID != "resume-1" { - t.Fatalf("resolved fields = model %q provider %q mode %q dir %q session %q", result.Model, result.Provider, result.Mode, result.Dir, result.SessionID) - } - - var encoded map[string]any - data, err := json.Marshal(result) - if err != nil { - t.Fatal(err) - } - if err := json.Unmarshal(data, &encoded); err != nil { - t.Fatal(err) - } - input, ok := encoded["input"].(map[string]any) - if !ok { - t.Fatalf("json input = %T, want object: %s", encoded["input"], data) - } - prompt, ok := input["prompt"].(map[string]any) - if !ok || prompt["user"] != "summarize" || prompt["system"] != "be precise" { - t.Fatalf("json input.prompt = %#v, want rendered prompt", input["prompt"]) - } - // input is the authored spec, so it carries the model and effort; the resolved - // runtime is run history and is published on the result itself. - if input["model"] != "claude-sonnet-4-6" || input["effort"] != "medium" { - t.Fatalf("json input model fields = %#v", input) - } - if encoded["provider"] != "anthropic" || encoded["mode"] != "api" { - t.Fatalf("json runtime = %#v/%#v, want the resolved pair", encoded["provider"], encoded["mode"]) - } - setup, ok := input["setup"].(map[string]any) - if !ok || setup["cwd"] != "/repo" { - t.Fatalf("json input.setup = %#v, want cwd /repo", input["setup"]) - } - if input["sessionId"] != "resume-1" || encoded["sessionId"] != "resume-1" || encoded["dir"] != "/repo" { - t.Fatalf("json session/dir fields = top %#v input %#v", encoded, input) - } - if got := encoded["inputTokens"]; got != float64(12) { - t.Fatalf("json inputTokens = %#v, want 12", got) - } -} - -func TestRunBuffered_PreservesStructuredOutput(t *testing.T) { - got, err := runBuffered(context.Background(), structuredPromptResultProvider{}, ai.Request{}) - if err != nil { - t.Fatal(err) - } - result, ok := got.(AIPromptResult) - if !ok { - t.Fatalf("runBuffered returned %T, want AIPromptResult", got) - } - if result.Text != `{"answer":"42"}` { - t.Fatalf("Text = %q, want JSON transcript text", result.Text) - } - if result.StructuredOutput["answer"] != "42" { - t.Fatalf("StructuredOutput = %#v, want decoded answer", result.StructuredOutput) - } -} - -func TestRunStreaming_JSONIncludesFullInputSpec(t *testing.T) { - req := ai.Request{ - Model: api.Model{Name: "gpt-5-codex", Mode: api.ModeCLI, Effort: api.EffortHigh}, - Prompt: api.Prompt{User: "fix tests"}, - Setup: &shell.Setup{Cwd: "/repo"}, - Permissions: api.Permissions{MCP: api.MCP{Disabled: true}}, - } - - got, err := runStreaming(context.Background(), promptResultStreamingProvider{}, req) - if err != nil { - t.Fatal(err) - } - result, ok := got.(AIPromptResult) - if !ok { - t.Fatalf("runStreaming returned %T, want AIPromptResult", got) - } - if result.Input.Prompt.User != "fix tests" || result.Input.Model.Mode != api.ModeCLI { - t.Fatalf("result input = %+v, want original request", result.Input) - } - if result.Dir != "/repo" || result.SessionID != "stream-session-1" || result.Input.SessionID != "stream-session-1" { - t.Fatalf("dir/session = dir %q session %q input session %q", result.Dir, result.SessionID, result.Input.SessionID) - } - if result.InputTokens != 21 || result.Output != 9 || result.CostUSD != 0.02 { - t.Fatalf("usage/cost = input %d output %d cost %f", result.InputTokens, result.Output, result.CostUSD) - } -} - -func TestRunStreaming_StructuredResultIsReturnedOnce(t *testing.T) { - for _, tc := range []struct { - name string - text string - }{ - {name: "result only"}, - {name: "replaces prior text", text: "discarded narrative"}, - } { - t.Run(tc.name, func(t *testing.T) { - got, err := runStreaming(context.Background(), structuredResultStreamingProvider{text: tc.text}, ai.Request{}) - if err != nil { - t.Fatal(err) - } - result, ok := got.(AIPromptResult) - if !ok { - t.Fatalf("runStreaming returned %T, want AIPromptResult", got) - } - if result.Text != `{"answer":"42"}` { - t.Fatalf("Text = %q, want authoritative structured JSON once", result.Text) - } - if result.StructuredOutput["answer"] != "42" { - t.Fatalf("StructuredOutput = %#v, want decoded answer", result.StructuredOutput) - } - }) - } -} - func defaultPromptOptions(t *testing.T) AIPromptOptions { t.Helper() return AIPromptOptions{ - AIRuntimeOptions: AIRuntimeOptions{AIProviderOptions: AIProviderOptions{ModelFlags: aiflags.ModelFlags{Temperature: "0"}}}, + AIRuntimeOptions: AIRuntimeOptions{AIProviderOptions: AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "claude-sonnet-5", Mode: "agent", Temperature: "0"}}}, Timeout: "120s", Prompt: "hello", } diff --git a/pkg/cli/attachments.go b/pkg/cli/attachments.go index 89ee891e..b72dcd37 100644 --- a/pkg/cli/attachments.go +++ b/pkg/cli/attachments.go @@ -157,7 +157,11 @@ func resolvePromptAttachments(ctx context.Context, req *ai.Request) error { } func newAttachmentStore(baseDir string) (*attachments.Store, error) { - defaults := loadSavedConfig().Attachments.WithDefaults() + saved, err := loadSavedConfig() + if err != nil { + return nil, err + } + defaults := saved.Attachments.WithDefaults() directory := defaults.Directory if !filepath.IsAbs(directory) { directory = filepath.Join(baseDir, directory) diff --git a/pkg/cli/attachments_gc.go b/pkg/cli/attachments_gc.go index 83a3f45d..dfeac5f7 100644 --- a/pkg/cli/attachments_gc.go +++ b/pkg/cli/attachments_gc.go @@ -27,7 +27,11 @@ func RunAttachmentsGC(opts AttachmentsGCOptions) (any, error) { if err != nil { return nil, err } - retention, err := parseAttachmentRetention(loadSavedConfig().Attachments.WithDefaults().Retention) + saved, err := loadSavedConfig() + if err != nil { + return nil, err + } + retention, err := parseAttachmentRetention(saved.Attachments.WithDefaults().Retention) if err != nil { return nil, err } diff --git a/pkg/cli/attachments_ginkgo_test.go b/pkg/cli/attachments_ginkgo_test.go index 15809339..0f13c643 100644 --- a/pkg/cli/attachments_ginkgo_test.go +++ b/pkg/cli/attachments_ginkgo_test.go @@ -109,9 +109,10 @@ var _ = Describe("attachment flags", func() { User: "What is this image of?", Attachments: []api.AttachmentRef{{Path: path}}, }} req.SetCwd(dir) - result, err := executeSyncBatch(context.Background(), PromptRenderResult{ - Name: "attachment batch", Input: req, - }, AIPromptOptions{MultiModels: []string{"*:sol"}}) + opts := AIPromptOptions{MultiModels: []string{"*:sol"}} + rendered, err := testRenderedVariants(req, opts) + Expect(err).NotTo(HaveOccurred()) + result, err := executeSyncBatch(context.Background(), rendered, opts) Expect(err).NotTo(HaveOccurred()) Expect(result.Status).To(Equal("partial")) diff --git a/pkg/cli/configure.go b/pkg/cli/configure.go index 5e073dab..5e2d7a91 100644 --- a/pkg/cli/configure.go +++ b/pkg/cli/configure.go @@ -3,6 +3,7 @@ package cli import ( "context" "fmt" + "maps" "strconv" "strings" "time" @@ -151,10 +152,11 @@ func runConfigureWizard() (any, error) { Timeout: timeout, Enabled: enabled, }) - for provider, defaults := range current.AI.Providers { - if _, exists := cfg.AI.Providers[provider]; !exists { - cfg.AI.Providers[provider] = defaults - } + cfg.AI, err = mergeConfiguredProvider(providerDefaultsMergeOptions{ + Current: current.AI, Next: cfg.AI, Provider: formProvider, + }) + if err != nil { + return nil, err } cfg.Prompts = current.Prompts cfg.Attachments = current.Attachments @@ -176,6 +178,28 @@ func runConfigureWizard() (any, error) { }, nil } +type providerDefaultsMergeOptions struct { + Current captainconfig.AIDefaults + Next captainconfig.AIDefaults + Provider *api.ModelProvider +} + +func mergeConfiguredProvider(options providerDefaultsMergeOptions) (captainconfig.AIDefaults, error) { + if options.Provider == nil { + return captainconfig.AIDefaults{}, fmt.Errorf("configured provider is required") + } + defaults, ok := options.Next.Providers[options.Provider.Name] + if !ok { + return captainconfig.AIDefaults{}, fmt.Errorf("form result has no defaults for provider %s", options.Provider.Name) + } + merged := options.Next + merged.Providers = maps.Clone(options.Current.Providers) + if err := merged.SetProvider(options.Provider, defaults); err != nil { + return captainconfig.AIDefaults{}, err + } + return merged, nil +} + // formInputs is the raw string/slice output of the huh form. Keeping it in a // struct makes buildConfigFromForm a pure function we can unit-test without a // TTY. diff --git a/pkg/cli/configure_alias_ginkgo_test.go b/pkg/cli/configure_alias_ginkgo_test.go new file mode 100644 index 00000000..dd00a36d --- /dev/null +++ b/pkg/cli/configure_alias_ginkgo_test.go @@ -0,0 +1,143 @@ +package cli + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("provider alias configuration", Serial, func() { + configurePath := func() { + captainconfig.SetPathForTesting(filepath.Join(GinkgoT().TempDir(), ".captain.yaml")) + DeferCleanup(func() { captainconfig.SetPathForTesting("") }) + } + stubGoogleModels := func() { + previous := configureDefaultsModels + configureDefaultsModels = func(_ context.Context, provider *api.ModelProvider, mode api.RuntimeMode) ([]ai.ModelDef, error) { + Expect(provider).To(BeIdenticalTo(api.Google)) + Expect(mode).To(Equal(api.ModeCLI)) + return []ai.ModelDef{{ID: "gemini-3.5-flash"}}, nil + } + DeferCleanup(func() { configureDefaultsModels = previous }) + } + seedAlias := func() { + Expect(captainconfig.Save(captainconfig.Config{AI: captainconfig.AIDefaults{ + Providers: map[string]captainconfig.ProviderDefaults{ + "gemini": {Mode: "cli", Model: "gemini-3.5-flash", ReasoningEffort: "high"}, + }, + }})).To(Succeed()) + } + + It("updates an existing agent-named provider block from the CLI", func() { + configurePath() + stubGoogleModels() + seedAlias() + + _, err := runProviderDefaultsConfigure(context.Background(), api.Google, ConfigureOptions{Effort: "default"}) + Expect(err).NotTo(HaveOccurred()) + + config, _, err := captainconfig.Load() + Expect(err).NotTo(HaveOccurred()) + Expect(config.AI.Providers).To(HaveKey("gemini")) + Expect(config.AI.Providers).NotTo(HaveKey("google")) + Expect(config.AI.Providers["gemini"].ReasoningEffort).To(BeEmpty()) + Expect(config.AI.Validate()).To(Succeed()) + }) + + It("updates an existing agent-named provider block from the HTTP API", func() { + configurePath() + stubGoogleModels() + seedAlias() + + mux := http.NewServeMux() + registerProviderDefaultsHandlers(mux) + request := httptest.NewRequest(http.MethodPut, "/api/captain/ai/providers/gemini/defaults", + strings.NewReader(`{"mode":"cli","model":"gemini-3.5-flash","effort":"medium"}`)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Origin", "http://localhost:9020") + request.Host = "localhost:9020" + request.RemoteAddr = "127.0.0.1:1234" + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + + config, _, err := captainconfig.Load() + Expect(err).NotTo(HaveOccurred()) + Expect(config.AI.Providers).To(HaveKey("gemini")) + Expect(config.AI.Providers).NotTo(HaveKey("google")) + Expect(config.AI.Providers["gemini"].ReasoningEffort).To(Equal("medium")) + Expect(config.AI.Validate()).To(Succeed()) + }) + + It("merges the form selection by provider identity", func() { + merged, err := mergeConfiguredProvider(providerDefaultsMergeOptions{ + Current: captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{ + "anthropic": {Mode: "agent", Model: "claude-existing"}, + "gemini": {Mode: "cli", Model: "gemini-existing"}, + }}, + Next: captainconfig.AIDefaults{ + DefaultProvider: "google", + BudgetUSD: 2, + Providers: map[string]captainconfig.ProviderDefaults{ + "google": {Mode: "cli", Model: "gemini-3.5-flash", ReasoningEffort: "medium"}, + }, + }, + Provider: api.Google, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(merged.Providers).To(Equal(map[string]captainconfig.ProviderDefaults{ + "anthropic": {Mode: "agent", Model: "claude-existing"}, + "gemini": {Mode: "cli", Model: "gemini-3.5-flash", ReasoningEffort: "medium"}, + })) + Expect(merged.BudgetUSD).To(Equal(2.0)) + Expect(merged.Validate()).To(Succeed()) + }) + + It("installs an agent-named provider opt-out under its provider identity", func() { + configurePath() + api.SetDisabled(api.DisabledSet{}) + DeferCleanup(func() { api.SetDisabled(api.DisabledSet{}) }) + + mux := http.NewServeMux() + registerDisabledHandlers(mux) + request := httptest.NewRequest(http.MethodPut, "/api/captain/ai/disabled", + strings.NewReader(`{"modes":[],"providers":["gemini"],"runtimes":[],"models":[],"efforts":[]}`)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Origin", "http://localhost:9020") + request.Host = "localhost:9020" + request.RemoteAddr = "127.0.0.1:1234" + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(api.Disabled().Provider(api.Google)).To(BeTrue()) + }) + + It("counts provider aliases by identity when checking that one remains enabled", func() { + configurePath() + api.SetDisabled(api.DisabledSet{}) + DeferCleanup(func() { api.SetDisabled(api.DisabledSet{}) }) + + mux := http.NewServeMux() + registerDisabledHandlers(mux) + request := httptest.NewRequest(http.MethodPut, "/api/captain/ai/disabled", + strings.NewReader(`{"modes":[],"providers":["google","gemini","anthropic","openai"],"runtimes":[],"models":[],"efforts":[]}`)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Origin", "http://localhost:9020") + request.Host = "localhost:9020" + request.RemoteAddr = "127.0.0.1:1234" + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(api.Disabled().Provider(api.Google)).To(BeTrue()) + Expect(api.Disabled().Provider(api.DeepSeek)).To(BeFalse()) + }) +}) diff --git a/pkg/cli/configure_provider.go b/pkg/cli/configure_provider.go index 5d4a9426..a5585975 100644 --- a/pkg/cli/configure_provider.go +++ b/pkg/cli/configure_provider.go @@ -99,11 +99,10 @@ func runProviderDefaultsConfigure(ctx context.Context, provider *api.ModelProvid return ConfigureProviderDefaultsResult{}, err } if err := captainconfig.Update(func(cfg *captainconfig.Config) error { - if cfg.AI.Providers == nil { - cfg.AI.Providers = map[string]captainconfig.ProviderDefaults{} - } - cfg.AI.Providers[provider.Name] = captainconfig.ProviderDefaults{ + if err := cfg.AI.SetProvider(provider, captainconfig.ProviderDefaults{ Mode: next.Mode, Model: next.Model, ReasoningEffort: next.Effort, + }); err != nil { + return err } if opts.Active { cfg.AI.DefaultProvider = provider.Name diff --git a/pkg/cli/gitagent_hook.go b/pkg/cli/gitagent_hook.go index 7a23e92a..809a169f 100644 --- a/pkg/cli/gitagent_hook.go +++ b/pkg/cli/gitagent_hook.go @@ -100,10 +100,12 @@ func hookJudgeProvider(runtime gitagent.HookRuntime) (ai.Provider, error) { if !runtime.RequiresJudge() { return nil, nil } - cfg, err := (AIProviderOptions{Sandbox: "none"}).ToConfig() + resolved, err := resolveInvocation(AIRuntimeOptions{AIProviderOptions: AIProviderOptions{Sandbox: "none"}}, nil) if err != nil { return nil, fmt.Errorf("configure hook judge: %w", err) } + cfg := resolved.Config + logRuntimeWarnings(resolved.Resolution.Warnings) if strings.TrimSpace(cfg.Model.Name) == "" { return nil, fmt.Errorf("verify prompts declared but no model is configured for the hook judge") } diff --git a/pkg/cli/gitagent_runtask.go b/pkg/cli/gitagent_runtask.go index 772e418a..e1cf5501 100644 --- a/pkg/cli/gitagent_runtask.go +++ b/pkg/cli/gitagent_runtask.go @@ -84,20 +84,21 @@ func runTaskPrompt(ctx context.Context, worktree string, payload gitagent.TaskPa ModelFlags: aiflags.ModelFlags{Model: payload.Model, Mode: string(payload.Mode), Effort: string(payload.Effort)}, Sandbox: string(api.SandboxOff), } - cfg, err := providerOpts.ToConfig() - if err != nil { - return err - } var req ai.Request req.Prompt.User = payload.Prompt req.Prompt.System = payload.System - req.Model = cfg.Model req.Budget.Timeout = payload.Timeout req.SetCwd(worktree) // Editing is the point: a coding agent that cannot write files produces an // empty result and an unexplained silence on the supervisor. req.Sandbox = &api.SandboxRef{Mode: api.SandboxNative} req.Permissions.Mode = api.PermissionAcceptEdits + resolved, err := resolveInvocation(AIRuntimeOptions{AIProviderOptions: providerOpts}, []api.SpecLayer{api.PromptSpecLayer("dispatched task", req)}) + if err != nil { + return err + } + req, cfg := resolved.Request, resolved.Config + logRuntimeWarnings(resolved.Resolution.Warnings) timeout, err := renderedTimeout(PromptRenderResult{Input: req, Config: cfg}) if err != nil { diff --git a/pkg/cli/model_selection_ginkgo_test.go b/pkg/cli/model_selection_ginkgo_test.go index c9409f80..74530705 100644 --- a/pkg/cli/model_selection_ginkgo_test.go +++ b/pkg/cli/model_selection_ginkgo_test.go @@ -26,7 +26,7 @@ var _ = Describe("CLI model selection", func() { opts := AIPromptOptions{} opts.Model = "gemini-3.5-flash" - req, cfg, err := overlayCLI(ai.Request{}, ai.Config{}, opts) + req, cfg, err := runtimeLayersForTest(ai.Request{}, opts) Expect(err).NotTo(HaveOccurred()) Expect(req.Model.Name).To(Equal("gemini-3.5-flash")) @@ -36,7 +36,7 @@ var _ = Describe("CLI model selection", func() { }) It("uses the same precedence for provider options", func() { - cfg, err := (AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "gemini-3.5-flash"}}).ToConfig() + cfg, err := providerConfigForTest(AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "gemini-3.5-flash"}}) Expect(err).NotTo(HaveOccurred()) Expect(cfg.Model.Name).To(Equal("gemini-3.5-flash")) @@ -44,15 +44,19 @@ var _ = Describe("CLI model selection", func() { Expect(cfg.Model.Mode).To(Equal(api.ModeAPI)) }) - It("passes an unsupported valid effort through for runtime degradation", func() { + It("reports effort normalization while preserving the authored selector", func() { // The compact grammar is mode:model[:effort]; the prefix is never an adapter. - cfg, err := (AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "api:gemini-3.6-flash:xhigh"}}).ToConfig() + resolved, err := resolveInvocation(AIRuntimeOptions{AIProviderOptions: AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "api:gemini-3.6-flash:xhigh"}}}, nil) Expect(err).NotTo(HaveOccurred()) + cfg := resolved.Config Expect(cfg.Model.Name).To(Equal("gemini-3.6-flash")) Expect(cfg.Model.Provider).To(Equal(api.Google)) Expect(cfg.Model.Mode).To(Equal(api.ModeAPI)) - Expect(cfg.Model.Effort).To(Equal(api.EffortXHigh)) + Expect(cfg.Model.Effort).To(Equal(api.EffortHigh)) + Expect(resolved.Resolution.Provenance["/effort"].Source.Name).To(Equal("CLI flags")) + Expect(resolved.Resolution.Provenance["/effort"].NormalizedBy).To(HaveField("Kind", api.FieldSourceCatalog)) + Expect(resolved.Resolution.Trace).To(HaveExactElements(HaveField("Spec.Model.Name", "api:gemini-3.6-flash:xhigh"))) }) It("accepts the whoami catalog model when configuring Gemini cli defaults", func() { @@ -62,7 +66,7 @@ var _ = Describe("CLI model selection", func() { }) It("keeps the saved model and runtime paired when there is no override", func() { - cfg, err := (AIProviderOptions{}).ToConfig() + cfg, err := providerConfigForTest(AIProviderOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(cfg.Model.Name).To(Equal("claude-opus-5")) @@ -70,20 +74,13 @@ var _ = Describe("CLI model selection", func() { Expect(cfg.Model.Mode).To(Equal(api.ModeAgent)) }) - // layeredModel resolves a request-layer model over a resolved frontmatter - // model exactly as renderPrompt does: expand, layer, fold, defaults, resolve. layeredModel := func(frontmatter, request api.Model) api.Model { GinkgoHelper() layers, err := promptLayers(nil, "selection.prompt", ai.Request{Model: frontmatter}, &api.Spec{Model: request}) Expect(err).NotTo(HaveOccurred()) - resolved, err := resolvePromptLayers(layers) + resolved, err := resolveInvocation(AIRuntimeOptions{}, layers) Expect(err).NotTo(HaveOccurred()) - req := resolved.Spec - cfg := configFromResolved(req) - Expect(applyPromptDefaults(&req, &cfg)).To(Succeed()) - model, err := ai.Resolve(cfg.Model) - Expect(err).NotTo(HaveOccurred()) - return model + return resolved.Request.Model } frontmatterModel := api.Model{Name: "claude-opus-4-6", Mode: api.ModeAPI, Provider: api.Anthropic} @@ -106,7 +103,7 @@ var _ = Describe("CLI model selection", func() { It("rejects a malformed request-layer selector before layering", func() { _, err := promptLayers(nil, "selection.prompt", ai.Request{Model: frontmatterModel}, &api.Spec{Model: api.Model{Name: "warp:gemini-3.5-flash"}}) - Expect(err).To(MatchError(ContainSubstring("render request model"))) + Expect(err).To(MatchError(ContainSubstring(`spec layer "render request": model:`))) }) It("omits single-runtime identity from multi-model parent labels", func() { @@ -128,11 +125,10 @@ var _ = Describe("CLI model selection", func() { return AIPromptResult{Text: "ok", Model: cfg.Model.Name, Provider: cfg.Model.Provider.Name, Mode: string(cfg.Model.Mode)}, nil } - result, err := executeSyncBatch( - context.Background(), - testRenderedPrompt(api.Model{Name: "opus", Mode: api.ModeAgent}), - AIPromptOptions{AIRuntimeOptions: AIRuntimeOptions{AIProviderOptions: AIProviderOptions{}}, MultiModels: []string{"gemini-3.5-flash"}}, - ) + opts := AIPromptOptions{MultiModels: []string{"gemini-3.5-flash"}} + rendered, err := testRenderedVariants(testRenderedPrompt(api.Model{Name: "opus"}).Input, opts) + Expect(err).NotTo(HaveOccurred()) + result, err := executeSyncBatch(context.Background(), rendered, opts) Expect(err).NotTo(HaveOccurred()) Expect(executed.Model.Name).To(Equal("gemini-3.5-flash")) @@ -148,7 +144,7 @@ var _ = Describe("CLI model selection", func() { configured := withCaps(api.Model{Name: "gpt-5.6-luna", Mode: api.ModeCLI}) selected := withCaps(api.Model{Name: "gemini-3.5-flash", Mode: api.ModeAPI, Effort: api.EffortHigh}) - variant := renderVariant(testRenderedPrompt(configured), selected, nil).Config.Model + variant := renderVariant(testRenderedPrompt(configured), testRuntimeVariant(selected)).Config.Model Expect(variant.Validate()).To(Succeed()) Expect(variant).To(Equal(selected)) @@ -162,7 +158,7 @@ var _ = Describe("CLI model selection", func() { Fallbacks: api.ModelList{{Name: "gemini-3-flash", Mode: api.ModeAPI}}, } - variant := renderVariant(testRenderedPrompt(api.Model{}), selected, nil).Config.Model + variant := renderVariant(testRenderedPrompt(api.Model{}), testRuntimeVariant(selected)).Config.Model Expect(variant.Fallbacks).To(Equal(selected.Fallbacks)) }) diff --git a/pkg/cli/permissions_matrix_test.go b/pkg/cli/permissions_matrix_test.go index 0b9f47d4..75d96b9c 100644 --- a/pkg/cli/permissions_matrix_test.go +++ b/pkg/cli/permissions_matrix_test.go @@ -136,8 +136,8 @@ func TestPermissionsMatrixCoversEveryRuntime(t *testing.T) { // producing an empty or full matrix — the same fail-loud rule the declaration // itself follows. func TestPermissionsMatrixRejectsUnknownSelectors(t *testing.T) { - if _, err := RunPermissionsMatrix(PermissionsMatrixOptions{Provider: "claude"}); err == nil { - t.Fatal("a family nickname that is not a provider key should be refused") + if _, err := RunPermissionsMatrix(PermissionsMatrixOptions{Provider: "acme"}); err == nil { + t.Fatal("an unknown provider should be refused") } if _, err := RunPermissionsMatrix(PermissionsMatrixOptions{Mode: "sdk"}); err == nil { t.Fatal("an unknown runtime mode should be refused") diff --git a/pkg/cli/prompt_batch_run.go b/pkg/cli/prompt_batch_run.go index 6ea25f92..5fef1a87 100644 --- a/pkg/cli/prompt_batch_run.go +++ b/pkg/cli/prompt_batch_run.go @@ -12,6 +12,9 @@ import ( ) func launchAsyncBatch(ctx context.Context, id string, rendered PromptRenderResult, runtimes []api.Model, chat bool) (PromptRunResult, error) { + if err := rendered.validateVariants(); err != nil { + return PromptRunResult{}, err + } if rendered.Input.SessionID != "" { return PromptRunResult{}, errors.New("a multi-model run cannot resume one provider session") } @@ -36,7 +39,7 @@ func launchAsyncBatch(ctx context.Context, id string, rendered PromptRenderResul i := i run := batch.Runs[i] binding := promptBinding(batch, i) - variant := renderVariant(rendered, run.Runtime, nil) + variant := renderVariant(rendered, rendered.variants[i]) runID := run.SessionID.String() stream := promptRuns.create(runID) capabilities := chatCapabilitiesFor(variant.Provider, variant.Mode) diff --git a/pkg/cli/prompt_batch_session.go b/pkg/cli/prompt_batch_session.go index 974aa197..565b18da 100644 --- a/pkg/cli/prompt_batch_session.go +++ b/pkg/cli/prompt_batch_session.go @@ -36,13 +36,9 @@ func createPromptBatchSessions(ctx context.Context, rendered PromptRenderResult, batch := promptBatchSession{ID: uuid.New(), Runs: make([]promptBatchRun, len(runtimes))} children := make([]database.CreateSessionInput, len(runtimes)) for i, runtime := range runtimes { - source := transcriptSource(runtime.Provider, runtime.Mode) - if source == "" { - source = "captain" - } batch.Runs[i] = promptBatchRun{SessionID: uuid.New(), Runtime: runtime} children[i] = database.CreateSessionInput{ - ID: batch.Runs[i].SessionID, Source: source, + ID: batch.Runs[i].SessionID, Source: "captain", Provider: providerName(runtime.Provider), HostID: captainHostID(), CWD: rendered.Input.Cwd(), Title: runtime.Name, InitialPrompt: rendered.Input.Prompt.User, AgentType: "model", Description: runtimeSelector(runtime), diff --git a/pkg/cli/prompt_batch_session_ginkgo_test.go b/pkg/cli/prompt_batch_session_ginkgo_test.go index 22c43032..d9a09b54 100644 --- a/pkg/cli/prompt_batch_session_ginkgo_test.go +++ b/pkg/cli/prompt_batch_session_ginkgo_test.go @@ -66,6 +66,7 @@ var _ = Describe("prompt batch sessions", func() { Expect(run.SessionID).NotTo(Equal(uuid.Nil)) child, err := db.GetSession(GinkgoT().Context(), run.SessionID) Expect(err).NotTo(HaveOccurred()) + Expect(child.Source).To(Equal("captain")) Expect(child.ParentSessionID).NotTo(BeNil()) Expect(*child.ParentSessionID).To(Equal(batch.ID)) Expect(child.RootSessionID).NotTo(BeNil()) @@ -74,4 +75,70 @@ var _ = Describe("prompt batch sessions", func() { Expect(child.Description).To(Equal(runtimeSelector(batch.Runs[i].Runtime))) } }) + + It("adopts a monitor-first transcript beneath its Captain admission session", func() { + handle := dbtest.ForGinkgo(dbtest.Options{Name: "captain_prompt_batch_monitor_first"}) + db, err := database.Open(GinkgoT().Context(), database.WithDSN(handle.DSN()), database.WithMigrations()) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + setCaptainDBForTest(nil) + Expect(db.Close()).To(Succeed()) + }) + setCaptainDBForTest(db) + + rendered := PromptRenderResult{Name: "compare", Provider: "openai", Mode: "cmux", Model: "gpt-5.6-sol"} + rendered.Input.Prompt.User = "Compare these approaches" + rendered.Input.SetCwd("/workspace/captain") + batch, err := createPromptBatchSessions(GinkgoT().Context(), rendered, resolveAll( + api.Model{Name: "gpt-5.6-sol", Mode: api.ModeCmux, Effort: api.EffortHigh}, + api.Model{Name: "gemini-2.5-flash", Mode: api.ModeAPI}, + )) + Expect(err).NotTo(HaveOccurred()) + + local := batch.Runs[0] + providerSessionID := "0195c1de-4ab8-7000-8000-00000000ba7c" + observed, err := db.CreateOrGetSession(GinkgoT().Context(), database.CreateSessionInput{ + ProviderSessionID: providerSessionID, + Source: transcriptSource(local.Runtime.Provider, local.Runtime.Mode), + Provider: providerName(local.Runtime.Provider), + HostID: captainHostID(), + CWD: rendered.Input.Cwd(), + }) + Expect(err).NotTo(HaveOccurred()) + + persistPromptRun(GinkgoT().Context(), promptRunRecordInput{ + Rendered: rendered, RunID: "monitor-first-run", Binding: promptBinding(batch, 0), + SessionID: providerSessionID, Model: local.Runtime.Name, + Provider: local.Runtime.Provider, Mode: local.Runtime.Mode, ResultText: "done", + }) + + admission, err := db.GetSession(GinkgoT().Context(), local.SessionID) + Expect(err).NotTo(HaveOccurred()) + Expect(admission.Source).To(Equal("captain")) + Expect(admission.ProviderSessionID).To(Equal(providerSessionID)) + transcript, err := db.GetTranscriptSession(GinkgoT().Context(), admission.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(transcript.ID).To(Equal(observed.ID)) + Expect(transcript.Source).To(Equal("codex")) + Expect(transcript.ParentRelation).To(Equal(database.SessionParentRelationTranscript)) + Expect(transcript.RootSessionID).To(Equal(&batch.ID)) + + runs, err := db.ListPromptRuns(GinkgoT().Context(), database.PromptRunFilter{SessionID: &admission.ID}) + Expect(err).NotTo(HaveOccurred()) + Expect(runs).To(HaveLen(1)) + Expect(runs[0].ExecutionSessionID).To(Equal(&observed.ID)) + Expect(runs[0].BatchID).To(Equal(&batch.ID)) + Expect(runs[0].State).To(Equal(database.PromptRunStateSucceeded)) + + apiRun := batch.Runs[1] + persistPromptRun(GinkgoT().Context(), promptRunRecordInput{ + Rendered: rendered, RunID: "api-run", Binding: promptBinding(batch, 1), + Model: apiRun.Runtime.Name, Provider: apiRun.Runtime.Provider, + Mode: apiRun.Runtime.Mode, ResultText: "done", + }) + apiRuns, err := db.ListPromptRuns(GinkgoT().Context(), database.PromptRunFilter{SessionID: &apiRun.SessionID}) + Expect(err).NotTo(HaveOccurred()) + Expect(apiRuns).To(HaveLen(1)) + Expect(apiRuns[0].ExecutionSessionID).To(BeNil()) + }) }) diff --git a/pkg/cli/prompt_entity.go b/pkg/cli/prompt_entity.go index 228e6b2c..60c9dfcf 100644 --- a/pkg/cli/prompt_entity.go +++ b/pkg/cli/prompt_entity.go @@ -126,6 +126,7 @@ type PromptRenderResult struct { InputDefault map[string]any `json:"inputDefault,omitempty"` OutputSchema map[string]any `json:"outputSchema,omitempty"` Runtimes []api.Model `json:"runtimes,omitempty"` + variants []AIRuntimeResolved // Resolution is the layer trace the render resolved through; its Spec is // the final request after saved defaults, so the trace explains Input. Resolution api.ResolvedSpec `json:"resolution"` @@ -253,7 +254,7 @@ func listPrompts(ctx context.Context, opts PromptListOptions) ([]PromptSummary, } func getPrompt(ctx context.Context, id string) (PromptDetail, error) { - record, err := resolvePromptRecord(ctx, id) + record, err := resolvePromptRecord(ctx, promptRecordOptions{ID: id}) if err != nil { return PromptDetail{}, err } @@ -269,7 +270,7 @@ func createPrompt(ctx context.Context, body map[string]any) (PromptDetail, error } func writeNewLocalPrompt(ctx context.Context, req PromptWriteRequest) (PromptDetail, error) { - sources, err := buildPromptSources(ctx) + sources, err := buildPromptSources(ctx, promptSourceOptions{}) if err != nil { return PromptDetail{}, err } @@ -315,7 +316,7 @@ func writeNewLocalPrompt(ctx context.Context, req PromptWriteRequest) (PromptDet } func updatePrompt(ctx context.Context, id string, body map[string]any) (PromptDetail, error) { - record, err := resolvePromptRecord(ctx, id) + record, err := resolvePromptRecord(ctx, promptRecordOptions{ID: id}) if err != nil { return PromptDetail{}, err } @@ -352,7 +353,7 @@ func updatePrompt(ctx context.Context, id string, body map[string]any) (PromptDe } func deletePrompt(ctx context.Context, id string) error { - record, err := resolvePromptRecord(ctx, id) + record, err := resolvePromptRecord(ctx, promptRecordOptions{ID: id}) if err != nil { return err } @@ -372,7 +373,7 @@ func deletePrompt(ctx context.Context, id string) error { // renderPromptAction renders a prompt for `captain prompt render`. HTTP callers // pass a structured api.Spec in the body (the request layer over the profile -// and frontmatter layers); the CLI passes flat flags (overlayCLI) plus +// and frontmatter layers); the CLI passes explicit flags plus // filepath/-p/stdin sources. func renderPromptAction(ctx context.Context, id string, flags map[string]string) (PromptRenderResult, error) { if _, isHTTP := clickyrpc.RequestFromContext(ctx); isHTTP { diff --git a/pkg/cli/prompt_layers.go b/pkg/cli/prompt_layers.go index 62f3a2e2..58860833 100644 --- a/pkg/cli/prompt_layers.go +++ b/pkg/cli/prompt_layers.go @@ -8,6 +8,7 @@ import ( "github.com/flanksource/captain/pkg/ai" promptlib "github.com/flanksource/captain/pkg/ai/prompt" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" "github.com/flanksource/captain/pkg/runtimeprofiles" ) @@ -17,15 +18,21 @@ const renderRequestLayer = "render request" // reference, else the prompt's frontmatter pin, else none. The catalog is only // built once a reference exists, so a plain render never opens the database; // a reference that resolves nowhere fails naming it. -func selectRuntimeProfile(ctx context.Context, requested, pin string) (*runtimeprofiles.Resolution, error) { - ref := strings.TrimSpace(requested) +type runtimeProfileSelection struct { + Requested string + Pin string + Config *captainconfig.Config +} + +func selectRuntimeProfile(ctx context.Context, options runtimeProfileSelection) (*runtimeprofiles.Resolution, error) { + ref := strings.TrimSpace(options.Requested) if ref == "" { - ref = strings.TrimSpace(pin) + ref = strings.TrimSpace(options.Pin) } if ref == "" { return nil, nil } - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{Config: options.Config}) if err != nil { return nil, fmt.Errorf("runtime profile %q: %w", ref, err) } @@ -57,31 +64,17 @@ func promptLayers(profile *runtimeprofiles.Resolution, source string, frontmatte return append(layers, api.RequestSpecLayer(renderRequestLayer, request)), nil } -func resolvePromptLayers(layers []api.SpecLayer) (api.ResolvedSpec, error) { - resolved, err := api.ResolveSpecLayers(layers...) - if err != nil { - return api.ResolvedSpec{}, fmt.Errorf("resolve prompt spec layers: %w", err) - } - return resolved, nil -} - -// resolveRenderLayers is the shared seam every render path goes through: -// select the profile (request reference, else the frontmatter pin), then -// resolve presets → profile → frontmatter → request. -func resolveRenderLayers(ctx context.Context, source, content string, frontmatter ai.Request, renderReq PromptRenderRequest) (api.ResolvedSpec, error) { +// renderLayers retains declarations until every request override is available. +func renderLayers(ctx context.Context, source, content string, frontmatter ai.Request, renderReq PromptRenderRequest, saved captainconfig.Config) ([]api.SpecLayer, error) { doc, err := promptlib.Parse(content) if err != nil { - return api.ResolvedSpec{}, err - } - profile, err := selectRuntimeProfile(ctx, renderReq.RuntimeProfile, doc.RuntimeProfile) - if err != nil { - return api.ResolvedSpec{}, err + return nil, err } - layers, err := promptLayers(profile, source, frontmatter, renderReq.Spec) + profile, err := selectRuntimeProfile(ctx, runtimeProfileSelection{Requested: renderReq.RuntimeProfile, Pin: doc.RuntimeProfile, Config: &saved}) if err != nil { - return api.ResolvedSpec{}, err + return nil, err } - return resolvePromptLayers(layers) + return promptLayers(profile, source, frontmatter, renderReq.Spec) } // configFromResolved projects the runtime knobs providers read off ai.Config. diff --git a/pkg/cli/prompt_layers_ginkgo_test.go b/pkg/cli/prompt_layers_ginkgo_test.go index 27be19b3..3842af9a 100644 --- a/pkg/cli/prompt_layers_ginkgo_test.go +++ b/pkg/cli/prompt_layers_ginkgo_test.go @@ -278,7 +278,8 @@ var _ = Describe("prompt layers", func() { Expect(rendered.Input.Budget.MaxTurns).To(Equal(2), "the flag stays above the resolved layers") Expect(rendered.Input.Permissions.Mode).To(Equal(api.PermissionPlan)) Expect(rendered.Input.Memory.SkipUser).To(BeTrue()) - Expect(traceNames(rendered.Resolution)).To(Equal([]string{"Org", "Review run spec", "layered.prompt"})) + Expect(traceNames(rendered.Resolution)).To(Equal([]string{"Org", "Review run spec", "layered.prompt", "CLI flags"})) + Expect(rendered.Resolution.Provenance["/budget/maxTurns"].Source.Name).To(Equal("CLI flags")) }) It("never consults the catalog for a render without a profile reference or pin", func() { diff --git a/pkg/cli/prompt_observe.go b/pkg/cli/prompt_observe.go index 6274d9c1..b544affd 100644 --- a/pkg/cli/prompt_observe.go +++ b/pkg/cli/prompt_observe.go @@ -8,7 +8,6 @@ import ( "io" "slices" "strings" - "time" "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/ai/observation" @@ -89,18 +88,15 @@ func observePromptAction(ctx context.Context, id string, flags map[string]string return api.RuntimeObservation{}, errors.New("prompt observe v1 does not support workflow-backed prompts") } - runtimes, err := ai.ResolveMulti([]string{selector}, rendered.Config.Model) - if err != nil { - return api.RuntimeObservation{}, err - } - if len(runtimes) != 1 { - return api.RuntimeObservation{}, fmt.Errorf("--runtime must resolve to exactly one runtime, got %d", len(runtimes)) - } - runtime := runtimes[0] + runtime := rendered.Config.Model if err := runtime.Validate(); err != nil { return api.RuntimeObservation{}, fmt.Errorf("invalid --runtime: %w", err) } - resolvedEffort, unsupported := resolveObservationEffort(runtime) + requestedRuntime := runtime + if requestedEffort.Value != nil { + requestedRuntime.Effort = api.Effort(*requestedEffort.Value) + } + resolvedEffort, unsupported := resolveObservationEffort(requestedRuntime) result := newRuntimeObservation(selector, runtime, requestedEffort, resolvedEffort) applyObservationCaptureRequest(&result, rendered.Input, flags) if unsupported != "" { @@ -113,11 +109,6 @@ func observePromptAction(ctx context.Context, id string, flags map[string]string } return checkedObservation(result) } - runtime.Effort = resolvedEffort - rendered = renderVariant(rendered, runtime, nil) - rendered.Input.Fallbacks = nil - rendered.Config.Model.Fallbacks = nil - recorder := observation.NewRecorder() req := rendered.Input cfg := rendered.Config @@ -283,63 +274,6 @@ func newRuntimeObservation(selector string, runtime api.Model, requested api.Obs } } -func executeObservationProvider(ctx context.Context, provider ai.Provider, req ai.Request, stream bool, recorder *observation.Recorder) (result observationRunResult) { - start := time.Now() - result = observationRunResult{model: provider.GetModel(), usedStream: stream} - defer func() { result.durationMS = time.Since(start).Milliseconds() }() - if !stream { - response, err := provider.Execute(ctx, req) - result.runtimeErr = err - result.usage = recorder.Snapshot().Usage - if response == nil { - return - } - result.costUSD = response.CostUSD - result.model = firstNonEmpty(response.Model, result.model) - result.terminal = err == nil - return - } - - streamer, ok := provider.(ai.StreamingProvider) - if !ok { - result.runtimeErr = errors.New("resolved provider does not expose streaming") - return - } - events, err := streamer.ExecuteStream(ctx, req) - if err != nil { - result.runtimeErr = err - return - } - for event := range events { - recorder.RecordEvent(event) - if event.Model != "" { - result.model = event.Model - } - switch event.Kind { - case ai.EventError: - result.runtimeErr = errors.New(firstNonEmpty(event.Error, "provider emitted an error event")) - case ai.EventResult: - result.terminal = event.Success - if event.Usage != nil { - usage := *event.Usage - result.usage = &usage - } - result.costUSD = event.CostUSD - if !event.Success && result.runtimeErr == nil { - result.runtimeErr = errors.New("provider reported an unsuccessful result") - } - } - } - if ctx.Err() != nil && result.runtimeErr == nil { - result.runtimeErr = ctx.Err() - } - usageSnapshot := recorder.Snapshot() - if usageSnapshot.UsageObserved { - result.usage = usageSnapshot.Usage - } - return -} - func applyObservationSnapshot(result *api.RuntimeObservation, snapshot observation.Snapshot, streamed, remote bool) { result.Capture.Dispatch.Events = nonNilObservationEvents(snapshot.Dispatch) result.Capture.Permissions.Events = nonNilObservationEvents(snapshot.Permissions) diff --git a/pkg/cli/prompt_observe_execute.go b/pkg/cli/prompt_observe_execute.go new file mode 100644 index 00000000..cbabf93f --- /dev/null +++ b/pkg/cli/prompt_observe_execute.go @@ -0,0 +1,67 @@ +package cli + +import ( + "context" + "errors" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/observation" +) + +func executeObservationProvider(ctx context.Context, provider ai.Provider, req ai.Request, stream bool, recorder *observation.Recorder) (result observationRunResult) { + start := time.Now() + result = observationRunResult{model: provider.GetModel(), usedStream: stream} + defer func() { result.durationMS = time.Since(start).Milliseconds() }() + if !stream { + response, err := provider.Execute(ctx, req) + result.runtimeErr = err + result.usage = recorder.Snapshot().Usage + if response == nil { + return + } + result.costUSD = response.CostUSD + result.model = firstNonEmpty(response.Model, result.model) + result.terminal = err == nil + return + } + + streamer, ok := provider.(ai.StreamingProvider) + if !ok { + result.runtimeErr = errors.New("resolved provider does not expose streaming") + return + } + events, err := streamer.ExecuteStream(ctx, req) + if err != nil { + result.runtimeErr = err + return + } + for event := range events { + recorder.RecordEvent(event) + if event.Model != "" { + result.model = event.Model + } + switch event.Kind { + case ai.EventError: + result.runtimeErr = errors.New(firstNonEmpty(event.Error, "provider emitted an error event")) + case ai.EventResult: + result.terminal = event.Success + if event.Usage != nil { + usage := *event.Usage + result.usage = &usage + } + result.costUSD = event.CostUSD + if !event.Success && result.runtimeErr == nil { + result.runtimeErr = errors.New("provider reported an unsuccessful result") + } + } + } + if ctx.Err() != nil && result.runtimeErr == nil { + result.runtimeErr = ctx.Err() + } + usageSnapshot := recorder.Snapshot() + if usageSnapshot.UsageObserved { + result.usage = usageSnapshot.Usage + } + return +} diff --git a/pkg/cli/prompt_profile_layers_ginkgo_test.go b/pkg/cli/prompt_profile_layers_ginkgo_test.go index 8e708b23..f54c888f 100644 --- a/pkg/cli/prompt_profile_layers_ginkgo_test.go +++ b/pkg/cli/prompt_profile_layers_ginkgo_test.go @@ -17,7 +17,7 @@ var _ = Describe("Composed prompt profile layers", func() { Model: api.Model{Name: "gpt-5.6-sol", Mode: api.ModeCLI}, Permissions: api.Permissions{Tools: api.Tools{"Bash": api.ToolPolicyDeny}}, }}) - profile, err := selectRuntimeProfile(f.ctx, "restricted", "") + profile, err := selectRuntimeProfile(f.ctx, runtimeProfileSelection{Requested: "restricted"}) Expect(err).NotTo(HaveOccurred()) Expect(profile.Resolved).To(Equal(api.ResolvedSpec{})) Expect(profile.Layers).To(HaveLen(1)) @@ -25,7 +25,7 @@ var _ = Describe("Composed prompt profile layers", func() { Model: api.Model{Name: "agent:claude-sonnet-5"}, }) Expect(err).NotTo(HaveOccurred()) - resolved, err := resolvePromptLayers(layers) + resolved, err := api.ResolveSpecLayers(api.ResolveSpecOptions{Layers: layers}) Expect(err).NotTo(HaveOccurred()) Expect(resolved.Spec.Mode).To(Equal(api.ModeAgent)) Expect(resolved.Spec.Permissions.Tools).To(Equal(api.Tools{"Bash": api.ToolPolicyDeny})) diff --git a/pkg/cli/prompt_records.go b/pkg/cli/prompt_records.go index d6d9b51a..ff11057b 100644 --- a/pkg/cli/prompt_records.go +++ b/pkg/cli/prompt_records.go @@ -14,11 +14,12 @@ import ( promptlib "github.com/flanksource/captain/pkg/ai/prompt" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" dp "github.com/google/dotprompt/go/dotprompt" ) func listPromptRecords(ctx context.Context) ([]promptRecord, error) { - sources, err := buildPromptSources(ctx) + sources, err := buildPromptSources(ctx, promptSourceOptions{}) if err != nil { return nil, err } @@ -96,8 +97,13 @@ func listPromptRecordsFromSource(source promptSource) ([]promptRecord, error) { return records, err } -func resolvePromptRecord(ctx context.Context, id string) (promptRecord, error) { - id = strings.TrimSpace(id) +type promptRecordOptions struct { + ID string + Config *captainconfig.Config +} + +func resolvePromptRecord(ctx context.Context, options promptRecordOptions) (promptRecord, error) { + id := strings.TrimSpace(options.ID) if looksLikePromptPath(id) { record, err := filePromptRecord(id) if err == nil { @@ -107,7 +113,7 @@ func resolvePromptRecord(ctx context.Context, id string) (promptRecord, error) { return promptRecord{}, err } } - sources, err := buildPromptSources(ctx) + sources, err := buildPromptSources(ctx, promptSourceOptions{Config: options.Config}) if err != nil { return promptRecord{}, err } @@ -281,6 +287,7 @@ func promptRunModels(models []api.Model) []api.Model { out := make([]api.Model, len(models)) for index, model := range models { out[index] = api.Model{ + Explicit: model.Explicit.Clone(), Name: model.Name, ID: model.ID, Mode: model.Mode, @@ -295,7 +302,7 @@ func promptRunModels(models []api.Model) []api.Model { func promptSummaryFromContent(record promptRecord, content string) (PromptSummary, error) { tmpl := promptlib.Load(content) - req, cfg, err := tmpl.Render(map[string]any{}, nil) + req, cfg, err := tmpl.Render(promptlib.RenderOptions{Data: map[string]any{}, Declared: true}) if err != nil { return PromptSummary{}, err } @@ -313,10 +320,7 @@ func promptSummaryFromContent(record promptRecord, content string) (PromptSummar summary.Model = firstNonEmpty(cfg.Model.Name, req.Name) summary.Mode = firstNonEmpty(string(cfg.Model.Mode), string(req.Mode)) summary.RuntimeProfile = inspection.RuntimeProfile - summary.Runtimes, err = resolvePromptRuntimes(inspection.Runtimes, cfg.Model) - if err != nil { - return PromptSummary{}, err - } + summary.Runtimes = inspection.Runtimes summary.Variables = inspection.Variables return summary, nil } diff --git a/pkg/cli/prompt_render.go b/pkg/cli/prompt_render.go index cb97f473..d2f6b777 100644 --- a/pkg/cli/prompt_render.go +++ b/pkg/cli/prompt_render.go @@ -9,6 +9,7 @@ import ( "github.com/flanksource/captain/pkg/ai" promptlib "github.com/flanksource/captain/pkg/ai/prompt" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" ) // renderPrompt is the HTTP/Spec render path: the caller's structured api.Spec @@ -19,7 +20,11 @@ func renderPrompt(ctx context.Context, id string, renderReq PromptRenderRequest) if strings.TrimSpace(id) == "" { return renderEphemeralPrompt(ctx, renderReq) } - record, err := resolvePromptRecord(ctx, id) + saved, err := loadSavedConfig() + if err != nil { + return PromptRenderResult{}, err + } + record, err := resolvePromptRecord(ctx, promptRecordOptions{ID: id, Config: &saved}) if err != nil { return PromptRenderResult{}, err } @@ -34,16 +39,16 @@ func renderPrompt(ctx context.Context, id string, renderReq PromptRenderRequest) if vars == nil { vars = map[string]any{} } - frontmatter, _, err := promptlib.Load(content).Render(vars, nil) + frontmatter, _, err := promptlib.Load(content).Render(promptlib.RenderOptions{Data: vars, Declared: true}) if err != nil { return PromptRenderResult{}, err } frontmatter.Prompt.Source = record.Rel - resolved, err := resolveRenderLayers(ctx, record.Rel, content, frontmatter, renderReq) + resolved, err := renderLayers(ctx, record.Rel, content, frontmatter, renderReq, saved) if err != nil { return PromptRenderResult{}, err } - return finishPromptRender(record, content, resolved, renderReq.Runtimes) + return completePromptRender(promptRenderInput{Record: record, Content: content, Layers: resolved, Runtimes: renderReq.Runtimes, Saved: saved}) } func renderEphemeralPrompt(ctx context.Context, renderReq PromptRenderRequest) (PromptRenderResult, error) { @@ -54,11 +59,15 @@ func renderEphemeralPrompt(ctx context.Context, renderReq PromptRenderRequest) ( } content := ephemeralPromptContent() frontmatter := ai.Request{Prompt: api.Prompt{Source: ""}} - resolved, err := resolveRenderLayers(ctx, record.Rel, content, frontmatter, renderReq) + saved, err := loadSavedConfig() + if err != nil { + return PromptRenderResult{}, err + } + resolved, err := renderLayers(ctx, record.Rel, content, frontmatter, renderReq, saved) if err != nil { return PromptRenderResult{}, err } - return finishPromptRender(record, content, resolved, renderReq.Runtimes) + return completePromptRender(promptRenderInput{Record: record, Content: content, Layers: resolved, Runtimes: renderReq.Runtimes, Saved: saved}) } func ephemeralPromptContent() string { @@ -70,110 +79,84 @@ description: Ephemeral prompt ` } -// finishPromptRender is the shared tail of the HTTP render paths: fold the -// resolved spec into request + config, resolve the sandbox, apply the saved -// defaults, normalize the context dir, then package the result. -func finishPromptRender(record promptRecord, content string, resolved api.ResolvedSpec, runtimes []api.Model) (PromptRenderResult, error) { - req := resolved.Spec - foldSkillPolicies(&req) - cfg := configFromResolved(req) - // There is no --sandbox over HTTP, so the ref carries the whole selection: - // the layered spec the caller sent, else the prompt's own frontmatter. - if err := applyRunSandbox(&req, &cfg, ""); err != nil { - return PromptRenderResult{}, err - } - if err := applyPromptDefaults(&req, &cfg); err != nil { - return PromptRenderResult{}, err - } - cwd, err := os.Getwd() - if err != nil { - return PromptRenderResult{}, fmt.Errorf("get working directory: %w", err) - } - if err := normalizePromptContextDir(&req, cwd); err != nil { - return PromptRenderResult{}, err - } - return finalizeRenderResult(record, content, req, cfg, runtimes, resolved) +type promptRenderInput struct { + Record promptRecord + Content string + Layers []api.SpecLayer + Runtimes []api.Model + Options AIPromptOptions + Saved captainconfig.Config } -// renderPromptCLI is the CLI render path: load from id | discovered name | -// .prompt filepath | -p | stdin, layer the selected runtime profile beneath -// the frontmatter, and overlay the flat CLI flags (overlayCLI). func renderPromptCLI(ctx context.Context, id string, opts AIPromptOptions, varsJSON, stdin string) (PromptRenderResult, error) { - content, source, usedStdin, record, err := loadPromptContent(ctx, id, opts, stdin) + saved, err := loadSavedConfig() if err != nil { return PromptRenderResult{}, err } - vars, err := promptVars(opts, varsJSON, stdin, usedStdin) + content, source, usedStdin, record, err := loadPromptContent(ctx, promptContentOptions{ID: id, Prompt: opts, Stdin: stdin, Config: &saved}) if err != nil { return PromptRenderResult{}, err } - req, cfg, resolved, err := renderLoadedContent(ctx, content, source, vars, opts) + vars, err := promptVars(opts, varsJSON, stdin, usedStdin) if err != nil { return PromptRenderResult{}, err } - result, err := finalizeRenderResult(record, content, req, cfg, nil, resolved) + layers, err := renderLoadedLayers(ctx, content, source, vars, opts, saved) if err != nil { return PromptRenderResult{}, err } - if len(opts.MultiModels) > 0 { - result.Runtimes, err = ai.ResolveMulti(opts.MultiModels, result.Config.Model) - if err != nil { - return PromptRenderResult{}, err - } - } - return result, nil + return completePromptRender(promptRenderInput{Record: record, Content: content, Layers: layers, Runtimes: fallbackModelsFromFlags(opts.MultiModels), Options: opts, Saved: saved}) } -// finalizeRenderResult packages the rendered request/config + prompt detail into -// a PromptRenderResult (shared by both paths). A comma-separated model is -// normalized into a clean primary + fallbacks so the displayed Model is a single -// name, and a mistyped model (primary or any fallback) is caught at render time, -// not just on run. The resolution's spec is replaced with the final request so -// the trace explains exactly what runs. -func finalizeRenderResult(record promptRecord, content string, req ai.Request, cfg ai.Config, runtimeOverride []api.Model, resolved api.ResolvedSpec) (PromptRenderResult, error) { - req.Model = req.ExpandCSV() - cfg.Model = cfg.Model.ExpandCSV() - var err error - req.Model, err = ai.Resolve(req.Model) +func completePromptRender(input promptRenderInput) (PromptRenderResult, error) { + saved := input.Saved + cwd, err := os.Getwd() if err != nil { - return PromptRenderResult{}, err + return PromptRenderResult{}, fmt.Errorf("get working directory: %w", err) } - cfg.Model, err = ai.Resolve(cfg.Model) + flags, err := input.Options.requestSpec() if err != nil { return PromptRenderResult{}, err } - for _, c := range cfg.Model.Candidates() { - warnIfLikelyModelTypo(c.Name) + layers := append([]api.SpecLayer(nil), input.Layers...) + if len(flags.Fields()) > 0 { + layers = append(layers, api.RequestSpecLayer("CLI flags", flags)) } - detail, err := parsedPromptDetail(record, content) + detail, err := parsedPromptDetail(input.Record, input.Content) if err != nil { return PromptRenderResult{}, err } - runtimes := detail.Runtimes - if len(runtimeOverride) > 0 { - runtimes = runtimeOverride + runtimes := input.Runtimes + if len(runtimes) == 0 { + runtimes = detail.Runtimes } - runtimes, err = resolvePromptRuntimes(runtimes, cfg.Model) + variants, err := resolvePromptRuntimes(promptRuntimeOptions{Models: runtimes, Layers: layers, Options: input.Options.AIRuntimeOptions, Saved: saved, Cwd: cwd, CLI: len(input.Options.MultiModels) > 0}) if err != nil { return PromptRenderResult{}, err } - resolved.Spec = req + var result AIRuntimeResolved + if len(variants) == 0 { + result, err = input.Options.resolveAuthored(AIRuntimeResolveOptions{Layers: layers, Saved: saved, Cwd: cwd}) + if err != nil { + return PromptRenderResult{}, err + } + } else { + result = variants[0] + runtimes = make([]api.Model, len(variants)) + result.Resolution.Warnings = nil + for i, variant := range variants { + runtimes[i] = variant.Request.Model + for _, warning := range variant.Resolution.Warnings { + result.Resolution.Warnings = append(result.Resolution.Warnings, fmt.Sprintf("runtime %d: %s", i+1, warning)) + } + } + } + req, cfg := result.Request, result.Config return PromptRenderResult{ - ID: detail.ID, - Name: detail.Name, - Model: cfg.Model.Name, - Provider: providerName(cfg.Model.Provider), - Mode: string(cfg.Model.Mode), - User: req.Prompt.User, - System: req.Prompt.System, - Input: req, - Config: cfg, - InputSchema: detail.InputSchema, - InputDefault: detail.InputDefault, - OutputSchema: detail.OutputSchema, - Runtimes: runtimes, - Resolution: resolved, - ValidationError: renderValidationError(req, cfg, runtimes), + ID: detail.ID, Name: detail.Name, Model: cfg.Model.Name, Provider: providerName(cfg.Model.Provider), Mode: string(cfg.Model.Mode), + User: req.Prompt.User, System: req.Prompt.System, Input: req, Config: cfg, + InputSchema: detail.InputSchema, InputDefault: detail.InputDefault, OutputSchema: detail.OutputSchema, + Runtimes: runtimes, variants: variants, Resolution: result.Resolution, ValidationError: renderValidationError(req, cfg, runtimes), }, nil } @@ -187,7 +170,8 @@ func renderValidationError(req ai.Request, cfg ai.Config, runtimes []api.Model) if err := req.ValidateRunnable(); err != nil { return err.Error() } - if cfg.Model.Name == "" && len(runtimes) == 0 { + needsModel := !req.IsVerifyOnly() || len(req.Workflow.Verify.Prompts) > 0 + if cfg.Model.Name == "" && len(runtimes) == 0 && needsModel { return "no model: set prompt frontmatter, pass a model override, or run 'captain configure'" } // A prompt that declares its own runtimes needs no singular base model: @@ -204,28 +188,3 @@ func renderValidationError(req ai.Request, cfg ai.Config, runtimes []api.Model) } return "" } - -func resolvePromptRuntimes(runtimes []api.Model, base api.Model) ([]api.Model, error) { - if len(runtimes) == 0 { - return nil, nil - } - resolved := make([]api.Model, len(runtimes)) - for i, runtime := range runtimes { - if runtime.Temperature == nil { - runtime.Temperature = base.Temperature - } - if runtime.Effort == api.EffortNone { - runtime.Effort = base.Effort - } - runtime.NoCache = runtime.NoCache || base.NoCache - var err error - resolved[i], err = ai.Resolve(runtime) - if err != nil { - return nil, fmt.Errorf("runtime %d: %w", i+1, err) - } - } - if err := validatePromptRuntimes(resolved); err != nil { - return nil, err - } - return resolved, nil -} diff --git a/pkg/cli/prompt_render_test.go b/pkg/cli/prompt_render_test.go index 7899b1ee..df4a2079 100644 --- a/pkg/cli/prompt_render_test.go +++ b/pkg/cli/prompt_render_test.go @@ -7,7 +7,6 @@ import ( "strings" "testing" - "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/api/registry" "github.com/flanksource/captain/pkg/captainconfig" @@ -47,10 +46,10 @@ Hello {{name}} Variables: map[string]any{"name": "Ada"}, Spec: &api.Spec{ Model: api.Model{ - Name: "gpt-4o", - ID: "openai/gpt-4o", - Provider: api.OpenAI, - Mode: api.ModeAPI, + Name: "claude-sonnet-5", + ID: "anthropic/claude-sonnet-5", + Provider: api.Anthropic, + Mode: api.ModeAgent, Temperature: &temp, Effort: api.EffortLow, NoCache: true, @@ -108,11 +107,11 @@ Hello {{name}} if rendered.ValidationError != "" { t.Fatalf("render validation error = %q", rendered.ValidationError) } - if rendered.Model != "gpt-4o" || rendered.Provider != "openai" || rendered.Mode != "api" { - t.Fatalf("rendered model/runtime = %s %s/%s, want openai api/gpt-4o", rendered.Provider, rendered.Mode, rendered.Model) + if rendered.Model != "claude-sonnet-5" || rendered.Provider != "anthropic" || rendered.Mode != "agent" { + t.Fatalf("rendered model/runtime = %s %s/%s, want anthropic agent/claude-sonnet-5", rendered.Provider, rendered.Mode, rendered.Model) } - if rendered.Config.Model.ID != "openai/gpt-4o" { - t.Fatalf("config model ID = %q, want openai/gpt-4o", rendered.Config.Model.ID) + if rendered.Config.Model.ID != "anthropic/claude-sonnet-5" { + t.Fatalf("config model ID = %q, want anthropic/claude-sonnet-5", rendered.Config.Model.ID) } if rendered.Input.Temperature == nil || *rendered.Input.Temperature != temp { t.Fatalf("temperature = %v, want %v", rendered.Input.Temperature, temp) @@ -439,17 +438,16 @@ func TestRenderPromptRunnability(t *testing.T) { const verifyFrontmatter = "workflow:\n verify:\n commands:\n - \"true\"\n" -func TestApplyPromptDefaultsSelectorEffortWins(t *testing.T) { +func TestResolvePromptSelectorEffortWins(t *testing.T) { isolateCaptainConfig(t) - req := ai.Request{Model: api.Model{Effort: api.EffortLow}} - cfg := ai.Config{Model: api.Model{ - Name: "gpt-5.6-sol", - Mode: api.ModeAgent, - Effort: api.EffortHigh, - }} - if err := applyPromptDefaults(&req, &cfg); err != nil { - t.Fatalf("applyPromptDefaults: %v", err) + resolved, err := resolveInvocation(AIRuntimeOptions{}, []api.SpecLayer{ + api.PromptSpecLayer("template", api.Spec{Model: api.Model{Effort: api.EffortLow}}), + api.RequestSpecLayer("request", api.Spec{Model: api.Model{Name: "gpt-5.6-sol", Mode: api.ModeAgent, Effort: api.EffortHigh}}), + }) + if err != nil { + t.Fatal(err) } + req, cfg := resolved.Request, resolved.Config if req.Name != "gpt-5.6-sol" || req.Mode != api.ModeAgent || req.Effort != api.EffortHigh { t.Fatalf("request = %+v, want selector model/effort", req.Model) } diff --git a/pkg/cli/prompt_run.go b/pkg/cli/prompt_run.go index 1679936a..d3cb6838 100644 --- a/pkg/cli/prompt_run.go +++ b/pkg/cli/prompt_run.go @@ -69,6 +69,9 @@ func runPromptAction(ctx context.Context, id string, flags map[string]string) (P if rendered.ValidationError != "" { return PromptRunResult{}, errors.New(rendered.ValidationError) } + if !isHTTP { + logRuntimeWarnings(rendered.Resolution.Warnings) + } if chatRequested { if rendered.Input.Prompt.HasSchema() { return PromptRunResult{}, errors.New("chat mode does not support structured-output prompts") @@ -236,18 +239,17 @@ func executeSyncWorkflowRun(t *task.Task, rendered PromptRenderResult, noStream func executeSyncBatch(ctx context.Context, rendered PromptRenderResult, opts AIPromptOptions) (PromptRunResult, error) { models := rendered.Runtimes - if len(models) == 0 { - var err error - models, err = ai.ResolveMulti(opts.MultiModels, rendered.Config.Model) - if err != nil { - return PromptRunResult{}, err - } + if err := rendered.validateVariants(); err != nil { + return PromptRunResult{}, err } if len(models) == 0 { + if len(opts.MultiModels) > 0 { + return PromptRunResult{}, errors.New("multi-model runtimes must be prepared by prompt rendering") + } return executeSyncRunSingle(ctx, rendered, opts) } if len(models) == 1 { - variant := renderVariant(rendered, models[0], fallbackModelsFromFlags(opts.Fallback)) + variant := renderVariant(rendered, rendered.variants[0]) opts.MultiModels = nil start := time.Now() single, runErr := executeSyncRunSingle(ctx, variant, opts) @@ -303,7 +305,7 @@ func executeSyncBatch(ctx context.Context, rendered PromptRenderResult, opts AIP selector += ":" + string(model.Effort) } tasks[i] = group.Add(selector, func(_ flanksourceContext.Context, t *task.Task) (PromptRunItem, error) { - variant := renderVariant(rendered, model, fallbackModelsFromFlags(opts.Fallback)) + variant := renderVariant(rendered, rendered.variants[i]) variantOpts := opts variantOpts.MultiModels = nil taskCtx := ai.ContextWithLogger(t.Context(), t) @@ -417,17 +419,3 @@ func promptTaskLabelsWithID(rendered PromptRenderResult, id, mode string) map[st } return labels } - -func renderVariant(rendered PromptRenderResult, model api.Model, fallbacks []api.Model) PromptRenderResult { - out := rendered - req := rendered.Input - cfg := rendered.Config - req.Model = variantModel(model, fallbacks) - cfg.Model = variantModel(model, fallbacks) - out.Input = req - out.Config = cfg - out.Model = cfg.Model.Name - out.Provider = providerName(cfg.Model.Provider) - out.Mode = string(cfg.Model.Mode) - return out -} diff --git a/pkg/cli/prompt_run_history.go b/pkg/cli/prompt_run_history.go index 5de7bf92..9d279e45 100644 --- a/pkg/cli/prompt_run_history.go +++ b/pkg/cli/prompt_run_history.go @@ -63,10 +63,3 @@ func codexHistoryFile(sessionID string) string { } return "" } - -func variantModel(model api.Model, fallbacks []api.Model) api.Model { - if len(fallbacks) > 0 { - model.Fallbacks = fallbacks - } - return model -} diff --git a/pkg/cli/prompt_run_persist.go b/pkg/cli/prompt_run_persist.go index 10682f0e..d5f3a0bc 100644 --- a/pkg/cli/prompt_run_persist.go +++ b/pkg/cli/prompt_run_persist.go @@ -42,46 +42,31 @@ func persistPromptRun(ctx context.Context, input promptRunRecordInput) { if input.Binding == nil && strings.TrimSpace(input.SessionID) == "" { return } - source := transcriptSource(input.Provider, input.Mode) - if source == "" { - source = "claude" - } db, err := captainDefaultDB(ctx) if err != nil { log.Errorf("persist prompt run for session %s: %v", input.SessionID, err) return } - var session *database.Session - if input.Binding != nil { - session, err = db.GetSession(ctx, input.Binding.SessionID) - if err == nil && strings.TrimSpace(input.SessionID) != "" { - providerSessionID := strings.TrimSpace(input.SessionID) - session, err = db.UpdateSessionState(ctx, database.UpdateSessionStateInput{ - ID: session.ID, ExpectedVersion: session.StateVersion, ProviderSessionID: &providerSessionID, - }) - } - } else { - session, err = db.CreateOrGetSession(ctx, database.CreateSessionInput{ - ProviderSessionID: input.SessionID, Source: source, HostID: captainHostID(), - Provider: providerName(input.Provider), CWD: input.Rendered.Input.Cwd(), - }) - } - if err != nil { - log.Errorf("persist prompt run for session %s: %v", firstNonEmpty(input.SessionID, bindingSessionID(input.Binding)), err) - return - } batchID := input.BatchID if input.Binding != nil { batchID = &input.Binding.BatchID } + var session *database.Session var runID uuid.UUID err = db.Transaction(ctx, func(tx *database.DB) error { + var executionSessionID *uuid.UUID + var sessionErr error + session, executionSessionID, sessionErr = preparePromptRunSession(ctx, tx, input) + if sessionErr != nil { + return sessionErr + } run, createErr := tx.CreatePromptRun(ctx, database.CreatePromptRunInput{ - SessionID: session.ID, - BatchID: batchID, - Origin: "captain", - AdmissionKey: input.RunID, - RenderedSpec: renderedSpecMap(input.Rendered), + SessionID: session.ID, + ExecutionSessionID: executionSessionID, + BatchID: batchID, + Origin: "captain", + AdmissionKey: input.RunID, + RenderedSpec: renderedSpecMap(input.Rendered), Runtime: database.PromptRunRuntime{ Mode: "run", Resolved: database.PromptRunRuntimeSelection{ @@ -132,7 +117,46 @@ func persistPromptRun(ctx context.Context, input promptRunRecordInput) { lifecycle = database.SessionLifecycleFailed } updatePromptSessionLifecycle(ctx, session.ID, lifecycle, input.Error) - trackLaunchedTranscript(input, source) + trackLaunchedTranscript(input, transcriptSource(input.Provider, input.Mode)) +} + +func preparePromptRunSession(ctx context.Context, tx *database.DB, input promptRunRecordInput) (*database.Session, *uuid.UUID, error) { + if input.Binding == nil { + source := transcriptSource(input.Provider, input.Mode) + if source == "" { + source = "claude" + } + session, err := tx.CreateOrGetSession(ctx, database.CreateSessionInput{ + ProviderSessionID: input.SessionID, Source: source, HostID: captainHostID(), + Provider: providerName(input.Provider), CWD: input.Rendered.Input.Cwd(), + }) + return session, nil, err + } + + session, err := tx.GetSession(ctx, input.Binding.SessionID) + providerSessionID := strings.TrimSpace(input.SessionID) + if err != nil || providerSessionID == "" { + return session, nil, err + } + session, err = tx.UpdateSessionState(ctx, database.UpdateSessionStateInput{ + ID: session.ID, ExpectedVersion: session.StateVersion, ProviderSessionID: &providerSessionID, + }) + if err != nil { + return nil, nil, err + } + source := transcriptSource(input.Provider, input.Mode) + if source == "" { + return session, nil, nil + } + transcript, err := tx.CreateOrGetSession(ctx, database.CreateSessionInput{ + ProviderSessionID: providerSessionID, Source: source, HostID: session.HostID, + Provider: providerName(input.Provider), CWD: session.CWD, + ParentSessionID: &session.ID, ParentRelation: database.SessionParentRelationTranscript, + }) + if err != nil { + return nil, nil, err + } + return session, &transcript.ID, nil } // upsertPromptRunIterations writes the run's per-turn rows after the run row diff --git a/pkg/cli/prompt_run_test.go b/pkg/cli/prompt_run_test.go index b8fc6a13..d83a15da 100644 --- a/pkg/cli/prompt_run_test.go +++ b/pkg/cli/prompt_run_test.go @@ -57,7 +57,12 @@ func TestExecuteSyncRunMultiModelsParallel(t *testing.T) { defer cancel() rendered := testRenderedPrompt(api.Model{Name: "claude-sonnet-5", Mode: api.ModeAPI}) rendered.Input.SetCwd(t.TempDir()) - got, err := executeSyncRun(ctx, rendered, AIPromptOptions{MultiModels: []string{"cli:sonnet-5,cmux:opus"}}) + opts := AIPromptOptions{MultiModels: []string{"cli:sonnet-5,cmux:opus"}} + rendered, err := testRenderedVariants(rendered.Input, opts) + if err != nil { + t.Fatal(err) + } + got, err := executeSyncRun(ctx, rendered, opts) if err != nil { t.Fatalf("executeSyncRun: %v", err) } @@ -109,7 +114,12 @@ func TestExecuteSyncRunMultiModelsHonorsNoStream(t *testing.T) { } rendered := testRenderedPrompt(api.Model{Name: "claude-sonnet-5", Mode: api.ModeAPI}) - got, err := executeSyncRun(context.Background(), rendered, AIPromptOptions{MultiModels: []string{"cli:sonnet-5"}, NoStream: true}) + opts := AIPromptOptions{MultiModels: []string{"cli:sonnet-5"}, NoStream: true} + rendered, err := testRenderedVariants(rendered.Input, opts) + if err != nil { + t.Fatal(err) + } + got, err := executeSyncRun(context.Background(), rendered, opts) if err != nil { t.Fatalf("executeSyncRun: %v", err) } @@ -139,7 +149,12 @@ func TestExecuteSyncRunMultiModelsPartialFailure(t *testing.T) { } rendered := testRenderedPrompt(api.Model{Name: "gpt-5.5", Mode: api.ModeAPI}) - got, err := executeSyncRun(context.Background(), rendered, AIPromptOptions{MultiModels: []string{"api:gpt-5.5,cmux:gpt-5.5"}}) + opts := AIPromptOptions{MultiModels: []string{"api:gpt-5.5,cmux:gpt-5.5"}} + rendered, err := testRenderedVariants(rendered.Input, opts) + if err != nil { + t.Fatal(err) + } + got, err := executeSyncRun(context.Background(), rendered, opts) if err != nil { t.Fatalf("executeSyncRun: %v", err) } @@ -154,7 +169,12 @@ func TestExecuteSyncRunMultiModelsPartialFailure(t *testing.T) { func TestExecuteSyncRunMultiModelsRejectsResume(t *testing.T) { rendered := testRenderedPrompt(api.Model{Name: "gpt-5.5", Mode: api.ModeAPI}) rendered.Input.SessionID = "session-1" - _, err := executeSyncRun(context.Background(), rendered, AIPromptOptions{MultiModels: []string{"api:gpt-5.5,cmux:gpt-5.5"}}) + opts := AIPromptOptions{MultiModels: []string{"api:gpt-5.5,cmux:gpt-5.5"}} + rendered, err := testRenderedVariants(rendered.Input, opts) + if err != nil { + t.Fatal(err) + } + _, err = executeSyncRun(context.Background(), rendered, opts) if err == nil || !strings.Contains(err.Error(), "--resume") { t.Fatalf("error = %v, want --resume rejection", err) } @@ -162,7 +182,7 @@ func TestExecuteSyncRunMultiModelsRejectsResume(t *testing.T) { func TestVariantModelUsesSelectorEffort(t *testing.T) { selector := api.Model{Name: "gpt-5.6-terra", Mode: api.ModeCmux, Effort: api.EffortUltra} - got := variantModel(selector, nil) + got := renderVariant(PromptRenderResult{}, testRuntimeVariant(selector)).Input.Model if got.Name != selector.Name || got.Provider != selector.Provider || got.Mode != selector.Mode || got.Effort != api.EffortUltra { t.Fatalf("variant = %+v, want selector model/runtime/effort", got) } @@ -314,3 +334,16 @@ func testRenderedPrompt(model api.Model) PromptRenderResult { Config: cfg, } } + +func testRuntimeVariant(model api.Model) AIRuntimeResolved { + spec := api.Spec{Model: model} + return AIRuntimeResolved{Request: spec, Config: ai.Config{Model: model}, Resolution: api.ResolvedSpec{Spec: spec}} +} + +func testRenderedVariants(spec api.Spec, options AIPromptOptions) (PromptRenderResult, error) { + return completePromptRender(promptRenderInput{ + Record: promptRecord{Rel: "test.prompt"}, Content: "hello", Options: options, + Layers: []api.SpecLayer{api.PromptSpecLayer("test.prompt", spec)}, + Runtimes: fallbackModelsFromFlags(options.MultiModels), + }) +} diff --git a/pkg/cli/prompt_runtime_variants.go b/pkg/cli/prompt_runtime_variants.go new file mode 100644 index 00000000..5ba1cf10 --- /dev/null +++ b/pkg/cli/prompt_runtime_variants.go @@ -0,0 +1,110 @@ +package cli + +import ( + "fmt" + "maps" + "reflect" + "slices" + "strings" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/captainconfig" +) + +type promptRuntimeOptions struct { + Models []api.Model + Layers []api.SpecLayer + Options AIRuntimeOptions + Saved captainconfig.Config + Cwd string + CLI bool +} + +func resolvePromptRuntimes(options promptRuntimeOptions) ([]AIRuntimeResolved, error) { + var variants []AIRuntimeResolved + var models []api.Model + seen := map[string]bool{} + for i, runtime := range options.Models { + candidates, err := promptRuntimeLayers(i, runtime) + if err != nil { + return nil, err + } + for _, candidate := range candidates { + layers := append(append([]api.SpecLayer(nil), options.Layers...), candidate) + result, err := options.Options.resolveAuthored(AIRuntimeResolveOptions{Layers: layers, Saved: options.Saved, Cwd: options.Cwd, RequireModel: true}) + if err != nil { + return nil, fmt.Errorf("%s: %w", candidate.Name, err) + } + key := result.Request.RuntimeKey() + if options.CLI && seen[key] { + continue + } + seen[key] = true + variants = append(variants, result) + models = append(models, result.Request.Model) + } + } + if len(models) > 0 && (len(models) != 1 || !options.CLI) { + if err := validatePromptRuntimes(models); err != nil { + return nil, err + } + } + return variants, nil +} + +func promptRuntimeLayers(index int, runtime api.Model) ([]api.SpecLayer, error) { + name := fmt.Sprintf("runtime %d", index+1) + selector := strings.TrimSpace(runtime.Name) + if strings.HasPrefix(selector, "*:") { + models, err := registry.ParseModelMulti(selector, registry.ParseOptions{}) + if err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + layers := make([]api.SpecLayer, len(models)) + for i, model := range models { + candidate := runtime + candidate.Name = runtimeSelector(model) + layers[i] = api.RequestSpecLayer(name+" ("+selector+")", api.Spec{Model: candidate}) + } + return layers, nil + } + prefix := strings.TrimSuffix(selector, ":") + if mode, valid := registry.ParseRuntimeMode(prefix); valid && selector != "" { + runtime.Name, runtime.Mode = "", mode + runtime.Explicit = runtime.Explicit.Clone() + delete(runtime.Explicit, "/model") + } else if strings.HasSuffix(selector, ":") { + return nil, fmt.Errorf("%s: invalid runtime mode %q", name, prefix) + } + return []api.SpecLayer{api.RequestSpecLayer(name, api.Spec{Model: runtime})}, nil +} + +func (rendered PromptRenderResult) validateVariants() error { + if len(rendered.variants) != len(rendered.Runtimes) { + return fmt.Errorf("prompt runtime variants are not prepared: got %d resolved candidates for %d runtimes", len(rendered.variants), len(rendered.Runtimes)) + } + return nil +} + +func renderVariant(rendered PromptRenderResult, variant AIRuntimeResolved) PromptRenderResult { + out := rendered + out.Input, out.Config, out.Resolution = variant.Request, variant.Config, variant.Resolution + if !reflect.DeepEqual(out.Input.Prompt.Attachments, rendered.Input.Prompt.Attachments) { + out.Input.Prompt.Attachments = slices.Clone(rendered.Input.Prompt.Attachments) + out.Resolution.Spec = out.Input + out.Resolution.Provenance = maps.Clone(out.Resolution.Provenance) + if out.Resolution.Provenance == nil { + out.Resolution.Provenance = map[string]api.FieldProvenance{} + } + path := "/prompt/attachments" + origin := out.Resolution.Provenance[path] + origin.NormalizedBy = &api.FieldSource{Kind: api.FieldSourceContext, Name: "attachment preparation", Key: path} + out.Resolution.Provenance[path] = origin + } + out.Model = out.Config.Model.Name + out.Provider = providerName(out.Config.Model.Provider) + out.Mode = string(out.Config.Model.Mode) + out.Runtimes, out.variants = nil, nil + return out +} diff --git a/pkg/cli/prompt_runtime_variants_ginkgo_test.go b/pkg/cli/prompt_runtime_variants_ginkgo_test.go new file mode 100644 index 00000000..cf218ade --- /dev/null +++ b/pkg/cli/prompt_runtime_variants_ginkgo_test.go @@ -0,0 +1,116 @@ +package cli + +import ( + "context" + "os" + "path/filepath" + "strings" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("admitted prompt runtime variants", func() { + var path string + var options AIPromptOptions + + BeforeEach(func() { + dir := GinkgoT().TempDir() + configPath := filepath.Join(dir, ".captain.yaml") + Expect(os.WriteFile(configPath, []byte("ai:\n providers:\n anthropic:\n mode: api\n reasoningEffort: high\n"), 0o600)).To(Succeed()) + captainconfig.SetPathForTesting(configPath) + DeferCleanup(captainconfig.SetPathForTesting, "") + path = filepath.Join(dir, "review.prompt") + Expect(os.WriteFile(path, []byte("---\nmodel: api:sonnet\n---\nReview the change."), 0o600)).To(Succeed()) + options = AIPromptOptions{MultiModels: []string{"agent:sonnet", "agent:opus"}} + options.Fallback = []string{"sonnet-4-6"} + }) + + It("dispatches the complete fallback settings admitted by preview", func() { + rendered, err := renderPromptCLI(context.Background(), path, options, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(rendered.Runtimes[0].Fallbacks).To(HaveExactElements(HaveField("Mode", api.ModeAPI))) + variant := renderVariant(rendered, rendered.variants[0]) + Expect(variant.Input.Model).To(Equal(rendered.Runtimes[0])) + Expect(variant.Config.Model).To(Equal(rendered.Runtimes[0])) + }) + + It("carries the admitted variant request and its authored provenance together", func() { + rendered, err := renderPromptCLI(context.Background(), path, options, "", "") + Expect(err).NotTo(HaveOccurred()) + variant := renderVariant(rendered, rendered.variants[1]) + Expect(variant.Resolution.Spec).To(Equal(variant.Input)) + Expect(variant.Resolution.Provenance["/model"].Source.Name).To(Equal("runtime 2")) + Expect(variant.Resolution.Trace).To(ContainElement(SatisfyAll(HaveField("Name", "runtime 2"), HaveField("Spec.Model.Name", "agent:opus")))) + }) + + It("validates the executing variants after they repair the lower runtime mode", func() { + Expect(os.WriteFile(path, []byte("---\nmodel: api:sonnet\nsandbox: docker\nruntimes:\n - cli:sonnet\n - cli:opus\n---\nReview the change."), 0o600)).To(Succeed()) + rendered, err := renderPromptCLI(context.Background(), path, AIPromptOptions{}, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(rendered.ValidationError).To(BeEmpty()) + Expect(rendered.Runtimes).To(HaveEach(HaveField("Mode", api.ModeCLI))) + }) + + It("expands an explicit wildcard through the shared runtime catalog before final composition", func() { + options.MultiModels, options.Fallback = []string{"*:sol"}, nil + rendered, err := renderPromptCLI(context.Background(), path, options, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(rendered.Runtimes).To(HaveLen(4)) + Expect(rendered.Runtimes).To(HaveEach(HaveField("Provider", api.OpenAI))) + }) + + It("fills mode-only variants from the lower authored model", func() { + Expect(os.WriteFile(path, []byte("---\nmodel: sonnet\nmode: api\n---\nReview."), 0o600)).To(Succeed()) + options.MultiModels, options.Fallback = []string{"cli:", "cmux:"}, nil + rendered, err := renderPromptCLI(context.Background(), path, options, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(rendered.Runtimes).To(HaveExactElements( + SatisfyAll(HaveField("Name", "claude-sonnet-5"), HaveField("Mode", api.ModeCLI)), + SatisfyAll(HaveField("Name", "claude-sonnet-5"), HaveField("Mode", api.ModeCmux)), + )) + }) + + It("accepts bare mode prefixes through the shared runtime grammar", func() { + Expect(os.WriteFile(path, []byte("---\nmodel: sonnet\n---\nReview."), 0o600)).To(Succeed()) + options.MultiModels, options.Fallback = []string{"cli", "cmux"}, nil + rendered, err := renderPromptCLI(context.Background(), path, options, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(rendered.Runtimes).To(HaveExactElements(HaveField("Mode", api.ModeCLI), HaveField("Mode", api.ModeCmux))) + }) + + It("deduplicates repeated CLI runtime selectors after final resolution", func() { + options.MultiModels, options.Fallback = []string{"agent:sonnet", "agent:sonnet", "cli:sonnet"}, nil + rendered, err := renderPromptCLI(context.Background(), path, options, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(rendered.Runtimes).To(HaveExactElements(HaveField("Mode", api.ModeAgent), HaveField("Mode", api.ModeCLI))) + }) + + It("leaves ordinary bare runtime selectors available for saved mode defaults", func() { + Expect(os.WriteFile(path, []byte("---\nmodel: sonnet\n---\nReview."), 0o600)).To(Succeed()) + options.MultiModels, options.Fallback = []string{"sonnet", "opus"}, nil + rendered, err := renderPromptCLI(context.Background(), path, options, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(rendered.Runtimes).To(HaveEach(HaveField("Mode", api.ModeAPI))) + Expect(rendered.variants[0].Resolution.Trace).To(ContainElement(SatisfyAll(HaveField("Name", "runtime 1"), HaveField("Spec.Model.Name", "sonnet")))) + Expect(rendered.variants[0].Resolution.Provenance["/mode"].Source.Kind).To(Equal(api.FieldSourceSaved)) + }) + + It("carries prepared attachment refs while preserving variant source and model provenance", func() { + options.Attach = []string{"diagram.png"} + rendered, err := renderPromptCLI(context.Background(), path, options, "", "") + Expect(err).NotTo(HaveOccurred()) + prepared := api.AttachmentRef{ID: api.AttachmentIDPrefix + strings.Repeat("a", 64), Filename: "diagram.png", MediaType: "image/png", Size: 512} + rendered.Input.Prompt.Attachments = []api.AttachmentRef{prepared} + variant := renderVariant(rendered, rendered.variants[1]) + Expect(variant.Input.Prompt.Attachments).To(Equal([]api.AttachmentRef{prepared})) + Expect(variant.Resolution.Spec).To(Equal(variant.Input)) + Expect(variant.Resolution.Provenance["/model"].Source.Name).To(Equal("runtime 2")) + Expect(variant.Resolution.Provenance["/prompt/attachments"].Source.Name).To(Equal("prompt flags")) + Expect(variant.Resolution.Provenance["/prompt/attachments"].NormalizedBy).To(HaveField("Name", "attachment preparation")) + Expect(rendered.variants[1].Request.Prompt.Attachments).To(Equal([]api.AttachmentRef{{Path: "diagram.png"}})) + Expect(rendered.variants[1].Resolution.Provenance["/prompt/attachments"].NormalizedBy).To(BeNil()) + }) +}) diff --git a/pkg/cli/prompt_runtimes_ginkgo_test.go b/pkg/cli/prompt_runtimes_ginkgo_test.go index 886473ef..4cba9d31 100644 --- a/pkg/cli/prompt_runtimes_ginkgo_test.go +++ b/pkg/cli/prompt_runtimes_ginkgo_test.go @@ -54,7 +54,7 @@ Review the screenshot. )) }) - It("serves the canonical prompt run request with the detail", func() { + It("serves authored model selectors and presence with the prompt detail", func() { record, err := filePromptRecord(path) Expect(err).NotTo(HaveOccurred()) @@ -65,18 +65,16 @@ Review the screenshot. Variables: map[string]any{}, Spec: &api.Spec{Model: api.Model{ Name: "gemini-3.5-flash", - Mode: api.ModeAPI, }}, Runtimes: []api.Model{ { - Name: "gemini-3.5-flash", - Mode: api.ModeAPI, - Effort: api.EffortHigh, + Name: "api:gemini-3.5-flash:high", }, { - Name: "claude-sonnet-5", - Mode: api.ModeAPI, - Effort: api.EffortMedium, + Name: "claude-sonnet-5", + Mode: api.ModeAPI, + Effort: api.EffortMedium, + Explicit: api.FieldPresence{"/model": true, "/mode": true, "/effort": true}, }, }, Chat: true, @@ -89,7 +87,7 @@ Review the screenshot. expectedPath, err := filepath.EvalSymlinks(path) Expect(err).NotTo(HaveOccurred()) - record, err := resolvePromptRecord(ctx, id) + record, err := resolvePromptRecord(ctx, promptRecordOptions{ID: id}) Expect(err).NotTo(HaveOccurred()) Expect(record.Path).To(Equal(expectedPath)) @@ -105,7 +103,7 @@ Review the screenshot. Expect(os.WriteFile(otherPath, []byte("Review this screenshot."), 0o600)).To(Succeed()) ctx := ContextWithPromptDirs(context.Background(), []string{filepath.Dir(path), otherDir}) - _, err := resolvePromptRecord(ctx, "compare") + _, err := resolvePromptRecord(ctx, promptRecordOptions{ID: "compare"}) Expect(err).To(MatchError(ContainSubstring("prompt name \"compare\" is ambiguous"))) }) diff --git a/pkg/cli/prompt_schema.go b/pkg/cli/prompt_schema.go index b346e104..36bb1893 100644 --- a/pkg/cli/prompt_schema.go +++ b/pkg/cli/prompt_schema.go @@ -11,6 +11,7 @@ import ( "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/runtimeprofiles" clickyrpc "github.com/flanksource/clicky/rpc" "github.com/spf13/cobra" ) @@ -83,7 +84,10 @@ func PromptSchemaDocument(ctx context.Context) (map[string]any, error) { if err != nil { return nil, err } - saved := loadSavedConfig() + saved, err := loadSavedConfig() + if err != nil { + return nil, err + } doc, err := buildPromptSchemaDocument(adapters, saved.Sandbox) if err != nil { return nil, err @@ -98,7 +102,7 @@ func PromptSchemaDocument(ctx context.Context) (map[string]any, error) { return nil, err } doc["sources"] = sources - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{Config: &saved}) if err != nil { return nil, err } diff --git a/pkg/cli/prompt_source.go b/pkg/cli/prompt_source.go index 3a9464cd..7cd7f6a4 100644 --- a/pkg/cli/prompt_source.go +++ b/pkg/cli/prompt_source.go @@ -9,9 +9,9 @@ import ( "strconv" "strings" - "github.com/flanksource/captain/pkg/ai" promptlib "github.com/flanksource/captain/pkg/ai/prompt" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" "github.com/flanksource/captain/pkg/claude" clickyrpc "github.com/flanksource/clicky/rpc" ) @@ -37,10 +37,18 @@ func readStdinIfCLI(ctx context.Context) string { // id) > --prompt/-p text > piped stdin. usedStdin reports whether stdin became // the prompt body (so the caller does not also expose it as the {{input}} // variable). -func loadPromptContent(ctx context.Context, id string, opts AIPromptOptions, stdin string) (content, source string, usedStdin bool, record promptRecord, err error) { +type promptContentOptions struct { + ID string + Prompt AIPromptOptions + Stdin string + Config *captainconfig.Config +} + +func loadPromptContent(ctx context.Context, options promptContentOptions) (content, source string, usedStdin bool, record promptRecord, err error) { + id, opts, stdin := options.ID, options.Prompt, options.Stdin switch { case strings.TrimSpace(id) != "": - record, err := resolvePromptRecord(ctx, id) + record, err := resolvePromptRecord(ctx, promptRecordOptions{ID: id, Config: options.Config}) if err != nil { return "", "", false, promptRecord{}, err } @@ -89,42 +97,32 @@ func promptVars(opts AIPromptOptions, varsJSON, stdin string, usedStdin bool) (m return data, nil } -// renderLoadedContent renders already-loaded .prompt content with vars, layers -// the selected runtime profile (--runtime-profile, else the frontmatter pin) -// beneath the frontmatter, overlays the CLI options, tags the source, and -// normalizes the context dir. The flags are not a spec layer yet: overlayCLI -// also folds saved defaults, API keys and the sandbox selection, none of which -// are Spec fields, so it stays the step above the resolved layers. -func renderLoadedContent(ctx context.Context, content, source string, vars map[string]any, opts AIPromptOptions) (ai.Request, ai.Config, api.ResolvedSpec, error) { - frontmatter, _, err := promptlib.Load(content).Render(vars, nil) +func renderLoadedLayers(ctx context.Context, content, source string, vars map[string]any, opts AIPromptOptions, saved captainconfig.Config) ([]api.SpecLayer, error) { + frontmatter, _, err := promptlib.Load(content).Render(promptlib.RenderOptions{Data: vars, Declared: true}) if err != nil { - return ai.Request{}, ai.Config{}, api.ResolvedSpec{}, err + return nil, err } frontmatter.Prompt.Source = source - resolved, err := resolveRenderLayers(ctx, source, content, frontmatter, PromptRenderRequest{RuntimeProfile: opts.RuntimeProfile}) + layers, err := renderLayers(ctx, source, content, frontmatter, PromptRenderRequest{RuntimeProfile: opts.RuntimeProfile}, saved) if err != nil { - return ai.Request{}, ai.Config{}, api.ResolvedSpec{}, err + return nil, err } - layered := resolved.Spec - foldSkillPolicies(&layered) - req, cfg, err := overlayCLI(layered, configFromResolved(layered), opts) + promptFlags, err := opts.promptSpec() if err != nil { - return ai.Request{}, ai.Config{}, api.ResolvedSpec{}, err + return nil, err } - req.Prompt.Source = source - cwd, err := os.Getwd() - if err != nil { - return ai.Request{}, ai.Config{}, api.ResolvedSpec{}, fmt.Errorf("get working directory: %w", err) + if len(promptFlags.Prompt.Attachments) > 0 { + promptFlags.Prompt.Attachments = append(append([]api.AttachmentRef(nil), frontmatter.Prompt.Attachments...), promptFlags.Prompt.Attachments...) } - if err := normalizePromptContextDir(&req, cwd); err != nil { - return ai.Request{}, ai.Config{}, api.ResolvedSpec{}, err + if len(promptFlags.Fields()) > 0 { + layers = append(layers, api.RequestSpecLayer("prompt flags", promptFlags)) } - return req, cfg, resolved, nil + return layers, nil } // actionFlagsToOptions reconstructs the typed AIPromptOptions from the entity // action's stringly-typed flag map (clicky CSV-encodes []string and "true"/"false" -// for bool), so the render/run core can reuse overlayCLI. +// for bool). Only changed flags are present, including explicit false values. func actionFlagsToOptions(f map[string]string) (AIPromptOptions, error) { var o AIPromptOptions o.Model = f["model"] @@ -171,6 +169,11 @@ func actionFlagsToOptions(f map[string]string) (AIPromptOptions, error) { o.MultiModels = flagSlice(f["multi-models"]) o.Timeout = f["timeout"] o.NoStream = flagBool(f["no-stream"]) + for flag, path := range runtimeFlagFields { + if _, present := f[flag]; present { + o.AIRuntimeOptions = o.WithExplicit(path) + } + } return o, nil } diff --git a/pkg/cli/prompt_source_test.go b/pkg/cli/prompt_source_test.go index 12e4a7b9..c6a1b254 100644 --- a/pkg/cli/prompt_source_test.go +++ b/pkg/cli/prompt_source_test.go @@ -77,22 +77,22 @@ func TestLoadPromptContent_Sources(t *testing.T) { } // filepath positional - content, _, usedStdin, _, err := loadPromptContent(ctx, path, AIPromptOptions{}, "") + content, _, usedStdin, _, err := loadPromptContent(ctx, promptContentOptions{ID: path}) if err != nil || content != body || usedStdin { t.Fatalf("file source: content=%q usedStdin=%v err=%v", content, usedStdin, err) } // --prompt/-p text (positional empty) - content, _, _, _, err = loadPromptContent(ctx, "", AIPromptOptions{AIRuntimeOptions: AIRuntimeOptions{}, Prompt: "inline body"}, "") + content, _, _, _, err = loadPromptContent(ctx, promptContentOptions{Prompt: AIPromptOptions{Prompt: "inline body"}}) if err != nil || content != "inline body" { t.Fatalf("prompt source: content=%q err=%v", content, err) } // stdin - content, _, usedStdin, _, err = loadPromptContent(ctx, "", AIPromptOptions{}, "piped body") + content, _, usedStdin, _, err = loadPromptContent(ctx, promptContentOptions{Stdin: "piped body"}) if err != nil || content != "piped body" || !usedStdin { t.Fatalf("stdin source: content=%q usedStdin=%v err=%v", content, usedStdin, err) } // nothing - if _, _, _, _, err = loadPromptContent(ctx, "", AIPromptOptions{}, ""); err == nil { + if _, _, _, _, err = loadPromptContent(ctx, promptContentOptions{}); err == nil { t.Error("expected error when no source is given") } } diff --git a/pkg/cli/prompt_sources.go b/pkg/cli/prompt_sources.go index 77952a49..dd293705 100644 --- a/pkg/cli/prompt_sources.go +++ b/pkg/cli/prompt_sources.go @@ -16,7 +16,11 @@ import ( clickyapi "github.com/flanksource/clicky/api" ) -func buildPromptSources(ctx context.Context) ([]promptSource, error) { +type promptSourceOptions struct { + Config *captainconfig.Config +} + +func buildPromptSources(ctx context.Context, options promptSourceOptions) ([]promptSource, error) { sources := []promptSource{{ Kind: "embedded", ID: "embedded", @@ -46,11 +50,15 @@ func buildPromptSources(ctx context.Context) ([]promptSource, error) { return nil } - cfg, exists, err := captainconfig.Load() - if err != nil { - return nil, err + cfg := options.Config + if cfg == nil { + loaded, err := loadSavedConfig() + if err != nil { + return nil, err + } + cfg = &loaded } - if exists { + if len(cfg.Prompts.Dirs) > 0 { configPath, err := captainconfig.Path() if err != nil { return nil, err @@ -237,7 +245,7 @@ type PromptSourceInfo struct { } func promptSourceInfos(ctx context.Context) ([]PromptSourceInfo, error) { - sources, err := buildPromptSources(ctx) + sources, err := buildPromptSources(ctx, promptSourceOptions{}) if err != nil { return nil, err } diff --git a/pkg/cli/prompt_spec.go b/pkg/cli/prompt_spec.go index d3f59572..7be4d423 100644 --- a/pkg/cli/prompt_spec.go +++ b/pkg/cli/prompt_spec.go @@ -9,60 +9,10 @@ import ( "strings" "time" - "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/api" clickyrpc "github.com/flanksource/clicky/rpc" ) -func applyPromptDefaults(req *ai.Request, cfg *ai.Config) error { - savedCfg := loadSavedConfig() - saved := savedCfg.AI - promptModel := req.Model - if promptModel.Name == "" { - promptModel.Name = cfg.Model.Name - } - if promptModel.ID == "" { - promptModel.ID = cfg.Model.ID - } - if promptModel.Mode == "" { - promptModel.Mode = cfg.Model.Mode - } - if promptModel.Provider == nil { - promptModel.Provider = cfg.Model.Provider - } - identity := selectModelIdentity( - api.Model{Name: promptModel.Name, ID: promptModel.ID, Mode: promptModel.Mode, Provider: promptModel.Provider}, - ) - req.Name, req.ID, req.Mode, req.Provider = identity.Name, identity.ID, identity.Mode, identity.Provider - if cfg.Model.Effort != api.EffortNone { - // An effort-qualified model selector (for example agent:sol:high) - // is model-local and intentionally overrides the request-wide flag/default. - req.Effort = cfg.Model.Effort - } else if req.Effort == "" { - req.Effort = cfg.Model.Effort - } - resolved, err := applyProviderDefaults(req.Model, saved) - if err != nil { - return err - } - req.Model = resolved - req.NoCache = req.NoCache || saved.NoCache - if req.Budget.MaxTokens == 0 { - req.Budget.MaxTokens = firstPositive(cfg.Budget.MaxTokens, saved.MaxTokens, 4096) - } - if req.Budget.Cost == 0 { - req.Budget.Cost = firstPositiveFloat(cfg.Budget.Cost, saved.BudgetUSD) - } - - cfg.Model = req.Model - cfg.Budget = req.Budget - cfg.NoCache = req.NoCache - if isZeroSchemaRepair(cfg.SchemaRepair) { - cfg.SchemaRepair = schemaRepairConfig(savedCfg.Prompts.SchemaRepair) - } - return nil -} - func dedupeStrings(in []string) []string { seen := map[string]bool{} var out []string @@ -122,6 +72,7 @@ func mergePromptActionFlags(req *PromptRenderRequest, flags map[string]string) e return fmt.Errorf("invalid --max-tokens %q: %w", v, err) } ensureRenderSpec(req).Budget.MaxTokens = n + *req.Spec = req.Spec.WithExplicit("/budget/maxTokens") } return nil } diff --git a/pkg/cli/provider_defaults.go b/pkg/cli/provider_defaults.go index f6bf41aa..53cabb79 100644 --- a/pkg/cli/provider_defaults.go +++ b/pkg/cli/provider_defaults.go @@ -1,7 +1,6 @@ package cli import ( - "fmt" "strings" "github.com/flanksource/captain/pkg/ai" @@ -48,40 +47,6 @@ func effectiveProviderDefaults(saved captainconfig.AIDefaults, provider *api.Mod return view, nil } -// savedProviderDefaults is effectiveProviderDefaults' run-path sibling: the same -// opt-out degradation, but over what the user actually configured rather than -// over registry-seeded values, and it never invents a model for an empty slot. -// -// The distinction is the point. effectiveProviderDefaults fills gaps from -// Provider.DefaultMode and the DefaultModelFor table, which is right for seeding -// `captain configure` and wrong for deciding what a run executes on — an unset -// field must survive as unset so ResolveForRun can refuse it. -func savedProviderDefaults(saved captainconfig.AIDefaults, provider *api.ModelProvider) (ProviderDefaultView, error) { - view, err := aiflags.SavedDefaults(saved, provider) - if err != nil { - return ProviderDefaultView{}, err - } - disabled := ai.Disabled() - mode := api.RuntimeMode(strings.TrimSpace(view.Mode)) - if mode != "" && disabled.Runtime(provider, mode) { - mode = firstEnabledMode(provider, mode) - } - model := strings.TrimSpace(view.Model) - if model != "" && disabled.Model(provider, mode, model) { - model = firstEnabledModel(provider, mode) - } - effort := api.Effort(strings.TrimSpace(view.Effort)) - if effort != api.EffortNone && disabled.Effort(effort) { - degraded, err := ai.ResolveModelEffort(provider, mode, model, effort) - if err != nil { - return ProviderDefaultView{}, err - } - effort = degraded - } - view.Mode, view.Model, view.Effort = string(mode), model, string(effort) - return view, nil -} - // firstEnabledMode replaces a disabled mode with another of the same provider // that is still enabled. When every one is off it returns the original so the // view still names what the user configured — the whoami disable card, not this @@ -106,73 +71,6 @@ func firstEnabledModel(provider *api.ModelProvider, mode api.RuntimeMode) string return defaultModelFor(provider, mode) } -func applyProviderDefaults(model api.Model, saved captainconfig.AIDefaults) (api.Model, error) { - var err error - if strings.TrimSpace(model.Name) != "" { - model, err = model.Expand() - if err != nil { - return api.Model{}, err - } - } - model, err = applyCandidateDefaults(model, saved, true) - if err != nil { - return api.Model{}, err - } - for i := range model.Fallbacks { - fallback := model.Fallbacks[i] - fallback, err = fallback.Expand() - if err != nil { - return api.Model{}, fmt.Errorf("fallback[%d]: %w", i, err) - } - fallback, err = applyCandidateDefaults(fallback, saved, false) - if err != nil { - return api.Model{}, fmt.Errorf("fallback[%d]: %w", i, err) - } - model.Fallbacks[i] = fallback - } - return model, nil -} - -func applyCandidateDefaults(model api.Model, saved captainconfig.AIDefaults, allowActive bool) (api.Model, error) { - provider := model.Provider - if provider == nil && strings.TrimSpace(model.Name) != "" { - inferred, err := api.ProviderFor(model.Name) - if err != nil { - return api.Model{}, err - } - provider = inferred - } - if provider == nil && allowActive { - provider, _ = api.ProviderByName(saved.ActiveProvider()) - } - if provider == nil { - return api.Model{}, fmt.Errorf("provider cannot be resolved for model %q", model.Name) - } - // Saved, not effective: this is the run path. Seeding from the registry's - // built-in tables here is how `captain ai prompt` kept silently defaulting - // after the flag path stopped. - defaults, err := savedProviderDefaults(saved, provider) - if err != nil { - return api.Model{}, err - } - if model.Mode == "" { - model.Mode = api.RuntimeMode(defaults.Mode) - } - if model.Provider == nil { - model.Provider = provider - } - if strings.TrimSpace(model.Name) == "" { - model.Name = defaults.Model - } - if model.Effort == api.EffortNone { - model.Effort = api.Effort(defaults.Effort) - } - if err := model.Effort.Validate(); err != nil { - return api.Model{}, err - } - return model, nil -} - // configurableProviders lists the provider families a user can configure // defaults for and toggle off. func configurableProviders() []*api.ModelProvider { return api.Providers() } diff --git a/pkg/cli/provider_defaults_test.go b/pkg/cli/provider_defaults_test.go index 92db1126..bd8662ad 100644 --- a/pkg/cli/provider_defaults_test.go +++ b/pkg/cli/provider_defaults_test.go @@ -15,10 +15,11 @@ func TestApplyProviderDefaultsFillsMissingFieldsAndFallbacks(t *testing.T) { "openai": {Mode: "agent", Model: "gpt-5.6-sol", ReasoningEffort: "medium"}, }, } - got, err := applyProviderDefaults(api.Model{Fallbacks: api.ModelList{{Name: "gpt-5.6"}}}, saved) + resolved, err := api.ResolveSpecLayers(api.ResolveSpecOptions{Layers: []api.SpecLayer{api.PromptSpecLayer("selection", api.Spec{Model: api.Model{Fallbacks: api.ModelList{{Name: "gpt-5.6-sol"}}}})}, Saved: &saved}) if err != nil { - t.Fatalf("applyProviderDefaults: %v", err) + t.Fatalf("ResolveSpecLayers: %v", err) } + got := resolved.Spec.Model if got.Mode != api.ModeAgent || got.Name != "claude-sonnet-5" || got.Effort != api.EffortHigh { t.Fatalf("primary = %+v", got) } @@ -32,10 +33,11 @@ func TestApplyProviderDefaultsPreservesExplicitFields(t *testing.T) { "openai": {Mode: "agent", Model: "gpt-5.6-sol", ReasoningEffort: "medium"}, }} want := api.Model{Name: "gpt-explicit", Mode: api.ModeAPI, Effort: api.EffortHigh} - got, err := applyProviderDefaults(want, saved) + resolved, err := api.ResolveSpecLayers(api.ResolveSpecOptions{Layers: []api.SpecLayer{api.PromptSpecLayer("selection", api.Spec{Model: want})}, Saved: &saved}) if err != nil { - t.Fatalf("applyProviderDefaults: %v", err) + t.Fatalf("ResolveSpecLayers: %v", err) } + got := resolved.Spec.Model if got.Name != want.Name || got.Mode != want.Mode || got.Effort != want.Effort { t.Fatalf("explicit model changed: got %+v want %+v", got, want) } diff --git a/pkg/cli/serve_chat_profile.go b/pkg/cli/serve_chat_profile.go index 93ca3bd4..defc61d1 100644 --- a/pkg/cli/serve_chat_profile.go +++ b/pkg/cli/serve_chat_profile.go @@ -9,7 +9,6 @@ import ( "github.com/flanksource/captain/pkg/aichat" "github.com/flanksource/captain/pkg/api" - "github.com/flanksource/captain/pkg/api/registry" "github.com/flanksource/captain/pkg/captainconfig" "github.com/flanksource/captain/pkg/runtimeprofiles" "github.com/flanksource/commons-db/shell" @@ -25,44 +24,53 @@ func captainChatProfileProvider(cwd string) aichat.RuntimeProfileProvider { base := api.SpecLayer{ Name: "captain serve", Scope: api.SpecLayerGlobal, Spec: api.Spec{ - Model: api.Model{Name: "sol", Mode: registry.ModeAgent}, Setup: &shell.Setup{Cwd: cwd}, }, } return aichat.RuntimeProfileProviderFunc(func(ctx context.Context, options ...aichat.RuntimeProfileOption) (aichat.RuntimeProfile, error) { + cfg, _, err := captainconfig.Load() + if err != nil { + return aichat.RuntimeProfile{}, fmt.Errorf("load chat settings: %w", err) + } + if err := cfg.AI.Validate(); err != nil { + return aichat.RuntimeProfile{}, fmt.Errorf("chat saved defaults: %w", err) + } selection := aichat.ApplyRuntimeProfileOptions(options...) - layers, err := chatProfileLayers(ctx, base, selection) + layers, err := chatProfileLayers(ctx, chatProfileLayerOptions{Base: base, Selection: selection, Config: cfg, Cwd: cwd}) if err != nil { return aichat.RuntimeProfile{}, err } - composed, err := api.ComposeSpecLayers(layers...) + composed, err := api.ComposeSpecLayers(api.ResolveSpecOptions{Layers: layers, Saved: &cfg.AI}) if err != nil { return aichat.RuntimeProfile{}, fmt.Errorf("resolve chat runtime profile: %w", err) } - return aichat.RuntimeProfile{System: captainChatSystemPrompt, Composed: composed}, nil + return aichat.RuntimeProfile{System: captainChatSystemPrompt, Composed: composed, Saved: &cfg.AI}, nil }) } +type chatProfileLayerOptions struct { + Base api.SpecLayer + Selection aichat.RuntimeProfileOptions + Config captainconfig.Config + Cwd string +} + // chatProfileLayers appends the selected profile's raw layers to the base. A // reference the caller supplied that resolves nowhere is the caller's error; a // configured default that fails stays a server error. -func chatProfileLayers(ctx context.Context, base api.SpecLayer, selection aichat.RuntimeProfileOptions) ([]api.SpecLayer, error) { - if err := api.ValidateSpecLayers(base); err != nil { +func chatProfileLayers(ctx context.Context, options chatProfileLayerOptions) ([]api.SpecLayer, error) { + if err := api.ValidateSpecLayers(options.Base); err != nil { return nil, fmt.Errorf("chat runtime profile base: %w", err) } - ref := strings.TrimSpace(selection.Ref) + ref := strings.TrimSpace(options.Selection.Ref) requested := ref != "" if !requested { - cfg, _, err := captainconfig.Load() - if err != nil { - return nil, fmt.Errorf("load chat runtime profile default: %w", err) - } - ref = strings.TrimSpace(cfg.Chat.RuntimeProfile) + ref = strings.TrimSpace(options.Config.Chat.RuntimeProfile) } if ref == "" { - return []api.SpecLayer{base}, nil + return []api.SpecLayer{options.Base}, nil } - catalog, err := buildRuntimeCatalog(ctx) + catalog, err := buildRuntimeCatalog(ctx, runtimeprofiles.DefaultCatalogOptions{Config: &options.Config, Cwd: options.Cwd}) if err != nil { return nil, fmt.Errorf("chat runtime profile %q: %w", ref, err) } @@ -74,5 +82,5 @@ func chatProfileLayers(ctx context.Context, base api.SpecLayer, selection aichat } return nil, fmt.Errorf("chat runtime profile %q: %w", ref, err) } - return append([]api.SpecLayer{base}, resolution.Layers...), nil + return append([]api.SpecLayer{options.Base}, resolution.Layers...), nil } diff --git a/pkg/cli/serve_chat_profile_ginkgo_test.go b/pkg/cli/serve_chat_profile_ginkgo_test.go index 8fff788d..9abff6cf 100644 --- a/pkg/cli/serve_chat_profile_ginkgo_test.go +++ b/pkg/cli/serve_chat_profile_ginkgo_test.go @@ -56,14 +56,37 @@ var _ = Describe("chat runtime profile provider", func() { Expect(profile.Composed.Spec.Cwd()).To(Equal(cwd), "the base layer survives") }) - It("serves the base layer alone when nothing selects a profile", func() { - f := chatCatalog() + It("serves a model-free base layer when nothing selects a profile or saved model", func() { + f, _, _ := newRuntimeCatalogFixture() profile, err := captainChatProfileProvider(GinkgoT().TempDir()).RuntimeProfile(f.ctx) Expect(err).NotTo(HaveOccurred()) Expect(profile.Composed.Trace).To(HaveExactElements(HaveField("Name", "captain serve"))) - Expect(profile.Composed.Spec.Model.Name).To(Equal("sol")) + Expect(profile.Composed.Spec.Model.Name).To(BeEmpty()) + }) + + It("takes model settings from one saved snapshot while keeping them out of authored layers", func() { + f, _, _ := newRuntimeCatalogFixture() + Expect(captainconfig.Save(captainconfig.Config{AI: captainconfig.AIDefaults{DefaultModel: "agent:sonnet:high", BudgetUSD: 4}})).To(Succeed()) + profile, err := captainChatProfileProvider(GinkgoT().TempDir()).RuntimeProfile(f.ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(profile.Saved).NotTo(BeNil()) + Expect(profile.Saved.DefaultModel).To(Equal("agent:sonnet:high")) + Expect(profile.Composed.Spec.Name).To(Equal("sonnet")) + Expect(profile.Composed.Spec.Effort).To(Equal(api.EffortHigh)) + Expect(profile.Composed.Spec.Budget.Cost).To(Equal(float64(4))) + Expect(profile.Composed.Trace[0].Spec.Name).To(BeEmpty()) + Expect(profile.Composed.Provenance["/model"].Source.Key).To(Equal("ai.defaultModel")) + Expect(captainconfig.Save(captainconfig.Config{AI: captainconfig.AIDefaults{DefaultModel: "api:sol"}})).To(Succeed()) + Expect(profile.Saved.DefaultModel).To(Equal("agent:sonnet:high")) + }) + + It("rejects malformed saved settings even when an explicit profile supplies a valid model", func() { + f := chatCatalog() + Expect(captainconfig.Save(captainconfig.Config{AI: captainconfig.AIDefaults{Temperature: 3}})).To(Succeed()) + _, err := captainChatProfileProvider(GinkgoT().TempDir()).RuntimeProfile(f.ctx, aichat.WithRuntimeProfileRef("plan")) + Expect(err).To(MatchError(ContainSubstring("ai.temperature"))) }) It("applies the configured chat default when the request names no profile", func() { diff --git a/pkg/cli/serve_disabled.go b/pkg/cli/serve_disabled.go index bb27f5eb..cd4e91c6 100644 --- a/pkg/cli/serve_disabled.go +++ b/pkg/cli/serve_disabled.go @@ -144,7 +144,14 @@ func validateDisabledSelections(saved captainconfig.AIDefaults, selections capta if len(set.EnabledEfforts()) == 0 { return fmt.Errorf("cannot disable every reasoning effort; leave at least one enabled") } - if len(selections.Providers) >= len(configurableProviders()) { + providers := configurableProviders() + disabledProviders := 0 + for _, provider := range providers { + if set.Provider(provider) { + disabledProviders++ + } + } + if disabledProviders == len(providers) { return fmt.Errorf("cannot disable every provider; leave at least one enabled") } // A flagless run resolves through ActiveProvider, so it is not enough that diff --git a/pkg/cli/serve_provider_defaults.go b/pkg/cli/serve_provider_defaults.go index 73c63f80..df6946b8 100644 --- a/pkg/cli/serve_provider_defaults.go +++ b/pkg/cli/serve_provider_defaults.go @@ -61,11 +61,10 @@ func handleProviderDefaults(w http.ResponseWriter, r *http.Request) { } active := false if err := captainconfig.Update(func(cfg *captainconfig.Config) error { - if cfg.AI.Providers == nil { - cfg.AI.Providers = map[string]captainconfig.ProviderDefaults{} - } - cfg.AI.Providers[provider.Name] = captainconfig.ProviderDefaults{ + if err := cfg.AI.SetProvider(provider, captainconfig.ProviderDefaults{ Mode: defaults.Mode, Model: defaults.Model, ReasoningEffort: defaults.Effort, + }); err != nil { + return err } active = cfg.AI.ActiveProvider() == provider.Name return nil diff --git a/pkg/cli/serve_sandbox.go b/pkg/cli/serve_sandbox.go index 905e1e31..e7da9679 100644 --- a/pkg/cli/serve_sandbox.go +++ b/pkg/cli/serve_sandbox.go @@ -107,7 +107,12 @@ func sandboxBackendParam(r *http.Request) string { // drift apart. func handleSandboxCatalog() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - writeServeJSON(w, http.StatusOK, buildSandboxCatalog(loadSavedConfig().Sandbox)) + saved, err := loadSavedConfig() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + writeServeJSON(w, http.StatusOK, buildSandboxCatalog(saved.Sandbox)) }) } diff --git a/pkg/cli/verify.go b/pkg/cli/verify.go index 430cdce8..34b97ca4 100644 --- a/pkg/cli/verify.go +++ b/pkg/cli/verify.go @@ -161,10 +161,12 @@ func (o VerifyOptions) judgeProvider() (ai.Provider, api.Model, error) { if len(o.Prompts) == 0 { return nil, api.Model{}, nil } - cfg, err := o.ToConfig() + resolved, err := resolveInvocation(AIRuntimeOptions{AIProviderOptions: o.AIProviderOptions}, nil) if err != nil { return nil, api.Model{}, err } + cfg := resolved.Config + logRuntimeWarnings(resolved.Resolution.Warnings) if cfg.Model.Name == "" { return nil, api.Model{}, fmt.Errorf("--prompt needs a model to judge with: pass --model or run 'captain configure'") } diff --git a/pkg/cli/webapp/dist/index.html b/pkg/cli/webapp/dist/index.html index 8a0d0706..3af11315 100644 --- a/pkg/cli/webapp/dist/index.html +++ b/pkg/cli/webapp/dist/index.html @@ -4,8 +4,8 @@ Captain - - + +
From ed8c594ee5eb3b14bf3ad54c3b08bc9bd7fd15c3 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 12:15:24 +0300 Subject: [PATCH 13/22] refactor(ai): forward complete prompt specs through agent execution Preserve the complete declared prompt configuration when executing named requests, including model, budget, memory, permissions, setup, and workflow settings. Extract AI runtime command registration into a dedicated module and update prompt rendering for the new options API. BREAKING CHANGE: PromptRequest no longer exposes individual prompt fields; callers must provide an api.Spec via Spec. --- cmd/captain/ai.go | 56 +++++++++++++++++ cmd/captain/main.go | 31 +--------- pkg/ai/agent.go | 33 ++-------- pkg/ai/agent/verify/llmjudge.go | 2 +- pkg/ai/agent/verify/registry.go | 2 +- pkg/ai/agent_spec_ginkgo_test.go | 97 ++++++++++++++++++++++++++++++ pkg/ai/agent_test.go | 22 +++---- pkg/ai/middleware/schema_repair.go | 4 +- 8 files changed, 173 insertions(+), 74 deletions(-) create mode 100644 cmd/captain/ai.go create mode 100644 pkg/ai/agent_spec_ginkgo_test.go diff --git a/cmd/captain/ai.go b/cmd/captain/ai.go new file mode 100644 index 00000000..b1178f14 --- /dev/null +++ b/cmd/captain/ai.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + + "github.com/flanksource/captain/pkg/cli" + "github.com/flanksource/clicky" + "github.com/spf13/cobra" +) + +func registerAIRuntimeCommands(rootCmd *cobra.Command) { + aiCmd := &cobra.Command{ + Use: "ai", + Short: "AI provider commands", + Long: "AI provider commands.\n\n" + + "Logging: increase application verbosity with -v/-vv or --log-level=debug. " + + "HTTP calls to the provider APIs are logged on the same ladder (with credentials " + + "redacted): failed requests are logged by default, -v adds an access line per " + + "request, -vv adds headers and query params, -vvv request bodies, -vvvv response " + + "bodies. Use -Plog.level.http= to raise only HTTP logging, or " + + "-Phttp.har= to write the exchanges to a HAR archive instead.", + } + rootCmd.AddCommand(aiCmd) + aiCmd.AddCommand(cli.NewCommandAlias(cli.CommandAliasOptions{ + Name: "prompt", + Short: "Alias for captain prompt run", + Root: rootCmd, + Target: []string{"prompt", "run"}, + })) + var agentCmd *cobra.Command + agentCmd = clicky.AddNamedCommand("agent", aiCmd, cli.AIAgentOptions{}, func(opts cli.AIAgentOptions) (any, error) { + opts.AIRuntimeOptions = opts.WithChangedFlags(agentCmd.Flags()) + return cli.RunAIAgent(opts) + }) + agentCmd.Short = "Run an iterative agent with verifiers, worktree, and commit" + clicky.AddNamedCommand("models", aiCmd, cli.AIModelsOptions{}, cli.RunAIModels) + var testCmd *cobra.Command + testCmd = clicky.AddNamedCommand("test", aiCmd, cli.AITestOptions{}, func(opts cli.AITestOptions) (any, error) { + opts.AIProviderOptions = (cli.AIRuntimeOptions{AIProviderOptions: opts.AIProviderOptions}).WithChangedFlags(testCmd.Flags()).AIProviderOptions + return cli.RunAITest(opts) + }) + clicky.AddNamedCommand("fixture", aiCmd, cli.AIFixtureOptions{}, cli.RunAIFixture).Short = "Run a YAML fixture across multiple Claude configurations" + clicky.AddNamedCommandWithContext("mock", aiCmd, cli.AIMockOptions{}, cli.RunAIMock).Short = "Serve scripted OpenAI/Anthropic replies so agent runs spend no tokens" + + // Local-only: --command is run through `sh -c` against a caller-chosen --cwd, + // so published as REST or MCP it would be unauthenticated remote execution. + var verifyCmd *cobra.Command + verifyCmd = clicky.AddNamedCommandWithContext("verify", rootCmd, cli.VerifyOptions{}, func(ctx context.Context, opts cli.VerifyOptions) (any, error) { + opts.AIProviderOptions = (cli.AIRuntimeOptions{AIProviderOptions: opts.AIProviderOptions}).WithChangedFlags(verifyCmd.Flags()).AIProviderOptions + return cli.RunVerify(ctx, opts) + }) + verifyCmd.Short = "Run a workflow's verification checks and report the verdict" + verifyCmd.Long = "Run the checks an api.Workflow declares — shell commands, LLM-judge prompts, and a fixture document handed to the configured fixture runner — against a working tree, and print each check's report. Exits non-zero when any check fails or cannot reach a verdict." + clicky.MarkLocalOnly(verifyCmd) + +} diff --git a/cmd/captain/main.go b/cmd/captain/main.go index 4e0b8dc5..4eabc434 100644 --- a/cmd/captain/main.go +++ b/cmd/captain/main.go @@ -251,29 +251,7 @@ func newRootCommand() *cobra.Command { clicky.AddNamedCommandWithContext("list", tokenCmd, cli.TokenListOptions{}, cli.RunTokenList).Short = "List tokens and what each can reach" clicky.AddNamedCommandWithContext("revoke", tokenCmd, cli.TokenRevokeOptions{}, cli.RunTokenRevoke).Short = "Refuse a token from now on" - aiCmd := &cobra.Command{ - Use: "ai", - Short: "AI provider commands", - Long: "AI provider commands.\n\n" + - "Logging: increase application verbosity with -v/-vv or --log-level=debug. " + - "HTTP calls to the provider APIs are logged on the same ladder (with credentials " + - "redacted): failed requests are logged by default, -v adds an access line per " + - "request, -vv adds headers and query params, -vvv request bodies, -vvvv response " + - "bodies. Use -Plog.level.http= to raise only HTTP logging, or " + - "-Phttp.har= to write the exchanges to a HAR archive instead.", - } - rootCmd.AddCommand(aiCmd) - aiCmd.AddCommand(cli.NewCommandAlias(cli.CommandAliasOptions{ - Name: "prompt", - Short: "Alias for captain prompt run", - Root: rootCmd, - Target: []string{"prompt", "run"}, - })) - clicky.AddNamedCommand("agent", aiCmd, cli.AIAgentOptions{}, cli.RunAIAgent).Short = "Run an iterative agent with verifiers, worktree, and commit" - clicky.AddNamedCommand("models", aiCmd, cli.AIModelsOptions{}, cli.RunAIModels) - clicky.AddNamedCommand("test", aiCmd, cli.AITestOptions{}, cli.RunAITest) - clicky.AddNamedCommand("fixture", aiCmd, cli.AIFixtureOptions{}, cli.RunAIFixture).Short = "Run a YAML fixture across multiple Claude configurations" - clicky.AddNamedCommandWithContext("mock", aiCmd, cli.AIMockOptions{}, cli.RunAIMock).Short = "Serve scripted OpenAI/Anthropic replies so agent runs spend no tokens" + registerAIRuntimeCommands(rootCmd) whoamiCmd := clicky.AddNamedCommand("whoami", rootCmd, cli.WhoamiOptions{}, cli.RunWhoami) whoamiCmd.Short = "List agent adapters, auth methods, and available models" @@ -294,13 +272,6 @@ func newRootCommand() *cobra.Command { rootCmd.AddCommand(attachmentsCmd) clicky.AddNamedCommand("gc", attachmentsCmd, cli.AttachmentsGCOptions{}, cli.RunAttachmentsGC).Short = "Remove old unreferenced attachments" - // Local-only: --command is run through `sh -c` against a caller-chosen --cwd, - // so published as REST or MCP it would be unauthenticated remote execution. - verifyCmd := clicky.AddNamedCommandWithContext("verify", rootCmd, cli.VerifyOptions{}, cli.RunVerify) - verifyCmd.Short = "Run a workflow's verification checks and report the verdict" - verifyCmd.Long = "Run the checks an api.Workflow declares — shell commands, LLM-judge prompts, and a fixture document handed to the configured fixture runner — against a working tree, and print each check's report. Exits non-zero when any check fails or cannot reach a verdict." - clicky.MarkLocalOnly(verifyCmd) - hookCmd := &cobra.Command{Use: "hook", Short: "Claude Code hook commands"} rootCmd.AddCommand(hookCmd) bashCheckCmd := &cobra.Command{Use: "bash-check", Short: "Scan bash command for violations (PreToolUse hook)", RunE: func(cmd *cobra.Command, args []string) error { diff --git a/pkg/ai/agent.go b/pkg/ai/agent.go index 8c296ff5..39464a03 100644 --- a/pkg/ai/agent.go +++ b/pkg/ai/agent.go @@ -2,7 +2,6 @@ package ai import ( "context" - "encoding/json" "fmt" "sync" "time" @@ -24,27 +23,10 @@ type Agent struct { costs Costs } -// PromptRequest is a single named prompt. Field names/types mirror the former -// clicky/ai.PromptRequest so consumers need only change the import path. +// PromptRequest carries the complete specification of one named model call. type PromptRequest struct { - Name string `json:"name"` - Prompt string `json:"prompt"` - SystemPrompt string `json:"system_prompt,omitempty"` - Context map[string]string `json:"context,omitempty"` - StructuredOutput any `json:"structured_output,omitempty"` - // SchemaJSON is a pre-built JSON Schema (e.g. from a .prompt frontmatter - // output block) forwarded verbatim to ai.Request.Prompt.SchemaJSON. Prefer it - // over StructuredOutput when the schema is declared in the prompt file rather - // than a Go type; the two are mutually exclusive. - SchemaJSON json.RawMessage `json:"schema_json,omitempty"` - // SchemaStrictness forwards api.Prompt.SchemaStrictness — the policy for a - // response that fails schema validation (warning/error/retry). "" (default) - // skips validation. - SchemaStrictness api.SchemaStrictness `json:"schema_strictness,omitempty"` - // Source identifies the prompt template (e.g. the .prompt filename) for - // diagnostics; forwarded to ai.Request.Source and printed by the logging - // middleware. - Source string `json:"source,omitempty"` + Name string `json:"name"` + Spec api.Spec `json:"spec"` } // PromptResponse is the result of one PromptRequest. @@ -95,14 +77,7 @@ func (a *Agent) ExecutePrompt(ctx context.Context, req PromptRequest) (*PromptRe return &PromptResponse{Request: req, Model: a.cfg.Model.Name, Error: err.Error()}, err } } - resp, err := a.provider.Execute(ctx, Request{Prompt: api.Prompt{ - User: req.Prompt, - System: req.SystemPrompt, - Source: req.Source, - Schema: req.StructuredOutput, - SchemaJSON: req.SchemaJSON, - SchemaStrictness: req.SchemaStrictness, - }}) + resp, err := a.provider.Execute(ctx, req.Spec) if err != nil { return &PromptResponse{Request: req, Model: a.cfg.Model.Name, Error: err.Error(), Duration: time.Since(start)}, err } diff --git a/pkg/ai/agent/verify/llmjudge.go b/pkg/ai/agent/verify/llmjudge.go index 9f83522b..a230dfa8 100644 --- a/pkg/ai/agent/verify/llmjudge.go +++ b/pkg/ai/agent/verify/llmjudge.go @@ -38,7 +38,7 @@ func (j *LLMJudgeVerifier) Verify(ctx context.Context, cwd string, changed []str } out := &judgeVerdict{} - req, _, err := j.Prompt.Render(data, out) + req, _, err := j.Prompt.Render(prompt.RenderOptions{Data: data, Output: out}) if err != nil { return Verdict{}, err } diff --git a/pkg/ai/agent/verify/registry.go b/pkg/ai/agent/verify/registry.go index 43000934..68208840 100644 --- a/pkg/ai/agent/verify/registry.go +++ b/pkg/ai/agent/verify/registry.go @@ -226,7 +226,7 @@ func promptFactory(_ context.Context, spec api.Verify, opts Options) ([]*Plugin, // prompt declaring a relocating sandbox is a validation error, never a silent // fallback). func rejectJudgeOverrides(path string, tmpl *prompt.Template, model string) error { - probe, _, err := tmpl.Render(map[string]any{"cwd": "", "changed": []string{}}, nil) + probe, _, err := tmpl.Render(prompt.RenderOptions{Data: map[string]any{"cwd": "", "changed": []string{}}}) if err != nil { return fmt.Errorf("verify prompt %q: %w", path, err) } diff --git a/pkg/ai/agent_spec_ginkgo_test.go b/pkg/ai/agent_spec_ginkgo_test.go new file mode 100644 index 00000000..38d7b87a --- /dev/null +++ b/pkg/ai/agent_spec_ginkgo_test.go @@ -0,0 +1,97 @@ +package ai_test + +import ( + "context" + "encoding/json" + "sync" + + captainai "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/commons-db/shell" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type namedSpecProvider struct { + mu sync.Mutex + requests []api.Spec +} + +func (p *namedSpecProvider) Execute(_ context.Context, spec api.Spec) (*api.Response, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.requests = append(p.requests, spec) + return &api.Response{Text: spec.Prompt.User, Model: spec.Model.Name, CostUSD: 0.02}, nil +} + +func (p *namedSpecProvider) GetModel() string { return "provider-model" } +func (p *namedSpecProvider) GetRuntime() api.Runtime { + return api.RuntimeOf(api.OpenAI, api.ModeAPI) +} + +var _ = Describe("Named prompt Spec transport", func() { + It("forwards the complete declared spec without borrowing provider config", func() { + temperature := 0.0 + spec := api.Spec{ + Explicit: api.FieldPresence{"/noCache": true, "/permissions": true}, + Model: api.Model{Name: "gpt-5.4", Mode: api.ModeAPI, Temperature: &temperature}, + Prompt: api.Prompt{ + User: "Review the change", System: "Explain findings", Source: "review.prompt", + SchemaJSON: json.RawMessage(`{"type":"object"}`), SchemaStrictness: "error", + }, + Budget: api.Budget{MaxTokens: 512, MaxTurns: 3, Timeout: "1m"}, + Memory: api.Memory{Skills: []string{"review"}, SkipUser: true}, + Permissions: api.Permissions{Mode: "plan"}, + Setup: &shell.Setup{Cwd: "/workspace/review", Env: []string{"REVIEW_MODE=focused"}}, + Sandbox: &api.SandboxRef{Mode: "off"}, + SessionID: "review-session", + CLIArgs: map[string]any{"resume": true}, + Workflow: &api.Workflow{Verify: &api.Verify{Commands: []string{"make test"}}}, + } + provider := &namedSpecProvider{} + agent := captainai.NewAgentWithProvider(provider, captainai.Config{ + Model: api.Model{Name: "config-model"}, Budget: api.Budget{MaxTokens: 2048}, + }) + request := captainai.PromptRequest{Name: "review", Spec: spec} + + response, err := agent.ExecutePrompt(context.Background(), request) + + Expect(err).NotTo(HaveOccurred()) + Expect(provider.requests).To(Equal([]api.Spec{spec})) + Expect(response.Request).To(Equal(request)) + Expect(response.Result).To(Equal("Review the change")) + Expect(agent.TotalCost()).To(BeNumerically("~", 0.02)) + }) + + It("keeps the native structured-output target attached to the prompt", func() { + target := &struct{ Summary string }{} + spec := api.Spec{Prompt: api.Prompt{User: "Summarize", Schema: target}} + provider := &namedSpecProvider{} + agent := captainai.NewAgentWithProvider(provider, captainai.Config{}) + + _, err := agent.ExecutePrompt(context.Background(), captainai.PromptRequest{Name: "summary", Spec: spec}) + + Expect(err).NotTo(HaveOccurred()) + Expect(provider.requests).To(Equal([]api.Spec{spec})) + Expect(provider.requests[0].Prompt.Schema).To(BeIdenticalTo(target)) + }) + + It("keeps each batch item's spec and response name independent", func() { + requests := []captainai.PromptRequest{ + {Name: "first", Spec: api.Spec{Prompt: api.Prompt{User: "First"}, SessionID: "first-session"}}, + {Name: "second", Spec: api.Spec{Prompt: api.Prompt{User: "Second"}, Memory: api.Memory{Skills: []string{"second"}}}}, + } + provider := &namedSpecProvider{} + agent := captainai.NewAgentWithProvider(provider, captainai.Config{MaxConcurrent: 2}) + + responses, err := agent.ExecuteBatch(context.Background(), requests) + + Expect(err).NotTo(HaveOccurred()) + Expect(provider.requests).To(ConsistOf(requests[0].Spec, requests[1].Spec)) + Expect(responses["first"].Request).To(Equal(requests[0])) + Expect(responses["second"].Request).To(Equal(requests[1])) + Expect(responses["first"].Result).To(Equal("First")) + Expect(responses["second"].Result).To(Equal("Second")) + Expect(agent.GetCosts()).To(HaveLen(2)) + }) +}) diff --git a/pkg/ai/agent_test.go b/pkg/ai/agent_test.go index 74b54e8a..05170e3b 100644 --- a/pkg/ai/agent_test.go +++ b/pkg/ai/agent_test.go @@ -45,7 +45,7 @@ func TestAgent_ExecutePromptCarriesTerminalOutcome(t *testing.T) { outcome := &TerminalOutcome{Kind: TerminalOutcomePlan, Plan: &TerminalPlan{Content: "1. Inspect"}} a := NewAgentWithProvider(&mockProvider{model: "m", outcome: outcome}, Config{Model: api.Model{Name: "m"}}) - resp, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "plan", Prompt: "plan this"}) + resp, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "plan", Spec: api.Spec{Prompt: api.Prompt{User: "plan this"}}}) require.NoError(t, err) assert.Same(t, outcome, resp.TerminalOutcome) } @@ -53,7 +53,7 @@ func TestAgent_ExecutePromptCarriesTerminalOutcome(t *testing.T) { func TestAgent_ExecutePromptAccruesCost(t *testing.T) { a := NewAgentWithProvider(&mockProvider{model: "test-model", text: "out"}, Config{Model: api.Model{Name: "test-model"}}) - resp, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p1", Prompt: "hi"}) + resp, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p1", Spec: api.Spec{Prompt: api.Prompt{User: "hi"}}}) require.NoError(t, err) assert.Equal(t, "out:hi", resp.Result) assert.Equal(t, "test-model", resp.Model) @@ -74,14 +74,14 @@ func TestAgent_ExecutePromptEnforcesBudget(t *testing.T) { }) // First two calls fit under the $0.10 budget (spend 0 → 0.08 → 0.16). - _, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p1", Prompt: "a"}) + _, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p1", Spec: api.Spec{Prompt: api.Prompt{User: "a"}}}) require.NoError(t, err) - _, err = a.ExecutePrompt(context.Background(), PromptRequest{Name: "p2", Prompt: "b"}) + _, err = a.ExecutePrompt(context.Background(), PromptRequest{Name: "p2", Spec: api.Spec{Prompt: api.Prompt{User: "b"}}}) require.NoError(t, err, "second call still under budget at pre-flight (spent 0.08)") // Third call trips the pre-flight (spent 0.16 ≥ 0.10) and must not execute. callsBefore := mp.calls - resp, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p3", Prompt: "c"}) + resp, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p3", Spec: api.Spec{Prompt: api.Prompt{User: "c"}}}) require.Error(t, err) assert.ErrorIs(t, err, ErrBudgetExceeded) assert.Equal(t, callsBefore, mp.calls, "provider must not be invoked once budget is exceeded") @@ -94,15 +94,15 @@ func TestAgent_ExecutePromptForwardsSchemaJSON(t *testing.T) { a := NewAgentWithProvider(mp, Config{Model: api.Model{Name: "m"}}) schema := json.RawMessage(`{"type":"object","required":["pass"]}`) - _, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p", Prompt: "hi", SchemaJSON: schema}) + _, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p", Spec: api.Spec{Prompt: api.Prompt{User: "hi", SchemaJSON: schema}}}) require.NoError(t, err) assert.JSONEq(t, string(schema), string(mp.lastReq.Prompt.SchemaJSON), - "PromptRequest.SchemaJSON must be forwarded to the provider request") + "PromptRequest.Spec.Prompt.SchemaJSON must be forwarded to the provider request") } func TestAgent_ExecutePromptError(t *testing.T) { a := NewAgentWithProvider(&mockProvider{model: "m", err: fmt.Errorf("boom")}, Config{Model: api.Model{Name: "m"}}) - resp, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p", Prompt: "x"}) + resp, err := a.ExecutePrompt(context.Background(), PromptRequest{Name: "p", Spec: api.Spec{Prompt: api.Prompt{User: "x"}}}) require.Error(t, err) assert.False(t, resp.IsOK()) assert.Contains(t, resp.Error, "boom") @@ -112,9 +112,9 @@ func TestAgent_ExecutePromptError(t *testing.T) { func TestAgent_ExecuteBatchKeyedByName(t *testing.T) { a := NewAgentWithProvider(&mockProvider{model: "m", text: "r"}, Config{Model: api.Model{Name: "m"}, MaxConcurrent: 2}) reqs := []PromptRequest{ - {Name: "a", Prompt: "1"}, - {Name: "b", Prompt: "2"}, - {Name: "c", Prompt: "3"}, + {Name: "a", Spec: api.Spec{Prompt: api.Prompt{User: "1"}}}, + {Name: "b", Spec: api.Spec{Prompt: api.Prompt{User: "2"}}}, + {Name: "c", Spec: api.Spec{Prompt: api.Prompt{User: "3"}}}, } got, err := a.ExecuteBatch(context.Background(), reqs) require.NoError(t, err) diff --git a/pkg/ai/middleware/schema_repair.go b/pkg/ai/middleware/schema_repair.go index e9da6115..7fd308f0 100644 --- a/pkg/ai/middleware/schema_repair.go +++ b/pkg/ai/middleware/schema_repair.go @@ -41,7 +41,7 @@ func (v *validatingProvider) repairRequest(parent ai.Request, schema json.RawMes if err != nil { return ai.Request{}, ai.Config{}, false, err } - renderedReq, renderedCfg, err := tmpl.Render(map[string]any{ + renderedReq, renderedCfg, err := tmpl.Render(promptlib.RenderOptions{Data: map[string]any{ "attempt": attempt, "mode": string(v.provider.GetRuntime().Mode), "provider": v.provider.GetRuntime().Provider, @@ -50,7 +50,7 @@ func (v *validatingProvider) repairRequest(parent ai.Request, schema json.RawMes "previousResponse": responseJSON(prev), "schema": string(schema), "validationErrors": verrs, - }, nil) + }}) if err != nil { return ai.Request{}, ai.Config{}, false, fmt.Errorf("render schema repair prompt: %w", err) } From 3322c656489bff241e21332a3ee57053f1ef9cb4 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 13:29:57 +0300 Subject: [PATCH 14/22] feat(api): enforce permission constraints Claude-Session-Id: 01a072be-045a-7831-a018-238a83d1a88c --- pkg/api/runtime_constraints.go | 25 ++ pkg/api/runtime_permission_constraints.go | 317 ++++++++++++++++++ ...time_permission_constraints_ginkgo_test.go | 129 +++++++ pkg/api/spec_layers.go | 19 +- pkg/cli/permissions_matrix_test.go | 2 +- .../preflight_constraints_ginkgo_test.go | 11 + 6 files changed, 499 insertions(+), 4 deletions(-) create mode 100644 pkg/api/runtime_permission_constraints.go create mode 100644 pkg/api/runtime_permission_constraints_ginkgo_test.go diff --git a/pkg/api/runtime_constraints.go b/pkg/api/runtime_constraints.go index 6bd6f2fa..0e33e647 100644 --- a/pkg/api/runtime_constraints.go +++ b/pkg/api/runtime_constraints.go @@ -14,6 +14,7 @@ const ( RuntimeConstraintInputTokens RuntimeConstraintViolation = "input_tokens" RuntimeConstraintTokenQuota RuntimeConstraintViolation = "token_quota" RuntimeConstraintCostQuota RuntimeConstraintViolation = "cost_quota" + RuntimeConstraintPermission RuntimeConstraintViolation = "permission" RuntimeConstraintInvalidInput RuntimeConstraintViolation = "invalid_input" ) @@ -24,6 +25,12 @@ type RuntimeConstraintError struct { Quota UsageQuota EstimatedInputTokens int MaxInputTokens int + Field string + Actual string + Constraint string + ActualLayer string + ConstraintLayer string + ConstraintSource SpecLayerSource } func (e *RuntimeConstraintError) Error() string { @@ -38,6 +45,18 @@ func (e *RuntimeConstraintError) Error() string { return fmt.Sprintf("%s quota %q from layer %q is exhausted: %d tokens used of %d", e.Quota.Scope, e.Quota.Name, e.Quota.Layer, e.Quota.TokensUsed, e.Quota.TokenLimit) case RuntimeConstraintCostQuota: return fmt.Sprintf("%s quota %q from layer %q is exhausted: $%.4f used of $%.4f", e.Quota.Scope, e.Quota.Name, e.Quota.Layer, e.Quota.CostUsedUSD, e.Quota.CostLimitUSD) + case RuntimeConstraintPermission: + actual := e.Actual + if actual == "" { + actual = "" + } + if e.ActualLayer != "" && e.ConstraintLayer != "" { + if e.ConstraintSource != "" { + return fmt.Sprintf("spec layer %q %s %q exceeds constraint %q from %s layer %q", e.ActualLayer, e.Field, actual, e.Constraint, e.ConstraintSource, e.ConstraintLayer) + } + return fmt.Sprintf("spec layer %q %s %q exceeds constraint %q from layer %q", e.ActualLayer, e.Field, actual, e.Constraint, e.ConstraintLayer) + } + return fmt.Sprintf("%s %q exceeds effective permission constraint %q", e.Field, actual, e.Constraint) case RuntimeConstraintInvalidInput: return fmt.Sprintf("estimated input tokens must be non-negative, got %d", e.EstimatedInputTokens) default: @@ -51,6 +70,9 @@ func ValidateRuntimeConstraints(resolved ResolvedSpec, model Model, estimatedInp if err := resolved.Constraints.Validate(); err != nil { return err } + if err := validatePermissionConstraints(resolved.Spec, resolved.Constraints.Permissions, resolved.Trace); err != nil { + return err + } if err := validateBudgetLimits(resolved.Spec.Budget, resolved.Constraints.Limits.Budget); err != nil { return err } @@ -89,6 +111,9 @@ func (constraints RuntimeConstraints) Validate() error { if _, err := strictRunLimits(RunLimits{}, constraints.Limits); err != nil { return fmt.Errorf("runtime constraints: %w", err) } + if err := constraints.Permissions.Validate(); err != nil { + return fmt.Errorf("runtime constraints: %w", err) + } for _, selector := range constraints.Models { if strings.TrimSpace(selector) == "" { return fmt.Errorf("runtime constraints model catalog contains an empty selector") diff --git a/pkg/api/runtime_permission_constraints.go b/pkg/api/runtime_permission_constraints.go new file mode 100644 index 00000000..9dc43c26 --- /dev/null +++ b/pkg/api/runtime_permission_constraints.go @@ -0,0 +1,317 @@ +package api + +import ( + "fmt" + "slices" + "strings" +) + +// PermissionConstraints are monotonic permission ceilings. Later layers may +// narrow them but cannot restore denied tools or skills, exceed the maximum +// posture, or select a sandbox outside the intersected allowlist. +type PermissionConstraints struct { + Mode PermissionMode `json:"mode,omitempty" yaml:"mode,omitempty"` + Tools Tools `json:"tools,omitempty" yaml:"tools,omitempty"` + Skills ResourcePolicies `json:"skills,omitempty" yaml:"skills,omitempty"` + SandboxModes []SandboxKind `json:"sandboxModes,omitempty" yaml:"sandboxModes,omitempty"` +} + +// PermissionConstraintsForSpec projects the restrictive part of an authored +// spec. An explicit unsandboxed boundary permits a later layer to add isolation; +// named external backends remain limited to the two backend sandbox kinds. +func PermissionConstraintsForSpec(spec Spec) PermissionConstraints { + constraints := PermissionConstraints{Mode: spec.Permissions.Mode} + for tool, policy := range spec.Permissions.Tools { + if policy == ToolPolicyDeny { + constraints.Tools = putTool(constraints.Tools, tool, policy) + } + } + for skill, mode := range spec.Permissions.Skills { + if mode == ResourceDisabled { + constraints.Skills = putResource(constraints.Skills, skill, mode) + } + } + if spec.Sandbox == nil { + return constraints + } + switch spec.Sandbox.Mode { + case SandboxOff: + constraints.SandboxModes = AllSandboxModes() + case "": + if spec.Sandbox.Backend != "" { + constraints.SandboxModes = []SandboxKind{SandboxDocker, SandboxGitAgent} + } + default: + constraints.SandboxModes = []SandboxKind{spec.Sandbox.Mode} + } + return constraints +} + +// ConstrainSpecLayerPermissions adds the restrictions authored by a layer to +// any permission constraints the embedding host already attached to it. +func ConstrainSpecLayerPermissions(layer SpecLayer) (SpecLayer, error) { + constraints, err := strictPermissionConstraints(layer.Constraints.Permissions, PermissionConstraintsForSpec(layer.Spec)) + if err != nil { + return SpecLayer{}, fmt.Errorf("spec layer %q permission constraints: %w", layer.Name, err) + } + layer.Constraints.Permissions = constraints + return layer, nil +} + +// Validate rejects values that do not describe a restrictive floor. +func (constraints PermissionConstraints) Validate() error { + if !constraints.Mode.Valid() { + return fmt.Errorf("invalid permission mode %q", constraints.Mode) + } + if _, ok := constraints.Tools[""]; ok { + return fmt.Errorf("permission constraint tool name is required") + } + for _, tool := range sortedKeys(constraints.Tools) { + if strings.TrimSpace(tool) == "" { + return fmt.Errorf("permission constraint tool name is required") + } + if constraints.Tools[tool] != ToolPolicyDeny { + return fmt.Errorf("permission constraint for tool %q must be deny, got %q", tool, constraints.Tools[tool]) + } + } + if _, ok := constraints.Skills[""]; ok { + return fmt.Errorf("permission constraint skill name is required") + } + for _, skill := range sortedKeys(constraints.Skills) { + if strings.TrimSpace(skill) == "" { + return fmt.Errorf("permission constraint skill name is required") + } + if constraints.Skills[skill] != ResourceDisabled { + return fmt.Errorf("permission constraint for skill %q must be disabled, got %q", skill, constraints.Skills[skill]) + } + } + seen := map[SandboxKind]bool{} + for _, mode := range constraints.SandboxModes { + if _, ok := ParseSandboxKind(string(mode)); !ok || mode == "" { + return fmt.Errorf("invalid sandbox mode %q in permission constraints", mode) + } + if seen[mode] { + return fmt.Errorf("permission constraints repeat sandbox mode %q", mode) + } + seen[mode] = true + } + return nil +} + +func strictPermissionConstraints(current, next PermissionConstraints) (PermissionConstraints, error) { + if err := current.Validate(); err != nil { + return PermissionConstraints{}, err + } + if err := next.Validate(); err != nil { + return PermissionConstraints{}, err + } + mode, err := strictPermissionMode(current.Mode, next.Mode) + if err != nil { + return PermissionConstraints{}, err + } + out := current.clone() + out.Mode = mode + for tool, policy := range next.Tools { + out.Tools = putTool(out.Tools, tool, policy) + } + for skill, resourceMode := range next.Skills { + out.Skills = putResource(out.Skills, skill, resourceMode) + } + out.SandboxModes = intersectSandboxModes(current.SandboxModes, next.SandboxModes) + if len(current.SandboxModes) > 0 && len(next.SandboxModes) > 0 && len(out.SandboxModes) == 0 { + return PermissionConstraints{}, fmt.Errorf("allowed sandbox modes have an empty intersection") + } + return out, nil +} + +func strictPermissionMode(current, next PermissionMode) (PermissionMode, error) { + if current == "" { + return next, nil + } + if next == "" || current == next { + return current, nil + } + currentRank, currentOrdered := permissionModeRank(current) + nextRank, nextOrdered := permissionModeRank(next) + if !currentOrdered || !nextOrdered { + return "", fmt.Errorf("permission modes %q and %q are incomparable constraints", current, next) + } + if nextRank < currentRank { + return next, nil + } + return current, nil +} + +func permissionModeRank(mode PermissionMode) (int, bool) { + switch mode { + case PermissionPlan: + return 0, true + case PermissionDefault: + return 1, true + case PermissionAcceptEdits: + return 2, true + case PermissionBypass: + return 3, true + default: + return 0, false + } +} + +func validatePermissionConstraints(spec Spec, constraints PermissionConstraints, trace []SpecLayer) error { + if err := constraints.Validate(); err != nil { + return fmt.Errorf("runtime constraints: %w", err) + } + if constraints.Mode != "" && !permissionModeWithin(spec.Permissions.Mode, constraints.Mode) { + return permissionConstraintError(trace, "permissions.mode", string(spec.Permissions.Mode), string(constraints.Mode)) + } + for _, tool := range sortedKeys(constraints.Tools) { + if spec.Permissions.Tools[tool] != ToolPolicyDeny { + return permissionConstraintError(trace, "permissions.tools."+tool, string(spec.Permissions.Tools[tool]), string(ToolPolicyDeny)) + } + } + for _, skill := range sortedKeys(constraints.Skills) { + if spec.Permissions.Skills[skill] != ResourceDisabled { + return permissionConstraintError(trace, "permissions.skills."+skill, string(spec.Permissions.Skills[skill]), string(ResourceDisabled)) + } + } + if len(constraints.SandboxModes) > 0 { + actual := SandboxOff + if spec.Sandbox != nil && spec.Sandbox.Mode != "" { + actual = spec.Sandbox.Mode + } + if !slices.Contains(constraints.SandboxModes, actual) { + allowed := make([]string, len(constraints.SandboxModes)) + for i, mode := range constraints.SandboxModes { + allowed[i] = string(mode) + } + return permissionConstraintError(trace, "sandbox.mode", string(actual), strings.Join(allowed, ",")) + } + } + return nil +} + +func permissionModeWithin(actual, limit PermissionMode) bool { + if actual == limit { + return true + } + actualRank, actualOrdered := permissionModeRank(actual) + limitRank, limitOrdered := permissionModeRank(limit) + return actualOrdered && limitOrdered && actualRank <= limitRank +} + +func permissionConstraintError(trace []SpecLayer, field, actual, constraint string) error { + err := &RuntimeConstraintError{Violation: RuntimeConstraintPermission, Field: field, Actual: actual, Constraint: constraint} + if layer, ok := lastSpecFieldLayer(trace, field); ok { + err.ActualLayer = layer.Name + } + if layer, ok := lastConstraintFieldLayer(trace, field, actual); ok { + err.ConstraintLayer = layer.Name + err.ConstraintSource = layer.Source + } + return err +} + +func lastSpecFieldLayer(trace []SpecLayer, field string) (SpecLayer, bool) { + for i := len(trace) - 1; i >= 0; i-- { + layer := trace[i] + if specFieldPresent(layer.Spec, field) { + return layer, true + } + switch { + case field == "permissions.mode" && layer.Spec.Permissions.Mode != "": + return layer, true + case strings.HasPrefix(field, "permissions.tools."): + if _, ok := layer.Spec.Permissions.Tools[strings.TrimPrefix(field, "permissions.tools.")]; ok { + return layer, true + } + case strings.HasPrefix(field, "permissions.skills."): + if _, ok := layer.Spec.Permissions.Skills[strings.TrimPrefix(field, "permissions.skills.")]; ok { + return layer, true + } + case field == "sandbox.mode" && layer.Spec.Sandbox != nil: + return layer, true + } + } + return SpecLayer{}, false +} + +func specFieldPresent(spec Spec, field string) bool { + path := "/" + strings.ReplaceAll(field, ".", "/") + for present := range spec.Fields() { + if present == path || strings.HasPrefix(path, present+"/") { + return true + } + } + return false +} + +func lastConstraintFieldLayer(trace []SpecLayer, field, actual string) (SpecLayer, bool) { + for i := len(trace) - 1; i >= 0; i-- { + layer := trace[i] + constraints := layer.Constraints.Permissions + switch { + case field == "permissions.mode" && constraints.Mode != "" && !permissionModeWithin(PermissionMode(actual), constraints.Mode): + return layer, true + case strings.HasPrefix(field, "permissions.tools.") && constraints.Tools[strings.TrimPrefix(field, "permissions.tools.")] == ToolPolicyDeny: + return layer, true + case strings.HasPrefix(field, "permissions.skills.") && constraints.Skills[strings.TrimPrefix(field, "permissions.skills.")] == ResourceDisabled: + return layer, true + case field == "sandbox.mode" && len(constraints.SandboxModes) > 0 && !slices.Contains(constraints.SandboxModes, SandboxKind(actual)): + return layer, true + } + } + return SpecLayer{}, false +} + +func intersectSandboxModes(current, next []SandboxKind) []SandboxKind { + if len(current) == 0 { + return append([]SandboxKind(nil), next...) + } + if len(next) == 0 { + return append([]SandboxKind(nil), current...) + } + out := make([]SandboxKind, 0, len(current)) + for _, mode := range current { + if slices.Contains(next, mode) { + out = append(out, mode) + } + } + return out +} + +func (constraints PermissionConstraints) clone() PermissionConstraints { + return PermissionConstraints{ + Mode: constraints.Mode, Tools: putTools(nil, constraints.Tools), Skills: putResources(nil, constraints.Skills), + SandboxModes: append([]SandboxKind(nil), constraints.SandboxModes...), + } +} + +func putTools(target, source Tools) Tools { + for name, policy := range source { + target = putTool(target, name, policy) + } + return target +} + +func putTool(target Tools, name string, policy ToolPolicy) Tools { + if target == nil { + target = Tools{} + } + target[name] = policy + return target +} + +func putResources(target, source ResourcePolicies) ResourcePolicies { + for name, mode := range source { + target = putResource(target, name, mode) + } + return target +} + +func putResource(target ResourcePolicies, name string, mode ResourceMode) ResourcePolicies { + if target == nil { + target = ResourcePolicies{} + } + target[name] = mode + return target +} diff --git a/pkg/api/runtime_permission_constraints_ginkgo_test.go b/pkg/api/runtime_permission_constraints_ginkgo_test.go new file mode 100644 index 00000000..64076fd8 --- /dev/null +++ b/pkg/api/runtime_permission_constraints_ginkgo_test.go @@ -0,0 +1,129 @@ +package api + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Runtime permission constraints", func() { + DescribeTable("rejects a later permission widening with both layer names", + func(constraints PermissionConstraints, request Spec, field string) { + _, err := ComposeSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{ + {Name: "project policy", Source: SpecLayerSourcePreset, Scope: SpecLayerGlobal, Constraints: RuntimeConstraints{Permissions: constraints}}, + RequestSpecLayer("request", request), + }}) + Expect(err).To(MatchError(And(ContainSubstring(field), ContainSubstring("project policy"), ContainSubstring("request")))) + }, + Entry("tool deny becomes allow", + PermissionConstraints{Tools: Tools{"Bash": ToolPolicyDeny}}, + Spec{Permissions: Permissions{Tools: Tools{"Bash": ToolPolicyAllow}}}, "permissions.tools.Bash"), + Entry("plan becomes acceptEdits", + PermissionConstraints{Mode: PermissionPlan}, + Spec{Permissions: Permissions{Mode: PermissionAcceptEdits}}, "permissions.mode"), + Entry("plan becomes incomparable auto", + PermissionConstraints{Mode: PermissionPlan}, + Spec{Permissions: Permissions{Mode: PermissionAuto}}, "permissions.mode"), + Entry("dontAsk becomes incomparable default", + PermissionConstraints{Mode: PermissionDontAsk}, + Spec{Permissions: Permissions{Mode: PermissionDefault}}, "permissions.mode"), + Entry("disabled skill becomes enabled", + PermissionConstraints{Skills: ResourcePolicies{"review": ResourceDisabled}}, + Spec{Permissions: Permissions{Skills: ResourcePolicies{"review": ResourceEnabled}}}, "permissions.skills.review"), + Entry("sandbox leaves its allowlist", + PermissionConstraints{SandboxModes: []SandboxKind{SandboxNative}}, + Spec{Sandbox: &SandboxRef{Mode: SandboxOff}}, "sandbox.mode"), + ) + + It("accepts permission and sandbox narrowing", func() { + resolved, err := ComposeSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{ + { + Name: "project policy", Scope: SpecLayerGlobal, + Constraints: RuntimeConstraints{Permissions: PermissionConstraints{ + Mode: PermissionAcceptEdits, Tools: Tools{"Bash": ToolPolicyDeny}, + Skills: ResourcePolicies{"review": ResourceDisabled}, SandboxModes: []SandboxKind{SandboxNative, SandboxDocker}, + }}, + }, + RequestSpecLayer("request", Spec{ + Permissions: Permissions{Mode: PermissionPlan, Tools: Tools{"Bash": ToolPolicyDeny}, Skills: ResourcePolicies{"review": ResourceDisabled}}, + Sandbox: &SandboxRef{Mode: SandboxDocker}, + }), + }}) + + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Permissions.Mode).To(Equal(PermissionPlan)) + Expect(resolved.Spec.Sandbox.Mode).To(Equal(SandboxDocker)) + }) + + It("intersects independently authored ceilings", func() { + resolved, err := ComposeSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{ + { + Name: "organization", Scope: SpecLayerGlobal, + Constraints: RuntimeConstraints{Permissions: PermissionConstraints{ + Mode: PermissionAcceptEdits, Tools: Tools{"Bash": ToolPolicyDeny}, SandboxModes: []SandboxKind{SandboxNative, SandboxDocker}, + }}, + }, + { + Name: "project", Scope: SpecLayerContext, + Constraints: RuntimeConstraints{Permissions: PermissionConstraints{ + Mode: PermissionDefault, Tools: Tools{"Write": ToolPolicyDeny}, SandboxModes: []SandboxKind{SandboxDocker, SandboxGitAgent}, + }}, + }, + RequestSpecLayer("request", Spec{ + Permissions: Permissions{Mode: PermissionPlan, Tools: Tools{"Bash": ToolPolicyDeny, "Write": ToolPolicyDeny}}, + Sandbox: &SandboxRef{Mode: SandboxDocker}, + }), + }}) + + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Constraints.Permissions).To(Equal(PermissionConstraints{ + Mode: PermissionDefault, Tools: Tools{"Bash": ToolPolicyDeny, "Write": ToolPolicyDeny}, SandboxModes: []SandboxKind{SandboxDocker}, + })) + }) + + DescribeTable("rejects invalid permission constraints", + func(constraints PermissionConstraints, message string) { + _, err := ComposeSpecLayers(ResolveSpecOptions{Layers: []SpecLayer{{ + Name: "project policy", Scope: SpecLayerGlobal, Constraints: RuntimeConstraints{Permissions: constraints}, + }}}) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("tool constraint is not deny", PermissionConstraints{Tools: Tools{"Bash": ToolPolicyAllow}}, "must be deny"), + Entry("tool constraint name is empty", PermissionConstraints{Tools: Tools{"": ToolPolicyDeny}}, "tool name is required"), + Entry("skill constraint is not disabled", PermissionConstraints{Skills: ResourcePolicies{"review": ResourceEnabled}}, "must be disabled"), + Entry("skill constraint name is empty", PermissionConstraints{Skills: ResourcePolicies{"": ResourceDisabled}}, "skill name is required"), + Entry("sandbox mode is invalid", PermissionConstraints{SandboxModes: []SandboxKind{"invalid"}}, "sandbox mode"), + Entry("mode is invalid", PermissionConstraints{Mode: "invalid"}, "permission mode"), + ) + + It("projects only restrictive fields from a spec", func() { + constraints := PermissionConstraintsForSpec(Spec{ + Permissions: Permissions{ + Mode: PermissionAcceptEdits, + Tools: Tools{"Bash": ToolPolicyDeny, "Read": ToolPolicyAllow}, + Skills: ResourcePolicies{"review": ResourceDisabled, "author": ResourceEnabled}, + }, + Sandbox: &SandboxRef{Mode: SandboxOff}, + }) + + Expect(constraints).To(Equal(PermissionConstraints{ + Mode: PermissionAcceptEdits, Tools: Tools{"Bash": ToolPolicyDeny}, Skills: ResourcePolicies{"review": ResourceDisabled}, + SandboxModes: AllSandboxModes(), + })) + }) + + It("adds authored restrictions to a layer without discarding its existing ceiling", func() { + layer, err := ConstrainSpecLayerPermissions(SpecLayer{ + Name: "profile", Scope: SpecLayerSurface, + Spec: Spec{Permissions: Permissions{Mode: PermissionPlan, Skills: ResourcePolicies{"review": ResourceDisabled}}}, + Constraints: RuntimeConstraints{Permissions: PermissionConstraints{ + Tools: Tools{"Bash": ToolPolicyDeny}, SandboxModes: []SandboxKind{SandboxNative}, + }}, + }) + + Expect(err).NotTo(HaveOccurred()) + Expect(layer.Constraints.Permissions).To(Equal(PermissionConstraints{ + Mode: PermissionPlan, Tools: Tools{"Bash": ToolPolicyDeny}, Skills: ResourcePolicies{"review": ResourceDisabled}, + SandboxModes: []SandboxKind{SandboxNative}, + })) + }) +}) diff --git a/pkg/api/spec_layers.go b/pkg/api/spec_layers.go index d95afa24..0fe4a277 100644 --- a/pkg/api/spec_layers.go +++ b/pkg/api/spec_layers.go @@ -48,9 +48,10 @@ type UsageQuota struct { // RuntimeConstraints restrict values a later Spec layer may select. type RuntimeConstraints struct { - Models []string `json:"models,omitempty" yaml:"models,omitempty"` - Limits RunLimits `json:"limits,omitempty" yaml:"limits,omitempty"` - Quotas []UsageQuota `json:"quotas,omitempty" yaml:"quotas,omitempty"` + Models []string `json:"models,omitempty" yaml:"models,omitempty"` + Limits RunLimits `json:"limits,omitempty" yaml:"limits,omitempty"` + Quotas []UsageQuota `json:"quotas,omitempty" yaml:"quotas,omitempty"` + Permissions PermissionConstraints `json:"permissions,omitempty" yaml:"permissions,omitempty"` } // SpecLayer is one named source of runtime defaults and constraints. @@ -114,6 +115,11 @@ func ComposeSpecLayers(options ResolveSpecOptions) (ComposedSpec, error) { return ComposedSpec{}, fmt.Errorf("spec layer %q limits: %w", layer.Name, err) } resolved.Constraints.Limits = limits + permissions, err := strictPermissionConstraints(resolved.Constraints.Permissions, layer.Constraints.Permissions) + if err != nil { + return ComposedSpec{}, fmt.Errorf("spec layer %q permission constraints: %w", layer.Name, err) + } + resolved.Constraints.Permissions = permissions limitSources.record(layer, limits.Budget) for _, quota := range layer.Constraints.Quotas { quota.Name = strings.TrimSpace(quota.Name) @@ -138,6 +144,9 @@ func ComposeSpecLayers(options ResolveSpecOptions) (ComposedSpec, error) { } resolved.Spec.Budget = budget resolved.recordLimits(limitSources) + if err := validatePermissionConstraints(resolved.Spec, resolved.Constraints.Permissions, resolved.Trace); err != nil { + return ComposedSpec{}, err + } return resolved, nil } @@ -168,6 +177,9 @@ func validateSpecLayer(layer SpecLayer) error { if _, err := strictRunLimits(RunLimits{}, layer.Constraints.Limits); err != nil { return fmt.Errorf("spec layer %q limits: %w", layer.Name, err) } + if err := layer.Constraints.Permissions.Validate(); err != nil { + return fmt.Errorf("spec layer %q permission constraints: %w", layer.Name, err) + } seenModels := map[string]bool{} for _, model := range layer.Constraints.Models { model = strings.TrimSpace(model) @@ -343,5 +355,6 @@ func cloneSpecLayer(layer SpecLayer) SpecLayer { layer.Spec = Spec{}.Merge(layer.Spec) layer.Constraints.Models = append([]string(nil), layer.Constraints.Models...) layer.Constraints.Quotas = append([]UsageQuota(nil), layer.Constraints.Quotas...) + layer.Constraints.Permissions = layer.Constraints.Permissions.clone() return layer } diff --git a/pkg/cli/permissions_matrix_test.go b/pkg/cli/permissions_matrix_test.go index 75d96b9c..b83ddc34 100644 --- a/pkg/cli/permissions_matrix_test.go +++ b/pkg/cli/permissions_matrix_test.go @@ -99,7 +99,7 @@ func TestPermissionsMatrixCells(t *testing.T) { {"anthropic agent does not", agent, "mcp disabled", "anthropic agent", "✗"}, {"no runtime enables MCP per server", agent, "mcp enabled", "openai agent", "✗"}, {"only anthropic cli loads skills", agent, "skills enabled", "anthropic cli", "✓"}, - {"nothing unloads a skill", agent, "skills disabled", "anthropic cli", "✗"}, + {"captain omits a disabled skill before dispatch", agent, "skills disabled", "anthropic cli", "✓"}, {"plugins are inert", agent, "plugins enabled", "anthropic cli", "✗"}, } for _, tc := range cases { diff --git a/pkg/promptrun/preflight_constraints_ginkgo_test.go b/pkg/promptrun/preflight_constraints_ginkgo_test.go index e985783f..eb56e08b 100644 --- a/pkg/promptrun/preflight_constraints_ginkgo_test.go +++ b/pkg/promptrun/preflight_constraints_ginkgo_test.go @@ -49,6 +49,17 @@ var _ = Describe("promptrun.Preflight constraints and runtimes", func() { Expect(in.Request.Budget.Timeout).To(Equal("30s")) }) + It("returns the same permission-constraint refusal from preview and Run", func() { + in.Request.Permissions.Tools = api.Tools{"Bash": api.ToolPolicyAllow} + in.Constraints.Permissions = api.PermissionConstraints{Tools: api.Tools{"Bash": api.ToolPolicyDeny}} + + _, previewErr := promptrun.Preflight(in) + _, runErr := promptrun.Run(context.Background(), in) + + Expect(previewErr).To(MatchError(ContainSubstring("permissions.tools.Bash"))) + Expect(runErr).To(MatchError(previewErr.Error())) + }) + DescribeTable("checks constraints against the actual run", func(mutate func(*promptrun.Input), message string) { mutate(&in) From 3cbbf9c256089e67acaf34d0dd5573d01dac6122 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 14:01:27 +0300 Subject: [PATCH 15/22] fix(webapp): accept runtime catalog metadata Claude-Session-Id: 01a072be-045a-7831-a018-238a83d1a88c --- pkg/cli/webapp/src/runtimeProfilesApi.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/cli/webapp/src/runtimeProfilesApi.ts b/pkg/cli/webapp/src/runtimeProfilesApi.ts index 9c225f85..89d3db3c 100644 --- a/pkg/cli/webapp/src/runtimeProfilesApi.ts +++ b/pkg/cli/webapp/src/runtimeProfilesApi.ts @@ -29,6 +29,10 @@ type StoredRecord = { updatedAt: string; }; +type EntityRecordMetadata = Partial & { + _id?: string; +}; + export type StoredRuntimePreset = RuntimePreset & StoredRecord; export type StoredRuntimeProfile = RuntimeProfile & StoredRecord; @@ -65,6 +69,11 @@ export type RuntimeProfileResolution = { resolved: ResolvedRuntimeSpec; }; +type RuntimeProfileResolveInput = { + profile: RuntimeProfile & EntityRecordMetadata; + presets: Array; +}; + /** The catalog's database source id, the default create target. */ export const RUNTIME_DB_TARGET = "db"; export const RUNTIME_PRESETS_URL = "/api/v1/runtime-preset"; @@ -133,7 +142,7 @@ export async function fetchRuntimeProfileResolution( * decodes its contract strictly, so only the contract fields are sent. */ export async function resolveRuntimeProfile( - request: RuntimeProfileResolveRequest, + request: RuntimeProfileResolveInput, signal?: AbortSignal, ): Promise { const body: RuntimeProfileResolveRequest = { From 327e0748c9a3ad4cee39ff1d06689132ad82b496 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 19:36:11 +0300 Subject: [PATCH 16/22] build(webapp): pin clicky UI commit Claude-Session-Id: 01a072be-045a-7831-a018-238a83d1a88c --- pkg/cli/webapp/package.json | 2 +- pkg/cli/webapp/pnpm-lock.yaml | 11 ++++++----- pkg/cli/webapp/pnpm-workspace.yaml | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/cli/webapp/package.json b/pkg/cli/webapp/package.json index d8b5bcfc..a1869e16 100644 --- a/pkg/cli/webapp/package.json +++ b/pkg/cli/webapp/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "@ai-sdk/react": "^3.0.201", - "@flanksource/clicky-ui": "0.3.34", + "@flanksource/clicky-ui": "github:flanksource/clicky-ui#e045560b6af4d717250ae66ad48b08c07772d0cd&path:packages/ui", "@shikijs/langs": "^1.24.0", "@shikijs/themes": "^1.24.0", "@shikijs/transformers": "^1.24.0", diff --git a/pkg/cli/webapp/pnpm-lock.yaml b/pkg/cli/webapp/pnpm-lock.yaml index 2099e3d8..7943aa45 100644 --- a/pkg/cli/webapp/pnpm-lock.yaml +++ b/pkg/cli/webapp/pnpm-lock.yaml @@ -16,8 +16,8 @@ importers: specifier: ^3.0.201 version: 3.0.216(react@18.3.1)(zod@4.4.3) '@flanksource/clicky-ui': - specifier: 0.3.34 - version: 0.3.34(@ai-sdk/react@3.0.216(react@18.3.1)(zod@4.4.3))(@babel/core@7.29.7)(@babel/template@7.29.7)(@shikijs/langs@1.29.2)(@shikijs/themes@1.29.2)(@shikijs/transformers@1.29.2)(@types/react@19.2.17)(ai@6.0.214(zod@4.4.3))(marked@15.0.12)(monaco-editor@0.48.0)(react-dom@18.3.1(react@18.3.1))(react-rnd@10.5.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)(recharts@3.10.1(@types/react@19.2.17)(react-dom@18.3.1(react@18.3.1))(react-is@17.0.2)(react@18.3.1)(redux@5.0.1))(shiki@1.29.2)(streamdown@2.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(tailwindcss@4.3.1) + specifier: github:flanksource/clicky-ui#e045560b6af4d717250ae66ad48b08c07772d0cd&path:packages/ui + version: https://codeload.github.com/flanksource/clicky-ui/tar.gz/e045560b6af4d717250ae66ad48b08c07772d0cd#path:packages/ui(@ai-sdk/react@3.0.216(react@18.3.1)(zod@4.4.3))(@babel/core@7.29.7)(@babel/template@7.29.7)(@shikijs/langs@1.29.2)(@shikijs/themes@1.29.2)(@shikijs/transformers@1.29.2)(@types/react@19.2.17)(ai@6.0.214(zod@4.4.3))(marked@15.0.12)(monaco-editor@0.48.0)(react-dom@18.3.1(react@18.3.1))(react-rnd@10.5.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)(recharts@3.10.1(@types/react@19.2.17)(react-dom@18.3.1(react@18.3.1))(react-is@17.0.2)(react@18.3.1)(redux@5.0.1))(shiki@1.29.2)(streamdown@2.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(tailwindcss@4.3.1) '@shikijs/langs': specifier: ^1.24.0 version: 1.29.2 @@ -426,8 +426,9 @@ packages: cpu: [x64] os: [win32] - '@flanksource/clicky-ui@0.3.34': - resolution: {integrity: sha512-5o1e8K2ZWbS9bWjsE0sd/rm0UkxZMbQ4/d0PRNQMHwGh3a55XG3oAkBhlBYxv2uUfhBj1zR1F6f195rDnKLvRg==} + '@flanksource/clicky-ui@https://codeload.github.com/flanksource/clicky-ui/tar.gz/e045560b6af4d717250ae66ad48b08c07772d0cd#path:packages/ui': + resolution: {gitHosted: true, integrity: sha512-US4YoTqe1mZPnqbdJv70EHo4CVXabShfHaC4OsLbd8M4TphCTr7z4rDwUo0+vs0SLoKm9s2dI4M/sk1rCwWmnw==, path: packages/ui, tarball: https://codeload.github.com/flanksource/clicky-ui/tar.gz/e045560b6af4d717250ae66ad48b08c07772d0cd} + version: 0.3.34 peerDependencies: '@ai-sdk/react': ^3.0.0 '@mdxeditor/editor': ^4.0.4 @@ -2771,7 +2772,7 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true - '@flanksource/clicky-ui@0.3.34(@ai-sdk/react@3.0.216(react@18.3.1)(zod@4.4.3))(@babel/core@7.29.7)(@babel/template@7.29.7)(@shikijs/langs@1.29.2)(@shikijs/themes@1.29.2)(@shikijs/transformers@1.29.2)(@types/react@19.2.17)(ai@6.0.214(zod@4.4.3))(marked@15.0.12)(monaco-editor@0.48.0)(react-dom@18.3.1(react@18.3.1))(react-rnd@10.5.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)(recharts@3.10.1(@types/react@19.2.17)(react-dom@18.3.1(react@18.3.1))(react-is@17.0.2)(react@18.3.1)(redux@5.0.1))(shiki@1.29.2)(streamdown@2.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(tailwindcss@4.3.1)': + '@flanksource/clicky-ui@https://codeload.github.com/flanksource/clicky-ui/tar.gz/e045560b6af4d717250ae66ad48b08c07772d0cd#path:packages/ui(@ai-sdk/react@3.0.216(react@18.3.1)(zod@4.4.3))(@babel/core@7.29.7)(@babel/template@7.29.7)(@shikijs/langs@1.29.2)(@shikijs/themes@1.29.2)(@shikijs/transformers@1.29.2)(@types/react@19.2.17)(ai@6.0.214(zod@4.4.3))(marked@15.0.12)(monaco-editor@0.48.0)(react-dom@18.3.1(react@18.3.1))(react-rnd@10.5.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)(recharts@3.10.1(@types/react@19.2.17)(react-dom@18.3.1(react@18.3.1))(react-is@17.0.2)(react@18.3.1)(redux@5.0.1))(shiki@1.29.2)(streamdown@2.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(tailwindcss@4.3.1)': dependencies: '@codemirror/autocomplete': 6.20.3 '@codemirror/commands': 6.11.0 diff --git a/pkg/cli/webapp/pnpm-workspace.yaml b/pkg/cli/webapp/pnpm-workspace.yaml index 6d3e09ac..5e741706 100644 --- a/pkg/cli/webapp/pnpm-workspace.yaml +++ b/pkg/cli/webapp/pnpm-workspace.yaml @@ -18,6 +18,7 @@ trustPolicyExclude: - semver@6.3.1 allowBuilds: + '@flanksource/clicky-ui@https://codeload.github.com/flanksource/clicky-ui/tar.gz/e045560b6af4d717250ae66ad48b08c07772d0cd#path:packages/ui': true esbuild: true overrides: '@types/react': 19.2.17 From 96cf01c98dbce8d3aba712d82cb62f8951ef2134 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 6 Sep 2026 19:55:25 +0300 Subject: [PATCH 17/22] refactor(captain): extract AI command registration into rootcmd Move AI runtime command registration into the internal root command package to keep CLI command wiring modular without changing behavior. Claude-Session-Id: 01a072be-045a-7831-a018-238a83d1a88c --- cmd/captain/{ => internal/rootcmd}/ai.go | 13 +++++-------- cmd/captain/main.go | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) rename cmd/captain/{ => internal/rootcmd}/ai.go (85%) diff --git a/cmd/captain/ai.go b/cmd/captain/internal/rootcmd/ai.go similarity index 85% rename from cmd/captain/ai.go rename to cmd/captain/internal/rootcmd/ai.go index b1178f14..6b9c0d94 100644 --- a/cmd/captain/ai.go +++ b/cmd/captain/internal/rootcmd/ai.go @@ -1,4 +1,4 @@ -package main +package rootcmd import ( "context" @@ -8,7 +8,7 @@ import ( "github.com/spf13/cobra" ) -func registerAIRuntimeCommands(rootCmd *cobra.Command) { +func RegisterAIRuntimeCommands(root *cobra.Command) { aiCmd := &cobra.Command{ Use: "ai", Short: "AI provider commands", @@ -20,11 +20,11 @@ func registerAIRuntimeCommands(rootCmd *cobra.Command) { "bodies. Use -Plog.level.http= to raise only HTTP logging, or " + "-Phttp.har= to write the exchanges to a HAR archive instead.", } - rootCmd.AddCommand(aiCmd) + root.AddCommand(aiCmd) aiCmd.AddCommand(cli.NewCommandAlias(cli.CommandAliasOptions{ Name: "prompt", Short: "Alias for captain prompt run", - Root: rootCmd, + Root: root, Target: []string{"prompt", "run"}, })) var agentCmd *cobra.Command @@ -42,15 +42,12 @@ func registerAIRuntimeCommands(rootCmd *cobra.Command) { clicky.AddNamedCommand("fixture", aiCmd, cli.AIFixtureOptions{}, cli.RunAIFixture).Short = "Run a YAML fixture across multiple Claude configurations" clicky.AddNamedCommandWithContext("mock", aiCmd, cli.AIMockOptions{}, cli.RunAIMock).Short = "Serve scripted OpenAI/Anthropic replies so agent runs spend no tokens" - // Local-only: --command is run through `sh -c` against a caller-chosen --cwd, - // so published as REST or MCP it would be unauthenticated remote execution. var verifyCmd *cobra.Command - verifyCmd = clicky.AddNamedCommandWithContext("verify", rootCmd, cli.VerifyOptions{}, func(ctx context.Context, opts cli.VerifyOptions) (any, error) { + verifyCmd = clicky.AddNamedCommandWithContext("verify", root, cli.VerifyOptions{}, func(ctx context.Context, opts cli.VerifyOptions) (any, error) { opts.AIProviderOptions = (cli.AIRuntimeOptions{AIProviderOptions: opts.AIProviderOptions}).WithChangedFlags(verifyCmd.Flags()).AIProviderOptions return cli.RunVerify(ctx, opts) }) verifyCmd.Short = "Run a workflow's verification checks and report the verdict" verifyCmd.Long = "Run the checks an api.Workflow declares — shell commands, LLM-judge prompts, and a fixture document handed to the configured fixture runner — against a working tree, and print each check's report. Exits non-zero when any check fails or cannot reach a verdict." clicky.MarkLocalOnly(verifyCmd) - } diff --git a/cmd/captain/main.go b/cmd/captain/main.go index 4eabc434..ea7780fd 100644 --- a/cmd/captain/main.go +++ b/cmd/captain/main.go @@ -251,7 +251,7 @@ func newRootCommand() *cobra.Command { clicky.AddNamedCommandWithContext("list", tokenCmd, cli.TokenListOptions{}, cli.RunTokenList).Short = "List tokens and what each can reach" clicky.AddNamedCommandWithContext("revoke", tokenCmd, cli.TokenRevokeOptions{}, cli.RunTokenRevoke).Short = "Refuse a token from now on" - registerAIRuntimeCommands(rootCmd) + rootcmd.RegisterAIRuntimeCommands(rootCmd) whoamiCmd := clicky.AddNamedCommand("whoami", rootCmd, cli.WhoamiOptions{}, cli.RunWhoami) whoamiCmd.Short = "List agent adapters, auth methods, and available models" From 637e8f6b23ef276f9ffe32d9d26f2de3a92d1583 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 7 Sep 2026 08:47:03 +0300 Subject: [PATCH 18/22] ci(lint): authenticate Gavel package resolution Claude-Session-Id: 01a072be-045a-7831-a018-238a83d1a88c --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a1d75dac..8bf6659e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,8 @@ jobs: go-version: "1.26.x" - name: Lint with Gavel uses: flanksource/gavel@43a9189b99e71ed928f418e01cd34aa797c2c0e0 # v0.0.54 + env: + GITHUB_TOKEN: ${{ github.token }} with: args: lint golangci-lint version: v0.0.54 From 36971af83caa9afc9a1d08c944340a1faa8685f6 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 8 Sep 2026 12:22:55 +0300 Subject: [PATCH 19/22] refactor(aichat): collapse the duplicate suspended-seed waits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit approval_execution.go carried two implementations of awaitSuspendedSeed: a free function with a 15s budget and a Service method with 5s. Production only ever called the method, so the copy the tests exercised was not the copy that ran, and the two had already drifted in both budget and error wording. Keep one — the free function, which takes an explicit store and is therefore testable — with the method's 5s budget and its richer errors, and reduce the method to resolving the store. Claude-Session: https://claude.ai/code/session_01Usd2NB1ZhiU5wMuXg76Y6W Claude-Session-Id: f046afbd-f74b-4513-b309-28a1d5c6b64c --- pkg/aichat/approval_execution.go | 66 ++++++++++---------------------- 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/pkg/aichat/approval_execution.go b/pkg/aichat/approval_execution.go index 481a8166..a86da331 100644 --- a/pkg/aichat/approval_execution.go +++ b/pkg/aichat/approval_execution.go @@ -11,30 +11,39 @@ import ( ) const ( - suspendedSeedTimeout = 15 * time.Second + // suspendedSeedWait bounds how long an approval resolution waits for the + // suspending turn's assistant message to land in the thread store. + suspendedSeedWait = 5 * time.Second suspendedSeedInterval = 25 * time.Millisecond ) -// awaitSuspendedSeed returns the assistant message the suspended turn ended on. -// The durable suspension (prompt run -> waiting) is committed while the same -// stream's persistence goroutine is still writing that assistant message, so an -// approval resolved the instant the run becomes resumable can observe the run -// before its transcript. Wait for that in-flight write instead of rejecting a -// legitimate approval, and fail loudly when it never lands. +// awaitSuspendedSeed returns the thread's trailing assistant message for the +// suspended turn. +// +// The durable suspension (prompt run -> waiting) is committed from the event +// pipeline while the same stream is still persisting the assistant message it +// suspended on, so an approval resolved the instant the run becomes resumable +// can observe the run before its transcript. The run's durable approval state +// guarantees that message is committed or imminent — wait the in-flight write +// out instead of failing a resolution that has already consumed the approval, +// and fail loudly when it never lands. func awaitSuspendedSeed(ctx context.Context, store ThreadStore, threadID, turnID string) (*UIMessage, error) { - deadline := time.Now().Add(suspendedSeedTimeout) + deadline := time.Now().Add(suspendedSeedWait) for { thread, err := store.Get(ctx, threadID) if err != nil { return nil, err } - if len(thread.Messages) > 0 { - seed := thread.Messages[len(thread.Messages)-1] + if count := len(thread.Messages); count > 0 { + seed := thread.Messages[count-1] if strings.EqualFold(seed.Role, string(api.RoleAssistant)) && seed.TurnID == turnID { return &seed, nil } } if time.Now().After(deadline) { + if len(thread.Messages) == 0 { + return nil, fmt.Errorf("captain chat session %s has no suspended assistant message", threadID) + } return nil, fmt.Errorf("captain chat session %s does not end with the suspended turn %s", threadID, turnID) } select { @@ -155,44 +164,11 @@ func enforceApprovalRuntimeProfile(spec api.Spec, resolved api.ComposedSpec) err return nil } -// suspendedSeedWait bounds how long an approval resolution waits for the -// suspending turn's assistant message to land in the thread store. -const suspendedSeedWait = 5 * time.Second - -// awaitSuspendedSeed returns the thread's trailing assistant message for the -// suspended turn. The prompt run reaches its waiting state from the event -// pipeline before the suspending stream persists that message on its final -// unwind, so an approval resolved from a session poll can arrive while the -// write is still in flight. The run's durable approval state guarantees the -// message is committed or imminent — wait it out instead of failing a -// resolution that has already consumed the approval. +// awaitSuspendedSeed resolves this service's thread store and waits there. func (s *Service) awaitSuspendedSeed(ctx context.Context, threadID, turnID string) (*UIMessage, error) { store, err := s.threads(ctx) if err != nil { return nil, err } - deadline := time.Now().Add(suspendedSeedWait) - for { - thread, err := store.Get(ctx, threadID) - if err != nil { - return nil, err - } - if count := len(thread.Messages); count > 0 { - seed := thread.Messages[count-1] - if strings.EqualFold(seed.Role, string(api.RoleAssistant)) && seed.TurnID == turnID { - return &seed, nil - } - } - if time.Now().After(deadline) { - if len(thread.Messages) == 0 { - return nil, fmt.Errorf("captain chat session %s has no suspended assistant message", threadID) - } - return nil, fmt.Errorf("captain chat session %s does not end with the suspended turn %s", threadID, turnID) - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(50 * time.Millisecond): - } - } + return awaitSuspendedSeed(ctx, store, threadID, turnID) } From e8a4ee2752e0ed5fc6913fd07f5995da818a4710 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 8 Sep 2026 12:23:10 +0300 Subject: [PATCH 20/22] fix(aichat): wait for a suspending run to park before refusing its approval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A provider approval is recorded — and so becomes visible on the session and goes out on the event stream carrying its ID — while the stream that raised it is still finishing the turn and encoding its checkpoint. Only afterwards does the prompt run reach `waiting`, the one state ResolveToolApprovalRequest accepts an answer in. Anything answering the question the moment it is asked lost that race and got a 409: a person clicking Approve promptly saw "Tool approval failed with status 409", and the mocked lifecycle suite failed the same way on CI. The store guard cannot simply be relaxed. A resolution applied before the run parks yields no continuation, and the suspension then parks the run on an already-answered approval that nothing ever resumes. So wait the parking out, bounded, the way awaitSuspendedSeed already waits out the other half of this same window — and fail loudly when it never happens, at once for a run that has already ended rather than burning the whole budget. The projection spec asserted that an early answer is refused, pinning the behaviour being changed; both halves are now covered explicitly instead. The prompt-run-conflict spec paused the first captain_prompt_runs query anywhere in the process, which the new pre-transaction read claims, so its interception is scoped to the read inside the transaction — what it always meant. Claude-Session: https://claude.ai/code/session_01Usd2NB1ZhiU5wMuXg76Y6W Claude-Session-Id: f046afbd-f74b-4513-b309-28a1d5c6b64c --- .../approval_settle_integration_test.go | 149 ++++++++++++++++++ .../database_threads_integration_test.go | 18 +-- pkg/aichat/execution_database_authority.go | 73 +++++++++ .../execution_database_integration_test.go | 7 +- 4 files changed, 229 insertions(+), 18 deletions(-) create mode 100644 pkg/aichat/approval_settle_integration_test.go diff --git a/pkg/aichat/approval_settle_integration_test.go b/pkg/aichat/approval_settle_integration_test.go new file mode 100644 index 00000000..4930932e --- /dev/null +++ b/pkg/aichat/approval_settle_integration_test.go @@ -0,0 +1,149 @@ +package aichat_test + +import ( + "context" + "encoding/json" + "time" + + "github.com/flanksource/captain/pkg/aichat" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/database" + "github.com/flanksource/commons-db/dbtest" + "github.com/google/uuid" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A provider approval is answerable — it is on the session, and its ID has gone +// out on the event stream — from the moment the permission frame is observed. +// The run it blocks only reaches `waiting` once the stream has finished the turn +// and encoded its checkpoint, several statements later. These specs pin what +// happens to an answer that arrives inside that window, which is where a person +// clicking Approve promptly, and the mocked lifecycle suite, both landed. +var _ = Describe("Approvals answered while the suspension is still landing", func() { + It("waits for the run to park rather than refusing the answer", func(ctx SpecContext) { + fixture := newApprovalFixture(ctx, "captain_aichat_approval_settles") + + // Nothing has parked the run yet; under the bare store guard this is the + // exact moment that produced "cannot be resolved before its prompt run is + // waiting" and a 409. + suspended := make(chan error, 1) + go func() { + time.Sleep(200 * time.Millisecond) + suspended <- suspendOnAccountsApproval(ctx, fixture.execution) + }() + + continuation, err := fixture.authority.ResolveToolApproval(ctx, aichat.ToolApprovalResolution{ + ThreadID: fixture.thread.ID, ApprovalID: fixture.approvalID, Approved: false, Reason: "not now", + }) + Expect(err).NotTo(HaveOccurred(), "an answer that raced the suspension is still a valid answer") + Expect(continuation).NotTo(BeNil(), "the resolution has to hand back the continuation that resumes the run") + DeferCleanup(continuation.Execution.Close) + Expect(<-suspended).To(Succeed()) + + resolved, err := fixture.store.GetSession(ctx, fixture.thread.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Requests).To(HaveLen(1)) + Expect(resolved.Requests[0].State).To(Equal(string(database.TurnRequestStateDenied))) + Expect(resolved.Requests[0].Reason).To(Equal("not now")) + }) + + It("refuses an answer once the run it blocks has already ended", func(ctx SpecContext) { + fixture := newApprovalFixture(ctx, "captain_aichat_approval_run_ended") + + runID, err := uuid.Parse(fixture.execution.PromptRunID()) + Expect(err).NotTo(HaveOccurred()) + run, err := fixture.db.GetPromptRun(ctx, runID) + Expect(err).NotTo(HaveOccurred()) + cancelled := database.PromptRunStateCancelled + _, err = fixture.db.UpdatePromptRun(ctx, database.UpdatePromptRunInput{ + ID: run.ID, ExpectedVersion: run.Version, State: &cancelled, + }) + Expect(err).NotTo(HaveOccurred()) + + // No suspension is coming, so this must fail on the run's state rather + // than burn the whole settle budget waiting for one. + started := time.Now() + _, err = fixture.authority.ResolveToolApproval(ctx, aichat.ToolApprovalResolution{ + ThreadID: fixture.thread.ID, ApprovalID: fixture.approvalID, Approved: true, + }) + Expect(err).To(MatchError(database.ErrTurnRequestConflict)) + Expect(err).To(MatchError(ContainSubstring("already ended (cancelled)"))) + Expect(time.Since(started)).To(BeNumerically("<", time.Second), + "a run that ended is a decided answer, not something to wait out") + }) +}) + +type approvalFixture struct { + db *database.DB + store *aichat.DatabaseThreadStore + authority *aichat.DatabaseExecutionAuthority + thread *aichat.Thread + execution aichat.Execution + approvalID string +} + +// newApprovalFixture drives a chat turn up to the point where the provider has +// asked for permission and the durable approval exists, but the run has not yet +// been parked. +func newApprovalFixture(ctx context.Context, name string) approvalFixture { + GinkgoHelper() + testDB := dbtest.ForGinkgo(dbtest.Options{Name: name}) + db, err := database.Open(ctx, database.WithDSN(testDB.DSN()), database.WithMigrations()) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(db.Close) + store, err := aichat.NewDatabaseThreadStore(db) + Expect(err).NotTo(HaveOccurred()) + thread, err := store.Create(ctx, "Accounts") + Expect(err).NotTo(HaveOccurred()) + authority, err := aichat.NewDatabaseExecutionAuthority(db) + Expect(err).NotTo(HaveOccurred()) + execution, err := authority.Begin(ctx, aichat.ExecutionRequest{ + ThreadID: thread.ID, RequestID: "user-message-1", Title: thread.Title, + Spec: api.Spec{Model: withCaps(api.Model{Name: "gemini", Mode: api.ModeAPI})}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(store.AppendMessage(ctx, thread.ID, aichat.UIMessage{ + ID: "user-message-1", TurnID: execution.TurnID(), Role: "user", + Parts: []aichat.UIPart{{Type: "text", Text: "Edit the account"}}, + })).To(Succeed()) + permission, err := execution.Observe(ctx, api.Event{ + Kind: api.EventPermission, ToolCallID: "call-account-1", Tool: "accounts_edit", + Input: map[string]any{"id": "acc-1"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(store.AppendMessage(ctx, thread.ID, aichat.UIMessage{ + ID: execution.TurnID() + "-assistant", TurnID: execution.TurnID(), Role: "assistant", + Parts: []aichat.UIPart{{ + Type: "dynamic-tool", ToolName: "accounts_edit", ToolCallID: "call-account-1", + State: "approval-requested", Input: json.RawMessage(`{"id":"acc-1"}`), + Approval: &aichat.Approval{ID: permission.ApprovalID}, + }}, + })).To(Succeed()) + return approvalFixture{ + db: db, store: store, authority: authority, thread: thread, + execution: execution, approvalID: permission.ApprovalID, + } +} + +// suspendOnAccountsApproval completes the turn the way a provider that needs an +// approval does: a terminal result carrying the approval state and the private +// checkpoint the resume replays from. This is what parks the run in `waiting`. +func suspendOnAccountsApproval(ctx context.Context, execution aichat.Execution) error { + _, err := execution.Observe(ctx, api.Event{ + Kind: api.EventResult, Success: true, + ToolApproval: &api.ToolApprovalState{ + Messages: []api.Message{{Role: api.RoleAssistant, Parts: []api.Part{{ + Type: api.PartToolRequest, ToolRequest: &api.ToolRequest{ + ToolCallID: "call-account-1", Name: "accounts_edit", Input: json.RawMessage(`{"id":"acc-1"}`), + }, + }}}}, + Calls: []api.ToolApprovalCall{{Request: api.ToolApprovalRequest{ + ToolCallID: "call-account-1", Tool: "accounts_edit", Input: json.RawMessage(`{"id":"acc-1"}`), + }}}, + ProviderCheckpoint: &api.ProviderCheckpoint{Codec: "test-provider", Version: 1, Payload: []byte("checkpoint")}, + }, + }) + return err +} diff --git a/pkg/aichat/database_threads_integration_test.go b/pkg/aichat/database_threads_integration_test.go index 5d3bba3f..3029dfac 100644 --- a/pkg/aichat/database_threads_integration_test.go +++ b/pkg/aichat/database_threads_integration_test.go @@ -218,8 +218,6 @@ var _ = Describe("Database chat sessions", func() { resolution := aichat.ToolApprovalResolution{ ThreadID: thread.ID, ApprovalID: permission.ApprovalID, Approved: false, Reason: "not now", } - _, err = authority.ResolveToolApproval(ctx, resolution) - Expect(err).To(MatchError(ContainSubstring("cannot be resolved before its prompt run is waiting"))) assistant := aichat.UIMessage{ ID: execution.TurnID() + "-assistant", TurnID: execution.TurnID(), Role: "assistant", Parts: []aichat.UIPart{{ @@ -229,21 +227,7 @@ var _ = Describe("Database chat sessions", func() { }}, } Expect(store.AppendMessage(ctx, thread.ID, assistant)).To(Succeed()) - _, err = execution.Observe(ctx, api.Event{ - Kind: api.EventResult, Success: true, - ToolApproval: &api.ToolApprovalState{ - Messages: []api.Message{{Role: api.RoleAssistant, Parts: []api.Part{{ - Type: api.PartToolRequest, ToolRequest: &api.ToolRequest{ - ToolCallID: "call-account-1", Name: "accounts_edit", Input: json.RawMessage(`{"id":"acc-1"}`), - }, - }}}}, - Calls: []api.ToolApprovalCall{{Request: api.ToolApprovalRequest{ - ToolCallID: "call-account-1", Tool: "accounts_edit", Input: json.RawMessage(`{"id":"acc-1"}`), - }}}, - ProviderCheckpoint: &api.ProviderCheckpoint{Codec: "test-provider", Version: 1, Payload: []byte("checkpoint")}, - }, - }) - Expect(err).NotTo(HaveOccurred()) + Expect(suspendOnAccountsApproval(ctx, execution)).To(Succeed()) aggregate, err := store.GetSession(ctx, thread.ID) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/aichat/execution_database_authority.go b/pkg/aichat/execution_database_authority.go index 65cb8e8a..16f4b78c 100644 --- a/pkg/aichat/execution_database_authority.go +++ b/pkg/aichat/execution_database_authority.go @@ -6,6 +6,7 @@ import ( "fmt" "maps" "strings" + "time" "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/database" @@ -132,10 +133,82 @@ func (a *DatabaseExecutionAuthority) Begin( return execution, nil } +const ( + // suspendedRunWait bounds how long a resolution waits for the run it answers + // to finish parking, and suspendedRunInterval is how often that is re-read. + // They match the seed wait's budget: the two wait out the two halves of the + // same in-flight suspension. + suspendedRunWait = 5 * time.Second + suspendedRunInterval = 25 * time.Millisecond +) + +// awaitSuspendedRun waits for a provider approval's prompt run to reach +// `waiting`, the one state ResolveToolApprovalRequest accepts an answer in. +// +// A provider approval is recorded — and so becomes visible on the session and +// goes out on the event stream carrying its approval ID — while the stream that +// raised it is still finishing the turn and encoding its checkpoint. The run +// only reaches `waiting` several statements later. So anything that answers the +// question the moment it is asked raced the suspension and got a 409 telling it +// to retry something that was never wrong: a person clicking Approve promptly, +// or a poller in a test. +// +// The guard being waited for is not removable. A resolution applied before the +// run parks yields no continuation (see resolveToolApproval), and the suspension +// then parks the run on an already-answered approval that nothing ever resumes. +// So wait the parking out — the same treatment awaitSuspendedSeed gives the +// other half of this window — and fail loudly when it never happens. +// +// This runs outside the resolving transaction deliberately: a snapshot taken +// inside one would never observe the suspending connection's commit. +func (a *DatabaseExecutionAuthority) awaitSuspendedRun(ctx context.Context, approvalID string) error { + requestID, err := uuid.Parse(approvalID) + if err != nil { + return nil // resolveToolApproval reports a malformed ID, with its own message + } + deadline := time.Now().Add(suspendedRunWait) + for { + request, err := a.db.GetTurnRequest(ctx, requestID) + if err != nil { + return nil // the resolve path owns not-found and read failures alike + } + // A caller-tool approval carries its own authority and is answerable + // whatever its run is doing. Anything already decided, or with no run to + // resume, is likewise the store's answer to give, not this wait's. + if request.CredentialID != nil || request.PromptRunID == nil || + request.State != database.TurnRequestStatePending { + return nil + } + run, err := a.db.GetPromptRun(ctx, *request.PromptRunID) + if err != nil { + return err + } + switch run.State { + case database.PromptRunStateWaiting: + return nil + case database.PromptRunStateSucceeded, database.PromptRunStateFailed, database.PromptRunStateCancelled: + return fmt.Errorf("%w: approval %s cannot be resolved, its prompt run %s already ended (%s)", + database.ErrTurnRequestConflict, request.ID, run.ID, run.State) + } + if time.Now().After(deadline) { + return fmt.Errorf("%w: approval %s is still pending after %s with its prompt run %s in state %q rather than waiting", + database.ErrTurnRequestConflict, request.ID, suspendedRunWait, run.ID, run.State) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(suspendedRunInterval): + } + } +} + func (a *DatabaseExecutionAuthority) ResolveToolApproval( ctx context.Context, resolution ToolApprovalResolution, ) (*ApprovalContinuation, error) { + if err := a.awaitSuspendedRun(ctx, resolution.ApprovalID); err != nil { + return nil, err + } var continuation *ApprovalContinuation err := a.db.Transaction(ctx, func(tx *database.DB) error { var resolveErr error diff --git a/pkg/aichat/execution_database_integration_test.go b/pkg/aichat/execution_database_integration_test.go index 89d7b9df..8ccb3b32 100644 --- a/pkg/aichat/execution_database_integration_test.go +++ b/pkg/aichat/execution_database_integration_test.go @@ -385,8 +385,13 @@ var _ = Describe("Database execution authority", func() { continueResolution := make(chan struct{}) var intercepted atomic.Bool const callback = "test:pause_approval_after_prompt_run_read" + // Pause the run read the resolution *resumes* from — the one inside its + // transaction, whose version the update then asserts on. Resolution also + // reads the run outside any transaction first, to wait out a suspension + // still landing; pausing there would stall a read this race is not about. Expect(db.Gorm().Callback().Query().After("gorm:query").Register(callback, func(tx *gorm.DB) { - if tx.Statement.Table == "captain_prompt_runs" && intercepted.CompareAndSwap(false, true) { + _, inTransaction := tx.Statement.ConnPool.(gorm.TxCommitter) + if inTransaction && tx.Statement.Table == "captain_prompt_runs" && intercepted.CompareAndSwap(false, true) { close(versionRead) <-continueResolution } From 05d59b9a84116cf5569611d954812fc6217d3c16 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 8 Sep 2026 12:23:22 +0300 Subject: [PATCH 21/22] fix(gitagent): publish the agent worktree only once it is complete git clone creates its target directory first and writes the local branch ref and its upstream config last, so cloning straight onto the published path left a window where the directory existed but its HEAD named a branch that did not: `git rev-parse @{u}` there fails with "no such branch", which is how the git-agent e2e cycle intermittently failed on CI. Build the workspace in a staging sibling and move it in with one rename. The path's existence now means what every observer already assumed, and an interrupted dispatch no longer leaves behind a partial worktree that the re-dispatch check mistakes for a finished one. Claude-Session: https://claude.ai/code/session_01Usd2NB1ZhiU5wMuXg76Y6W Claude-Session-Id: f046afbd-f74b-4513-b309-28a1d5c6b64c --- pkg/gitagent/workspace.go | 25 ++++++++++++++++-- pkg/gitagent/workspace_test.go | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/pkg/gitagent/workspace.go b/pkg/gitagent/workspace.go index 173c09b3..8a2157d7 100644 --- a/pkg/gitagent/workspace.go +++ b/pkg/gitagent/workspace.go @@ -39,18 +39,39 @@ func SetupAgentWorkspace(ctx context.Context, sidecarRepo, task, dispatchCommit, if _, err := os.Stat(workdir); err == nil { return workdir, nil // re-dispatch onto an existing workspace is a no-op } + // Build the workspace beside its final path and move it in with one rename. + // `git clone` creates its target directory first and writes the local branch + // ref and its upstream config last, so cloning straight onto workdir + // publishes a directory whose HEAD names a branch that does not exist yet — + // `git rev-parse @{u}` there fails with "no such branch". Anything that + // watches the task directory to know the workspace is ready (the e2e suite, + // an operator, the re-dispatch check above) would be reading a worktree that + // is not one yet, and an interrupted clone would leave a partial workspace + // that every later dispatch mistakes for a finished one. + staging, err := os.MkdirTemp(filepath.Dir(workdir), ".worktree-") + if err != nil { + return "", err + } + defer func() { _ = os.RemoveAll(staging) }() // the rename leaves nothing to remove + // MkdirTemp is 0700; a clone is not. Restore the mode the agent would see. + if err := os.Chmod(staging, 0o755); err != nil { + return "", err + } branchName := "captain/" + task if _, err := runGit(ctx, filepath.Dir(workdir), env, - "clone", "--quiet", "--shared", "--branch", branchName, sidecarRepo, workdir); err != nil { + "clone", "--quiet", "--shared", "--branch", branchName, sidecarRepo, staging); err != nil { return "", err } // Pin the selected runtime so a bare `git commit` needs no global config // and still records which model and effort produced it. for _, kv := range [][2]string{{"user.name", runtimeIdentity}, {"user.email", "agent@captain.local"}} { - if _, err := runGit(ctx, workdir, env, "config", kv[0], kv[1]); err != nil { + if _, err := runGit(ctx, staging, env, "config", kv[0], kv[1]); err != nil { return "", err } } + if err := os.Rename(staging, workdir); err != nil { + return "", fmt.Errorf("publish agent workspace %s: %w", workdir, err) + } return workdir, nil } diff --git a/pkg/gitagent/workspace_test.go b/pkg/gitagent/workspace_test.go index b96037bb..e496aa73 100644 --- a/pkg/gitagent/workspace_test.go +++ b/pkg/gitagent/workspace_test.go @@ -49,6 +49,52 @@ func TestSetupAgentWorkspacePinsRuntimeIdentity(t *testing.T) { } } +// The workspace path is published only once it is a workspace. `git clone` +// creates its target first and writes the branch ref and upstream config last, +// so cloning straight onto the final path let an observer read a directory +// whose HEAD named a branch that did not exist yet — which is how the git-agent +// e2e cycle intermittently failed its `@{u}` check on CI. +func TestSetupAgentWorkspaceIsCompleteWhenItAppears(t *testing.T) { + ctx := context.Background() + repo := filepath.Join(t.TempDir(), "sidecar.git") + if err := InitSidecar(ctx, repo); err != nil { + t.Fatal(err) + } + commit, err := BuildControlCommit(ctx, repo, nil, map[string][]byte{"seed.txt": []byte("seed\n")}) + if err != nil { + t.Fatal(err) + } + const task = "t-complete" + if err := SaveTaskState(repo, &TaskState{Task: task}); err != nil { + t.Fatal(err) + } + workdir, err := SetupAgentWorkspace(ctx, repo, task, commit, "captain-agent") + if err != nil { + t.Fatal(err) + } + // A bare `git push` needs an upstream, and it has to be there the moment the + // path exists — not a few milliseconds later (H17). + env := ScrubGitEnv(os.Environ()) + upstream, err := runGit(ctx, workdir, env, "rev-parse", "--abbrev-ref", "@{u}") + if err != nil { + t.Fatalf("the published workspace has no upstream: %v", err) + } + if !strings.HasSuffix(upstream, task) { + t.Fatalf("upstream = %q, want one ending in %q", upstream, task) + } + // The staging directory the clone was built in is gone, so a later dispatch + // cannot mistake a half-built workspace for a finished one. + entries, err := os.ReadDir(taskStateDir(repo, task)) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".worktree-") { + t.Fatalf("staging directory %q survived the setup", entry.Name()) + } + } +} + // A dispatch that launches nothing leaves the supervisor waiting out its whole // budget on work that never started — a silence indistinguishable from an // agent still thinking. Empty must therefore be an error, and "no agent" must From 48f4d2885cb89eca32f5893418863e6deaa8846d Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 8 Sep 2026 12:23:41 +0300 Subject: [PATCH 22/22] test(cli): keep the Go toolchain caches across the test HOME override The package hands itself a throwaway HOME so a developer's ~/.captain.yaml cannot fail specs, including the ones that shell out to a captain subprocess. But the Go toolchain derives GOPATH, GOCACHE and GOMODCACHE from HOME whenever they are unset, so the `go build ./cmd/captain` the git-agent e2e tests run inherited an empty module cache and an empty build cache and re-downloaded and recompiled the entire dependency tree, cgo sqlite3 included, on every run. On CI that was ten minutes for this one package, with the runner's warm caches sitting untouched. Resolve the three variables to absolute paths before HOME is replaced, so the isolation covers captain's config and nothing else. Measured on one e2e test against an already-warm cache: 408s before, 57s after. Claude-Session: https://claude.ai/code/session_01Usd2NB1ZhiU5wMuXg76Y6W Claude-Session-Id: f046afbd-f74b-4513-b309-28a1d5c6b64c --- pkg/cli/main_test.go | 58 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/pkg/cli/main_test.go b/pkg/cli/main_test.go index 7c58c873..dd211260 100644 --- a/pkg/cli/main_test.go +++ b/pkg/cli/main_test.go @@ -1,8 +1,12 @@ package cli import ( + "encoding/json" + "fmt" "os" + "os/exec" "path/filepath" + "strings" "testing" "github.com/flanksource/captain/pkg/captainconfig" @@ -23,6 +27,9 @@ func TestMain(m *testing.M) { if _, ok := os.LookupEnv("CAPTAIN_SESSION_DB_URL"); !ok { _ = os.Setenv("CAPTAIN_SESSION_DB_URL", "off") } + if err := pinGoToolchainEnv(); err != nil { + panic("pkg/cli tests: " + err.Error()) + } home, err := os.MkdirTemp("", "captain-cli-home") if err != nil { panic("pkg/cli tests: isolate HOME: " + err.Error()) @@ -33,3 +40,54 @@ func TestMain(m *testing.M) { _ = os.RemoveAll(home) os.Exit(code) } + +// pinGoToolchainEnv resolves the Go toolchain's cache locations to absolute +// paths and exports them, so the HOME override above cannot relocate them. +// +// The HOME this package hands itself is for captain's own config. But the Go +// toolchain derives GOPATH, GOCACHE and GOMODCACHE from HOME whenever they are +// unset, so every `go build` a test shells out to — captainBinary builds +// ./cmd/captain — inherited an empty module cache and an empty build cache and +// re-downloaded and recompiled the entire dependency tree, cgo sqlite3 +// included. On CI that was ten minutes for this one package, with the runner's +// warm caches sitting untouched. It must run before HOME is replaced. +func pinGoToolchainEnv() error { + names := []string{"GOCACHE", "GOMODCACHE", "GOPATH"} + out, err := exec.Command("go", append([]string{"env", "-json"}, names...)...).Output() + if err != nil { + return fmt.Errorf("resolve the Go toolchain environment: %w", err) + } + var resolved map[string]string + if err := json.Unmarshal(out, &resolved); err != nil { + return fmt.Errorf("decode `go env -json %v`: %w", names, err) + } + for _, name := range names { + if resolved[name] == "" { + return fmt.Errorf("`go env -json` reported no %s; a toolchain subprocess would derive it from the throwaway HOME", name) + } + if err := os.Setenv(name, resolved[name]); err != nil { + return fmt.Errorf("pin %s: %w", name, err) + } + } + return nil +} + +// A toolchain subprocess started from this package must reach the same caches +// as the rest of the build. Left to inherit the throwaway HOME it reaches none +// of them, which is invisible locally and cost ten minutes per CI run. +func TestGoToolchainCachesSurviveTheHomeOverride(t *testing.T) { + home := os.Getenv("HOME") + if home == "" { + t.Fatal("TestMain is expected to hand this package its own HOME") + } + for _, name := range []string{"GOCACHE", "GOMODCACHE", "GOPATH"} { + value := os.Getenv(name) + if value == "" { + t.Errorf("%s is unset, so `go build` derives it from HOME and gets a cold cache", name) + continue + } + if rel, err := filepath.Rel(home, value); err == nil && !strings.HasPrefix(rel, "..") { + t.Errorf("%s = %q sits inside the throwaway HOME %q, so every toolchain subprocess gets a cold cache", name, value, home) + } + } +}