Unify runtime composition and preflight

.github/workflows/test.yml

@@ -24,6 +24,8 @@ jobs:
           go-version: "1.26.x"
       - name: Lint with Gavel
         uses: flanksource/gavel@43a9189b99e71ed928f418e01cd34aa797c2c0e0 # v0.0.54
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         with:
           args: lint golangci-lint
           version: v0.0.54

cmd/captain/internal/rootcmd/ai.go

@@ -0,0 +1,53 @@
+package rootcmd
+
+import (
+	"context"
+
+	"github.com/flanksource/captain/pkg/cli"
+	"github.com/flanksource/clicky"
+	"github.com/spf13/cobra"
+)
+
+func RegisterAIRuntimeCommands(root *cobra.Command) {
+	aiCmd := &cobra.Command{
+		Use:   "ai",
+		Short: "AI provider commands",
+		Long: "AI provider commands.\n\n" +
+			"Logging: increase application verbosity with -v/-vv or --log-level=debug. " +
+			"HTTP calls to the provider APIs are logged on the same ladder (with credentials " +
+			"redacted): failed requests are logged by default, -v adds an access line per " +
+			"request, -vv adds headers and query params, -vvv request bodies, -vvvv response " +
+			"bodies. Use -Plog.level.http=<level> to raise only HTTP logging, or " +
+			"-Phttp.har=<path> to write the exchanges to a HAR archive instead.",
+	}
+	root.AddCommand(aiCmd)
+	aiCmd.AddCommand(cli.NewCommandAlias(cli.CommandAliasOptions{
+		Name:   "prompt",
+		Short:  "Alias for captain prompt run",
+		Root:   root,
+		Target: []string{"prompt", "run"},
+	}))
+	var agentCmd *cobra.Command
+	agentCmd = clicky.AddNamedCommand("agent", aiCmd, cli.AIAgentOptions{}, func(opts cli.AIAgentOptions) (any, error) {
+		opts.AIRuntimeOptions = opts.WithChangedFlags(agentCmd.Flags())
+		return cli.RunAIAgent(opts)
+	})
+	agentCmd.Short = "Run an iterative agent with verifiers, worktree, and commit"
+	clicky.AddNamedCommand("models", aiCmd, cli.AIModelsOptions{}, cli.RunAIModels)
+	var testCmd *cobra.Command
+	testCmd = clicky.AddNamedCommand("test", aiCmd, cli.AITestOptions{}, func(opts cli.AITestOptions) (any, error) {
+		opts.AIProviderOptions = (cli.AIRuntimeOptions{AIProviderOptions: opts.AIProviderOptions}).WithChangedFlags(testCmd.Flags()).AIProviderOptions
+		return cli.RunAITest(opts)
+	})
+	clicky.AddNamedCommand("fixture", aiCmd, cli.AIFixtureOptions{}, cli.RunAIFixture).Short = "Run a YAML fixture across multiple Claude configurations"
+	clicky.AddNamedCommandWithContext("mock", aiCmd, cli.AIMockOptions{}, cli.RunAIMock).Short = "Serve scripted OpenAI/Anthropic replies so agent runs spend no tokens"
+
+	var verifyCmd *cobra.Command
+	verifyCmd = clicky.AddNamedCommandWithContext("verify", root, cli.VerifyOptions{}, func(ctx context.Context, opts cli.VerifyOptions) (any, error) {
+		opts.AIProviderOptions = (cli.AIRuntimeOptions{AIProviderOptions: opts.AIProviderOptions}).WithChangedFlags(verifyCmd.Flags()).AIProviderOptions
+		return cli.RunVerify(ctx, opts)
+	})
+	verifyCmd.Short = "Run a workflow's verification checks and report the verdict"
+	verifyCmd.Long = "Run the checks an api.Workflow declares — shell commands, LLM-judge prompts, and a fixture document handed to the configured fixture runner — against a working tree, and print each check's report. Exits non-zero when any check fails or cannot reach a verdict."
+	clicky.MarkLocalOnly(verifyCmd)
+}

cmd/captain/main.go

@@ -251,29 +251,7 @@ func newRootCommand() *cobra.Command {
 	clicky.AddNamedCommandWithContext("list", tokenCmd, cli.TokenListOptions{}, cli.RunTokenList).Short = "List tokens and what each can reach"
 	clicky.AddNamedCommandWithContext("revoke", tokenCmd, cli.TokenRevokeOptions{}, cli.RunTokenRevoke).Short = "Refuse a token from now on"
 
-	aiCmd := &cobra.Command{
-		Use:   "ai",
-		Short: "AI provider commands",
-		Long: "AI provider commands.\n\n" +
-			"Logging: increase application verbosity with -v/-vv or --log-level=debug. " +
-			"HTTP calls to the provider APIs are logged on the same ladder (with credentials " +
-			"redacted): failed requests are logged by default, -v adds an access line per " +
-			"request, -vv adds headers and query params, -vvv request bodies, -vvvv response " +
-			"bodies. Use -Plog.level.http=<level> to raise only HTTP logging, or " +
-			"-Phttp.har=<path> to write the exchanges to a HAR archive instead.",
-	}
-	rootCmd.AddCommand(aiCmd)
-	aiCmd.AddCommand(cli.NewCommandAlias(cli.CommandAliasOptions{
-		Name:   "prompt",
-		Short:  "Alias for captain prompt run",
-		Root:   rootCmd,
-		Target: []string{"prompt", "run"},
-	}))
-	clicky.AddNamedCommand("agent", aiCmd, cli.AIAgentOptions{}, cli.RunAIAgent).Short = "Run an iterative agent with verifiers, worktree, and commit"
-	clicky.AddNamedCommand("models", aiCmd, cli.AIModelsOptions{}, cli.RunAIModels)
-	clicky.AddNamedCommand("test", aiCmd, cli.AITestOptions{}, cli.RunAITest)
-	clicky.AddNamedCommand("fixture", aiCmd, cli.AIFixtureOptions{}, cli.RunAIFixture).Short = "Run a YAML fixture across multiple Claude configurations"
-	clicky.AddNamedCommandWithContext("mock", aiCmd, cli.AIMockOptions{}, cli.RunAIMock).Short = "Serve scripted OpenAI/Anthropic replies so agent runs spend no tokens"
+	rootcmd.RegisterAIRuntimeCommands(rootCmd)
 
 	whoamiCmd := clicky.AddNamedCommand("whoami", rootCmd, cli.WhoamiOptions{}, cli.RunWhoami)
 	whoamiCmd.Short = "List agent adapters, auth methods, and available models"
@@ -294,13 +272,6 @@ func newRootCommand() *cobra.Command {
 	rootCmd.AddCommand(attachmentsCmd)
 	clicky.AddNamedCommand("gc", attachmentsCmd, cli.AttachmentsGCOptions{}, cli.RunAttachmentsGC).Short = "Remove old unreferenced attachments"
 
-	// Local-only: --command is run through `sh -c` against a caller-chosen --cwd,
-	// so published as REST or MCP it would be unauthenticated remote execution.
-	verifyCmd := clicky.AddNamedCommandWithContext("verify", rootCmd, cli.VerifyOptions{}, cli.RunVerify)
-	verifyCmd.Short = "Run a workflow's verification checks and report the verdict"
-	verifyCmd.Long = "Run the checks an api.Workflow declares — shell commands, LLM-judge prompts, and a fixture document handed to the configured fixture runner — against a working tree, and print each check's report. Exits non-zero when any check fails or cannot reach a verdict."
-	clicky.MarkLocalOnly(verifyCmd)
-
 	hookCmd := &cobra.Command{Use: "hook", Short: "Claude Code hook commands"}
 	rootCmd.AddCommand(hookCmd)
 	bashCheckCmd := &cobra.Command{Use: "bash-check", Short: "Scan bash command for violations (PreToolUse hook)", RunE: func(cmd *cobra.Command, args []string) error {

pkg/ai/agent.go

@@ -2,7 +2,6 @@ package ai
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"sync"
 	"time"
@@ -24,27 +23,10 @@ type Agent struct {
 	costs Costs
 }
 
-// PromptRequest is a single named prompt. Field names/types mirror the former
-// clicky/ai.PromptRequest so consumers need only change the import path.
+// PromptRequest carries the complete specification of one named model call.
 type PromptRequest struct {
-	Name             string            `json:"name"`
-	Prompt           string            `json:"prompt"`
-	SystemPrompt     string            `json:"system_prompt,omitempty"`
-	Context          map[string]string `json:"context,omitempty"`
-	StructuredOutput any               `json:"structured_output,omitempty"`
-	// SchemaJSON is a pre-built JSON Schema (e.g. from a .prompt frontmatter
-	// output block) forwarded verbatim to ai.Request.Prompt.SchemaJSON. Prefer it
-	// over StructuredOutput when the schema is declared in the prompt file rather
-	// than a Go type; the two are mutually exclusive.
-	SchemaJSON json.RawMessage `json:"schema_json,omitempty"`
-	// SchemaStrictness forwards api.Prompt.SchemaStrictness — the policy for a
-	// response that fails schema validation (warning/error/retry). "" (default)
-	// skips validation.
-	SchemaStrictness api.SchemaStrictness `json:"schema_strictness,omitempty"`
-	// Source identifies the prompt template (e.g. the .prompt filename) for
-	// diagnostics; forwarded to ai.Request.Source and printed by the logging
-	// middleware.
-	Source string `json:"source,omitempty"`
+	Name string   `json:"name"`
+	Spec api.Spec `json:"spec"`
 }
 
 // PromptResponse is the result of one PromptRequest.
@@ -95,14 +77,7 @@ func (a *Agent) ExecutePrompt(ctx context.Context, req PromptRequest) (*PromptRe
 			return &PromptResponse{Request: req, Model: a.cfg.Model.Name, Error: err.Error()}, err
 		}
 	}
-	resp, err := a.provider.Execute(ctx, Request{Prompt: api.Prompt{
-		User:             req.Prompt,
-		System:           req.SystemPrompt,
-		Source:           req.Source,
-		Schema:           req.StructuredOutput,
-		SchemaJSON:       req.SchemaJSON,
-		SchemaStrictness: req.SchemaStrictness,
-	}})
+	resp, err := a.provider.Execute(ctx, req.Spec)
 	if err != nil {
 		return &PromptResponse{Request: req, Model: a.cfg.Model.Name, Error: err.Error(), Duration: time.Since(start)}, err
 	}

pkg/ai/agent/verify/declarations.go

@@ -0,0 +1,46 @@
+package verify
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/flanksource/captain/pkg/ai"
+	"github.com/flanksource/captain/pkg/ai/prompt"
+	"github.com/flanksource/captain/pkg/api"
+)
+
+type DeclarationOptions struct {
+	// Provider is authoritative when supplied; Model names a provider not built yet.
+	Provider ai.Provider
+	Model    string
+}
+
+// ValidateDeclarations inspects verifier wiring and judge files without invoking
+// registered factories, executing commands, or constructing a provider.
+func ValidateDeclarations(wf *api.Workflow, opts DeclarationOptions) error {
+	if wf == nil || wf.Verify == nil {
+		return nil
+	}
+	if strings.TrimSpace(wf.Verify.Fixture) != "" && !Registered(KindFixture) {
+		return fmt.Errorf("workflow.verify.fixture declared but no fixture verifier is registered " +
+			"(link a fixture runner in-process or set verify.fixtureRunner in ~/.captain.yaml)")
+	}
+	model := opts.Model
+	if opts.Provider != nil {
+		model = opts.Provider.GetModel()
+	}
+	for i, path := range wf.Verify.Prompts {
+		path = strings.TrimSpace(path)
+		if path == "" {
+			return fmt.Errorf("workflow.verify.prompts[%d] is empty", i)
+		}
+		tmpl, err := prompt.LoadFile(path)
+		if err != nil {
+			return fmt.Errorf("verify prompt %q: %w", path, err)
+		}
+		if err := rejectJudgeOverrides(path, tmpl, model); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/ai/agent/verify/llmjudge.go

@@ -38,7 +38,7 @@ func (j *LLMJudgeVerifier) Verify(ctx context.Context, cwd string, changed []str
 	}
 
 	out := &judgeVerdict{}
-	req, _, err := j.Prompt.Render(data, out)
+	req, _, err := j.Prompt.Render(prompt.RenderOptions{Data: data, Output: out})
 	if err != nil {
 		return Verdict{}, err
 	}

pkg/ai/agent/verify/registry.go

@@ -127,9 +127,8 @@ func HooksFor(ctx context.Context, wf *api.Workflow, opts Options) ([]any, error
 	if wf == nil || wf.Verify == nil {
 		return nil, nil
 	}
-	if strings.TrimSpace(wf.Verify.Fixture) != "" && !Registered(KindFixture) {
-		return nil, fmt.Errorf("workflow.verify.fixture declared but no fixture verifier is registered " +
-			"(link a fixture runner in-process or set verify.fixtureRunner in ~/.captain.yaml)")
+	if err := ValidateDeclarations(wf, DeclarationOptions{Provider: opts.Provider}); err != nil {
+		return nil, err
 	}
 	var hooks []any
 	for _, kind := range kindOrder {
@@ -149,25 +148,6 @@ func HooksFor(ctx context.Context, wf *api.Workflow, opts Options) ([]any, error
 	return hooks, nil
 }
 
-// ValidatePromptDeclarations loads every declared judge prompt before a run
-// constructs its provider. This keeps a broken workflow attributable to the
-// prompt declaration even when the selected provider is unavailable.
-func ValidatePromptDeclarations(wf *api.Workflow) error {
-	if wf == nil || wf.Verify == nil {
-		return nil
-	}
-	for i, path := range wf.Verify.Prompts {
-		path = strings.TrimSpace(path)
-		if path == "" {
-			return fmt.Errorf("workflow.verify.prompts[%d] is empty", i)
-		}
-		if _, err := prompt.LoadFile(path); err != nil {
-			return fmt.Errorf("verify prompt %q: %w", path, err)
-		}
-	}
-	return nil
-}
-
 // DeclaresExec reports whether the workflow declares a check that starts a
 // process — a shell command or a fixture handed to an external runner. A
 // receive path asks before it has any hooks, because the confinement wrapper is
@@ -231,7 +211,7 @@ func promptFactory(_ context.Context, spec api.Verify, opts Options) ([]*Plugin,
 		if err != nil {
 			return nil, fmt.Errorf("verify prompt %q: %w", path, err)
 		}
-		if err := rejectJudgeOverrides(path, tmpl, opts.Provider); err != nil {
+		if err := rejectJudgeOverrides(path, tmpl, opts.Provider.GetModel()); err != nil {
 			return nil, err
 		}
 		plugins = append(plugins, New("judge:"+path, &LLMJudgeVerifier{Provider: opts.Provider, Prompt: tmpl}))
@@ -245,17 +225,17 @@ func promptFactory(_ context.Context, spec api.Verify, opts Options) ([]*Plugin,
 // ignored, which is exactly the downgrade issue #39 forbids (R5.4: a hook
 // prompt declaring a relocating sandbox is a validation error, never a silent
 // fallback).
-func rejectJudgeOverrides(path string, tmpl *prompt.Template, provider ai.Provider) error {
-	probe, _, err := tmpl.Render(map[string]any{"cwd": "", "changed": []string{}}, nil)
+func rejectJudgeOverrides(path string, tmpl *prompt.Template, model string) error {
+	probe, _, err := tmpl.Render(prompt.RenderOptions{Data: map[string]any{"cwd": "", "changed": []string{}}})
 	if err != nil {
 		return fmt.Errorf("verify prompt %q: %w", path, err)
 	}
 	if probe.Sandbox != nil {
 		return fmt.Errorf("verify prompt %q declares a sandbox; judge hooks run on the run's provider and cannot relocate", path)
 	}
-	if declared := strings.TrimSpace(probe.Name); declared != "" && declared != provider.GetModel() {
+	if declared := strings.TrimSpace(probe.Name); declared != "" && declared != model {
 		return fmt.Errorf("verify prompt %q declares model %q but judge hooks run on the run's provider (%s); remove the model or match it",
-			path, declared, provider.GetModel())
+			path, declared, model)
 	}
 	return nil
 }

pkg/ai/agent_spec_ginkgo_test.go

@@ -0,0 +1,97 @@
+package ai_test
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+
+	captainai "github.com/flanksource/captain/pkg/ai"
+	"github.com/flanksource/captain/pkg/api"
+	"github.com/flanksource/commons-db/shell"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+type namedSpecProvider struct {
+	mu       sync.Mutex
+	requests []api.Spec
+}
+
+func (p *namedSpecProvider) Execute(_ context.Context, spec api.Spec) (*api.Response, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.requests = append(p.requests, spec)
+	return &api.Response{Text: spec.Prompt.User, Model: spec.Model.Name, CostUSD: 0.02}, nil
+}
+
+func (p *namedSpecProvider) GetModel() string { return "provider-model" }
+func (p *namedSpecProvider) GetRuntime() api.Runtime {
+	return api.RuntimeOf(api.OpenAI, api.ModeAPI)
+}
+
+var _ = Describe("Named prompt Spec transport", func() {
+	It("forwards the complete declared spec without borrowing provider config", func() {
+		temperature := 0.0
+		spec := api.Spec{
+			Explicit: api.FieldPresence{"/noCache": true, "/permissions": true},
+			Model:    api.Model{Name: "gpt-5.4", Mode: api.ModeAPI, Temperature: &temperature},
+			Prompt: api.Prompt{
+				User: "Review the change", System: "Explain findings", Source: "review.prompt",
+				SchemaJSON: json.RawMessage(`{"type":"object"}`), SchemaStrictness: "error",
+			},
+			Budget:      api.Budget{MaxTokens: 512, MaxTurns: 3, Timeout: "1m"},
+			Memory:      api.Memory{Skills: []string{"review"}, SkipUser: true},
+			Permissions: api.Permissions{Mode: "plan"},
+			Setup:       &shell.Setup{Cwd: "/workspace/review", Env: []string{"REVIEW_MODE=focused"}},
+			Sandbox:     &api.SandboxRef{Mode: "off"},
+			SessionID:   "review-session",
+			CLIArgs:     map[string]any{"resume": true},
+			Workflow:    &api.Workflow{Verify: &api.Verify{Commands: []string{"make test"}}},
+		}
+		provider := &namedSpecProvider{}
+		agent := captainai.NewAgentWithProvider(provider, captainai.Config{
+			Model: api.Model{Name: "config-model"}, Budget: api.Budget{MaxTokens: 2048},
+		})
+		request := captainai.PromptRequest{Name: "review", Spec: spec}
+
+		response, err := agent.ExecutePrompt(context.Background(), request)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(provider.requests).To(Equal([]api.Spec{spec}))
+		Expect(response.Request).To(Equal(request))
+		Expect(response.Result).To(Equal("Review the change"))
+		Expect(agent.TotalCost()).To(BeNumerically("~", 0.02))
+	})
+
+	It("keeps the native structured-output target attached to the prompt", func() {
+		target := &struct{ Summary string }{}
+		spec := api.Spec{Prompt: api.Prompt{User: "Summarize", Schema: target}}
+		provider := &namedSpecProvider{}
+		agent := captainai.NewAgentWithProvider(provider, captainai.Config{})
+
+		_, err := agent.ExecutePrompt(context.Background(), captainai.PromptRequest{Name: "summary", Spec: spec})
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(provider.requests).To(Equal([]api.Spec{spec}))
+		Expect(provider.requests[0].Prompt.Schema).To(BeIdenticalTo(target))
+	})
+
+	It("keeps each batch item's spec and response name independent", func() {
+		requests := []captainai.PromptRequest{
+			{Name: "first", Spec: api.Spec{Prompt: api.Prompt{User: "First"}, SessionID: "first-session"}},
+			{Name: "second", Spec: api.Spec{Prompt: api.Prompt{User: "Second"}, Memory: api.Memory{Skills: []string{"second"}}}},
+		}
+		provider := &namedSpecProvider{}
+		agent := captainai.NewAgentWithProvider(provider, captainai.Config{MaxConcurrent: 2})
+
+		responses, err := agent.ExecuteBatch(context.Background(), requests)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(provider.requests).To(ConsistOf(requests[0].Spec, requests[1].Spec))
+		Expect(responses["first"].Request).To(Equal(requests[0]))
+		Expect(responses["second"].Request).To(Equal(requests[1]))
+		Expect(responses["first"].Result).To(Equal("First"))
+		Expect(responses["second"].Result).To(Equal("Second"))
+		Expect(agent.GetCosts()).To(HaveLen(2))
+	})
+})