diff --git a/README.md b/README.md index c2af5bca..2db333b5 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ It provides tools to: - summarize tool usage, paths, binaries, and API cost - list files changed by a session - install Claude Code hooks -- enforce a session-specific **Definition of Done** gate +- run a workflow's **verification** checks — commands, LLM judges, and fixtures — as a definition of done - test and use AI providers from the command line - run iterative AI agents with verifiers, worktrees, and commits - generate/build/run containerized Claude Code sandboxes @@ -86,21 +86,20 @@ It analyzes: Captain can install hook commands into Claude settings for: - **PreToolUse bash scanning** via `hook bash-check` -- **Stop hook gating** via `hook dod install` The bash-check hook scans bash commands and can deny unsafe or disallowed commands. -### 3. Definition of Done (DoD) +### 3. Verification -Captain supports a per-session Definition of Done workflow: +`captain verify` runs the checks an `api.Workflow` declares against a working tree and reports each one's verdict: -- `dod set` — attach one or more validation commands to a Claude session -- `dod check` — intended for Claude Stop hooks -- `dod run` — manually execute DoD checks -- `dod status` — show current DoD config/results -- `dod clear` — remove the DoD gate +- `--command` — a shell command run as a pass/fail check (repeatable) +- `--prompt` — a `.prompt` LLM judge, judged by the run's provider (repeatable) +- `--fixture` — a fixture document handed to the runner configured as `verify.fixtureRunner` in `~/.captain.yaml` -This lets Claude continue iterating until required checks pass. +The same checks are the generate→verify loop's definition of done: a failing verdict's output feeds the next iteration. A declared check with nothing to run it is an error, never a silent pass. + +`captain verify` is local only — it is excluded from both the REST API and the MCP tool set, because `--command` runs through `sh -c` against a caller-chosen `--cwd`, and published as REST or MCP that would be unauthenticated remote code execution. ### 4. Session changes @@ -138,7 +137,7 @@ Supported backends are inferred from code and dependencies, including: ### 7. Web UI and MCP server - `serve` — starts an HTTP API and embedded web UI for launching AI agents and opening follow-up chat sessions; supports `--dev` to proxy to the Vite dev server -- `mcp` — exposes captain commands (history, info, cost, changes, dod, etc.) as MCP tools so Claude Code can invoke them directly +- `mcp` — exposes captain commands (history, info, cost, changes, verify, etc.) as MCP tools so Claude Code can invoke them directly ### 8. Utility commands @@ -189,7 +188,6 @@ captain/ ├── pkg/collections/ # Generic collection utilities ├── pkg/container/ # Sandbox discovery, generation, build/run logic ├── pkg/container/base/ # Embedded agent base image (Dockerfile, deps.yaml, entrypoint.sh) -├── pkg/dod/ # Definition of Done persistence and execution ├── pkg/git/ # Git worktree helpers ├── pkg/sandbox/ # Token/preset/sandbox helpers ├── Makefile # Thin wrapper around Taskfile @@ -210,7 +208,7 @@ captain ai captain whoami captain configure captain serve -captain dod +captain verify captain hook captain projects captain container @@ -299,22 +297,24 @@ captain hook bash-check install captain hook bash-check install --user ``` -Install the DoD stop hook and related skill files: +### Verification ```bash -captain hook dod install -captain hook dod install --user +captain verify --command "go test ./..." --command "golangci-lint run" +captain verify --fixture acceptance.md --cwd /path/to/repo +captain verify --prompt review-diff.prompt --model claude-sonnet-4-6 ``` -### Definition of Done +Running fixtures needs a runner, since captain declares fixtures but does not execute them: -```bash -captain dod set --session-id "go test ./..." "golangci-lint run" -captain dod status --session-id -captain dod run --session-id -captain dod clear --session-id +```yaml +# ~/.captain.yaml +verify: + fixtureRunner: [gavel, fixture, verify] ``` +The runner is handed the fixture document on stdin, plus `--cwd ` and one `--changed ` per changed file, and answers on stdout with NDJSON: any number of `{"progress": }` lines, then exactly one `{"report": }`. + ### AI utilities ```bash @@ -536,7 +536,7 @@ Starts an HTTP API and embedded web UI. The UI launches `captain ai agent` opera captain mcp ``` -Exposes captain commands as MCP tools. Auto-exposes all commands except `sandbox`, `projects`, `container`, `hook`, `ai`, `dod set/clear/run`. +Exposes captain commands as MCP tools. Auto-exposes all commands except `sandbox`, `projects`, `container`, `hook`, and `ai`. ### Utility commands @@ -803,7 +803,6 @@ If you want to use hooks: ```bash .bin/captain hook bash-check install --user -.bin/captain hook dod install --user .bin/captain hook monitor install # Opt in to Claude CLI estimate capture: .bin/captain hook monitor install --capture-cost diff --git a/cmd/captain/help_test.go b/cmd/captain/help_test.go index 8f86eb20..3973efc5 100644 --- a/cmd/captain/help_test.go +++ b/cmd/captain/help_test.go @@ -12,8 +12,10 @@ import ( var _ = Describe("root help", func() { // commonsMarker is a property only the commons help block documents, so its - // presence distinguishes the appended block from cobra's own output. - const commonsMarker = "http.har.maxBodySize" + // presence distinguishes the appended block from cobra's own output. It is + // a knob that exists in every commons release captain builds against, not + // one whose name is still moving (the body-size cap was renamed after v1.57.0). + const commonsMarker = "http.har.level" newRoot := func() (*cobra.Command, *cobra.Command, *bytes.Buffer) { out := &bytes.Buffer{} diff --git a/cmd/captain/local_only_test.go b/cmd/captain/local_only_test.go index 80fb704b..daad8abf 100644 --- a/cmd/captain/local_only_test.go +++ b/cmd/captain/local_only_test.go @@ -59,6 +59,7 @@ var _ = Describe("REST executor exposure", func() { Entry("git-agent add", http.MethodPost, "/api/v1/sandbox/git-agent"), Entry("git-agent list", http.MethodGet, "/api/v1/sandbox/git-agent"), Entry("captain serve", http.MethodPost, "/api/v1/serve"), + Entry("captain verify", http.MethodPost, "/api/v1/verify"), ) // The token group is the load-bearing case: these routes are what stands in diff --git a/cmd/captain/main.go b/cmd/captain/main.go index 06d1de21..4e0b8dc5 100644 --- a/cmd/captain/main.go +++ b/cmd/captain/main.go @@ -61,6 +61,13 @@ func newRootCommand() *cobra.Command { fmt.Fprintln(os.Stderr, err) os.Exit(1) } + // The configured fixture runner claims the `fixture` verifier before + // any command can build verify hooks: a workflow declaring a fixture + // must dispatch it, never fall through to an empty hook list. + if err := cli.InstallFixtureVerifier(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } // Bind the database context every command reads from, so an unknown // --context fails before the command runs rather than at first query. name, err := cli.ResolveDatabaseContextName(cmd.Context()) @@ -287,25 +294,12 @@ func newRootCommand() *cobra.Command { rootCmd.AddCommand(attachmentsCmd) clicky.AddNamedCommand("gc", attachmentsCmd, cli.AttachmentsGCOptions{}, cli.RunAttachmentsGC).Short = "Remove old unreferenced attachments" - dodCmd := &cobra.Command{ - Use: "dod", - Short: "Definition of Done checks", - Long: "Manage Definition of Done gates that must pass before Claude Code stops. Use 'status' to check current gate state and 'check' to run the gate commands.", - } - rootCmd.AddCommand(dodCmd) - clicky.AddNamedCommand("set", dodCmd, cli.DodSetOptions{}, cli.RunDodSet) - - dodCheckCmd := clicky.AddNamedCommand("check", dodCmd, cli.DodCheckOptions{}, cli.RunDodCheck) - dodCheckCmd.Short = "Run Definition of Done gate checks" - dodCheckCmd.Long = "Execute the configured DoD commands and report pass/fail status for each gate." - - clicky.AddNamedCommand("clear", dodCmd, cli.DodClearOptions{}, cli.RunDodClear) - - dodStatusCmd := clicky.AddNamedCommand("status", dodCmd, cli.DodStatusOptions{}, cli.RunDodStatus) - dodStatusCmd.Short = "Show current Definition of Done gate status" - dodStatusCmd.Long = "Display which DoD gates are configured and their last pass/fail state." - - clicky.AddNamedCommand("run", dodCmd, cli.DodRunOptions{}, cli.RunDodRun) + // Local-only: --command is run through `sh -c` against a caller-chosen --cwd, + // so published as REST or MCP it would be unauthenticated remote execution. + verifyCmd := clicky.AddNamedCommandWithContext("verify", rootCmd, cli.VerifyOptions{}, cli.RunVerify) + verifyCmd.Short = "Run a workflow's verification checks and report the verdict" + verifyCmd.Long = "Run the checks an api.Workflow declares — shell commands, LLM-judge prompts, and a fixture document handed to the configured fixture runner — against a working tree, and print each check's report. Exits non-zero when any check fails or cannot reach a verdict." + clicky.MarkLocalOnly(verifyCmd) hookCmd := &cobra.Command{Use: "hook", Short: "Claude Code hook commands"} rootCmd.AddCommand(hookCmd) @@ -315,9 +309,6 @@ func newRootCommand() *cobra.Command { }} hookCmd.AddCommand(bashCheckCmd) clicky.AddNamedCommand("install", bashCheckCmd, cli.HookInstallOptions{}, cli.RunBashCheckInstall) - dodHookCmd := &cobra.Command{Use: "dod", Short: "Definition of Done hook"} - hookCmd.AddCommand(dodHookCmd) - clicky.AddNamedCommand("install", dodHookCmd, cli.HookInstallOptions{}, cli.RunDodInstall) monitorHookCmd := &cobra.Command{Use: "monitor", Short: "Session monitoring hooks (hooks-first session tracking)"} hookCmd.AddCommand(monitorHookCmd) @@ -396,9 +387,7 @@ func newRootCommand() *cobra.Command { "^container", "^hook", "^ai", - "^dod set", - "^dod clear", - "^dod run", + "^verify", }, }, } diff --git a/docs/pnpm-lock.yaml b/docs/pnpm-lock.yaml index caab0829..106438f3 100644 --- a/docs/pnpm-lock.yaml +++ b/docs/pnpm-lock.yaml @@ -5,7 +5,8 @@ settings: excludeLinksFromLockfile: false overrides: - js-yaml: '>=4.3.0' + cytoscape: 3.34.0 + js-yaml: '>=4.3.0 <5' importers: @@ -72,12 +73,18 @@ importers: specifier: ^4.3.2 version: 4.3.2 devDependencies: + '@astrojs/check': + specifier: ^0.9.10 + version: 0.9.10(prettier@3.9.6)(typescript@6.0.3) '@types/react': specifier: ^18.3.28 version: 18.3.31 '@types/react-dom': specifier: ^18.3.7 version: 18.3.7(@types/react@18.3.31) + cookie: + specifier: 2.0.1 + version: 2.0.1 typescript: specifier: ~6.0.3 version: 6.0.3 @@ -125,6 +132,12 @@ packages: '@antfu/install-pkg@1.1.0': resolution: {integrity: sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==} + '@astrojs/check@0.9.10': + resolution: {integrity: sha512-zgx/UQMozdjOa3bOxjgeCFdtpE3c9rRX6xHwa+2QXvy8z8Akifu2AtubHyv/zzC2znO8dl8fFWL4K+Ba9kS8HQ==} + hasBin: true + peerDependencies: + typescript: ^5.0.0 || ^6.0.0 + '@astrojs/compiler-binding-darwin-arm64@0.3.1': resolution: {integrity: sha512-IEmEF2fUIlTHtpeE/isyEGVOB14cEyh/LZOFYt6wn3jNyVpdC8aR5OZ+RzFUR/f+8ZDM1LaMwZKvoA7eMyJeFw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -190,9 +203,24 @@ packages: resolution: {integrity: sha512-aT7xkgsbNoS6nriY5qKpbihK43slFHO41iqgHCTdOvn1ifaQxLCc5yXy+6GzAtiafoaC1zA7OwVXCXMsvUZOkg==} engines: {node: '>=22.12.0'} + '@astrojs/compiler@2.13.1': + resolution: {integrity: sha512-f3FN83d2G/v32ipNClRKgYv30onQlMZX1vCeZMjPsMMPl1mDpmbl0+N5BYo4S/ofzqJyS5hvwacEo0CCVDn/Qg==} + '@astrojs/internal-helpers@0.10.1': resolution: {integrity: sha512-5phcroT/vmOOrYuuAxtkbPixy5hePtlz9i8K4OeDv3dNK6/UQRuXPOSRTxIOBbUY5Sonw2UaxjbuVc43Mcir6Q==} + '@astrojs/language-server@2.16.15': + resolution: {integrity: sha512-hy7nMT1YGDAaRi9Q/SgywovZ8ThBgaRniMUaIT2/lBkFlJyolLjVxDGkhn3qT+zFUQ7GC/yrEBjsb7Xnj+yeaA==} + hasBin: true + peerDependencies: + prettier: ^3.0.0 + prettier-plugin-astro: '>=0.11.0' + peerDependenciesMeta: + prettier: + optional: true + prettier-plugin-astro: + optional: true + '@astrojs/markdown-remark@7.2.1': resolution: {integrity: sha512-jPVNIqTvk+yKviikszv/Y1U4jGUSKpp/Nw48QZV4qjWgp70j4Lkq3lhSDRbWwCfgKvEyO9GHuVbV1dM2WYXy1w==} @@ -226,6 +254,9 @@ packages: resolution: {integrity: sha512-C1TLn5sPJr0x4vk56piHWKbnqlEB8BKyte5Y45V02U+D7BGO5eMqZDH5aPjnkXQWJggvmsTXxH03QMZ9NgWLzQ==} engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0} + '@astrojs/yaml2ts@0.2.4': + resolution: {integrity: sha512-8oddpOae35pJsXPQXhTkM0ypfKPskVsh2bCxRtbf7e+/Epw2nReakFYpLKjZMEr75CsoF203PMnCocpfz0s69A==} + '@babel/code-frame@7.29.7': resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} engines: {node: '>=6.9.0'} @@ -473,6 +504,27 @@ packages: '@codemirror/view@6.43.4': resolution: {integrity: sha512-YImu23iyKfncJzT7sRy+rEqEhSc8RhOHqDxwy4WzXRKJwYm6iwf/9OJk5ctCAdZ6yi2ZqaGEvmf55fSVqMDrgg==} + '@emmetio/abbreviation@2.3.3': + resolution: {integrity: sha512-mgv58UrU3rh4YgbE/TzgLQwJ3pFsHHhCLqY20aJq+9comytTXUDNGG/SMtSeMJdkpxgXSXunBGLD8Boka3JyVA==} + + '@emmetio/css-abbreviation@2.1.8': + resolution: {integrity: sha512-s9yjhJ6saOO/uk1V74eifykk2CBYi01STTK3WlXWGOepyKa23ymJ053+DNQjpFcy1ingpaO7AxCcwLvHFY9tuw==} + + '@emmetio/css-parser@0.4.1': + resolution: {integrity: sha512-2bC6m0MV/voF4CTZiAbG5MWKbq5EBmDPKu9Sb7s7nVcEzNQlrZP6mFFFlIaISM8X6514H9shWMme1fCm8cWAfQ==} + + '@emmetio/html-matcher@1.3.0': + resolution: {integrity: sha512-NTbsvppE5eVyBMuyGfVu2CRrLvo7J4YHb6t9sBFLyY03WYhXET37qA4zOYUjBWFCRHO7pS1B9khERtY0f5JXPQ==} + + '@emmetio/scanner@1.0.4': + resolution: {integrity: sha512-IqRuJtQff7YHHBk4G8YZ45uB9BaAGcwQeVzgj/zj8/UdOhtQpEIupUhSk8dys6spFIWVZVeK20CzGEnqR5SbqA==} + + '@emmetio/stream-reader-utils@0.1.0': + resolution: {integrity: sha512-ZsZ2I9Vzso3Ho/pjZFsmmZ++FWeEd/txqybHTm4OgaZzdS8V9V/YYWQwg5TC38Z7uLWUV1vavpLLbjJtKubR1A==} + + '@emmetio/stream-reader@2.2.0': + resolution: {integrity: sha512-fXVXEyFA5Yv3M3n8sUGT7+fvecGrZP4k6FnWWMSZVQf69kAq0LLpaBQLGcPR30m3zMmKYhECP4k/ZkzvhEW5kw==} + '@emnapi/core@1.11.1': resolution: {integrity: sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==} @@ -1888,6 +1940,47 @@ packages: peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 + '@volar/kit@2.4.28': + resolution: {integrity: sha512-cKX4vK9dtZvDRaAzeoUdaAJEew6IdxHNCRrdp5Kvcl6zZOqb6jTOfk3kXkIkG3T7oTFXguEMt5+9ptyqYR84Pg==} + peerDependencies: + typescript: '*' + + '@volar/language-core@2.4.28': + resolution: {integrity: sha512-w4qhIJ8ZSitgLAkVay6AbcnC7gP3glYM3fYwKV3srj8m494E3xtrCv6E+bWviiK/8hs6e6t1ij1s2Endql7vzQ==} + + '@volar/language-server@2.4.28': + resolution: {integrity: sha512-NqcLnE5gERKuS4PUFwlhMxf6vqYo7hXtbMFbViXcbVkbZ905AIVWhnSo0ZNBC2V127H1/2zP7RvVOVnyITFfBw==} + peerDependencies: + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + + '@volar/language-service@2.4.28': + resolution: {integrity: sha512-Rh/wYCZJrI5vCwMk9xyw/Z+MsWxlJY1rmMZPsxUoJKfzIRjS/NF1NmnuEcrMbEVGja00aVpCsInJfixQTMdvLw==} + peerDependencies: + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + + '@volar/source-map@2.4.28': + resolution: {integrity: sha512-yX2BDBqJkRXfKw8my8VarTyjv48QwxdJtvRgUpNE5erCsgEUdI2DsLbpa+rOQVAJYshY99szEcRDmyHbF10ggQ==} + + '@volar/typescript@2.4.28': + resolution: {integrity: sha512-Ja6yvWrbis2QtN4ClAKreeUZPVYMARDYZl9LMEv1iQ1QdepB6wn0jTRxA9MftYmYa4DQ4k/DaSZpFPUfxl8giw==} + peerDependencies: + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + + '@vscode/emmet-helper@2.11.0': + resolution: {integrity: sha512-QLxjQR3imPZPQltfbWRnHU6JecWTF1QSWhx3GAKQpslx7y3Dp6sIIXhKjiUJ/BR9FX8PVthjr9PD6pNwOJfAzw==} + + '@vscode/l10n@0.0.18': + resolution: {integrity: sha512-KYSIHVmslkaCDyw013pphY+d7x1qV8IZupYfeIfzNA+nsaWHbn5uPuQRvdRFsa9zFzGeudPuoGoZ1Op4jrJXIQ==} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -1910,10 +2003,34 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + ajv-draft-04@1.0.0: + resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} + peerDependencies: + ajv: ^8.5.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv-i18n@4.2.0: + resolution: {integrity: sha512-v/ei2UkCEeuKNXh8RToiFsUclmU+G57LO1Oo22OagNMENIw+Yb8eMwvHu7Vn9fmkjJyv6XclhJ8TbuigSglPkg==} + peerDependencies: + ajv: ^8.0.0-beta.0 + + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + am-i-vibing@0.4.0: resolution: {integrity: sha512-MxT4XZL7pzLHpuvhDKdMaQHMGGkJDLluKBLsbstn+8wv9sWcFT6h+0ve9qkml95amVTZtZV83gQe2hY+ojgHLg==} hasBin: true + ansi-regex@6.3.0: + resolution: {integrity: sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==} + engines: {node: '>=12'} + + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + anymatch@3.1.3: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} @@ -1984,6 +2101,10 @@ packages: character-reference-invalid@2.0.1: resolution: {integrity: sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==} + chokidar@4.0.3: + resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} + engines: {node: '>= 14.16.0'} + chokidar@5.0.0: resolution: {integrity: sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==} engines: {node: '>= 20.19.0'} @@ -1998,6 +2119,10 @@ packages: classnames@2.5.1: resolution: {integrity: sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==} + cliui@9.0.1: + resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} + engines: {node: '>=20'} + clsx@2.1.1: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} @@ -2088,12 +2213,12 @@ packages: cytoscape-cose-bilkent@4.1.0: resolution: {integrity: sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==} peerDependencies: - cytoscape: ^3.2.0 + cytoscape: 3.34.0 cytoscape-fcose@2.2.0: resolution: {integrity: sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==} peerDependencies: - cytoscape: ^3.2.0 + cytoscape: 3.34.0 cytoscape@3.34.0: resolution: {integrity: sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg==} @@ -2324,9 +2449,15 @@ packages: electron-to-chromium@1.5.385: resolution: {integrity: sha512-78sa/M08MNAYHQfjoWMvOlKQqZ0ElhSm/L5HNUc96VZ3b+KvDVnngFm8sYQy0XrhTRgAhggHr5abA7yTvRdo4Q==} + emmet@2.4.11: + resolution: {integrity: sha512-23QPJB3moh/U9sT4rQzGgeyyGIrcM+GH5uVYg2C6wZIxAIJq7Ng3QLT79tl8FUwDXhyq9SusfknOrofAKqvgyQ==} + emoji-regex-xs@1.0.0: resolution: {integrity: sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg==} + emoji-regex@10.6.0: + resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + enhanced-resolve@5.21.6: resolution: {integrity: sha512-aNnGCvbJ/RIyWo1IuhNdVjnNF+EjH9wpzpNHt+ci/m9He9LJvUN8wrCcXjp9cWsGNAuvSpVFTx/vraAFQ8qGjQ==} engines: {node: '>=10.13.0'} @@ -2398,12 +2529,18 @@ packages: extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + fast-string-truncated-width@3.0.3: resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} fast-string-width@3.0.2: resolution: {integrity: sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==} + fast-uri@3.1.6: + resolution: {integrity: sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==} + fast-wrap-ansi@0.2.2: resolution: {integrity: sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==} @@ -2443,6 +2580,14 @@ packages: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + get-east-asian-width@1.6.0: + resolution: {integrity: sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==} + engines: {node: '>=18'} + get-nonce@1.0.1: resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} engines: {node: '>=6'} @@ -2595,6 +2740,9 @@ packages: engines: {node: '>=6'} hasBin: true + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + json-schema@0.4.0: resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} @@ -2603,6 +2751,9 @@ packages: engines: {node: '>=6'} hasBin: true + jsonc-parser@2.3.1: + resolution: {integrity: sha512-H8jvkz1O50L3dMZCsLqiuB2tA7muqbSg1AtGEkN0leAqGjsUzDJir3Zwr02BhqdcITPg3ei3mZ+HjMocAknhhg==} + jsonc-parser@3.3.1: resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} @@ -2971,6 +3122,9 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + muggle-string@0.4.1: + resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==} + nanoid@3.3.15: resolution: {integrity: sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -3082,6 +3236,11 @@ packages: engines: {node: '>=10.13.0'} hasBin: true + prettier@3.9.6: + resolution: {integrity: sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==} + engines: {node: '>=14'} + hasBin: true + prismjs@1.30.0: resolution: {integrity: sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==} engines: {node: '>=6'} @@ -3190,6 +3349,10 @@ packages: resolution: {integrity: sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==} engines: {node: '>=0.10.0'} + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} + engines: {node: '>= 14.18.0'} + readdirp@5.0.0: resolution: {integrity: sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==} engines: {node: '>= 20.19.0'} @@ -3276,6 +3439,16 @@ packages: remend@1.3.0: resolution: {integrity: sha512-iIhggPkhW3hFImKtB10w0dz4EZbs28mV/dmbcYVonWEJ6UGHHpP+bFZnTh6GNWJONg5m+U56JrL+8IxZRdgWjw==} + request-light@0.5.8: + resolution: {integrity: sha512-3Zjgh+8b5fhRJBQZoy+zbVKpAQGLyka0MPgW3zruTF4dFFJ8Fqcfu9YsAvi/rvdcaTeWG3MkbZv4WKxAn/84Lg==} + + request-light@0.7.0: + resolution: {integrity: sha512-lMbBMrDoxgsyO+yB3sDcrDuX85yYt7sS8BfQd11jtbW/z5ZWgLZRcEGLsLoYw7I0WSUGQBs8CC8ScIxkTX1+6Q==} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + reselect@5.2.0: resolution: {integrity: sha512-AgZ3UOZm3YndfrJ4OYjgrT7bmCm/1iqkjvEfH/oYjzh6PD2qw4QuT3jjnXIrpdt4MTpMXclMT3lXbmRY+XRakw==} @@ -3377,9 +3550,21 @@ packages: react: ^18.0.0 || ^19.0.0 react-dom: ^18.0.0 || ^19.0.0 + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + + string-width@8.2.2: + resolution: {integrity: sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==} + engines: {node: '>=20'} + stringify-entities@4.0.4: resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==} + strip-ansi@7.2.0: + resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} + engines: {node: '>=12'} + style-mod@4.1.3: resolution: {integrity: sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==} @@ -3453,6 +3638,12 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + typesafe-path@0.2.2: + resolution: {integrity: sha512-OJabfkAg1WLZSqJAJ0Z6Sdt3utnbzr/jh+NAHoyWHJe8CMSy79Gm085094M9nvTPy22KzTVn5Zq5mbapCI/hPA==} + + typescript-auto-import-cache@0.3.6: + resolution: {integrity: sha512-RpuHXrknHdVdK7wv/8ug3Fr0WNsNi5l5aB8MYYuXhq2UH5lnEB1htJ1smhtD5VeCsGr2p8mUDtd83LCQDFVgjQ==} + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -3675,19 +3866,111 @@ packages: vite: optional: true + volar-service-css@0.0.71: + resolution: {integrity: sha512-wRRFt9BpjMKCazcgOh67MSjUjiWUCAh99DyYSDIOTuxaRjEtDC7PpB0k1Y1wbJIW/pVtMUSVbpPo3UGSm0Byxw==} + peerDependencies: + '@volar/language-service': ~2.4.0 + peerDependenciesMeta: + '@volar/language-service': + optional: true + + volar-service-emmet@0.0.71: + resolution: {integrity: sha512-zqjzt6bN95e3CUstBm0PBFAJnrfz0ZAARka87fart46/gNCLLuP3Vujy8V/J8HEziTFLnfkgIASLFYPUhonJcA==} + peerDependencies: + '@volar/language-service': ~2.4.0 + peerDependenciesMeta: + '@volar/language-service': + optional: true + + volar-service-html@0.0.71: + resolution: {integrity: sha512-e8tHPhgQ7ooLfudAEIku+kgd9pWkq3SSz8RbnQDI1+Eb8wbenkLGHqoirLqz5ORLV6wIMr2Iv08RWBG5eOcgpw==} + peerDependencies: + '@volar/language-service': ~2.4.0 + peerDependenciesMeta: + '@volar/language-service': + optional: true + + volar-service-prettier@0.0.71: + resolution: {integrity: sha512-Rz7JVH3qD108UCdmIEiZvOBNljMt2nLFdbN8AXcDfn7xD9F5I2aCIsDVqBbXw21PsnxG0b7MfwtNF+zPS/NKUg==} + peerDependencies: + '@volar/language-service': ~2.4.0 + prettier: ^2.2 || ^3.0 + peerDependenciesMeta: + '@volar/language-service': + optional: true + prettier: + optional: true + + volar-service-typescript-twoslash-queries@0.0.71: + resolution: {integrity: sha512-9K2k72s4n7rV9s4bX0MyjbX9iBribvKZbBJKuEmTCZfeWJXs6Yh7bGpY4eoc7UufAjvpheBqwyZCOIPBvxCv0A==} + peerDependencies: + '@volar/language-service': ~2.4.0 + typescript: '*' + peerDependenciesMeta: + '@volar/language-service': + optional: true + typescript: + optional: true + + volar-service-typescript@0.0.71: + resolution: {integrity: sha512-yTtM/BVT6hoyEYnDtaCyAtNhdNeS/mhTTABlBOdw3NNiRBUin3IznFJpgfjer4c6RYopiPjjQjc9VFhxVl1mLw==} + peerDependencies: + '@volar/language-service': ~2.4.0 + typescript: '*' + peerDependenciesMeta: + '@volar/language-service': + optional: true + typescript: + optional: true + + volar-service-yaml@0.0.71: + resolution: {integrity: sha512-qYGWGuVpUTnZGu5P/CR4KLK4aIR8RrcVnmfZ2eRcj9q/I8VZCoC5yy9FtEvfNvnDp4MU17yhdJcvpQPIqhJS2Q==} + peerDependencies: + '@volar/language-service': ~2.4.0 + peerDependenciesMeta: + '@volar/language-service': + optional: true + + vscode-css-languageservice@6.3.10: + resolution: {integrity: sha512-eq5N9Er3fC4vA9zd9EFhyBG90wtCCuXgRSpAndaOgXMh1Wgep5lBgRIeDgjZBW9pa+332yC9+49cZMW8jcL3MA==} + + vscode-html-languageservice@5.6.2: + resolution: {integrity: sha512-ulCrSnFnfQ16YzvwnYUgEbUEl/ZG7u2eV27YhvLObSHKkb8fw1Z9cgsnUwjTEeDIdJDoTDTDpxuhQwoenoLNMg==} + + vscode-json-languageservice@4.1.8: + resolution: {integrity: sha512-0vSpg6Xd9hfV+eZAaYN63xVVMOTmJ4GgHxXnkLCh+9RsQBkWKIghzLhW2B9ebfG+LQQg8uLtsQ2aUKjTgE+QOg==} + engines: {npm: '>=7.0.0'} + + vscode-jsonrpc@8.2.0: + resolution: {integrity: sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==} + engines: {node: '>=14.0.0'} + vscode-jsonrpc@9.0.1: resolution: {integrity: sha512-rfuA6T75H6m5EkbhtEPzre9pT0HPcDI2MMy4+nPFIBks5J8JBAUHD4tRYSgaBOijIEC7SRkC1kKyXTLqbmh9jw==} engines: {node: '>=14.0.0'} + vscode-languageserver-protocol@3.17.5: + resolution: {integrity: sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==} + vscode-languageserver-protocol@3.18.2: resolution: {integrity: sha512-XRyDbT0Pp3sSNti3JmxVEUMySWCSi1hhM+/KUlCy1hV1zmrqpM1OwO12EAki8blhmLuIMpaJrYbo0OzGVfK2Qg==} vscode-languageserver-textdocument@1.0.12: resolution: {integrity: sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA==} + vscode-languageserver-types@3.17.5: + resolution: {integrity: sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==} + vscode-languageserver-types@3.18.0: resolution: {integrity: sha512-8TsGPNMIMiiBdkORgRSvLjuiEIiAFtO+KssmYWxQ+uSVvlf7RjK8YKCOjPzZ+YA04jXEV7+7LvkSmHkhpNS99g==} + vscode-languageserver@9.0.1: + resolution: {integrity: sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==} + hasBin: true + + vscode-nls@5.2.0: + resolution: {integrity: sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==} + vscode-uri@3.1.0: resolution: {integrity: sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==} @@ -3697,12 +3980,29 @@ packages: web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + xxhash-wasm@1.1.0: resolution: {integrity: sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==} + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + yaml-language-server@1.23.0: + resolution: {integrity: sha512-3qVyCOexLCWw06PQa5kRPwvMWMZ/eZeCRWUvgD6a0OkqL/4iCnxy2WumbWifa937Uo5xhyWJ0uxlU39ljhNh7A==} + hasBin: true + + yaml@2.8.3: + resolution: {integrity: sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==} + engines: {node: '>= 14.6'} + hasBin: true + yaml@2.9.0: resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} @@ -3712,6 +4012,10 @@ packages: resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yargs@18.1.0: + resolution: {integrity: sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yjs@13.6.31: resolution: {integrity: sha512-Eq+5BRfbeGyqGVrTJL3bEcr8gKkxPuyuoHmAwpk52fDb8kOVMrfVSTRPd6yiGgX5Fskb96qCRjzjbRjrL4YEnw==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} @@ -3780,6 +4084,17 @@ snapshots: package-manager-detector: 1.7.0 tinyexec: 1.2.4 + '@astrojs/check@0.9.10(prettier@3.9.6)(typescript@6.0.3)': + dependencies: + '@astrojs/language-server': 2.16.15(prettier@3.9.6)(typescript@6.0.3) + chokidar: 4.0.3 + kleur: 4.1.5 + typescript: 6.0.3 + yargs: 18.1.0 + transitivePeerDependencies: + - prettier + - prettier-plugin-astro + '@astrojs/compiler-binding-darwin-arm64@0.3.1': optional: true @@ -3834,6 +4149,8 @@ snapshots: - '@emnapi/core' - '@emnapi/runtime' + '@astrojs/compiler@2.13.1': {} + '@astrojs/internal-helpers@0.10.1': dependencies: '@types/hast': 3.0.5 @@ -3845,6 +4162,31 @@ snapshots: smol-toml: 1.7.0 unified: 11.0.5 + '@astrojs/language-server@2.16.15(prettier@3.9.6)(typescript@6.0.3)': + dependencies: + '@astrojs/compiler': 2.13.1 + '@astrojs/yaml2ts': 0.2.4 + '@jridgewell/sourcemap-codec': 1.5.5 + '@volar/kit': 2.4.28(typescript@6.0.3) + '@volar/language-core': 2.4.28 + '@volar/language-server': 2.4.28(typescript@6.0.3) + '@volar/language-service': 2.4.28(typescript@6.0.3) + muggle-string: 0.4.1 + tinyglobby: 0.2.17 + volar-service-css: 0.0.71(@volar/language-service@2.4.28(typescript@6.0.3)) + volar-service-emmet: 0.0.71(@volar/language-service@2.4.28(typescript@6.0.3)) + volar-service-html: 0.0.71(@volar/language-service@2.4.28(typescript@6.0.3)) + volar-service-prettier: 0.0.71(@volar/language-service@2.4.28(typescript@6.0.3))(prettier@3.9.6) + volar-service-typescript: 0.0.71(@volar/language-service@2.4.28(typescript@6.0.3))(typescript@6.0.3) + volar-service-typescript-twoslash-queries: 0.0.71(@volar/language-service@2.4.28(typescript@6.0.3))(typescript@6.0.3) + volar-service-yaml: 0.0.71(@volar/language-service@2.4.28(typescript@6.0.3)) + vscode-html-languageservice: 5.6.2 + vscode-uri: 3.1.0 + optionalDependencies: + prettier: 3.9.6 + transitivePeerDependencies: + - typescript + '@astrojs/markdown-remark@7.2.1': dependencies: '@astrojs/internal-helpers': 0.10.1 @@ -3934,6 +4276,10 @@ snapshots: is-docker: 4.0.0 package-manager-detector: 1.7.0 + '@astrojs/yaml2ts@0.2.4': + dependencies: + yaml: 2.9.0 + '@babel/code-frame@7.29.7': dependencies: '@babel/helper-validator-identifier': 7.29.7 @@ -4358,6 +4704,29 @@ snapshots: style-mod: 4.1.3 w3c-keyname: 2.2.8 + '@emmetio/abbreviation@2.3.3': + dependencies: + '@emmetio/scanner': 1.0.4 + + '@emmetio/css-abbreviation@2.1.8': + dependencies: + '@emmetio/scanner': 1.0.4 + + '@emmetio/css-parser@0.4.1': + dependencies: + '@emmetio/stream-reader': 2.2.0 + '@emmetio/stream-reader-utils': 0.1.0 + + '@emmetio/html-matcher@1.3.0': + dependencies: + '@emmetio/scanner': 1.0.4 + + '@emmetio/scanner@1.0.4': {} + + '@emmetio/stream-reader-utils@0.1.0': {} + + '@emmetio/stream-reader@2.2.0': {} + '@emnapi/core@1.11.1': dependencies: '@emnapi/wasi-threads': 1.2.2 @@ -5842,6 +6211,62 @@ snapshots: transitivePeerDependencies: - supports-color + '@volar/kit@2.4.28(typescript@6.0.3)': + dependencies: + '@volar/language-service': 2.4.28(typescript@6.0.3) + '@volar/typescript': 2.4.28(typescript@6.0.3) + typesafe-path: 0.2.2 + typescript: 6.0.3 + vscode-languageserver-textdocument: 1.0.12 + vscode-uri: 3.1.0 + + '@volar/language-core@2.4.28': + dependencies: + '@volar/source-map': 2.4.28 + + '@volar/language-server@2.4.28(typescript@6.0.3)': + dependencies: + '@volar/language-core': 2.4.28 + '@volar/language-service': 2.4.28(typescript@6.0.3) + '@volar/typescript': 2.4.28(typescript@6.0.3) + path-browserify: 1.0.1 + request-light: 0.7.0 + vscode-languageserver: 9.0.1 + vscode-languageserver-protocol: 3.18.2 + vscode-languageserver-textdocument: 1.0.12 + vscode-uri: 3.1.0 + optionalDependencies: + typescript: 6.0.3 + + '@volar/language-service@2.4.28(typescript@6.0.3)': + dependencies: + '@volar/language-core': 2.4.28 + vscode-languageserver-protocol: 3.18.2 + vscode-languageserver-textdocument: 1.0.12 + vscode-uri: 3.1.0 + optionalDependencies: + typescript: 6.0.3 + + '@volar/source-map@2.4.28': {} + + '@volar/typescript@2.4.28(typescript@6.0.3)': + dependencies: + '@volar/language-core': 2.4.28 + path-browserify: 1.0.1 + vscode-uri: 3.1.0 + optionalDependencies: + typescript: 6.0.3 + + '@vscode/emmet-helper@2.11.0': + dependencies: + emmet: 2.4.11 + jsonc-parser: 2.3.1 + vscode-languageserver-textdocument: 1.0.12 + vscode-languageserver-types: 3.18.0 + vscode-uri: 3.1.0 + + '@vscode/l10n@0.0.18': {} + acorn-jsx@5.3.2(acorn@8.17.0): dependencies: acorn: 8.17.0 @@ -5864,10 +6289,29 @@ snapshots: '@opentelemetry/api': 1.9.1 zod: 4.4.3 + ajv-draft-04@1.0.0(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + + ajv-i18n@4.2.0(ajv@8.20.0): + dependencies: + ajv: 8.20.0 + + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.6 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + am-i-vibing@0.4.0: dependencies: process-ancestry: 0.1.0 + ansi-regex@6.3.0: {} + + ansi-styles@6.2.3: {} + anymatch@3.1.3: dependencies: normalize-path: 3.0.0 @@ -6006,6 +6450,10 @@ snapshots: character-reference-invalid@2.0.1: {} + chokidar@4.0.3: + dependencies: + readdirp: 4.1.2 + chokidar@5.0.0: dependencies: readdirp: 5.0.0 @@ -6018,6 +6466,12 @@ snapshots: classnames@2.5.1: {} + cliui@9.0.1: + dependencies: + string-width: 7.2.0 + strip-ansi: 7.2.0 + wrap-ansi: 9.0.2 + clsx@2.1.1: {} cm6-theme-basic-light@0.2.0(@codemirror/language@6.12.4)(@codemirror/state@6.7.0)(@codemirror/view@6.43.4)(@lezer/highlight@1.2.3): @@ -6358,8 +6812,15 @@ snapshots: electron-to-chromium@1.5.385: {} + emmet@2.4.11: + dependencies: + '@emmetio/abbreviation': 2.3.3 + '@emmetio/css-abbreviation': 2.1.8 + emoji-regex-xs@1.0.0: {} + emoji-regex@10.6.0: {} + enhanced-resolve@5.21.6: dependencies: graceful-fs: 4.2.11 @@ -6461,12 +6922,16 @@ snapshots: extend@3.0.2: {} + fast-deep-equal@3.1.3: {} + fast-string-truncated-width@3.0.3: {} fast-string-width@3.0.2: dependencies: fast-string-truncated-width: 3.0.3 + fast-uri@3.1.6: {} + fast-wrap-ansi@0.2.2: dependencies: fast-string-width: 3.0.2 @@ -6496,6 +6961,10 @@ snapshots: gensync@1.0.0-beta.2: {} + get-caller-file@2.0.5: {} + + get-east-asian-width@1.6.0: {} + get-nonce@1.0.1: {} get-tsconfig@5.0.0-beta.4: @@ -6712,10 +7181,14 @@ snapshots: jsesc@3.1.0: {} + json-schema-traverse@1.0.0: {} + json-schema@0.4.0: {} json5@2.2.3: {} + jsonc-parser@2.3.1: {} + jsonc-parser@3.3.1: {} katex@0.16.47: @@ -7380,6 +7853,8 @@ snapshots: ms@2.1.3: {} + muggle-string@0.4.1: {} + nanoid@3.3.15: {} neotraverse@1.0.1: {} @@ -7489,6 +7964,8 @@ snapshots: prettier@2.8.8: {} + prettier@3.9.6: {} + prismjs@1.30.0: {} process-ancestry@0.1.0: {} @@ -7584,6 +8061,8 @@ snapshots: dependencies: loose-envify: 1.4.0 + readdirp@4.1.2: {} + readdirp@5.0.0: {} recharts@3.9.1(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react-is@17.0.2)(react@18.3.1)(redux@5.0.1): @@ -7739,6 +8218,12 @@ snapshots: remend@1.3.0: {} + request-light@0.5.8: {} + + request-light@0.7.0: {} + + require-from-string@2.0.2: {} + reselect@5.2.0: {} resolve-pkg-maps@1.0.0: {} @@ -7923,11 +8408,26 @@ snapshots: transitivePeerDependencies: - supports-color + string-width@7.2.0: + dependencies: + emoji-regex: 10.6.0 + get-east-asian-width: 1.6.0 + strip-ansi: 7.2.0 + + string-width@8.2.2: + dependencies: + get-east-asian-width: 1.6.0 + strip-ansi: 7.2.0 + stringify-entities@4.0.4: dependencies: character-entities-html4: 2.1.0 character-entities-legacy: 3.0.0 + strip-ansi@7.2.0: + dependencies: + ansi-regex: 6.3.0 + style-mod@4.1.3: {} style-to-js@1.1.21: @@ -7989,6 +8489,12 @@ snapshots: tslib@2.8.1: {} + typesafe-path@0.2.2: {} + + typescript-auto-import-cache@0.3.6: + dependencies: + semver: 7.8.5 + typescript@6.0.3: {} ufo@1.6.4: {} @@ -8159,8 +8665,95 @@ snapshots: optionalDependencies: vite: 8.1.3(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) + volar-service-css@0.0.71(@volar/language-service@2.4.28(typescript@6.0.3)): + dependencies: + vscode-css-languageservice: 6.3.10 + vscode-languageserver-textdocument: 1.0.12 + vscode-uri: 3.1.0 + optionalDependencies: + '@volar/language-service': 2.4.28(typescript@6.0.3) + + volar-service-emmet@0.0.71(@volar/language-service@2.4.28(typescript@6.0.3)): + dependencies: + '@emmetio/css-parser': 0.4.1 + '@emmetio/html-matcher': 1.3.0 + '@vscode/emmet-helper': 2.11.0 + vscode-uri: 3.1.0 + optionalDependencies: + '@volar/language-service': 2.4.28(typescript@6.0.3) + + volar-service-html@0.0.71(@volar/language-service@2.4.28(typescript@6.0.3)): + dependencies: + vscode-html-languageservice: 5.6.2 + vscode-languageserver-textdocument: 1.0.12 + vscode-uri: 3.1.0 + optionalDependencies: + '@volar/language-service': 2.4.28(typescript@6.0.3) + + volar-service-prettier@0.0.71(@volar/language-service@2.4.28(typescript@6.0.3))(prettier@3.9.6): + dependencies: + vscode-uri: 3.1.0 + optionalDependencies: + '@volar/language-service': 2.4.28(typescript@6.0.3) + prettier: 3.9.6 + + volar-service-typescript-twoslash-queries@0.0.71(@volar/language-service@2.4.28(typescript@6.0.3))(typescript@6.0.3): + dependencies: + vscode-uri: 3.1.0 + optionalDependencies: + '@volar/language-service': 2.4.28(typescript@6.0.3) + typescript: 6.0.3 + + volar-service-typescript@0.0.71(@volar/language-service@2.4.28(typescript@6.0.3))(typescript@6.0.3): + dependencies: + path-browserify: 1.0.1 + semver: 7.8.5 + typescript-auto-import-cache: 0.3.6 + vscode-languageserver-textdocument: 1.0.12 + vscode-nls: 5.2.0 + vscode-uri: 3.1.0 + optionalDependencies: + '@volar/language-service': 2.4.28(typescript@6.0.3) + typescript: 6.0.3 + + volar-service-yaml@0.0.71(@volar/language-service@2.4.28(typescript@6.0.3)): + dependencies: + vscode-uri: 3.1.0 + yaml-language-server: 1.23.0 + optionalDependencies: + '@volar/language-service': 2.4.28(typescript@6.0.3) + + vscode-css-languageservice@6.3.10: + dependencies: + '@vscode/l10n': 0.0.18 + vscode-languageserver-textdocument: 1.0.12 + vscode-languageserver-types: 3.17.5 + vscode-uri: 3.1.0 + + vscode-html-languageservice@5.6.2: + dependencies: + '@vscode/l10n': 0.0.18 + vscode-languageserver-textdocument: 1.0.12 + vscode-languageserver-types: 3.18.0 + vscode-uri: 3.1.0 + + vscode-json-languageservice@4.1.8: + dependencies: + jsonc-parser: 3.3.1 + vscode-languageserver-textdocument: 1.0.12 + vscode-languageserver-types: 3.18.0 + vscode-nls: 5.2.0 + vscode-uri: 3.1.0 + + vscode-jsonrpc@8.2.0: {} + vscode-jsonrpc@9.0.1: {} + vscode-languageserver-protocol@3.17.5: + dependencies: + vscode-jsonrpc: 8.2.0 + vscode-languageserver-types: 3.17.5 + vscode-languageserver-protocol@3.18.2: dependencies: vscode-jsonrpc: 9.0.1 @@ -8168,22 +8761,64 @@ snapshots: vscode-languageserver-textdocument@1.0.12: {} + vscode-languageserver-types@3.17.5: {} + vscode-languageserver-types@3.18.0: {} + vscode-languageserver@9.0.1: + dependencies: + vscode-languageserver-protocol: 3.17.5 + + vscode-nls@5.2.0: {} + vscode-uri@3.1.0: {} w3c-keyname@2.2.8: {} web-namespaces@2.0.1: {} + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.3 + string-width: 7.2.0 + strip-ansi: 7.2.0 + xxhash-wasm@1.1.0: {} + y18n@5.0.8: {} + yallist@3.1.1: {} + yaml-language-server@1.23.0: + dependencies: + '@vscode/l10n': 0.0.18 + ajv: 8.20.0 + ajv-draft-04: 1.0.0(ajv@8.20.0) + ajv-i18n: 4.2.0(ajv@8.20.0) + prettier: 3.9.6 + request-light: 0.5.8 + vscode-json-languageservice: 4.1.8 + vscode-languageserver: 9.0.1 + vscode-languageserver-textdocument: 1.0.12 + vscode-languageserver-types: 3.18.0 + vscode-uri: 3.1.0 + yaml: 2.8.3 + + yaml@2.8.3: {} + yaml@2.9.0: {} yargs-parser@22.0.0: {} + yargs@18.1.0: + dependencies: + cliui: 9.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + string-width: 8.2.2 + y18n: 5.0.8 + yargs-parser: 22.0.0 + yjs@13.6.31: dependencies: lib0: 0.2.117 diff --git a/docs/src/data/navigation.ts b/docs/src/data/navigation.ts index a82152fc..9f907dd5 100644 --- a/docs/src/data/navigation.ts +++ b/docs/src/data/navigation.ts @@ -74,6 +74,35 @@ export const docSections: DocSection[] = [ }, ], }, + { + label: "AI Agents", + items: [ + { + label: "AI Agents Overview", + href: "/agents/", + description: "The generate→verify loop, its hooks, and how a host embeds it.", + status: "ready", + }, + { + label: "Verification", + href: "/agents/verification/", + description: "Verifier kinds, workflow.verify, the fixture-runner contract, and captain verify.", + status: "ready", + }, + { + label: "Embedding a Run", + href: "/agents/embedding/", + description: "pkg/promptrun.Run — Input, hooks, timeouts, verify-only runs, and Result.", + status: "ready", + }, + { + label: "Approvals", + href: "/agents/approvals/", + description: "The durable tool-approval broker (pkg/ai/approval.Broker).", + status: "ready", + }, + ], + }, { label: "Next Sections", items: [ @@ -83,12 +112,6 @@ export const docSections: DocSection[] = [ description: "Embedded API and web UI.", status: "stub", }, - { - label: "AI Agents", - href: "/agents/", - description: "Agent loop, verifiers, worktrees, and judges.", - status: "stub", - }, { label: "Sessions", href: "/sessions/", diff --git a/docs/src/pages/agents/approvals.mdx b/docs/src/pages/agents/approvals.mdx new file mode 100644 index 00000000..0c24dc24 --- /dev/null +++ b/docs/src/pages/agents/approvals.mdx @@ -0,0 +1,85 @@ +--- +layout: ../../layouts/DocsLayout.astro +title: Approvals +description: The durable tool-approval broker (pkg/ai/approval.Broker). +section: AI Agents +--- + +import CodeExample from "../../components/CodeExample.tsx"; + +# Approvals + +`pkg/ai/approval.Broker` is the `api.PermissionFunc` every execution path that owns a session and a prompt run shares: it records one pending `captain_turn_requests` row, hands the host an `api.EventPermission` frame to surface, and blocks until that row is resolved, expires, or the caller's context ends. + +The aichat execution path supplies its caller-tool credential, turn, and model call; a streaming provider run (`captain prompt run`) or an external host such as a dashboard supplies none of the three and is identified by its prompt run and tool call alone. + +## Fields + + + +`Validate()` refuses to broker anything without a database, a session ID, a prompt run ID, a `Notify` callback, and a positive `Timeout`. + +## `Notify`, `OnWaiting` / `OnRunning` + +`CanUseTool` (the `PermissionFunc`) records the pending approval idempotently, calls `Notify` with the `EventPermission` frame, and blocks. `OnWaiting` and `OnRunning` bracket that wait with the host's own state: once the host has been told the run is waiting, **every** way out of `CanUseTool` tells it the run is running again — a cancelled caller resumes on a context with cancellation stripped (`context.WithoutCancel`), because ending the wait is the response to the cancellation, not a victim of it. A credential-less (provider or host) approval depends on `OnWaiting`: the store only resolves such an approval while its prompt run is in the `waiting` state. + +## The provider path: no turn, no model call + +A `captain prompt run` or external-host approval — as opposed to one raised inside an aichat turn — leaves `TurnID`, `ModelCallID`, and `CredentialID` all unset (`uuid.Nil` / `nil`). The durable row still records the session and prompt run it belongs to, and is resolved the same way: by session ID and request ID, with no credential or turn to check against. + +## Listing and resolving + + + +- **`ListTurnRequests`** requires `SessionID`; filter to one run with `PromptRunID`. A host listing what a user needs to answer filters the results for `state == "pending"` itself. +- **`ResolveToolApprovalRequest`** flips a pending row to `approved` or `denied`. A credential-backed (caller-tool) approval can only resolve while its expected turn is still active; a credential-less (provider/host) approval can only resolve while its prompt run is `waiting`. +- **`CancelPendingTurnRequests`** cancels every still-pending approval for one session + prompt run — used when a run ends with approvals nobody ever answered. + +## Per-tool `ask` needs a broker + +`api.RequireToolPolicySupport` is checked before a run's first model call. A per-tool policy of `deny` or `allow` is refused outright on a runtime that cannot carry an allow/deny list at all — running without it would grant the agent more than the spec allows. A per-tool policy of `ask` is refused unconditionally: no transport exposes a per-tool prompt to the *runtime's own* declarative tool filter, so it would resolve to "allowed" on every runtime that does support allow/deny lists. + +That check is about the runtime's declarative filter, not about whether a broker exists — an `ask` decision made per call, live, is a different mechanism: it is what `approval.Broker.CanUseTool` implements as the `PermissionFunc` itself, and it is how the caller-tool path (aichat) honours a per-tool `ask` policy today. A streaming provider run's declared `permissions.tools` still cannot say `ask` — only `allow`/`deny` — regardless of whether a broker is attached. diff --git a/docs/src/pages/agents/embedding.mdx b/docs/src/pages/agents/embedding.mdx new file mode 100644 index 00000000..c9eb47ec --- /dev/null +++ b/docs/src/pages/agents/embedding.mdx @@ -0,0 +1,87 @@ +--- +layout: ../../layouts/DocsLayout.astro +title: Embedding a Run +description: pkg/promptrun.Run — Input, hooks, timeouts, verify-only runs, and Result. +section: AI Agents +--- + +import CodeExample from "../../components/CodeExample.tsx"; + +# Embedding a run + +`pkg/promptrun.Run` executes one resolved prompt spec through captain's generate→verify loop: attachment checks, provider construction, tool-policy enforcement, the workflow's commit and verify hooks, the caller's own hooks, the setup plugin, and finally `agent.Runner`. It is the seam `captain prompt run` and an embedding host (gavel's todo lifecycle) share, so the two never assemble the same pieces by hand and drift. + +What stays with the caller: rendering the prompt and resolving its spec, resolving attachments against a store, persisting the run, and streaming events to whoever is watching (`OnEvent` is the tap for that). + +## `Input` + + + +- **`Provider`** — when set, `Run` takes it as owning the workspace: a remote-executing sandbox that materialises the checkout on its own side, or a test double. `Run` adds no setup hook in that case. When nil, `Run` constructs the provider from `Config` and prepares `Request.Setup` through the setup plugin, in-process. +- **`CallerOwnsCommits`** — for a host whose own commit pipeline runs the commit (gavel's pre-commit gates and trailers): `Run` builds no commit hook from `Workflow.Commits`, and leaves the declaration on the request so the recorded spec still says what the run commits, and validation of that declaration still runs. `Hooks` must be non-empty when this is set — nothing would commit otherwise, and `Hooks()` refuses the run rather than silently doing nothing. + +## Timeout rule + + + +There is deliberately no compiled-in ceiling. An earlier fallback capped a run that had declared nothing, so a job that legitimately needed an hour died at a couple of minutes with a deadline nobody had chosen and nothing naming it. + +## Verify-only runs + +A spec with an empty `prompt.user` but a declared `workflow.verify` is **verify-only**: generation is skipped and the run scores the state that already exists (`api.Spec.IsVerifyOnly`). `Run` classifies every request exactly once, through `Request.ValidateRunnable`, so the caller building the provider and the runner deciding whether to call a model can never disagree about what kind of run this is. + +The rule this exists for is the near miss: attachments or a message history with no prompt body and no `workflow.verify` declared is **not** verify-only (there is nothing to verify) and is **not** runnable either — attachments and messages accompany a prompt, they never stand in for one. `ValidateRunnable` refuses that shape outright rather than let it silently report a pass having generated nothing and verified nothing. + +A verify-only run of only `commands` and a `fixture` needs no model and builds no provider — it must work without credentials to hand. A verify-only run that also declares `prompts` (LLM judges) still builds one, because the judge needs it. + +## `OnEvent` + + + +`OnEvent` taps the live event stream — the model's events and the hooks' — with the runner's own signature: `iter` names the turn an event belongs to, which a renderer needs and a single-turn caller ignores. + +## `Result`, final report and merging + + + +`Passed`, `FailureReason`, and `FinalReport` are exported functions over `[]agent.VerifyResult`, not methods that could disagree with what the runner itself decided: + +- **`Passed(verdicts)`** is `agent.VerifyPassed(verdicts)` — the rule the runner sets `HookContext.Verified` from — so a caller reading `Result.Passed` and a `Post` hook reading its context never disagree about whether the run verified. +- **`FailureReason(verdicts)`** is the last failing verdict's reason, empty when the run passed. +- **`FinalReport(verdicts)`** is the run's verdict as one report: a round runs every verifier the workflow declares, so it produces one report per verifier. Taking only the last of them used to throw the rest away — a round of `commands` + `fixture` came back as the fixture's tree alone. A round of one report is that report unwrapped; a round of several is `api.MergeReports`, which nests each report under its own group node (its own `summary` carried on the node, not recomputed, so an elided suite still contributes its whole tally) and passes only when every report in the round passed. diff --git a/docs/src/pages/agents/index.mdx b/docs/src/pages/agents/index.mdx index 902a429f..669603fa 100644 --- a/docs/src/pages/agents/index.mdx +++ b/docs/src/pages/agents/index.mdx @@ -1,11 +1,43 @@ --- layout: ../../layouts/DocsLayout.astro title: AI Agents -description: Placeholder for Captain AI agent documentation. -section: Next Sections +description: The generate→verify loop, its hooks, and how a host embeds it. +section: AI Agents --- # AI agents -This section is scaffolded for the iterative agent loop, verification commands, git worktrees, commit finalization, and judge checks. Prompts are the first authored section; agent lifecycle docs can build on the same runtime spec vocabulary. +An `api.Spec` (a rendered `.prompt` file plus any runtime overlay) declares a **workflow**: what to generate, how to verify it, and how to commit it. `pkg/promptrun.Run` is the seam that turns a spec into one executed run — `captain prompt run` and an embedding host (gavel's todo lifecycle) both go through it, so the two never drift on what a run actually does. +
+
+

Verification

+

`workflow.verify` declares commands, LLM-judge prompts, and a fixture document as the loop's definition of done. See Verification.

+
+
+

Embedding a run

+

`promptrun.Run` assembles the provider, the hook list, and the `agent.Runner` loop from one `Input`. See Embedding a run.

+
+
+

Approvals

+

`approval.Broker` is the durable `PermissionFunc` that records a pending tool call and blocks until a host resolves it. See Approvals.

+
+
+ +## Hook order + +`promptrun.Hooks` assembles one run's `agent.Runner` hooks in a fixed order: + +``` +commit → cmd → prompt → fixture → caller → setup +``` + +Commit hooks lead so a squash is cut before any teardown can take the tree it commits from. The workflow's own checks run cheap to expensive — commands, then LLM judges, then the fixture — so a run that is going to fail fails on the fast check first. The caller's own hooks sit between the checks and setup: a host's commit pipeline at `PhaseRun` still sees a live worktree, and a host's `PreRun` hook runs before the tree is relocated. Setup trails so its teardown is the last thing to happen. + +A caller that sets `Input.CallerOwnsCommits` drops the leading commit hooks — the host commits the tree itself — and must supply at least one hook of its own, or nothing would commit at all. + +## Where to go next + +- [Verification](/agents/verification/) — verifier kinds, `workflow.verify`, the fixture-runner contract, and `captain verify`. +- [Embedding a run](/agents/embedding/) — `promptrun.Input`, timeouts, verify-only runs, and `Result`. +- [Approvals](/agents/approvals/) — the durable tool-approval broker. diff --git a/docs/src/pages/agents/verification.mdx b/docs/src/pages/agents/verification.mdx new file mode 100644 index 00000000..89594eaa --- /dev/null +++ b/docs/src/pages/agents/verification.mdx @@ -0,0 +1,135 @@ +--- +layout: ../../layouts/DocsLayout.astro +title: Verification +description: Verifier kinds, workflow.verify, the fixture-runner contract, and captain verify. +section: AI Agents +--- + +import CodeExample from "../../components/CodeExample.tsx"; + +# Verification + +`workflow.verify` is a run's definition of done. It runs after each generation and votes; a non-passing verdict appends feedback to the prompt and triggers another iteration, up to `maxIterations`. Everything below is `pkg/ai/agent/verify` and `pkg/api.Verify`/`api.VerifyReport`. + +## Verifier kinds + +Three kinds of check exist, each mapped from one field of `workflow.verify` by a registered `verify.Factory`: + +| Kind | Field | Factory | +| --- | --- | --- | +| `cmd` | `commands` | built in, registered at package init | +| `prompt` | `prompts` | built in, registered at package init | +| `fixture` | `fixture` | not registered by default — a host links one in-process, or configures an external runner | + +A round runs every declared kind in `kindOrder`: `cmd`, then `prompt`, then `fixture` — cheap and deterministic first, so a run that is going to fail fails on the fast check rather than after a model call or a whole test run. + +## `workflow.verify` + + + +- **`commands`** (`[]string`) — shell commands run as pass/fail checks (exit 0 = pass); their output tail becomes the re-run feedback. This is the only part of `verify` captain itself executes, via `agent/verify.CmdVerifier`. +- **`fixture`** (`string`) — a clicky-FixtureEditor markdown document (acceptance criteria / LLM-judge checklist). Captain does not implement a fixture engine; it dispatches this document to whatever claimed the `fixture` kind. +- **`prompts`** (`[]string`) — `.prompt` template paths run as LLM-judge checks. Each renders against the run's context and must yield `{ok, reason, feedback}` (`agent/verify.LLMJudgeVerifier`). A judge prompt **cannot** declare its own `sandbox` or a `model` different from the run's provider — both are hard validation errors, not a silent downgrade. +- **`scope`** (`""` \| `all` \| `changed`) — narrows verification to the agent's changed files instead of the whole tree. +- **`maxIterations`** (`int`, `>= 0`) — caps the generate→verify loop; `0` means the run default of `1` (a single generation, verification votes once with no automatic re-run). + +## A declared fixture with no verifier is an error + +Declaring `workflow.verify.fixture` with no `fixture` factory registered does **not** silently produce zero hooks — a workflow whose only verification is a fixture would otherwise pass vacuously. `verify.HooksFor` refuses the run outright with: + +``` +workflow.verify.fixture declared but no fixture verifier is registered +(link a fixture runner in-process or set verify.fixtureRunner in ~/.captain.yaml) +``` + +A host claims the `fixture` kind one of two ways: linking a fixture engine in-process (`verify.Register(verify.KindFixture, factory)` — gavel does this from `fixtures/verifier`, panicking if something has already registered it), or configuring an external runner in `~/.captain.yaml`. + +## External fixture runner (`verify.fixtureRunner`) + + + +When `verify.fixtureRunner` names a program, captain installs it as the `fixture` verifier (`agent/verify.ExternalVerifier`) and dispatches every declared fixture to it as **one process, three streams**: + +- **stdin** — the fixture markdown document, verbatim. +- **argv** — the configured command, then `--cwd ` and one `--changed ` per file the agent changed (scope permitting). +- **stdout** — NDJSON: zero or more `{"progress": }` lines while it runs, then exactly one `{"report": }`. +- **stderr** — diagnostics; its tail is appended to any error captain reports. + +The contract fails loudly rather than guessing a verdict: + +| Runner behavior | Result | +| --- | --- | +| Exits 0 with exactly one `report` line | The verdict. A failing fixture is still a verdict (the runner exits non-zero for it and the report says `passed: false`). | +| Non-zero exit with no `report` line | Error: `"...exited N with no report line"`, plus the process error (e.g. a missing binary) and the stderr tail. | +| More than one `report` line | Error: `"emitted more than one report line"`. | +| Malformed JSON on a line | Error naming the malformed line (truncated to 200 bytes). | +| Wall clock exceeds `Timeout` | Error: `"...timed out after "`, with the stderr tail. | + +`gavel fixtures run --format captain-verify-report` implements exactly this contract — it is gavel's own external-runner entry point, reading the fixture document from stdin and writing NDJSON progress/report lines to stdout. + +## `VerifyReport` + +Every verifier's verdict is a typed `api.VerifyReport` — what a `Verify` hook returns, what captain persists per iteration, and what a renderer draws. + +| Field | Type | Notes | +| --- | --- | --- | +| `kind` | `string` | `cmd` \| `prompt` \| `fixture` \| `func` \| `round` (a merged round whose reports don't share one kind) | +| `name` | `string` | | +| `ran` | `bool` | | +| `passed` | `bool` | Only `true` when `state == "passed"` | +| `reason`, `feedback` | `string` | Feedback is what a failing verdict appends to the next turn's prompt | +| `iteration` | `int` | **1-based** loop turn this report judged — the same numbering `captain_prompt_run_iterations` is keyed on; always present, never omitted | +| `summary` | `VerifySummary` | `total`, `passed`, `failed`, `warned`, `skipped`, `pending`, `running`, and `timedout` (its own bucket, spelled `timedout` to match clicky-ui's `StatusCounts`) | +| `tests` | `[]VerifyNode` | The tree of what ran; a node can carry its own `summary` and no `children`, which lets a producer report a suite's totals without listing every row it elided | +| `checklist` | `[]VerifyChecklistItem` | `{item, passed, message}` — `passed` is `nil` while unjudged | +| `state` | `VerifyState` | `queued`, `running`, `passed`, `failed`, `errored`, `warned`, `skipped`, `cancelled`, `timed_out` | +| `started_at`, `finished_at`, `duration` | | | + +`state` normally follows the tree (`Failed > TimedOut > Warned > Running > Queued > Skipped > Passed`, in that precedence), but `errored` and `cancelled` are **host-stamped**: a runner that could not schedule its checks, or a run stopped mid-check, asserts one of these directly rather than have it derived from whatever queued leaves it left behind. Neither can pass, and `VerifyReport.Validate()` skips the tree-consistency check for a host-stamped state. + +## Progress streaming, end to end + +A running check can report where it has got to before it has a verdict, at up to one snapshot every `ProgressInterval` (`500ms`), with the last snapshot always flushed before the verdict: + +1. A `ProgressVerifier` (the `ExternalVerifier`, gavel's fixture engine) calls its progress sink. +2. `verify.Plugin` coalesces snapshots through a `progressEmitter` and fans each one out: onto the run's own event stream as `ai.EventVerifyProgress`, and into whatever extra sinks a caller registered via `verify.Options.Progress`. +3. `captain prompt run`'s stream republishes the newest snapshot as its own SSE `verify` event: `{"report": , "done": bool}` (`cli.VerifyFrame`). Only the latest is kept — a late subscriber gets the run's current verification state on connect, the same way it gets `run` and `state`. +4. The session-message model carries the same shape as a `data-verify` part (`session.PartVerify`) alongside the human-readable text of the same verdict — a renderer that knows the verification tree draws it, one that doesn't still has the text. +5. Once a turn is judged, its rolled-up report (see [Embedding a run](/agents/embedding/#result-final-report-and-merging)) is persisted as the `verification_result` `jsonb` column on `captain_prompt_run_iterations`. +6. The run's own final report rides on `result_json.verify` in `captain_prompt_runs`, alongside the prompt's own structured output. + +## `captain verify` — local only + +`captain verify` runs a workflow's checks against a working tree without spending a generation on it — the same checks the loop votes with, driven out of the generate→verify loop: + + + +| Flag | Meaning | +| --- | --- | +| `--fixture` | Fixture document run by the configured `verify.fixtureRunner` | +| `--command`, `-c` | Shell command run as a pass/fail check (repeatable) | +| `--prompt` | LLM-judge `.prompt` template, judged by the run's provider (repeatable) | +| `--cwd` | Directory the checks run in (default `.`) | +| `--timeout` | Wall-clock bound per check (default `10m`) | + +`captain verify` is **local only** — it is excluded from both the REST API (`clicky.MarkLocalOnly`) and the MCP tool set (`^verify` is in `mcp.ToolsConfig.Exclude`). The reason is in the command's own registration: `--command` is run through `sh -c` against a caller-chosen `--cwd`, so published as REST or MCP it would be unauthenticated remote code execution. diff --git a/docs/src/pages/prompts/sources-api.mdx b/docs/src/pages/prompts/sources-api.mdx index efa89a81..aeeecca2 100644 --- a/docs/src/pages/prompts/sources-api.mdx +++ b/docs/src/pages/prompts/sources-api.mdx @@ -57,6 +57,24 @@ Every summary and detail also carries `version` — a hash of the exact file con The schema document at `/api/captain/ai/prompt/schema` lists every discovered source under `sources` (`id`, `kind`, `label`, `root`, `writable`, `implicit`), so a save destination can be chosen even when the directory holds no prompts yet. +It also carries `verifiers`, one entry per `workflow.verify.*` kind, so the workbench can show whether a check it is about to author will ever run: + + + +`cmd` is always available — it is captain's own factory, registered at package init. `prompt` is available unless every probed runtime is unauthenticated (`reason` names that). `fixture` reflects whether the registry has a `fixture` factory — an in-process registration, or a configured `verify.fixtureRunner` — and its `reason`, when unavailable, is the exact sentence `verify.HooksFor` would refuse a run with (see [Verification](/agents/verification/)). + +When `verify.fixtureRunner` is configured, the document also carries `fixtureSchemas`: the raw JSON that runner prints for ` --schema`, so the fixture editor can complete a fixture document against that runner's own fence types. It is omitted when no runner is configured, and a runner that fails to answer `--schema` still leaves `fixture` available — the schema is advisory editor metadata, not a precondition for dispatch. + ## Render request Rendering accepts template variables and an optional runtime `spec` overlay. It may also carry `content`: an unsaved draft that is rendered (or run) in place of the saved file. A `spec.model` that merely echoes the saved prompt's own frontmatter is ignored for a draft, so a draft that changes `model:` runs with the draft's model; any other override is kept. It returns the rendered user/system prompt, full typed request input, config mirror, schema metadata, and a validation error field when the request is incomplete. @@ -82,3 +100,5 @@ Rendering accepts template variables and an optional runtime `spec` overlay. It ## Run stream The `run` action renders synchronously, creates a prompt run id, then streams session entries through `/api/captain/prompt/runs/{runId}/stream`. A snapshot endpoint at `/api/captain/prompt/runs/{runId}` lets the UI recover entries, completion state, summary, and error text. + +A workflow that declares `workflow.verify` also publishes its own `verify` SSE event: `{"report": , "done": boolean}`. Only the newest snapshot is kept — a late subscriber gets the run's current verification state on connect, the same way it gets `run` and `state` — and it stops updating once the run is done. See [Verification](/agents/verification/) for the report shape and how it is persisted. diff --git a/migrations/32_execution_approvals.pg.hcl b/migrations/32_execution_approvals.pg.hcl index 9b0ae8d3..89bdf23b 100644 --- a/migrations/32_execution_approvals.pg.hcl +++ b/migrations/32_execution_approvals.pg.hcl @@ -250,6 +250,6 @@ table "captain_turn_requests" { expr = "resolved_at IS NULL OR resolved_at >= created_at" } check "captain_turn_requests_tool_approval_identity" { - expr = "kind <> 'tool_approval' OR (prompt_run_id IS NOT NULL AND turn_id IS NOT NULL AND model_call_id IS NOT NULL AND tool_call_id IS NOT NULL)" + expr = "kind <> 'tool_approval' OR (prompt_run_id IS NOT NULL AND tool_call_id IS NOT NULL AND (credential_id IS NULL OR (turn_id IS NOT NULL AND model_call_id IS NOT NULL)))" } } diff --git a/migrations/74_turn_request_approval_identity.sql b/migrations/74_turn_request_approval_identity.sql index e453e82f..0ce6e178 100644 --- a/migrations/74_turn_request_approval_identity.sql +++ b/migrations/74_turn_request_approval_identity.sql @@ -1,5 +1,24 @@ -- phase: post +-- RETIRED: this script's strict four-column identity is superseded by +-- 81_turn_request_provider_approval_identity.sql, and what remains here is the +-- legacy caller-tool backfill it was written for. +-- +-- A script re-runs whenever its content hash changes or its ledger row is +-- dropped (a restore, a rebuilt environment, an edit to this file). In its +-- original form the re-run raised on any tool_approval row with a NULL turn_id +-- -- which is exactly the shape a credential-less provider approval has, and +-- exactly what 81 legitimises. One stored provider approval was therefore +-- enough to block every later Apply, and with it startup. +-- +-- Two changes retire it. The backfill and its ambiguity guard are scoped to +-- caller-tool rows (credential_id IS NOT NULL), so a re-run is a no-op for +-- provider rows. And the constraint it installs is now character-for-character +-- the one 81 installs, because a re-run of this script alone -- 81's hash is +-- unchanged, so 81 does not follow -- would otherwise leave the database +-- holding a superseded constraint that rejects every provider approval. The +-- final shape has to be reachable from either script on its own. + WITH candidates AS ( SELECT request.id AS request_id, @@ -15,6 +34,7 @@ WITH candidates AS ( ON turn.id = model_call.turn_id AND turn.session_id = request.session_id WHERE request.kind = 'tool_approval' + AND request.credential_id IS NOT NULL AND (request.turn_id IS NULL OR request.model_call_id IS NULL) ), unique_candidates AS ( SELECT request_id, model_call_id, turn_id @@ -36,6 +56,7 @@ BEGIN INTO invalid_ids FROM public.captain_turn_requests request WHERE request.kind = 'tool_approval' + AND request.credential_id IS NOT NULL AND ( request.prompt_run_id IS NULL OR request.turn_id IS NULL @@ -68,9 +89,11 @@ ALTER TABLE public.captain_turn_requests kind <> 'tool_approval' OR ( prompt_run_id IS NOT NULL - AND turn_id IS NOT NULL - AND model_call_id IS NOT NULL AND tool_call_id IS NOT NULL + AND ( + credential_id IS NULL + OR (turn_id IS NOT NULL AND model_call_id IS NOT NULL) + ) ) ) NOT VALID; diff --git a/migrations/81_turn_request_provider_approval_identity.sql b/migrations/81_turn_request_provider_approval_identity.sql new file mode 100644 index 00000000..ca013cb9 --- /dev/null +++ b/migrations/81_turn_request_provider_approval_identity.sql @@ -0,0 +1,41 @@ +-- phase: post + +-- A caller-tool approval is raised inside an aichat turn, so it always has a +-- captain_turns row and a captain_model_calls row to hang off. A provider +-- approval raised by `captain prompt run` -- or by an external host driving a +-- streaming provider -- has a session and a prompt run and nothing else: those +-- providers never open a turn or a model call. Demanding all four columns is +-- what kept the durable approval broker private to the aichat execution path. +-- +-- Identity is therefore conditional on the credential. The caller-tool path +-- (credential_id IS NOT NULL) keeps the full four-column identity that +-- 74_turn_request_approval_identity.sql backfilled and validated. The +-- credential-less provider path is identified by (prompt_run_id, tool_call_id) +-- alone, which is exactly what its "provider::" +-- idempotency key already keys on. +-- +-- 74 is retired to a caller-tool-only backfill and now installs this exact +-- constraint too. That duplication is deliberate: a script re-runs when its +-- content hash changes or its ledger row is dropped, and either script can +-- re-run without the other, so the final shape has to be what both of them +-- leave behind. Keep the two CHECK bodies identical. + +ALTER TABLE public.captain_turn_requests + DROP CONSTRAINT IF EXISTS captain_turn_requests_tool_approval_identity; + +ALTER TABLE public.captain_turn_requests + ADD CONSTRAINT captain_turn_requests_tool_approval_identity + CHECK ( + kind <> 'tool_approval' + OR ( + prompt_run_id IS NOT NULL + AND tool_call_id IS NOT NULL + AND ( + credential_id IS NULL + OR (turn_id IS NOT NULL AND model_call_id IS NOT NULL) + ) + ) + ) NOT VALID; + +ALTER TABLE public.captain_turn_requests + VALIDATE CONSTRAINT captain_turn_requests_tool_approval_identity; diff --git a/migrations/approval_identity_upgrade_integration_test.go b/migrations/approval_identity_upgrade_integration_test.go index 0266246e..0ff3eac5 100644 --- a/migrations/approval_identity_upgrade_integration_test.go +++ b/migrations/approval_identity_upgrade_integration_test.go @@ -13,7 +13,10 @@ import ( . "github.com/onsi/gomega" ) -const approvalIdentityMigration = "74_turn_request_approval_identity.sql" +const ( + approvalIdentityMigration = "74_turn_request_approval_identity.sql" + providerApprovalIdentityMigration = "81_turn_request_provider_approval_identity.sql" +) var _ = Describe("Tool approval identity migration", func() { It("backfills an unambiguous legacy approval and replaces the credential constraint", func(ctx SpecContext) { @@ -22,7 +25,7 @@ var _ = Describe("Tool approval identity migration", func() { Expect(Apply(ctx, dsn)).To(Succeed()) ids := seedLegacyToolApproval(ctx, db, 1) - Expect(resetApprovalIdentityMigration(ctx, db)).To(Succeed()) + Expect(resetMigrationScripts(ctx, db, approvalIdentityMigration)).To(Succeed()) Expect(Apply(ctx, dsn)).To(Succeed()) var turnID, modelCallID uuid.UUID @@ -34,28 +37,78 @@ var _ = Describe("Tool approval identity migration", func() { Expect(turnID).To(Equal(ids.turn)) Expect(modelCallID).To(Equal(ids.modelCalls[0])) - var definition string - Expect(db.QueryRowContext(ctx, ` - SELECT pg_get_constraintdef(oid) - FROM pg_constraint - WHERE conname = 'captain_turn_requests_tool_approval_identity' - `).Scan(&definition)).To(Succeed()) - Expect(definition).To(And( + // Identity is conditional on the credential: the caller-tool half keeps + // the four-column shape, and the provider half is identified by its + // prompt run and tool call alone. + Expect(approvalIdentityConstraint(ctx, db)).To(And( + ContainSubstring("credential_id IS NULL"), ContainSubstring("turn_id IS NOT NULL"), ContainSubstring("model_call_id IS NOT NULL"), Not(ContainSubstring("credential_id IS NOT NULL")), )) + Expect(insertProviderApproval(ctx, db, ids, "provider-call")).To(Succeed()) + Expect(Apply(ctx, dsn)).To(Succeed()) + }) + + It("accepts a provider approval with no credential, turn or model call", func(ctx SpecContext) { + handle := dbtest.ForGinkgo(dbtest.Options{Name: "captain_approval_identity_provider"}) + dsn, db := handle.DSN(), handle.SQL() + Expect(Apply(ctx, dsn)).To(Succeed()) + + ids := seedLegacyToolApproval(ctx, db, 1) + Expect(resetMigrationScripts(ctx, db, approvalIdentityMigration)).To(Succeed()) + Expect(Apply(ctx, dsn)).To(Succeed()) + + // This is the row `captain prompt run` writes: a session and a prompt run, + // and nothing else. Rejecting it is what kept the durable approval broker + // private to the aichat execution path. + Expect(insertProviderApproval(ctx, db, ids, "credential-less-call")).To(Succeed()) + }) + + It("still rejects a caller-tool approval that names no turn", func(ctx SpecContext) { + handle := dbtest.ForGinkgo(dbtest.Options{Name: "captain_approval_identity_caller"}) + dsn, db := handle.DSN(), handle.SQL() + Expect(Apply(ctx, dsn)).To(Succeed()) + + ids := seedLegacyToolApproval(ctx, db, 1) + Expect(resetMigrationScripts(ctx, db, approvalIdentityMigration)).To(Succeed()) + Expect(Apply(ctx, dsn)).To(Succeed()) + + // A credential means the approval was raised inside a turn, so the turn + // and model call are not optional: relaxing them for the provider path + // must not relax them here. _, err := db.ExecContext(ctx, ` INSERT INTO captain_turn_requests ( - id, session_id, turn_id, prompt_run_id, model_call_id, tool_call_id, + id, session_id, prompt_run_id, credential_id, tool_call_id, kind, request, idempotency_key, requested_by, expires_at - ) VALUES ($1, $2, $3, $4, $5, 'provider-call', 'tool_approval', - '{"tool":"accounts_edit","input":{}}', $6, 'provider', $7) - `, uuid.New(), ids.session, ids.turn, ids.promptRun, ids.modelCalls[0], - "provider:"+ids.promptRun.String()+":provider-call", time.Now().Add(time.Hour)) - Expect(err).NotTo(HaveOccurred()) + ) VALUES ($1, $2, $3, $4, 'turnless-caller-call', 'tool_approval', + '{"tool":"accounts_edit","input":{}}', $5, 'caller_tool', $6) + `, uuid.New(), ids.session, ids.promptRun, ids.credential, + "mcp:"+ids.credential.String()+":turnless-caller-call", time.Now().Add(time.Hour)) + Expect(err).To(MatchError(ContainSubstring("captain_turn_requests_tool_approval_identity"))) + }) + + It("re-applies the retired 74 and its successor with a provider approval already stored", func(ctx SpecContext) { + handle := dbtest.ForGinkgo(dbtest.Options{Name: "captain_approval_identity_rerun"}) + dsn, db := handle.DSN(), handle.SQL() + Expect(Apply(ctx, dsn)).To(Succeed()) + + ids := seedLegacyToolApproval(ctx, db, 1) + Expect(resetMigrationScripts(ctx, db, approvalIdentityMigration)).To(Succeed()) + Expect(Apply(ctx, dsn)).To(Succeed()) + Expect(insertProviderApproval(ctx, db, ids, "provider-call")).To(Succeed()) + + // A content-hash change, a ledger reset, a restore: any of them re-runs + // both scripts. 74's original guard read this NULL-turn provider row as + // an ambiguous legacy row and raised, so one stored provider approval + // blocked startup forever after. + Expect(resetMigrationScripts(ctx, db, + approvalIdentityMigration, providerApprovalIdentityMigration)).To(Succeed()) Expect(Apply(ctx, dsn)).To(Succeed()) + + Expect(approvalIdentityConstraint(ctx, db)).To(ContainSubstring("credential_id IS NULL"), + "81 runs after 74 and reinstates the final shape") }) It("fails when a legacy approval cannot be correlated to one model call", func(ctx SpecContext) { @@ -64,7 +117,7 @@ var _ = Describe("Tool approval identity migration", func() { Expect(Apply(ctx, dsn)).To(Succeed()) ids := seedLegacyToolApproval(ctx, db, 2) - Expect(resetApprovalIdentityMigration(ctx, db)).To(Succeed()) + Expect(resetMigrationScripts(ctx, db, approvalIdentityMigration)).To(Succeed()) err := Apply(ctx, dsn) Expect(err).To(MatchError(And( ContainSubstring("ambiguous legacy tool approval identity"), @@ -91,9 +144,35 @@ type legacyApprovalIDs struct { turn uuid.UUID promptRun uuid.UUID request uuid.UUID + credential uuid.UUID modelCalls []uuid.UUID } +// insertProviderApproval writes the row a credential-less provider run raises: +// a session and a prompt run, with no credential, turn or model call. +func insertProviderApproval(ctx context.Context, db *sql.DB, ids legacyApprovalIDs, toolCallID string) error { + _, err := db.ExecContext(ctx, ` + INSERT INTO captain_turn_requests ( + id, session_id, prompt_run_id, tool_call_id, + kind, request, idempotency_key, requested_by, expires_at + ) VALUES ($1, $2, $3, $4, 'tool_approval', + '{"tool":"accounts_edit","input":{}}', $5, 'provider', $6) + `, uuid.New(), ids.session, ids.promptRun, toolCallID, + "provider:"+ids.promptRun.String()+":"+toolCallID, time.Now().Add(time.Hour)) + return err +} + +func approvalIdentityConstraint(ctx context.Context, db *sql.DB) string { + GinkgoHelper() + var definition string + Expect(db.QueryRowContext(ctx, ` + SELECT pg_get_constraintdef(oid) + FROM pg_constraint + WHERE conname = 'captain_turn_requests_tool_approval_identity' + `).Scan(&definition)).To(Succeed()) + return definition +} + func seedLegacyToolApproval(ctx context.Context, db *sql.DB, modelCallCount int) legacyApprovalIDs { ids := legacyApprovalIDs{ session: uuid.New(), turn: uuid.New(), promptRun: uuid.New(), request: uuid.New(), @@ -125,6 +204,7 @@ func seedLegacyToolApproval(ctx context.Context, db *sql.DB, modelCallCount int) } credentialID := uuid.New() + ids.credential = credentialID _, err = db.ExecContext(ctx, ` INSERT INTO captain_session_mcp_credentials ( id, session_id, prompt_run_id, provider, mode, secret_hash, policy, expires_at @@ -160,20 +240,25 @@ func installLegacyApprovalConstraint(ctx context.Context, db *sql.DB) error { return err } -func resetApprovalIdentityMigration(ctx context.Context, db *sql.DB) error { - result, err := db.ExecContext(ctx, ` - DELETE FROM schema_migration_scripts - WHERE scope = $1 AND path = $2 - `, Scope, approvalIdentityMigration) - if err != nil { - return err - } - affected, err := result.RowsAffected() - if err != nil { - return fmt.Errorf("read reset migration result: %w", err) - } - if affected != 1 { - return fmt.Errorf("reset migration ledger: deleted %d rows, want 1", affected) +// resetMigrationScripts forgets the named scripts' recorded hashes so the next +// Apply re-runs them — the ledger state a restore, a rebuilt environment or a +// content-hash change puts a database in. +func resetMigrationScripts(ctx context.Context, db *sql.DB, paths ...string) error { + for _, path := range paths { + result, err := db.ExecContext(ctx, ` + DELETE FROM schema_migration_scripts + WHERE scope = $1 AND path = $2 + `, Scope, path) + if err != nil { + return err + } + affected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("read reset migration result for %s: %w", path, err) + } + if affected != 1 { + return fmt.Errorf("reset migration ledger for %s: deleted %d rows, want 1", path, affected) + } } return nil } diff --git a/migrations/migrations.go b/migrations/migrations.go index 8b1e47c8..a85ccd8d 100644 --- a/migrations/migrations.go +++ b/migrations/migrations.go @@ -162,14 +162,11 @@ func verifyToolApprovalIdentity(ctx context.Context, request applyRequest) (resu if err != nil { return fmt.Errorf("read captain_turn_requests_tool_approval_identity: %w", err) } - normalized := strings.ToLower(strings.Join(strings.Fields(definition), " ")) + normalized := flattenConstraintDefinition(definition) if !validated || strings.Contains(normalized, "credential_id is not null") { return fmt.Errorf("captain_turn_requests_tool_approval_identity is invalid: %s", definition) } - for _, required := range []string{ - "prompt_run_id is not null", "turn_id is not null", - "model_call_id is not null", "tool_call_id is not null", - } { + for _, required := range approvalIdentityFragments { if !strings.Contains(normalized, required) { return fmt.Errorf("captain_turn_requests_tool_approval_identity omits %q: %s", required, definition) } @@ -177,6 +174,32 @@ func verifyToolApprovalIdentity(ctx context.Context, request applyRequest) (resu return nil } +// approvalIdentityFragments are what the tool-approval identity constraint must +// say, as 81_turn_request_provider_approval_identity.sql leaves it. +// +// The third fragment is the whole point of that migration and the one this +// check exists to defend: identity is conditional on the credential, so a +// credential-less provider approval — `captain prompt run`, or an external host +// driving a streaming provider, neither of which ever opens a turn or a model +// call — is identified by its prompt run and tool call alone. A definition that +// demands turn_id and model_call_id unconditionally is 74's retired shape, and +// finding it here means the database drifted back to a constraint that rejects +// every provider approval the broker writes. +var approvalIdentityFragments = []string{ + "prompt_run_id is not null", + "tool_call_id is not null", + "credential_id is null or turn_id is not null and model_call_id is not null", +} + +// flattenConstraintDefinition renders pg_get_constraintdef's output as one +// lower-case line with its parenthesis nesting dropped, so a fragment can be +// matched without reproducing exactly how PostgreSQL chose to bracket the +// expression it deparsed. +func flattenConstraintDefinition(definition string) string { + unbracketed := strings.NewReplacer("(", " ", ")", " ").Replace(definition) + return strings.ToLower(strings.Join(strings.Fields(unbracketed), " ")) +} + type migrationLock struct { db *sql.DB conn *sql.Conn diff --git a/migrations/schema_ginkgo_test.go b/migrations/schema_ginkgo_test.go index bde5c12a..d7dd7e46 100644 --- a/migrations/schema_ginkgo_test.go +++ b/migrations/schema_ginkgo_test.go @@ -8,6 +8,18 @@ import ( . "github.com/onsi/gomega" ) +// addedCheckConstraint extracts the CHECK body of the script's +// ADD CONSTRAINT ... statement, with whitespace collapsed so the two files are +// compared on what they say rather than how they are indented. +func addedCheckConstraint(script, name string) string { + GinkgoHelper() + _, after, found := strings.Cut(script, "ADD CONSTRAINT captain_turn_requests_tool_approval_identity") + Expect(found).To(BeTrue(), "%s no longer adds the constraint", name) + body, _, found := strings.Cut(after, ") NOT VALID;") + Expect(found).To(BeTrue(), "%s no longer ends the constraint with ) NOT VALID;", name) + return strings.Join(strings.Fields(body), " ") +} + var _ = Describe("schema-scoped Captain migrations", func() { It("leaves the public bundle unchanged", func() { filesystem, err := schemaFilesystem(DefaultSchema) @@ -54,6 +66,27 @@ var _ = Describe("schema-scoped Captain migrations", func() { Expect(err).To(HaveOccurred()) }) + // 74 is retired but still installs the tool-approval identity constraint, + // because either script can re-run without the other (a content-hash change, + // a dropped ledger row) and whichever runs last decides the shape the + // database ends up with. They only agree by staying byte-identical. + It("installs one tool-approval identity constraint from both 74 and 81", func() { + bodies := map[string]string{} + for _, name := range []string{ + "74_turn_request_approval_identity.sql", + "81_turn_request_provider_approval_identity.sql", + } { + content, err := schemaFS.ReadFile(name) + Expect(err).NotTo(HaveOccurred()) + bodies[name] = addedCheckConstraint(string(content), name) + } + Expect(bodies["74_turn_request_approval_identity.sql"]). + To(Equal(bodies["81_turn_request_provider_approval_identity.sql"])) + for name, body := range bodies { + Expect(body).To(ContainSubstring("credential_id IS NULL"), name) + } + }) + It("uses a stable schema-specific advisory lock", func() { Expect(migrationLockKey(DefaultSchema)).To(Equal(captainMigrationLockKey)) Expect(migrationLockKey("agent_namespace_one")).To(Equal(migrationLockKey("agent_namespace_one"))) diff --git a/pkg/ai/agent/agent_suite_test.go b/pkg/ai/agent/agent_suite_test.go new file mode 100644 index 00000000..0bcc984f --- /dev/null +++ b/pkg/ai/agent/agent_suite_test.go @@ -0,0 +1,13 @@ +package agent + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAgentSuite(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/ai/agent") +} diff --git a/pkg/ai/agent/commit/workflow.go b/pkg/ai/agent/commit/workflow.go index 1a7651ca..e29cc66b 100644 --- a/pkg/ai/agent/commit/workflow.go +++ b/pkg/ai/agent/commit/workflow.go @@ -6,7 +6,7 @@ import "github.com/flanksource/captain/pkg/api" // per declared policy, in declaration order. Returns nil when the workflow // commits nothing, which is the default: captain never commits unless asked to. // -// Sibling of verify.HooksForWorkflow, and returns []any for the same reason — +// Sibling of verify.HooksFor, and returns []any for the same reason — // the runner's hook list is heterogeneous. Register these ahead of the worktree // plugin so a run-phase commit is cut while the worktree is still live, and // therefore has something for the merge to take. diff --git a/pkg/ai/agent/hook_context.go b/pkg/ai/agent/hook_context.go new file mode 100644 index 00000000..2572b6ed --- /dev/null +++ b/pkg/ai/agent/hook_context.go @@ -0,0 +1,128 @@ +package agent + +import ( + "context" + "fmt" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" +) + +// HookContext carries what hooks read/mutate: the request for this iteration and +// the accumulating response, whose Workspace holds the run's working-dir state. +type HookContext struct { + context.Context + // Request is the CURRENT request: PreRun hooks rewrite it (setup replaces the + // checkout it performed with where that landed), and the runner replaces it + // wholesale with each verify-driven retry. + Request *ai.Request + // Original is the request the run started from, before any hook rewrote it. + // It is the only place a Post hook can read what the run was *asked* to do — + // which repo to check out, which branch to isolate on — while Request says + // where that ended up. Cloned once by Run, so mutating it affects nothing. + Original ai.Request + Response *ai.Response + // Iteration is the loop's 0-BASED index of the turn currently in scope: the + // turn about to run, or — while verify hooks vote and PhaseTurn dispatches — + // the one that just completed. A hook reporting the turn to a person or to + // the iteration store adds one (see VerifyResult.Iteration), which is 1-based. + // + // It only ever names a turn that actually executes; the loop's final + // BuildRequest call, the one that ends the run, does not advance it. + Iteration int + Scope Scope + + // Hooks is the run's hook list, so a hook can detect an incompatible peer + // rather than silently competing with it (see EnsureSingleIsolator). + Hooks []any + + // Phase is the boundary currently being dispatched; it is only meaningful + // inside a Post hook, which also receives it as an argument. + Phase Phase + // Turn is the loop iteration that just completed. Set only during PhaseTurn; + // nil everywhere else. + Turn *ai.LoopIteration + + // Verified and Failed describe the run's outcome so PhaseAgent/PhaseRun + // hooks (e.g. the worktree merge/cleanup gate) can act on it. Runner.Run + // sets both right after the loop ends; they are meaningless beforehand, and + // in particular are not yet settled during PhaseTurn. + // + // Verified mirrors VerifyPassed(result.Verdicts) — true when the last verify + // verdict passed, or trivially true when no Verify hooks ran at all. + Verified bool + // Failed is true when the generate/verify run itself returned an error + // (a provider failure, not a failing verdict). + Failed bool + + // emit publishes an event on the run's stream, so what a hook does between + // turns reaches the same renderers as what the model does during them. Set by + // Runner.Run; nil in a hand-built context, which Notify tolerates. + emit func(ai.Event) +} + +// Notify reports one thing this hook did, in the run's own voice. It reaches the +// live stream as an ai.EventSystem and is buffered on the workspace with its +// timestamp, so a caller can persist it into the transcript once the run's +// session id is known — a hook firing mid-turn cannot know it yet. +// +// Purely informational: a hook that failed returns an error, it does not Notify. +func (hc *HookContext) Notify(format string, args ...any) { + hc.NotifyEvent(ai.Event{Kind: ai.EventSystem, Text: fmt.Sprintf(format, args...)}) +} + +// NotifyEvent is Notify for a hook whose report has an event kind of its own and +// structured fields to go with it — a verify verdict is not a generic system +// line, and a renderer that must colour a pass and a failure differently, or a +// dashboard that filters on them, cannot recover that from the text. +// +// ev.Text is the human-readable report and is what the notice records; a typed +// verify report on ev.Raw rides along on the notice so a stored transcript +// carries the tree the live stream drew rather than only its headline. The +// remaining fields travel on the live stream only. An event with no text records +// nothing: a notice exists to be read. +func (hc *HookContext) NotifyEvent(ev ai.Event) { + if ev.Text == "" { + return + } + if ev.Kind == "" { + ev.Kind = ai.EventSystem + } + notice := api.Notice{At: time.Now(), Phase: string(hc.Phase), Text: ev.Text, Kind: ev.Kind} + if report, ok := ev.Raw.(*api.VerifyReport); ok { + notice.Report = report + } + hc.Workspace().AddNoticeRecord(notice) + hc.Emit(ev) +} + +// Emit publishes one event on the run's live stream and records nothing. It is +// for the reports that are true only while they are being read — a verifier's +// in-flight progress, redrawn in place by whatever is watching — as opposed to +// the ones a reader should still find in the transcript afterwards. +// +// Routing progress through NotifyEvent wrote every coalesced snapshot into the +// workspace's notices and from there into the persisted transcript, so a long +// check buried its own verdict under a stack of superseded counts. Such an event +// needs no Text: it carries the structure a renderer draws, not prose about it. +// +// nil-safe: a hand-built HookContext has no stream, and emitting into it is a +// no-op rather than a panic. +func (hc *HookContext) Emit(ev ai.Event) { + if ev.Kind == "" { + ev.Kind = ai.EventSystem + } + if hc.emit != nil { + hc.emit(ev) + } +} + +// Workspace returns the run's working-dir state, allocating it if needed (so it +// is never nil for a hook to read/mutate). +func (hc *HookContext) Workspace() *api.Workspace { + if hc.Response.Workspace == nil { + hc.Response.Workspace = &api.Workspace{} + } + return hc.Response.Workspace +} diff --git a/pkg/ai/agent/hooks.go b/pkg/ai/agent/hooks.go new file mode 100644 index 00000000..472d7fc5 --- /dev/null +++ b/pkg/ai/agent/hooks.go @@ -0,0 +1,78 @@ +package agent + +import ( + "fmt" + "strings" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" +) + +// WorkspaceIsolator is implemented by a hook that relocates the run into its own +// working tree — a git worktree from Spec.Setup.Checkout, or the `wt` worktree +// plugin. Two of them in one run create two trees and use one, so each declares +// itself here and calls EnsureSingleIsolator before acting. +type WorkspaceIsolator interface { + Name() string + // IsolatesWorkspace reports whether this hook will relocate the run given + // hc's request; a hook whose isolation is configured on the spec answers + // from hc.Request rather than from its own fields. + IsolatesWorkspace(*HookContext) bool +} + +// EnsureSingleIsolator fails when more than one registered hook would relocate +// the run into its own working tree. Silently creating two and working in one is +// the failure this exists to prevent: the run edits a tree nobody merges. +func (hc *HookContext) EnsureSingleIsolator() error { + var names []string + for _, h := range hc.Hooks { + if iso, ok := h.(WorkspaceIsolator); ok && iso.IsolatesWorkspace(hc) { + names = append(names, iso.Name()) + } + } + if len(names) > 1 { + return fmt.Errorf("agent: hooks %s each isolate the run in their own working tree; register exactly one", strings.Join(names, " and ")) + } + return nil +} + +// VerifyResult is a Verify hook's judgement on an iteration. When !Valid, Retry +// (if non-nil) is the exact next request to run — the hook bakes its feedback +// into that request's prompt. Report is the typed verdict every verifier +// produces (what captain persists per iteration and the webapp renders); +// Iteration is the loop iteration it judged. +type VerifyResult struct { + Valid bool + Retry *ai.Request + Report *api.VerifyReport + Iteration int +} + +// PreRun runs once before the loop. +type PreRun interface { + Name() string + PreRun(*HookContext) error +} + +// Verify runs after each completed iteration and votes. +type Verify interface { + Name() string + Verify(*HookContext) (VerifyResult, error) +} + +// Post runs after a lifecycle phase completes (commit, teardown, checkpointing). +// A hook declares which phases it handles via Phases(); the runner dispatches +// only those, and always — including on the failure path, so a hook can make +// work durable or tear down after an error. +type Post interface { + Name() string + Phases() []Phase + Post(*HookContext, Phase) error +} + +// Output produces the workflow's typed final result. Optional; when absent the +// runner leaves Result.Output at its zero value. +type Output[T any] interface { + Name() string + Output(*HookContext) (T, error) +} diff --git a/pkg/ai/agent/runner.go b/pkg/ai/agent/runner.go index 3cbad8a0..5d7a1660 100644 --- a/pkg/ai/agent/runner.go +++ b/pkg/ai/agent/runner.go @@ -16,7 +16,6 @@ import ( "errors" "fmt" "strings" - "time" "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/ai/history" @@ -29,183 +28,9 @@ import ( // fallbackLog is used when ctx carries no task-scoped logger (ai.ContextWithLogger). var fallbackLog = logger.GetLogger("agent") -// Scope controls how much hooks act on. ScopeChanged restricts them to the files -// the agent edited; ScopeAll lets each act on the whole tree. -type Scope string - -const ( - ScopeChanged Scope = "changed" - ScopeAll Scope = "all" -) - -// AllScopes lists every scope in canonical order. -func AllScopes() []Scope { return []Scope{ScopeAll, ScopeChanged} } - -// Valid reports whether s is one of the supported scopes. -func (s Scope) Valid() bool { - for _, x := range AllScopes() { - if s == x { - return true - } - } - return false -} - -// ScopeList renders the supported scopes as a comma-separated string. -func ScopeList() string { - parts := make([]string, len(AllScopes())) - for i, s := range AllScopes() { - parts[i] = string(s) - } - return strings.Join(parts, ", ") -} - -// ParseScope resolves a CLI/flag value into a Scope, defaulting empty to ScopeAll. -func ParseScope(s string) (Scope, error) { - switch Scope(s) { - case "", ScopeAll: - return ScopeAll, nil - case ScopeChanged: - return ScopeChanged, nil - default: - return "", fmt.Errorf("invalid --scope %q (valid: %s)", s, ScopeList()) - } -} - -// HookContext carries what hooks read/mutate: the request for this iteration and -// the accumulating response, whose Workspace holds the run's working-dir state. -type HookContext struct { - context.Context - // Request is the CURRENT request: PreRun hooks rewrite it (setup replaces the - // checkout it performed with where that landed), and the runner replaces it - // wholesale with each verify-driven retry. - Request *ai.Request - // Original is the request the run started from, before any hook rewrote it. - // It is the only place a Post hook can read what the run was *asked* to do — - // which repo to check out, which branch to isolate on — while Request says - // where that ended up. Cloned once by Run, so mutating it affects nothing. - Original ai.Request - Response *ai.Response - Iteration int - Scope Scope - - // Hooks is the run's hook list, so a hook can detect an incompatible peer - // rather than silently competing with it (see EnsureSingleIsolator). - Hooks []any - - // Phase is the boundary currently being dispatched; it is only meaningful - // inside a Post hook, which also receives it as an argument. - Phase Phase - // Turn is the loop iteration that just completed. Set only during PhaseTurn; - // nil everywhere else. - Turn *ai.LoopIteration - - // Verified and Failed describe the run's outcome so PhaseAgent/PhaseRun - // hooks (e.g. the worktree merge/cleanup gate) can act on it. Runner.Run - // sets both right after the loop ends; they are meaningless beforehand, and - // in particular are not yet settled during PhaseTurn. - // - // Verified mirrors VerifyPassed(result.Verdicts) — true when the last verify - // verdict passed, or trivially true when no Verify hooks ran at all. - Verified bool - // Failed is true when the generate/verify run itself returned an error - // (a provider failure, not a failing verdict). - Failed bool - - // emit publishes an event on the run's stream, so what a hook does between - // turns reaches the same renderers as what the model does during them. Set by - // Runner.Run; nil in a hand-built context, which Notify tolerates. - emit func(ai.Event) -} - -// Notify reports one thing this hook did, in the run's own voice. It reaches the -// live stream as an ai.EventSystem and is buffered on the workspace with its -// timestamp, so a caller can persist it into the transcript once the run's -// session id is known — a hook firing mid-turn cannot know it yet. -// -// Purely informational: a hook that failed returns an error, it does not Notify. -func (hc *HookContext) Notify(format string, args ...any) { - text := fmt.Sprintf(format, args...) - hc.Workspace().AddNotice(time.Now(), string(hc.Phase), text) - if hc.emit != nil { - hc.emit(ai.Event{Kind: ai.EventSystem, Text: text}) - } -} - -// Workspace returns the run's working-dir state, allocating it if needed (so it -// is never nil for a hook to read/mutate). -func (hc *HookContext) Workspace() *api.Workspace { - if hc.Response.Workspace == nil { - hc.Response.Workspace = &api.Workspace{} - } - return hc.Response.Workspace -} - -// WorkspaceIsolator is implemented by a hook that relocates the run into its own -// working tree — a git worktree from Spec.Setup.Checkout, or the `wt` worktree -// plugin. Two of them in one run create two trees and use one, so each declares -// itself here and calls EnsureSingleIsolator before acting. -type WorkspaceIsolator interface { - Name() string - // IsolatesWorkspace reports whether this hook will relocate the run given - // hc's request; a hook whose isolation is configured on the spec answers - // from hc.Request rather than from its own fields. - IsolatesWorkspace(*HookContext) bool -} - -// EnsureSingleIsolator fails when more than one registered hook would relocate -// the run into its own working tree. Silently creating two and working in one is -// the failure this exists to prevent: the run edits a tree nobody merges. -func (hc *HookContext) EnsureSingleIsolator() error { - var names []string - for _, h := range hc.Hooks { - if iso, ok := h.(WorkspaceIsolator); ok && iso.IsolatesWorkspace(hc) { - names = append(names, iso.Name()) - } - } - if len(names) > 1 { - return fmt.Errorf("agent: hooks %s each isolate the run in their own working tree; register exactly one", strings.Join(names, " and ")) - } - return nil -} - -// VerifyResult is a Verify hook's judgement on an iteration. When !Valid, Retry -// (if non-nil) is the exact next request to run — the hook bakes its feedback -// into that request's prompt. Output carries structured verify output. -type VerifyResult struct { - Valid bool - Retry *ai.Request - Output any -} - -// PreRun runs once before the loop. -type PreRun interface { - Name() string - PreRun(*HookContext) error -} - -// Verify runs after each completed iteration and votes. -type Verify interface { - Name() string - Verify(*HookContext) (VerifyResult, error) -} - -// Post runs after a lifecycle phase completes (commit, teardown, checkpointing). -// A hook declares which phases it handles via Phases(); the runner dispatches -// only those, and always — including on the failure path, so a hook can make -// work durable or tear down after an error. -type Post interface { - Name() string - Phases() []Phase - Post(*HookContext, Phase) error -} - -// Output produces the workflow's typed final result. Optional; when absent the -// runner leaves Result.Output at its zero value. -type Output[T any] interface { - Name() string - Output(*HookContext) (T, error) -} +// HookContext and its Notify/Emit/Workspace methods live in hook_context.go; +// Scope lives in scope.go; the hook interfaces (PreRun, Verify, Post, Output), +// VerifyResult and WorkspaceIsolator live in hooks.go. // Runner drives a generate→verify loop composed of hooks, producing a typed // result T. Hooks is a heterogeneous list; each element may implement any of @@ -236,6 +61,9 @@ type Result[T any] struct { // Verify hooks runs generate-only. func (r *Runner[T]) Run(ctx context.Context) (Result[T], error) { var zero Result[T] + if err := r.Request.ValidateRunnable(); err != nil { + return zero, fmt.Errorf("agent: %w", err) + } // The spec's budget.timeout bounds the whole run, not one model call. Only // pkg/cli applied it before, so every caller driving the Runner directly // (gavel's `pr status --ai-fix`) ran unbounded and a wedged turn could hang @@ -286,7 +114,11 @@ func (r *Runner[T]) Run(ctx context.Context) (Result[T], error) { } } - verifyOnly := strings.TrimSpace(r.Request.Prompt.User) == "" + // One rule, shared with every caller: api.Spec.IsVerifyOnly. Deciding here + // with a second test of its own ("is the user prompt blank?") is how a request + // carrying attachments or a message history got a provider built for it and + // then never generated — reporting a pass with nothing verified. + verifyOnly := r.Request.IsVerifyOnly() var runErr error if verifyOnly { runErr = r.runVerifyOnce(hc, &result) @@ -297,7 +129,7 @@ func (r *Runner[T]) Run(ctx context.Context) (Result[T], error) { } hc.Failed = runErr != nil - hc.Verified = verifyPassed(result.Verdicts) + hc.Verified = VerifyPassed(result.Verdicts) // PhaseAgent closes out the loop while the working tree is still live (the // last point a hook can commit an isolated worktree); PhaseRun then tears it // down. Both run even when the loop failed — that is what makes a failed @@ -416,7 +248,14 @@ func (r *Runner[T]) runLoop(ctx context.Context, hc *HookContext, result *Result } req = *retry } - hc.Iteration = iter + // RunUntil calls BuildRequest once more after the final executed turn + // and only then notices it is out of iterations, so advancing the + // index here unconditionally left the run — and every PhaseAgent / + // PhaseRun hook and event after it — attributed to a turn that never + // ran. Name the turn only when it is really about to execute. + if iter < maxIter { + hc.Iteration = iter + } hc.Request = &req // A hook that relocated the run recorded where on the workspace; // propagate it, because a verify-driven retry is built fresh and @@ -445,17 +284,19 @@ func (r *Runner[T]) runLoop(ctx context.Context, hc *HookContext, result *Result return verifyErr } -// verifyPassed reports whether the run's last verify verdict passed, or -// trivially true when no Verify hooks ran at all. Runner.Run uses it to set -// HookContext.Verified for Post hooks; mirrors pkg/cli's own verifyPassed, -// which summarizes the same Result.Verdicts for CLI output. +// VerifyPassed reports whether the run's last verify verdict passed, or +// trivially true when no Verify hooks ran at all. It is the single definition of +// "did this run verify": Runner.Run sets HookContext.Verified from it, and +// promptrun.Passed is it — a caller reading Result.Verdicts must never re-derive +// the rule, because a second copy is free to drift into calling a failed run +// green. // // Reading only the last verdict is sound because verify() stops a round at // its first failure: within any round a failure is that round's final // verdict, so the last entry of the accumulated list is always the final // round's outcome. Earlier invalid entries are the history of rounds whose // feedback drove a retry, not the run's result. -func verifyPassed(verdicts []VerifyResult) bool { +func VerifyPassed(verdicts []VerifyResult) bool { if len(verdicts) == 0 { return true } @@ -465,7 +306,7 @@ func verifyPassed(verdicts []VerifyResult) bool { // verify runs the Verify hooks in declaration order and stops at the first // failure (issue #40 R5.1). Stopping is not just economy — a cheap failing // gate short-circuits the expensive judges behind it — it is what keeps -// verifyPassed's last-verdict read correct: continuing past a failure lets a +// VerifyPassed's last-verdict read correct: continuing past a failure lets a // later passing hook become the round's final verdict and mask the failure. // retry is the failing hook's proposed next request. func (r *Runner[T]) verify(hc *HookContext) (verdicts []VerifyResult, retry *ai.Request, allValid bool, err error) { diff --git a/pkg/ai/agent/runner_test.go b/pkg/ai/agent/runner_test.go index c9aebb9c..5889ce72 100644 --- a/pkg/ai/agent/runner_test.go +++ b/pkg/ai/agent/runner_test.go @@ -233,8 +233,10 @@ func TestRunner_TurnPhaseFiresPerTurnBeforeVerifiers(t *testing.T) { func TestRunner_VerifyOnlyEmitsNoTurnPhase(t *testing.T) { var log []string - // Empty prompt body ⇒ verify-only: nothing generated, so no turn boundary. + // No prompt body + a declared verification ⇒ verify-only: nothing generated, + // so no turn boundary. r := &Runner[string]{ + Request: ai.Request{Workflow: &api.Workflow{Verify: &api.Verify{}}}, Hooks: []any{ &lifecycleHook{log: &log}, verifyHook{name: "score", fn: func(*HookContext) (VerifyResult, error) { @@ -413,9 +415,11 @@ func TestRunner_RunPhaseSeesVerifiedAndFailed(t *testing.T) { func TestRunner_VerifyOnlySkipsGeneration(t *testing.T) { var verifyCalls int - // Empty prompt body ⇒ verify-only: no provider, no generation loop. + // No prompt body + a declared verification ⇒ verify-only: no provider, no + // generation loop. r := &Runner[string]{ - Scope: ScopeAll, + Request: ai.Request{Workflow: &api.Workflow{Verify: &api.Verify{}}}, + Scope: ScopeAll, Hooks: []any{ verifyHook{name: "score", fn: func(*HookContext) (VerifyResult, error) { verifyCalls++ diff --git a/pkg/ai/agent/runner_verify_only_ginkgo_test.go b/pkg/ai/agent/runner_verify_only_ginkgo_test.go new file mode 100644 index 00000000..fb98c922 --- /dev/null +++ b/pkg/ai/agent/runner_verify_only_ginkgo_test.go @@ -0,0 +1,80 @@ +package agent + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" +) + +// countingVerifier votes once and records that it was asked. +type countingVerifier struct { + name string + calls *int + valid bool +} + +func (v *countingVerifier) Name() string { return v.name } +func (v *countingVerifier) Verify(hc *HookContext) (VerifyResult, error) { + *v.calls++ + report := api.NewNodeReport(api.VerifyKindFunc, v.name, api.VerifyNode{ + Name: v.name, Passed: v.valid, Failed: !v.valid, + }) + report.Iteration = hc.Iteration + 1 + return VerifyResult{Valid: v.valid, Report: &report, Iteration: report.Iteration}, nil +} + +var _ = Describe("Runner: what makes a run verify-only", func() { + // The runner used to decide with its own test — "is the user prompt blank?" — + // while every caller decided with Spec.IsVerifyOnly. The two disagreed on a + // request carrying attachments or a message history and no user text: the + // caller built a provider for a generating run, the runner skipped generation, + // no Verify hook voted, and the run reported a pass having done nothing. + It("takes the decision from the request, not from the prompt body", func() { + calls := 0 + provider := &fakeProvider{events: func(int) []ai.Event { + return []ai.Event{{Kind: ai.EventResult, Success: true}} + }} + req := ai.Request{Workflow: &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}}}} + Expect(req.IsVerifyOnly()).To(BeTrue()) + + r := &Runner[string]{Provider: provider, Request: req, Hooks: []any{&countingVerifier{name: "check", calls: &calls, valid: true}}} + res, err := r.Run(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(provider.calls).To(BeZero(), "a verify-only run never generates") + Expect(calls).To(Equal(1)) + Expect(res.Loop).To(BeNil()) + }) + + DescribeTable("refuses a request that neither generates nor verifies", + func(mutate func(*ai.Request)) { + req := ai.Request{} + mutate(&req) + provider := &fakeProvider{events: func(int) []ai.Event { return nil }} + + _, err := (&Runner[string]{Provider: provider, Request: req}).Run(context.Background()) + Expect(err).To(MatchError(ContainSubstring("workflow.verify"))) + Expect(provider.calls).To(BeZero(), "the refusal comes before the first model call") + }, + Entry("nothing at all", func(*ai.Request) {}), + Entry("attachments but no prompt and no verification", func(r *ai.Request) { + r.Prompt.Attachments = []api.AttachmentRef{{Path: "invoice.pdf"}} + }), + Entry("a message history but no prompt and no verification", func(r *ai.Request) { + r.Messages = []api.Message{{Role: api.RoleUser, Parts: []api.Part{{Type: api.PartText, Text: "hi"}}}} + }), + ) + + It("still generates for a prompt whose body is only whitespace-free text", func() { + provider := &fakeProvider{events: func(int) []ai.Event { + return []ai.Event{{Kind: ai.EventResult, Success: true}} + }} + r := &Runner[string]{Provider: provider, Request: ai.Request{Prompt: api.Prompt{User: "fix it"}}} + _, err := r.Run(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(provider.calls).To(Equal(1)) + }) +}) diff --git a/pkg/ai/agent/scope.go b/pkg/ai/agent/scope.go new file mode 100644 index 00000000..9fe10ced --- /dev/null +++ b/pkg/ai/agent/scope.go @@ -0,0 +1,49 @@ +package agent + +import ( + "fmt" + "strings" +) + +// Scope controls how much hooks act on. ScopeChanged restricts them to the files +// the agent edited; ScopeAll lets each act on the whole tree. +type Scope string + +const ( + ScopeChanged Scope = "changed" + ScopeAll Scope = "all" +) + +// AllScopes lists every scope in canonical order. +func AllScopes() []Scope { return []Scope{ScopeAll, ScopeChanged} } + +// Valid reports whether s is one of the supported scopes. +func (s Scope) Valid() bool { + for _, x := range AllScopes() { + if s == x { + return true + } + } + return false +} + +// ScopeList renders the supported scopes as a comma-separated string. +func ScopeList() string { + parts := make([]string, len(AllScopes())) + for i, s := range AllScopes() { + parts[i] = string(s) + } + return strings.Join(parts, ", ") +} + +// ParseScope resolves a CLI/flag value into a Scope, defaulting empty to ScopeAll. +func ParseScope(s string) (Scope, error) { + switch Scope(s) { + case "", ScopeAll: + return ScopeAll, nil + case ScopeChanged: + return ScopeChanged, nil + default: + return "", fmt.Errorf("invalid --scope %q (valid: %s)", s, ScopeList()) + } +} diff --git a/pkg/ai/agent/verify/exec.go b/pkg/ai/agent/verify/exec.go new file mode 100644 index 00000000..df2a8ace --- /dev/null +++ b/pkg/ai/agent/verify/exec.go @@ -0,0 +1,184 @@ +package verify + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync" + "syscall" + "time" +) + +// DefaultCmdTimeout bounds a verify command that declares no timeout of its +// own. A hook with no bound is a denial-of-service against whatever is waiting +// on the verdict — locally a stuck loop, remotely a blocked push. +const DefaultCmdTimeout = 10 * time.Minute + +// CommandWrapFunc rewrites a command for confined execution. It mirrors +// api.CommandWrapper's Wrap signature so a resolved sandbox adapter plugs in +// directly — hook inputs are untrusted, so a receive path must never exec +// them bare on the host (issue #40 R5.2). +type CommandWrapFunc func(ctx context.Context, cmd string, args, env []string) (string, []string, []string, error) + +// execRequest is one child process a verifier runs: what to execute, where, +// under what confinement and bounds, and where its streams go. +type execRequest struct { + Cmd string + Args []string + Dir string + Env []string // nil ⇒ inherit the process's + Wrap CommandWrapFunc + Timeout time.Duration // 0 ⇒ DefaultCmdTimeout + Stdin string + Stdout io.Writer + Stderr io.Writer +} + +// execOutcome is how a child process ended. Err is the non-nil exit error of a +// command that ran and failed; a process that could not be started or was torn +// down by the caller's context is reported through runProcess's error instead. +type execOutcome struct { + State *os.ProcessState + Elapsed time.Duration + Err error + TimedOut bool +} + +// effectiveTimeout is the wall clock a check actually gets. +func effectiveTimeout(d time.Duration) time.Duration { + if d <= 0 { + return DefaultCmdTimeout + } + return d +} + +// runProcess starts the command in its own process group, bounds it by the +// request's timeout, and reports how it ended. +// +// The bounds are the point of the helper: the caller's context and Timeout cap +// its wall clock, it runs in its own process group so a kill reaches its +// children, and a grandchild that survives the kill holding an output pipe +// cannot hold Wait open indefinitely. +// +// An error means no verdict is available: the wrapper refused, or the parent +// context ended (cancellation, or its own earlier deadline) and the run is being +// torn down — which is not a judgement on the work. +func runProcess(ctx context.Context, req execRequest) (execOutcome, error) { + timeout := effectiveTimeout(req.Timeout) + // The verifier's own timeout lives on a derived context; the parent is + // consulted separately below, so a parent deadline shorter than Timeout is + // reported as the run's cancellation, not misattributed to the command. + runCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + command, args, env, err := wrapCommand(ctx, req) + if err != nil { + return execOutcome{}, err + } + + cmd := exec.CommandContext(runCtx, command, args...) + cmd.Dir = req.Dir + if env != nil { + cmd.Env = env + } + if req.Stdin != "" { + cmd.Stdin = strings.NewReader(req.Stdin) + } + cmd.Stdout, cmd.Stderr = req.Stdout, req.Stderr + // Own process group, and cancellation kills the group: signalling only the + // pid leaves a hook's children running after their parent is dead. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } + cmd.WaitDelay = 10 * time.Second + + started := time.Now() + runErr := cmd.Run() + outcome := execOutcome{State: cmd.ProcessState, Elapsed: time.Since(started), Err: runErr} + switch { + case runErr == nil: + return outcome, nil + case ctx.Err() != nil: + return execOutcome{}, ctx.Err() + case errors.Is(runCtx.Err(), context.DeadlineExceeded): + outcome.TimedOut = true + } + return outcome, nil +} + +// wrapCommand applies the confinement seam, preserving the caller's environment +// boundary. A wrapper that supplies no environment keeps the pre-wrap one: +// leaving env nil would hand the wrapped process the full inherited environment, +// silently widening a caller's deliberately reduced Env (the git-agent hook +// path, issue #40). +func wrapCommand(ctx context.Context, req execRequest) (string, []string, []string, error) { + if req.Wrap == nil { + return req.Cmd, req.Args, req.Env, nil + } + wrapEnv := req.Env + if wrapEnv == nil { + wrapEnv = os.Environ() + } + command, args, env, err := req.Wrap(ctx, req.Cmd, req.Args, wrapEnv) + if err != nil { + return "", nil, nil, fmt.Errorf("wrapping %s for sandboxed execution: %w", req.Cmd, err) + } + if env == nil { + env = wrapEnv + } + return command, args, env, nil +} + +// exitCodeOf reads a finished process's status; -1 when it never ran. +func exitCodeOf(state *os.ProcessState) int { + if state == nil { + return -1 + } + return state.ExitCode() +} + +// tailBuffer keeps the last max bytes written through it, so a chatty command +// is bounded while it streams instead of being buffered whole and truncated +// afterwards. +type tailBuffer struct { + mu sync.Mutex + max int + buf []byte + truncated bool +} + +func newTailBuffer(max int) *tailBuffer { + if max <= 0 { + max = defaultFeedbackTail + } + return &tailBuffer{max: max} +} + +// defaultFeedbackTail bounds how much of a failing check's output is fed back +// into the next iteration's prompt. +const defaultFeedbackTail = 4096 + +func (b *tailBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.buf = append(b.buf, p...) + if len(b.buf) > b.max { + copy(b.buf, b.buf[len(b.buf)-b.max:]) + b.buf = b.buf[:b.max] + b.truncated = true + } + return len(p), nil +} + +func (b *tailBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + out := strings.TrimSpace(string(b.buf)) + if b.truncated && out != "" { + return "[output truncated]\n" + out + } + return out +} diff --git a/pkg/ai/agent/verify/external.go b/pkg/ai/agent/verify/external.go new file mode 100644 index 00000000..27e76af1 --- /dev/null +++ b/pkg/ai/agent/verify/external.go @@ -0,0 +1,205 @@ +package verify + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/flanksource/captain/pkg/api" +) + +// maxReportLineBytes bounds one NDJSON line from a fixture runner. A report is +// a tree of every test that ran, so the bound is generous; without one a runner +// that streams unterminated output buys unbounded memory in the process waiting +// on its verdict. +const maxReportLineBytes = 4 << 20 + +// ExternalVerifier runs a fixture document through a separate process — the +// host's fixture runner (gavel, today) — and reads its verdict back as a typed +// report. It is how captain dispatches Verify.Fixture without linking a fixture +// engine: captain owns the contract, the runner owns the fixtures. +// +// The contract is one process, three streams: +// +// - stdin — the fixture markdown, verbatim. +// - argv — the runner's own command, then `--cwd ` and one `--changed +// ` per changed file. +// - stdout — NDJSON: zero or more {"progress": } lines while it +// runs, then exactly one {"report": }. +// - stderr — diagnostics, kept as a tail for the error message. +// +// A runner that emits no report, malformed JSON, or more than one report never +// produces a verdict: it produces an error. A fixture whose tests failed is a +// verdict (runners exit non-zero for that), and the report says so. +type ExternalVerifier struct { + Command []string // argv of the fixture runner; required + Fixture string // the fixture document, handed over on stdin + Timeout time.Duration + Env []string + Wrap CommandWrapFunc + + progress func(api.VerifyReport) +} + +// SetProgress implements ProgressVerifier: the runner's progress lines are +// forwarded as they arrive. +func (e *ExternalVerifier) SetProgress(fn func(api.VerifyReport)) { e.progress = fn } + +func (e *ExternalVerifier) Verify(ctx context.Context, cwd string, changed []string) (Verdict, error) { + if len(e.Command) == 0 { + return Verdict{}, fmt.Errorf("fixture verifier: no runner command configured") + } + args := append([]string(nil), e.Command[1:]...) + args = append(args, "--cwd", cwd) + for _, path := range changed { + args = append(args, "--changed", path) + } + + reader := &ndjsonReader{onLine: e.consume} + stderr := newTailBuffer(defaultFeedbackTail) + outcome, err := runProcess(ctx, execRequest{ + Cmd: e.Command[0], Args: args, Dir: cwd, Env: e.Env, Wrap: e.Wrap, Timeout: e.Timeout, + Stdin: e.Fixture, Stdout: reader, Stderr: stderr, + }) + if err != nil { + return Verdict{}, err + } + reader.close() + + name := strings.Join(e.Command, " ") + if reader.err != nil { + return Verdict{}, fmt.Errorf("fixture verifier %s: %w%s", name, reader.err, diagnostics(stderr)) + } + if outcome.TimedOut { + return Verdict{}, fmt.Errorf("fixture verifier %s timed out after %s%s", + name, effectiveTimeout(e.Timeout), diagnostics(stderr)) + } + if reader.report == nil { + return Verdict{}, fmt.Errorf("fixture verifier %s exited %d with no report line%s%s", + name, exitCodeOf(outcome.State), processError(outcome.Err), diagnostics(stderr)) + } + report := *reader.report + if err := report.Validate(); err != nil { + return Verdict{}, fmt.Errorf("fixture verifier %s: %w", name, err) + } + return Verdict{OK: report.Passed, Reason: report.Reason, Feedback: report.Feedback, Report: &report}, nil +} + +// consume dispatches one decoded NDJSON line: a progress snapshot is forwarded +// live, and the report is kept for the verdict. +func (e *ExternalVerifier) consume(line ndjsonLine) error { + switch { + case line.Progress != nil: + if e.progress != nil { + e.progress(*line.Progress) + } + case line.Report == nil: + return fmt.Errorf(`a stdout line carried neither "progress" nor "report"`) + } + return nil +} + +// processError names why the process itself failed. A runner binary that does +// not exist never runs, exits -1 and says nothing on stderr, so "exited -1 with +// no report line" alone points at the protocol when the real fault is the argv: +// exec's own error is the only thing that names the missing or misspelled path. +func processError(err error) string { + if err == nil { + return "" + } + return ": " + err.Error() +} + +// diagnostics appends the runner's stderr tail to an error, when it said +// anything: an external runner that fails is usually explaining why on stderr, +// and an error without it sends the reader looking for a log that scrolled past. +func diagnostics(stderr *tailBuffer) string { + if tail := stderr.String(); tail != "" { + return "\n" + tail + } + return "" +} + +// ndjsonLine is one line of the fixture runner's stdout protocol. +type ndjsonLine struct { + Progress *api.VerifyReport `json:"progress,omitempty"` + Report *api.VerifyReport `json:"report,omitempty"` +} + +// ndjsonReader splits the runner's stdout into lines as they stream and decodes +// each one, so a progress line reaches a reader while the runner is still +// working rather than after it exits. The first failure is kept and stops +// further decoding: a runner that has gone off-protocol has no verdict to give. +type ndjsonReader struct { + onLine func(ndjsonLine) error + + buf bytes.Buffer + report *api.VerifyReport + err error +} + +func (r *ndjsonReader) Write(p []byte) (int, error) { + if r.err != nil { + // Keep draining: a writer that stops reading its child's stdout + // deadlocks the child rather than ending the run. + return len(p), nil + } + r.buf.Write(p) + for r.err == nil { + line, err := r.buf.ReadBytes('\n') + if err != nil { + // No complete line yet; put the fragment back for the next write. + r.buf.Write(line) + if r.buf.Len() > maxReportLineBytes { + r.err = fmt.Errorf("a stdout line exceeded %d bytes without a newline", maxReportLineBytes) + } + break + } + r.decode(line) + } + return len(p), nil +} + +func (r *ndjsonReader) decode(raw []byte) { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 { + return + } + var line ndjsonLine + if err := json.Unmarshal(trimmed, &line); err != nil { + r.err = fmt.Errorf("malformed stdout line %q: %w", truncateLine(trimmed), err) + return + } + if line.Report != nil { + if r.report != nil { + r.err = fmt.Errorf("emitted more than one report line") + return + } + r.report = line.Report + } + if err := r.onLine(line); err != nil { + // The raw line is the diagnosis: "carried neither key" without it leaves + // the reader diffing the runner's whole stdout against the protocol. + r.err = fmt.Errorf("%w: %s", err, truncateLine(trimmed)) + } +} + +// close decodes a final line the runner left unterminated. +func (r *ndjsonReader) close() { + if r.err != nil || r.buf.Len() == 0 { + return + } + r.decode(r.buf.Bytes()) + r.buf.Reset() +} + +func truncateLine(line []byte) string { + const max = 200 + if len(line) <= max { + return string(line) + } + return string(line[:max]) + "…" +} diff --git a/pkg/ai/agent/verify/external_contract_ginkgo_test.go b/pkg/ai/agent/verify/external_contract_ginkgo_test.go new file mode 100644 index 00000000..b10f5531 --- /dev/null +++ b/pkg/ai/agent/verify/external_contract_ginkgo_test.go @@ -0,0 +1,133 @@ +package verify + +import ( + "context" + "os" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/api" +) + +// writeScript drops an executable shell script and returns its path. +func writeScript(dir, name, body string) string { + GinkgoHelper() + path := filepath.Join(dir, name) + Expect(os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0o755)).To(Succeed()) + return path +} + +// The protocol is the whole contract between captain and a host's fixture +// runner, so every way a runner can break it has to end in an error rather than +// a verdict: a verdict invented from broken output is a definition of done that +// silently passed. +var _ = Describe("the external fixture verifier's stdout contract", func() { + ctx := context.Background() + + It("reports a runner that never finished as a timeout, not a verdict", func() { + dir := GinkgoT().TempDir() + verifier := &ExternalVerifier{ + Command: []string{writeScript(dir, "slow.sh", "sleep 30\n")}, + Fixture: "# acceptance\n", + Timeout: 200 * time.Millisecond, + } + + vd, err := verifier.Verify(ctx, dir, nil) + Expect(err).To(MatchError(ContainSubstring("timed out after 200ms"))) + Expect(vd.Report).To(BeNil(), "a check with no answer must not report one") + }) + + It("names the process error when the runner binary cannot be executed", func() { + dir := GinkgoT().TempDir() + missing := filepath.Join(dir, "gavle") // the classic transposed typo + verifier := &ExternalVerifier{Command: []string{missing}, Fixture: "# acceptance\n"} + + _, err := verifier.Verify(ctx, dir, nil) + Expect(err).To(MatchError(And( + ContainSubstring("no report line"), + ContainSubstring(missing), + ContainSubstring("no such file or directory"), + )), "exec's own error is the only thing that names the misspelled runner") + }) + + It("refuses a runner that emits two report lines", func() { + dir := GinkgoT().TempDir() + stdout := ndjson("report", passingReport("first")) + ndjson("report", passingReport("second")) + verifier := &ExternalVerifier{ + Command: []string{fixtureRunnerScript(dir, stdout, 0)}, Fixture: "# acceptance\n", + } + + _, err := verifier.Verify(ctx, dir, nil) + Expect(err).To(MatchError(ContainSubstring("more than one report line"))) + }) + + It("refuses a stdout line carrying neither key, and quotes the line", func() { + dir := GinkgoT().TempDir() + verifier := &ExternalVerifier{ + Command: []string{fixtureRunnerScript(dir, "{\"summary\":{\"passed\":1}}\n", 0)}, + Fixture: "# acceptance\n", + } + + _, err := verifier.Verify(ctx, dir, nil) + Expect(err).To(MatchError(And( + ContainSubstring(`carried neither "progress" nor "report"`), + ContainSubstring(`{"summary":{"passed":1}}`), + ))) + }) + + It("still parses a final report line the runner left unterminated", func() { + dir := GinkgoT().TempDir() + stdout := strings.TrimSuffix(ndjson("report", passingReport("go test ./...")), "\n") + verifier := &ExternalVerifier{ + Command: []string{fixtureRunnerScript(dir, stdout, 0)}, Fixture: "# acceptance\n", + } + + vd, err := verifier.Verify(ctx, dir, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(vd.OK).To(BeTrue()) + Expect(vd.Report.Tests[0].Name).To(Equal("go test ./...")) + }) + + It("refuses a report whose verdict contradicts its own state", func() { + dir := GinkgoT().TempDir() + contradictory := api.VerifyReport{ + Kind: "fixture", Name: "acceptance", Ran: true, Passed: true, State: api.VerifyStateFailed, + } + verifier := &ExternalVerifier{ + Command: []string{fixtureRunnerScript(dir, ndjson("report", contradictory), 0)}, + Fixture: "# acceptance\n", + } + + _, err := verifier.Verify(ctx, dir, nil) + Expect(err).To(MatchError(ContainSubstring(`passed=true with state "failed"`))) + }) + + It("runs the runner through the confinement wrapper, argv intact", func() { + dir := GinkgoT().TempDir() + runner := fixtureRunnerScript(dir, ndjson("report", passingReport("ok")), 0) + marker := filepath.Join(dir, "wrapped.txt") + wrapper := writeScript(dir, "wrapper.sh", "echo wrapped > "+marker+"\nexec \"$@\"\n") + + var sawCmd string + var sawArgs []string + verifier := &ExternalVerifier{ + Command: []string{runner, "check"}, + Fixture: "# acceptance\n", + Wrap: func(_ context.Context, cmd string, args, env []string) (string, []string, []string, error) { + sawCmd, sawArgs = cmd, args + return wrapper, append([]string{cmd}, args...), env, nil + }, + } + + vd, err := verifier.Verify(ctx, dir, []string{"pkg/a.go"}) + Expect(err).NotTo(HaveOccurred()) + Expect(vd.OK).To(BeTrue()) + Expect(sawCmd).To(Equal(runner), "the wrapper is handed the runner, not a shell") + Expect(sawArgs).To(Equal([]string{"check", "--cwd", dir, "--changed", "pkg/a.go"})) + Expect(marker).To(BeAnExistingFile(), "the wrapped command is what actually ran") + }) +}) diff --git a/pkg/ai/agent/verify/external_ginkgo_test.go b/pkg/ai/agent/verify/external_ginkgo_test.go new file mode 100644 index 00000000..b5e4892b --- /dev/null +++ b/pkg/ai/agent/verify/external_ginkgo_test.go @@ -0,0 +1,128 @@ +package verify + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strconv" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/api" +) + +// fixtureRunnerScript writes a throwaway runner: it records the fixture it was +// given on stdin and the arguments it was called with, replays a canned NDJSON +// stdout, and exits with the given status — the whole contract of a real +// fixture runner, in the shape a test can assert on. +func fixtureRunnerScript(dir, stdout string, exitCode int) string { + Expect(os.WriteFile(filepath.Join(dir, "stdout.ndjson"), []byte(stdout), 0o644)).To(Succeed()) + script := "#!/bin/sh\n" + + "cat > " + filepath.Join(dir, "stdin.txt") + "\n" + + "echo \"$@\" > " + filepath.Join(dir, "args.txt") + "\n" + + "cat " + filepath.Join(dir, "stdout.ndjson") + "\n" + + "exit " + strconv.Itoa(exitCode) + "\n" + path := filepath.Join(dir, "runner.sh") + Expect(os.WriteFile(path, []byte(script), 0o755)).To(Succeed()) + return path +} + +// ndjson renders one protocol line: {"progress": …} or {"report": …}. +func ndjson(key string, report api.VerifyReport) string { + raw, err := json.Marshal(map[string]api.VerifyReport{key: report}) + Expect(err).NotTo(HaveOccurred()) + return string(raw) + "\n" +} + +func passingReport(name string) api.VerifyReport { + return api.NewNodeReport("fixture", "acceptance", api.VerifyNode{Name: name, Passed: true}) +} + +func runningReport(name string) api.VerifyReport { + return api.NewNodeReport("fixture", "acceptance", api.VerifyNode{Name: name, Running: true}) +} + +var _ = Describe("the external fixture verifier", func() { + ctx := context.Background() + + It("forwards each progress line and returns the report as the verdict", func() { + dir := GinkgoT().TempDir() + stdout := ndjson("progress", runningReport("check 1")) + + ndjson("progress", runningReport("check 2")) + + ndjson("report", passingReport("go test ./...")) + verifier := &ExternalVerifier{Command: []string{fixtureRunnerScript(dir, stdout, 0)}, Fixture: "# acceptance\n"} + + var snapshots []api.VerifyReport + verifier.SetProgress(func(r api.VerifyReport) { snapshots = append(snapshots, r) }) + + vd, err := verifier.Verify(ctx, dir, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(vd.OK).To(BeTrue()) + Expect(snapshots).To(HaveLen(2)) + Expect(snapshots[1].Tests[0].Name).To(Equal("check 2")) + Expect(vd.Report).NotTo(BeNil()) + Expect(vd.Report.Validate()).To(Succeed()) + Expect(vd.Report.Kind).To(Equal("fixture")) + Expect(vd.Report.Tests[0].Name).To(Equal("go test ./...")) + Expect(vd.Report.Summary).To(Equal(api.VerifySummary{Total: 1, Passed: 1})) + }) + + It("reports a failing fixture as a verdict, not an error, even on a non-zero exit", func() { + dir := GinkgoT().TempDir() + failing := api.NewNodeReport("fixture", "acceptance", api.VerifyNode{ + Name: "go test ./...", Failed: true, Message: "TestFoo failed", + }) + failing.Reason = "1 test failed" + failing.Feedback = "TestFoo: want 3, got 4" + verifier := &ExternalVerifier{ + Command: []string{fixtureRunnerScript(dir, ndjson("report", failing), 1)}, + Fixture: "# acceptance\n", + } + + vd, err := verifier.Verify(ctx, dir, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(vd.OK).To(BeFalse()) + Expect(vd.Reason).To(Equal("1 test failed")) + Expect(vd.Feedback).To(Equal("TestFoo: want 3, got 4")) + Expect(vd.Report.State).To(Equal(api.VerifyStateFailed)) + }) + + It("refuses to invent a verdict when the runner exits without a report", func() { + dir := GinkgoT().TempDir() + verifier := &ExternalVerifier{Command: []string{fixtureRunnerScript(dir, "", 1)}, Fixture: "# acceptance\n"} + + _, err := verifier.Verify(ctx, dir, nil) + Expect(err).To(MatchError(ContainSubstring("no report line"))) + }) + + It("refuses to invent a verdict from malformed output", func() { + dir := GinkgoT().TempDir() + verifier := &ExternalVerifier{ + Command: []string{fixtureRunnerScript(dir, "{\"report\": not-json}\n", 0)}, + Fixture: "# acceptance\n", + } + + _, err := verifier.Verify(ctx, dir, nil) + Expect(err).To(MatchError(ContainSubstring("malformed stdout line"))) + }) + + It("hands the fixture over on stdin and names the cwd and every changed file", func() { + dir := GinkgoT().TempDir() + verifier := &ExternalVerifier{ + Command: []string{fixtureRunnerScript(dir, ndjson("report", passingReport("ok")), 0), "check"}, + Fixture: "# acceptance\n- [ ] it works\n", + } + + _, err := verifier.Verify(ctx, dir, []string{"pkg/a.go", "pkg/b.go"}) + Expect(err).NotTo(HaveOccurred()) + + Expect(os.ReadFile(filepath.Join(dir, "stdin.txt"))).To(Equal([]byte("# acceptance\n- [ ] it works\n"))) + args, err := os.ReadFile(filepath.Join(dir, "args.txt")) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(string(args))).To(Equal( + "check --cwd " + dir + " --changed pkg/a.go --changed pkg/b.go")) + }) +}) diff --git a/pkg/ai/agent/verify/llmjudge.go b/pkg/ai/agent/verify/llmjudge.go index b9382b27..9f83522b 100644 --- a/pkg/ai/agent/verify/llmjudge.go +++ b/pkg/ai/agent/verify/llmjudge.go @@ -3,9 +3,11 @@ package verify import ( "context" "fmt" + "time" "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/ai/prompt" + "github.com/flanksource/captain/pkg/api" ) // judgeVerdict is the structured output an LLM judge returns. @@ -40,8 +42,19 @@ func (j *LLMJudgeVerifier) Verify(ctx context.Context, cwd string, changed []str if err != nil { return Verdict{}, err } + started := time.Now() if _, err := j.Provider.Execute(ctx, req); err != nil { return Verdict{}, err } - return Verdict{OK: out.OK, Reason: out.Reason, Feedback: out.Feedback}, nil + // One leaf per judgement; the Plugin names the report after the hook + // ("judge:") since the template does not carry its path. + report := api.NewNodeReport(api.VerifyKindPrompt, "", api.VerifyNode{ + // The framework is the report's own kind, not a second name for it: a + // renderer grouping a mixed tree by framework would otherwise show a + // judge's node and a judge report as two different families. + Name: "llm judge", Framework: api.VerifyKindPrompt, Passed: out.OK, Failed: !out.OK, + Message: out.Reason, Duration: time.Since(started), + }) + report.Feedback = out.Feedback + return Verdict{OK: out.OK, Reason: out.Reason, Feedback: out.Feedback, Report: &report}, nil } diff --git a/pkg/ai/agent/verify/notice_test.go b/pkg/ai/agent/verify/notice_test.go new file mode 100644 index 00000000..38a8db38 --- /dev/null +++ b/pkg/ai/agent/verify/notice_test.go @@ -0,0 +1,179 @@ +package verify + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/claude/tools" +) + +// TestVerdictsReachTheRunsEventStream drives the hook through the real Runner, +// because the link under test is the one between the verifier and the run's +// event stream. Verification is the loop's definition of done and the only +// participant that used to be silent: the model's turns stream, but a verify +// command's output went into the retry prompt and nowhere else, so a run showed +// a turn, then a long pause, then another turn, with no record of what the check +// had said in between. +func TestVerdictsReachTheRunsEventStream(t *testing.T) { + for _, test := range []struct { + name string + command string + wantKind api.EventKind + contains []string + }{{ + name: "a passing command still reports, or a green run says nothing at all", + command: "true", + wantKind: api.EventVerified, + contains: []string{"passed in ", "verify:true"}, + }, { + name: "a failing command reports its reason and its output", + command: "echo 'Please Enter a Valid Cover Amount'; exit 1", + wantKind: api.EventVerifyFailed, + contains: []string{"failed in ", "sh failed", "Please Enter a Valid Cover Amount"}, + }} { + t.Run(test.name, func(t *testing.T) { + hooks, err := HooksFor(context.Background(), + &api.Workflow{Verify: &api.Verify{Commands: []string{test.command}}}, Options{}) + if err != nil { + t.Fatalf("hooks: %v", err) + } + var streamed []ai.Event + runner := &agent.Runner[string]{ + Provider: &silentProvider{}, + Cwd: t.TempDir(), + Request: ai.Request{Prompt: api.Prompt{User: "fix it"}}, + Hooks: hooks, + MaxIterations: 1, + OnEvent: func(_ int, ev ai.Event) { + if ev.Kind == api.EventVerified || ev.Kind == api.EventVerifyFailed { + streamed = append(streamed, ev) + } + }, + } + res, runErr := runner.Run(context.Background()) + if runErr != nil { + t.Fatalf("run: %v", runErr) + } + + if len(streamed) != 1 { + t.Fatalf("streamed verdicts = %+v, want exactly one", streamed) + } + got := streamed[0] + // The kind is the verdict: a consumer selects on it instead of + // reading the sentence to find out whether anything passed. + if got.Kind != test.wantKind { + t.Errorf("kind = %q, want %q", got.Kind, test.wantKind) + } + if got.Success != (test.wantKind == api.EventVerified) { + t.Errorf("Success = %t, want it to agree with kind %q", got.Success, got.Kind) + } + // The check's identity and wall clock travel as fields, not only in + // the prose, so a dashboard need not parse them back out. + if !strings.HasPrefix(got.Tool, "verify:") { + t.Errorf("Tool = %q, want the hook's name", got.Tool) + } + if got.Duration <= 0 { + t.Errorf("Duration = %s, want the verifier's wall clock", got.Duration) + } + for _, want := range test.contains { + if !strings.Contains(got.Text, want) { + t.Errorf("verdict text %q does not contain %q", got.Text, want) + } + } + + // The stream dies with the terminal; the buffered copy is what a + // caller persists into the session transcript once the run's session + // id is known, so the two must not drift — in text or in kind. + notices := res.Response.Workspace.Notices + if len(notices) != 1 { + t.Fatalf("buffered notices = %+v, want exactly one", notices) + } + if notices[0].Text != got.Text { + t.Errorf("buffered notice = %q, want the streamed text %q", notices[0].Text, got.Text) + } + if notices[0].Kind != test.wantKind { + t.Errorf("buffered notice kind = %q, want %q — a stored verdict must stay selectable", + notices[0].Kind, test.wantKind) + } + }) + } +} + +// TestVerdictSurvivesTheTranscriptPreview pins why the verdict leads the notice +// and the hook's name trails it. A transcript row shows only the first +// MessagePreviewChars runes, and the cmd factory names a hook after the entire +// shell command it runs — so naming it first spends the whole preview on a +// command line the reader already knows, and the row never says whether +// anything passed. +func TestVerdictSurvivesTheTranscriptPreview(t *testing.T) { + command := "oipa-cli test " + strings.Repeat("fixtures/some/deeply/nested/fixture.yaml ", 6) + hc := &agent.HookContext{ + Context: context.Background(), + Request: &ai.Request{}, + Response: &ai.Response{Workspace: &api.Workspace{Cwd: t.TempDir()}}, + } + New("verify:"+command, FuncVerifier(func(context.Context, string, []string) (Verdict, error) { + return Verdict{OK: false, Reason: "sh failed", Feedback: "Please Enter a Valid Cover Amount"}, nil + })).notify(hc, Verdict{OK: false, Reason: "sh failed", Feedback: "boom"}, 5*time.Minute) + + notices := hc.Workspace().Notices + if len(notices) != 1 { + t.Fatalf("notices = %+v, want exactly one", notices) + } + preview := []rune(notices[0].Text) + if len(preview) > tools.MessagePreviewChars { + preview = preview[:tools.MessagePreviewChars] + } + for _, want := range []string{"failed in 5m0s", "sh failed"} { + if !strings.Contains(string(preview), want) { + t.Errorf("the first %d runes (%q) must still say %q", + tools.MessagePreviewChars, string(preview), want) + } + } +} + +// TestVerifierErrorIsNotNotified pins Notify's contract: it is purely +// informational, and a hook that failed reports by returning an error rather +// than narrating one. A verifier that could not run at all aborts the run, and +// the abort is the report. +func TestVerifierErrorIsNotNotified(t *testing.T) { + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + + hc := &agent.HookContext{ + Context: cancelled, + Request: &ai.Request{}, + Response: &ai.Response{Workspace: &api.Workspace{Cwd: t.TempDir()}}, + } + _, err := New("verify:true", &CmdVerifier{Cmd: "true"}).Verify(hc) + + if err == nil { + t.Fatal("a cancelled run must surface the cancellation, not a verdict") + } + if notices := hc.Workspace().Notices; len(notices) != 0 { + t.Errorf("notices = %+v, want none for a verifier that could not reach a verdict", notices) + } +} + +// silentProvider is an agent that edits nothing and says nothing, so each test's +// only events are the ones its verifier produces. +type silentProvider struct{} + +func (*silentProvider) GetModel() string { return "fake" } +func (*silentProvider) GetRuntime() ai.Runtime { return ai.RuntimeOf(ai.Anthropic, ai.ModeAgent) } + +func (*silentProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { + return &ai.Response{}, nil +} + +func (*silentProvider) ExecuteStream(context.Context, ai.Request) (<-chan ai.Event, error) { + ch := make(chan ai.Event, 1) + ch <- ai.Event{Kind: ai.EventResult, Success: true} + close(ch) + return ch, nil +} diff --git a/pkg/ai/agent/verify/progress.go b/pkg/ai/agent/verify/progress.go new file mode 100644 index 00000000..78e436cf --- /dev/null +++ b/pkg/ai/agent/verify/progress.go @@ -0,0 +1,82 @@ +package verify + +import ( + "sync" + "time" + + "github.com/flanksource/captain/pkg/api" +) + +// ProgressInterval bounds how often an in-flight snapshot reaches a reader. A +// fixture runner reports per test; a reader needs to see that something is +// moving, not every row as it lands, and every snapshot costs an event on the +// run's stream and a redraw in whatever is watching it. +const ProgressInterval = 500 * time.Millisecond + +// ProgressVerifier is a Verifier that can report where it has got to before it +// has a verdict. The Plugin hands it a sink; a verifier that implements nothing +// here is simply silent until it finishes. +type ProgressVerifier interface { + SetProgress(func(api.VerifyReport)) +} + +// progressEmitter rate-limits in-flight snapshots to one per interval and +// guarantees the last one is delivered: a check that reports ten rows in a +// hundred milliseconds and then blocks for a minute must still leave the reader +// looking at the tenth row, not the first. +type progressEmitter struct { + mu sync.Mutex + interval time.Duration + sinks []func(api.VerifyReport) + lastAt time.Time + pending *api.VerifyReport +} + +func newProgressEmitter(interval time.Duration, sinks ...func(api.VerifyReport)) *progressEmitter { + live := make([]func(api.VerifyReport), 0, len(sinks)) + for _, sink := range sinks { + if sink != nil { + live = append(live, sink) + } + } + return &progressEmitter{interval: interval, sinks: live} +} + +// publish takes one snapshot from the verifier. It emits immediately when the +// window has elapsed, and otherwise holds the snapshot for flush. +func (e *progressEmitter) publish(report api.VerifyReport) { + e.mu.Lock() + defer e.mu.Unlock() + if len(e.sinks) == 0 { + return + } + now := time.Now() + if !e.lastAt.IsZero() && now.Sub(e.lastAt) < e.interval { + e.pending = &report + return + } + e.lastAt = now + e.pending = nil + e.deliver(report) +} + +// flush delivers the snapshot the window held back. It runs before the verdict +// so the two arrive in the order they happened. +func (e *progressEmitter) flush() { + e.mu.Lock() + defer e.mu.Unlock() + if e.pending == nil { + return + } + report := *e.pending + e.pending = nil + e.lastAt = time.Now() + e.deliver(report) +} + +// deliver fans one snapshot out to every sink. Callers hold e.mu. +func (e *progressEmitter) deliver(report api.VerifyReport) { + for _, sink := range e.sinks { + sink(report) + } +} diff --git a/pkg/ai/agent/verify/prompt_hooks_test.go b/pkg/ai/agent/verify/prompt_hooks_test.go index 3c43feee..68835d87 100644 --- a/pkg/ai/agent/verify/prompt_hooks_test.go +++ b/pkg/ai/agent/verify/prompt_hooks_test.go @@ -33,21 +33,26 @@ func writeJudgePrompt(t *testing.T) string { return path } -func TestPromptHooksForWorkflow(t *testing.T) { +// judgeHooks builds the hooks for a workflow that declares only prompts. +func judgeHooks(t *testing.T, provider ai.Provider, prompts ...string) ([]any, error) { + t.Helper() + return HooksFor(context.Background(), &api.Workflow{Verify: &api.Verify{Prompts: prompts}}, Options{Provider: provider}) +} + +func TestPromptHooks(t *testing.T) { provider := &judgeStubProvider{} t.Run("nothing declared yields no hooks", func(t *testing.T) { - if hooks, err := PromptHooksForWorkflow(nil, provider); err != nil || hooks != nil { + if hooks, err := HooksFor(context.Background(), nil, Options{Provider: provider}); err != nil || hooks != nil { t.Fatalf("hooks = %v, err = %v", hooks, err) } - if hooks, err := PromptHooksForWorkflow(&api.Workflow{Verify: &api.Verify{}}, provider); err != nil || hooks != nil { + if hooks, err := judgeHooks(t, provider); err != nil || hooks != nil { t.Fatalf("hooks = %v, err = %v", hooks, err) } }) t.Run("a blank prompt entry fails instead of dropping the check", func(t *testing.T) { - path := writeJudgePrompt(t) - _, err := PromptHooksForWorkflow(&api.Workflow{Verify: &api.Verify{Prompts: []string{path, " "}}}, provider) + _, err := judgeHooks(t, provider, writeJudgePrompt(t), " ") if err == nil || !strings.Contains(err.Error(), "prompts[1] is empty") { t.Fatalf("err = %v, want blank entry rejected", err) } @@ -55,9 +60,7 @@ func TestPromptHooksForWorkflow(t *testing.T) { t.Run("builds a named LLM judge per prompt", func(t *testing.T) { path := writeJudgePrompt(t) - wf := &api.Workflow{Verify: &api.Verify{Prompts: []string{path}}} - - hooks, err := PromptHooksForWorkflow(wf, provider) + hooks, err := judgeHooks(t, provider, path) if err != nil { t.Fatal(err) } @@ -74,8 +77,7 @@ func TestPromptHooksForWorkflow(t *testing.T) { }) t.Run("the judge consults the provider, not a live model", func(t *testing.T) { - path := writeJudgePrompt(t) - hooks, err := PromptHooksForWorkflow(&api.Workflow{Verify: &api.Verify{Prompts: []string{path}}}, provider) + hooks, err := judgeHooks(t, provider, writeJudgePrompt(t)) if err != nil { t.Fatal(err) } @@ -93,8 +95,28 @@ func TestPromptHooksForWorkflow(t *testing.T) { } }) + // The node's framework names the verifier family that produced it, using the + // same kind string the report carries — a renderer grouping a mixed tree by + // framework must not see "judge" and "prompt" as two families. + t.Run("the judgement is one node in the prompt framework", func(t *testing.T) { + hooks, err := judgeHooks(t, provider, writeJudgePrompt(t)) + if err != nil { + t.Fatal(err) + } + vd, err := hooks[0].(*Plugin).v.Verify(context.Background(), "/work", nil) + if err != nil { + t.Fatal(err) + } + if vd.Report.Kind != api.VerifyKindPrompt { + t.Fatalf("report kind = %q, want %q", vd.Report.Kind, api.VerifyKindPrompt) + } + if got := vd.Report.Tests[0].Framework; got != api.VerifyKindPrompt { + t.Fatalf("node framework = %q, want %q — the kind string, not a second name for it", got, api.VerifyKindPrompt) + } + }) + t.Run("a missing prompt file is an error, not a skipped check", func(t *testing.T) { - _, err := PromptHooksForWorkflow(&api.Workflow{Verify: &api.Verify{Prompts: []string{"/does/not/exist.prompt"}}}, provider) + _, err := judgeHooks(t, provider, "/does/not/exist.prompt") if err == nil || !strings.Contains(err.Error(), "/does/not/exist.prompt") { t.Fatalf("err = %v", err) } @@ -106,7 +128,7 @@ func TestPromptHooksForWorkflow(t *testing.T) { if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } - _, err := PromptHooksForWorkflow(&api.Workflow{Verify: &api.Verify{Prompts: []string{path}}}, provider) + _, err := judgeHooks(t, provider, path) if err == nil || !strings.Contains(err.Error(), "declares a sandbox") { t.Fatalf("err = %v, want sandbox declaration rejected (R5.4)", err) } @@ -118,7 +140,7 @@ func TestPromptHooksForWorkflow(t *testing.T) { if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } - _, err := PromptHooksForWorkflow(&api.Workflow{Verify: &api.Verify{Prompts: []string{path}}}, provider) + _, err := judgeHooks(t, provider, path) if err == nil || !strings.Contains(err.Error(), `declares model "gpt-5.5"`) { t.Fatalf("err = %v, want model mismatch rejected", err) } @@ -130,15 +152,14 @@ func TestPromptHooksForWorkflow(t *testing.T) { if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } - hooks, err := PromptHooksForWorkflow(&api.Workflow{Verify: &api.Verify{Prompts: []string{path}}}, provider) + hooks, err := judgeHooks(t, provider, path) if err != nil || len(hooks) != 1 { t.Fatalf("hooks = %v, err = %v", hooks, err) } }) t.Run("declared prompts with no provider fail loud", func(t *testing.T) { - path := writeJudgePrompt(t) - _, err := PromptHooksForWorkflow(&api.Workflow{Verify: &api.Verify{Prompts: []string{path}}}, nil) + _, err := judgeHooks(t, nil, writeJudgePrompt(t)) if err == nil || !strings.Contains(err.Error(), "no provider") { t.Fatalf("err = %v", err) } diff --git a/pkg/ai/agent/verify/registry.go b/pkg/ai/agent/verify/registry.go new file mode 100644 index 00000000..50367c8f --- /dev/null +++ b/pkg/ai/agent/verify/registry.go @@ -0,0 +1,261 @@ +package verify + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/prompt" + "github.com/flanksource/captain/pkg/api" +) + +// Kind names a family of verifier — one field of api.Verify and the factory that +// turns that field into hooks. +type Kind string + +const ( + KindCmd Kind = "cmd" + KindPrompt Kind = "prompt" + KindFixture Kind = "fixture" +) + +// kindOrder is the order a workflow's checks run in: the cheap deterministic +// commands first, then the judges and fixtures that cost a model call or a +// whole test run, so a run that is going to fail fails on the fast check. +var kindOrder = []Kind{KindCmd, KindPrompt, KindFixture} + +// Options is what every factory is given besides the spec: the provider a judge +// executes on, the confinement and bounds every child process inherits, and an +// optional sink for live progress snapshots. +type Options struct { + // Provider judges prompt hooks. Required when Verify.Prompts is non-empty. + Provider ai.Provider + // Env, Wrap and Timeout apply to every verifier that starts a process: the + // allowlisted environment, the confinement seam (a receive path must never + // exec agent-authored input bare on the host — R5.2), and the wall clock. + Env []string + Wrap CommandWrapFunc + Timeout time.Duration + // Progress receives each coalesced in-flight snapshot a verifier reports. + // Nil means the caller wants only the final verdict. + Progress func(api.VerifyReport) + // RunSpec is the resolved spec of the run these checks belong to — the model, + // permissions, budget and workflow it was started under — and is read-only to + // a factory. A factory that runs an agent of its own (a fixture grader + // judging a document's acceptance criteria) inherits the run's posture from + // it; without it such a grader has to invent a model and a permission mode, + // and grades outside the bounds the run declared. Nil when the checks are + // driven with no run behind them. + RunSpec *api.Spec +} + +// Factory builds the hooks one kind contributes. It returns *Plugin rather than +// the runner's []any so a caller that drives verifiers out of loop — the +// git-agent receive path, `captain verify` — keeps the typed handle. +type Factory func(ctx context.Context, spec api.Verify, opts Options) ([]*Plugin, error) + +var ( + registryMu sync.RWMutex + factories = map[Kind]Factory{} +) + +func init() { + Register(KindCmd, cmdFactory) + Register(KindPrompt, promptFactory) +} + +// Register installs the factory for one kind. Registering a kind twice panics: +// two factories for one field means half the declared checks silently never run, +// and which half depends on init order. +func Register(kind Kind, f Factory) { + if f == nil { + panic(fmt.Sprintf("verify: nil factory registered for kind %q", kind)) + } + registryMu.Lock() + defer registryMu.Unlock() + if _, exists := factories[kind]; exists { + panic(fmt.Sprintf("verify: a factory for kind %q is already registered", kind)) + } + factories[kind] = f +} + +// Registered reports whether a kind has a factory, so a host can install its own +// only when nothing has claimed the kind yet. +func Registered(kind Kind) bool { + registryMu.RLock() + defer registryMu.RUnlock() + _, ok := factories[kind] + return ok +} + +// Unregister removes a kind's factory and reports whether one was installed. +// +// It is a test and host seam, not part of the dispatch path. The registry is +// process-global, so a spec that installs a factory has to take it back out or +// the next spec inherits it; and a host that owns the whole process — it linked +// the fixture runner, nothing else can be mid-verification — may replace a kind +// by unregistering it first, since Register refuses to overwrite a live one. +func Unregister(kind Kind) bool { + registryMu.Lock() + defer registryMu.Unlock() + _, existed := factories[kind] + delete(factories, kind) + return existed +} + +func factoryFor(kind Kind) (Factory, bool) { + registryMu.RLock() + defer registryMu.RUnlock() + f, ok := factories[kind] + return f, ok +} + +// HooksFor builds the generate→verify loop's Verify hooks from a spec's +// Workflow, dispatching each declared kind to its registered factory in +// kindOrder. Returns nil when there is nothing to verify. +// +// It returns []any — the runner's heterogeneous hook list — and every plugin it +// returns carries opts.Progress, so a factory cannot forget to wire the sink. +// +// A declared check with no factory is an error, never an empty hook list: a +// workflow whose only verification is a fixture would otherwise produce zero +// hooks, and a run with zero verify hooks passes vacuously. +func HooksFor(ctx context.Context, wf *api.Workflow, opts Options) ([]any, error) { + if wf == nil || wf.Verify == nil { + return nil, nil + } + if strings.TrimSpace(wf.Verify.Fixture) != "" && !Registered(KindFixture) { + return nil, fmt.Errorf("workflow.verify.fixture declared but no fixture verifier is registered " + + "(link a fixture runner in-process or set verify.fixtureRunner in ~/.captain.yaml)") + } + var hooks []any + for _, kind := range kindOrder { + factory, ok := factoryFor(kind) + if !ok { + continue + } + plugins, err := factory(ctx, *wf.Verify, opts) + if err != nil { + return nil, err + } + for _, p := range plugins { + p.OnProgress(opts.Progress) + hooks = append(hooks, p) + } + } + return hooks, nil +} + +// ValidatePromptDeclarations loads every declared judge prompt before a run +// constructs its provider. This keeps a broken workflow attributable to the +// prompt declaration even when the selected provider is unavailable. +func ValidatePromptDeclarations(wf *api.Workflow) error { + if wf == nil || wf.Verify == nil { + return nil + } + for i, path := range wf.Verify.Prompts { + path = strings.TrimSpace(path) + if path == "" { + return fmt.Errorf("workflow.verify.prompts[%d] is empty", i) + } + if _, err := prompt.LoadFile(path); err != nil { + return fmt.Errorf("verify prompt %q: %w", path, err) + } + } + return nil +} + +// DeclaresExec reports whether the workflow declares a check that starts a +// process — a shell command or a fixture handed to an external runner. A +// receive path asks before it has any hooks, because the confinement wrapper is +// built against the materialized tree and must exist before the checks do. +func DeclaresExec(wf *api.Workflow) bool { + if wf == nil || wf.Verify == nil { + return false + } + if strings.TrimSpace(wf.Verify.Fixture) != "" { + return true + } + for _, cmd := range wf.Verify.Commands { + if strings.TrimSpace(cmd) != "" { + return true + } + } + return false +} + +// cmdFactory turns each verify command into a pass/fail check whose failure +// output drives the re-run. A blank entry is skipped rather than run as an empty +// shell command. +func cmdFactory(_ context.Context, spec api.Verify, opts Options) ([]*Plugin, error) { + var plugins []*Plugin + for _, cmd := range spec.Commands { + cmd = strings.TrimSpace(cmd) + if cmd == "" { + continue + } + plugins = append(plugins, New("verify:"+cmd, &CmdVerifier{ + Cmd: "sh", Args: []string{"-c", cmd}, + Env: opts.Env, Wrap: opts.Wrap, Timeout: opts.Timeout, + })) + } + return plugins, nil +} + +// promptFactory builds the LLM-judge hooks from Verify.Prompts: each entry is a +// .prompt template whose output schema is {ok, reason, feedback}, judged by the +// run's provider. +// +// A prompt that fails to load is an error, not a skipped hook: a declared check +// that silently never runs is a false accept. +func promptFactory(_ context.Context, spec api.Verify, opts Options) ([]*Plugin, error) { + if len(spec.Prompts) == 0 { + return nil, nil + } + if opts.Provider == nil { + return nil, fmt.Errorf("verify prompts declared but no provider available to judge them") + } + var plugins []*Plugin + for i, path := range spec.Prompts { + path = strings.TrimSpace(path) + if path == "" { + // Same rule as api.Workflow.Validate: a blank entry is a broken + // declaration, and skipping it would silently drop a configured check + // for callers that never ran Validate. + return nil, fmt.Errorf("workflow.verify.prompts[%d] is empty", i) + } + tmpl, err := prompt.LoadFile(path) + if err != nil { + return nil, fmt.Errorf("verify prompt %q: %w", path, err) + } + if err := rejectJudgeOverrides(path, tmpl, opts.Provider); err != nil { + return nil, err + } + plugins = append(plugins, New("judge:"+path, &LLMJudgeVerifier{Provider: opts.Provider, Prompt: tmpl})) + } + return plugins, nil +} + +// rejectJudgeOverrides refuses judge frontmatter the hook cannot honour. A +// judge executes on the run's provider, so a declared sandbox — relocating or +// not — and a model different from the provider's would both be silently +// ignored, which is exactly the downgrade issue #39 forbids (R5.4: a hook +// prompt declaring a relocating sandbox is a validation error, never a silent +// fallback). +func rejectJudgeOverrides(path string, tmpl *prompt.Template, provider ai.Provider) error { + probe, _, err := tmpl.Render(map[string]any{"cwd": "", "changed": []string{}}, nil) + if err != nil { + return fmt.Errorf("verify prompt %q: %w", path, err) + } + if probe.Sandbox != nil { + return fmt.Errorf("verify prompt %q declares a sandbox; judge hooks run on the run's provider and cannot relocate", path) + } + if declared := strings.TrimSpace(probe.Name); declared != "" && declared != provider.GetModel() { + return fmt.Errorf("verify prompt %q declares model %q but judge hooks run on the run's provider (%s); remove the model or match it", + path, declared, provider.GetModel()) + } + return nil +} diff --git a/pkg/ai/agent/verify/registry_ginkgo_test.go b/pkg/ai/agent/verify/registry_ginkgo_test.go new file mode 100644 index 00000000..42fc5bb9 --- /dev/null +++ b/pkg/ai/agent/verify/registry_ginkgo_test.go @@ -0,0 +1,205 @@ +package verify + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/api" +) + +// withFactory registers a factory for the duration of one spec. The registry is +// process-global on purpose (a host links its fixture runner once at startup), +// so a spec that installs one must take it back out. +func withFactory(kind Kind, f Factory) func() { + Register(kind, f) + return func() { Unregister(kind) } +} + +// recordingFactory captures what HooksFor handed it and yields one named plugin. +type recordingFactory struct { + spec api.Verify + opts Options + calls int + verifier Verifier +} + +func (r *recordingFactory) factory(_ context.Context, spec api.Verify, opts Options) ([]*Plugin, error) { + r.calls++ + r.spec, r.opts = spec, opts + v := r.verifier + if v == nil { + v = FuncVerifier(func(context.Context, string, []string) (Verdict, error) { + return Verdict{OK: true}, nil + }) + } + return []*Plugin{New("fixture", v)}, nil +} + +// progressStub reports five in-flight snapshots inside 100ms, so the emitter's +// coalescing window (500ms) is exercised by a verifier that reports faster than +// a reader can use. +type progressStub struct { + progress func(api.VerifyReport) + reported int +} + +func (p *progressStub) SetProgress(fn func(api.VerifyReport)) { p.progress = fn } + +func (p *progressStub) Verify(context.Context, string, []string) (Verdict, error) { + for i := 1; i <= 5; i++ { + snapshot := api.NewNodeReport(api.VerifyKindFunc, "fixture", api.VerifyNode{ + Name: fmt.Sprintf("check %d", i), Running: true, + }) + p.progress(snapshot) + p.reported++ + time.Sleep(20 * time.Millisecond) + } + final := api.NewNodeReport(api.VerifyKindFunc, "fixture", api.VerifyNode{Name: "check 5", Passed: true}) + return Verdict{OK: true, Report: &final}, nil +} + +var _ = Describe("the verifier registry", func() { + ctx := context.Background() + + It("yields no hooks for a workflow with nothing to verify", func() { + hooks, err := HooksFor(ctx, nil, Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(hooks).To(BeNil()) + + hooks, err = HooksFor(ctx, &api.Workflow{}, Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(hooks).To(BeNil()) + }) + + It("refuses a declared fixture when no fixture verifier is registered", func() { + Expect(Registered(KindFixture)).To(BeFalse(), "no fixture runner is linked in this test binary") + + _, err := HooksFor(ctx, &api.Workflow{Verify: &api.Verify{Fixture: "# acceptance\n"}}, Options{}) + Expect(err).To(MatchError(ContainSubstring("no fixture verifier is registered"))) + }) + + It("hands the fixture document and the run's options to the registered factory", func() { + recorder := &recordingFactory{} + defer withFactory(KindFixture, recorder.factory)() + + wrap := func(_ context.Context, cmd string, args, env []string) (string, []string, []string, error) { + return cmd, args, env, nil + } + hooks, err := HooksFor(ctx, &api.Workflow{Verify: &api.Verify{Fixture: "# acceptance\n"}}, Options{ + Env: []string{"PATH=/usr/bin"}, Wrap: wrap, Timeout: 42 * time.Second, + Progress: func(api.VerifyReport) {}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(hooks).To(HaveLen(1)) + Expect(recorder.calls).To(Equal(1)) + Expect(recorder.spec.Fixture).To(Equal("# acceptance\n")) + Expect(recorder.opts.Env).To(Equal([]string{"PATH=/usr/bin"})) + Expect(recorder.opts.Timeout).To(Equal(42 * time.Second)) + Expect(recorder.opts.Wrap).NotTo(BeNil()) + Expect(recorder.opts.Progress).NotTo(BeNil()) + }) + + It("orders the hooks command, then prompt, then fixture", func() { + defer withFactory(KindFixture, (&recordingFactory{}).factory)() + promptPath := filepath.Join(GinkgoT().TempDir(), "judge.prompt") + Expect(os.WriteFile(promptPath, []byte("{{role \"user\"}}\nJudge {{cwd}}."), 0o644)).To(Succeed()) + + hooks, err := HooksFor(ctx, &api.Workflow{Verify: &api.Verify{ + Commands: []string{"go test ./...", " "}, + Prompts: []string{promptPath}, + Fixture: "# acceptance\n", + }}, Options{Provider: &judgeStubProvider{}}) + Expect(err).NotTo(HaveOccurred()) + + names := make([]string, 0, len(hooks)) + for _, h := range hooks { + names = append(names, h.(*Plugin).Name()) + } + Expect(names).To(Equal([]string{"verify:go test ./...", "judge:" + promptPath, "fixture"}), + "a blank command is skipped and the kinds keep their declared order") + }) + + It("applies the run's command environment and timeout to every command verifier", func() { + hooks, err := HooksFor(ctx, &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}}}, Options{ + Env: []string{"PATH=/usr/bin"}, Timeout: 7 * time.Second, + }) + Expect(err).NotTo(HaveOccurred()) + cv, ok := hooks[0].(*Plugin).Verifier().(*CmdVerifier) + Expect(ok).To(BeTrue()) + Expect(cv.Env).To(Equal([]string{"PATH=/usr/bin"})) + Expect(cv.Timeout).To(Equal(7 * time.Second)) + }) + + It("refuses to register two factories for one kind", func() { + Expect(func() { Register(KindCmd, (&recordingFactory{}).factory) }).To(PanicWith( + ContainSubstring("cmd"))) + }) + + It("lets a kind be taken back out and re-registered", func() { + Expect(Unregister(KindFixture)).To(BeFalse(), "nothing claims the fixture kind yet") + + Register(KindFixture, (&recordingFactory{}).factory) + Expect(Registered(KindFixture)).To(BeTrue()) + Expect(Unregister(KindFixture)).To(BeTrue()) + Expect(Registered(KindFixture)).To(BeFalse()) + + // Register panics on a live kind, so unregistering is what makes a + // replacement possible at all. + Expect(func() { Register(KindFixture, (&recordingFactory{}).factory) }).NotTo(Panic()) + Expect(Unregister(KindFixture)).To(BeTrue()) + }) + + It("coalesces a verifier's progress into at most one event per window, last snapshot always delivered", func() { + var sunk []api.VerifyReport + defer withFactory(KindFixture, (&recordingFactory{verifier: &progressStub{}}).factory)() + + var streamed []ai.Event + hooks, err := HooksFor(ctx, &api.Workflow{Verify: &api.Verify{Fixture: "# acceptance\n"}}, Options{ + Progress: func(r api.VerifyReport) { sunk = append(sunk, r) }, + }) + Expect(err).NotTo(HaveOccurred()) + + runner := &agent.Runner[string]{ + Provider: &silentProvider{}, + Cwd: GinkgoT().TempDir(), + Request: ai.Request{Prompt: api.Prompt{User: "fix it"}}, + Hooks: hooks, + MaxIterations: 1, + OnEvent: func(_ int, ev ai.Event) { + switch ev.Kind { + case api.EventVerifyProgress, api.EventVerified, api.EventVerifyFailed: + streamed = append(streamed, ev) + } + }, + } + _, err = runner.Run(ctx) + Expect(err).NotTo(HaveOccurred()) + + var progress []ai.Event + for _, ev := range streamed { + if ev.Kind == api.EventVerifyProgress { + progress = append(progress, ev) + } + } + Expect(len(progress)).To(BeNumerically(">=", 1)) + Expect(len(progress)).To(BeNumerically("<=", 2), "five snapshots inside one 500ms window collapse") + Expect(streamed[len(streamed)-1].Kind).To(Equal(api.EventVerified), + "the last snapshot is flushed before the verdict, never after it") + + last, ok := progress[len(progress)-1].Raw.(*api.VerifyReport) + Expect(ok).To(BeTrue(), "Raw carries the *api.VerifyReport") + Expect(last.Tests[0].Name).To(Equal("check 5"), "the final snapshot always reaches the reader") + Expect(progress[len(progress)-1].Tool).To(Equal("fixture")) + + Expect(sunk).To(HaveLen(len(progress)), "the Options sink sees the same coalesced snapshots") + Expect(sunk[len(sunk)-1].Tests[0].Name).To(Equal("check 5")) + }) +}) diff --git a/pkg/ai/agent/verify/report_ginkgo_test.go b/pkg/ai/agent/verify/report_ginkgo_test.go new file mode 100644 index 00000000..51ae1007 --- /dev/null +++ b/pkg/ai/agent/verify/report_ginkgo_test.go @@ -0,0 +1,239 @@ +package verify + +import ( + "context" + "encoding/json" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/api" +) + +var _ = Describe("typed verify reports", func() { + var ( + cwd string + hc *agent.HookContext + ) + + BeforeEach(func() { + cwd = GinkgoT().TempDir() + hc = &agent.HookContext{ + Context: context.Background(), + Request: &ai.Request{}, + Response: &ai.Response{Workspace: &api.Workspace{Cwd: cwd}}, + Scope: agent.ScopeAll, + // HookContext.Iteration is the loop's 0-based index; a report and a + // VerifyResult name the turn 1-based, so this is turn 3 of the run. + Iteration: 2, + } + }) + + Describe("CmdVerifier", func() { + It("reports a failing command as one failed node carrying the command, cwd and output tail", func() { + plugin := New("verify:sh", &CmdVerifier{Cmd: "sh", Args: []string{"-c", "echo out; exit 1"}}) + + result, err := plugin.Verify(hc) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Valid).To(BeFalse()) + Expect(result.Iteration).To(Equal(3)) + Expect(result.Retry).NotTo(BeNil()) + + report := result.Report + Expect(report).NotTo(BeNil()) + Expect(report.Validate()).To(Succeed()) + Expect(report.Kind).To(Equal(api.VerifyKindCmd)) + Expect(report.Name).To(Equal("sh -c echo out; exit 1")) + Expect(report.Iteration).To(Equal(3)) + Expect(report.Ran).To(BeTrue()) + Expect(report.Passed).To(BeFalse()) + Expect(report.State).To(Equal(api.VerifyStateFailed)) + Expect(report.Summary).To(Equal(api.VerifySummary{Total: 1, Failed: 1})) + Expect(report.Feedback).To(ContainSubstring("out")) + + Expect(report.Tests).To(HaveLen(1)) + node := report.Tests[0] + Expect(node.Failed).To(BeTrue()) + Expect(node.Command).To(Equal("sh -c echo out; exit 1")) + Expect(node.WorkDir).To(Equal(cwd)) + Expect(node.Stderr).To(ContainSubstring("out")) + Expect(node.Context.ExitCode).To(Equal(1)) + Expect(node.Duration).To(BeNumerically(">", time.Duration(0))) + + raw, err := json.Marshal(report) + Expect(err).NotTo(HaveOccurred()) + Expect(string(raw)).To(ContainSubstring(`"work_dir"`)) + Expect(string(raw)).To(ContainSubstring(`"exit_code":1`)) + }) + + It("reports a passing command as a passed report", func() { + plugin := New("verify:true", &CmdVerifier{Cmd: "sh", Args: []string{"-c", "echo fine"}}) + + result, err := plugin.Verify(hc) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Valid).To(BeTrue()) + Expect(result.Retry).To(BeNil()) + Expect(result.Report.Validate()).To(Succeed()) + Expect(result.Report.Passed).To(BeTrue()) + Expect(result.Report.State).To(Equal(api.VerifyStatePassed)) + Expect(result.Report.Tests[0].Stdout).To(Equal("fine")) + Expect(result.Report.Tests[0].Context.ExitCode).To(Equal(0)) + }) + + It("marks a timed-out command as timed out, not merely failed", func() { + plugin := New("verify:sleep", &CmdVerifier{Cmd: "sh", Args: []string{"-c", "sleep 5"}, Timeout: 50 * time.Millisecond}) + + result, err := plugin.Verify(hc) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Valid).To(BeFalse()) + Expect(result.Report.State).To(Equal(api.VerifyStateTimedOut)) + Expect(result.Report.Tests[0].TimedOut).To(BeTrue()) + // Timed out, not merely failed, in the counters too: a check that + // never finished is a different problem from one that disagreed. + Expect(result.Report.Summary).To(Equal(api.VerifySummary{Total: 1, TimedOut: 1})) + }) + }) + + // A verifier that reports OK while its own report says otherwise has two + // answers and no way to choose: silently trusting one of them is how a + // failing check lands in the store as a pass. + Describe("a verdict that disagrees with its own report", func() { + It("is an error rather than a silently reconciled verdict", func() { + contradiction := api.NewNodeReport("fixture", "acceptance", api.VerifyNode{Name: "t", Failed: true}) + plugin := New("fixture:acceptance", FuncVerifier(func(context.Context, string, []string) (Verdict, error) { + return Verdict{OK: true, Report: &contradiction}, nil + })) + + _, err := plugin.Verify(hc) + Expect(err).To(MatchError(ContainSubstring("reports passed=false but its verdict says OK=true"))) + }) + }) + + Describe("a Verifier that returns no report", func() { + It("gets a one-node report synthesised from its verdict", func() { + plugin := New("custom", FuncVerifier(func(context.Context, string, []string) (Verdict, error) { + return Verdict{OK: false, Reason: "not yet", Feedback: "do more"}, nil + })) + + result, err := plugin.Verify(hc) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Valid).To(BeFalse()) + Expect(result.Report).NotTo(BeNil()) + Expect(result.Report.Validate()).To(Succeed()) + Expect(result.Report.Kind).To(Equal(api.VerifyKindFunc)) + Expect(result.Report.Name).To(Equal("custom")) + Expect(result.Report.Reason).To(Equal("not yet")) + Expect(result.Report.Feedback).To(Equal("do more")) + Expect(result.Report.Iteration).To(Equal(3)) + Expect(result.Report.Tests).To(HaveLen(1)) + Expect(result.Report.Tests[0].Name).To(Equal("custom")) + Expect(result.Report.Tests[0].Failed).To(BeTrue()) + }) + + It("keeps a report the Verifier supplied and fills only what is missing", func() { + supplied := api.NewNodeReport("fixture", "", api.VerifyNode{Name: "go test ./x", Passed: true}) + plugin := New("fixture:acceptance", FuncVerifier(func(context.Context, string, []string) (Verdict, error) { + return Verdict{OK: true, Report: &supplied}, nil + })) + + result, err := plugin.Verify(hc) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Report.Kind).To(Equal("fixture")) + Expect(result.Report.Name).To(Equal("fixture:acceptance")) + Expect(result.Report.Iteration).To(Equal(3)) + Expect(result.Report.Tests[0].Name).To(Equal("go test ./x")) + }) + }) + + It("attaches the report to the verdict event on the run stream", func() { + var streamed []ai.Event + runner := &agent.Runner[string]{ + Provider: &silentProvider{}, + Cwd: cwd, + Request: ai.Request{Prompt: api.Prompt{User: "fix it"}}, + Hooks: []any{New("verify:false", &CmdVerifier{Cmd: "false"})}, + MaxIterations: 1, + OnEvent: func(_ int, ev ai.Event) { + if ev.Kind == api.EventVerifyFailed || ev.Kind == api.EventVerified { + streamed = append(streamed, ev) + } + }, + } + + res, err := runner.Run(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(streamed).To(HaveLen(1)) + Expect(streamed[0].Kind).To(Equal(ai.EventVerifyFailed)) + report, ok := streamed[0].Raw.(*api.VerifyReport) + Expect(ok).To(BeTrue(), "Raw should carry the *api.VerifyReport") + Expect(report.State).To(Equal(api.VerifyStateFailed)) + Expect(res.Verdicts).To(HaveLen(1)) + Expect(res.Verdicts[0].Report).To(BeIdenticalTo(report)) + // The loop indexes its turns from 0; a verdict names the turn it judged + // the way a person does — and the way the iteration store is keyed. + Expect(res.Verdicts[0].Iteration).To(Equal(1)) + Expect(report.Iteration).To(Equal(1)) + }) + + // Progress is ephemeral: it exists so a reader watching a long check sees + // that something is moving. Recording each snapshot as a workspace notice + // wrote every one of them into the persisted transcript, burying the verdict + // the notices exist to surface. + Describe("in-flight progress", func() { + It("streams snapshots without leaving a notice behind, while the verdict still records one", func() { + var streamed []ai.Event + runner := &agent.Runner[string]{ + Provider: &silentProvider{}, + Cwd: cwd, + Request: ai.Request{Prompt: api.Prompt{User: "fix it"}}, + Hooks: []any{New("fixture", &threeSnapshotVerifier{})}, + MaxIterations: 1, + OnEvent: func(_ int, ev ai.Event) { + switch ev.Kind { + case ai.EventVerifyProgress, ai.EventVerified, ai.EventVerifyFailed: + streamed = append(streamed, ev) + } + }, + } + + res, err := runner.Run(context.Background()) + Expect(err).NotTo(HaveOccurred()) + + var progress []ai.Event + for _, ev := range streamed { + if ev.Kind == ai.EventVerifyProgress { + progress = append(progress, ev) + } + } + Expect(len(progress)).To(BeNumerically(">=", 1)) + Expect(progress[0].Tool).To(Equal("fixture")) + Expect(progress[0].Text).To(BeEmpty(), "a stream-only event carries the report, not prose about it") + snapshot, ok := progress[len(progress)-1].Raw.(*api.VerifyReport) + Expect(ok).To(BeTrue(), "Raw carries the *api.VerifyReport") + Expect(snapshot.Tests[0].Name).To(Equal("check 3"), "the last snapshot is always flushed") + + notices := res.Response.Workspace.Notices + Expect(notices).To(HaveLen(1), "only the verdict is a notice; the snapshots are not") + Expect(notices[0].Kind).To(Equal(ai.EventVerified)) + }) + }) +}) + +// threeSnapshotVerifier reports three in-flight snapshots and then passes. +type threeSnapshotVerifier struct{ progress func(api.VerifyReport) } + +func (v *threeSnapshotVerifier) SetProgress(fn func(api.VerifyReport)) { v.progress = fn } + +func (v *threeSnapshotVerifier) Verify(context.Context, string, []string) (Verdict, error) { + for i := 1; i <= 3; i++ { + v.progress(api.NewNodeReport(api.VerifyKindFunc, "fixture", api.VerifyNode{ + Name: fmt.Sprintf("check %d", i), Running: true, + })) + } + final := api.NewNodeReport(api.VerifyKindFunc, "fixture", api.VerifyNode{Name: "check 3", Passed: true}) + return Verdict{OK: true, Report: &final}, nil +} diff --git a/pkg/ai/agent/verify/verify.go b/pkg/ai/agent/verify/verify.go index d33039bb..24301910 100644 --- a/pkg/ai/agent/verify/verify.go +++ b/pkg/ai/agent/verify/verify.go @@ -6,25 +6,26 @@ package verify import ( "context" - "errors" "fmt" "os" - "os/exec" "strings" - "sync" - "syscall" "time" "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/api" ) // Verdict is a Verifier's judgement. Feedback is fed back into the next -// iteration's prompt when OK is false. +// iteration's prompt when OK is false. Report is the typed form of the same +// verdict — the tree of what ran and the checklist — for persistence and +// rendering; a Verifier that leaves it nil gets a one-node report synthesised +// from OK/Reason/Feedback by the Plugin. type Verdict struct { OK bool Reason string Feedback string + Report *api.VerifyReport } // Verifier checks the state of a working tree after an agent turn. cwd is where @@ -40,11 +41,25 @@ type Verifier interface { type Plugin struct { name string v Verifier + // progress are the extra sinks a caller registered for in-flight snapshots, + // alongside the run's own event stream. See OnProgress. + progress []func(api.VerifyReport) } // New wraps a Verifier as a named agent.Verify hook. func New(name string, v Verifier) *Plugin { return &Plugin{name: name, v: v} } +// OnProgress adds a sink for the in-flight snapshots a ProgressVerifier +// reports, on top of the run's event stream. HooksFor calls it with +// Options.Progress so a factory cannot forget to wire the caller's sink; a nil +// sink registers nothing. +func (p *Plugin) OnProgress(sink func(api.VerifyReport)) *Plugin { + if sink != nil { + p.progress = append(p.progress, sink) + } + return p +} + func (p *Plugin) Name() string { return p.name } // Verifier exposes the wrapped Verifier so a caller that runs checks outside @@ -58,14 +73,161 @@ func (p *Plugin) Verify(hc *agent.HookContext) (agent.VerifyResult, error) { if hc.Scope == agent.ScopeChanged { changed = ws.Changed } + emitter := p.watchProgress(hc) + started := time.Now() vd, err := p.v.Verify(hc, ws.Cwd, changed) if err != nil { + // No notice: Notify is purely informational, and a hook that failed + // reports by returning an error. The abort is the report — including the + // snapshot the window was holding, which is dropped with it. return agent.VerifyResult{}, err } + // Before the verdict, so a reader sees the last thing that ran and then how + // it ended, in that order. + emitter.flush() + elapsed := time.Since(started) + // HookContext.Iteration is the loop's 0-based index; a report and a verdict + // name the turn the way a person and the iteration store do, from 1. + iteration := hc.Iteration + 1 + report, err := p.report(vd, iteration, elapsed) + if err != nil { + return agent.VerifyResult{}, err + } + vd.Report = report + p.notify(hc, vd, elapsed) + result := agent.VerifyResult{Valid: vd.OK, Report: vd.Report, Iteration: iteration} + if !vd.OK { + result.Retry = retryWithFeedback(hc.Request, vd.Feedback) + } + return result, nil +} + +// watchProgress hands a ProgressVerifier a coalescing sink that reports onto +// the run's stream and into whatever sinks the caller registered, and returns +// the emitter so the caller can flush the held snapshot before the verdict. A +// verifier that reports no progress gets an emitter nobody ever feeds. +func (p *Plugin) watchProgress(hc *agent.HookContext) *progressEmitter { + sinks := append([]func(api.VerifyReport){p.notifyProgress(hc)}, p.progress...) + emitter := newProgressEmitter(ProgressInterval, sinks...) + if pv, ok := p.v.(ProgressVerifier); ok { + pv.SetProgress(emitter.publish) + } + return emitter +} + +// notifyProgress reports one snapshot on the run's stream under its own kind, +// with the typed report on Raw exactly as the verdict carries it — a renderer +// that draws the verification tree redraws it from the same shape while the +// check is still running. +// +// It Emits rather than Notifies: a snapshot is true only while it is on screen, +// and recording each one as a workspace notice wrote every superseded count into +// the persisted transcript and buried the verdict underneath them. There is +// deliberately no text — the event carries the report a renderer draws, and a +// sentence describing it would be the thing that got recorded. +func (p *Plugin) notifyProgress(hc *agent.HookContext) func(api.VerifyReport) { + return func(report api.VerifyReport) { + if report.Name == "" { + report.Name = p.name + } + hc.Emit(ai.Event{Kind: ai.EventVerifyProgress, Tool: p.name, Raw: &report}) + } +} + +// report completes the verifier's typed report: a Verifier that returned none +// gets a one-node report synthesised from its verdict, and every report carries +// the hook's name, the verdict's reason/feedback, the iteration it judged and +// the wall clock it took, so a consumer never has to reach back into the +// Verdict for those. +// +// A report that disagrees with the verdict it came with is an error. The two are +// the same judgement written twice, and quietly preferring one of them is how a +// check that failed reaches the store, the webapp and the next turn as a pass. +func (p *Plugin) report(vd Verdict, iteration int, elapsed time.Duration) (*api.VerifyReport, error) { + report := vd.Report + if report == nil { + synthesised := api.NewNodeReport(api.VerifyKindFunc, p.name, api.VerifyNode{ + Name: p.name, Passed: vd.OK, Failed: !vd.OK, Message: vd.Reason, Duration: elapsed, + }) + report = &synthesised + } + if report.Passed != vd.OK { + return nil, fmt.Errorf("verify %q: report %q reports passed=%t but its verdict says OK=%t", + p.name, report.Name, report.Passed, vd.OK) + } + if report.Name == "" { + report.Name = p.name + } + if report.Reason == "" { + report.Reason = vd.Reason + } + if report.Feedback == "" { + report.Feedback = vd.Feedback + } + if report.Duration == 0 { + report.Duration = elapsed + } + report.Iteration = iteration + return report, nil +} + +// notify records the verdict on the run's stream and workspace, the way every +// other lifecycle hook records what it did. +// +// Verification is the loop's definition of done and was the only participant +// that left no trace. A failing verdict's output travels into the next turn's +// prompt and nowhere else, so it is lost entirely on the last iteration — the +// one that decides the run — and a passing verdict was never recorded at all. A +// reader was left with a turn, a long pause, and another turn, with nothing +// saying what the check had reported or how long it took. +// +// It reports under EventVerified / EventVerifyFailed rather than as a generic +// system line, so a consumer selects on the verdict instead of parsing it, and +// carries the check's name and wall clock as fields rather than only in prose. +// +// The feedback is included rather than summarized: it is the verdict's whole +// content, it is already tail-bounded by the verifier, and on the final +// iteration this is the only place it survives. +// +// The verdict leads and the hook's name follows it, because a row that has to +// fit one line shows the front of the text: a workflow hook is named after the +// whole shell command it runs, so naming it first spends the whole line before +// saying whether anything passed. +// +// The typed report (vd.Report, when the verdict carries one) rides on Raw so a +// renderer that wants the tree — the webapp's verification panel, a transcript +// rehydrating a stored run — reads it from the same event the prose came from. +func (p *Plugin) notify(hc *agent.HookContext, vd Verdict, elapsed time.Duration) { + event := ai.Event{Kind: ai.EventVerifyFailed, Tool: p.name, Duration: elapsed} + if vd.Report != nil { + event.Raw = vd.Report + } if vd.OK { - return agent.VerifyResult{Valid: true, Output: vd}, nil + event.Kind, event.Success = ai.EventVerified, true + event.Text = fmt.Sprintf("passed in %s — %s", took(elapsed), p.name) + hc.NotifyEvent(event) + return + } + reason := strings.TrimSpace(vd.Reason) + if reason == "" { + reason = "no reason reported" } - return agent.VerifyResult{Valid: false, Retry: retryWithFeedback(hc.Request, vd.Feedback), Output: vd}, nil + event.Reason = reason + event.Text = fmt.Sprintf("failed in %s: %s — %s", took(elapsed), reason, p.name) + if feedback := strings.TrimSpace(vd.Feedback); feedback != "" { + event.Text += "\n" + feedback + } + hc.NotifyEvent(event) +} + +// took renders a verify's wall clock at a precision that stays useful across +// both scales it runs at: milliseconds for a lint that finishes instantly, +// whole seconds for a test suite where the fractions are noise. +func took(d time.Duration) time.Duration { + if d >= time.Minute { + return d.Round(time.Second) + } + return d.Round(time.Millisecond) } // retryWithFeedback builds the next request: the current one with the verifier's @@ -84,11 +246,6 @@ func (f FuncVerifier) Verify(ctx context.Context, cwd string, changed []string) return f(ctx, cwd, changed) } -// DefaultCmdTimeout bounds a verify command that declares no timeout of its -// own. A hook with no bound is a denial-of-service against whatever is waiting -// on the verdict — locally a stuck loop, remotely a blocked push. -const DefaultCmdTimeout = 10 * time.Minute - // CmdVerifier runs an external command (lint, test, build, …) in the run's cwd. // A zero exit code is a pass; a non-zero exit is a failure whose feedback is the // tail of the combined output. When PerFile is set the changed files are @@ -96,128 +253,72 @@ const DefaultCmdTimeout = 10 * time.Minute // // The command is bounded three ways: the caller's context and Timeout cap its // wall clock, it runs in its own process group so a kill reaches its children, -// and its output is tail-bounded as it streams rather than buffered in full. +// and its output is tail-bounded as it streams rather than buffered in full +// (see exec.go, which ExternalVerifier shares). type CmdVerifier struct { Cmd string Args []string PerFile bool - FeedbackTail int // max bytes of output fed back; 0 ⇒ 4096 + FeedbackTail int // max bytes of output fed back; 0 ⇒ defaultFeedbackTail Timeout time.Duration // wall-clock bound; 0 ⇒ DefaultCmdTimeout Env []string // command environment; nil ⇒ inherit the process's Wrap CommandWrapFunc // optional confinement seam; see CommandWrapFunc } -// CommandWrapFunc rewrites a command for confined execution. It mirrors -// api.CommandWrapper's Wrap signature so a resolved sandbox adapter plugs in -// directly — hook inputs are untrusted, so a receive path must never exec -// them bare on the host (issue #40 R5.2). -type CommandWrapFunc func(ctx context.Context, cmd string, args, env []string) (string, []string, []string, error) - func (c *CmdVerifier) Verify(ctx context.Context, cwd string, changed []string) (Verdict, error) { args := append([]string(nil), c.Args...) if c.PerFile { args = append(args, changed...) } - timeout := c.Timeout - if timeout <= 0 { - timeout = DefaultCmdTimeout - } - // The verifier's own timeout lives on a derived context; the parent is - // consulted separately below, so a parent deadline shorter than Timeout is - // reported as the run's cancellation, not misattributed to the command. - runCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - tail := c.FeedbackTail - if tail <= 0 { - tail = 4096 - } - output := &tailBuffer{max: tail} - - command, cmdArgs, env := c.Cmd, args, c.Env - if c.Wrap != nil { - wrapEnv := env - if wrapEnv == nil { - wrapEnv = os.Environ() - } - var err error - command, cmdArgs, env, err = c.Wrap(ctx, command, cmdArgs, wrapEnv) - if err != nil { - return Verdict{}, fmt.Errorf("wrapping %s for sandboxed execution: %w", c.Cmd, err) - } - if env == nil { - // A wrapper that supplies no environment keeps the pre-wrap - // boundary. Leaving env nil here would hand the wrapped process - // the full inherited environment — silently widening a caller's - // deliberately reduced Env (the git-agent hook path, issue #40). - env = wrapEnv - } - } - - cmd := exec.CommandContext(runCtx, command, cmdArgs...) - cmd.Dir = cwd - if env != nil { - cmd.Env = env + output := newTailBuffer(c.FeedbackTail) + outcome, err := runProcess(ctx, execRequest{ + Cmd: c.Cmd, Args: args, Dir: cwd, Env: c.Env, Wrap: c.Wrap, Timeout: c.Timeout, + Stdout: output, Stderr: output, + }) + if err != nil { + return Verdict{}, err } - cmd.Stdout, cmd.Stderr = output, output - // Own process group, and cancellation kills the group: signalling only the - // pid leaves a hook's children running after their parent is dead. - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } - // A grandchild that survives the kill holding our output pipe must not hold - // Wait open indefinitely. - cmd.WaitDelay = 10 * time.Second - err := cmd.Run() + node := c.node(args, cwd, outcome.Elapsed, outcome.State) switch { - case err == nil: - return Verdict{OK: true}, nil - case ctx.Err() != nil: - // The parent context ended (cancellation or its own, earlier deadline): - // the run is being torn down, which is not a verdict on the work. - return Verdict{}, ctx.Err() - case errors.Is(runCtx.Err(), context.DeadlineExceeded): - return Verdict{ - OK: false, - Reason: fmt.Sprintf("%s timed out after %s", c.Cmd, timeout), - Feedback: output.String(), - }, nil + case outcome.Err == nil: + node.Passed = true + node.Stdout = output.String() + return c.verdict(Verdict{OK: true}, node), nil + case outcome.TimedOut: + node.Failed, node.TimedOut = true, true + node.Message = fmt.Sprintf("%s timed out after %s", c.Cmd, effectiveTimeout(c.Timeout)) + node.Stderr = output.String() + return c.verdict(Verdict{OK: false, Reason: node.Message, Feedback: output.String()}, node), nil } feedback := output.String() if feedback == "" { - feedback = err.Error() + feedback = outcome.Err.Error() } - return Verdict{OK: false, Reason: c.Cmd + " failed", Feedback: feedback}, nil + node.Failed = true + node.Message = c.Cmd + " failed" + node.Stderr = feedback + return c.verdict(Verdict{OK: false, Reason: node.Message, Feedback: feedback}, node), nil } -// tailBuffer keeps the last max bytes written through it, so a chatty command -// is bounded while it streams instead of being buffered whole and truncated -// afterwards. -type tailBuffer struct { - mu sync.Mutex - max int - buf []byte - truncated bool -} - -func (b *tailBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - b.buf = append(b.buf, p...) - if len(b.buf) > b.max { - copy(b.buf, b.buf[len(b.buf)-b.max:]) - b.buf = b.buf[:b.max] - b.truncated = true +// node is the single leaf a command verifier reports: the command as declared +// (not as wrapped), where it ran, how long it took and how it exited. +func (c *CmdVerifier) node(args []string, cwd string, elapsed time.Duration, state *os.ProcessState) api.VerifyNode { + command := strings.TrimSpace(strings.Join(append([]string{c.Cmd}, args...), " ")) + exitCode := exitCodeOf(state) + return api.VerifyNode{ + Name: command, + Framework: api.VerifyKindCmd, + Command: command, + WorkDir: cwd, + Duration: elapsed, + Context: &api.VerifyNodeContext{Command: command, ExitCode: exitCode, Cwd: cwd}, } - return len(p), nil } -func (b *tailBuffer) String() string { - b.mu.Lock() - defer b.mu.Unlock() - out := strings.TrimSpace(string(b.buf)) - if b.truncated && out != "" { - return "[output truncated]\n" + out - } - return out +func (c *CmdVerifier) verdict(vd Verdict, node api.VerifyNode) Verdict { + report := api.NewNodeReport(api.VerifyKindCmd, node.Command, node) + report.Feedback = vd.Feedback + vd.Report = &report + return vd } diff --git a/pkg/ai/agent/verify/verify_suite_test.go b/pkg/ai/agent/verify/verify_suite_test.go new file mode 100644 index 00000000..3b38b14e --- /dev/null +++ b/pkg/ai/agent/verify/verify_suite_test.go @@ -0,0 +1,13 @@ +package verify + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestVerify(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Agent Verify Suite") +} diff --git a/pkg/ai/agent/verify/workflow.go b/pkg/ai/agent/verify/workflow.go index 4e9a0bc3..2143b15b 100644 --- a/pkg/ai/agent/verify/workflow.go +++ b/pkg/ai/agent/verify/workflow.go @@ -1,126 +1,12 @@ package verify import ( - "fmt" - "strings" - - "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/ai/agent" - "github.com/flanksource/captain/pkg/ai/prompt" "github.com/flanksource/captain/pkg/api" ) -// HooksForWorkflow builds the generate→verify loop's Verify hooks from a spec's -// Workflow: each verify command becomes a pass/fail CmdVerifier whose failure -// output drives a re-run. Returns nil when there is nothing to verify. -// -// Shared by captain prompt-run and gavel so both construct the loop identically -// from an api.Spec.Workflow. Returns []any — the runner's heterogeneous hook list. -func HooksForWorkflow(wf *api.Workflow) []any { - if wf == nil || wf.Verify == nil { - return nil - } - var hooks []any - for _, cmd := range wf.Verify.Commands { - cmd = strings.TrimSpace(cmd) - if cmd == "" { - continue - } - hooks = append(hooks, New("verify:"+cmd, &CmdVerifier{Cmd: "sh", Args: []string{"-c", cmd}})) - } - return hooks -} - -// JudgePrompt is a loaded Verify.Prompts entry awaiting the provider that will -// judge with it. -type JudgePrompt struct { - Path string - Template *prompt.Template -} - -// LoadJudgePrompts loads every Verify.Prompts entry declared by a workflow. -// -// Loading is separate from binding so a caller can reject a broken declaration -// before it builds a provider: a workflow naming a prompt that does not exist -// is wrong on every machine, while the runtime a provider needs is missing only -// on some, and reporting the environment first hides the real defect. -func LoadJudgePrompts(wf *api.Workflow) ([]JudgePrompt, error) { - if wf == nil || wf.Verify == nil || len(wf.Verify.Prompts) == 0 { - return nil, nil - } - var prompts []JudgePrompt - for i, path := range wf.Verify.Prompts { - path = strings.TrimSpace(path) - if path == "" { - // Same rule as api.Workflow.Validate: a blank entry is a broken - // declaration, and skipping it would silently drop a configured check - // for callers that never ran Validate. - return nil, fmt.Errorf("workflow.verify.prompts[%d] is empty", i) - } - tmpl, err := prompt.LoadFile(path) - if err != nil { - return nil, fmt.Errorf("verify prompt %q: %w", path, err) - } - prompts = append(prompts, JudgePrompt{Path: path, Template: tmpl}) - } - return prompts, nil -} - -// JudgeHooks binds loaded judge prompts to the provider that executes them: -// each is a template whose output schema is {ok, reason, feedback}. -func JudgeHooks(prompts []JudgePrompt, provider ai.Provider) ([]any, error) { - if len(prompts) == 0 { - return nil, nil - } - if provider == nil { - return nil, fmt.Errorf("verify prompts declared but no provider available to judge them") - } - var hooks []any - for _, p := range prompts { - if err := rejectJudgeOverrides(p.Path, p.Template, provider); err != nil { - return nil, err - } - hooks = append(hooks, New("judge:"+p.Path, &LLMJudgeVerifier{Provider: provider, Prompt: p.Template})) - } - return hooks, nil -} - -// PromptHooksForWorkflow builds the LLM-judge hooks from Verify.Prompts: each -// entry is a .prompt template whose output schema is {ok, reason, feedback}, -// judged by the given provider. It is separate from HooksForWorkflow because -// judge hooks need a provider and command hooks do not — keeping the original -// signature stable for gavel, which shares it. -// -// A prompt that fails to load is an error, not a skipped hook: a declared -// check that silently never runs is a false accept. -func PromptHooksForWorkflow(wf *api.Workflow, provider ai.Provider) ([]any, error) { - prompts, err := LoadJudgePrompts(wf) - if err != nil { - return nil, err - } - return JudgeHooks(prompts, provider) -} - -// rejectJudgeOverrides refuses judge frontmatter the hook cannot honour. A -// judge executes on the run's provider, so a declared sandbox — relocating or -// not — and a model different from the provider's would both be silently -// ignored, which is exactly the downgrade issue #39 forbids (R5.4: a hook -// prompt declaring a relocating sandbox is a validation error, never a silent -// fallback). -func rejectJudgeOverrides(path string, tmpl *prompt.Template, provider ai.Provider) error { - probe, _, err := tmpl.Render(map[string]any{"cwd": "", "changed": []string{}}, nil) - if err != nil { - return fmt.Errorf("verify prompt %q: %w", path, err) - } - if probe.Sandbox != nil { - return fmt.Errorf("verify prompt %q declares a sandbox; judge hooks run on the run's provider and cannot relocate", path) - } - if declared := strings.TrimSpace(probe.Name); declared != "" && declared != provider.GetModel() { - return fmt.Errorf("verify prompt %q declares model %q but judge hooks run on the run's provider (%s); remove the model or match it", - path, declared, provider.GetModel()) - } - return nil -} +// The workflow → hooks mapping itself lives in registry.go (HooksFor); what +// remains here is the loop shape a Workflow declares around those hooks. // MaxIterationsForWorkflow is the loop cap: the declared maxIterations, else 1 // (a single generation; verification votes once with no automatic re-run). diff --git a/pkg/ai/agent/verify/workflow_test.go b/pkg/ai/agent/verify/workflow_test.go index 967e3282..926db7c0 100644 --- a/pkg/ai/agent/verify/workflow_test.go +++ b/pkg/ai/agent/verify/workflow_test.go @@ -7,22 +7,23 @@ import ( "github.com/flanksource/captain/pkg/api" ) -func TestHooksForWorkflow(t *testing.T) { - if HooksForWorkflow(nil) != nil { - t.Errorf("nil workflow should yield no hooks") - } - if HooksForWorkflow(&api.Workflow{}) != nil { - t.Errorf("workflow without verify should yield no hooks") - } - - wf := &api.Workflow{Verify: &api.Verify{Commands: []string{"go test ./...", " ", "go vet ./..."}}} - hooks := HooksForWorkflow(wf) - if len(hooks) != 2 { - t.Fatalf("want 2 hooks (blank command skipped), got %d", len(hooks)) - } - named, ok := hooks[0].(interface{ Name() string }) - if !ok || named.Name() != "verify:go test ./..." { - t.Errorf("unexpected first hook %v", hooks[0]) +func TestDeclaresExec(t *testing.T) { + for _, test := range []struct { + name string + wf *api.Workflow + want bool + }{ + {name: "nil workflow", wf: nil}, + {name: "no verify stage", wf: &api.Workflow{}}, + {name: "only blank commands", wf: &api.Workflow{Verify: &api.Verify{Commands: []string{" "}}}}, + {name: "a command", wf: &api.Workflow{Verify: &api.Verify{Commands: []string{"go test ./..."}}}, want: true}, + {name: "a fixture", wf: &api.Workflow{Verify: &api.Verify{Fixture: "# acceptance"}}, want: true}, + } { + t.Run(test.name, func(t *testing.T) { + if got := DeclaresExec(test.wf); got != test.want { + t.Errorf("DeclaresExec = %t, want %t", got, test.want) + } + }) } } diff --git a/pkg/ai/agent/verify_order_test.go b/pkg/ai/agent/verify_order_test.go index 4962696b..34df6b72 100644 --- a/pkg/ai/agent/verify_order_test.go +++ b/pkg/ai/agent/verify_order_test.go @@ -31,7 +31,7 @@ func TestVerify_StopsAtFirstFailure(t *testing.T) { require.Len(t, verdicts, 1) assert.False(t, verdicts[len(verdicts)-1].Valid, "the round's last verdict must be the failure") assert.NotNil(t, retry, "the failing hook's retry must be proposed") - assert.False(t, verifyPassed(verdicts)) + assert.False(t, VerifyPassed(verdicts)) } func TestVerify_AllPassingRunsEveryHook(t *testing.T) { @@ -49,5 +49,5 @@ func TestVerify_AllPassingRunsEveryHook(t *testing.T) { assert.Nil(t, retry) assert.Equal(t, 3, calls) assert.Len(t, verdicts, 3) - assert.True(t, verifyPassed(verdicts)) + assert.True(t, VerifyPassed(verdicts)) } diff --git a/pkg/ai/approval/approval_suite_test.go b/pkg/ai/approval/approval_suite_test.go new file mode 100644 index 00000000..4bcca945 --- /dev/null +++ b/pkg/ai/approval/approval_suite_test.go @@ -0,0 +1,13 @@ +package approval_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestApproval(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "AI Approval Broker Suite") +} diff --git a/pkg/ai/approval/broker.go b/pkg/ai/approval/broker.go new file mode 100644 index 00000000..3e44bba1 --- /dev/null +++ b/pkg/ai/approval/broker.go @@ -0,0 +1,233 @@ +// Package approval brokers durable tool approvals for any Captain execution +// that owns a session and a prompt run. +// +// Broker is the api.PermissionFunc every path shares: it records one pending +// captain_turn_requests row, hands the host an api.EventPermission frame to +// surface, and blocks until that row is resolved, expires, or the caller's +// context ends. The aichat execution path supplies its caller-tool credential, +// turn and model call; a streaming provider run (`captain prompt run`) or an +// external host such as a dashboard supplies none of the three and is +// identified by its prompt run and tool call alone. +package approval + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/database" + "github.com/google/uuid" +) + +const ( + // DefaultPoll is how often an unresolved approval is re-read. + DefaultPoll = 100 * time.Millisecond + // CallerToolTimeout bounds an approval raised by a caller tool, which a + // person answers inside a live chat turn. + CallerToolTimeout = 5 * time.Minute + // ProviderTimeout bounds an approval raised by a provider, which suspends + // the run and may be answered long after the process that raised it exited. + ProviderTimeout = 24 * time.Hour +) + +// ErrInvalidBroker reports a Broker that cannot broker anything. +var ErrInvalidBroker = errors.New("invalid approval broker") + +// Broker answers tool-permission requests from the durable approval table. +type Broker struct { + DB *database.DB + SessionID uuid.UUID // captain_sessions.id; required + PromptRunID uuid.UUID // captain_prompt_runs.id; required + + // TurnID, ModelCallID and CredentialID identify a caller-tool approval + // raised inside an aichat turn. A provider or host approval leaves all + // three empty: those executions never open a turn or a model call. + TurnID *uuid.UUID + ModelCallID *uuid.UUID + CredentialID uuid.UUID + + RequestedBy string // who raised it, e.g. "provider" or "caller_tool" + Timeout time.Duration // approval expiry; required + Poll time.Duration // re-read interval; DefaultPoll when zero + + // Notify receives the EventPermission frame carrying the tool, its input, + // the provider tool-call ID and the durable approval ID, so the host can + // surface a request it is expected to answer. Required. + Notify func(context.Context, api.Event) error + + // OnWaiting and OnRunning bracket the wait with the host's own state + // transitions. A credential-less approval depends on OnWaiting: the store + // only resolves one while its prompt run is waiting. + OnWaiting func(context.Context) error + OnRunning func(context.Context) error + + // ClaimToolUseID resolves a request whose tool-use ID the runtime generated + // locally onto the provider's own tool-call ID. Required only when a caller + // can set PermissionRequest.ToolUseIDGenerated. + ClaimToolUseID func(context.Context, api.PermissionRequest) (string, error) +} + +// Validate reports whether the broker names everything it needs to record and +// surface an approval. +func (b *Broker) Validate() error { + var missing []string + if b.DB == nil { + missing = append(missing, "a database") + } + if b.SessionID == uuid.Nil { + missing = append(missing, "a session ID") + } + if b.PromptRunID == uuid.Nil { + missing = append(missing, "a prompt run ID") + } + if b.Notify == nil { + missing = append(missing, "a notify callback") + } + if b.Timeout <= 0 { + missing = append(missing, "a positive timeout") + } + if len(missing) > 0 { + return fmt.Errorf("%w: approvals need %s", ErrInvalidBroker, strings.Join(missing, ", ")) + } + return nil +} + +// CanUseTool is the api.PermissionFunc. It records the pending approval +// idempotently, surfaces it, and blocks until it is answered. +// +// OnWaiting and OnRunning bracket the wait: once the host has been told the run +// is waiting, every way out of this function tells it the run is running again. +// The results are named so the deferred half of that bracket can join its error +// onto whichever exit path fired. +func (b *Broker) CanUseTool( + ctx context.Context, + req api.PermissionRequest, +) (decision api.PermissionDecision, err error) { + if err := b.Validate(); err != nil { + return api.PermissionDecision{}, err + } + if req.ToolUseIDGenerated { + if b.ClaimToolUseID == nil { + return api.PermissionDecision{}, fmt.Errorf( + "%w: tool %q generated its own tool-use ID with no ClaimToolUseID to correlate it", ErrInvalidBroker, req.Tool) + } + toolUseID, err := b.ClaimToolUseID(ctx, req) + if err != nil { + return api.PermissionDecision{}, err + } + req.ToolUseID = toolUseID + } + pending, err := b.DB.CreateToolApprovalRequest(ctx, database.CreateToolApprovalRequestInput{ + CredentialID: b.CredentialID, SessionID: b.SessionID, PromptRunID: b.PromptRunID, + TurnID: optionalUUID(b.TurnID), ModelCallID: optionalUUID(b.ModelCallID), + RequestedBy: b.RequestedBy, ToolCallID: req.ToolUseID, Tool: req.Tool, Input: req.Input, + ExpiresAt: time.Now().Add(b.Timeout), + }) + if err != nil { + return api.PermissionDecision{}, err + } + if b.OnWaiting != nil { + if waitingErr := b.OnWaiting(ctx); waitingErr != nil { + return api.PermissionDecision{}, waitingErr + } + } + // From here the host believes the run is waiting, so every exit has to put it + // back — not just the one that reaches a verdict. A Notify that failed used + // to return straight out, leaving the run parked on an approval no reader was + // ever shown; and a cancelled caller resumed on its own dead context, so the + // transition failed exactly when it mattered. context.WithoutCancel is the + // point: ending the wait is the response to the cancellation, not a victim + // of it. + defer func() { + err = errors.Join(err, b.resume(context.WithoutCancel(ctx))) + }() + if notifyErr := b.Notify(ctx, api.Event{ + Kind: api.EventPermission, Tool: req.Tool, ToolCallID: req.ToolUseID, + ApprovalID: pending.ID.String(), Input: req.Input, + }); notifyErr != nil { + return api.PermissionDecision{}, notifyErr + } + return b.wait(ctx, pending.ID) +} + +func (b *Broker) resume(ctx context.Context) error { + if b.OnRunning == nil { + return nil + } + return b.OnRunning(ctx) +} + +func (b *Broker) wait(ctx context.Context, requestID uuid.UUID) (api.PermissionDecision, error) { + ticker := time.NewTicker(b.poll()) + defer ticker.Stop() + for { + request, err := b.DB.GetTurnRequest(ctx, requestID) + if err != nil { + return api.PermissionDecision{}, err + } + if decision, resolved, err := decide(request); resolved { + return decision, err + } + if request.ExpiresAt != nil && !time.Now().Before(*request.ExpiresAt) { + if err := b.DB.ExpireToolApprovalRequest(ctx, request.ID, + database.TurnRequestStateExpired, "approval timed out"); err != nil { + return api.PermissionDecision{}, err + } + continue + } + // A caller-tool approval outlives its credential only as a leak: the + // credential is the authority the tool would run under. + if request.CredentialID != nil { + if err := b.DB.ValidateCallerToolCredential(ctx, *request.CredentialID); err != nil { + _ = b.DB.ExpireToolApprovalRequest(ctx, request.ID, database.TurnRequestStateCancelled, err.Error()) + return api.PermissionDecision{}, err + } + } + select { + case <-ctx.Done(): + _ = b.DB.ExpireToolApprovalRequest(context.Background(), request.ID, + database.TurnRequestStateCancelled, ctx.Err().Error()) + return api.PermissionDecision{}, ctx.Err() + case <-ticker.C: + } + } +} + +func (b *Broker) poll() time.Duration { + if b.Poll > 0 { + return b.Poll + } + return DefaultPoll +} + +// decide maps a terminal approval row onto its decision; the second result +// reports whether the row is terminal at all. +func decide(request *database.TurnRequest) (api.PermissionDecision, bool, error) { + switch request.State { + case database.TurnRequestStateApproved: + decision := api.PermissionDecision{Allow: true} + if updated, ok := request.Response["updatedInput"].(map[string]any); ok { + decision.UpdatedInput = updated + } + return decision, true, nil + case database.TurnRequestStateDenied: + message := request.Reason + if message == "" { + message = "tool call denied" + } + return api.PermissionDecision{Message: message}, true, nil + case database.TurnRequestStateExpired, database.TurnRequestStateCancelled: + return api.PermissionDecision{}, true, fmt.Errorf("tool approval %s", request.State) + } + return api.PermissionDecision{}, false, nil +} + +func optionalUUID(id *uuid.UUID) uuid.UUID { + if id == nil { + return uuid.Nil + } + return *id +} diff --git a/pkg/ai/approval/broker_ginkgo_test.go b/pkg/ai/approval/broker_ginkgo_test.go new file mode 100644 index 00000000..ab833b32 --- /dev/null +++ b/pkg/ai/approval/broker_ginkgo_test.go @@ -0,0 +1,399 @@ +package approval_test + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/flanksource/captain/pkg/ai/approval" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/database" + "github.com/flanksource/commons-db/dbtest" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" +) + +// brokerPoll keeps the specs responsive without changing what is exercised: the +// broker re-reads the durable row on this interval instead of the package +// default. +const brokerPoll = 20 * time.Millisecond + +type outcome struct { + decision api.PermissionDecision + err error +} + +var _ = Describe("Approval broker", Ordered, func() { + var db *database.DB + + BeforeAll(func(ctx SpecContext) { + handle := dbtest.ForGinkgo(dbtest.Options{Name: "captain_approval_broker"}) + opened, err := database.Open(ctx, database.WithDSN(handle.DSN()), database.WithMigrations()) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(opened.Close()).To(Succeed()) }) + db = opened + }) + + It("blocks on a durable pending row and unblocks with the approved input", func(ctx SpecContext) { + run := newProviderRun(ctx, db) + outcomes := run.callTool(ctx, run.broker(time.Minute), api.PermissionRequest{ + Tool: "Bash", Input: map[string]any{"command": "ls"}, ToolUseID: "toolu_approve", + }) + + event := run.awaitPermission() + Expect(event).To(MatchFields(IgnoreExtras, Fields{ + "Tool": Equal("Bash"), + "ToolCallID": Equal("toolu_approve"), + "Input": Equal(map[string]any{"command": "ls"}), + })) + + pending, err := db.GetTurnRequest(ctx, uuid.MustParse(event.ApprovalID)) + Expect(err).NotTo(HaveOccurred()) + Expect(*pending).To(MatchFields(IgnoreExtras, Fields{ + "State": Equal(database.TurnRequestStatePending), + "RequestedBy": Equal("provider"), + "ToolCallID": Equal("toolu_approve"), + "PromptRunID": PointTo(Equal(run.run)), + "TurnID": BeNil(), + "ModelCallID": BeNil(), + "Request": Equal(map[string]any{"tool": "Bash", "input": map[string]any{"command": "ls"}}), + })) + Consistently(outcomes, 5*brokerPoll).ShouldNot(Receive()) + + _, err = db.ResolveToolApprovalRequest(ctx, database.ResolveToolApprovalRequestInput{ + SessionID: run.session, RequestID: pending.ID, Approved: true, + UpdatedInput: map[string]any{"command": "ls -al"}, ResolvedBy: "dashboard", + }) + Expect(err).NotTo(HaveOccurred()) + + Eventually(outcomes).Should(Receive(Equal(outcome{decision: api.PermissionDecision{ + Allow: true, UpdatedInput: map[string]any{"command": "ls -al"}, + }}))) + Expect(run.hooks()).To(Equal([2]int{1, 1})) + }) + + It("returns the denial reason as the decision message", func(ctx SpecContext) { + run := newProviderRun(ctx, db) + outcomes := run.callTool(ctx, run.broker(time.Minute), api.PermissionRequest{ + Tool: "Write", Input: map[string]any{"path": "go.mod"}, ToolUseID: "toolu_deny", + }) + + event := run.awaitPermission() + _, err := db.ResolveToolApprovalRequest(ctx, database.ResolveToolApprovalRequestInput{ + SessionID: run.session, RequestID: uuid.MustParse(event.ApprovalID), + Approved: false, Reason: "go.mod is off limits", ResolvedBy: "dashboard", + }) + Expect(err).NotTo(HaveOccurred()) + + Eventually(outcomes).Should(Receive(Equal(outcome{ + decision: api.PermissionDecision{Message: "go.mod is off limits"}, + }))) + }) + + It("reuses one durable row when the same tool call is brokered twice", func(ctx SpecContext) { + run := newProviderRun(ctx, db) + request := api.PermissionRequest{ + Tool: "Edit", Input: map[string]any{"path": "main.go"}, ToolUseID: "toolu_retry", + } + first := run.callTool(ctx, run.broker(time.Minute), request) + firstEvent := run.awaitPermission() + second := run.callTool(ctx, run.broker(time.Minute), request) + secondEvent := run.awaitPermission() + Expect(secondEvent.ApprovalID).To(Equal(firstEvent.ApprovalID)) + + requests, err := db.ListTurnRequests(ctx, database.TurnRequestFilter{ + SessionID: run.session, PromptRunID: &run.run, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(requests).To(HaveLen(1)) + + _, err = db.ResolveToolApprovalRequest(ctx, database.ResolveToolApprovalRequestInput{ + SessionID: run.session, RequestID: requests[0].ID, Approved: true, ResolvedBy: "dashboard", + }) + Expect(err).NotTo(HaveOccurred()) + Eventually(first).Should(Receive(Equal(outcome{decision: api.PermissionDecision{Allow: true}}))) + Eventually(second).Should(Receive(Equal(outcome{decision: api.PermissionDecision{Allow: true}}))) + }) + + It("expires an approval nobody answered before its timeout", func(ctx SpecContext) { + run := newProviderRun(ctx, db) + outcomes := run.callTool(ctx, run.broker(150*time.Millisecond), api.PermissionRequest{ + Tool: "Bash", Input: map[string]any{"command": "sleep 1"}, ToolUseID: "toolu_expire", + }) + event := run.awaitPermission() + + var got outcome + Eventually(outcomes, 2*time.Second).Should(Receive(&got)) + Expect(got.err).To(MatchError(ContainSubstring("expired"))) + expired, err := db.GetTurnRequest(ctx, uuid.MustParse(event.ApprovalID)) + Expect(err).NotTo(HaveOccurred()) + Expect(expired.State).To(Equal(database.TurnRequestStateExpired)) + }) + + It("cancels the durable row when the calling context ends", func(ctx SpecContext) { + run := newProviderRun(ctx, db) + callCtx, cancel := context.WithCancel(ctx) + outcomes := run.callTool(callCtx, run.broker(time.Minute), api.PermissionRequest{ + Tool: "Bash", Input: map[string]any{"command": "git push"}, ToolUseID: "toolu_cancel", + }) + event := run.awaitPermission() + cancel() + + var got outcome + Eventually(outcomes).Should(Receive(&got)) + Expect(got.err).To(MatchError(context.Canceled)) + cancelled, err := db.GetTurnRequest(ctx, uuid.MustParse(event.ApprovalID)) + Expect(err).NotTo(HaveOccurred()) + Expect(cancelled.State).To(Equal(database.TurnRequestStateCancelled)) + + // Resuming is the response to the cancellation, not a casualty of it: the + // hook runs on a detached context, so the run leaves `waiting` even though + // the caller's own context is already dead. + Expect(run.hooks()).To(Equal([2]int{1, 1})) + Expect(run.state(ctx)).NotTo(Equal(database.PromptRunStateWaiting), + "a cancelled approval that never resumed leaves the run waiting forever") + }) + + It("resumes the run when the host cannot surface the request", func(ctx SpecContext) { + run := newProviderRun(ctx, db) + broker := run.broker(time.Minute) + unreachable := errors.New("event stream closed") + broker.Notify = func(context.Context, api.Event) error { return unreachable } + + _, err := broker.CanUseTool(ctx, api.PermissionRequest{ + Tool: "Bash", Input: map[string]any{"command": "ls"}, ToolUseID: "toolu_notify_fail", + }) + Expect(err).To(MatchError(unreachable)) + Expect(run.hooks()).To(Equal([2]int{1, 1}), + "a run marked waiting for an approval nobody was shown has to be put back") + Expect(run.state(ctx)).NotTo(Equal(database.PromptRunStateWaiting)) + }) + + It("refuses a generated tool-use ID it cannot correlate, before writing any row", func(ctx SpecContext) { + run := newProviderRun(ctx, db) + broker := run.broker(time.Minute) + broker.ClaimToolUseID = nil + + _, err := broker.CanUseTool(ctx, api.PermissionRequest{ + Tool: "Bash", Input: map[string]any{"command": "ls"}, + ToolUseID: "local_1", ToolUseIDGenerated: true, + }) + Expect(err).To(MatchError(approval.ErrInvalidBroker)) + + requests, listErr := db.ListTurnRequests(ctx, database.TurnRequestFilter{ + SessionID: run.session, PromptRunID: &run.run, + }) + Expect(listErr).NotTo(HaveOccurred()) + Expect(requests).To(BeEmpty(), + "a row keyed on a locally invented ID is one no provider decision could ever match") + Expect(run.hooks()).To(Equal([2]int{0, 0})) + }) + + It("records the provider's own tool-call ID once the claim resolves it", func(ctx SpecContext) { + run := newProviderRun(ctx, db) + broker := run.broker(time.Minute) + var claimed api.PermissionRequest + broker.ClaimToolUseID = func(_ context.Context, req api.PermissionRequest) (string, error) { + claimed = req + return "toolu_provider_1", nil + } + + outcomes := run.callTool(ctx, broker, api.PermissionRequest{ + Tool: "Bash", Input: map[string]any{"command": "ls"}, + ToolUseID: "local_1", ToolUseIDGenerated: true, + }) + event := run.awaitPermission() + Expect(claimed.ToolUseID).To(Equal("local_1"), "the claim is handed the locally generated ID") + Expect(event.ToolCallID).To(Equal("toolu_provider_1")) + + pending, err := db.GetTurnRequest(ctx, uuid.MustParse(event.ApprovalID)) + Expect(err).NotTo(HaveOccurred()) + Expect(pending.ToolCallID).To(Equal("toolu_provider_1"), + "the durable row is keyed on the ID the provider will send a result for") + + _, err = db.ResolveToolApprovalRequest(ctx, database.ResolveToolApprovalRequestInput{ + SessionID: run.session, RequestID: pending.ID, Approved: true, ResolvedBy: "dashboard", + }) + Expect(err).NotTo(HaveOccurred()) + Eventually(outcomes).Should(Receive(Equal(outcome{decision: api.PermissionDecision{Allow: true}}))) + }) + + It("brokers a caller-tool approval under its credential, turn and model call", func(ctx SpecContext) { + run := newCallerToolRun(ctx, db) + outcomes := run.callTool(ctx, run.callerBroker(), api.PermissionRequest{ + Tool: "Bash", Input: map[string]any{"command": "ls"}, ToolUseID: "toolu_caller", + }) + + event := run.awaitPermission() + pending, err := db.GetTurnRequest(ctx, uuid.MustParse(event.ApprovalID)) + Expect(err).NotTo(HaveOccurred()) + Expect(*pending).To(MatchFields(IgnoreExtras, Fields{ + "RequestedBy": Equal("caller_tool"), + "TurnID": PointTo(Equal(run.turn)), + "ModelCallID": PointTo(Equal(run.modelCall)), + "CredentialID": PointTo(Equal(run.credential)), + })) + + _, err = db.ResolveToolApprovalRequest(ctx, database.ResolveToolApprovalRequestInput{ + SessionID: run.session, RequestID: pending.ID, Approved: true, ResolvedBy: "chat", + }) + Expect(err).NotTo(HaveOccurred()) + Eventually(outcomes).Should(Receive(Equal(outcome{decision: api.PermissionDecision{Allow: true}}))) + }) + + It("cancels a caller-tool approval whose credential is revoked mid-wait", func(ctx SpecContext) { + run := newCallerToolRun(ctx, db) + outcomes := run.callTool(ctx, run.callerBroker(), api.PermissionRequest{ + Tool: "Bash", Input: map[string]any{"command": "rm -rf /"}, ToolUseID: "toolu_revoked", + }) + event := run.awaitPermission() + + // The credential is the authority the tool would run under; an approval + // that outlives it is a decision nobody is still entitled to make. + Expect(db.RevokeCallerToolCredential(ctx, run.credential, "session ended")).To(Succeed()) + + var got outcome + Eventually(outcomes, 2*time.Second).Should(Receive(&got)) + Expect(got.err).To(MatchError(database.ErrCallerToolCredentialInactive)) + cancelled, err := db.GetTurnRequest(ctx, uuid.MustParse(event.ApprovalID)) + Expect(err).NotTo(HaveOccurred()) + Expect(cancelled.State).To(Equal(database.TurnRequestStateCancelled)) + }) + + It("accepts a broker that names its database, session, run, notifier and timeout", func() { + Expect(completeBroker(db).Validate()).To(Succeed()) + }) + + DescribeTable("rejects an incomplete broker", + func(strip func(*approval.Broker), missing string) { + broker := completeBroker(db) + strip(broker) + Expect(broker.Validate()).To(MatchError(ContainSubstring(missing))) + }, + Entry("without a database", func(b *approval.Broker) { b.DB = nil }, "database"), + Entry("without a session", func(b *approval.Broker) { b.SessionID = uuid.Nil }, "session"), + Entry("without a prompt run", func(b *approval.Broker) { b.PromptRunID = uuid.Nil }, "prompt run"), + Entry("without a notifier", func(b *approval.Broker) { b.Notify = nil }, "notify"), + Entry("without a timeout", func(b *approval.Broker) { b.Timeout = 0 }, "timeout"), + ) +}) + +func completeBroker(db *database.DB) *approval.Broker { + return &approval.Broker{ + DB: db, SessionID: uuid.New(), PromptRunID: uuid.New(), Timeout: time.Minute, + Notify: func(context.Context, api.Event) error { return nil }, + } +} + +// providerRun is a captain session and prompt run with no turn, model call or +// caller-tool credential — the shape `captain prompt run` and an external host +// present to the broker. +type providerRun struct { + db *database.DB + session uuid.UUID + run uuid.UUID + events chan api.Event + + mu sync.Mutex + waiting int + running int +} + +func newProviderRun(ctx context.Context, db *database.DB) *providerRun { + GinkgoHelper() + session, err := db.CreateOrGetSession(ctx, database.CreateSessionInput{ + ID: uuid.New(), Source: "captain", Provider: "anthropic", + }) + Expect(err).NotTo(HaveOccurred()) + run, err := db.CreatePromptRun(ctx, database.CreatePromptRunInput{SessionID: session.ID}) + Expect(err).NotTo(HaveOccurred()) + return &providerRun{db: db, session: session.ID, run: run.ID, events: make(chan api.Event, 4)} +} + +func (r *providerRun) broker(timeout time.Duration) *approval.Broker { + return &approval.Broker{ + DB: r.db, SessionID: r.session, PromptRunID: r.run, RequestedBy: "provider", + Timeout: timeout, Poll: brokerPoll, Notify: r.notify, + OnWaiting: r.markWaiting, OnRunning: r.markRunning, + } +} + +func (r *providerRun) callTool( + ctx context.Context, + broker *approval.Broker, + request api.PermissionRequest, +) chan outcome { + outcomes := make(chan outcome, 1) + go func() { + defer GinkgoRecover() + decision, err := broker.CanUseTool(ctx, request) + outcomes <- outcome{decision: decision, err: err} + }() + return outcomes +} + +func (r *providerRun) awaitPermission() api.Event { + GinkgoHelper() + var event api.Event + Eventually(r.events).Should(Receive(&event)) + Expect(event.Kind).To(Equal(api.EventPermission)) + Expect(event.ApprovalID).NotTo(BeEmpty()) + return event +} + +func (r *providerRun) notify(ctx context.Context, event api.Event) error { + select { + case r.events <- event: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// markWaiting is the host callback a credential-less approval depends on: +// ResolveToolApprovalRequest only accepts one while its prompt run is waiting. +func (r *providerRun) markWaiting(ctx context.Context) error { + r.mu.Lock() + r.waiting++ + r.mu.Unlock() + return r.setState(ctx, database.PromptRunStateWaiting) +} + +func (r *providerRun) markRunning(ctx context.Context) error { + r.mu.Lock() + r.running++ + r.mu.Unlock() + return r.setState(ctx, database.PromptRunStateRunning) +} + +func (r *providerRun) setState(ctx context.Context, state database.PromptRunState) error { + r.mu.Lock() + defer r.mu.Unlock() + current, err := r.db.GetPromptRun(ctx, r.run) + if err != nil { + return err + } + _, err = r.db.UpdatePromptRun(ctx, database.UpdatePromptRunInput{ + ID: current.ID, ExpectedVersion: current.Version, State: &state, + }) + return err +} + +// state is the prompt run's durable state, which is what a host reading the +// dashboard sees: a run still parked in `waiting` after its approval ended is +// the symptom every unpaired OnWaiting produces. +func (r *providerRun) state(ctx context.Context) database.PromptRunState { + GinkgoHelper() + current, err := r.db.GetPromptRun(ctx, r.run) + Expect(err).NotTo(HaveOccurred()) + return current.State +} + +func (r *providerRun) hooks() [2]int { + r.mu.Lock() + defer r.mu.Unlock() + return [2]int{r.waiting, r.running} +} diff --git a/pkg/ai/approval/caller_tool_run_ginkgo_test.go b/pkg/ai/approval/caller_tool_run_ginkgo_test.go new file mode 100644 index 00000000..2d9ca453 --- /dev/null +++ b/pkg/ai/approval/caller_tool_run_ginkgo_test.go @@ -0,0 +1,81 @@ +package approval_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai/approval" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/database" + "github.com/google/uuid" +) + +// callerToolRun is the other shape a broker is handed: an aichat turn, with the +// model call it was raised inside and the caller-tool credential the tool would +// run under. Its approvals are identified by all four columns, and stay alive +// only as long as that credential does. +type callerToolRun struct { + *providerRun + + turn uuid.UUID + modelCall uuid.UUID + credential uuid.UUID +} + +func newCallerToolRun(ctx context.Context, db *database.DB) *callerToolRun { + GinkgoHelper() + session, err := db.CreateOrGetSession(ctx, database.CreateSessionInput{ + ID: uuid.New(), Source: "aichat", Provider: "anthropic", + }) + Expect(err).NotTo(HaveOccurred()) + + turn, _, err := db.CreateChatTurn(ctx, database.CreateChatTurnInput{ + SessionID: session.ID, ProviderTurnID: "turn-" + uuid.NewString(), + }) + Expect(err).NotTo(HaveOccurred()) + + run, err := db.CreatePromptRun(ctx, database.CreatePromptRunInput{ + SessionID: session.ID, TurnID: &turn.ID, + }) + Expect(err).NotTo(HaveOccurred()) + + modelCall, err := db.CreateChatModelCall(ctx, database.CreateChatModelCallInput{ + TurnID: turn.ID, PromptRunID: run.ID, + Model: "claude-sonnet-5", Provider: "anthropic", Mode: string(api.ModeAPI), + }) + Expect(err).NotTo(HaveOccurred()) + + credential, err := db.CreateCallerToolCredential(ctx, database.CreateCallerToolCredentialInput{ + SessionID: session.ID, PromptRunID: run.ID, Provider: "anthropic", Mode: api.ModeAPI, + SecretHash: uniqueSecretHash(), + Policy: map[string]api.ToolPolicy{"Bash": api.ToolPolicyAsk}, + }) + Expect(err).NotTo(HaveOccurred()) + + return &callerToolRun{ + providerRun: &providerRun{ + db: db, session: session.ID, run: run.ID, events: make(chan api.Event, 4), + }, + turn: turn.ID, modelCall: modelCall, credential: credential.ID, + } +} + +// uniqueSecretHash is the 32 bytes a credential is keyed on. The column is +// unique, so two credentials in one suite cannot share a constant. +func uniqueSecretHash() []byte { + first, second := uuid.New(), uuid.New() + return append(first[:], second[:]...) +} + +// callerBroker names all three identity columns, which is what makes the row a +// caller-tool approval rather than a provider one. +func (r *callerToolRun) callerBroker() *approval.Broker { + return &approval.Broker{ + DB: r.db, SessionID: r.session, PromptRunID: r.run, + TurnID: &r.turn, ModelCallID: &r.modelCall, CredentialID: r.credential, + RequestedBy: "caller_tool", Timeout: approval.CallerToolTimeout, Poll: brokerPoll, + Notify: r.notify, OnWaiting: r.markWaiting, OnRunning: r.markRunning, + } +} diff --git a/pkg/ai/loop.go b/pkg/ai/loop.go index 0d0e214f..28ab2e77 100644 --- a/pkg/ai/loop.go +++ b/pkg/ai/loop.go @@ -3,6 +3,7 @@ package ai import ( "context" "fmt" + "time" ) // LoopOptions configures a RunUntil run. The driver is provider-agnostic but @@ -36,6 +37,12 @@ type LoopIteration struct { Usage Usage Success bool Err error + // StartedAt and FinishedAt bracket the provider call. A host persisting one + // row per turn needs both: sending neither leaves the store's own trigger to + // invent them from the clock at write time, which is after the whole run + // ended and makes every turn look instantaneous and simultaneous. + StartedAt time.Time + FinishedAt time.Time } // LoopResult bundles the entire run. @@ -95,8 +102,10 @@ func RunUntil(ctx context.Context, opts LoopOptions) (*LoopResult, error) { iter := &LoopIteration{ Iteration: len(result.Iterations), Request: req, + StartedAt: time.Now(), } runOneIteration(ctx, opts, req, iter) + iter.FinishedAt = time.Now() result.Iterations = append(result.Iterations, iter) result.TotalCost += iter.CostUSD diff --git a/pkg/ai/types.go b/pkg/ai/types.go index c9e99776..57673707 100644 --- a/pkg/ai/types.go +++ b/pkg/ai/types.go @@ -45,6 +45,10 @@ const ( EventError = api.EventError EventSystem = api.EventSystem EventPermission = api.EventPermission + + EventVerified = api.EventVerified + EventVerifyFailed = api.EventVerifyFailed + EventVerifyProgress = api.EventVerifyProgress ) // Usage is an alias for the canonical api.Usage (per-call token breakdown). diff --git a/pkg/aichat/execution_database.go b/pkg/aichat/execution_database.go index dd5af6e8..c3359388 100644 --- a/pkg/aichat/execution_database.go +++ b/pkg/aichat/execution_database.go @@ -6,21 +6,15 @@ import ( "fmt" "strings" "sync" - "time" "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/approval" "github.com/flanksource/captain/pkg/ai/callertools" "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/database" "github.com/google/uuid" ) -const ( - callerToolApprovalTimeout = 5 * time.Minute - providerApprovalTimeout = 24 * time.Hour - approvalPollInterval = 100 * time.Millisecond -) - type databaseExecution struct { db *database.DB ctx context.Context @@ -144,7 +138,7 @@ func (e *databaseExecution) startCallerTools(ctx context.Context, provider *api. var credentialID uuid.UUID runtime, err := callertools.New(callertools.Options{ Definitions: e.definitions, SessionID: e.session.ID.String(), - ApprovalTimeout: callerToolApprovalTimeout, + ApprovalTimeout: approval.CallerToolTimeout, ValidateCredential: func(ctx context.Context) error { if credentialID == uuid.Nil { return fmt.Errorf("caller-tool credential has not been issued") @@ -152,7 +146,7 @@ func (e *databaseExecution) startCallerTools(ctx context.Context, provider *api. return e.db.ValidateCallerToolCredential(ctx, credentialID) }, CanUseTool: func(ctx context.Context, request api.PermissionRequest) (api.PermissionDecision, error) { - return e.requestApproval(ctx, credentialID, request) + return e.approvalBroker(credentialID).CanUseTool(ctx, request) }, }) if err != nil { @@ -180,44 +174,28 @@ func (e *databaseExecution) startCallerTools(ctx context.Context, provider *api. return nil } -func (e *databaseExecution) requestApproval( - ctx context.Context, - credentialID uuid.UUID, - request api.PermissionRequest, -) (api.PermissionDecision, error) { - if request.ToolUseIDGenerated { - toolUseID, err := e.claimProviderToolUse(ctx, request) - if err != nil { - return api.PermissionDecision{}, err - } - request.ToolUseID = toolUseID - } - expiresAt := time.Now().Add(callerToolApprovalTimeout) - pending, err := e.db.CreateToolApprovalRequest(ctx, database.CreateToolApprovalRequestInput{ - CredentialID: credentialID, SessionID: e.session.ID, TurnID: e.turn.ID, PromptRunID: e.run.ID, - ModelCallID: e.modelCallID, RequestedBy: "caller_tool", - ToolCallID: request.ToolUseID, Tool: request.Tool, Input: request.Input, - ExpiresAt: expiresAt, - }) - if err != nil { - return api.PermissionDecision{}, err - } - if err := e.markWaiting(ctx); err != nil { - return api.PermissionDecision{}, err - } - if err := e.emitApproval(ctx, pending.ID, request); err != nil { - return api.PermissionDecision{}, err +// approvalBroker is this execution's durable tool-approval seam. The caller-tool +// path names its credential, turn and model call, so a host answers it from the +// same captain_turn_requests row a credential-less provider run writes. +func (e *databaseExecution) approvalBroker(credentialID uuid.UUID) *approval.Broker { + // The broker outlives this call and keeps whatever it is handed, so it gets + // copies rather than pointers into the execution: &e.turn.ID and + // &e.modelCallID aim inside state this execution mutates under e.mu (the + // prompt run is swapped wholesale on every runtime bind), which is a live + // read of shared memory from whichever goroutine later records the approval. + e.mu.Lock() + turnID, modelCallID, runID := e.turn.ID, e.modelCallID, e.run.ID + e.mu.Unlock() + return &approval.Broker{ + DB: e.db, SessionID: e.session.ID, PromptRunID: runID, + TurnID: &turnID, ModelCallID: &modelCallID, CredentialID: credentialID, + RequestedBy: "caller_tool", Timeout: approval.CallerToolTimeout, + Notify: e.emit, OnWaiting: e.markWaiting, OnRunning: e.markRunning, + ClaimToolUseID: e.claimProviderToolUse, } - decision, err := e.waitForApproval(ctx, pending.ID) - restoreErr := e.markRunning(ctx) - return decision, errors.Join(err, restoreErr) } -func (e *databaseExecution) emitApproval(ctx context.Context, approvalID uuid.UUID, request api.PermissionRequest) error { - event := api.Event{ - Kind: api.EventPermission, Tool: request.Tool, - ToolCallID: request.ToolUseID, ApprovalID: approvalID.String(), Input: request.Input, - } +func (e *databaseExecution) emit(ctx context.Context, event api.Event) error { select { case e.events <- event: return nil @@ -226,52 +204,6 @@ func (e *databaseExecution) emitApproval(ctx context.Context, approvalID uuid.UU } } -func (e *databaseExecution) waitForApproval( - ctx context.Context, - requestID uuid.UUID, -) (api.PermissionDecision, error) { - ticker := time.NewTicker(approvalPollInterval) - defer ticker.Stop() - for { - request, err := e.db.GetTurnRequest(ctx, requestID) - if err != nil { - return api.PermissionDecision{}, err - } - switch request.State { - case database.TurnRequestStateApproved: - decision := api.PermissionDecision{Allow: true} - if updated, ok := request.Response["updatedInput"].(map[string]any); ok { - decision.UpdatedInput = updated - } - return decision, nil - case database.TurnRequestStateDenied: - message := request.Reason - if message == "" { - message = "tool call denied" - } - return api.PermissionDecision{Message: message}, nil - case database.TurnRequestStateExpired, database.TurnRequestStateCancelled: - return api.PermissionDecision{}, fmt.Errorf("tool approval %s", request.State) - } - if request.ExpiresAt != nil && !time.Now().Before(*request.ExpiresAt) { - if err := e.db.ExpireToolApprovalRequest(ctx, request.ID, database.TurnRequestStateExpired, "approval timed out"); err != nil { - return api.PermissionDecision{}, err - } - continue - } - if err := e.db.ValidateCallerToolCredential(ctx, *request.CredentialID); err != nil { - _ = e.db.ExpireToolApprovalRequest(ctx, request.ID, database.TurnRequestStateCancelled, err.Error()) - return api.PermissionDecision{}, err - } - select { - case <-ctx.Done(): - _ = e.db.ExpireToolApprovalRequest(context.Background(), request.ID, database.TurnRequestStateCancelled, ctx.Err().Error()) - return api.PermissionDecision{}, ctx.Err() - case <-ticker.C: - } - } -} - func (e *databaseExecution) Observe(ctx context.Context, event api.Event) (api.Event, error) { if event.SessionID != "" { if err := e.bindProviderSession(ctx, event.SessionID); err != nil { @@ -286,11 +218,11 @@ func (e *databaseExecution) Observe(ctx context.Context, event api.Event) (api.E if event.ApprovalID != "" { return event, nil } - approval, err := e.createProviderApproval(ctx, event) + pending, err := e.createProviderApproval(ctx, event) if err != nil { return event, err } - event.ApprovalID = approval.ID.String() + event.ApprovalID = pending.ID.String() return event, nil case api.EventResult: if event.ToolApproval != nil { @@ -441,25 +373,6 @@ func (e *databaseExecution) bindProviderSession(ctx context.Context, providerID return nil } -func (e *databaseExecution) markRunning(ctx context.Context) error { - phase := database.PromptRunPhaseGenerate - state := database.PromptRunStateRunning - activity := database.SessionActivityWorking - if err := e.updateRun(ctx, runUpdate{Phase: &phase, State: &state}); err != nil { - return err - } - return e.updateSessionActivity(ctx, activity) -} - -func (e *databaseExecution) markWaiting(ctx context.Context) error { - state := database.PromptRunStateWaiting - activity := database.SessionActivityApproval - if err := e.updateRun(ctx, runUpdate{State: &state}); err != nil { - return err - } - return e.updateSessionActivity(ctx, activity) -} - func (e *databaseExecution) finish(ctx context.Context, success bool, message string, event api.Event) error { e.finishMu.Lock() defer e.finishMu.Unlock() @@ -515,64 +428,3 @@ func (e *databaseExecution) finish(ctx context.Context, success bool, message st e.mu.Unlock() return nil } - -type runUpdate struct { - Phase *database.PromptRunPhase - State *database.PromptRunState - Message *string - ApprovalState *api.ToolApprovalState - ProviderCheckpoint *database.PromptRunCheckpoint - ClearApprovalState bool - ClearProviderCheckpoint bool -} - -func (e *databaseExecution) updateRun(ctx context.Context, update runUpdate) error { - e.mu.Lock() - defer e.mu.Unlock() - input := database.UpdatePromptRunInput{ - ID: e.run.ID, ExpectedVersion: e.run.Version, Phase: update.Phase, State: update.State, - } - if update.Message != nil && *update.Message != "" { - input.Error = update.Message - } - input.ApprovalState = update.ApprovalState - input.ProviderCheckpoint = update.ProviderCheckpoint - input.ClearApprovalState = update.ClearApprovalState - input.ClearProviderCheckpoint = update.ClearProviderCheckpoint - run, err := e.db.UpdatePromptRun(ctx, input) - if err != nil { - return err - } - e.run = run - return nil -} - -func (e *databaseExecution) updateSessionActivity( - ctx context.Context, - activity database.SessionActivityState, -) error { - return e.updateSessionState(ctx, database.SessionLifecycleRunning, activity, "") -} - -func (e *databaseExecution) updateSessionState( - ctx context.Context, - lifecycle database.SessionLifecycleStatus, - activity database.SessionActivityState, - reason string, -) error { - e.mu.Lock() - defer e.mu.Unlock() - session, err := e.db.GetSession(ctx, e.session.ID) - if err != nil { - return err - } - updated, err := e.db.UpdateSessionState(ctx, database.UpdateSessionStateInput{ - ID: session.ID, ExpectedVersion: session.StateVersion, - LifecycleStatus: &lifecycle, ActivityState: &activity, StateReason: &reason, - }) - if err != nil { - return err - } - e.session = updated - return nil -} diff --git a/pkg/aichat/execution_database_correlation.go b/pkg/aichat/execution_database_correlation.go index 814384f8..f4ca261d 100644 --- a/pkg/aichat/execution_database_correlation.go +++ b/pkg/aichat/execution_database_correlation.go @@ -6,6 +6,7 @@ import ( "reflect" "time" + "github.com/flanksource/captain/pkg/ai/approval" "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/database" ) @@ -53,7 +54,7 @@ func (e *databaseExecution) createProviderApproval(ctx context.Context, event ap request, err := e.db.CreateToolApprovalRequest(ctx, database.CreateToolApprovalRequestInput{ SessionID: e.session.ID, TurnID: e.turn.ID, PromptRunID: e.run.ID, ModelCallID: e.modelCallID, ToolCallID: event.ToolCallID, Tool: event.Tool, Input: event.Input, - RequestedBy: "provider", ExpiresAt: time.Now().Add(providerApprovalTimeout), + RequestedBy: "provider", ExpiresAt: time.Now().Add(approval.ProviderTimeout), }) if err != nil { return nil, err diff --git a/pkg/aichat/execution_database_state.go b/pkg/aichat/execution_database_state.go new file mode 100644 index 00000000..98f20a23 --- /dev/null +++ b/pkg/aichat/execution_database_state.go @@ -0,0 +1,88 @@ +package aichat + +import ( + "context" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/database" +) + +func (e *databaseExecution) markRunning(ctx context.Context) error { + phase := database.PromptRunPhaseGenerate + state := database.PromptRunStateRunning + activity := database.SessionActivityWorking + if err := e.updateRun(ctx, runUpdate{Phase: &phase, State: &state}); err != nil { + return err + } + return e.updateSessionActivity(ctx, activity) +} + +func (e *databaseExecution) markWaiting(ctx context.Context) error { + state := database.PromptRunStateWaiting + activity := database.SessionActivityApproval + if err := e.updateRun(ctx, runUpdate{State: &state}); err != nil { + return err + } + return e.updateSessionActivity(ctx, activity) +} + +type runUpdate struct { + Phase *database.PromptRunPhase + State *database.PromptRunState + Message *string + ApprovalState *api.ToolApprovalState + ProviderCheckpoint *database.PromptRunCheckpoint + ClearApprovalState bool + ClearProviderCheckpoint bool +} + +func (e *databaseExecution) updateRun(ctx context.Context, update runUpdate) error { + e.mu.Lock() + defer e.mu.Unlock() + input := database.UpdatePromptRunInput{ + ID: e.run.ID, ExpectedVersion: e.run.Version, Phase: update.Phase, State: update.State, + } + if update.Message != nil && *update.Message != "" { + input.Error = update.Message + } + input.ApprovalState = update.ApprovalState + input.ProviderCheckpoint = update.ProviderCheckpoint + input.ClearApprovalState = update.ClearApprovalState + input.ClearProviderCheckpoint = update.ClearProviderCheckpoint + run, err := e.db.UpdatePromptRun(ctx, input) + if err != nil { + return err + } + e.run = run + return nil +} + +func (e *databaseExecution) updateSessionActivity( + ctx context.Context, + activity database.SessionActivityState, +) error { + return e.updateSessionState(ctx, database.SessionLifecycleRunning, activity, "") +} + +func (e *databaseExecution) updateSessionState( + ctx context.Context, + lifecycle database.SessionLifecycleStatus, + activity database.SessionActivityState, + reason string, +) error { + e.mu.Lock() + defer e.mu.Unlock() + session, err := e.db.GetSession(ctx, e.session.ID) + if err != nil { + return err + } + updated, err := e.db.UpdateSessionState(ctx, database.UpdateSessionStateInput{ + ID: session.ID, ExpectedVersion: session.StateVersion, + LifecycleStatus: &lifecycle, ActivityState: &activity, StateReason: &reason, + }) + if err != nil { + return err + } + e.session = updated + return nil +} diff --git a/pkg/aiflags/defaults.go b/pkg/aiflags/defaults.go index edc3fa71..bfef7731 100644 --- a/pkg/aiflags/defaults.go +++ b/pkg/aiflags/defaults.go @@ -34,26 +34,76 @@ func LoadDefaults() (captainconfig.AIDefaults, error) { return cfg.AI, nil } -// EffectiveDefaults resolves one provider's saved defaults: the per-provider block -// back-filled from the legacy flat keys, with the mode defaulting to api and the -// model to that runtime's built-in default. +// EffectiveDefaults SEEDS a form: one provider's saved defaults with the gaps +// filled from the registry — the mode from Provider.DefaultMode and the model +// from the built-in DefaultModelFor table. +// +// It is for `captain configure` and the whoami/picker surfaces, where proposing +// a value the user can accept or change is exactly right. It must NOT be used to +// decide what a run executes on: proposing there is the silent defaulting this +// package now refuses. See SavedDefaults. func EffectiveDefaults(saved captainconfig.AIDefaults, provider *registry.Provider) (ProviderDefaultView, error) { + view, err := SavedDefaults(saved, provider) + if err != nil { + return ProviderDefaultView{}, err + } + if view.Mode == "" { + view.Mode = string(provider.DefaultMode) + } + if _, err := provider.RequireMode(registry.RuntimeMode(view.Mode)); err != nil { + return ProviderDefaultView{}, err + } + if view.Model == "" { + view.Model = DefaultModelFor(provider, registry.RuntimeMode(view.Mode)) + } + return view, nil +} + +// SavedDefaults RESOLVES a run: strictly what ~/.captain.yaml records for this +// provider, plus the global ai.defaultModel selector as a last config-owned +// fallback. Unset fields come back empty so the caller fails loudly instead of +// inheriting a compiled-in model or mode. +// +// The global default is a compact selector, so it can supply a mode as well as a +// name — which is the whole reason it can stand in for a missing provider block. +func SavedDefaults(saved captainconfig.AIDefaults, provider *registry.Provider) (ProviderDefaultView, error) { if provider == nil { return ProviderDefaultView{}, fmt.Errorf("provider is required") } configured, exists := saved.Providers[provider.Name] mode := registry.RuntimeMode(strings.TrimSpace(configured.Mode)) - if mode == "" { - mode = provider.DefaultMode + model := strings.TrimSpace(configured.Model) + effort := registry.Effort(strings.TrimSpace(configured.ReasoningEffort)) + + if model == "" || mode == "" { + global, err := globalDefaultModel(saved) + if err != nil { + return ProviderDefaultView{}, err + } + // Only adopt the global model when it belongs to this provider; its mode + // is a mechanism and travels regardless. + if model == "" && global.Provider == provider { + model = global.Name + } + if mode == "" { + mode = global.Mode + } } - if _, err := provider.RequireMode(mode); err != nil { - return ProviderDefaultView{}, err + + // A provider that serves exactly one mode leaves nothing to guess: naming it + // is arithmetic, not a default. Without this, configuring one provider and + // then naming a model from another would demand a second `captain configure` + // for a mechanism that was never ambiguous. + if mode == "" { + if modes := provider.Modes(); len(modes) == 1 { + mode = modes[0] + } } - model := strings.TrimSpace(configured.Model) - if model == "" { - model = DefaultModelFor(provider, mode) + if mode != "" { + if _, err := provider.RequireMode(mode); err != nil { + return ProviderDefaultView{}, err + } } - effort := registry.Effort(strings.TrimSpace(configured.ReasoningEffort)) if err := effort.Validate(); err != nil { return ProviderDefaultView{}, err } @@ -62,6 +112,23 @@ func EffectiveDefaults(saved captainconfig.AIDefaults, provider *registry.Provid }, nil } +// globalDefaultModel expands the ai.defaultModel compact selector. An unset key +// yields the zero model, which contributes nothing. +func globalDefaultModel(saved captainconfig.AIDefaults) (registry.Model, error) { + name := strings.TrimSpace(saved.DefaultModel) + if name == "" { + return registry.Model{}, nil + } + model, err := (registry.Model{Name: name}).Expand() + if err != nil { + return registry.Model{}, fmt.Errorf("invalid ai.defaultModel %q in captain config: %w", name, err) + } + if p, _, ok := registry.ProviderForToken(model.Name); ok { + model.Provider = p + } + return model, nil +} + // ApplyDefaults fills a model's unset fields from the saved per-provider defaults, // primary and fallbacks alike. It expects an already-expanded model (see the // package doc) and does not resolve — the caller resolves once, afterwards. @@ -99,13 +166,26 @@ func applyCandidateDefaults(model registry.Model, saved captainconfig.AIDefaults } provider = p } + // A nameless model takes the configured provider, preferring the one + // ai.defaultModel names over the coarser defaultProvider key. + if provider == nil && allowActive { + if global, err := globalDefaultModel(saved); err != nil { + return registry.Model{}, err + } else if global.Provider != nil { + provider = global.Provider + } + } if provider == nil && allowActive { provider, _ = registry.ProviderByName(saved.ActiveProvider()) } if provider == nil { return registry.Model{}, fmt.Errorf("provider cannot be resolved for model %q", model.Name) } - defaults, err := EffectiveDefaults(saved, provider) + // SavedDefaults, not EffectiveDefaults: a run inherits only what the user + // configured. EffectiveDefaults would fill the gaps from the registry's + // built-in tables, which is right for seeding a form and wrong here — it is + // exactly how an unconfigured `--ai-model haiku` silently acquired agent mode. + defaults, err := SavedDefaults(saved, provider) if err != nil { return registry.Model{}, err } diff --git a/pkg/aiflags/unconfigured.go b/pkg/aiflags/unconfigured.go new file mode 100644 index 00000000..12a4d5cf --- /dev/null +++ b/pkg/aiflags/unconfigured.go @@ -0,0 +1,121 @@ +package aiflags + +import ( + "errors" + "fmt" + "strings" + + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/captainconfig" +) + +// ErrUnconfigured marks the "nothing chose a model or a mode" failure so callers +// can recognise it without matching on message text. +var ErrUnconfigured = errors.New("no model configured") + +// UnconfiguredError reports that a run reached the execution boundary without a +// model or a mechanism, and names the commands that fix it. +// +// It exists because the alternative is guessing. captain used to fill the gap +// from a compiled-in table — Provider.DefaultMode and the DefaultModelFor list — +// so an unconfigured `--ai-model haiku` silently became `agent:claude-haiku-4-5` +// and no configuration file said so. Those tables now seed `captain configure` +// only; a run that nothing configured stops here. +type UnconfiguredError struct { + // Field is "model" or "mode": which half is missing. + Field string + // Provider is the family the caller named, when it is known. A missing model + // often means no family is known either, in which case it is nil. + Provider *registry.Provider + // Model is whatever the caller did supply, for context in the message. + Model string +} + +func (e *UnconfiguredError) Unwrap() error { return ErrUnconfigured } + +func (e *UnconfiguredError) Error() string { + var b strings.Builder + if e.Field == "mode" && e.Provider != nil { + // AgentName, not Name: users speak in families ("gemini models"), not + // provider keys ("google models"). + fmt.Fprintf(&b, "no runtime mode configured for %s models", e.Provider.AgentName) + } else if e.Field == "mode" { + b.WriteString("no runtime mode configured") + } else { + b.WriteString("no model configured") + } + if e.Model != "" { + fmt.Fprintf(&b, " (selecting %q)", e.Model) + } + b.WriteString("\n set one with either:") + b.WriteString("\n captain configure") + if e.Provider != nil { + fmt.Fprintf(&b, " %s", e.Provider.Name) + } + b.WriteString("\n gavel configure (per-repo, writes .gavel.yaml)") + b.WriteString("\n or name one inline, e.g. --model agent:claude-sonnet-5") + if e.Field == "mode" && e.Provider != nil { + fmt.Fprintf(&b, "\n modes available for %s: %s", e.Provider.AgentName, modeList(e.Provider.Modes())) + } + return b.String() +} + +func modeList(modes []registry.RuntimeMode) string { + parts := make([]string, len(modes)) + for i, m := range modes { + parts[i] = string(m) + } + return strings.Join(parts, ", ") +} + +// IsUnconfigured reports whether err is the "nothing configured a model" failure. +func IsUnconfigured(err error) bool { return errors.Is(err, ErrUnconfigured) } + +// ResolveForRun is the execution boundary: it applies the saved ~/.captain.yaml +// defaults, refuses to proceed when the selection is still incomplete, and then +// resolves against the catalog. +// +// The refusal happens BEFORE registry.ResolveModel deliberately. The registry is +// the grammar — it parses "api:haiku" into a concrete triple and is used to +// render catalogs, validate prompts and replay recorded history, all of which +// legitimately resolve names they never intend to run. Leaving its provider +// fallback intact and gating here means only runs are strict. +// +// Every path that is about to execute goes through this rather than bare +// ResolveModel; that is what makes configuration the single source of a model. +func ResolveForRun(model registry.Model) (registry.Model, error) { + saved, err := LoadDefaults() + if err != nil { + return registry.Model{}, err + } + return ResolveForRunWith(model, saved) +} + +// ResolveForRunWith is ResolveForRun against an already-loaded config, for +// callers that read ~/.captain.yaml once and resolve many models. +func ResolveForRunWith(model registry.Model, saved captainconfig.AIDefaults) (registry.Model, error) { + applied, err := ApplyDefaults(model, saved) + if err != nil { + return registry.Model{}, err + } + if err := requireConfigured(applied); err != nil { + return registry.Model{}, err + } + return registry.ResolveModel(applied) +} + +func requireConfigured(model registry.Model) error { + if strings.TrimSpace(model.Name) == "" { + return &UnconfiguredError{Field: "model", Provider: model.Provider} + } + if model.Mode == "" { + provider := model.Provider + if provider == nil { + if p, _, ok := registry.ProviderForToken(model.Name); ok { + provider = p + } + } + return &UnconfiguredError{Field: "mode", Provider: provider, Model: model.Name} + } + return nil +} diff --git a/pkg/aiflags/unconfigured_test.go b/pkg/aiflags/unconfigured_test.go new file mode 100644 index 00000000..4153c221 --- /dev/null +++ b/pkg/aiflags/unconfigured_test.go @@ -0,0 +1,144 @@ +package aiflags + +import ( + "strings" + "testing" + + "github.com/flanksource/captain/pkg/api/registry" + "github.com/flanksource/captain/pkg/captainconfig" +) + +// The reported bug in its most reduced form: an unconfigured captain filled the +// mode from Provider.DefaultMode, so a bare model silently became an agent run. +// The registry still does that for parsing and display; a run must not. +func TestResolveForRunRefusesAModeNobodyConfigured(t *testing.T) { + _, err := ResolveForRunWith(registry.Model{Name: "haiku"}, captainconfig.AIDefaults{}) + + if !IsUnconfigured(err) { + t.Fatalf("want an unconfigured error, got %v", err) + } + for _, want := range []string{"no runtime mode configured", "captain configure", "gavel configure"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error must mention %q:\n%s", want, err) + } + } +} + +func TestResolveForRunRefusesAModelNobodyConfigured(t *testing.T) { + _, err := ResolveForRunWith(registry.Model{}, captainconfig.AIDefaults{}) + + if !IsUnconfigured(err) { + t.Fatalf("want an unconfigured error, got %v", err) + } + if !strings.Contains(err.Error(), "no model configured") { + t.Errorf("unexpected message:\n%s", err) + } +} + +// An explicit compact selector is complete on its own and must not need config. +func TestResolveForRunAcceptsAnExplicitSelector(t *testing.T) { + resolved, err := ResolveForRunWith(registry.Model{Name: "api:haiku"}, captainconfig.AIDefaults{}) + if err != nil { + t.Fatalf("explicit selector must resolve without config: %v", err) + } + if resolved.Mode != registry.ModeAPI { + t.Errorf("mode = %q, want api", resolved.Mode) + } + if resolved.Name != "claude-haiku-4-5" { + t.Errorf("name = %q, want claude-haiku-4-5", resolved.Name) + } +} + +// The per-provider block is the primary source, and it supplies the mode a bare +// name is missing. +func TestResolveForRunTakesTheProviderBlock(t *testing.T) { + saved := captainconfig.AIDefaults{Providers: map[string]captainconfig.ProviderDefaults{ + registry.Anthropic.Name: {Mode: "api"}, + }} + + resolved, err := ResolveForRunWith(registry.Model{Name: "haiku"}, saved) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if resolved.Mode != registry.ModeAPI { + t.Errorf("mode = %q, want api from the configured provider block", resolved.Mode) + } +} + +// ai.defaultModel is a compact selector precisely so it can carry a mode; a bare +// name there could not answer the question it exists to answer. +func TestGlobalDefaultModelSuppliesBothHalves(t *testing.T) { + saved := captainconfig.AIDefaults{DefaultModel: "api:claude-haiku-4-5"} + + resolved, err := ResolveForRunWith(registry.Model{}, saved) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if resolved.Name != "claude-haiku-4-5" || resolved.Mode != registry.ModeAPI { + t.Errorf("got %s:%s, want api:claude-haiku-4-5", resolved.Mode, resolved.Name) + } +} + +// A selector naming another provider still needs a mechanism, and the global +// default's mode is a mechanism, so it travels even across families. +func TestGlobalDefaultModeAppliesToAnotherProvidersModel(t *testing.T) { + saved := captainconfig.AIDefaults{DefaultModel: "api:claude-haiku-4-5"} + + resolved, err := ResolveForRunWith(registry.Model{Name: "gemini-3.5-flash"}, saved) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if resolved.Mode != registry.ModeAPI { + t.Errorf("mode = %q, want api", resolved.Mode) + } + if resolved.Name != "gemini-3.5-flash" { + t.Errorf("the global default must not replace an explicitly named model, got %q", resolved.Name) + } +} + +// EffectiveDefaults still seeds forms from the registry — configure and whoami +// need a value to propose. Only the run path is strict. +func TestEffectiveDefaultsStillSeedsAProposal(t *testing.T) { + view, err := EffectiveDefaults(captainconfig.AIDefaults{}, registry.Anthropic) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if view.Mode == "" || view.Model == "" { + t.Errorf("configure/whoami need a proposal, got mode=%q model=%q", view.Mode, view.Model) + } + if view.Configured { + t.Error("an unconfigured provider must still report Configured=false") + } +} + +// SavedDefaults is the run-path view: it reports only what was configured. +func TestSavedDefaultsInventsNothing(t *testing.T) { + view, err := SavedDefaults(captainconfig.AIDefaults{}, registry.Anthropic) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if view.Model != "" { + t.Errorf("model = %q, want empty", view.Model) + } + if view.Mode != "" { + t.Errorf("mode = %q, want empty: anthropic serves several modes, so none is implied", view.Mode) + } +} + +// A provider that serves exactly one mode leaves nothing to guess, so naming it +// is arithmetic rather than a default. Without this, configuring one provider +// and then naming a model from a single-mode family would demand a second +// `captain configure` for a mechanism that was never ambiguous. +func TestSingleModeProviderNeedsNoConfiguredMode(t *testing.T) { + if len(registry.DeepSeek.Modes()) != 1 { + t.Skipf("deepseek now serves %d modes; this test asserts the single-mode case", len(registry.DeepSeek.Modes())) + } + + view, err := SavedDefaults(captainconfig.AIDefaults{}, registry.DeepSeek) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if view.Mode == "" { + t.Error("a provider serving one mode must resolve it without configuration") + } +} diff --git a/pkg/api/runtime_event.go b/pkg/api/runtime_event.go index 79f3c586..88b1dc7a 100644 --- a/pkg/api/runtime_event.go +++ b/pkg/api/runtime_event.go @@ -46,6 +46,25 @@ const ( // the requested tool; the decision itself flows back through the CanUseTool // callback, not through the event stream. EventPermission EventKind = "permission" + + // EventVerified and EventVerifyFailed carry one verify hook's verdict on the + // turn that just finished — the loop's definition of done, reported as it is + // reached. Tool names the check, Text is the report, Duration is its wall + // clock, and on a failure Text carries the output the next turn will be given. + // + // They are distinct kinds rather than an EventSystem line because the verdict + // is the run's outcome, not a narration of it: a renderer has to colour a pass + // and a failure differently, a dashboard filters on them, and a transcript + // reader must be able to find them without matching on prose. + EventVerified EventKind = "verified" + EventVerifyFailed EventKind = "verify_failed" + + // EventVerifyProgress is one in-flight snapshot of a check that has not + // reached a verdict yet: a fixture run's tests as they land, rate-limited so + // a chatty runner does not flood the stream. Tool names the check and Raw + // carries the same *VerifyReport the verdict will, so a renderer redraws the + // tree from one shape whether the check is running or done. + EventVerifyProgress EventKind = "verify_progress" ) // Event is one item in a streaming provider's output channel. @@ -70,7 +89,12 @@ type Event struct { SessionID string // when Kind == EventSystem Model string Error string // when Kind == EventError - Reason string // when Kind == EventInterrupted + Reason string // when Kind == EventInterrupted or EventVerifyFailed + + // Duration is how long the reported work took, when the event reports a + // completed unit of work rather than a fragment of one (EventVerified / + // EventVerifyFailed). Zero elsewhere. + Duration time.Duration // StructuredData is the validated structured output (raw JSON) carried on an // EventResult when the request supplied a schema; nil for text-mode runs. It diff --git a/pkg/api/spec.go b/pkg/api/spec.go index 54449f19..a1a2757e 100644 --- a/pkg/api/spec.go +++ b/pkg/api/spec.go @@ -234,6 +234,25 @@ func (s Spec) IsVerifyOnly() bool { return s.ToolApproval == nil && len(s.Messages) == 0 && s.Prompt.User == "" && len(s.Prompt.Attachments) == 0 && s.Workflow != nil && s.Workflow.Verify != nil } +// ValidateRunnable refuses a spec that names neither work to generate nor work +// to verify — the one classification a run is allowed to make, so that a caller +// building the provider and the runner deciding whether to call it can never +// answer it differently. +// +// The shape this exists for is the near miss: attachments or a message history, +// no prompt body, and no workflow.verify. IsVerifyOnly says no (there is nothing +// to verify), while the runner's own "is the user prompt blank" test said yes — +// so the caller built a provider for a generating run, the runner skipped +// generation, no Verify hook voted, and the run reported a pass having done +// nothing at all. A run with no instruction is an error, never a quiet pass. +func (s Spec) ValidateRunnable() error { + if s.IsVerifyOnly() || s.ToolApproval != nil || strings.TrimSpace(s.Prompt.User) != "" { + return nil + } + return fmt.Errorf("a run needs something to do: prompt.user is empty and workflow.verify is not declared" + + " (attachments and messages accompany a prompt, they do not stand in for one)") +} + func (s Spec) hasPromptBody() bool { return s.Prompt.User != "" || s.Prompt.System != "" || s.Prompt.AppendSystem != "" || len(s.Prompt.Attachments) > 0 } diff --git a/pkg/api/verify_merge.go b/pkg/api/verify_merge.go new file mode 100644 index 00000000..18c1388c --- /dev/null +++ b/pkg/api/verify_merge.go @@ -0,0 +1,141 @@ +package api + +import ( + "fmt" + "strings" + "time" +) + +// MergeReports rolls one verification round's reports into a single report. +// +// A round runs every verifier the workflow declares — `commands` then `fixture`, +// say — and each returns its own report. Keeping the last one threw the rest +// away: the round's row and its result_json.verify carried the fixture's tree +// and nothing else, and the run's summary counted half of what actually ran. +// +// The merged shape keeps each report whole and addressable: one group node per +// report, named after the report and framed by its kind, holding that report's +// tests as children and its summary as the group's own (see VerifyNode.Summary, +// which is what makes the counts add up without re-walking the children). The +// checklists concatenate, the summaries total, and only a round in which every +// report passed passes. +// +// It is an error rather than a silent choice when the reports disagree about the +// turn they judged: two turns' verdicts are not one verdict, and picking one of +// the iteration numbers is how a turn-2 failure gets filed under turn 1. +func MergeReports(name string, reports ...VerifyReport) (VerifyReport, error) { + if len(reports) == 0 { + return VerifyReport{}, fmt.Errorf("merge verify reports %q: no reports to merge", name) + } + iteration, err := sharedIteration(name, reports) + if err != nil { + return VerifyReport{}, err + } + + merged := VerifyReport{Kind: mergedKind(reports), Name: name, Iteration: iteration, Passed: true, Ran: true} + var reasons, feedback []string + for _, r := range reports { + merged.Tests = append(merged.Tests, reportNode(r)) + merged.Checklist = append(merged.Checklist, r.Checklist...) + merged.Summary = AddSummaries(merged.Summary, r.Summary) + merged.Duration += r.Duration + merged.Passed = merged.Passed && r.Passed + merged.Ran = merged.Ran && r.Ran + merged.StartedAt = earliest(merged.StartedAt, r.StartedAt) + merged.FinishedAt = latest(merged.FinishedAt, r.FinishedAt) + if r.Passed { + continue + } + if reason := strings.TrimSpace(r.Reason); reason != "" { + reasons = append(reasons, reason) + } + if f := strings.TrimSpace(r.Feedback); f != "" { + feedback = append(feedback, f) + } + } + merged.Reason = strings.Join(reasons, "; ") + merged.Feedback = strings.Join(feedback, "\n\n") + merged.State = mergedState(merged.Tests, reports) + + if err := merged.Validate(); err != nil { + return VerifyReport{}, fmt.Errorf("merge verify reports %q: %w", name, err) + } + return merged, nil +} + +// reportNode is one report as a group node: its own tests below it, its own +// summary on it. The summary is carried rather than recomputed so a report that +// elided rows still contributes every one of them to the round's totals. +func reportNode(r VerifyReport) VerifyNode { + summary := r.Summary + return VerifyNode{ + Name: r.Name, + Framework: r.Kind, + Message: r.Reason, + Duration: r.Duration, + Summary: &summary, + Children: r.Tests, + } +} + +// mergedState is the tree's state, except that a host-stamped state (errored, +// cancelled) is the host's word about a runner that never reported and no node +// flag maps to it — so it survives the merge rather than being overwritten by +// whatever the queued leaves it left behind imply. errored outranks cancelled: +// a round that broke did not merely stop. +func mergedState(tests []VerifyNode, reports []VerifyReport) VerifyState { + stamped := VerifyState("") + for _, r := range reports { + switch { + case !r.State.HostStamped(): + case r.State == VerifyStateErrored: + return VerifyStateErrored + case stamped == "": + stamped = r.State + } + } + if stamped != "" { + return stamped + } + return StateForReport(tests) +} + +// mergedKind keeps the reports' kind when they share one; a mixed round is a +// round, not a member of any one verifier family. +func mergedKind(reports []VerifyReport) string { + kind := reports[0].Kind + for _, r := range reports[1:] { + if r.Kind != kind { + return VerifyKindRound + } + } + if kind == "" { + return VerifyKindRound + } + return kind +} + +func sharedIteration(name string, reports []VerifyReport) (int, error) { + iteration := reports[0].Iteration + for _, r := range reports[1:] { + if r.Iteration != iteration { + return 0, fmt.Errorf("merge verify reports %q: %q judged iteration %d but %q judged iteration %d; a round's reports judge one turn", + name, reports[0].Name, iteration, r.Name, r.Iteration) + } + } + return iteration, nil +} + +func earliest(into, from *time.Time) *time.Time { + if from == nil || (into != nil && into.Before(*from)) { + return into + } + return from +} + +func latest(into, from *time.Time) *time.Time { + if from == nil || (into != nil && into.After(*from)) { + return into + } + return from +} diff --git a/pkg/api/verify_merge_ginkgo_test.go b/pkg/api/verify_merge_ginkgo_test.go new file mode 100644 index 00000000..7b5bb63a --- /dev/null +++ b/pkg/api/verify_merge_ginkgo_test.go @@ -0,0 +1,126 @@ +package api_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/api" +) + +var _ = Describe("MergeReports", func() { + var ( + earlier = time.Date(2026, 9, 3, 8, 0, 0, 0, time.UTC) + later = time.Date(2026, 9, 3, 8, 5, 0, 0, time.UTC) + ) + + // A round runs every verifier the workflow declares. Keeping the last + // verdict's report threw the others away, so a round of `commands` + `fixture` + // persisted the fixture's tree and nothing else — and the run's summary + // counted half of what actually ran. + cmd := func() api.VerifyReport { + report := api.NewNodeReport(api.VerifyKindCmd, "verify:go test ./...", api.VerifyNode{ + Name: "go test ./...", Framework: api.VerifyKindCmd, Passed: true, Duration: 2 * time.Second, + }) + report.Iteration = 2 + report.Ran = true + started, finished := earlier, earlier.Add(2*time.Second) + report.StartedAt, report.FinishedAt = &started, &finished + return report + } + fixture := func() api.VerifyReport { + report := api.VerifyReport{ + Kind: api.VerifyKindFixture, Name: "acceptance", Ran: true, Iteration: 2, + Passed: false, Reason: "2 of 40 failed", Feedback: "TestFoo: want 3, got 4", + State: api.VerifyStateFailed, + Tests: []api.VerifyNode{ + {Name: "TestFoo", Framework: api.VerifyKindFixture, Failed: true}, + {Name: "TestBar", Framework: api.VerifyKindFixture, Passed: true}, + }, + Summary: api.VerifySummary{Total: 2, Failed: 1, Passed: 1}, + Checklist: []api.VerifyChecklistItem{{Item: "adds a test", Passed: boolPtr(false)}}, + Duration: 30 * time.Second, + } + started, finished := later, later.Add(30*time.Second) + report.StartedAt, report.FinishedAt = &started, &finished + return report + } + + It("nests each report under a group node named after it and totals the counts", func() { + merged, err := api.MergeReports("verify", cmd(), fixture()) + Expect(err).NotTo(HaveOccurred()) + + Expect(merged.Name).To(Equal("verify")) + Expect(merged.Iteration).To(Equal(2)) + Expect(merged.Tests).To(HaveLen(2)) + Expect(merged.Tests[0].Name).To(Equal("verify:go test ./...")) + Expect(merged.Tests[0].Framework).To(Equal(api.VerifyKindCmd)) + Expect(merged.Tests[0].Children).To(HaveLen(1)) + Expect(merged.Tests[0].Summary).To(Equal(&api.VerifySummary{Total: 1, Passed: 1})) + Expect(merged.Tests[1].Name).To(Equal("acceptance")) + Expect(merged.Tests[1].Framework).To(Equal(api.VerifyKindFixture)) + Expect(merged.Tests[1].Children).To(HaveLen(2)) + + Expect(merged.Summary).To(Equal(api.VerifySummary{Total: 3, Passed: 2, Failed: 1})) + Expect(merged.Passed).To(BeFalse()) + Expect(merged.Ran).To(BeTrue()) + Expect(merged.State).To(Equal(api.VerifyStateFailed)) + Expect(merged.Reason).To(ContainSubstring("2 of 40 failed")) + Expect(merged.Feedback).To(ContainSubstring("TestFoo: want 3, got 4")) + Expect(merged.Checklist).To(HaveLen(1)) + Expect(merged.Validate()).To(Succeed()) + }) + + It("takes the earliest start and the latest finish", func() { + merged, err := api.MergeReports("verify", cmd(), fixture()) + Expect(err).NotTo(HaveOccurred()) + Expect(merged.StartedAt).NotTo(BeNil()) + Expect(merged.FinishedAt).NotTo(BeNil()) + Expect(*merged.StartedAt).To(Equal(earlier)) + Expect(*merged.FinishedAt).To(Equal(later.Add(30 * time.Second))) + Expect(merged.Duration).To(Equal(32 * time.Second)) + }) + + It("keeps a shared kind and names a mixed round `round`", func() { + one, two := cmd(), cmd() + two.Name = "verify:make lint" + merged, err := api.MergeReports("verify", one, two) + Expect(err).NotTo(HaveOccurred()) + Expect(merged.Kind).To(Equal(api.VerifyKindCmd)) + Expect(merged.Passed).To(BeTrue()) + Expect(merged.State).To(Equal(api.VerifyStatePassed)) + Expect(merged.Validate()).To(Succeed()) + + mixed, err := api.MergeReports("verify", cmd(), fixture()) + Expect(err).NotTo(HaveOccurred()) + Expect(mixed.Kind).To(Equal(api.VerifyKindRound)) + }) + + // Two reports from different turns are not one verdict, and silently keeping + // one of the iteration numbers is how a turn-2 failure gets filed under turn 1. + It("refuses reports from different iterations", func() { + second := fixture() + second.Iteration = 3 + _, err := api.MergeReports("verify", cmd(), second) + Expect(err).To(MatchError(ContainSubstring("iteration"))) + }) + + It("refuses an empty round", func() { + _, err := api.MergeReports("verify") + Expect(err).To(MatchError(ContainSubstring("no reports"))) + }) + + // errored/cancelled are the host's word and no node flag maps to them, so a + // round containing one keeps it rather than reporting the tree's own state. + It("keeps a host-stamped state over the state the tree implies", func() { + stopped := fixture() + stopped.State = api.VerifyStateCancelled + stopped.Passed = false + merged, err := api.MergeReports("verify", cmd(), stopped) + Expect(err).NotTo(HaveOccurred()) + Expect(merged.State).To(Equal(api.VerifyStateCancelled)) + Expect(merged.Passed).To(BeFalse()) + Expect(merged.Validate()).To(Succeed()) + }) +}) diff --git a/pkg/api/verify_report.go b/pkg/api/verify_report.go new file mode 100644 index 00000000..0f4829a6 --- /dev/null +++ b/pkg/api/verify_report.go @@ -0,0 +1,360 @@ +package api + +import ( + "encoding/json" + "fmt" + "time" +) + +// VerifyState is the terminal (or live) state of a verification report. +type VerifyState string + +const ( + VerifyStateQueued VerifyState = "queued" + VerifyStateRunning VerifyState = "running" + VerifyStatePassed VerifyState = "passed" + VerifyStateFailed VerifyState = "failed" + VerifyStateErrored VerifyState = "errored" + VerifyStateWarned VerifyState = "warned" + VerifyStateSkipped VerifyState = "skipped" + VerifyStateCancelled VerifyState = "cancelled" + VerifyStateTimedOut VerifyState = "timed_out" +) + +// AllVerifyStates lists every state in canonical order. +func AllVerifyStates() []VerifyState { + return []VerifyState{ + VerifyStateQueued, VerifyStateRunning, VerifyStatePassed, VerifyStateFailed, VerifyStateErrored, + VerifyStateWarned, VerifyStateSkipped, VerifyStateCancelled, VerifyStateTimedOut, + } +} + +// Validate rejects a state outside AllVerifyStates. +func (s VerifyState) Validate() error { + for _, known := range AllVerifyStates() { + if s == known { + return nil + } + } + return fmt.Errorf("invalid verify state %q", s) +} + +// Report kinds produced by captain's own verifiers. A registry kind names the +// verifier family that produced a report; hosts add their own (e.g. "fixture"). +const ( + VerifyKindCmd = "cmd" + VerifyKindPrompt = "prompt" + VerifyKindFunc = "func" + VerifyKindFixture = "fixture" + // VerifyKindRound is what MergeReports stamps on a round whose reports do not + // share one kind: the merged document is a whole verification round rather + // than the output of any one verifier family. + VerifyKindRound = "round" +) + +// VerifySummary counts the leaves of a report's test tree. It is the wire twin +// of clicky-ui's TestSummary — a node carrying one is summarised from it rather +// than from its children (see VerifyNode.Summary) — and SummarizeNodes tallies +// leaves exactly as that package's countsFromLeaf does, timed-out bucket +// included. +type VerifySummary struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Warned int `json:"warned"` + Skipped int `json:"skipped"` + Pending int `json:"pending"` + Running int `json:"running"` + // TimedOut is its own bucket rather than a flavour of failed, and is spelled + // `timedout` because that is the key clicky-ui's StatusCounts reads. + TimedOut int `json:"timedout"` +} + +// VerifyNodeProgress is a running node's in-flight position. +type VerifyNodeProgress struct { + Phase string `json:"phase,omitempty"` + Done int `json:"done"` + Total int `json:"total"` +} + +// VerifyNodeContext mirrors clicky-ui's FixtureContext: what a command-shaped +// node ran and, for a CEL expectation, what it evaluated. +type VerifyNodeContext struct { + Command string `json:"command,omitempty"` + ExitCode int `json:"exit_code"` + Cwd string `json:"cwd,omitempty"` + CELExpression string `json:"cel_expression,omitempty"` + CELVars map[string]any `json:"cel_vars,omitempty"` + Expected any `json:"expected,omitempty"` + Actual any `json:"actual,omitempty"` +} + +// VerifyNode is one node of a verification tree. Its JSON is the snake_case +// wire shape of clicky-ui's Test so a TestRunner renders it unchanged; a node +// with Children is a group and carries no verdict of its own. +type VerifyNode struct { + Name string `json:"name"` + Framework string `json:"framework,omitempty"` + TaskID string `json:"task_id,omitempty"` + File string `json:"file,omitempty"` + Line int `json:"line,omitempty"` + Message string `json:"message,omitempty"` + Command string `json:"command,omitempty"` + WorkDir string `json:"work_dir,omitempty"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + Duration time.Duration `json:"duration,omitempty"` + Passed bool `json:"passed,omitempty"` + Failed bool `json:"failed,omitempty"` + Warned bool `json:"warned,omitempty"` + Skipped bool `json:"skipped,omitempty"` + Pending bool `json:"pending,omitempty"` + Running bool `json:"running,omitempty"` + TimedOut bool `json:"timed_out,omitempty"` + Progress *VerifyNodeProgress `json:"progress,omitempty"` + Context *VerifyNodeContext `json:"context,omitempty"` + Detail json.RawMessage `json:"detail,omitempty"` + // Summary is this node's own tally, and it wins over its children — exactly + // as clicky-ui's sum() reads `t.summary` before it recurses. It is what lets + // a producer ship the counts of a suite whose rows it elided (a merged round + // nests one group per report; a 40-test fixture may send its totals and only + // the failures), where recursing would report that suite as empty. + Summary *VerifySummary `json:"summary,omitempty"` + Children []VerifyNode `json:"children,omitempty"` +} + +// VerifyChecklistItem is one acceptance-criteria verdict. Passed is nil while +// the item has not been judged. +type VerifyChecklistItem struct { + Item string `json:"item"` + Passed *bool `json:"passed"` + Message string `json:"message,omitempty"` +} + +// VerifyReport is a verifier's typed judgement: the verdict, the tree of what +// ran, and the acceptance-criteria checklist. It is what a Verify hook returns, +// what captain persists per iteration, and what the webapp renders. +type VerifyReport struct { + Kind string `json:"kind"` + Name string `json:"name,omitempty"` + Ran bool `json:"ran"` + Passed bool `json:"passed"` + Reason string `json:"reason,omitempty"` + Feedback string `json:"feedback,omitempty"` + // Iteration is the 1-based loop turn this report judged ("turn 1 of 3"), the + // same numbering captain_prompt_run_iterations is keyed on. It is always on + // the wire: with omitempty, an unstamped report and the first turn's report + // arrived as the same document, and the store cannot tell them apart. + Iteration int `json:"iteration"` + Summary VerifySummary `json:"summary"` + Tests []VerifyNode `json:"tests,omitempty"` + Checklist []VerifyChecklistItem `json:"checklist,omitempty"` + State VerifyState `json:"state"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Duration time.Duration `json:"duration,omitempty"` +} + +// SummarizeNodes counts the leaves of a tree; a node with children is a group, +// not a result. It mirrors clicky-ui's sum() exactly, so the counters captain +// persists and the ones the webapp recomputes never disagree: +// +// - a node carrying its own Summary is counted from it and never recursed +// into, so an elided child list still contributes the whole suite; +// - a timed-out leaf counts only in TimedOut, never in Failed; +// - a leaf carrying no status flag at all is not counted, Total included — it +// is a placeholder row, not a queued test; +// - otherwise the leaf's own flags are counted, one bucket each. +func SummarizeNodes(nodes []VerifyNode) VerifySummary { + var s VerifySummary + for i := range nodes { + n := &nodes[i] + switch { + case n.Summary != nil: + s = AddSummaries(s, *n.Summary) + case len(n.Children) > 0: + s = AddSummaries(s, SummarizeNodes(n.Children)) + default: + s = AddSummaries(s, summarizeLeaf(n)) + } + } + return s +} + +// summarizeLeaf is clicky-ui's countsFromLeaf: timed out wins outright, and a +// flagless leaf contributes nothing. +func summarizeLeaf(n *VerifyNode) VerifySummary { + if n.TimedOut { + return VerifySummary{Total: 1, TimedOut: 1} + } + var s VerifySummary + if n.Passed { + s.Passed = 1 + } + if n.Failed { + s.Failed = 1 + } + if n.Warned { + s.Warned = 1 + } + if n.Skipped { + s.Skipped = 1 + } + if n.Pending { + s.Pending = 1 + } + if n.Running { + s.Running = 1 + } + if s != (VerifySummary{}) { + s.Total = 1 + } + return s +} + +// AddSummaries totals two summaries, bucket by bucket, so a caller rolling +// several reports into one verdict never re-lists the fields (and never forgets +// the one added last). +func AddSummaries(into, from VerifySummary) VerifySummary { + into.Total += from.Total + into.Passed += from.Passed + into.Failed += from.Failed + into.Warned += from.Warned + into.Skipped += from.Skipped + into.Pending += from.Pending + into.Running += from.Running + into.TimedOut += from.TimedOut + return into +} + +// StateForNode derives a state from one node — the one-node case of +// StateForReport, and defined as it rather than as a second hand-ordered switch. +// Two switches drifted on the leaves that carry contradictory flags (a node both +// skipped and running, both pending and skipped): NewNodeReport stamped the +// state one of them chose and Validate then rejected the report against the +// other. There is one precedence, and it lives in StateForReport. +func StateForNode(n VerifyNode) VerifyState { + return StateForReport([]VerifyNode{n}) +} + +// StateForReport derives a report's state from its whole tree, in the order a +// reader cares about: an outright failure first, then a check that never +// finished, then a soft failure, then anything still moving, then the states a +// finished-and-uneventful tree can be in. An empty tree has not started, so it +// is queued. +// +// It is the one definition of that precedence: StateForNode is this function +// applied to a single node, so the state NewNodeReport stamps and the one +// Validate checks can never disagree. +func StateForReport(tests []VerifyNode) VerifyState { + s := SummarizeNodes(tests) + switch { + case s.Failed > 0: + return VerifyStateFailed + case s.TimedOut > 0: + return VerifyStateTimedOut + case s.Warned > 0: + return VerifyStateWarned + case s.Running > 0: + return VerifyStateRunning + case s.Pending > 0: + return VerifyStateQueued + case s.Skipped > 0: + return VerifyStateSkipped + case s.Passed > 0: + return VerifyStatePassed + default: + return VerifyStateQueued + } +} + +// NewNodeReport builds a report from a single leaf: the verdict is the leaf's +// state, and only a passed leaf passes. +func NewNodeReport(kind, name string, node VerifyNode) VerifyReport { + state := StateForNode(node) + return VerifyReport{ + Kind: kind, + Name: name, + Ran: state != VerifyStateQueued, + Passed: state == VerifyStatePassed, + Reason: node.Message, + Summary: SummarizeNodes([]VerifyNode{node}), + Tests: []VerifyNode{node}, + State: state, + Duration: node.Duration, + } +} + +// HostStamped reports whether the state is one the producing host asserts +// rather than one the tree implies. A runner that could not schedule its nodes +// (errored) or a run stopped mid-check (cancelled) leaves queued leaves behind +// that say nothing about why; no node flag maps to either, so Validate takes +// them as stamped. Neither can pass. +func (s VerifyState) HostStamped() bool { + return s == VerifyStateErrored || s == VerifyStateCancelled +} + +// Validate checks the report's internal consistency: only a passed report +// passes, a report that never ran is queued, running, errored or cancelled, +// the summary matches the leaves of its tree, and — unless the host stamped a +// terminal state of its own — the state is the one that tree justifies. +// +// The state check is what stops a red run reading as green downstream: the +// webapp colours its badge from State while the panel lists the tree, and a CEL +// predicate written against the state would pass on a report whose tests failed. +// +// A live snapshot is a valid report: a ProgressVerifier publishes one that is +// already running before it has run, so `running` joins `queued` as a state a +// not-yet-finished report is allowed to be in. +func (r VerifyReport) Validate() error { + if r.Kind == "" { + return fmt.Errorf("verify report: kind is required") + } + if err := r.State.Validate(); err != nil { + return fmt.Errorf("verify report %q: %w", r.Name, err) + } + if r.Passed && r.State != VerifyStatePassed { + return fmt.Errorf("verify report %q: passed=true with state %q", r.Name, r.State) + } + if want := SummarizeNodes(r.Tests); want != r.Summary { + return fmt.Errorf("verify report %q: summary %+v does not match its %d leaf node(s) %+v", r.Name, r.Summary, want.Total, want) + } + if r.State.HostStamped() { + return nil + } + if !r.Ran && r.State != VerifyStateQueued && r.State != VerifyStateRunning { + return fmt.Errorf("verify report %q: ran=false with state %q; a report that did not run is queued, running, errored or cancelled", r.Name, r.State) + } + if len(r.Tests) > 0 { + if want := StateForReport(r.Tests); want != r.State { + return fmt.Errorf("verify report %q: state %q but its tests are %q", r.Name, r.State, want) + } + } + return nil +} + +// CELVars renders the report as the plain map a host binds to the CEL variable +// `verify` (wire field names, numbers as float64) so predicates such as +// `verify.summary.failed > 0` and `verify.checklist.all(i, i.passed)` read the +// same shape the webapp does. +// +// A report a verifier built with an unencodable detail is an error rather than +// a panic: the caller is a host evaluating a predicate, and it can report a +// broken report far better than a stack unwinding through its evaluator can. +func (r VerifyReport) CELVars() (map[string]any, error) { + raw, err := json.Marshal(r) + if err != nil { + return nil, fmt.Errorf("verify report %q: marshal: %w", r.Name, err) + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("verify report %q: unmarshal: %w", r.Name, err) + } + if out["checklist"] == nil { + out["checklist"] = []any{} + } + if out["tests"] == nil { + out["tests"] = []any{} + } + return out, nil +} diff --git a/pkg/api/verify_report_ginkgo_test.go b/pkg/api/verify_report_ginkgo_test.go new file mode 100644 index 00000000..39063bba --- /dev/null +++ b/pkg/api/verify_report_ginkgo_test.go @@ -0,0 +1,280 @@ +package api_test + +import ( + "encoding/json" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/api" +) + +var _ = Describe("VerifyReport", func() { + leaf := func(name string, set func(*api.VerifyNode)) api.VerifyNode { + n := api.VerifyNode{Name: name} + set(&n) + return n + } + passed := func(n *api.VerifyNode) { n.Passed = true } + failed := func(n *api.VerifyNode) { n.Failed = true } + + Describe("wire shape", func() { + It("marshals with clicky-ui's snake_case Test keys", func() { + started := time.Date(2026, 9, 3, 8, 0, 0, 0, time.UTC) + report := api.NewNodeReport(api.VerifyKindCmd, "verify:go test", api.VerifyNode{ + Name: "go test", TaskID: "t1", WorkDir: "/repo", TimedOut: true, Duration: time.Second, + Context: &api.VerifyNodeContext{Command: "go test", ExitCode: 2, Cwd: "/repo", CELExpression: "exitCode == 0"}, + }) + report.StartedAt = &started + + raw, err := json.Marshal(report) + Expect(err).NotTo(HaveOccurred()) + var doc map[string]any + Expect(json.Unmarshal(raw, &doc)).To(Succeed()) + + Expect(doc).To(HaveKey("started_at")) + node := doc["tests"].([]any)[0].(map[string]any) + Expect(node).To(HaveKeyWithValue("task_id", "t1")) + Expect(node).To(HaveKeyWithValue("work_dir", "/repo")) + Expect(node).To(HaveKeyWithValue("timed_out", true)) + context := node["context"].(map[string]any) + Expect(context).To(HaveKeyWithValue("exit_code", 2.0)) + Expect(context).To(HaveKeyWithValue("cel_expression", "exitCode == 0")) + Expect(report.State).To(Equal(api.VerifyStateTimedOut)) + Expect(report.Passed).To(BeFalse()) + }) + }) + + Describe("SummarizeNodes", func() { + It("counts nested leaves only and never a group", func() { + tree := []api.VerifyNode{{ + Name: "suite", + Children: []api.VerifyNode{ + leaf("a", passed), + leaf("b", failed), + leaf("c", func(n *api.VerifyNode) { n.Warned = true }), + {Name: "nested", Children: []api.VerifyNode{ + leaf("d", func(n *api.VerifyNode) { n.Running = true }), + leaf("e", func(n *api.VerifyNode) { n.Pending = true }), + }}, + }, + }} + Expect(api.SummarizeNodes(tree)).To(Equal(api.VerifySummary{Total: 5, Passed: 1, Failed: 1, Warned: 1, Running: 1, Pending: 1})) + }) + + // clicky-ui's countsFromLeaf gives a flagless leaf total 0: it is a + // placeholder row, not a queued test, and counting it inflates every + // denominator a progress bar divides by. + It("does not count a leaf carrying no status flag at all", func() { + Expect(api.SummarizeNodes([]api.VerifyNode{leaf("blank", func(*api.VerifyNode) {})})). + To(Equal(api.VerifySummary{})) + }) + + // A timed-out leaf is its own bucket, exactly as countsFromLeaf splits it: + // folding it into failed loses the only signal that says "this never + // finished" rather than "this ran and disagreed". + It("counts a timed-out leaf as timed out and not as failed", func() { + timedOut := leaf("t", func(n *api.VerifyNode) { n.TimedOut, n.Failed = true, true }) + Expect(api.SummarizeNodes([]api.VerifyNode{timedOut})). + To(Equal(api.VerifySummary{Total: 1, TimedOut: 1})) + }) + + It("marshals the timed-out counter under clicky-ui's `timedout` key", func() { + raw, err := json.Marshal(api.VerifySummary{Total: 1, TimedOut: 1}) + Expect(err).NotTo(HaveOccurred()) + Expect(string(raw)).To(ContainSubstring(`"timedout":1`)) + }) + + // clicky-ui's sum() reads a node's own summary before it looks at children, + // which is what lets a producer ship the counts of a suite whose child list + // it elided. Recursing anyway reported that suite as empty. + It("counts a group from its own summary rather than from its children", func() { + group := api.VerifyNode{ + Name: "acceptance", Framework: api.VerifyKindFixture, + Summary: &api.VerifySummary{Total: 40, Passed: 39, Failed: 1}, + Children: []api.VerifyNode{leaf("the one row that was kept", failed)}, + } + Expect(api.SummarizeNodes([]api.VerifyNode{group})). + To(Equal(api.VerifySummary{Total: 40, Passed: 39, Failed: 1})) + }) + + It("validates a report whose group carries a summary and an elided child list", func() { + report := api.VerifyReport{ + Kind: api.VerifyKindFixture, Name: "acceptance", Ran: true, State: api.VerifyStateFailed, + Tests: []api.VerifyNode{{ + Name: "acceptance", Framework: api.VerifyKindFixture, + Summary: &api.VerifySummary{Total: 40, Passed: 39, Failed: 1}, + }}, + Summary: api.VerifySummary{Total: 40, Passed: 39, Failed: 1}, + } + Expect(report.Validate()).To(Succeed()) + }) + + It("marshals a node summary under `summary` and omits it when absent", func() { + raw, err := json.Marshal(api.VerifyNode{Name: "suite", Summary: &api.VerifySummary{Total: 2, Passed: 2}}) + Expect(err).NotTo(HaveOccurred()) + Expect(string(raw)).To(ContainSubstring(`"summary":{"total":2,"passed":2`)) + + raw, err = json.Marshal(api.VerifyNode{Name: "leaf", Passed: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(string(raw)).NotTo(ContainSubstring("summary")) + }) + }) + + Describe("StateForReport", func() { + DescribeTable("derives the report's state from the whole tree", + func(nodes []api.VerifyNode, want api.VerifyState) { + Expect(api.StateForReport(nodes)).To(Equal(want)) + }, + Entry("no tests at all is queued", []api.VerifyNode{}, api.VerifyStateQueued), + Entry("a failure outranks everything", []api.VerifyNode{ + leaf("a", passed), leaf("b", failed), leaf("c", func(n *api.VerifyNode) { n.TimedOut = true }), + }, api.VerifyStateFailed), + Entry("a timeout outranks a warning", []api.VerifyNode{ + leaf("a", func(n *api.VerifyNode) { n.TimedOut = true }), leaf("b", func(n *api.VerifyNode) { n.Warned = true }), + }, api.VerifyStateTimedOut), + Entry("a warning outranks a still-running leaf", []api.VerifyNode{ + leaf("a", func(n *api.VerifyNode) { n.Warned = true }), leaf("b", func(n *api.VerifyNode) { n.Running = true }), + }, api.VerifyStateWarned), + Entry("anything still running keeps the report running", []api.VerifyNode{ + leaf("a", passed), leaf("b", func(n *api.VerifyNode) { n.Running = true }), + }, api.VerifyStateRunning), + Entry("anything still pending keeps the report queued", []api.VerifyNode{ + leaf("a", passed), leaf("b", func(n *api.VerifyNode) { n.Pending = true }), + }, api.VerifyStateQueued), + Entry("all skipped is skipped", []api.VerifyNode{leaf("a", func(n *api.VerifyNode) { n.Skipped = true })}, api.VerifyStateSkipped), + Entry("all passed is passed", []api.VerifyNode{leaf("a", passed), leaf("b", passed)}, api.VerifyStatePassed), + ) + + It("agrees with StateForNode for every single-leaf report", func() { + for _, node := range []api.VerifyNode{ + leaf("p", passed), leaf("f", failed), + leaf("t", func(n *api.VerifyNode) { n.TimedOut, n.Failed = true, true }), + leaf("w", func(n *api.VerifyNode) { n.Warned = true }), + leaf("s", func(n *api.VerifyNode) { n.Skipped = true }), + leaf("r", func(n *api.VerifyNode) { n.Running = true }), + leaf("blank", func(*api.VerifyNode) {}), + // Contradictory flags are where two hand-ordered switches drift: the + // leaf branch preferred skipped over running and skipped over + // pending, so NewNodeReport stamped a state Validate then rejected. + leaf("skipped+running", func(n *api.VerifyNode) { n.Skipped, n.Running = true, true }), + leaf("pending+skipped", func(n *api.VerifyNode) { n.Pending, n.Skipped = true, true }), + leaf("failed+passed", func(n *api.VerifyNode) { n.Failed, n.Passed = true, true }), + } { + Expect(api.StateForReport([]api.VerifyNode{node})).To(Equal(api.StateForNode(node)), + "NewNodeReport stamps StateForNode, and Validate checks StateForReport: the two must not disagree") + Expect(api.NewNodeReport(api.VerifyKindCmd, node.Name, node).Validate()).To(Succeed(), + "a report NewNodeReport built must survive its own Validate") + } + }) + }) + + Describe("Validate", func() { + It("accepts a report built from a passing leaf", func() { + Expect(api.NewNodeReport(api.VerifyKindCmd, "ok", leaf("ok", passed)).Validate()).To(Succeed()) + }) + + It("rejects passed=true with a failed state", func() { + report := api.NewNodeReport(api.VerifyKindCmd, "bad", leaf("bad", failed)) + report.Passed = true + Expect(report.Validate()).To(MatchError(ContainSubstring("passed=true with state \"failed\""))) + }) + + It("rejects a summary that disagrees with the leaves", func() { + report := api.NewNodeReport(api.VerifyKindCmd, "drift", leaf("drift", passed)) + report.Summary.Failed = 1 + Expect(report.Validate()).To(MatchError(ContainSubstring("summary"))) + }) + + It("rejects a report that did not run but claims a verdict state", func() { + report := api.VerifyReport{Kind: api.VerifyKindCmd, State: api.VerifyStatePassed} + Expect(report.Validate()).To(MatchError(ContainSubstring("ran=false"))) + }) + + It("requires a kind and a known state", func() { + Expect(api.VerifyReport{State: api.VerifyStateQueued}.Validate()).To(MatchError(ContainSubstring("kind is required"))) + Expect(api.VerifyReport{Kind: api.VerifyKindCmd, State: "nope"}.Validate()).To(MatchError(ContainSubstring("invalid verify state"))) + }) + + // A state that disagrees with the tree is how a red run reads as green + // downstream: the webapp colours the badge from State while the panel + // lists the failures, and a CEL predicate on the state passes. + It("rejects a state that its own tests do not justify", func() { + report := api.NewNodeReport(api.VerifyKindCmd, "drift", leaf("drift", failed)) + report.State = api.VerifyStateWarned + Expect(report.Validate()).To(MatchError(ContainSubstring(`state "warned" but its tests are "failed"`))) + }) + + // errored and cancelled are the host's word, not the tree's: a runner + // that could not schedule its nodes, or a run stopped mid-check, leaves + // queued leaves behind that say nothing about why. No node flag maps to + // either state, so they are accepted as stamped — and only as failures. + DescribeTable("accepts a host-stamped terminal state over nodes that never ran", + func(state api.VerifyState) { + report := api.VerifyReport{ + Kind: api.VerifyKindFixture, Name: "acceptance", State: state, Reason: "runner exited", + Tests: []api.VerifyNode{leaf("go test", func(n *api.VerifyNode) { n.Pending = true })}, + Summary: api.VerifySummary{Total: 1, Pending: 1}, + } + Expect(report.Validate()).To(Succeed()) + + report.Ran = true + Expect(report.Validate()).To(Succeed(), "a check that started and then was cancelled did run") + + report.Passed = true + Expect(report.Validate()).To(MatchError(ContainSubstring("passed=true with state"))) + }, + Entry("errored", api.VerifyStateErrored), + Entry("cancelled", api.VerifyStateCancelled), + ) + + // The report is also the live snapshot a ProgressVerifier publishes: it + // has not finished, so Ran is false while the tree is already running. + It("accepts an in-flight snapshot that is running before it has run", func() { + snapshot := api.VerifyReport{ + Kind: api.VerifyKindCmd, Ran: false, State: api.VerifyStateRunning, + Tests: []api.VerifyNode{leaf("go test", func(n *api.VerifyNode) { n.Running = true })}, + Summary: api.VerifySummary{Total: 1, Running: 1}, + } + Expect(snapshot.Validate()).To(Succeed()) + }) + }) + + Describe("CELVars", func() { + It("exposes the summary counters as numbers and never a nil checklist", func() { + vars, err := api.NewNodeReport(api.VerifyKindCmd, "cmd", leaf("cmd", failed)).CELVars() + Expect(err).NotTo(HaveOccurred()) + summary := vars["summary"].(map[string]any) + Expect(summary["failed"]).To(BeNumerically("==", 1)) + Expect(vars["passed"]).To(BeFalse()) + Expect(vars["checklist"]).To(Equal([]any{})) + }) + + // A detail a verifier could not encode is a broken report, and a host + // binding it into a CEL predicate deserves the error rather than a + // panic unwinding through its evaluator. + It("returns an error for a detail that cannot be marshalled", func() { + report := api.NewNodeReport(api.VerifyKindCmd, "cmd", leaf("cmd", failed)) + report.Tests[0].Detail = json.RawMessage(`{"unterminated":`) + _, err := report.CELVars() + Expect(err).To(MatchError(ContainSubstring("cmd"))) + }) + }) + + Describe("Iteration", func() { + // 1-based, and always on the wire: `omitempty` erased iteration 0, which + // meant "unstamped" and "the first turn" arrived as the same document. + It("marshals iteration 1 rather than dropping it as a zero value", func() { + report := api.NewNodeReport(api.VerifyKindCmd, "cmd", leaf("cmd", passed)) + raw, err := json.Marshal(report) + Expect(err).NotTo(HaveOccurred()) + Expect(string(raw)).To(ContainSubstring(`"iteration":0`)) + + report.Iteration = 1 + raw, err = json.Marshal(report) + Expect(err).NotTo(HaveOccurred()) + Expect(string(raw)).To(ContainSubstring(`"iteration":1`)) + }) + }) +}) diff --git a/pkg/api/workflow.go b/pkg/api/workflow.go index b20a0b9e..98c6367f 100644 --- a/pkg/api/workflow.go +++ b/pkg/api/workflow.go @@ -38,9 +38,15 @@ type Verify struct { // executes. Commands []string `json:"commands,omitempty" yaml:"commands,omitempty"` // Fixture is a clicky-FixtureEditor markdown document (acceptance criteria / - // LLM-judge checklist). Captain declares and reflects it in the spec schema - // for the SpecRuntimeEditor, but does not execute it — only gavel runs - // fixtures. + // LLM-judge checklist). Captain does not implement a fixture engine, but it + // does dispatch this document: the declaration is handed to whatever claimed + // the `fixture` verifier — a runner the host linked in-process, or the + // external program named by `verify.fixtureRunner` in ~/.captain.yaml, which + // receives the document on stdin and answers with a report. + // + // Declaring a fixture with neither registered is an error, not a pass: a + // definition of done that contributes no check would let the run succeed + // without ever being verified. Fixture string `json:"fixture,omitempty" yaml:"fixture,omitempty"` // Prompts are .prompt template paths run as LLM-judge checks: each renders // against the run's context and must yield {ok, reason, feedback}; a false diff --git a/pkg/api/workspace.go b/pkg/api/workspace.go index 2621769b..07f706a5 100644 --- a/pkg/api/workspace.go +++ b/pkg/api/workspace.go @@ -48,12 +48,32 @@ type Notice struct { At time.Time `json:"at" yaml:"at"` Phase string `json:"phase,omitempty" yaml:"phase,omitempty"` Text string `json:"text" yaml:"text"` + // Kind is the event kind this notice was reported as, so a reader can tell a + // verify verdict from a commit line without matching on prose. Empty means + // EventSystem — the generic lifecycle narration most hooks emit. + Kind EventKind `json:"kind,omitempty" yaml:"kind,omitempty"` + // Report is the typed verdict a verify notice reports on. Text is that + // verdict's headline; the tree, the checklist and the counters live here, so a + // stored transcript carries the same document the live stream did instead of + // one sentence about it. + Report *VerifyReport `json:"report,omitempty" yaml:"report,omitempty"` } -// AddNotice appends a notice; nil-safe convenience for hooks. +// AddNotice appends a generic lifecycle notice; nil-safe convenience for hooks. func (w *Workspace) AddNotice(at time.Time, phase, text string) { + w.AddKindNotice(at, phase, text, EventSystem) +} + +// AddKindNotice appends a notice reported under a specific event kind. +func (w *Workspace) AddKindNotice(at time.Time, phase, text string, kind EventKind) { + w.AddNoticeRecord(Notice{At: at, Phase: phase, Text: text, Kind: kind}) +} + +// AddNoticeRecord appends a fully-formed notice; nil-safe. It is the way to +// record one that carries a typed report alongside its prose. +func (w *Workspace) AddNoticeRecord(n Notice) { if w == nil { return } - w.Notices = append(w.Notices, Notice{At: at, Phase: phase, Text: text}) + w.Notices = append(w.Notices, n) } diff --git a/pkg/captainconfig/config.go b/pkg/captainconfig/config.go index 6b32d575..f122f830 100644 --- a/pkg/captainconfig/config.go +++ b/pkg/captainconfig/config.go @@ -26,8 +26,24 @@ type Config struct { Credentials CredentialDefaults `yaml:"credentials,omitempty"` Runtime RuntimeDefaults `yaml:"runtime,omitempty"` Chat ChatDefaults `yaml:"chat,omitempty"` + Verify VerifyDefaults `yaml:"verify,omitempty"` } +// VerifyDefaults is the verify block of ~/.captain.yaml: how this host runs the +// parts of a workflow's verification captain does not implement itself. +type VerifyDefaults struct { + // FixtureRunner is the argv of the external program that executes a + // workflow's `verify.fixture` document — captain declares fixtures but does + // not run them. It is handed the fixture on stdin and answers with a + // VerifyReport (see agent/verify.ExternalVerifier). Empty means this host + // runs no fixtures, and a workflow that declares one fails rather than + // passing without its definition of done. + FixtureRunner []string `yaml:"fixtureRunner,omitempty"` +} + +// IsZero lets yaml omit an empty verify block instead of writing `verify: {}`. +func (v VerifyDefaults) IsZero() bool { return len(v.FixtureRunner) == 0 } + // RuntimeDefaults names extra directories of runtime preset and profile YAML // files, one record per file. Relative paths resolve against the config file's // directory, like Prompts.Dirs; the implicit ~/.config/captain and repo-local @@ -147,9 +163,17 @@ func (a AttachmentDefaults) WithDefaults() AttachmentDefaults { } type AIDefaults struct { - DefaultProvider string `yaml:"defaultProvider,omitempty"` - Providers map[string]ProviderDefaults `yaml:"providers,omitempty"` - Disabled DisabledSelections `yaml:"disabled,omitempty"` + DefaultProvider string `yaml:"defaultProvider,omitempty"` + // DefaultModel is the global fallback when no provider block, prompt, spec or + // flag names a model. It is a COMPACT SELECTOR ("agent:claude-sonnet-5"), not + // a bare name: a name alone cannot carry a mode, and a mode is exactly what + // the caller is missing when it falls through to here. + // + // It is the single value that makes a one-line ~/.captain.yaml sufficient, and + // the last stop before ResolveForRun refuses to guess. + DefaultModel string `yaml:"defaultModel,omitempty"` + Providers map[string]ProviderDefaults `yaml:"providers,omitempty"` + Disabled DisabledSelections `yaml:"disabled,omitempty"` // File-wide generation settings. These are global on purpose: they are // properties of a run, not of a provider. diff --git a/pkg/captainconfig/config_test.go b/pkg/captainconfig/config_test.go index 107d74f7..32caa703 100644 --- a/pkg/captainconfig/config_test.go +++ b/pkg/captainconfig/config_test.go @@ -88,6 +88,42 @@ func TestSaveLoad_RoundTrip(t *testing.T) { } } +func TestVerifyDefaults_RoundTrip(t *testing.T) { + path := withTempPath(t) + want := VerifyDefaults{FixtureRunner: []string{"gavel", "fixture", "verify"}} + if err := Save(Config{Verify: want}); err != nil { + t.Fatalf("Save() err = %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read config: %v", err) + } + if !strings.Contains(string(data), "fixtureRunner:") { + t.Fatalf("saved config has no fixture runner:\n%s", data) + } + + got, _, err := Load() + if err != nil { + t.Fatalf("Load() err = %v", err) + } + if !reflect.DeepEqual(got.Verify, want) { + t.Errorf("verify round-trip:\n got = %+v\n want = %+v", got.Verify, want) + } + + // An unconfigured host writes no verify block at all, rather than an empty + // one a reader would have to interpret. + if err := Save(Config{}); err != nil { + t.Fatalf("Save() err = %v", err) + } + data, err = os.ReadFile(path) + if err != nil { + t.Fatalf("read config: %v", err) + } + if strings.Contains(string(data), "verify:") { + t.Errorf("empty verify block written:\n%s", data) + } +} + func TestUpdatePreservesUnrelatedConfiguration(t *testing.T) { withTempPath(t) wantPrompt := PromptDefaults{Dirs: []string{"/repo/prompts"}} diff --git a/pkg/claude/tools/tool.go b/pkg/claude/tools/tool.go index 849da032..0aa6cb2c 100644 --- a/pkg/claude/tools/tool.go +++ b/pkg/claude/tools/tool.go @@ -241,6 +241,10 @@ func NewTool(base BaseTool) Tool { return &UserTool{BaseTool: base} case "System": return &SystemTool{BaseTool: base} + case "Verified": + return &VerifyTool{BaseTool: base} + case "Verify failed": + return &VerifyTool{BaseTool: base, Failed: true} case "Assistant": return &AssistantTool{BaseTool: base} case "Reasoning": diff --git a/pkg/claude/tools/verify.go b/pkg/claude/tools/verify.go new file mode 100644 index 00000000..ce9225ee --- /dev/null +++ b/pkg/claude/tools/verify.go @@ -0,0 +1,52 @@ +package tools + +import ( + "github.com/flanksource/clicky/api" + "github.com/flanksource/clicky/api/icons" +) + +// VerifyTool renders one verify verdict — the loop's definition of done, voting +// on the turn that just finished. +// +// It is its own row rather than a system line because it is the run's outcome: +// a reader scanning a transcript for why a run stopped is looking for exactly +// these, and a pass and a failure have to be told apart at a glance rather than +// by reading the sentence. +type VerifyTool struct { + BaseTool + // Failed distinguishes the two roles this renders; NewTool sets it from the + // name so the row can colour itself without re-reading the text. + Failed bool +} + +func (t *VerifyTool) Name() string { + if t.Failed { + return "Verify failed" + } + return "Verified" +} + +func (t *VerifyTool) Category() string { return "verify" } +func (t *VerifyTool) FilePath() string { return "" } +func (t *VerifyTool) ExtractPath() string { return "" } + +func (t *VerifyTool) Pretty() api.Text { + icon := icons.Icon{Unicode: "✓", Iconify: "mdi:check-circle", Style: "text-green-500"} + label, color := "verified", "text-green-500 font-medium" + if t.Failed { + icon = icons.Icon{Unicode: "✗", Iconify: "mdi:close-circle", Style: "text-red-500"} + label, color = "verify", "text-red-500 font-medium" + } + text := t.header(icon, label, color) + if body := t.Str("text"); body != "" { + text = text.Append(" "+messagePreview(body), "text-muted") + } + return text +} + +func (t *VerifyTool) Detail() api.Textable { + if denied := t.BaseTool.Detail(); denied != nil { + return denied + } + return messageDetail(t.Str("text")) +} diff --git a/pkg/cli/ai.go b/pkg/cli/ai.go index ee77a868..0b1d2539 100644 --- a/pkg/cli/ai.go +++ b/pkg/cli/ai.go @@ -340,7 +340,10 @@ func (o AIPromptOptions) ToRequest() (ai.Request, error) { } func executePromptRequest(parent context.Context, req ai.Request, cfg ai.Config, timeout time.Duration, noStream bool) (any, error) { - ctx, cancel := runContext(parent, req, remoteAwareTimeout(req, cfg, timeout)) + ctx, cancel, err := runContext(parent, req, remoteAwareTimeout(req, cfg, timeout)) + if err != nil { + return nil, err + } defer cancel() if err := preparePromptAttachments(ctx, &req, cfg); err != nil { return nil, err @@ -360,19 +363,25 @@ func executePromptRequest(parent context.Context, req ai.Request, cfg ai.Config, } // runContext derives the timeout-bounded context for a prompt execution. A -// non-empty req.Budget.Timeout overrides the caller-supplied timeout; a -// non-positive timeout falls back to 120s. -func runContext(parent context.Context, req ai.Request, timeout time.Duration) (context.Context, context.CancelFunc) { +// parseable req.Budget.Timeout overrides the caller-supplied timeout; an +// unparseable one is an error, not a silent substitution. A caller-supplied +// timeout of zero falls back to the CLI default. +func runContext(parent context.Context, req ai.Request, timeout time.Duration) (context.Context, context.CancelFunc, error) { if parent == nil { parent = context.Background() } - if req.Budget.Timeout != "" { - timeout = runtimeTimeout(req.Budget.Timeout) + declared, err := runtimeTimeout(req.Budget.Timeout) + if err != nil { + return nil, nil, err + } + if declared > 0 { + timeout = declared } if timeout <= 0 { - timeout = 120 * time.Second + timeout = defaultRunTimeout } - return context.WithTimeout(parent, timeout) + ctx, cancel := context.WithTimeout(parent, timeout) + return ctx, cancel, nil } // warnIfLikelyModelTypo emits a "did you mean" hint when the model name is not a diff --git a/pkg/cli/ai_agent.go b/pkg/cli/ai_agent.go index 090b9443..f8c755b8 100644 --- a/pkg/cli/ai_agent.go +++ b/pkg/cli/ai_agent.go @@ -16,6 +16,7 @@ import ( "github.com/flanksource/captain/pkg/ai/middleware" "github.com/flanksource/captain/pkg/ai/prompt" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/promptrun" ) // AIAgentOptions runs the plugin-based agent loop (pkg/ai/agent) from the CLI: @@ -84,7 +85,7 @@ func scopeFromFlag(s string) (agent.Scope, error) { // the worktree plugin so the chain is squashed into one commit *before* the // merge, leaving `wt merge` real commits to take rather than a dirty tree it // would have to invent an LLM message for. -func buildAgentPlugins(opts AIAgentOptions, p ai.Provider) ([]any, *worktree.Plugin, error) { +func buildAgentPlugins(ctx context.Context, opts AIAgentOptions, p ai.Provider) ([]any, *worktree.Plugin, error) { if opts.Commit && !opts.Worktree { return nil, nil, fmt.Errorf("--commit requires --worktree (captain commits the isolated branch, not your working tree)") } @@ -130,14 +131,24 @@ func buildAgentPlugins(opts AIAgentOptions, p ai.Provider) ([]any, *worktree.Plu hooks = append(hooks, wt) } - for _, cmd := range opts.Verify { - cmd = strings.TrimSpace(cmd) - if cmd == "" { - continue - } - hooks = append(hooks, verify.New("verify:"+cmd, &verify.CmdVerifier{Cmd: "sh", Args: []string{"-c", cmd}})) + // --verify is a workflow declaration like any other, so it is dispatched + // through the verifier registry rather than rebuilt here. One place decides + // how a declared check becomes a hook — including the process bounds and the + // confinement seam a hand-built CmdVerifier silently left unset — and a kind + // the host has claimed is honoured instead of being bypassed. + wf := &api.Workflow{Verify: &api.Verify{Commands: opts.Verify}} + if err := wf.Validate(); err != nil { + return nil, nil, err + } + checks, err := verify.HooksFor(ctx, wf, verify.Options{Provider: p}) + if err != nil { + return nil, nil, err } + hooks = append(hooks, checks...) + // The judge is not a registry kind: --judge carries a rubric typed on the + // command line, while api.Verify.Prompts names .prompt files on disk. There + // is no declaration to dispatch, so this hook is built here on purpose. if opts.Judge != "" { judge := &verify.LLMJudgeVerifier{ Provider: p, @@ -175,19 +186,28 @@ func RunAIAgent(opts AIAgentOptions) (any, error) { return nil, err } - p, err := ai.NewProvider(cfg) - if err != nil { - return nil, err - } - if p, err = middleware.Wrap(p, middleware.WithLogging()); err != nil { - return nil, err - } - sp, ok := p.(ai.StreamingProvider) - if !ok { - return nil, fmt.Errorf("runtime %s does not support the streaming agent loop", api.RuntimeOf(cfg.Model.Provider, cfg.Model.Mode)) + timeout, _ := time.ParseDuration(opts.Timeout) + if timeout <= 0 { + timeout = 600 * time.Second } + // The run's context is created before the hooks are: a factory may need it + // (loading a judge template, reaching a host's fixture runner), and a hook + // built outside the run's deadline is a hook the deadline never bounds. + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() - hooks, _, err := buildAgentPlugins(opts, p) + // --judge is the one hook that needs a model of its own before the run starts, + // because it is built here rather than from the workflow promptrun assembles. + // Nothing else does, so nothing else pays for a provider: promptrun builds the + // run's, with the whole middleware stack behind it. + var judgeProvider ai.Provider + if opts.Judge != "" { + if judgeProvider, err = middleware.NewProvider(cfg); err != nil { + return nil, err + } + defer closeProvider(judgeProvider) + } + hooks, _, err := buildAgentPlugins(ctx, opts, judgeProvider) if err != nil { return nil, err } @@ -196,39 +216,34 @@ func RunAIAgent(opts AIAgentOptions) (any, error) { if err != nil { return nil, err } + baseReq.SetCwd(cwd) renderer := NewEventRenderer(os.Stderr) - runner := &agent.Runner[string]{ - Provider: sp, + start := time.Now() + // promptrun.Run is the one definition of a run: the tool-policy refusal before + // the first model call, the setup plugin, the middleware provider and the + // deadline all arrive with it. `captain ai agent` used to assemble those by + // hand and had none of them. + result, runErr := promptrun.Run(ctx, promptrun.Input{ Request: baseReq, + Config: cfg, Hooks: hooks, MaxIterations: opts.MaxIterations, - Repo: cwd, - Cwd: cwd, Scope: scope, + Repo: cwd, + Timeout: timeout, OnEvent: renderer.Handle, - } - - timeout, _ := time.ParseDuration(opts.Timeout) - if timeout <= 0 { - timeout = 600 * time.Second - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - start := time.Now() - result, runErr := runner.Run(ctx) + }) renderErr := renderer.Flush() - ws := result.Response.Workspace res := AIAgentResult{ Duration: time.Since(start).Round(time.Millisecond).String(), - Passed: verifyPassed(result.Verdicts), - Model: firstNonEmpty(result.Response.Model, sp.GetModel(), cfg.Model.Name), - Provider: firstRuntime(result.Response.Runtime, sp.GetRuntime(), api.RuntimeOf(cfg.Model.Provider, cfg.Model.Mode)).Provider, - Mode: string(firstRuntime(result.Response.Runtime, sp.GetRuntime(), api.RuntimeOf(cfg.Model.Provider, cfg.Model.Mode)).Mode), + Passed: result.Passed, + Model: firstNonEmpty(result.Model, cfg.Model.Name), } - if ws != nil { + runtime := firstRuntime(responseRuntime(result.Response), api.RuntimeOf(cfg.Model.Provider, cfg.Model.Mode)) + res.Provider, res.Mode = runtime.Provider, string(runtime.Mode) + if ws := responseWorkspace(result.Response); ws != nil { res.ChangedFiles = ws.Changed res.SessionID = ws.SessionID res.Branch = ws.Branch @@ -246,3 +261,21 @@ func RunAIAgent(opts AIAgentOptions) (any, error) { } return res, nil } + +// responseRuntime / responseWorkspace read a run's response, which is nil when +// the run failed before the runner ever built one (an unenforceable tool policy, +// a refused attachment). Reading through it unguarded panicked on exactly the +// runs whose error the caller most needs to see. +func responseRuntime(resp *ai.Response) api.Runtime { + if resp == nil { + return api.Runtime{} + } + return resp.Runtime +} + +func responseWorkspace(resp *ai.Response) *api.Workspace { + if resp == nil { + return nil + } + return resp.Workspace +} diff --git a/pkg/cli/ai_agent_test.go b/pkg/cli/ai_agent_test.go index daef31dc..6fab4d0b 100644 --- a/pkg/cli/ai_agent_test.go +++ b/pkg/cli/ai_agent_test.go @@ -1,6 +1,7 @@ package cli import ( + "context" "strings" "testing" @@ -46,7 +47,7 @@ func TestScopeFromFlag(t *testing.T) { } func TestBuildAgentPlugins_CommitRequiresWorktree(t *testing.T) { - _, _, err := buildAgentPlugins(AIAgentOptions{Commit: true}, nil) + _, _, err := buildAgentPlugins(context.Background(), AIAgentOptions{Commit: true}, nil) if err == nil || !strings.Contains(err.Error(), "worktree") { t.Fatalf("err = %v, want a --commit-requires-worktree error", err) } @@ -60,7 +61,7 @@ func TestBuildAgentPlugins_WorktreeBranchAndCommit(t *testing.T) { Squash: true, } opts.Prompt = "fix the failing lint\nsecond line ignored" - plugins, wt, err := buildAgentPlugins(opts, nil) + plugins, wt, err := buildAgentPlugins(context.Background(), opts, nil) if err != nil { t.Fatalf("buildAgentPlugins: %v", err) } @@ -88,7 +89,7 @@ func TestBuildAgentPlugins_WorktreeBranchAndCommit(t *testing.T) { // fail at flag-parse time, not silently register a hook that never fires. func TestBuildAgentPlugins_CommitPhaseIsValidated(t *testing.T) { opts := AIAgentOptions{Worktree: true, Commit: true, CommitOn: "whenever", Squash: true} - if _, _, err := buildAgentPlugins(opts, nil); err == nil || !strings.Contains(err.Error(), "whenever") { + if _, _, err := buildAgentPlugins(context.Background(), opts, nil); err == nil || !strings.Contains(err.Error(), "whenever") { t.Fatalf("err = %v, want the unknown phase rejected by name", err) } } @@ -98,7 +99,7 @@ func TestBuildAgentPlugins_CommitPhaseIsValidated(t *testing.T) { // `--commit-on=run` into a validation error the user never asked for. func TestBuildAgentPlugins_SquashDefaultSurvivesRunPhase(t *testing.T) { opts := AIAgentOptions{Worktree: true, Commit: true, CommitOn: string(api.CommitOnRun), Squash: true} - plugins, _, err := buildAgentPlugins(opts, nil) + plugins, _, err := buildAgentPlugins(context.Background(), opts, nil) if err != nil { t.Fatalf("buildAgentPlugins: %v", err) } @@ -109,7 +110,7 @@ func TestBuildAgentPlugins_SquashDefaultSurvivesRunPhase(t *testing.T) { func TestBuildAgentPlugins_WorktreeWithoutCommit(t *testing.T) { opts := AIAgentOptions{Worktree: true} - _, wt, err := buildAgentPlugins(opts, nil) + _, wt, err := buildAgentPlugins(context.Background(), opts, nil) if err != nil { t.Fatalf("buildAgentPlugins: %v", err) } @@ -126,7 +127,7 @@ func TestBuildAgentPlugins_VerifyAndJudge(t *testing.T) { Verify: []string{"make lint", " ", "go test ./..."}, Judge: "the change must include a test", } - plugins, wt, err := buildAgentPlugins(opts, nil) + plugins, wt, err := buildAgentPlugins(context.Background(), opts, nil) if err != nil { t.Fatalf("buildAgentPlugins: %v", err) } @@ -139,15 +140,3 @@ func TestBuildAgentPlugins_VerifyAndJudge(t *testing.T) { t.Errorf("plugin names = %v, want %v (blank --verify entries skipped)", got, want) } } - -func TestVerifyPassed(t *testing.T) { - if !verifyPassed(nil) { - t.Error("no verdicts should pass") - } - if !verifyPassed([]agent.VerifyResult{{Valid: false}, {Valid: true}}) { - t.Error("last verdict valid should pass") - } - if verifyPassed([]agent.VerifyResult{{Valid: true}, {Valid: false}}) { - t.Error("last verdict not valid should fail") - } -} diff --git a/pkg/cli/ai_sandbox_remote.go b/pkg/cli/ai_sandbox_remote.go index d627be24..1153a6ed 100644 --- a/pkg/cli/ai_sandbox_remote.go +++ b/pkg/cli/ai_sandbox_remote.go @@ -36,9 +36,18 @@ func remoteAwareTimeout(req ai.Request, cfg ai.Config, timeout time.Duration) ti } // renderedTimeout is remoteAwareTimeout for an already-rendered prompt, used -// by the stream and batch paths that size their own deadline. -func renderedTimeout(rendered PromptRenderResult) time.Duration { - return remoteAwareTimeout(rendered.Input, rendered.Config, runtimeTimeout(rendered.Input.Budget.Timeout)) +// by the stream and batch paths that size their own deadline. A spec that +// declares no budget.timeout gets the CLI default; one that declares an +// unparseable value is an error, not a run on a substituted deadline. +func renderedTimeout(rendered PromptRenderResult) (time.Duration, error) { + declared, err := runtimeTimeout(rendered.Input.Budget.Timeout) + if err != nil { + return 0, err + } + if declared <= 0 { + declared = defaultRunTimeout + } + return remoteAwareTimeout(rendered.Input, rendered.Config, declared), nil } // remoteExecProviderFor returns a provider backed by the resolved sandbox's diff --git a/pkg/cli/attachments_ginkgo_test.go b/pkg/cli/attachments_ginkgo_test.go index 2083c504..15809339 100644 --- a/pkg/cli/attachments_ginkgo_test.go +++ b/pkg/cli/attachments_ginkgo_test.go @@ -57,13 +57,17 @@ var _ = Describe("attachment flags", func() { })) }) - It("renders an attachment-only canonical prompt", func() { + // Attachments plumb through the render, but they are not an instruction: + // api.Spec.ValidateRunnable (which promptrun.Run enforces) refuses a spec + // with no prompt.user and no workflow.verify, and render now gives the same + // answer instead of accepting the run and failing after the provider exists. + It("carries an attachment-only canonical prompt but reports it as not runnable", func() { rendered, err := renderPromptCLI(context.Background(), "", AIPromptOptions{ AIRuntimeOptions: AIRuntimeOptions{AIProviderOptions: AIProviderOptions{ModelFlags: aiflags.ModelFlags{Model: "gemini-2.5-pro"}}}, Attach: []string{"diagram.png"}, }, "", "") Expect(err).NotTo(HaveOccurred()) - Expect(rendered.ValidationError).To(BeEmpty()) + Expect(rendered.ValidationError).To(Equal(api.Spec{}.ValidateRunnable().Error())) Expect(rendered.Input.Prompt.Attachments).To(Equal([]api.AttachmentRef{{Path: "diagram.png"}})) }) @@ -76,7 +80,7 @@ var _ = Describe("attachment flags", func() { } rendered, err := renderPrompt(context.Background(), "", PromptRenderRequest{Spec: &api.Spec{ Model: api.Model{Name: "gemini-2.5-pro", Mode: api.ModeAPI}, - Prompt: api.Prompt{Attachments: []api.AttachmentRef{attachment}}, + Prompt: api.Prompt{User: "Describe this diagram", Attachments: []api.AttachmentRef{attachment}}, }}) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/cli/captain_config_once.go b/pkg/cli/captain_config_once.go new file mode 100644 index 00000000..3f2456b8 --- /dev/null +++ b/pkg/cli/captain_config_once.go @@ -0,0 +1,61 @@ +package cli + +import ( + "sync" + + "github.com/flanksource/captain/pkg/captainconfig" +) + +// captainConfigResult is one memoized parse of ~/.captain.yaml, together with +// the path it was read from. The path is what keeps the cache honest: +// captainconfig.SetPath redirects the file (a git receive hook runs under the +// pusher's HOME, and every spec redirects it to a temp dir), and a bare +// sync.Once would hand the second caller the first caller's file. +type captainConfigResult struct { + loaded bool + path string + config captainconfig.Config + exists bool + err error +} + +var ( + captainConfigMu sync.Mutex + captainConfigCache captainConfigResult +) + +// LoadCaptainConfigOnce reads ~/.captain.yaml at most once per process and path, +// returning the same result — success or failure — to every later caller. +// +// The root command installs several things out of that one file before any +// subcommand runs (the opt-out set, the fixture verifier); each doing its own +// Load meant re-reading and re-parsing the same YAML on every invocation, and +// left open the window where two installers disagree because the file changed +// between them. The bool is captainconfig.Load's: false means the file does not +// exist, which is a normal first-run state rather than an error. +func LoadCaptainConfigOnce() (captainconfig.Config, bool, error) { + path, err := captainconfig.Path() + if err != nil { + return captainconfig.Config{}, false, err + } + captainConfigMu.Lock() + defer captainConfigMu.Unlock() + if captainConfigCache.loaded && captainConfigCache.path == path { + return captainConfigCache.config, captainConfigCache.exists, captainConfigCache.err + } + config, exists, loadErr := captainconfig.Load() + captainConfigCache = captainConfigResult{ + loaded: true, path: path, config: config, exists: exists, err: loadErr, + } + return config, exists, loadErr +} + +// ResetCaptainConfigCache drops the memoized parse. It exists for specs that +// rewrite the config file in place under one path; the production path never +// needs it, because the file is read once at startup and the process that reads +// it is not the one that edits it. +func ResetCaptainConfigCache() { + captainConfigMu.Lock() + defer captainConfigMu.Unlock() + captainConfigCache = captainConfigResult{} +} diff --git a/pkg/cli/dod.go b/pkg/cli/dod.go deleted file mode 100644 index 8221cc56..00000000 --- a/pkg/cli/dod.go +++ /dev/null @@ -1,188 +0,0 @@ -package cli - -import ( - "encoding/json" - "fmt" - "os" - "time" - - "github.com/flanksource/captain/pkg/claude" - "github.com/flanksource/captain/pkg/dod" -) - -type DodSetOptions struct { - SessionID string `flag:"session-id" help:"Claude session ID (auto-detected from stdin if not set)" short:"s"` - Workdir string `flag:"workdir" help:"Working directory for commands (defaults to cwd)" short:"w"` - Timeout int `flag:"timeout" help:"Timeout per command in seconds" default:"300" short:"t"` - Commands []string `args:"true" help:"Commands to run as Definition of Done checks" required:"true"` -} - -func RunDodSet(opts DodSetOptions) (any, error) { - if len(opts.Commands) == 0 { - return nil, fmt.Errorf("usage: captain dod set ") - } - - sessionID := opts.SessionID - if sessionID == "" { - return nil, fmt.Errorf("--session-id is required (Claude provides this via hook stdin)") - } - - workdir := opts.Workdir - if workdir == "" { - var err error - workdir, err = os.Getwd() - if err != nil { - return nil, err - } - } - - dodFile := &dod.DodFile{ - Commands: opts.Commands, - Workdir: workdir, - Timeout: opts.Timeout, - CreatedAt: time.Now().UTC(), - } - - if err := dod.Write(sessionID, dodFile); err != nil { - return nil, fmt.Errorf("writing dod: %w", err) - } - - return map[string]any{ - "session_id": sessionID, - "commands": opts.Commands, - "workdir": workdir, - "timeout": opts.Timeout, - }, nil -} - -type DodCheckOptions struct{} - -// StopHookInput is the JSON Claude Code passes to Stop hooks via stdin. -type StopHookInput struct { - SessionID string `json:"session_id"` - TranscriptPath string `json:"transcript_path,omitempty"` - CWD string `json:"cwd,omitempty"` - StopHookActive bool `json:"stop_hook_active"` - LastAssistantMsg string `json:"last_assistant_message,omitempty"` - HookEventName string `json:"hook_event_name,omitempty"` -} - -func RunDodCheck(_ DodCheckOptions) (any, error) { - if !claude.IsStdinPiped() { - return nil, fmt.Errorf("dod check must be called as a Claude Code Stop hook (reads JSON from stdin)") - } - - data, err := os.ReadFile("/dev/stdin") - if err != nil { - return nil, fmt.Errorf("reading stdin: %w", err) - } - - var input StopHookInput - if err := json.Unmarshal(data, &input); err != nil { - return nil, fmt.Errorf("parsing hook input: %w", err) - } - - if input.SessionID == "" { - return nil, fmt.Errorf("no session_id in hook input") - } - - if !dod.Exists(input.SessionID) { - return nil, nil // no DoD set, allow stop - } - - dodFile, err := dod.Read(input.SessionID) - if err != nil { - // corrupt file — allow stop rather than blocking forever - fmt.Fprintf(os.Stderr, "warning: could not read dod file: %v\n", err) - return nil, nil - } - - // If stop_hook_active is true, Claude is already retrying. - // Run checks again — if they still fail, block again (user can /dod-clear to escape). - run := dod.RunCommands(dodFile) - dodFile.LastRun = run - _ = dod.Write(input.SessionID, dodFile) // persist last run results - - allPassed := true - for _, r := range run.Results { - if !r.Passed { - allPassed = false - break - } - } - - if allPassed { - _ = dod.Delete(input.SessionID) - return nil, nil // allow stop - } - - // Block stop: write failure message to stderr (exit 2 triggers re-prompt) - fmt.Fprint(os.Stderr, dod.FormatFailureMessage(run)) - os.Exit(2) - return nil, nil // unreachable -} - -type DodClearOptions struct { - SessionID string `flag:"session-id" help:"Claude session ID" short:"s"` -} - -func RunDodClear(opts DodClearOptions) (any, error) { - if opts.SessionID == "" { - return nil, fmt.Errorf("--session-id is required") - } - - if !dod.Exists(opts.SessionID) { - return "No DoD set for this session", nil - } - - if err := dod.Delete(opts.SessionID); err != nil { - return nil, fmt.Errorf("clearing dod: %w", err) - } - return "DoD cleared", nil -} - -type DodStatusOptions struct { - SessionID string `flag:"session-id" help:"Claude session ID" short:"s"` -} - -func RunDodStatus(opts DodStatusOptions) (any, error) { - if opts.SessionID == "" { - return nil, fmt.Errorf("--session-id is required") - } - - if !dod.Exists(opts.SessionID) { - return "No DoD set for this session", nil - } - - dodFile, err := dod.Read(opts.SessionID) - if err != nil { - return nil, err - } - - return dodFile, nil -} - -type DodRunOptions struct { - SessionID string `flag:"session-id" help:"Claude session ID" short:"s"` -} - -func RunDodRun(opts DodRunOptions) (any, error) { - if opts.SessionID == "" { - return nil, fmt.Errorf("--session-id is required") - } - - if !dod.Exists(opts.SessionID) { - return nil, fmt.Errorf("no DoD set for session %s", opts.SessionID) - } - - dodFile, err := dod.Read(opts.SessionID) - if err != nil { - return nil, err - } - - run := dod.RunCommands(dodFile) - dodFile.LastRun = run - _ = dod.Write(opts.SessionID, dodFile) - - return run, nil -} diff --git a/pkg/cli/event_renderer.go b/pkg/cli/event_renderer.go index 626a7388..f1b8fcb5 100644 --- a/pkg/cli/event_renderer.go +++ b/pkg/cli/event_renderer.go @@ -5,32 +5,57 @@ import ( "fmt" "io" "os" + "strings" "github.com/charmbracelet/x/ansi" "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/session" + "github.com/flanksource/clicky" "golang.org/x/term" ) type EventRenderer struct { output io.Writer interactive bool + // width is the output's own terminal width, not the process's stdout. A run + // streaming to stderr while stdout is redirected to a file is the normal + // case for a long command, and sizing to the wrong one of the two is how a + // wide terminal ends up showing a line cut for an 80-column guess. + width int accumulator *promptEventAccumulator pending *session.Message rendered map[string]bool err error iteration int hasIter bool + // progressDrawn records that the cursor is sitting on an in-place verify + // status line, so the next thing written erases it first instead of landing + // on top of it. + progressDrawn bool } func NewEventRenderer(output *os.File) *EventRenderer { - return newEventRenderer(output, output != nil && term.IsTerminal(int(output.Fd()))) + interactive := output != nil && term.IsTerminal(int(output.Fd())) + renderer := newEventRenderer(output, interactive) + if output != nil { + if w, _, err := term.GetSize(int(output.Fd())); err == nil { + renderer.width = w + } + } + return renderer } +// defaultRenderWidth is the fallback when the output is not a terminal — a +// pipe, a file, a CI log. It matches tools.MessagePreviewChars so a redirected +// run reads the same as it did before there was a width at all. +const defaultRenderWidth = 120 + func newEventRenderer(output io.Writer, interactive bool) *EventRenderer { renderer := &EventRenderer{ output: output, interactive: interactive, + width: defaultRenderWidth, rendered: map[string]bool{}, } renderer.accumulator = newPromptEventAccumulator(renderer.consume, discardTaskSink{}, "", "") @@ -48,6 +73,26 @@ func (r *EventRenderer) Handle(iteration int, event ai.Event) { } r.iteration, r.hasIter = iteration, true + // An in-flight snapshot is redrawn over the last one and never committed to + // the scrollback: a fixture runner reports every few hundred milliseconds, + // and one line each would bury the run's own output under superseded counts. + if event.Kind == ai.EventVerifyProgress { + r.renderProgress(event) + return + } + r.clearProgress() + + // A verdict is rendered here rather than through the transcript row the + // accumulator would build, because a row is a one-line preview cut to a + // fixed budget: it would elide the very output the verdict exists to show. + // This is the only consumer that knows the width it is writing into, so it + // is the only one that can fit the report instead of guessing. + if event.Kind == ai.EventVerified || event.Kind == ai.EventVerifyFailed { + r.flushPending() + r.renderVerdict(event) + return + } + if r.pendingBoundary(event.Kind) { r.flushPending() } @@ -58,10 +103,36 @@ func (r *EventRenderer) Handle(iteration int, event ai.Event) { } func (r *EventRenderer) Flush() error { + r.clearProgress() r.flushPending() return r.err } +// renderProgress redraws the verify status line in place. Only on a terminal: a +// file or a CI log cannot redraw, so writing there would produce exactly the +// line-per-snapshot spam the in-place line exists to avoid, and a run's log +// would end up mostly counts. +func (r *EventRenderer) renderProgress(event ai.Event) { + report, ok := event.Raw.(*api.VerifyReport) + if !ok || report == nil || !r.interactive { + return + } + r.flushPending() + line := clicky.Text("⟳ ", "text-blue-500").Append(verifyProgressStatus(*report), "text-muted").ANSI() + r.write("\r" + ansi.EraseEntireLine + truncateANSI(line, r.width)) + r.progressDrawn = true +} + +// clearProgress wipes the in-place status line before anything else is written, +// so a verdict never lands on top of the counts it supersedes. +func (r *EventRenderer) clearProgress() { + if !r.progressDrawn { + return + } + r.progressDrawn = false + r.write("\r" + ansi.EraseEntireLine) +} + func (r *EventRenderer) pendingBoundary(kind ai.EventKind) bool { if r.pending == nil || len(r.pending.Parts) == 0 { return false @@ -128,6 +199,26 @@ func (r *EventRenderer) flushPending() { r.pending = nil } +// renderVerdict writes one verify verdict at the output's own width: the +// headline on the first line, and the verifier's output — the failure the next +// turn is about to be told about — beneath it, one line per line so a test +// runner's tables and traces keep their alignment instead of wrapping. +func (r *EventRenderer) renderVerdict(event ai.Event) { + headline, body, _ := strings.Cut(strings.TrimRight(event.Text, "\n"), "\n") + icon, style := "✓", "text-green-500 font-medium" + if event.Kind == ai.EventVerifyFailed { + icon, style = "✗", "text-red-500 font-medium" + } + prefix := clicky.Text(icon+" verify ", style) + r.write(truncateANSI(prefix.Append(headline, "text-muted").ANSI(), r.width) + "\n") + for _, line := range strings.Split(body, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + r.write(truncateANSI(line, r.width) + "\n") + } +} + func (r *EventRenderer) renderMessage(message session.Message) { text, ok := transcriptMessageANSI(message) if ok { diff --git a/pkg/cli/event_renderer_ginkgo_test.go b/pkg/cli/event_renderer_ginkgo_test.go index d5063a32..3a169e5f 100644 --- a/pkg/cli/event_renderer_ginkgo_test.go +++ b/pkg/cli/event_renderer_ginkgo_test.go @@ -117,4 +117,105 @@ var _ = Describe("Captain event renderer", func() { ContainSubstring("second command"), )) }) + + // A verdict is the loop's outcome, so it is rendered here rather than as the + // one-line transcript row the accumulator would build for it: the row's fixed + // preview budget would elide the very output the verdict exists to show. + Describe("verify verdicts", func() { + const failure = "fixtures/funeral-policy.yaml: policy Digital Funeral\n" + + " savePolicy rejected the policy: Please Enter a Valid Cover Amount" + + It("prints the verifier's output in full beneath the headline", func() { + var output bytes.Buffer + renderer := newEventRenderer(&output, false) + + renderer.Handle(0, ai.Event{Kind: ai.EventText, Text: "applying the fix"}) + renderer.Handle(0, ai.Event{ + Kind: ai.EventVerifyFailed, Tool: "verify:oipa-cli test fixtures/funeral-policy.yaml", + Text: "failed in 5m7s: sh failed — verify:oipa-cli test fixtures/funeral-policy.yaml\n" + failure, + }) + Expect(renderer.Flush()).To(Succeed()) + + text := output.String() + Expect(text).To(ContainSubstring("failed in 5m7s")) + // Every line of the body survives: this is the failure the next turn + // is about to be told about, and summarizing it here would leave the + // reader with the same "why did it try again?" the verdict answers. + for _, line := range strings.Split(failure, "\n") { + Expect(text).To(ContainSubstring(line)) + } + // It stands outside the model's prose, like a hook notice. + Expect(text).NotTo(ContainSubstring("applying the fix✗")) + }) + + It("fits the headline to the output's width rather than a fixed budget", func() { + var narrow, wide bytes.Buffer + long := "failed in 5m7s: sh failed — verify:" + strings.Repeat("fixtures/a-long-path.yaml ", 12) + + for _, spec := range []struct { + out *bytes.Buffer + width int + }{{&narrow, 60}, {&wide, 200}} { + renderer := newEventRenderer(spec.out, false) + renderer.width = spec.width + renderer.Handle(0, ai.Event{Kind: ai.EventVerifyFailed, Text: long}) + Expect(renderer.Flush()).To(Succeed()) + } + + Expect(len(narrow.String())).To(BeNumerically("<", len(wide.String())), + "a wider terminal must show more of the line, not the same fixed cut") + Expect(wide.String()).To(ContainSubstring("failed in 5m7s")) + }) + + // A fixture runner reports a snapshot every few hundred milliseconds. One + // log line each turns the run's output into a column of superseded counts + // that scrolls the model's own work off the screen, so the counts update + // one line in place and only the verdict is committed to the scrollback. + It("draws progress as one status line that updates in place", func() { + var output bytes.Buffer + renderer := newEventRenderer(&output, true) + + for done := 1; done <= 3; done++ { + renderer.Handle(0, ai.Event{ + Kind: ai.EventVerifyProgress, Tool: "fixture", Raw: verifyReport(done, 5), + }) + } + renderer.Handle(0, ai.Event{ + Kind: ai.EventVerified, Success: true, Text: "passed in 4ms — fixture", Raw: verifyReport(5, 5), + }) + Expect(renderer.Flush()).To(Succeed()) + + text := output.String() + Expect(text).To(ContainSubstring("3/5")) + Expect(strings.Count(text, "1/5")).To(Equal(1), "each snapshot is drawn once, over the last one") + Expect(strings.Count(text, "\n")).To(Equal(1), + "only the verdict ends a line; three snapshots must not become three lines") + Expect(text).To(ContainSubstring("passed in 4ms")) + }) + + It("keeps progress out of a redirected run's output entirely", func() { + var output bytes.Buffer + renderer := newEventRenderer(&output, false) + + renderer.Handle(0, ai.Event{Kind: ai.EventVerifyProgress, Tool: "fixture", Raw: verifyReport(1, 5)}) + Expect(renderer.Flush()).To(Succeed()) + + // A file or CI log cannot redraw, so an in-place status line would + // become exactly the per-snapshot log spam it exists to avoid. + Expect(output.String()).To(BeEmpty()) + }) + + It("renders a pass as a single line, since a passing check reports no output", func() { + var output bytes.Buffer + renderer := newEventRenderer(&output, false) + + renderer.Handle(0, ai.Event{ + Kind: ai.EventVerified, Success: true, Text: "passed in 4ms — verify:true", + }) + Expect(renderer.Flush()).To(Succeed()) + + Expect(output.String()).To(ContainSubstring("passed in 4ms")) + Expect(strings.Count(strings.TrimRight(output.String(), "\n"), "\n")).To(Equal(0)) + }) + }) }) diff --git a/pkg/cli/gitagent_e2e_test.go b/pkg/cli/gitagent_e2e_test.go index 2355d485..690d5183 100644 --- a/pkg/cli/gitagent_e2e_test.go +++ b/pkg/cli/gitagent_e2e_test.go @@ -200,6 +200,35 @@ func (h *host) configBytes() string { return string(data) } +// configureDefaultModel writes ai.defaultModel into this host's config, which a +// dispatch now requires: captain no longer falls back to a compiled-in model, so +// a host with no configured model refuses to run rather than silently picking +// one. These tests are about git-agent dispatch mechanics, so they configure the +// model the way `captain configure` would and get on with it. +func (h *host) configureDefaultModel(t *testing.T, selector string) { + t.Helper() + path := filepath.Join(h.home, ".captain.yaml") + cfg := map[string]any{} + if data, err := os.ReadFile(path); err == nil { + if err := yaml.Unmarshal(data, &cfg); err != nil { + t.Fatal(err) + } + } + aiCfg, _ := cfg["ai"].(map[string]any) + if aiCfg == nil { + aiCfg = map[string]any{} + } + aiCfg["defaultModel"] = selector + cfg["ai"] = aiCfg + out, err := yaml.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, out, 0o600); err != nil { + t.Fatal(err) + } +} + // setBackendOption edits one option under sandbox.backends.git-agent in this // host's config, the way an operator would. func (h *host) setBackendOption(t *testing.T, key, value string) { @@ -424,6 +453,7 @@ func TestFullCycleWithAManualAgent(t *testing.T) { t.Skip("builds the captain binary and runs two endpoints") } supervisor, agent, repo, _, _ := enrollPair(t) + supervisor.configureDefaultModel(t, "agent:claude-sonnet-5") // Opt out of the launcher: this test drives the agent itself. agent.setBackendOption(t, "agentCommand", gitagent.NoAgentCommand) @@ -689,6 +719,7 @@ func TestOneEndpointRoutesTwoRepositories(t *testing.T) { t.Skip("builds the captain binary and runs two endpoints") } supervisor, agent, repoA, _, _ := enrollPair(t) + supervisor.configureDefaultModel(t, "agent:claude-sonnet-5") repoB := newRepo(t) // same basename, different canonical path agent.setBackendOption(t, "agentCommand", `echo "// completed $CAPTAIN_TASK" >> pkg/main.go `+ @@ -807,6 +838,10 @@ func TestUnconfiguredDispatchLaunchesTheDefaultAgent(t *testing.T) { t.Skip("builds the captain binary and runs two endpoints") } supervisor, agent, repo, _, _ := enrollPair(t) + // "Unconfigured" here means no agentCommand — the backend must fall back to + // captain's own agent launcher. The MODEL is configured, because that is no + // longer something captain will choose on your behalf. + supervisor.configureDefaultModel(t, "agent:claude-sonnet-5") if strings.Contains(agent.configBytes(), "agentCommand") { t.Fatalf("this test requires an unconfigured backend:\n%s", agent.configBytes()) } diff --git a/pkg/cli/gitagent_runtask.go b/pkg/cli/gitagent_runtask.go index 6b09290f..772e418a 100644 --- a/pkg/cli/gitagent_runtask.go +++ b/pkg/cli/gitagent_runtask.go @@ -99,7 +99,11 @@ func runTaskPrompt(ctx context.Context, worktree string, payload gitagent.TaskPa req.Sandbox = &api.SandboxRef{Mode: api.SandboxNative} req.Permissions.Mode = api.PermissionAcceptEdits - if _, err := executePromptRequestFunc(ctx, req, cfg, renderedTimeout(PromptRenderResult{Input: req, Config: cfg}), true); err != nil { + timeout, err := renderedTimeout(PromptRenderResult{Input: req, Config: cfg}) + if err != nil { + return err + } + if _, err := executePromptRequestFunc(ctx, req, cfg, timeout, true); err != nil { return err } return nil diff --git a/pkg/cli/hook.go b/pkg/cli/hook.go index b6dbc814..ceaabb45 100644 --- a/pkg/cli/hook.go +++ b/pkg/cli/hook.go @@ -4,12 +4,10 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" "strings" "github.com/flanksource/captain/pkg/bash" "github.com/flanksource/captain/pkg/claude" - "github.com/flanksource/captain/pkg/dod" ) type HookInstallOptions struct { @@ -75,42 +73,13 @@ func RunBashCheckInstall(opts HookInstallOptions) (any, error) { } hookCommand := fmt.Sprintf("%s hook bash-check", captainPath) - result, err := installHook(target, "PreToolUse", "Bash", hookCommand, "dod check", opts.Timeout) + result, err := installHook(target, "PreToolUse", "Bash", hookCommand, "hook bash-check", opts.Timeout) if err != nil { return nil, err } return result, nil } -func RunDodInstall(opts HookInstallOptions) (any, error) { - captainPath, err := os.Executable() - if err != nil { - captainPath = "captain" - } - - target, err := resolveSettingsTarget(opts.User) - if err != nil { - return nil, err - } - - var results []string - - hookCommand := fmt.Sprintf("%s dod check", captainPath) - hookResult, err := installHook(target, "Stop", "", hookCommand, "dod check", opts.Timeout) - if err != nil { - return nil, err - } - results = append(results, hookResult) - - skillResult, err := installSkills() - if err != nil { - return nil, err - } - results = append(results, skillResult) - - return strings.Join(results, "\n"), nil -} - func resolveSettingsTarget(userGlobal bool) (string, error) { if userGlobal { target := claude.GetClaudeHome() + "/settings.json" @@ -197,30 +166,6 @@ findExisting: return fmt.Sprintf("%s hook: %s in %s (%s)", eventType, action, target, hookCommand), nil } -func installSkills() (string, error) { - skillsDir := filepath.Join(claude.GetClaudeHome(), "skills", "captain-dod") - if err := os.MkdirAll(skillsDir, 0755); err != nil { - return "", fmt.Errorf("creating skills dir: %w", err) - } - - skills := map[string]string{ - "dod.md": dod.SkillDod, - "dod-clear.md": dod.SkillDodClear, - "dod-status.md": dod.SkillDodStatus, - "dod-run.md": dod.SkillDodRun, - } - - installed := 0 - for name, content := range skills { - path := filepath.Join(skillsDir, name) - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - return "", fmt.Errorf("writing skill %s: %w", name, err) - } - installed++ - } - return fmt.Sprintf("Skills: installed %d skill files to %s", installed, skillsDir), nil -} - func asSlice(v any) []any { if s, ok := v.([]any); ok { return s diff --git a/pkg/cli/prompt_batch_run.go b/pkg/cli/prompt_batch_run.go index 87be4fab..6ea25f92 100644 --- a/pkg/cli/prompt_batch_run.go +++ b/pkg/cli/prompt_batch_run.go @@ -51,12 +51,16 @@ func launchAsyncBatch(ctx context.Context, id string, rendered PromptRenderResul Effort: string(run.Runtime.Effort), Chat: chat, Capabilities: capabilities, } handles[i] = group.Add(runtimeSelector(run.Runtime), func(_ flanksourceContext.Context, t *task.Task) (PromptRunSummary, error) { + timeout, err := renderedTimeout(variant) + if err != nil { + return PromptRunSummary{}, err + } if chat { - chatSession := newChatSession(runID, variant, renderedTimeout(variant), stream, binding) + chatSession := newChatSession(runID, variant, timeout, stream, binding) promptChats.register(chatSession) return chatSession.run(t) } - summary, runErr := runPromptStream(t, variant, renderedTimeout(variant), runID, stream, binding) + summary, runErr := runPromptStream(t, variant, timeout, runID, stream, binding) if runErr != nil { persistPromptRun(context.WithoutCancel(t.Context()), promptRunRecordInput{ Rendered: variant, RunID: runID, Binding: binding, diff --git a/pkg/cli/prompt_chat.go b/pkg/cli/prompt_chat.go index 25843d46..33e5c028 100644 --- a/pkg/cli/prompt_chat.go +++ b/pkg/cli/prompt_chat.go @@ -120,7 +120,10 @@ func (c *chatSession) run(t *task.Task) (PromptRunSummary, error) { var errChatIdle = errors.New("chat idle timeout") func (c *chatSession) runTurn(baseCtx context.Context, t *task.Task, req ai.Request) (PromptRunSummary, bool, error) { - turnCtx, cancel := runContext(baseCtx, req, c.timeout) + turnCtx, cancel, err := runContext(baseCtx, req, c.timeout) + if err != nil { + return PromptRunSummary{}, false, err + } turnDone := make(chan struct{}) c.mu.Lock() c.state.Turn++ diff --git a/pkg/cli/prompt_chat_http.go b/pkg/cli/prompt_chat_http.go index 380981c5..3afcb810 100644 --- a/pkg/cli/prompt_chat_http.go +++ b/pkg/cli/prompt_chat_http.go @@ -193,7 +193,10 @@ func resumeSessionMessage(item SessionGetItem, request ChatMessageRequest) (Chat User: req.Prompt.User, Input: req, Config: ai.Config{Model: modelSpec, SessionID: item.ProviderSessionID}, } - run := launchAsyncRun(item.CaptainID, rendered, true) + run, err := launchAsyncRun(item.CaptainID, rendered, true) + if err != nil { + return ChatMessageResponse{}, err + } if stream, ok := promptRuns.get(run.RunID); ok { stream.publish(userChatMessage(messageID, req.Prompt.User)) } diff --git a/pkg/cli/prompt_observe.go b/pkg/cli/prompt_observe.go index cc0b81f6..6274d9c1 100644 --- a/pkg/cli/prompt_observe.go +++ b/pkg/cli/prompt_observe.go @@ -133,11 +133,21 @@ func observePromptAction(ctx context.Context, id string, flags map[string]string } contextWithCapture := capture.Context(observation.ContextWithRecorder(ctx, recorder)) - runCtx, cancel := runContext( + flagTimeout, err := runtimeTimeout(opts.Timeout) + if err != nil { + return api.RuntimeObservation{}, fmt.Errorf("--timeout: %w", err) + } + if flagTimeout <= 0 { + flagTimeout = defaultRunTimeout + } + runCtx, cancel, err := runContext( contextWithCapture, rendered.Input, - remoteAwareTimeout(rendered.Input, rendered.Config, runtimeTimeout(opts.Timeout)), + remoteAwareTimeout(rendered.Input, rendered.Config, flagTimeout), ) + if err != nil { + return api.RuntimeObservation{}, err + } defer cancel() cfg.CanUseTool = recorder.PermissionBroker(cfg.CanUseTool) diff --git a/pkg/cli/prompt_render.go b/pkg/cli/prompt_render.go index 905e43cc..056e3826 100644 --- a/pkg/cli/prompt_render.go +++ b/pkg/cli/prompt_render.go @@ -196,18 +196,32 @@ func finalizeRenderResult(record promptRecord, content string, req ai.Request, c }, nil } +// renderValidationError is the render-time verdict `captain prompt run` refuses +// on. Runnability is asked with api.Spec.ValidateRunnable — the same rule +// promptrun.Run enforces — so a spec that the run seam will reject is rejected +// here, before a provider or a stream exists, with the same message. Render used +// to ask a looser question ("prompt text or attachment"), so an attachments-only +// or messages-only prompt passed here and failed later. func renderValidationError(req ai.Request, cfg ai.Config, runtimes []api.Model) string { - switch { - case req.Prompt.User == "" && len(req.Prompt.Attachments) == 0 && !req.IsVerifyOnly(): - return "prompt text or attachment required" - case cfg.Model.Name == "" && len(runtimes) == 0: + if err := req.ValidateRunnable(); err != nil { + return err.Error() + } + if cfg.Model.Name == "" && len(runtimes) == 0 { return "no model: set prompt frontmatter, pass a model override, or run 'captain configure'" - default: - if err := req.Validate(); err != nil { - return err.Error() - } - return "" } + // A prompt that declares its own runtimes needs no singular base model: + // each runtime names one and the run fans out across them. Validate + // against the first so the rest of the request is still checked. This + // used to pass only because the base name was back-filled from a + // compiled-in default table, which made an unconfigured captain look + // configured. + if req.Name == "" && len(runtimes) > 0 { + req.Model = runtimes[0] + } + if err := req.Validate(); err != nil { + return err.Error() + } + return "" } func resolvePromptRuntimes(runtimes []api.Model, base api.Model) ([]api.Model, error) { diff --git a/pkg/cli/prompt_render_test.go b/pkg/cli/prompt_render_test.go index 9baf1e47..7899b1ee 100644 --- a/pkg/cli/prompt_render_test.go +++ b/pkg/cli/prompt_render_test.go @@ -363,6 +363,82 @@ func TestRenderPromptPreservesSandboxMetadata(t *testing.T) { } } +// TestRenderPromptRunnability pins render-time agreement with +// api.Spec.ValidateRunnable, which promptrun.Run enforces. Render used to accept +// an attachments-only prompt ("prompt text or attachment required" fired only +// when both were absent), so `captain prompt run` built a provider and opened a +// stream before the run seam rejected the very same spec — the failure arrived +// after the expensive part, with a different message. +func TestRenderPromptRunnability(t *testing.T) { + notRunnable := api.Spec{}.ValidateRunnable().Error() + + tests := []struct { + name string + frontmatter string + body string + want string + }{ + { + name: "attachments without a prompt body or verify are not a run", + frontmatter: "prompt:\n attachments:\n - path: diagram.png\n", + want: notRunnable, + }, + { + // api.Spec.IsVerifyOnly requires no attachments: attachments are + // input to a generation, so a spec carrying them but no instruction + // is under-specified even with a verify declared. Render must give + // the same answer promptrun.Run does, not a friendlier one. + name: "attachments plus workflow.verify are still not a run", + frontmatter: "prompt:\n attachments:\n - path: diagram.png\n" + verifyFrontmatter, + want: notRunnable, + }, + { + name: "a prompt body is runnable on its own", + body: "Summarize the diagram\n", + want: "", + }, + { + name: "an empty prompt with workflow.verify is verify-only", + frontmatter: verifyFrontmatter, + want: "", + }, + { + name: "an empty prompt with neither is not a run", + want: notRunnable, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + isolateCaptainConfig(t) + dir := t.TempDir() + t.Chdir(t.TempDir()) + ctx := ContextWithPromptDirs(context.Background(), []string{dir}) + created, err := createPrompt(ctx, map[string]any{ + "name": "Runnable", + "content": "---\nname: Runnable\nmodel: claude-sonnet-4-6\n" + tt.frontmatter + + "---\n{{role \"user\"}}\n" + tt.body, + }) + if err != nil { + t.Fatalf("createPrompt() err = %v", err) + } + + rendered, err := renderPrompt(ctx, created.ID, PromptRenderRequest{}) + if err != nil { + t.Fatalf("renderPrompt() err = %v", err) + } + if rendered.ValidationError != tt.want { + t.Fatalf("validation error = %q, want %q", rendered.ValidationError, tt.want) + } + if tt.want == "" && tt.body == "" && !rendered.Input.IsVerifyOnly() { + t.Fatalf("spec %+v is not verify-only; render accepted a body-less prompt for another reason", rendered.Input.Prompt) + } + }) + } +} + +const verifyFrontmatter = "workflow:\n verify:\n commands:\n - \"true\"\n" + func TestApplyPromptDefaultsSelectorEffortWins(t *testing.T) { isolateCaptainConfig(t) req := ai.Request{Model: api.Model{Effort: api.EffortLow}} diff --git a/pkg/cli/prompt_run.go b/pkg/cli/prompt_run.go index e02f3aaa..1679936a 100644 --- a/pkg/cli/prompt_run.go +++ b/pkg/cli/prompt_run.go @@ -82,7 +82,7 @@ func runPromptAction(ctx context.Context, id string, flags map[string]string) (P if len(rendered.Runtimes) > 0 { return launchAsyncBatch(ctx, id, rendered, rendered.Runtimes, chatRequested) } - return launchAsyncRun(id, rendered, chatRequested), nil + return launchAsyncRun(id, rendered, chatRequested) } return executeSyncRun(ctx, rendered, opts) } @@ -91,10 +91,13 @@ var executePromptRequestFunc = executePromptRequest // launchAsyncRun starts the background clicky task + SSE stream and returns the // run handle (the serve/web-UI contract). -func launchAsyncRun(id string, rendered PromptRenderResult, chat bool) PromptRunResult { +func launchAsyncRun(id string, rendered PromptRenderResult, chat bool) (PromptRunResult, error) { + timeout, err := renderedTimeout(rendered) + if err != nil { + return PromptRunResult{}, err + } runID := uuid.NewString() stream := promptRuns.create(runID) - timeout := renderedTimeout(rendered) capabilities := chatCapabilitiesFor(rendered.Provider, rendered.Mode) stream.setRun(PromptRunFrame{ RunID: runID, Status: "running", Chat: chat, Model: rendered.Model, @@ -121,7 +124,7 @@ func launchAsyncRun(id string, rendered PromptRenderResult, chat bool) PromptRun return PromptRunResult{ RunID: runID, Status: "running", Model: rendered.Model, Provider: rendered.Provider, Mode: rendered.Mode, Chat: chat, Capabilities: capabilities, - } + }, nil } func workflowConfigured(workflow *api.Workflow) bool { @@ -157,7 +160,11 @@ func executeSyncRunSingleDirect(ctx context.Context, t *task.Task, rendered Prom if workflowConfigured(rendered.Input.Workflow) { return executeSyncWorkflowRun(t, rendered, opts.NoStream, binding) } - out, err := executePromptRequestFunc(ctx, rendered.Input, rendered.Config, renderedTimeout(rendered), opts.NoStream) + timeout, err := renderedTimeout(rendered) + if err != nil { + return PromptRunResult{}, err + } + out, err := executePromptRequestFunc(ctx, rendered.Input, rendered.Config, timeout, opts.NoStream) if err != nil { return PromptRunResult{}, err } @@ -195,9 +202,11 @@ func executeSyncWorkflowRun(t *task.Task, rendered PromptRenderResult, noStream // loop only runs under `captain serve` — deregister so embedders don't // accumulate finished runs. defer promptRuns.remove(runID) - timeout := renderedTimeout(rendered) + timeout, err := renderedTimeout(rendered) + if err != nil { + return PromptRunResult{}, err + } var summary PromptRunSummary - var err error if noStream { summary, err = runPromptBufferedWorkflow(t, rendered, timeout, runID, stream, binding) } else { diff --git a/pkg/cli/prompt_run_events.go b/pkg/cli/prompt_run_events.go index 938faa5f..5aa4ef44 100644 --- a/pkg/cli/prompt_run_events.go +++ b/pkg/cli/prompt_run_events.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/bash" "github.com/flanksource/captain/pkg/session" "github.com/segmentio/encoding/json" @@ -30,7 +31,14 @@ type taskSink interface { type promptEventAccumulator struct { mu sync.Mutex emit func(session.Message) - task taskSink + // verify receives the run's verification state — every in-flight snapshot and + // the verdict — on its own channel rather than as transcript frames. Nil for + // a consumer with no stream behind it (the terminal renderer). + verify func(VerifyFrame) + // lastVerify is the newest report any verify event carried, kept so a verdict + // that arrives without one can still close the frame without blanking it. + lastVerify *api.VerifyReport + task taskSink sessionID string model string @@ -96,6 +104,21 @@ func (a *promptEventAccumulator) handle(_ int, ev ai.Event) { a.emitToolResult(ev) case ai.EventPermission: a.task.Warnf("permission: %s awaiting approval", ev.Tool) + case ai.EventVerifyProgress: + // Stream state, never a transcript frame and never a log line: a check + // reporting every few hundred milliseconds would otherwise write a + // superseded count into the replay buffer, and into the task log, for + // each one. The task gets its status updated in place instead. + a.publishVerify(ev, false) + case ai.EventVerified, ai.EventVerifyFailed: + // The loop's definition of done, reported as it is reached. Like a + // lifecycle notice it stands on its own rather than joining the in-flight + // turn, so the buffers are flushed first; unlike one it carries its own + // role, so a reader can find the verdicts in a transcript without + // matching on prose. + a.flush() + a.publishVerify(ev, true) + a.emitVerdict(ev) case ai.EventError: a.flush() a.emitError(ev) @@ -250,6 +273,80 @@ func (a *promptEventAccumulator) emitNotice(text string) { }) } +// publishVerify forwards the event's typed report as the run's current +// verification state. +// +// A progress event without a report publishes nothing: a frame with a nil report +// would blank whatever the last real snapshot put on screen. A verdict without +// one still publishes — Done is the only thing that turns the verification panel +// from a running check into a result, and withholding it left the panel spinning +// on a superseded snapshot for the rest of the run. It carries the last snapshot +// forward rather than a nil report, so the panel keeps what it was showing and +// merely stops. +func (a *promptEventAccumulator) publishVerify(ev ai.Event, done bool) { + report, _ := ev.Raw.(*api.VerifyReport) + if report != nil { + a.lastVerify = report + } else if !done { + return + } + if !done && report != nil { + a.task.SetDescription(verifyProgressStatus(*report)) + } + if a.verify != nil { + a.verify(VerifyFrame{Report: a.lastVerify, Done: done}) + } +} + +// verifyProgressStatus is the one-line count a task's status shows while a check +// runs: how far it has got, and whether anything has gone red yet. +func verifyProgressStatus(report api.VerifyReport) string { + s := report.Summary + name := report.Name + if s.Total == 0 { + return "verifying " + name + } + // A producer that reports more outstanding rows than its tree has — a suite + // still settling its own totals — must not read as negative progress in the + // one line a person is watching. + done := max(s.Total-s.Pending-s.Running, 0) + if failed := s.Failed + s.TimedOut; failed > 0 { + return fmt.Sprintf("verifying %s %d/%d, %d failed", name, done, s.Total, failed) + } + return fmt.Sprintf("verifying %s %d/%d", name, done, s.Total) +} + +// emitVerdict records one verify verdict as its own transcript role, so it is +// selectable in a stored session rather than being one more system line. The +// typed report rides beside the prose as a data part: the verdict is a tree a +// viewer draws, and the sentence is only its headline. +func (a *promptEventAccumulator) emitVerdict(ev ai.Event) { + role := session.RoleVerifyFailed + if ev.Kind == ai.EventVerified { + role = session.RoleVerified + a.task.Infof("verified: %s", ev.Text) + } else { + a.task.Warnf("not verified: %s", ev.Text) + } + parts := []session.Part{{Type: session.PartText, Text: ev.Text}} + if report, ok := ev.Raw.(*api.VerifyReport); ok && report != nil { + encoded, err := json.Marshal(report) + if err != nil { + // The verdict itself still lands; losing the tree is a defect worth + // naming rather than a frame worth dropping. + a.task.Warnf("verify report for %s could not be encoded: %v", report.Name, err) + } else { + parts = append(parts, session.Part{Type: session.PartVerify, Data: encoded}) + } + } + a.emit(session.Message{ + ID: a.nextID(string(ev.Kind)), + Role: role, + Parts: parts, + Provenance: a.provenance(), + }) +} + func (a *promptEventAccumulator) nextID(kind string) string { a.seq++ if a.idPrefix == "" { diff --git a/pkg/cli/prompt_run_events_test.go b/pkg/cli/prompt_run_events_test.go index e3d9a576..4b2ae0a9 100644 --- a/pkg/cli/prompt_run_events_test.go +++ b/pkg/cli/prompt_run_events_test.go @@ -1,10 +1,12 @@ package cli import ( + "fmt" "strings" "testing" "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/session" "github.com/segmentio/encoding/json" ) @@ -179,3 +181,183 @@ func TestPromptRunAccumulator_CapturesSessionAndUsage(t *testing.T) { t.Fatalf("cost = %v, want 0.02", acc.cost) } } + +// TestPromptRunAccumulator_VerdictsGetTheirOwnRole covers the session half of a +// verify verdict: it is the run's outcome, so a stored transcript must let a +// reader select the verdicts by role rather than searching the system lines for +// prose that looks like one. +func TestPromptRunAccumulator_VerdictsGetTheirOwnRole(t *testing.T) { + msgs := collectEntries("m", "b", + ai.Event{Kind: ai.EventText, Text: "applying the fix"}, + ai.Event{Kind: ai.EventVerifyFailed, Text: "failed in 5m7s: sh failed — verify:oipa-cli test x.yaml"}, + ai.Event{Kind: ai.EventVerified, Success: true, Text: "passed in 4m2s — verify:oipa-cli test x.yaml"}, + ) + + var roles []string + for _, m := range msgs { + roles = append(roles, m.Role) + } + // The in-flight text is flushed first, so the verdict never lands inside the + // sentence the model was part-way through. + want := []string{"assistant", session.RoleVerifyFailed, session.RoleVerified} + if strings.Join(roles, ",") != strings.Join(want, ",") { + t.Fatalf("roles = %v, want %v", roles, want) + } + last := msgs[len(msgs)-1] + if last.Parts[0].Text != "passed in 4m2s — verify:oipa-cli test x.yaml" { + t.Fatalf("verdict parts = %+v, want the report verbatim", last.Parts) + } +} + +// verifyReport builds a report of `done` passing leaves out of `total`, the +// shape a fixture runner streams while it works. +func verifyReport(done, total int) *api.VerifyReport { + tests := make([]api.VerifyNode, 0, total) + for i := 0; i < total; i++ { + node := api.VerifyNode{Name: fmt.Sprintf("check %d", i+1)} + if i < done { + node.Passed = true + } else { + node.Pending = true + } + tests = append(tests, node) + } + report := api.VerifyReport{ + Kind: api.VerifyKindFunc, Name: "fixture", Ran: true, Tests: tests, + Summary: api.SummarizeNodes(tests), State: api.StateForReport(tests), + } + report.Passed = report.State == api.VerifyStatePassed + return &report +} + +// A verdict is a structure as much as a sentence: the webapp draws the +// verification tree, and the only place the transcript could carry it was the +// prose. The typed report rides alongside the text as an AI SDK data part. +func TestPromptRunAccumulator_VerdictCarriesTheTypedReport(t *testing.T) { + report := verifyReport(3, 3) + msgs := collectEntries("m", "b", ai.Event{ + Kind: ai.EventVerified, Success: true, Text: "passed in 4ms — fixture", Raw: report, + }) + + if len(msgs) != 1 { + t.Fatalf("want one verdict frame, got %d", len(msgs)) + } + parts := msgs[0].Parts + if len(parts) != 2 { + t.Fatalf("verdict parts = %+v, want the text then the report", parts) + } + if parts[0].Type != session.PartText || parts[1].Type != session.PartVerify { + t.Fatalf("part types = %q/%q, want %q then %q", parts[0].Type, parts[1].Type, session.PartText, session.PartVerify) + } + var round api.VerifyReport + if err := json.Unmarshal(parts[1].Data, &round); err != nil { + t.Fatalf("data-verify part does not decode as a report: %v", err) + } + if err := round.Validate(); err != nil { + t.Fatalf("round-tripped report is not valid: %v", err) + } + if round.Summary != report.Summary || round.State != report.State { + t.Fatalf("round-tripped report = %+v, want %+v", round, *report) + } +} + +// Progress is stream state, not transcript: it is superseded within the second, +// and the replay buffer every later subscriber receives in full is the wrong +// place for it. +func TestPromptRunAccumulator_ProgressPublishesFramesAndNoEntries(t *testing.T) { + var frames []VerifyFrame + var entries []session.Message + acc := newPromptEventAccumulator(func(m session.Message) { entries = append(entries, m) }, fakeTaskSink{}, "m", "b") + acc.verify = func(f VerifyFrame) { frames = append(frames, f) } + + for done := 1; done <= 3; done++ { + acc.handle(0, ai.Event{Kind: ai.EventVerifyProgress, Tool: "fixture", Raw: verifyReport(done, 3)}) + } + if len(entries) != 0 { + t.Fatalf("progress produced %d transcript entries, want none: %+v", len(entries), entries) + } + if len(frames) != 3 { + t.Fatalf("progress frames = %d, want one per snapshot", len(frames)) + } + for i, frame := range frames { + if frame.Done { + t.Fatalf("frame %d is marked done while the check is still running", i) + } + if frame.Report.Summary.Passed != i+1 { + t.Fatalf("frame %d passed = %d, want %d", i, frame.Report.Summary.Passed, i+1) + } + } + + acc.handle(0, ai.Event{Kind: ai.EventVerified, Success: true, Text: "passed in 4ms — fixture", Raw: verifyReport(3, 3)}) + if len(frames) != 4 || !frames[3].Done { + t.Fatalf("final frame = %+v, want the verdict marked done", frames[len(frames)-1]) + } + if len(entries) != 1 { + t.Fatalf("verdict produced %d entries, want exactly one", len(entries)) + } +} + +// A count divided by a total the producer has not finished settling — more +// running leaves reported than the tree has rows — must not read as negative +// progress in the one line a person is watching. +func TestVerifyProgressStatus_ClampsDoneAtZero(t *testing.T) { + report := api.VerifyReport{Name: "fixture", Summary: api.VerifySummary{Total: 2, Pending: 2, Running: 1}} + if got := verifyProgressStatus(report); got != "verifying fixture 0/2" { + t.Fatalf("verifyProgressStatus = %q, want the count clamped at zero", got) + } + failing := api.VerifyReport{Name: "fixture", Summary: api.VerifySummary{Total: 2, Running: 3, Failed: 1}} + if got := verifyProgressStatus(failing); got != "verifying fixture 0/2, 1 failed" { + t.Fatalf("verifyProgressStatus = %q, want the count clamped at zero", got) + } +} + +// The verdict is what turns the verification panel from "running" to a result. +// A verdict event whose Raw carries no report used to publish nothing at all, so +// the panel sat on the last progress snapshot forever, spinner and all. +func TestPromptRunAccumulator_VerdictWithoutAReportStillFinishesTheFrame(t *testing.T) { + var frames []VerifyFrame + acc := newPromptEventAccumulator(func(session.Message) {}, fakeTaskSink{}, "m", "b") + acc.verify = func(f VerifyFrame) { frames = append(frames, f) } + + snapshot := verifyReport(1, 3) + acc.handle(0, ai.Event{Kind: ai.EventVerifyProgress, Tool: "fixture", Raw: snapshot}) + acc.handle(0, ai.Event{Kind: ai.EventVerifyFailed, Text: "failed in 4ms — fixture"}) + + if len(frames) != 2 { + t.Fatalf("frames = %d, want the snapshot and then the verdict", len(frames)) + } + if !frames[1].Done { + t.Fatalf("verdict frame = %+v, want Done", frames[1]) + } + if frames[1].Report != snapshot { + t.Fatalf("verdict frame report = %+v, want the last snapshot kept rather than blanked", frames[1].Report) + } +} + +// A progress snapshot with no report is the one frame that must be dropped: +// publishing it would blank whatever the last real snapshot put on screen. +func TestPromptRunAccumulator_ProgressWithoutAReportPublishesNothing(t *testing.T) { + var frames []VerifyFrame + acc := newPromptEventAccumulator(func(session.Message) {}, fakeTaskSink{}, "m", "b") + acc.verify = func(f VerifyFrame) { frames = append(frames, f) } + + acc.handle(0, ai.Event{Kind: ai.EventVerifyProgress, Tool: "fixture"}) + if len(frames) != 0 { + t.Fatalf("frames = %+v, want none", frames) + } +} + +// Every other publisher on the stream stops at done; setVerify did not, so a +// check reporting after the run's terminal frame appended to a buffer whose +// subscribers were already closed. +func TestRunStreamSetVerify_StopsAtDone(t *testing.T) { + stream := newRunStream() + stream.fail("stopped") + stream.setVerify(VerifyFrame{Report: verifyReport(1, 1), Done: true}) + + snapshot, ch := stream.subscribeEvents() + stream.unsubscribeEvents(ch) + if snapshot.Verify != nil { + t.Fatalf("verify frame = %+v, want nothing published after the run ended", snapshot.Verify) + } +} diff --git a/pkg/cli/prompt_run_iterations.go b/pkg/cli/prompt_run_iterations.go new file mode 100644 index 00000000..d726bf7a --- /dev/null +++ b/pkg/cli/prompt_run_iterations.go @@ -0,0 +1,36 @@ +package cli + +import ( + "encoding/json" + + "github.com/flanksource/captain/pkg/api" +) + +// resultJSONWithVerify puts the run's final report on result_json under +// `verify`, beside the prompt's own structured output. It copies rather than +// mutating: the same map is the CLI summary's StructuredOutput, which is the +// prompt's answer and nothing else. +// +// The per-turn rows that accompany it come from promptrun.IterationRecords — +// the one derivation every host shares. +func resultJSONWithVerify(structured map[string]any, report *api.VerifyReport) map[string]any { + if report == nil { + return structured + } + raw, err := json.Marshal(report) + if err != nil { + log.Errorf("prompt run result_json: encoding the verify report of %q failed: %v", report.Name, err) + return structured + } + var encoded map[string]any + if err := json.Unmarshal(raw, &encoded); err != nil { + log.Errorf("prompt run result_json: decoding the verify report of %q failed: %v", report.Name, err) + return structured + } + merged := make(map[string]any, len(structured)) + for key, value := range structured { + merged[key] = value + } + merged["verify"] = encoded + return merged +} diff --git a/pkg/cli/prompt_run_iterations_test.go b/pkg/cli/prompt_run_iterations_test.go new file mode 100644 index 00000000..9e541d4b --- /dev/null +++ b/pkg/cli/prompt_run_iterations_test.go @@ -0,0 +1,40 @@ +package cli + +import ( + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" +) + +// The record derivation itself is specified in pkg/promptrun +// (iterations_ginkgo_test.go); these helpers build the runs the persistence +// specs here file through it. + +// verdictReport is the report one turn's verifier produced, stamped for that +// 1-based turn exactly as verify.Plugin stamps it. +func verdictReport(iteration int, passed bool) *api.VerifyReport { + node := api.VerifyNode{Name: "go test ./...", Passed: passed, Failed: !passed} + report := api.NewNodeReport(api.VerifyKindCmd, "verify:go test ./...", node) + report.Iteration = iteration + return &report +} + +func loopWith(turns int, base time.Time, err error) *ai.LoopResult { + loop := &ai.LoopResult{StopReason: "condition-met"} + for i := 0; i < turns; i++ { + started := base.Add(time.Duration(i) * time.Minute) + iteration := &ai.LoopIteration{ + Iteration: i, + Request: ai.Request{Prompt: api.Prompt{User: "attempt " + string(rune('A'+i))}}, + StartedAt: started, + FinishedAt: started.Add(30 * time.Second), + Success: true, + } + if i == turns-1 { + iteration.Err = err + } + loop.Iterations = append(loop.Iterations, iteration) + } + return loop +} diff --git a/pkg/cli/prompt_run_live.go b/pkg/cli/prompt_run_live.go index c420d31b..e9c05a25 100644 --- a/pkg/cli/prompt_run_live.go +++ b/pkg/cli/prompt_run_live.go @@ -2,16 +2,14 @@ package cli import ( "context" - "encoding/json" "errors" - "fmt" "time" "github.com/flanksource/captain/pkg/ai" - "github.com/flanksource/captain/pkg/ai/agent" - "github.com/flanksource/captain/pkg/ai/agent/commit" - "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/ai/middleware" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/database" + "github.com/flanksource/captain/pkg/promptrun" "github.com/flanksource/clicky/task" ) @@ -23,193 +21,172 @@ func runPromptBufferedWorkflow(t *task.Task, rendered PromptRenderResult, timeou return runPromptWorkflow(t, rendered, timeout, runID, stream, binding, true) } +// runPromptWorkflow is `captain prompt run`'s caller of promptrun.Run: it owns +// what is specific to this process — the stop button, the live stream and its +// transcript frames, attachment resolution against the local store, the remote +// sandbox provider, and persistence — and hands the run itself to the shared +// seam so it means the same thing here as in an embedding host. func runPromptWorkflow(t *task.Task, rendered PromptRenderResult, timeout time.Duration, runID string, stream *runStream, binding *promptSessionBinding, noStream bool) (PromptRunSummary, error) { - ctx, cancel := runContext(t.Context(), rendered.Input, timeout) + ctx, cancel := context.WithCancel(t.Context()) stream.setCancel(cancel) defer cancel() ctx = ai.ContextWithLogger(ctx, t) req := rendered.Input - cfg := rendered.Config - if err := preparePromptAttachments(ctx, &req, cfg); err != nil { - return failRun(t, stream, err) - } - // Judge prompts load before the provider is built: a workflow naming a - // prompt that does not exist is broken everywhere, and building first would - // report whichever runtime binary this machine lacks instead. - judgePrompts, err := verify.LoadJudgePrompts(req.Workflow) - if err != nil { - return failRun(t, stream, err) - } - - // A verify-only run makes no model call — agent.Runner takes runVerifyOnce - // and never touches the provider — so constructing one, and demanding its - // runtime binary on PATH, would fail a run that only executes shell hooks. - // A declared judge does execute on the provider, so it still needs one. - var p ai.Provider - if !req.IsVerifyOnly() || len(judgePrompts) > 0 { - built, cleanup, err := buildProvider(ctx, &req, cfg) - if err != nil { - return failRun(t, stream, err) - } - defer cleanup() - defer closeProvider(built) - p = built - } - - streamer, err := workflowRunnerProvider(p, noStream, req.IsVerifyOnly()) + remote, err := preparePromptRun(ctx, &req, rendered.Config) if err != nil { return failRun(t, stream, err) } - - judgeHooks, err := verify.JudgeHooks(judgePrompts, p) - if err != nil { - return failRun(t, stream, err) + if remote != nil { + defer closeProvider(remote) } start := time.Now() acc := newPromptEventAccumulator(stream.publish, t, rendered.Model, rendered.Mode) - acc.cwd = req.Cwd() - acc.idPrefix = runID - runner := &agent.Runner[string]{ - Provider: streamer, + acc.cwd, acc.idPrefix, acc.verify = req.Cwd(), runID, stream.setVerify + result, err := promptrun.Run(ctx, promptrun.Input{ Request: req, - // Commit hooks lead so that at PhaseRun they squash before any teardown - // hook (a worktree merge) runs and takes the result. - Hooks: append(append(commit.HooksForWorkflow(req.Workflow), verify.HooksForWorkflow(req.Workflow)...), judgeHooks...), - MaxIterations: verify.MaxIterationsForWorkflow(req.Workflow), - Repo: req.Cwd(), - Cwd: req.Cwd(), - Scope: verify.ScopeForWorkflow(req.Workflow), - OnEvent: acc.handle, - } - runResult, err := runner.Run(ctx) - session, model, usage, cost := acc.snapshot() - loop := runResult.Loop - if session == "" && runResult.Response.Workspace != nil { - session = runResult.Response.Workspace.SessionID - } - if session == "" && loop != nil && len(loop.Iterations) > 0 { - session = loop.Iterations[0].SessionID - } - stream.setRunMetadata(session, model) + Config: rendered.Config, + Provider: remote, + OnEvent: acc.handle, + Timeout: timeout, + NoStream: noStream, + }) + stream.setRunMetadata(result.SessionID, firstNonEmpty(result.Model, rendered.Model)) + + interrupted := contextEndedRun(ctx, err) + record := promptRunRecord(rendered, runID, binding, result, interrupted) if err != nil { if stream.wasStopped() { err = errors.New("stopped") } + persistPromptRun(context.WithoutCancel(ctx), failedRunRecord(record, err, interrupted)) return failRun(t, stream, err) } - passed := verifyPassed(runResult.Verdicts) - structuredOutput, err := structuredOutputMap(runResult.Response.StructuredData) - if err != nil { - return failRun(t, stream, err) - } - resultText, err := structuredOutputText(runResult.Response.Text, structuredOutput) + structured, err := completeRunRecord(&record, result) if err != nil { + persistPromptRun(context.WithoutCancel(ctx), failedRunRecord(record, err, false)) return failRun(t, stream, err) } - record := promptRunRecordInput{ - Rendered: rendered, RunID: runID, Binding: binding, SessionID: session, - Model: model, Provider: providerOf(api.Runtime{Provider: rendered.Provider, Mode: api.RuntimeMode(rendered.Mode)}), Mode: api.RuntimeMode(rendered.Mode), ResultText: resultText, ResultJSON: structuredOutput, - } - if !passed { - record.Error = verifyReason(runResult.Verdicts) - } persistPromptRun(context.WithoutCancel(ctx), record) - summarySessionID := session - if binding != nil { - summarySessionID = binding.SessionID.String() - } - summary := PromptRunSummary{ - RunID: runID, - SessionID: summarySessionID, - Model: model, - Provider: rendered.Provider, - Mode: rendered.Mode, - InputTokens: usage.InputTokens, - OutputTokens: usage.OutputTokens, - CostUSD: cost, - Duration: time.Since(start).Round(time.Millisecond).String(), - Success: passed, - Text: resultText, - StructuredOutput: structuredOutput, - } - if !passed { - summary.Error = verifyReason(runResult.Verdicts) - } + + summary := completedRunSummary(record, result, binding, structured, time.Since(start)) stream.complete(summary) t.Success() return summary, nil } -func workflowRunnerProvider(provider ai.Provider, noStream, verifyOnly bool) (ai.StreamingProvider, error) { - if provider == nil { - if verifyOnly { - return nil, nil - } - return nil, fmt.Errorf("a generating run needs a provider") - } - if noStream { - return bufferedWorkflowProvider{Provider: provider}, nil +// preparePromptRun is everything this process must do to the request before the +// shared seam sees it: warn on a model name that looks mistyped, resolve the +// prompt's attachments against the local store, and — when the resolved sandbox +// executes elsewhere — build the provider that owns the whole run. A nil +// provider means the run executes here. +func preparePromptRun(ctx context.Context, req *ai.Request, cfg ai.Config) (ai.Provider, error) { + for _, c := range cfg.Model.Candidates() { + warnIfLikelyModelTypo(c.Name) } - streamer, ok := provider.(ai.StreamingProvider) - if ok || verifyOnly { - return streamer, nil + if err := resolvePromptAttachments(ctx, req); err != nil { + return nil, err } - return bufferedWorkflowProvider{Provider: provider}, nil + return remoteWorkflowProvider(req, cfg) } -// bufferedWorkflowProvider preserves the agent runner's event contract while -// forcing generation through Provider.Execute. It emits only completed response -// events, so --no-stream never invokes an underlying ExecuteStream method. -type bufferedWorkflowProvider struct { - ai.Provider +// promptRunRecord is the run as it will be persisted, assembled before the error +// branch so that an interrupted run — or one that broke on turn 2 of 3 — is +// still written down. Its verdict travels two ways: one row per turn (what was +// asked, what the check said, how long it took) and result_json.verify, the +// round's report beside the prompt's own structured output. Returning on the +// error path before this left a stopped run with no rows and no report at all. +func promptRunRecord(rendered PromptRenderResult, runID string, binding *promptSessionBinding, result promptrun.Result, interrupted bool) promptRunRecordInput { + runtime := api.Runtime{Provider: rendered.Provider, Mode: api.RuntimeMode(rendered.Mode)} + return promptRunRecordInput{ + Rendered: rendered, RunID: runID, Binding: binding, SessionID: result.SessionID, + Model: firstNonEmpty(result.Model, rendered.Model), Provider: providerOf(runtime), Mode: runtime.Mode, + ResultJSON: resultJSONWithVerify(nil, result.Report), + Iterations: promptrun.IterationRecords(result, interrupted), + } } -func (p bufferedWorkflowProvider) ExecuteStream(ctx context.Context, req ai.Request) (<-chan ai.Event, error) { - resp, err := p.Execute(ctx, req) +// completeRunRecord fills in what only a run that reached its own end has: the +// answer, as text and as structured output, and the failure reason of a run +// whose checks said no. +func completeRunRecord(record *promptRunRecordInput, result promptrun.Result) (map[string]any, error) { + structured, err := structuredOutputMap(result.StructuredData) if err != nil { return nil, err } - if resp == nil { - return nil, errors.New("buffered workflow provider returned a nil response") - } - structured, err := bufferedStructuredData(resp.StructuredData) - if err != nil { + if record.ResultText, err = structuredOutputText(result.Response.Text, structured); err != nil { return nil, err } - - events := make(chan ai.Event, 3) - if resp.Workspace != nil && resp.Workspace.SessionID != "" { - events <- ai.Event{Kind: ai.EventSystem, SessionID: resp.Workspace.SessionID, Model: resp.Model} - } - if resp.Text != "" { - events <- ai.Event{Kind: ai.EventText, Text: resp.Text, Model: resp.Model} + record.ResultJSON = resultJSONWithVerify(structured, result.Report) + if !result.Passed { + record.Error = promptrun.FailureReason(result.Verdicts) } - usage := resp.Usage - events <- ai.Event{ - Kind: ai.EventResult, Success: true, Model: resp.Model, Usage: &usage, - CostUSD: resp.CostUSD, StructuredData: structured, ToolApproval: resp.ToolApproval, - } - close(events) - return events, nil + return structured, nil } -func bufferedStructuredData(value any) (json.RawMessage, error) { - if value == nil { - return nil, nil +// completedRunSummary is the finished run as the CLI prints it and the stream +// reports it. The session it names is the captain session when the run is bound +// to one, not the provider's own id. +func completedRunSummary(record promptRunRecordInput, result promptrun.Result, binding *promptSessionBinding, structured map[string]any, elapsed time.Duration) PromptRunSummary { + sessionID := record.SessionID + if binding != nil { + sessionID = binding.SessionID.String() + } + return PromptRunSummary{ + RunID: record.RunID, + SessionID: sessionID, + Model: record.Model, + Provider: record.Rendered.Provider, + Mode: record.Rendered.Mode, + InputTokens: result.Usage.InputTokens, + OutputTokens: result.Usage.OutputTokens, + CostUSD: result.CostUSD, + Duration: elapsed.Round(time.Millisecond).String(), + Success: result.Passed, + Text: record.ResultText, + StructuredOutput: structured, + Error: record.Error, } - if raw, ok := value.(json.RawMessage); ok { - return raw, nil +} + +// remoteWorkflowProvider is the whole-run relocation branch: when the resolved +// sandbox executes remotely, the run happens on another machine and comes back +// whole, so the provider it returns owns the workspace (promptrun adds no setup +// hook) and never streams. Nil means the run executes here. +func remoteWorkflowProvider(req *ai.Request, cfg ai.Config) (ai.Provider, error) { + remote, err := remoteExecProviderFor(req, cfg) + if err != nil || remote == nil { + return nil, err } - raw, err := json.Marshal(value) + wrapped, err := middleware.Wrap(remote, middleware.WithLogging(), middleware.WithSchemaValidation(cfg)) if err != nil { - return nil, fmt.Errorf("encode buffered workflow structured output: %w", err) + closeProvider(remote) + return nil, err } - if string(raw) == "null" { - return nil, nil + return bufferedOnlyProvider{Provider: wrapped}, nil +} + +// contextEndedRun reports whether the loop stopped because its context did — +// the stop button, or the run's own deadline. Both leave the last turn cut off +// rather than judged, and neither is the work's fault. +func contextEndedRun(ctx context.Context, err error) bool { + if err == nil { + return false + } + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil +} + +// failedRunRecord stamps the record of a run that ended in an error: an +// interrupted run is cancelled — its work was cut off, not judged — and anything +// else failed. +func failedRunRecord(record promptRunRecordInput, err error, interrupted bool) promptRunRecordInput { + record.Error = err.Error() + record.State = database.PromptRunStateFailed + if interrupted { + record.State = database.PromptRunStateCancelled } - return raw, nil + return record } func failRun(t *task.Task, stream *runStream, err error) (PromptRunSummary, error) { diff --git a/pkg/cli/prompt_run_persist.go b/pkg/cli/prompt_run_persist.go index 2f4008d5..10682f0e 100644 --- a/pkg/cli/prompt_run_persist.go +++ b/pkg/cli/prompt_run_persist.go @@ -3,6 +3,8 @@ package cli import ( "context" "encoding/json" + "errors" + "fmt" "strings" "github.com/flanksource/captain/pkg/api" @@ -23,6 +25,14 @@ type promptRunRecordInput struct { ResultText string ResultJSON map[string]any Error string + // State overrides the state the run row is recorded under. Empty derives it + // from Error, which is what a run that reached its own end needs. An + // interrupted run is neither succeeded nor failed — its work was cut off, not + // judged — so the run path stamps `cancelled` explicitly. + State database.PromptRunState + // Iterations is one row per executed loop turn (1-based), built by + // promptRunIterationRecords from the runner's own loop and verdicts. + Iterations []database.UpsertPromptRunIterationInput } // persistPromptRun records a captain-launched run against its session in the @@ -64,6 +74,7 @@ func persistPromptRun(ctx context.Context, input promptRunRecordInput) { if input.Binding != nil { batchID = &input.Binding.BatchID } + var runID uuid.UUID err = db.Transaction(ctx, func(tx *database.DB) error { run, createErr := tx.CreatePromptRun(ctx, database.CreatePromptRunInput{ SessionID: session.ID, @@ -84,9 +95,12 @@ func persistPromptRun(ctx context.Context, input promptRunRecordInput) { return createErr } finished := database.PromptRunPhaseFinished - state := database.PromptRunStateSucceeded - if input.Error != "" { - state = database.PromptRunStateFailed + state := input.State + if state == "" { + state = database.PromptRunStateSucceeded + if input.Error != "" { + state = database.PromptRunStateFailed + } } update := database.UpdatePromptRunInput{ ID: run.ID, ExpectedVersion: run.Version, Phase: &finished, State: &state, @@ -100,21 +114,44 @@ func persistPromptRun(ctx context.Context, input promptRunRecordInput) { if input.Error != "" { update.Error = &input.Error } - _, updateErr := tx.UpdatePromptRun(ctx, update) - return updateErr + if _, updateErr := tx.UpdatePromptRun(ctx, update); updateErr != nil { + return updateErr + } + runID = run.ID + return nil }) if err != nil { log.Errorf("persist prompt run for session %s: %v", firstNonEmpty(input.SessionID, bindingSessionID(input.Binding)), err) return } + if err := upsertPromptRunIterations(ctx, db, runID, input.Iterations); err != nil { + log.Errorf("persist prompt run %s iterations for session %s: %v", input.RunID, firstNonEmpty(input.SessionID, bindingSessionID(input.Binding)), err) + } lifecycle := database.SessionLifecycleSucceeded - if input.Error != "" { + if input.Error != "" || input.State == database.PromptRunStateCancelled { lifecycle = database.SessionLifecycleFailed } updatePromptSessionLifecycle(ctx, session.ID, lifecycle, input.Error) trackLaunchedTranscript(input, source) } +// upsertPromptRunIterations writes the run's per-turn rows after the run row +// has been committed, one row at a time and all of them: a turn the store +// refuses (a report that fails its own validation) must cost exactly that row. +// Writing them inside the run's transaction took the run itself down with a bad +// turn, leaving no record that the run had happened at all; and stopping at the +// first refusal hid the turns after it. Every refusal is reported. +func upsertPromptRunIterations(ctx context.Context, db *database.DB, runID uuid.UUID, records []database.UpsertPromptRunIterationInput) error { + var errs []error + for _, record := range records { + record.PromptRunID = runID + if _, err := db.UpsertPromptRunIteration(ctx, record); err != nil { + errs = append(errs, fmt.Errorf("iteration %d: %w", record.Iteration, err)) + } + } + return errors.Join(errs...) +} + func bindingSessionID(binding *promptSessionBinding) string { if binding == nil { return "" diff --git a/pkg/cli/prompt_run_persist_test.go b/pkg/cli/prompt_run_persist_test.go index 94e8d3e0..b56ed94a 100644 --- a/pkg/cli/prompt_run_persist_test.go +++ b/pkg/cli/prompt_run_persist_test.go @@ -1,10 +1,14 @@ package cli import ( + "errors" "testing" + "time" + "github.com/flanksource/captain/pkg/ai/agent" "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/database" + "github.com/flanksource/captain/pkg/promptrun" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -70,3 +74,163 @@ func TestPersistPromptRunRecordsNativeRun(t *testing.T) { assert.Equal(t, "verify failed: tests red", failed.Error) }) } + +// A run that had to try twice is only legible if both attempts survive: the +// failing turn with the feedback that drove the retry, and the passing one that +// ended it. Without the rows a reader sees a green run and no account of the red +// one it started as. +func TestPersistPromptRunRecordsEveryIteration(t *testing.T) { + db := withTestCaptainDB(t) + const providerSession = "0195c1de-4ab8-7000-8000-0000000abcde" + rendered := PromptRenderResult{Name: "fix-bug", Model: "claude-sonnet-5", Provider: "anthropic", Mode: "agent"} + rendered.Input.Prompt.User = "fix the failing test" + + base := time.Date(2026, 9, 3, 9, 0, 0, 0, time.UTC) + loop := loopWith(2, base, nil) + verdicts := []agent.VerifyResult{ + {Valid: false, Iteration: 1, Report: verdictReport(1, false)}, + {Valid: true, Iteration: 2, Report: verdictReport(2, true)}, + } + verdicts[0].Report.Feedback = "TestFoo failed" + final, err := promptrun.FinalReport(verdicts) + require.NoError(t, err) + + persistPromptRun(t.Context(), promptRunRecordInput{ + Rendered: rendered, RunID: "run-iter", SessionID: providerSession, + Model: "claude-sonnet-5", Provider: api.Anthropic, Mode: api.ModeAgent, + ResultText: "fixed", + ResultJSON: resultJSONWithVerify(map[string]any{"answer": "42"}, final), + Iterations: promptrun.IterationRecords(promptrun.Result{Loop: loop, Verdicts: verdicts}, false), + }) + + session, err := db.GetSessionByIdentity(t.Context(), providerSession, "claude", "", "") + require.NoError(t, err) + runs, err := db.ListPromptRuns(t.Context(), database.PromptRunFilter{SessionID: &session.ID}) + require.NoError(t, err) + require.Len(t, runs, 1) + + iterations, err := db.ListPromptRunIterations(t.Context(), runs[0].ID) + require.NoError(t, err) + require.Len(t, iterations, 2) + + first := iterations[0] + assert.Equal(t, 1, first.Iteration) + assert.Equal(t, database.PromptRunIterationStateFailed, first.State) + assert.Equal(t, "TestFoo failed", first.Feedback) + require.NotNil(t, first.VerificationResult) + assert.False(t, first.VerificationResult.Passed) + assert.Equal(t, 1, first.VerificationResult.Iteration) + assert.Equal(t, map[string]any{"prompt": "attempt A"}, first.Request) + require.NotNil(t, first.StartedAt) + require.NotNil(t, first.FinishedAt) + // The turn's own clock, not the clock at write time: the trigger's back-fill + // would stamp both attempts with the moment the run ended. + assert.Equal(t, base.UTC(), first.StartedAt.UTC()) + assert.Equal(t, base.Add(30*time.Second).UTC(), first.FinishedAt.UTC()) + + second := iterations[1] + assert.Equal(t, 2, second.Iteration) + assert.Equal(t, database.PromptRunIterationStateSucceeded, second.State) + require.NotNil(t, second.VerificationResult) + assert.True(t, second.VerificationResult.Passed) + + // The final report also lands on the run itself, beside the prompt's answer. + require.NotNil(t, runs[0].ResultJSON) + assert.Equal(t, "42", runs[0].ResultJSON["answer"]) + verify, ok := runs[0].ResultJSON["verify"].(map[string]any) + require.True(t, ok, "result_json.verify = %#v", runs[0].ResultJSON["verify"]) + assert.Equal(t, true, verify["passed"]) + + report, iteration, err := db.LatestPromptRunVerification(t.Context(), runs[0].ID) + require.NoError(t, err) + assert.Equal(t, 2, iteration, "the run's verdict is the last turn's, not the first's") + require.NotNil(t, report) + assert.True(t, report.Passed) +} + +// A run the stop button ended on turn 2 of 3 is still a run: the turn that +// completed, the verdict that judged it, and a row that says it was cancelled +// rather than that it failed. Returning before persistence left a stopped run +// with no row at all — no iterations, no result_json.verify, nothing to read. +func TestPersistPromptRunRecordsACancelledRun(t *testing.T) { + db := withTestCaptainDB(t) + const providerSession = "0195c1de-4ab8-7000-8000-0000000abce0" + rendered := PromptRenderResult{Name: "fix-bug", Model: "claude-sonnet-5", Provider: "anthropic", Mode: "agent"} + rendered.Input.Prompt.User = "fix the failing test" + + base := time.Date(2026, 9, 3, 9, 0, 0, 0, time.UTC) + verdicts := []agent.VerifyResult{{Valid: false, Iteration: 1, Report: verdictReport(1, false)}} + + persistPromptRun(t.Context(), promptRunRecordInput{ + Rendered: rendered, RunID: "run-stopped", SessionID: providerSession, + Model: "claude-sonnet-5", Provider: api.Anthropic, Mode: api.ModeAgent, + Error: "stopped", State: database.PromptRunStateCancelled, + Iterations: promptrun.IterationRecords(promptrun.Result{Loop: loopWith(1, base, nil), Verdicts: verdicts}, true), + }) + + session, err := db.GetSessionByIdentity(t.Context(), providerSession, "claude", "", "") + require.NoError(t, err) + runs, err := db.ListPromptRuns(t.Context(), database.PromptRunFilter{SessionID: &session.ID}) + require.NoError(t, err) + require.Len(t, runs, 1) + assert.Equal(t, database.PromptRunStateCancelled, runs[0].State, + "an interrupted run is neither succeeded nor failed") + assert.Equal(t, "stopped", runs[0].Error) + + iterations, err := db.ListPromptRunIterations(t.Context(), runs[0].ID) + require.NoError(t, err) + require.Len(t, iterations, 1, "the turn that ran before the stop is the record of how far it got") + assert.Equal(t, database.PromptRunIterationStateCancelled, iterations[0].State) +} + +// failedRunRecord is the stamp the run-path applies when promptrun.Run returns +// an error, and it is the only thing that decides cancelled from failed. +func TestFailedRunRecord_CancelledVersusFailed(t *testing.T) { + base := promptRunRecordInput{RunID: "r"} + + stopped := failedRunRecord(base, errors.New("stopped"), true) + assert.Equal(t, database.PromptRunStateCancelled, stopped.State) + assert.Equal(t, "stopped", stopped.Error) + + broke := failedRunRecord(base, errors.New("upstream 529"), false) + assert.Equal(t, database.PromptRunStateFailed, broke.State) + assert.Equal(t, "upstream 529", broke.Error) +} + +// A turn whose report the store refuses must not take the run down with it: the +// run row is the record that the run happened at all, and the other turns are +// still true. The bad row is the only thing that goes missing, and loudly. +func TestPersistPromptRunKeepsTheRunWhenAnIterationIsRejected(t *testing.T) { + db := withTestCaptainDB(t) + const providerSession = "0195c1de-4ab8-7000-8000-0000000abcdf" + rendered := PromptRenderResult{Name: "fix-bug", Model: "claude-sonnet-5", Provider: "anthropic", Mode: "agent"} + rendered.Input.Prompt.User = "fix the failing test" + + base := time.Date(2026, 9, 3, 9, 0, 0, 0, time.UTC) + corrupt := verdictReport(2, true) + corrupt.State = api.VerifyStateFailed // passed=true with a failed state: Validate rejects it + verdicts := []agent.VerifyResult{ + {Valid: false, Iteration: 1, Report: verdictReport(1, false)}, + {Valid: true, Iteration: 2, Report: corrupt}, + } + + persistPromptRun(t.Context(), promptRunRecordInput{ + Rendered: rendered, RunID: "run-partial", SessionID: providerSession, + Model: "claude-sonnet-5", Provider: api.Anthropic, Mode: api.ModeAgent, + ResultText: "fixed", + Iterations: promptrun.IterationRecords(promptrun.Result{Loop: loopWith(2, base, nil), Verdicts: verdicts}, false), + }) + + session, err := db.GetSessionByIdentity(t.Context(), providerSession, "claude", "", "") + require.NoError(t, err) + runs, err := db.ListPromptRuns(t.Context(), database.PromptRunFilter{SessionID: &session.ID}) + require.NoError(t, err) + require.Len(t, runs, 1, "the run row survives a rejected iteration") + assert.Equal(t, database.PromptRunStateSucceeded, runs[0].State) + assert.Equal(t, "fixed", runs[0].ResultText) + + iterations, err := db.ListPromptRunIterations(t.Context(), runs[0].ID) + require.NoError(t, err) + require.Len(t, iterations, 1, "the good turn is kept; only the rejected one is missing") + assert.Equal(t, 1, iterations[0].Iteration) +} diff --git a/pkg/cli/prompt_run_stream.go b/pkg/cli/prompt_run_stream.go index 937142ea..fe41c568 100644 --- a/pkg/cli/prompt_run_stream.go +++ b/pkg/cli/prompt_run_stream.go @@ -18,6 +18,8 @@ const runSubBuffer = 64 // runStream is the in-process pub/sub buffer for one prompt run's session.Message // frames. Every frame is buffered for replay to late/reconnecting subscribers // and fanned out to current subscribers. +// VerifyFrame, setVerify and cloneVerify live in prompt_run_stream_verify.go. + type runStream struct { mu sync.Mutex entries []session.Message @@ -25,6 +27,7 @@ type runStream struct { eventSubs map[chan runStreamEvent]struct{} run PromptRunFrame chatState *ChatStateFrame + verify *VerifyFrame cancel context.CancelFunc stopRequested bool done bool @@ -205,17 +208,22 @@ func (s *runStream) subscribe() (replay []session.Message, ch chan session.Messa return replay, ch, false, nil, "" } -func (s *runStream) subscribeEvents() (PromptRunFrame, []session.Message, *ChatStateFrame, chan runStreamEvent, bool, *PromptRunSummary, string) { +func (s *runStream) subscribeEvents() (promptRunSnapshotBody, chan runStreamEvent) { s.mu.Lock() defer s.mu.Unlock() - replay := append([]session.Message(nil), s.entries...) - state := cloneChatState(s.chatState) + snapshot := promptRunSnapshotBody{ + Run: s.run, + Entries: append([]session.Message(nil), s.entries...), + State: cloneChatState(s.chatState), + Verify: cloneVerify(s.verify), + } if s.done { - return s.run, replay, state, nil, true, s.summary, s.errMsg + snapshot.Done, snapshot.Summary, snapshot.Error = true, s.summary, s.errMsg + return snapshot, nil } ch := make(chan runStreamEvent, runSubBuffer) s.eventSubs[ch] = struct{}{} - return s.run, replay, state, ch, false, nil, "" + return snapshot, ch } func (s *runStream) unsubscribeEvents(ch chan runStreamEvent) { @@ -299,10 +307,15 @@ func (b *runBroker) prune(maxAge time.Duration) { } } +// promptRunSnapshotBody is everything a subscriber needs to render the run as +// it stands at the moment it connects, before a single live event arrives. It +// is both what subscribeEvents hands a new SSE subscriber and the JSON body of +// the snapshot endpoint, so the two can never describe different runs. type promptRunSnapshotBody struct { Run PromptRunFrame `json:"run"` Entries []session.Message `json:"entries"` State *ChatStateFrame `json:"state,omitempty"` + Verify *VerifyFrame `json:"verify,omitempty"` Done bool `json:"done"` Summary *PromptRunSummary `json:"summary,omitempty"` Error string `json:"error,omitempty"` @@ -311,6 +324,7 @@ type promptRunSnapshotBody struct { // handlePromptRunStream streams a run's session.Message frames as SSE: // // event: entry data: (one per frame; replayed on connect) +// event: verify data: (latest verification state; sent on connect) // event: done data: (terminal, success) // event: error data: (terminal, failure) func handlePromptRunStream(b *runBroker) http.HandlerFunc { @@ -327,17 +341,20 @@ func handlePromptRunStream(b *runBroker) http.HandlerFunc { } setSSEHeaders(w) - run, replay, state, ch, done, summary, errMsg := stream.subscribeEvents() + snapshot, ch := stream.subscribeEvents() defer stream.unsubscribeEvents(ch) - writeSSE(w, "run", run) - for _, e := range replay { + writeSSE(w, "run", snapshot.Run) + for _, e := range snapshot.Entries { writeSSE(w, "entry", e) } - if state != nil { - writeSSE(w, "state", state) + if snapshot.State != nil { + writeSSE(w, "state", snapshot.State) + } + if snapshot.Verify != nil { + writeSSE(w, "verify", snapshot.Verify) } - if done { - writeTerminal(w, summary, errMsg) + if snapshot.Done { + writeTerminal(w, snapshot.Summary, snapshot.Error) flusher.Flush() return } @@ -376,10 +393,10 @@ func handlePromptRunSnapshot(b *runBroker) http.HandlerFunc { http.Error(w, "unknown run", http.StatusNotFound) return } - run, entries, state, ch, done, summary, errMsg := stream.subscribeEvents() + snapshot, ch := stream.subscribeEvents() stream.unsubscribeEvents(ch) w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(promptRunSnapshotBody{Run: run, Entries: entries, State: state, Done: done, Summary: summary, Error: errMsg}) + _ = json.NewEncoder(w).Encode(snapshot) } } diff --git a/pkg/cli/prompt_run_stream_verify.go b/pkg/cli/prompt_run_stream_verify.go new file mode 100644 index 00000000..78cd6c8d --- /dev/null +++ b/pkg/cli/prompt_run_stream_verify.go @@ -0,0 +1,40 @@ +package cli + +import "github.com/flanksource/captain/pkg/api" + +// VerifyFrame is the run's current verification state: the newest report a +// verifier produced, and whether it is the verdict or a snapshot of a check +// still running. Only the latest is kept — a superseded progress snapshot has +// no reader — so a late subscriber gets the current state of the check on +// connect the same way it gets the current `run` and `state`. +type VerifyFrame struct { + Report *api.VerifyReport `json:"report"` + Done bool `json:"done"` +} + +// setVerify replaces the run's verification state and publishes it as its own +// SSE event. It is deliberately not a transcript frame: a check reporting every +// few hundred milliseconds would otherwise append a message per snapshot to a +// buffer that is replayed in full to every later subscriber, so the transcript +// would grow with superseded counts and the verdict would be buried in them. +// +// It stops at done like publish does. A finished run's subscribers are already +// closed and its snapshot is the terminal one a late reader gets; a check still +// reporting after that would rewrite the state of a run that has ended. +func (s *runStream) setVerify(frame VerifyFrame) { + s.mu.Lock() + defer s.mu.Unlock() + if s.done { + return + } + s.verify = &frame + s.publishEventLocked(runStreamEvent{name: "verify", data: frame}) +} + +func cloneVerify(frame *VerifyFrame) *VerifyFrame { + if frame == nil { + return nil + } + clone := *frame + return &clone +} diff --git a/pkg/cli/prompt_run_workflow_test.go b/pkg/cli/prompt_run_workflow_test.go index cb19f828..49d76986 100644 --- a/pkg/cli/prompt_run_workflow_test.go +++ b/pkg/cli/prompt_run_workflow_test.go @@ -26,105 +26,6 @@ func workflowRendered(verify *api.Verify) PromptRenderResult { } } -type bufferedOnlyWorkflowProvider struct { - executeCalls int -} - -func (p *bufferedOnlyWorkflowProvider) GetModel() string { return "buffered-model" } -func (p *bufferedOnlyWorkflowProvider) GetRuntime() ai.Runtime { - return api.RuntimeOf(api.DeepSeek, api.ModeAPI) -} -func (p *bufferedOnlyWorkflowProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { - p.executeCalls++ - return &ai.Response{ - Text: "done", - StructuredData: map[string]any{"status": "ok"}, - Model: "buffered-model", - Runtime: api.RuntimeOf(api.DeepSeek, api.ModeAPI), - Usage: ai.Usage{InputTokens: 2, OutputTokens: 3}, - CostUSD: 0.01, - }, nil -} - -type streamingWorkflowProvider struct { - bufferedOnlyWorkflowProvider - streamCalls int -} - -func (p *streamingWorkflowProvider) ExecuteStream(context.Context, ai.Request) (<-chan ai.Event, error) { - p.streamCalls++ - events := make(chan ai.Event, 1) - events <- ai.Event{Kind: ai.EventResult, Success: true} - close(events) - return events, nil -} - -func TestWorkflowRunnerProviderHonorsNoStream(t *testing.T) { - t.Run("buffered-only provider uses completed events", func(t *testing.T) { - provider := &bufferedOnlyWorkflowProvider{} - runner, err := workflowRunnerProvider(provider, false, false) - if err != nil { - t.Fatal(err) - } - events, err := runner.ExecuteStream(context.Background(), ai.Request{}) - if err != nil { - t.Fatal(err) - } - var got []ai.Event - for event := range events { - got = append(got, event) - } - if provider.executeCalls != 1 { - t.Fatalf("Execute calls = %d, want 1", provider.executeCalls) - } - if len(got) != 2 || got[0].Kind != ai.EventText || got[0].Text != "done" || got[1].Kind != ai.EventResult { - t.Fatalf("events = %+v, want final text and result", got) - } - }) - - t.Run("buffered-only provider", func(t *testing.T) { - provider := &bufferedOnlyWorkflowProvider{} - runner, err := workflowRunnerProvider(provider, true, false) - if err != nil { - t.Fatal(err) - } - events, err := runner.ExecuteStream(context.Background(), ai.Request{}) - if err != nil { - t.Fatal(err) - } - var got []ai.Event - for event := range events { - got = append(got, event) - } - if provider.executeCalls != 1 { - t.Fatalf("Execute calls = %d, want 1", provider.executeCalls) - } - if len(got) != 2 || got[0].Kind != ai.EventText || got[0].Text != "done" || got[1].Kind != ai.EventResult { - t.Fatalf("events = %+v, want final text and result", got) - } - if string(got[1].StructuredData) != `{"status":"ok"}` || got[1].Usage == nil || got[1].Usage.InputTokens != 2 || got[1].CostUSD != 0.01 { - t.Fatalf("result event = %+v", got[1]) - } - }) - - t.Run("streaming remains unchanged", func(t *testing.T) { - provider := &streamingWorkflowProvider{} - runner, err := workflowRunnerProvider(provider, false, false) - if err != nil { - t.Fatal(err) - } - events, err := runner.ExecuteStream(context.Background(), ai.Request{}) - if err != nil { - t.Fatal(err) - } - for range events { - } - if provider.streamCalls != 1 || provider.executeCalls != 0 { - t.Fatalf("stream calls = %d, execute calls = %d", provider.streamCalls, provider.executeCalls) - } - }) -} - // The CLI path must execute declared hooks, not skip them: a verify command // that fails has to fail the run. func TestExecuteSyncRun_CLIRunsVerifyHooks(t *testing.T) { diff --git a/pkg/cli/prompt_schema.go b/pkg/cli/prompt_schema.go index d475835f..b346e104 100644 --- a/pkg/cli/prompt_schema.go +++ b/pkg/cli/prompt_schema.go @@ -73,15 +73,26 @@ func WritePromptSchema(ctx context.Context, w io.Writer) error { // request). The discovered prompt sources ride along so the editor's save // destination picker sees every writable directory, including empty ones; the // runtime catalog's sources do the same for the preset/profile editor. +// +// `verifiers` says which workflow.verify.* kinds this host can actually run, and +// `fixtureSchemas` carries the configured fixture runner's own fence schemas +// (omitted when no runner is configured), so the editor can author a fixture +// document knowing whether it will ever run. func PromptSchemaDocument(ctx context.Context) (map[string]any, error) { adapters, err := schemaAdapters() if err != nil { return nil, err } - doc, err := buildPromptSchemaDocument(adapters, loadSavedConfig().Sandbox) + saved := loadSavedConfig() + doc, err := buildPromptSchemaDocument(adapters, saved.Sandbox) if err != nil { return nil, err } + verifiers, fixtureSchemas := verifierCatalog(ctx, enabledAdapters(adapters), saved.Verify.FixtureRunner) + doc["verifiers"] = verifiers + if len(fixtureSchemas) > 0 { + doc["fixtureSchemas"] = fixtureSchemas + } sources, err := promptSourceInfos(ctx) if err != nil { return nil, err diff --git a/pkg/cli/prompt_schema_verifiers.go b/pkg/cli/prompt_schema_verifiers.go new file mode 100644 index 00000000..eb135ace --- /dev/null +++ b/pkg/cli/prompt_schema_verifiers.go @@ -0,0 +1,176 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "strings" + "sync" + "time" + + "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/api" +) + +// VerifierAvailability is one verify kind and whether this host can actually run +// it. The webapp authors `workflow.verify.*` against it: a kind that cannot run +// here must be shown as such before a run declares it, because HooksFor refuses +// a declared check with no factory rather than passing vacuously. +type VerifierAvailability struct { + Kind string `json:"kind"` + Available bool `json:"available"` + // Reason is empty when the kind is available and nothing went wrong. On an + // unavailable kind it is the runtime's own refusal text; on an available + // fixture kind it can still carry a schema-probe failure, which is advisory. + Reason string `json:"reason"` +} + +// fixtureSchemaProbeTimeout bounds one ` --schema` call. The schema +// document is served per request, so a runner that hangs must not hang the +// editor; ten seconds is far more than printing a reflected schema takes. +const fixtureSchemaProbeTimeout = 10 * time.Second + +// verifierCatalog answers, for every kind in the verify registry, whether this +// host can run it, and — when a fixture runner is configured — that runner's own +// fence schemas so the editor can complete a fixture document. +// +// It never fails: a broken fixture runner is reported on the fixture entry, not +// raised, because the rest of the schema document is still correct and the +// editor still needs it. +func verifierCatalog(ctx context.Context, adapters []AdapterStatus, runner []string) ([]VerifierAvailability, json.RawMessage) { + fixture, schemas := fixtureVerifierAvailability(ctx, runner) + return []VerifierAvailability{ + // cmd is captain's own factory, registered in the verify package's init: + // there is no host state that can take it away. + {Kind: string(verify.KindCmd), Available: true}, + promptVerifierAvailability(adapters), + fixture, + }, schemas +} + +// promptVerifierAvailability reads the judge's precondition off the adapters the +// document already probed: promptFactory needs a provider, and a provider needs +// a runtime that is authenticated and installed. With nothing probed there is no +// evidence against it, so the kind is offered. +func promptVerifierAvailability(adapters []AdapterStatus) VerifierAvailability { + entry := VerifierAvailability{Kind: string(verify.KindPrompt), Available: true} + if len(adapters) == 0 { + return entry + } + for _, adapter := range adapters { + if adapter.Ready() { + return entry + } + } + entry.Available = false + entry.Reason = "verify prompts declared but no provider available to judge them: " + + "no probed runtime is authenticated (run 'captain whoami')" + return entry +} + +// fixtureVerifierAvailability reports whether a declared fixture would dispatch, +// and fetches the configured runner's schemas when there is one to ask. +// +// The two are independent: the kind is claimed by an in-process registration or +// by the configured argv, while the schemas are advisory editor metadata. A +// runner that cannot print its schemas still runs fixtures. +func fixtureVerifierAvailability(ctx context.Context, runner []string) (VerifierAvailability, json.RawMessage) { + entry := VerifierAvailability{Kind: string(verify.KindFixture)} + entry.Available = verify.Registered(verify.KindFixture) || len(runner) > 0 + if !entry.Available { + entry.Reason = fixtureUnavailableReason(ctx) + return entry, nil + } + if len(runner) == 0 { + return entry, nil + } + schemas, err := fixtureSchemas(runner) + if err != nil { + entry.Reason = err.Error() + return entry, nil + } + return entry, schemas +} + +// fixtureUnavailableReason is HooksFor's own refusal, asked rather than +// re-worded: the editor shows the exact sentence a run would fail with. The +// probe workflow declares only a fixture, so dispatch stops at the registry +// check and nothing is executed. +func fixtureUnavailableReason(ctx context.Context) string { + _, err := verify.HooksFor(ctx, + &api.Workflow{Verify: &api.Verify{Fixture: "# availability probe\n"}}, verify.Options{}) + if err != nil { + return err.Error() + } + return "" +} + +// fixtureSchemaEntry memoizes one runner argv's schema probe — result or +// failure — so a serve process answers the document endpoint without spawning a +// process per request. +type fixtureSchemaEntry struct { + once sync.Once + schemas json.RawMessage + err error +} + +var fixtureSchemaCache sync.Map // argv key -> *fixtureSchemaEntry + +func fixtureSchemas(runner []string) (json.RawMessage, error) { + value, _ := fixtureSchemaCache.LoadOrStore(strings.Join(runner, "\x00"), &fixtureSchemaEntry{}) + entry := value.(*fixtureSchemaEntry) + entry.once.Do(func() { entry.schemas, entry.err = probeFixtureSchemas(runner) }) + return entry.schemas, entry.err +} + +// ResetFixtureSchemaCache drops every memoized schema probe. It exists for specs +// that swap the configured runner within one process; a serve process keeps the +// cache for its lifetime, since the runner argv comes from a config file read at +// startup. +func ResetFixtureSchemaCache() { + fixtureSchemaCache.Range(func(key, _ any) bool { + fixtureSchemaCache.Delete(key) + return true + }) +} + +// probeFixtureSchemas runs ` --schema` and returns its stdout +// verbatim once it is confirmed to be JSON. The context is this call's own, not +// the request's: the result is cached process-wide, so one cancelled HTTP +// request must not poison every later document with a context error. +func probeFixtureSchemas(runner []string) (json.RawMessage, error) { + ctx, cancel := context.WithTimeout(context.Background(), fixtureSchemaProbeTimeout) + defer cancel() + + args := append(append([]string{}, runner[1:]...), "--schema") + cmd := exec.CommandContext(ctx, runner[0], args...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + label := strings.Join(append(append([]string{}, runner...), "--schema"), " ") + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("%s: %w%s", label, err, stderrTail(stderr.String())) + } + schemas := bytes.TrimSpace(stdout.Bytes()) + var probe any + if err := json.Unmarshal(schemas, &probe); err != nil { + return nil, fmt.Errorf("%s: printed no JSON schema document: %w", label, err) + } + return json.RawMessage(schemas), nil +} + +// stderrTail is the last few lines of a failed runner's stderr — enough to name +// the failure without pasting a whole run into the schema document. +func stderrTail(text string) string { + lines := strings.Split(strings.TrimRight(text, "\n"), "\n") + if len(lines) > 5 { + lines = lines[len(lines)-5:] + } + tail := strings.TrimSpace(strings.Join(lines, "\n")) + if tail == "" { + return "" + } + return ": " + tail +} diff --git a/pkg/cli/prompt_schema_verifiers_ginkgo_test.go b/pkg/cli/prompt_schema_verifiers_ginkgo_test.go new file mode 100644 index 00000000..b64351fb --- /dev/null +++ b/pkg/cli/prompt_schema_verifiers_ginkgo_test.go @@ -0,0 +1,251 @@ +package cli + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" +) + +// hooksForUnregisteredFixtureText is HooksFor's exact refusal, copied rather +// than derived: the webapp renders this sentence, so the document must keep +// saying it even if the availability probe stops asking HooksFor. +const hooksForUnregisteredFixtureText = "workflow.verify.fixture declared but no fixture verifier is registered " + + "(link a fixture runner in-process or set verify.fixtureRunner in ~/.captain.yaml)" + +// fakeFixtureRunner writes an executable that appends one byte to a counter file +// and then prints stdout/stderr and exits with code — enough to prove how many +// times the schema probe actually spawned it. +type fakeFixtureRunner struct { + path string + counter string +} + +func newFakeFixtureRunner(stdout, stderr string, code int) fakeFixtureRunner { + GinkgoHelper() + dir := GinkgoT().TempDir() + runner := fakeFixtureRunner{ + path: filepath.Join(dir, "fake-gavel"), + counter: filepath.Join(dir, "runs"), + } + script := "#!/bin/sh\n" + + "printf x >> " + shellQuote(runner.counter) + "\n" + + "printf %s " + shellQuote(stdout) + "\n" + + "printf %s " + shellQuote(stderr) + " 1>&2\n" + + "exit " + strconv.Itoa(code) + "\n" + Expect(os.WriteFile(runner.path, []byte(script), 0o755)).To(Succeed()) + return runner +} + +// argv is the configured runner argv: the script plus the sub-command a real +// host configures, so the probe is proven to append --schema after both. +func (r fakeFixtureRunner) argv() []string { return []string{r.path, "fixtures", "--stdin"} } + +func (r fakeFixtureRunner) runs() int { + data, err := os.ReadFile(r.counter) + if os.IsNotExist(err) { + return 0 + } + Expect(err).NotTo(HaveOccurred()) + return len(data) +} + +func verifierEntry(entries []VerifierAvailability, kind verify.Kind) VerifierAvailability { + GinkgoHelper() + for _, entry := range entries { + if entry.Kind == string(kind) { + return entry + } + } + Fail("no verifier entry for kind " + string(kind)) + return VerifierAvailability{} +} + +// readyAdapter is a probed runtime that can execute a judge prompt. +func readyAdapter() []AdapterStatus { + return []AdapterStatus{{ + Type: "api", Provider: api.Anthropic.Name, Mode: string(api.ModeAPI), Authenticated: true, + }} +} + +func unauthenticatedAdapter() []AdapterStatus { + return []AdapterStatus{{Type: "api", Provider: api.Anthropic.Name, Mode: string(api.ModeAPI)}} +} + +var _ = Describe("prompt schema verifier availability", func() { + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + ResetFixtureSchemaCache() + }) + + AfterEach(func() { + verify.Unregister(verify.KindFixture) + ResetFixtureSchemaCache() + }) + + It("reports cmd available and prompt available when a probed runtime is ready", func() { + entries, schemas := verifierCatalog(ctx, readyAdapter(), nil) + + Expect(schemas).To(BeNil()) + Expect(verifierEntry(entries, verify.KindCmd)).To(Equal( + VerifierAvailability{Kind: "cmd", Available: true})) + Expect(verifierEntry(entries, verify.KindPrompt).Available).To(BeTrue()) + Expect(verifierEntry(entries, verify.KindPrompt).Reason).To(BeEmpty()) + }) + + It("reports prompt unavailable when no probed runtime can judge", func() { + entries, _ := verifierCatalog(ctx, unauthenticatedAdapter(), nil) + + prompt := verifierEntry(entries, verify.KindPrompt) + Expect(prompt.Available).To(BeFalse()) + Expect(prompt.Reason).To(ContainSubstring("no provider available to judge them")) + }) + + It("reports the fixture kind unavailable with the HooksFor text when nothing is registered", func() { + entries, schemas := verifierCatalog(ctx, readyAdapter(), nil) + + Expect(schemas).To(BeNil()) + Expect(verifierEntry(entries, verify.KindFixture)).To(Equal(VerifierAvailability{ + Kind: "fixture", Available: false, Reason: hooksForUnregisteredFixtureText, + })) + }) + + It("reports an in-process registration available without spawning anything", func() { + runner := newFakeFixtureRunner(`{"schemaVersion":1}`, "", 0) + verify.Register(verify.KindFixture, func( + context.Context, api.Verify, verify.Options, + ) ([]*verify.Plugin, error) { + return nil, nil + }) + + entries, schemas := verifierCatalog(ctx, readyAdapter(), nil) + + Expect(verifierEntry(entries, verify.KindFixture)).To(Equal( + VerifierAvailability{Kind: "fixture", Available: true})) + Expect(schemas).To(BeNil(), "there is no runner to ask for fence schemas") + Expect(runner.runs()).To(Equal(0)) + }) + + It("embeds the configured runner's JSON schema document verbatim", func() { + document := `{"schemaVersion":1,"source":"gavel fixtures --schema","fences":{"test":{"aliases":["yaml test"]}}}` + runner := newFakeFixtureRunner(document+"\n", "", 0) + + entries, schemas := verifierCatalog(ctx, readyAdapter(), runner.argv()) + + Expect(verifierEntry(entries, verify.KindFixture)).To(Equal( + VerifierAvailability{Kind: "fixture", Available: true})) + Expect(string(schemas)).To(Equal(document)) + Expect(runner.runs()).To(Equal(1)) + }) + + It("names the runner and the parse error when the runner prints garbage", func() { + runner := newFakeFixtureRunner("not json at all", "", 0) + + entries, schemas := verifierCatalog(ctx, readyAdapter(), runner.argv()) + + Expect(schemas).To(BeNil()) + fixture := verifierEntry(entries, verify.KindFixture) + Expect(fixture.Available).To(BeTrue(), + "the schema is advisory; a configured runner still claims the kind") + Expect(fixture.Reason).To(ContainSubstring(runner.path)) + Expect(fixture.Reason).To(ContainSubstring("--schema")) + Expect(fixture.Reason).To(ContainSubstring("invalid character")) + }) + + It("names the runner and its stderr tail when the runner fails", func() { + runner := newFakeFixtureRunner("", "unknown flag: --schema", 2) + + entries, schemas := verifierCatalog(ctx, readyAdapter(), runner.argv()) + + Expect(schemas).To(BeNil()) + fixture := verifierEntry(entries, verify.KindFixture) + Expect(fixture.Available).To(BeTrue()) + Expect(fixture.Reason).To(ContainSubstring("exit status 2")) + Expect(fixture.Reason).To(ContainSubstring("unknown flag: --schema")) + }) + + It("spawns the runner once per argv and serves later documents from the cache", func() { + runner := newFakeFixtureRunner(`{"schemaVersion":1}`, "", 0) + + _, first := verifierCatalog(ctx, readyAdapter(), runner.argv()) + _, second := verifierCatalog(ctx, readyAdapter(), runner.argv()) + + Expect(string(second)).To(Equal(string(first))) + Expect(runner.runs()).To(Equal(1)) + + ResetFixtureSchemaCache() + _, third := verifierCatalog(ctx, readyAdapter(), runner.argv()) + Expect(string(third)).To(Equal(string(first))) + Expect(runner.runs()).To(Equal(2), "the reset seam re-runs the probe for the next spec") + }) +}) + +var _ = Describe("prompt schema document verifier fields", func() { + var previousAdapters func() ([]AdapterStatus, error) + + BeforeEach(func() { + ResetFixtureSchemaCache() + previousAdapters = schemaAdapters + schemaAdapters = func() ([]AdapterStatus, error) { + return ai.ProbeAdapters(ai.WhoamiOptions{}, ai.AuthProbe{ + Getenv: func(string) string { return "" }, + LookPath: func(bin string) (string, error) { return "/usr/local/bin/" + bin, nil }, + FileExists: func(string) bool { return false }, + Home: "/home/test", + }) + } + }) + + AfterEach(func() { + schemaAdapters = previousAdapters + verify.Unregister(verify.KindFixture) + captainconfig.SetPathForTesting("") + ResetCaptainConfigCache() + ResetFixtureSchemaCache() + }) + + It("serves verifiers[] and the configured runner's fixtureSchemas", func() { + runner := newFakeFixtureRunner(`{"schemaVersion":1,"fences":{}}`, "", 0) + seedCaptainConfig("verify:\n fixtureRunner: [" + runner.path + ", fixtures]\n") + + doc, err := PromptSchemaDocument(context.Background()) + Expect(err).NotTo(HaveOccurred()) + + entries, ok := doc["verifiers"].([]VerifierAvailability) + Expect(ok).To(BeTrue(), "document carries typed verifier availability") + Expect(entries).To(HaveLen(3)) + Expect(verifierEntry(entries, verify.KindFixture).Available).To(BeTrue()) + + raw, ok := doc["fixtureSchemas"].(json.RawMessage) + Expect(ok).To(BeTrue()) + Expect(string(raw)).To(Equal(`{"schemaVersion":1,"fences":{}}`)) + + encoded, err := json.Marshal(doc) + Expect(err).NotTo(HaveOccurred()) + var decoded map[string]any + Expect(json.Unmarshal(encoded, &decoded)).To(Succeed()) + Expect(decoded["fixtureSchemas"]).To(HaveKeyWithValue("schemaVersion", float64(1))) + }) + + It("omits fixtureSchemas when no runner is configured", func() { + seedCaptainConfig("") + + doc, err := PromptSchemaDocument(context.Background()) + Expect(err).NotTo(HaveOccurred()) + + Expect(doc).NotTo(HaveKey("fixtureSchemas")) + entries := doc["verifiers"].([]VerifierAvailability) + Expect(verifierEntry(entries, verify.KindFixture).Reason).To(Equal(hooksForUnregisteredFixtureText)) + }) +}) diff --git a/pkg/cli/prompt_spec.go b/pkg/cli/prompt_spec.go index adbe51be..d3f59572 100644 --- a/pkg/cli/prompt_spec.go +++ b/pkg/cli/prompt_spec.go @@ -164,10 +164,18 @@ func decodePromptBody(ctx context.Context, flat map[string]any, dst any) error { return nil } -func runtimeTimeout(raw string) time.Duration { - timeout, _ := time.ParseDuration(raw) - if timeout <= 0 { - return 120 * time.Second - } - return timeout +// defaultRunTimeout is the CLI's own deadline for a run whose spec declares no +// budget.timeout. It belongs at the call sites that need a bound — the value +// pkg/cli hands promptrun.Input.Timeout — not inside the parser, where it would +// stand in for a value the author got wrong. +const defaultRunTimeout = 120 * time.Second + +// runtimeTimeout resolves a declared budget.timeout string through the same +// parser api.Spec validation uses. Zero means no bound was declared and the +// caller's own default applies; an unparseable or non-positive value is an +// error naming the field and the raw value. Substituting a default here made +// `budget.timeout: "2 minutes"` run to a deadline nobody asked for, reported as +// if the declared ceiling had been honoured. +func runtimeTimeout(raw string) (time.Duration, error) { + return api.Budget{Timeout: raw}.ParseTimeout() } diff --git a/pkg/cli/prompt_spec_test.go b/pkg/cli/prompt_spec_test.go new file mode 100644 index 00000000..29670591 --- /dev/null +++ b/pkg/cli/prompt_spec_test.go @@ -0,0 +1,93 @@ +package cli + +import ( + "strings" + "testing" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" +) + +// TestPromptSpecRuntimeTimeout pins the parse of a declared budget.timeout. +// runtimeTimeout used to discard time.ParseDuration's error and hand back 120s, +// so `budget.timeout: "2 minutes"` — a plausible typo — ran to a two-minute +// deadline the author never asked for and never learned about. A declared +// ceiling that quietly does nothing reads as enforced. +func TestPromptSpecRuntimeTimeout(t *testing.T) { + tests := []struct { + name string + raw string + want time.Duration + wantErr string + }{ + {name: "a valid duration parses", raw: "90s", want: 90 * time.Second}, + {name: "hours parse", raw: "2h", want: 2 * time.Hour}, + { + name: "a humanized duration is a loud error", + raw: "2 minutes", + wantErr: `invalid budget timeout "2 minutes"`, + }, + { + name: "a non-positive duration is a loud error", + raw: "0s", + wantErr: `invalid budget timeout "0s" (must be > 0)`, + }, + { + name: "a negative duration is a loud error", + raw: "-5s", + wantErr: `invalid budget timeout "-5s" (must be > 0)`, + }, + {name: "an absent timeout declares no bound", raw: "", want: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := runtimeTimeout(tt.raw) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("runtimeTimeout(%q) = %s, want error %q", tt.raw, got, tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("runtimeTimeout(%q) error = %q, want it to name the field and the raw value (%q)", tt.raw, err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("runtimeTimeout(%q) err = %v", tt.raw, err) + } + if got != tt.want { + t.Fatalf("runtimeTimeout(%q) = %s, want %s", tt.raw, got, tt.want) + } + }) + } +} + +// TestPromptSpecRenderedTimeoutDefault pins where the CLI's own 120s lives: at +// the call site that needs a deadline, not inside the parser. A spec that +// declares none gets it; a spec that declares one gets exactly that. +func TestPromptSpecRenderedTimeoutDefault(t *testing.T) { + absent, err := renderedTimeout(PromptRenderResult{}) + if err != nil { + t.Fatalf("renderedTimeout(absent) err = %v", err) + } + if absent != defaultRunTimeout { + t.Fatalf("renderedTimeout(absent) = %s, want the CLI default %s", absent, defaultRunTimeout) + } + + declared, err := renderedTimeout(PromptRenderResult{ + Input: ai.Request{Budget: api.Budget{Timeout: "90s"}}, + }) + if err != nil { + t.Fatalf("renderedTimeout(declared) err = %v", err) + } + if declared != 90*time.Second { + t.Fatalf("renderedTimeout(declared) = %s, want 90s", declared) + } + + if _, err := renderedTimeout(PromptRenderResult{ + Input: ai.Request{Budget: api.Budget{Timeout: "2 minutes"}}, + }); err == nil { + t.Fatal("renderedTimeout accepted an unparseable budget.timeout; the run would silently use a default deadline") + } +} diff --git a/pkg/cli/prompt_workflow.go b/pkg/cli/prompt_workflow.go deleted file mode 100644 index 92b69b9e..00000000 --- a/pkg/cli/prompt_workflow.go +++ /dev/null @@ -1,32 +0,0 @@ -package cli - -import ( - "github.com/flanksource/captain/pkg/ai/agent" - "github.com/flanksource/captain/pkg/ai/agent/verify" -) - -// verifyPassed reports whether the last verify verdict passed. With no verify -// hooks, the run passed. The last-verdict read is sound because the runner -// stops each verification round at its first failure (see agent.verifyPassed), -// so the list's final entry is always the final round's outcome. -func verifyPassed(verdicts []agent.VerifyResult) bool { - if len(verdicts) == 0 { - return true - } - return verdicts[len(verdicts)-1].Valid -} - -// verifyReason surfaces the last failing verifier's reason in a run summary. -func verifyReason(verdicts []agent.VerifyResult) string { - if len(verdicts) == 0 { - return "" - } - last := verdicts[len(verdicts)-1] - if last.Valid { - return "" - } - if vd, ok := last.Output.(verify.Verdict); ok && vd.Reason != "" { - return vd.Reason - } - return "verification failed" -} diff --git a/pkg/cli/provider_defaults.go b/pkg/cli/provider_defaults.go index f6b9b798..f6bf41aa 100644 --- a/pkg/cli/provider_defaults.go +++ b/pkg/cli/provider_defaults.go @@ -48,6 +48,40 @@ func effectiveProviderDefaults(saved captainconfig.AIDefaults, provider *api.Mod return view, nil } +// savedProviderDefaults is effectiveProviderDefaults' run-path sibling: the same +// opt-out degradation, but over what the user actually configured rather than +// over registry-seeded values, and it never invents a model for an empty slot. +// +// The distinction is the point. effectiveProviderDefaults fills gaps from +// Provider.DefaultMode and the DefaultModelFor table, which is right for seeding +// `captain configure` and wrong for deciding what a run executes on — an unset +// field must survive as unset so ResolveForRun can refuse it. +func savedProviderDefaults(saved captainconfig.AIDefaults, provider *api.ModelProvider) (ProviderDefaultView, error) { + view, err := aiflags.SavedDefaults(saved, provider) + if err != nil { + return ProviderDefaultView{}, err + } + disabled := ai.Disabled() + mode := api.RuntimeMode(strings.TrimSpace(view.Mode)) + if mode != "" && disabled.Runtime(provider, mode) { + mode = firstEnabledMode(provider, mode) + } + model := strings.TrimSpace(view.Model) + if model != "" && disabled.Model(provider, mode, model) { + model = firstEnabledModel(provider, mode) + } + effort := api.Effort(strings.TrimSpace(view.Effort)) + if effort != api.EffortNone && disabled.Effort(effort) { + degraded, err := ai.ResolveModelEffort(provider, mode, model, effort) + if err != nil { + return ProviderDefaultView{}, err + } + effort = degraded + } + view.Mode, view.Model, view.Effort = string(mode), model, string(effort) + return view, nil +} + // firstEnabledMode replaces a disabled mode with another of the same provider // that is still enabled. When every one is off it returns the original so the // view still names what the user configured — the whoami disable card, not this @@ -114,7 +148,10 @@ func applyCandidateDefaults(model api.Model, saved captainconfig.AIDefaults, all if provider == nil { return api.Model{}, fmt.Errorf("provider cannot be resolved for model %q", model.Name) } - defaults, err := effectiveProviderDefaults(saved, provider) + // Saved, not effective: this is the run path. Seeding from the registry's + // built-in tables here is how `captain ai prompt` kept silently defaulting + // after the flag path stopped. + defaults, err := savedProviderDefaults(saved, provider) if err != nil { return api.Model{}, err } diff --git a/pkg/cli/serve_disabled.go b/pkg/cli/serve_disabled.go index 46939670..bb27f5eb 100644 --- a/pkg/cli/serve_disabled.go +++ b/pkg/cli/serve_disabled.go @@ -31,7 +31,7 @@ func registerDisabledHandlers(mux *http.ServeMux) { // CLI invocation honours the same disables the whoami page writes. Loading the // config is deliberately side-effect free, which is why this is explicit. func InstallDisabledSelections() error { - config, _, err := captainconfig.Load() + config, _, err := LoadCaptainConfigOnce() if err != nil { return err } diff --git a/pkg/cli/verify.go b/pkg/cli/verify.go new file mode 100644 index 00000000..430cdce8 --- /dev/null +++ b/pkg/cli/verify.go @@ -0,0 +1,246 @@ +package cli + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/ai/middleware" + "github.com/flanksource/captain/pkg/api" + clickyapi "github.com/flanksource/clicky/api" +) + +// VerifyOptions runs a workflow's verification stage on its own: the same +// checks the generate→verify loop votes with, against a tree that is already +// written. It is how a definition of done is exercised without spending a +// generation on it — locally before a push, or by a host wiring captain's +// checks into its own pipeline. +// +// There is deliberately no --json flag: clicky already binds --json/--format +// globally, and the result renders through that pipeline. +type VerifyOptions struct { + Fixture string `flag:"fixture" help:"Fixture document run by the configured verify.fixtureRunner"` + Commands []string `flag:"command" help:"Shell command run as a pass/fail check (repeatable)" short:"c"` + Prompts []string `flag:"prompt" help:"LLM-judge .prompt template, judged by the run's provider (repeatable)"` + Cwd string `flag:"cwd" help:"Directory the checks run in" default:"."` + Timeout string `flag:"timeout" help:"Wall-clock bound per check" default:"10m"` + + AIProviderOptions +} + +// VerifyResult is what one `captain verify` run reports: the verdict, and every +// check's typed report exactly as a Verify hook produces it inside a run. +type VerifyResult struct { + Passed bool `json:"passed" pretty:"label=Passed"` + Reports []api.VerifyReport `json:"reports"` + Summary api.VerifySummary `json:"summary"` +} + +// verifyRejected carries the reports through the failure path. clicky renders +// an error that implements a rendering interface through the same format +// pipeline as a success and still exits non-zero — so a failing verification +// prints its reports rather than a bare sentence. +type verifyRejected struct{ VerifyResult } + +func (e verifyRejected) Error() string { + failed := 0 + for _, report := range e.Reports { + if !report.Passed { + failed++ + } + } + return fmt.Sprintf("verification failed: %d of %d checks did not pass", failed, len(e.Reports)) +} + +func RunVerify(ctx context.Context, opts VerifyOptions) (any, error) { + cwd, err := filepath.Abs(firstNonEmpty(strings.TrimSpace(opts.Cwd), ".")) + if err != nil { + return nil, err + } + wf, err := opts.workflow() + if err != nil { + return nil, err + } + timeout, err := verifyTimeout(opts.Timeout) + if err != nil { + return nil, err + } + provider, model, err := opts.judgeProvider() + if err != nil { + return nil, err + } + if provider != nil { + defer closeProvider(provider) + } + + // The spec this verification runs under, so a verifier that grades with an + // agent of its own inherits the model resolved here instead of choosing one. + spec := &api.Spec{Model: model, Workflow: wf} + hooks, err := verify.HooksFor(ctx, wf, verify.Options{Provider: provider, Timeout: timeout, RunSpec: spec}) + if err != nil { + return nil, err + } + if len(hooks) == 0 { + return nil, fmt.Errorf("nothing to verify: pass --command, --prompt or --fixture") + } + + result := VerifyResult{Passed: true} + for _, hook := range hooks { + plugin, ok := hook.(*verify.Plugin) + if !ok { + return nil, fmt.Errorf("unexpected hook type %T", hook) + } + report, err := runVerifyPlugin(ctx, plugin, cwd) + if err != nil { + // A check that could not reach a verdict is not a failing check: the + // run has no answer, and saying "failed" would invent one. + return nil, fmt.Errorf("%s: %w", plugin.Name(), err) + } + result.Reports = append(result.Reports, report) + result.Passed = result.Passed && report.Passed + result.Summary = api.AddSummaries(result.Summary, report.Summary) + } + if !result.Passed { + return result, verifyRejected{result} + } + return result, nil +} + +// runVerifyPlugin drives one check out of loop — the same path the git-agent +// receive side uses — and completes its report with the hook's name. +func runVerifyPlugin(ctx context.Context, plugin *verify.Plugin, cwd string) (api.VerifyReport, error) { + started := time.Now() + vd, err := plugin.Verifier().Verify(ctx, cwd, nil) + if err != nil { + return api.VerifyReport{}, err + } + if vd.Report == nil { + // Every verifier captain ships reports; a host-supplied one need not. + synthesised := api.NewNodeReport(api.VerifyKindFunc, plugin.Name(), api.VerifyNode{ + Name: plugin.Name(), Passed: vd.OK, Failed: !vd.OK, Message: vd.Reason, + Duration: time.Since(started), + }) + vd.Report = &synthesised + } + report := *vd.Report + if report.Name == "" { + report.Name = plugin.Name() + } + return report, nil +} + +// workflow is the api.Workflow the flags describe: the same declaration a spec +// carries, so the CLI and a run verify through one code path. +func (o VerifyOptions) workflow() (*api.Workflow, error) { + spec := &api.Verify{Commands: o.Commands, Prompts: o.Prompts} + if path := strings.TrimSpace(o.Fixture); path != "" { + document, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read fixture %s: %w", path, err) + } + spec.Fixture = string(document) + } + wf := &api.Workflow{Verify: spec} + if err := wf.Validate(); err != nil { + return nil, err + } + return wf, nil +} + +// judgeProvider builds the provider the LLM-judge prompts execute on, and only +// then: a verification made of commands and fixtures needs no model, and +// constructing one would demand credentials the run does not use. It returns the +// resolved model alongside, because that is what the run's spec declares — a +// zero model when nothing here needs one. +func (o VerifyOptions) judgeProvider() (ai.Provider, api.Model, error) { + if len(o.Prompts) == 0 { + return nil, api.Model{}, nil + } + cfg, err := o.ToConfig() + if err != nil { + return nil, api.Model{}, err + } + if cfg.Model.Name == "" { + return nil, api.Model{}, fmt.Errorf("--prompt needs a model to judge with: pass --model or run 'captain configure'") + } + provider, err := ai.NewProvider(cfg) + if err != nil { + return nil, api.Model{}, err + } + wrapped, err := middleware.Wrap(provider, middleware.WithLogging()) + if err != nil { + return nil, api.Model{}, err + } + return wrapped, cfg.Model, nil +} + +func verifyTimeout(value string) (time.Duration, error) { + value = strings.TrimSpace(value) + if value == "" { + return 0, nil + } + timeout, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("invalid --timeout %q: %w", value, err) + } + return timeout, nil +} + +// Pretty renders one line per check — the verdict first, then what it was — +// with a failing check's feedback beneath it, since that is the whole reason +// the command was run. +func (r VerifyResult) Pretty() clickyapi.Text { + text := clickyapi.Text{} + for i, report := range r.Reports { + if i > 0 { + text = text.Append("\n") + } + icon, style := "✓", "text-green-500 font-medium" + if !report.Passed { + icon, style = "✗", "text-red-500 font-medium" + } + text = text.Append(icon+" ", style).Append(report.Name, "text-muted") + if reason := strings.TrimSpace(report.Reason); reason != "" { + text = text.Append(" — " + reason) + } + if feedback := strings.TrimSpace(report.Feedback); feedback != "" && !report.Passed { + text = text.Append("\n" + feedback) + } + } + return text +} + +// InstallFixtureVerifier registers the configured external fixture runner as +// the `fixture` verifier, so a workflow that declares a fixture is dispatched +// rather than silently contributing no hooks. Called once at CLI startup; a +// host that linked its own fixture runner in-process keeps it. +func InstallFixtureVerifier() error { + cfg, _, err := LoadCaptainConfigOnce() + if err != nil { + return err + } + if len(cfg.Verify.FixtureRunner) == 0 || verify.Registered(verify.KindFixture) { + return nil + } + verify.Register(verify.KindFixture, externalFixtureFactory(cfg.Verify.FixtureRunner)) + return nil +} + +// externalFixtureFactory builds the one hook a declared fixture contributes: +// the configured runner, handed the fixture document and the run's bounds. +func externalFixtureFactory(command []string) verify.Factory { + return func(_ context.Context, spec api.Verify, opts verify.Options) ([]*verify.Plugin, error) { + if strings.TrimSpace(spec.Fixture) == "" { + return nil, nil + } + return []*verify.Plugin{verify.New("fixture", &verify.ExternalVerifier{ + Command: command, Fixture: spec.Fixture, + Timeout: opts.Timeout, Env: opts.Env, Wrap: opts.Wrap, + })}, nil + } +} diff --git a/pkg/cli/verify_ginkgo_test.go b/pkg/cli/verify_ginkgo_test.go new file mode 100644 index 00000000..3dcb57cb --- /dev/null +++ b/pkg/cli/verify_ginkgo_test.go @@ -0,0 +1,107 @@ +package cli + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/api" + clickyapi "github.com/flanksource/clicky/api" +) + +var _ = Describe("captain verify", func() { + ctx := context.Background() + + It("reports a passing command as a passed report", func() { + result, err := RunVerify(ctx, VerifyOptions{Commands: []string{"true"}, Cwd: GinkgoT().TempDir()}) + Expect(err).NotTo(HaveOccurred()) + + verdict, ok := result.(VerifyResult) + Expect(ok).To(BeTrue()) + Expect(verdict.Passed).To(BeTrue()) + Expect(verdict.Reports).To(HaveLen(1)) + Expect(verdict.Reports[0].Validate()).To(Succeed()) + Expect(verdict.Reports[0].Kind).To(Equal(api.VerifyKindCmd)) + Expect(verdict.Summary).To(Equal(api.VerifySummary{Total: 1, Passed: 1})) + + raw, err := json.Marshal(result) + Expect(err).NotTo(HaveOccurred()) + Expect(string(raw)).To(ContainSubstring(`"passed":true`)) + Expect(string(raw)).To(ContainSubstring(`"reports"`)) + }) + + It("fails the command when a check fails, and still carries its reports", func() { + result, err := RunVerify(ctx, VerifyOptions{Commands: []string{"echo boom-detail; exit 1"}, Cwd: GinkgoT().TempDir()}) + Expect(err).To(HaveOccurred(), "a failing check must exit non-zero") + Expect(err.Error()).To(ContainSubstring("1 of 1 checks did not pass")) + + verdict, ok := result.(VerifyResult) + Expect(ok).To(BeTrue()) + Expect(verdict.Passed).To(BeFalse()) + Expect(verdict.Reports[0].Feedback).To(ContainSubstring("boom-detail")) + + Expect(clickyapi.TryTypedValue(err)).NotTo(BeNil(), + "the error renders through the format pipeline, so a failure prints its reports") + }) + + It("counts a check that ran out of wall clock in the timed-out bucket", func() { + result, err := RunVerify(ctx, VerifyOptions{ + Commands: []string{"sleep 5"}, Timeout: "200ms", Cwd: GinkgoT().TempDir(), + }) + Expect(err).To(HaveOccurred(), "a check that never finished has not passed") + + verdict, ok := result.(VerifyResult) + Expect(ok).To(BeTrue()) + Expect(verdict.Reports).To(HaveLen(1)) + Expect(verdict.Reports[0].State).To(Equal(api.VerifyStateTimedOut)) + // The run summary is rolled up by api.AddSummaries, so every bucket the + // wire shape carries survives — a re-listing of the fields here is what + // dropped `timedout` on the floor and reported the run as 0 of 0. + Expect(verdict.Summary).To(Equal(api.VerifySummary{Total: 1, TimedOut: 1})) + + raw, err := json.Marshal(result) + Expect(err).NotTo(HaveOccurred()) + Expect(string(raw)).To(ContainSubstring(`"timedout":1`)) + }) + + It("refuses a declared fixture with no fixture runner configured", func() { + fixture := filepath.Join(GinkgoT().TempDir(), "acceptance.md") + Expect(os.WriteFile(fixture, []byte("# acceptance\n"), 0o644)).To(Succeed()) + + _, err := RunVerify(ctx, VerifyOptions{Fixture: fixture, Cwd: GinkgoT().TempDir()}) + Expect(err).To(MatchError(ContainSubstring("no fixture verifier is registered"))) + }) + + // A fixture runner that grades with its own agent inherits the run's model + // and permissions from the spec `captain verify` resolved, rather than + // inventing a posture of its own. + It("hands the spec it built to the verifier factory", func() { + fixture := filepath.Join(GinkgoT().TempDir(), "acceptance.md") + Expect(os.WriteFile(fixture, []byte("# acceptance\n"), 0o644)).To(Succeed()) + + var captured *api.Spec + verify.Register(verify.KindFixture, func(_ context.Context, _ api.Verify, opts verify.Options) ([]*verify.Plugin, error) { + captured = opts.RunSpec + return []*verify.Plugin{verify.New("fixture", verify.FuncVerifier( + func(context.Context, string, []string) (verify.Verdict, error) { + return verify.Verdict{OK: true}, nil + }))}, nil + }) + DeferCleanup(func() { verify.Unregister(verify.KindFixture) }) + + _, err := RunVerify(ctx, VerifyOptions{Fixture: fixture, Cwd: GinkgoT().TempDir()}) + Expect(err).NotTo(HaveOccurred()) + Expect(captured).NotTo(BeNil()) + Expect(captured.Workflow.Verify.Fixture).To(Equal("# acceptance\n")) + }) + + It("refuses a run with nothing declared rather than passing vacuously", func() { + _, err := RunVerify(ctx, VerifyOptions{Cwd: GinkgoT().TempDir()}) + Expect(err).To(MatchError(ContainSubstring("nothing to verify"))) + }) +}) diff --git a/pkg/cli/verify_install_ginkgo_test.go b/pkg/cli/verify_install_ginkgo_test.go new file mode 100644 index 00000000..bce93d96 --- /dev/null +++ b/pkg/cli/verify_install_ginkgo_test.go @@ -0,0 +1,175 @@ +package cli + +import ( + "context" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/captainconfig" +) + +// seedCaptainConfig points ~/.captain.yaml at a temp file with the given body, +// or at a path that does not exist when body is empty. +func seedCaptainConfig(body string) { + GinkgoHelper() + path := filepath.Join(GinkgoT().TempDir(), ".captain.yaml") + if body != "" { + Expect(os.WriteFile(path, []byte(body), 0o644)).To(Succeed()) + } + captainconfig.SetPathForTesting(path) +} + +// fixtureHook builds the one hook a declared fixture contributes, through the +// same registry dispatch a run uses. +func fixtureHook(opts verify.Options) *verify.Plugin { + GinkgoHelper() + hooks, err := verify.HooksFor(context.Background(), + &api.Workflow{Verify: &api.Verify{Fixture: "# acceptance\n"}}, opts) + Expect(err).NotTo(HaveOccurred()) + Expect(hooks).To(HaveLen(1)) + plugin, ok := hooks[0].(*verify.Plugin) + Expect(ok).To(BeTrue()) + return plugin +} + +// The verifier registry is process-global, so every spec here takes its +// registration back out: a leaked `fixture` factory would make the sibling spec +// asserting "no fixture verifier is registered" pass or fail on ordering. +var _ = Describe("installing the configured fixture runner", func() { + AfterEach(func() { + verify.Unregister(verify.KindFixture) + captainconfig.SetPathForTesting("") + ResetCaptainConfigCache() + }) + + It("claims the fixture kind with the configured runner's argv", func() { + seedCaptainConfig("verify:\n fixtureRunner: [gavel, fixtures, --stdin]\n") + + Expect(InstallFixtureVerifier()).To(Succeed()) + Expect(verify.Registered(verify.KindFixture)).To(BeTrue()) + + external, ok := fixtureHook(verify.Options{}).Verifier().(*verify.ExternalVerifier) + Expect(ok).To(BeTrue(), "a configured runner is dispatched as an external process") + Expect(external.Command).To(Equal([]string{"gavel", "fixtures", "--stdin"})) + Expect(external.Fixture).To(Equal("# acceptance\n")) + }) + + It("forwards the run's environment, confinement wrapper and timeout to the runner", func() { + seedCaptainConfig("verify:\n fixtureRunner: [gavel]\n") + Expect(InstallFixtureVerifier()).To(Succeed()) + + wrap := func(_ context.Context, cmd string, args, env []string) (string, []string, []string, error) { + return cmd, args, env, nil + } + external, ok := fixtureHook(verify.Options{ + Env: []string{"PATH=/usr/bin"}, Wrap: wrap, Timeout: 42 * time.Second, + }).Verifier().(*verify.ExternalVerifier) + Expect(ok).To(BeTrue()) + Expect(external.Env).To(Equal([]string{"PATH=/usr/bin"})) + Expect(external.Timeout).To(Equal(42 * time.Second)) + Expect(external.Wrap).NotTo(BeNil(), + "an external runner is agent-adjacent input; it must never escape the run's confinement") + }) + + It("contributes no hook when the workflow declares no fixture", func() { + seedCaptainConfig("verify:\n fixtureRunner: [gavel]\n") + Expect(InstallFixtureVerifier()).To(Succeed()) + + hooks, err := verify.HooksFor(context.Background(), + &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}}}, verify.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(hooks).To(HaveLen(1)) + Expect(hooks[0].(*verify.Plugin).Name()).To(Equal("verify:true")) + }) + + It("is a no-op on a host with no config file at all", func() { + seedCaptainConfig("") + + Expect(InstallFixtureVerifier()).To(Succeed()) + Expect(verify.Registered(verify.KindFixture)).To(BeFalse(), + "no configured runner means the host runs no fixtures, which HooksFor reports as an error") + }) + + It("fails loudly on a malformed config rather than running no fixtures", func() { + seedCaptainConfig("verify:\n fixtureRunner: 42\n") + + Expect(InstallFixtureVerifier()).To(MatchError(ContainSubstring(".captain.yaml"))) + Expect(verify.Registered(verify.KindFixture)).To(BeFalse()) + }) + + It("leaves a fixture verifier the host already linked in place", func() { + seedCaptainConfig("verify:\n fixtureRunner: [gavel]\n") + verify.Register(verify.KindFixture, func( + _ context.Context, _ api.Verify, _ verify.Options, + ) ([]*verify.Plugin, error) { + return []*verify.Plugin{verify.New("linked-fixture", verify.FuncVerifier( + func(context.Context, string, []string) (verify.Verdict, error) { + return verify.Verdict{OK: true}, nil + }))}, nil + }) + + Expect(InstallFixtureVerifier()).To(Succeed()) + Expect(fixtureHook(verify.Options{}).Name()).To(Equal("linked-fixture"), + "an in-process runner outranks the configured external one") + }) +}) + +var _ = Describe("loading ~/.captain.yaml once", func() { + AfterEach(func() { + captainconfig.SetPathForTesting("") + ResetCaptainConfigCache() + }) + + It("parses the file once and serves every later caller from the cache", func() { + path := filepath.Join(GinkgoT().TempDir(), ".captain.yaml") + Expect(os.WriteFile(path, []byte("verify:\n fixtureRunner: [gavel]\n"), 0o644)).To(Succeed()) + captainconfig.SetPathForTesting(path) + + first, exists, err := LoadCaptainConfigOnce() + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + Expect(first.Verify.FixtureRunner).To(Equal([]string{"gavel"})) + + // Rewriting the file behind the cache is how the second call proves it + // never re-read: the root command installs several things out of one + // parse, and they must all see the same file. + Expect(os.WriteFile(path, []byte("verify:\n fixtureRunner: [other]\n"), 0o644)).To(Succeed()) + second, _, err := LoadCaptainConfigOnce() + Expect(err).NotTo(HaveOccurred()) + Expect(second.Verify.FixtureRunner).To(Equal([]string{"gavel"})) + + ResetCaptainConfigCache() + reloaded, _, err := LoadCaptainConfigOnce() + Expect(err).NotTo(HaveOccurred()) + Expect(reloaded.Verify.FixtureRunner).To(Equal([]string{"other"})) + }) + + It("re-reads when the config path itself changes", func() { + seedCaptainConfig("verify:\n fixtureRunner: [first]\n") + first, _, err := LoadCaptainConfigOnce() + Expect(err).NotTo(HaveOccurred()) + Expect(first.Verify.FixtureRunner).To(Equal([]string{"first"})) + + seedCaptainConfig("verify:\n fixtureRunner: [second]\n") + second, _, err := LoadCaptainConfigOnce() + Expect(err).NotTo(HaveOccurred()) + Expect(second.Verify.FixtureRunner).To(Equal([]string{"second"}), + "the cache is keyed on the path, so a redirected config is never stale") + }) + + It("replays a malformed file's error to every caller", func() { + seedCaptainConfig("verify:\n fixtureRunner: 42\n") + + _, _, first := LoadCaptainConfigOnce() + _, _, second := LoadCaptainConfigOnce() + Expect(first).To(HaveOccurred()) + Expect(second).To(MatchError(first.Error()), + "a broken config is a hard stop for the second installer too, not a silent zero value") + }) +}) diff --git a/pkg/cli/webapp/src/PromptRunStream.test.tsx b/pkg/cli/webapp/src/PromptRunStream.test.tsx index 70b4c42d..07e9144f 100644 --- a/pkg/cli/webapp/src/PromptRunStream.test.tsx +++ b/pkg/cli/webapp/src/PromptRunStream.test.tsx @@ -1,6 +1,7 @@ -import { render, screen } from "@testing-library/react"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { PromptRunStream } from "./PromptRunStream"; +import type { VerifyReport } from "./types/verifyReport"; const useSessionChatMock = vi.hoisted(() => vi.fn()); @@ -8,6 +9,30 @@ vi.mock("./hooks/useSessionChat", () => ({ useSessionChat: useSessionChatMock, })); +afterEach(cleanup); + +function verifyReport(overrides: Partial = {}): VerifyReport { + return { + kind: "fixture", + name: "acceptance", + ran: true, + passed: false, + iteration: 1, + summary: { + total: 5, + passed: 3, + failed: 0, + warned: 0, + skipped: 0, + pending: 2, + running: 0, + timedout: 0, + }, + state: "running", + ...overrides, + }; +} + describe("PromptRunStream", () => { beforeEach(() => { useSessionChatMock.mockReturnValue({ @@ -32,6 +57,7 @@ describe("PromptRunStream", () => { provider: "anthropic", mode: "api", }, + verify: null, }); }); @@ -53,4 +79,76 @@ describe("PromptRunStream", () => { ).toBeInTheDocument(); expect(screen.queryByText("Starting run…")).not.toBeInTheDocument(); }); + + it("shows nothing when there is no verify frame yet", () => { + render(); + expect(screen.queryByTestId("verify-status")).not.toBeInTheDocument(); + }); + + it("shows a live progress line while a check is still running", () => { + useSessionChatMock.mockReturnValue({ + messages: [], + status: "streaming", + run: { runId: "run-verify", status: "running" }, + verify: { report: verifyReport(), done: false }, + }); + + render(); + + expect(screen.getByTestId("verify-status")).toHaveTextContent( + "verifying · 3/5 passed", + ); + }); + + it("shows a plain verdict once the check has passed", () => { + useSessionChatMock.mockReturnValue({ + messages: [], + status: "done", + run: { runId: "run-verify", status: "done" }, + verify: { + report: verifyReport({ passed: true, state: "passed" }), + done: true, + }, + }); + + render(); + + expect(screen.getByTestId("verify-status")).toHaveTextContent("verified"); + }); + + it("renders a malformed verify frame's error without dropping the transcript", () => { + useSessionChatMock.mockReturnValue({ + messages: [], + status: "streaming", + run: { runId: "run-verify", status: "running" }, + verify: null, + error: 'invalid verify frame: verify report: "state" must be one of queued, running, passed, failed, errored, warned, skipped, cancelled, timed_out, got "bogus"', + }); + + render(); + + expect(screen.getByRole("alert")).toHaveTextContent(/invalid verify frame/); + expect(screen.queryByTestId("verify-status")).not.toBeInTheDocument(); + }); + + it("shows the failure reason once the check has failed", () => { + useSessionChatMock.mockReturnValue({ + messages: [], + status: "done", + run: { runId: "run-verify", status: "done" }, + verify: { + report: verifyReport({ + state: "failed", + reason: "2 of 5 checks failed", + }), + done: true, + }, + }); + + render(); + + expect(screen.getByTestId("verify-status")).toHaveTextContent( + "verification failed · 2 of 5 checks failed", + ); + }); }); diff --git a/pkg/cli/webapp/src/PromptRunStream.tsx b/pkg/cli/webapp/src/PromptRunStream.tsx index ef6dc1b7..f5b3534e 100644 --- a/pkg/cli/webapp/src/PromptRunStream.tsx +++ b/pkg/cli/webapp/src/PromptRunStream.tsx @@ -5,6 +5,7 @@ import { type PromptRunSummary, } from "./hooks/usePromptRunStream"; import { useSessionChat } from "./hooks/useSessionChat"; +import type { VerifyFrame, VerifyState } from "./types/verifyReport"; /** * PromptRunStream renders a prompt run's session history live: it subscribes to @@ -13,8 +14,9 @@ import { useSessionChat } from "./hooks/useSessionChat"; */ export function PromptRunStream({ runID }: { runID: string }) { const chat = useSessionChat({ initialRunID: runID }); - const { messages, summary, status, error, run, chatState } = chat; + const { messages, summary, status, error, run, chatState, verify } = chat; const empty = messages.length === 0; + const verifyLine = verifyStatusLine(verify); return (
@@ -26,6 +28,14 @@ export function PromptRunStream({ runID }: { runID: string }) { ) : null}
+ {verifyLine && ( +
+ {verifyLine} +
+ )} {error && ( = { error: "bg-destructive", }; +const VERIFY_FAILURE_STATES: readonly VerifyState[] = [ + "failed", + "errored", + "timed_out", + "cancelled", +]; + +/** + * verifyStatusLine renders the run's latest `verify` frame as a single line: + * a live progress count while a check is still running, and a terse verdict + * once it is done. Returns null when there is nothing to show yet. + */ +function verifyStatusLine(verify: VerifyFrame | null): string | null { + const report = verify?.report; + if (!report) return null; + if (report.state === "passed") return "verified"; + if (VERIFY_FAILURE_STATES.includes(report.state)) { + return report.reason + ? `verification failed · ${report.reason}` + : "verification failed"; + } + if (report.state === "warned") { + return report.reason + ? `verification warned · ${report.reason}` + : "verification warned"; + } + if (report.state === "skipped") return "verification skipped"; + const { passed, total } = report.summary; + return `verifying · ${passed}/${total} passed`; +} + function StatusPill({ status }: { status: PromptRunStreamStatus }) { return ( diff --git a/pkg/cli/webapp/src/StateMessage.tsx b/pkg/cli/webapp/src/StateMessage.tsx index 63582663..56c4f1b8 100644 --- a/pkg/cli/webapp/src/StateMessage.tsx +++ b/pkg/cli/webapp/src/StateMessage.tsx @@ -10,5 +10,6 @@ export function StateMessage({ const classes = tone === "error" ? "border-destructive/30 bg-destructive/10 text-destructive" : "border-border bg-muted/30 text-muted-foreground"; - return
{children}
; + return
{children}
; } diff --git a/pkg/cli/webapp/src/WhoamiPage.test.tsx b/pkg/cli/webapp/src/WhoamiPage.test.tsx index b85760d3..11d1c779 100644 --- a/pkg/cli/webapp/src/WhoamiPage.test.tsx +++ b/pkg/cli/webapp/src/WhoamiPage.test.tsx @@ -303,6 +303,23 @@ describe("WhoamiPage", () => { ); }); + it("shows a rejected model exclusion as a banner above the capability tree", async () => { + const rejection = "configuration changes require a loopback request host"; + vi.stubGlobal("fetch", vi.fn() + .mockResolvedValueOnce(jsonResponse(WHOAMI_RESULT)) + .mockResolvedValueOnce(new Response(rejection, { status: 403 }))); + renderWhoamiPage(); + + const checkbox = await screen.findByRole("checkbox", { name: "Enable gpt-5.5-codex model" }); + fireEvent.click(checkbox); + + const banner = await screen.findByRole("alert"); + const tree = screen.getByRole("tree", { name: "Capability topology" }); + expect(banner).toHaveTextContent(rejection); + expect(banner.compareDocumentPosition(tree) & Node.DOCUMENT_POSITION_FOLLOWING).not.toBe(0); + expect(checkbox).toBeChecked(); + }); + it("shows inherited provider policy without erasing child selections", async () => { const disabled = { ...NOTHING_DISABLED, providers: ["openai"] }; vi.stubGlobal("fetch", vi.fn().mockResolvedValue(jsonResponse({ diff --git a/pkg/cli/webapp/src/WhoamiTopology.tsx b/pkg/cli/webapp/src/WhoamiTopology.tsx index 9a146d88..5af35bbf 100644 --- a/pkg/cli/webapp/src/WhoamiTopology.tsx +++ b/pkg/cli/webapp/src/WhoamiTopology.tsx @@ -107,6 +107,7 @@ function CapabilityTopology({ Expand a provider, runtime mode, or model. Child selections remain intact when a parent policy is disabled.

+ {controller.error && {controller.error}} - {controller.error && {controller.error}} ); } diff --git a/pkg/cli/webapp/src/hooks/useEventSource.test.ts b/pkg/cli/webapp/src/hooks/useEventSource.test.ts new file mode 100644 index 00000000..e67bcd6d --- /dev/null +++ b/pkg/cli/webapp/src/hooks/useEventSource.test.ts @@ -0,0 +1,124 @@ +import { act, renderHook } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useEventSource } from "./useEventSource"; + +/** + * jsdom does not implement EventSource, so tests install this minimal double + * as the global constructor and drive it directly via `emit`. + */ +class MockEventSource { + static instances: MockEventSource[] = []; + onopen: (() => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + onerror: (() => void) | null = null; + closed = false; + private readonly listeners = new Map< + string, + Set<(event: MessageEvent) => void> + >(); + + constructor(readonly url: string) { + MockEventSource.instances.push(this); + } + + addEventListener(name: string, handler: EventListener) { + const set = this.listeners.get(name) ?? new Set(); + set.add(handler as (event: MessageEvent) => void); + this.listeners.set(name, set); + } + + removeEventListener(name: string, handler: EventListener) { + this.listeners.get(name)?.delete(handler as (event: MessageEvent) => void); + } + + close() { + this.closed = true; + } + + emit(name: string, data: string) { + const event = { data } as MessageEvent; + for (const handler of this.listeners.get(name) ?? []) { + handler(event); + } + } +} + +describe("useEventSource", () => { + let originalEventSource: typeof EventSource | undefined; + + beforeEach(() => { + originalEventSource = globalThis.EventSource; + MockEventSource.instances = []; + globalThis.EventSource = + MockEventSource as unknown as typeof EventSource; + }); + + afterEach(() => { + globalThis.EventSource = originalEventSource as typeof EventSource; + }); + + it("routes a handler error to onError and keeps delivering later events", () => { + const onEvent = vi.fn((event: string) => { + if (event === "verify") throw new Error("drifted verify frame"); + }); + const onError = vi.fn(); + + renderHook(() => + useEventSource("http://test/stream", { + events: ["verify", "entry"], + onEvent, + onError, + }), + ); + const source = MockEventSource.instances[0]!; + + act(() => { + source.emit("verify", "bad-payload"); + }); + expect(onError).toHaveBeenCalledWith("verify", "drifted verify frame"); + expect(source.closed).toBe(false); + + act(() => { + source.emit("entry", "good-payload"); + }); + expect(onEvent).toHaveBeenLastCalledWith("entry", "good-payload"); + expect(onError).toHaveBeenCalledTimes(1); + }); + + it("does not call onError when a handler completes normally", () => { + const onEvent = vi.fn(); + const onError = vi.fn(); + + renderHook(() => + useEventSource("http://test/stream", { + events: ["entry"], + onEvent, + onError, + }), + ); + const source = MockEventSource.instances[0]!; + + act(() => { + source.emit("entry", "fine"); + }); + expect(onEvent).toHaveBeenCalledWith("entry", "fine"); + expect(onError).not.toHaveBeenCalled(); + }); + + it("wraps the default message dispatch the same way", () => { + const onEvent = vi.fn(() => { + throw new Error("boom"); + }); + const onError = vi.fn(); + + renderHook(() => + useEventSource("http://test/stream", { onEvent, onError }), + ); + const source = MockEventSource.instances[0]!; + + act(() => { + source.onmessage?.({ data: "payload" } as MessageEvent); + }); + expect(onError).toHaveBeenCalledWith("message", "boom"); + }); +}); diff --git a/pkg/cli/webapp/src/hooks/useEventSource.ts b/pkg/cli/webapp/src/hooks/useEventSource.ts index 906df05c..167b3652 100644 --- a/pkg/cli/webapp/src/hooks/useEventSource.ts +++ b/pkg/cli/webapp/src/hooks/useEventSource.ts @@ -10,6 +10,12 @@ export interface UseEventSourceOptions { /** Called for every frame: the event name ("message" for default) and raw data. */ onEvent: (event: string, data: string) => void; onStatus?: (status: SSEStatus) => void; + /** + * Called when `onEvent` throws while handling a frame (e.g. wire-shape + * drift). The stream stays open and later events keep being delivered — + * a bad frame must never go silent nor tear down the connection. + */ + onError?: (event: string, message: string) => void; } /** @@ -18,13 +24,15 @@ export interface UseEventSourceOptions { * refs so callers can pass inline closures without forcing reconnects. */ export function useEventSource(url: string | undefined, options: UseEventSourceOptions): void { - const { enabled = true, events = [], onEvent, onStatus } = options; + const { enabled = true, events = [], onEvent, onStatus, onError } = options; const onEventRef = useRef(onEvent); const onStatusRef = useRef(onStatus); + const onErrorRef = useRef(onError); useEffect(() => { onEventRef.current = onEvent; onStatusRef.current = onStatus; - }, [onEvent, onStatus]); + onErrorRef.current = onError; + }, [onEvent, onStatus, onError]); // Stable dependency: only re-bind when the set of named events actually changes. const eventsKey = events.join(","); @@ -36,6 +44,7 @@ export function useEventSource(url: string | undefined, options: UseEventSourceO eventsKey, onEventRef, onStatusRef, + onErrorRef, ); }, [url, enabled, eventsKey]); } @@ -45,22 +54,31 @@ function subscribeToEventSource( eventsKey: string, onEventRef: MutableRefObject, onStatusRef: MutableRefObject, + onErrorRef: MutableRefObject, ) { const setStatus = (status: SSEStatus) => onStatusRef.current?.(status); + const dispatch = (name: string, data: string) => { + try { + onEventRef.current(name, data); + } catch (error) { + onErrorRef.current?.( + name, + error instanceof Error ? error.message : String(error), + ); + } + }; setStatus("connecting"); const eventSource = new EventSource(url); let closed = false; eventSource.onopen = () => setStatus("open"); - eventSource.onmessage = (event) => - onEventRef.current("message", event.data); + eventSource.onmessage = (event) => dispatch("message", event.data); eventSource.onerror = () => { if (!closed) setStatus("reconnecting"); }; const listeners = (eventsKey ? eventsKey.split(",") : []).map((name) => { - const handler = (event: MessageEvent) => - onEventRef.current(name, event.data); + const handler = (event: MessageEvent) => dispatch(name, event.data); eventSource.addEventListener(name, handler as EventListener); return { name, handler }; }); diff --git a/pkg/cli/webapp/src/hooks/usePromptRunStream.test.tsx b/pkg/cli/webapp/src/hooks/usePromptRunStream.test.tsx index 2beeacf4..afd1959a 100644 --- a/pkg/cli/webapp/src/hooks/usePromptRunStream.test.tsx +++ b/pkg/cli/webapp/src/hooks/usePromptRunStream.test.tsx @@ -1,6 +1,7 @@ import { act, renderHook } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import { usePromptRunStream } from "./usePromptRunStream"; +import type { VerifyReport } from "../types/verifyReport"; const useEventSourceMock = vi.hoisted(() => vi.fn()); @@ -8,11 +9,37 @@ vi.mock("./useEventSource", () => ({ useEventSource: useEventSourceMock, })); +function verifyReport(overrides: Partial = {}): VerifyReport { + return { + kind: "fixture", + name: "acceptance", + ran: true, + passed: false, + iteration: 1, + summary: { + total: 5, + passed: 3, + failed: 0, + warned: 0, + skipped: 0, + pending: 2, + running: 0, + timedout: 0, + }, + state: "running", + ...overrides, + }; +} + +function lastOnEvent() { + const calls = useEventSourceMock.mock.calls; + return calls[calls.length - 1]?.[1].onEvent; +} + describe("usePromptRunStream", () => { it("retains the terminal failure summary from the error event", () => { const { result } = renderHook(() => usePromptRunStream("run-4a82")); - const calls = useEventSourceMock.mock.calls; - const onEvent = calls[calls.length - 1]?.[1].onEvent; + const onEvent = lastOnEvent(); act(() => { onEvent( @@ -44,6 +71,108 @@ describe("usePromptRunStream", () => { error: "provider rejected the request", run: undefined, chatState: undefined, + verify: null, + }); + }); + + it("keeps the latest verify snapshot, ending on the done verdict", () => { + const { result } = renderHook(() => usePromptRunStream("run-verify-1")); + const onEvent = lastOnEvent(); + + const first = verifyReport({ summary: { ...verifyReport().summary, passed: 1, pending: 4 } }); + const second = verifyReport({ summary: { ...verifyReport().summary, passed: 3, pending: 2 } }); + const verdict = verifyReport({ + passed: true, + state: "passed", + summary: { ...verifyReport().summary, passed: 5, pending: 0 }, + }); + + act(() => { + onEvent("verify", JSON.stringify({ report: first, done: false })); + }); + expect(result.current.verify).toEqual({ report: first, done: false }); + + act(() => { + onEvent("verify", JSON.stringify({ report: second, done: false })); + }); + expect(result.current.verify).toEqual({ report: second, done: false }); + + act(() => { + onEvent("verify", JSON.stringify({ report: verdict, done: true })); + }); + expect(result.current.verify).toEqual({ report: verdict, done: true }); + }); + + it("resets verify to null when a frame reports no report", () => { + const { result } = renderHook(() => usePromptRunStream("run-verify-2")); + const onEvent = lastOnEvent(); + + act(() => { + onEvent( + "verify", + JSON.stringify({ report: verifyReport(), done: false }), + ); + }); + expect(result.current.verify?.report).not.toBeNull(); + + act(() => { + onEvent("verify", JSON.stringify({ report: null, done: false })); + }); + expect(result.current.verify).toEqual({ report: null, done: false }); + }); + + it("clears verify and surfaces an error on a malformed frame, without swallowing it", () => { + const { result } = renderHook(() => usePromptRunStream("run-verify-3")); + const onEvent = lastOnEvent(); + + act(() => { + onEvent( + "verify", + JSON.stringify({ report: verifyReport(), done: false }), + ); + }); + expect(result.current.verify?.report).not.toBeNull(); + + expect(() => + act(() => { + onEvent( + "verify", + JSON.stringify({ + report: { ...verifyReport(), state: "bogus" }, + done: false, + }), + ); + }), + ).not.toThrow(); + + expect(result.current.verify).toBeNull(); + expect(result.current.error).toMatch(/verify/); + expect(result.current.error).toMatch(/state/); + }); + + it("preserves the data-verify part on a verdict transcript message", () => { + const { result } = renderHook(() => usePromptRunStream("run-verify-4")); + const onEvent = lastOnEvent(); + const report = verifyReport({ passed: true, state: "passed" }); + + act(() => { + onEvent( + "entry", + JSON.stringify({ + id: "msg-1", + role: "verified", + parts: [ + { type: "text", text: "verified: acceptance" }, + { type: "data-verify", data: report }, + ], + }), + ); + }); + + expect(result.current.messages).toHaveLength(1); + expect(result.current.messages[0]?.parts[1]).toEqual({ + type: "data-verify", + data: report, }); }); }); diff --git a/pkg/cli/webapp/src/hooks/usePromptRunStream.ts b/pkg/cli/webapp/src/hooks/usePromptRunStream.ts index 4cbc433b..96c18ebb 100644 --- a/pkg/cli/webapp/src/hooks/usePromptRunStream.ts +++ b/pkg/cli/webapp/src/hooks/usePromptRunStream.ts @@ -7,6 +7,7 @@ import { } from "react"; import type { SessionUIMessage } from "@flanksource/clicky-ui/ai"; import { useEventSource } from "./useEventSource"; +import { parseVerifyFrame, type VerifyFrame } from "../types/verifyReport"; /** Immediate response of the async prompt "run" action. */ export interface PromptRunHandle { @@ -111,6 +112,7 @@ export interface PromptRunStreamState { error?: string; run?: PromptRunFrame; chatState?: ChatStateFrame; + verify: VerifyFrame | null; } const PROMPT_RUN_BASE = "/api/captain/prompt/runs"; @@ -129,7 +131,10 @@ type PromptRunStreamAction = messages: SessionUIMessage[]; } | { type: "done"; summary?: PromptRunSummary } - | { type: "error"; summary: PromptRunSummary }; + | { type: "error"; summary: PromptRunSummary } + | { type: "verify"; verify: VerifyFrame } + | { type: "verify-error"; message: string } + | { type: "stream-handler-error"; message: string }; type MessageIndex = { byId: Map; @@ -152,9 +157,16 @@ function streamReducer( done: !action.runID, run: undefined, chatState: undefined, + verify: null, }; case "run": return { ...state, run: action.run }; + case "verify": + return { ...state, verify: action.verify }; + case "verify-error": + return { ...state, verify: null, error: action.message }; + case "stream-handler-error": + return { ...state, error: action.message }; case "chat-state": return { ...state, @@ -198,6 +210,7 @@ function initialStreamState(): PromptRunStreamReducerState { done: true, run: undefined, chatState: undefined, + verify: null, }; } @@ -268,6 +281,18 @@ export function usePromptRunStream( }); return; } + if (event === "verify") { + try { + const raw: unknown = JSON.parse(data); + update({ type: "verify", verify: parseVerifyFrame(raw) }); + } catch (error) { + update({ + type: "verify-error", + message: `invalid verify frame: ${describeError(error)}`, + }); + } + return; + } if (event !== "entry") return; const message = parse(data); if (!message) return; @@ -281,10 +306,21 @@ export function usePromptRunStream( const url = runID ? `${basePath}/${encodeURIComponent(runID)}/stream` : undefined; + const onError = useCallback( + (event: string, message: string) => { + update({ + type: "stream-handler-error", + message: `${event} handler error: ${message}`, + }); + }, + [update], + ); + useEventSource(url, { enabled: Boolean(url) && !state.done, - events: ["run", "entry", "state", "done", "error"], + events: ["run", "entry", "state", "done", "error", "verify"], onEvent, + onError, }); return { @@ -294,6 +330,7 @@ export function usePromptRunStream( error: state.error, run: state.run, chatState: state.chatState, + verify: state.verify, }; } @@ -326,3 +363,7 @@ function parse(data: string): T | undefined { return undefined; } } + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/pkg/cli/webapp/src/types/verifyReport.test.ts b/pkg/cli/webapp/src/types/verifyReport.test.ts new file mode 100644 index 00000000..c5054f9e --- /dev/null +++ b/pkg/cli/webapp/src/types/verifyReport.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from "vitest"; +import { parseVerifyFrame, type VerifyReport } from "./verifyReport"; + +const BASE_REPORT: VerifyReport = { + kind: "fixture", + name: "acceptance", + ran: true, + passed: false, + reason: "2 of 5 checks failed", + iteration: 1, + summary: { + total: 5, + passed: 3, + failed: 2, + warned: 0, + skipped: 0, + pending: 0, + running: 0, + timedout: 0, + }, + tests: [ + { + name: "renders the panel", + framework: "vitest", + passed: true, + children: [], + }, + { + name: "handles the null report", + framework: "vitest", + failed: true, + message: "expected null, got object", + }, + ], + checklist: [ + { item: "shows a status line", passed: true }, + { item: "never swallows an error", passed: null, message: "not judged yet" }, + ], + state: "failed", +}; + +describe("parseVerifyFrame", () => { + it("parses a running snapshot with a null report", () => { + expect(parseVerifyFrame({ report: null, done: false })).toEqual({ + report: null, + done: false, + }); + }); + + it("parses a full verdict report and preserves every field", () => { + const frame = parseVerifyFrame({ report: BASE_REPORT, done: true }); + expect(frame).toEqual({ report: BASE_REPORT, done: true }); + }); + + it("throws when the frame is not an object", () => { + expect(() => parseVerifyFrame("not-a-frame")).toThrow(/expected an object/); + }); + + it('throws when "done" is missing or not a boolean', () => { + expect(() => parseVerifyFrame({ report: null })).toThrow(/"done"/); + expect(() => parseVerifyFrame({ report: null, done: "yes" })).toThrow( + /"done"/, + ); + }); + + it("throws when the report's state is not a known VerifyState", () => { + const drifted = { ...BASE_REPORT, state: "in_progress" }; + expect(() => parseVerifyFrame({ report: drifted, done: false })).toThrow( + /state/, + ); + }); + + it("throws when a required report field has the wrong type", () => { + const drifted = { ...BASE_REPORT, kind: 42 }; + expect(() => parseVerifyFrame({ report: drifted, done: false })).toThrow( + /"kind"/, + ); + }); + + it("throws when the summary is missing a required counter", () => { + const { timedout: _timedout, ...incompleteSummary } = BASE_REPORT.summary; + const drifted = { ...BASE_REPORT, summary: incompleteSummary }; + expect(() => parseVerifyFrame({ report: drifted, done: false })).toThrow( + /summary/, + ); + }); + + it("throws when a checklist item's passed field is not boolean-or-null", () => { + const drifted = { + ...BASE_REPORT, + checklist: [{ item: "x", passed: "true" }], + }; + expect(() => parseVerifyFrame({ report: drifted, done: false })).toThrow( + /checklist/, + ); + }); + + it("throws when a test node is missing its name", () => { + const drifted = { ...BASE_REPORT, tests: [{ passed: true }] }; + expect(() => parseVerifyFrame({ report: drifted, done: false })).toThrow( + /name/, + ); + }); + + it("parses a test node's rolled-up summary field", () => { + const nodeSummary = { + total: 2, + passed: 1, + failed: 1, + warned: 0, + skipped: 0, + pending: 0, + running: 0, + timedout: 0, + }; + const drifted = { + ...BASE_REPORT, + tests: [{ name: "group", passed: false, summary: nodeSummary }], + }; + const frame = parseVerifyFrame({ report: drifted, done: false }); + expect(frame.report?.tests?.[0]?.summary).toEqual(nodeSummary); + }); + + it("throws when a test node's summary has a wrong-typed counter", () => { + const drifted = { + ...BASE_REPORT, + tests: [{ name: "group", summary: { ...BASE_REPORT.summary, total: "2" } }], + }; + expect(() => parseVerifyFrame({ report: drifted, done: false })).toThrow( + /"total"/, + ); + }); +}); diff --git a/pkg/cli/webapp/src/types/verifyReport.ts b/pkg/cli/webapp/src/types/verifyReport.ts new file mode 100644 index 00000000..f71ff27d --- /dev/null +++ b/pkg/cli/webapp/src/types/verifyReport.ts @@ -0,0 +1,320 @@ +/** + * Local mirror of captain's wire-level verification types (Go source of truth: + * `pkg/api/verify_report.go`). Kept as a plain type module — no clicky-ui + * import — so it can later be swapped for that package's own export (once the + * release carrying `VerificationResults` lands) without touching consumers. + * + * Field names stay snake_case to match the JSON the server actually sends; + * this module does not camelCase or otherwise reshape the wire. + */ + +/** Canonical order mirrors Go's `AllVerifyStates()`. */ +export const VERIFY_STATES = [ + "queued", + "running", + "passed", + "failed", + "errored", + "warned", + "skipped", + "cancelled", + "timed_out", +] as const; + +export type VerifyState = (typeof VERIFY_STATES)[number]; + +export interface VerifySummary { + total: number; + passed: number; + failed: number; + warned: number; + skipped: number; + pending: number; + running: number; + timedout: number; +} + +export interface VerifyNodeProgress { + phase?: string; + done: number; + total: number; +} + +export interface VerifyNodeContext { + command?: string; + exit_code: number; + cwd?: string; + cel_expression?: string; + cel_vars?: Record; + expected?: unknown; + actual?: unknown; +} + +export interface VerifyNode { + name: string; + framework?: string; + task_id?: string; + file?: string; + line?: number; + message?: string; + command?: string; + work_dir?: string; + stdout?: string; + stderr?: string; + duration?: number; + passed?: boolean; + failed?: boolean; + warned?: boolean; + skipped?: boolean; + pending?: boolean; + running?: boolean; + timed_out?: boolean; + progress?: VerifyNodeProgress; + context?: VerifyNodeContext; + /** Rolled-up counts for this node's subtree, same shape as the report summary. */ + summary?: VerifySummary; + detail?: unknown; + children?: VerifyNode[]; +} + +export interface VerifyChecklistItem { + item: string; + passed: boolean | null; + message?: string; +} + +export interface VerifyReport { + kind: string; + name?: string; + ran: boolean; + passed: boolean; + reason?: string; + feedback?: string; + /** 1-based loop turn ("turn 1 of 3"); always on the wire. */ + iteration: number; + summary: VerifySummary; + tests?: VerifyNode[]; + checklist?: VerifyChecklistItem[]; + state: VerifyState; + started_at?: string; + finished_at?: string; + duration?: number; +} + +/** The `verify` SSE event payload: the newest report, and whether it is the + * verdict (`done: true`) or a still-running snapshot (`done: false`). */ +export interface VerifyFrame { + report: VerifyReport | null; + done: boolean; +} + +/** + * parseVerifyFrame validates an already-JSON-decoded value against the + * VerifyFrame wire shape and throws a descriptive error on any drift — a + * missing field, a wrong type, or a state string the Go side no longer emits. + * There is no silent default: a malformed frame must fail loudly rather than + * render a stale or blank verification panel. + */ +export function parseVerifyFrame(data: unknown): VerifyFrame { + const frame = requireRecord(data, "verify frame"); + const done = requireBoolean(frame.done, 'verify frame: "done"'); + const report = + frame.report === null || frame.report === undefined + ? null + : parseVerifyReport(frame.report); + return { report, done }; +} + +function parseVerifyReport(value: unknown): VerifyReport { + const r = requireRecord(value, "verify report"); + const state = requireString(r.state, 'verify report: "state"'); + if (!isVerifyState(state)) { + throw new Error( + `verify report: "state" must be one of ${VERIFY_STATES.join(", ")}, got ${JSON.stringify(state)}`, + ); + } + return { + kind: requireString(r.kind, 'verify report: "kind"'), + name: optionalString(r.name, 'verify report: "name"'), + ran: requireBoolean(r.ran, 'verify report: "ran"'), + passed: requireBoolean(r.passed, 'verify report: "passed"'), + reason: optionalString(r.reason, 'verify report: "reason"'), + feedback: optionalString(r.feedback, 'verify report: "feedback"'), + iteration: requireNumber(r.iteration, 'verify report: "iteration"'), + summary: parseVerifySummary(r.summary), + tests: optionalArray(r.tests, 'verify report: "tests"', parseVerifyNode), + checklist: optionalArray( + r.checklist, + 'verify report: "checklist"', + parseChecklistItem, + ), + state, + started_at: optionalString(r.started_at, 'verify report: "started_at"'), + finished_at: optionalString(r.finished_at, 'verify report: "finished_at"'), + duration: optionalNumber(r.duration, 'verify report: "duration"'), + }; +} + +function parseVerifySummary(value: unknown): VerifySummary { + const s = requireRecord(value, "verify report summary"); + return { + total: requireNumber(s.total, 'verify report summary: "total"'), + passed: requireNumber(s.passed, 'verify report summary: "passed"'), + failed: requireNumber(s.failed, 'verify report summary: "failed"'), + warned: requireNumber(s.warned, 'verify report summary: "warned"'), + skipped: requireNumber(s.skipped, 'verify report summary: "skipped"'), + pending: requireNumber(s.pending, 'verify report summary: "pending"'), + running: requireNumber(s.running, 'verify report summary: "running"'), + timedout: requireNumber(s.timedout, 'verify report summary: "timedout"'), + }; +} + +function parseVerifyNode(value: unknown): VerifyNode { + const n = requireRecord(value, "verify test node"); + return { + name: requireString(n.name, 'verify test node: "name"'), + framework: optionalString(n.framework, 'verify test node: "framework"'), + task_id: optionalString(n.task_id, 'verify test node: "task_id"'), + file: optionalString(n.file, 'verify test node: "file"'), + line: optionalNumber(n.line, 'verify test node: "line"'), + message: optionalString(n.message, 'verify test node: "message"'), + command: optionalString(n.command, 'verify test node: "command"'), + work_dir: optionalString(n.work_dir, 'verify test node: "work_dir"'), + stdout: optionalString(n.stdout, 'verify test node: "stdout"'), + stderr: optionalString(n.stderr, 'verify test node: "stderr"'), + duration: optionalNumber(n.duration, 'verify test node: "duration"'), + passed: optionalBoolean(n.passed, 'verify test node: "passed"'), + failed: optionalBoolean(n.failed, 'verify test node: "failed"'), + warned: optionalBoolean(n.warned, 'verify test node: "warned"'), + skipped: optionalBoolean(n.skipped, 'verify test node: "skipped"'), + pending: optionalBoolean(n.pending, 'verify test node: "pending"'), + running: optionalBoolean(n.running, 'verify test node: "running"'), + timed_out: optionalBoolean(n.timed_out, 'verify test node: "timed_out"'), + progress: n.progress === undefined ? undefined : parseProgress(n.progress), + context: n.context === undefined ? undefined : parseContext(n.context), + summary: n.summary === undefined ? undefined : parseVerifySummary(n.summary), + detail: n.detail, + children: optionalArray( + n.children, + 'verify test node: "children"', + parseVerifyNode, + ), + }; +} + +function parseProgress(value: unknown): VerifyNodeProgress { + const p = requireRecord(value, "verify test node progress"); + return { + phase: optionalString(p.phase, 'verify test node progress: "phase"'), + done: requireNumber(p.done, 'verify test node progress: "done"'), + total: requireNumber(p.total, 'verify test node progress: "total"'), + }; +} + +function parseContext(value: unknown): VerifyNodeContext { + const c = requireRecord(value, "verify test node context"); + return { + command: optionalString(c.command, 'verify test node context: "command"'), + exit_code: requireNumber( + c.exit_code, + 'verify test node context: "exit_code"', + ), + cwd: optionalString(c.cwd, 'verify test node context: "cwd"'), + cel_expression: optionalString( + c.cel_expression, + 'verify test node context: "cel_expression"', + ), + cel_vars: + c.cel_vars === undefined + ? undefined + : requireRecord(c.cel_vars, 'verify test node context: "cel_vars"'), + expected: c.expected, + actual: c.actual, + }; +} + +function parseChecklistItem(value: unknown): VerifyChecklistItem { + const c = requireRecord(value, "verify report checklist item"); + const passed = c.passed; + if (passed !== null && typeof passed !== "boolean") { + throw new Error( + `verify report checklist item: "passed" must be a boolean or null, got ${typeOf(passed)}`, + ); + } + return { + item: requireString(c.item, 'verify report checklist item: "item"'), + passed, + message: optionalString( + c.message, + 'verify report checklist item: "message"', + ), + }; +} + +function isVerifyState(value: string): value is VerifyState { + return (VERIFY_STATES as readonly string[]).includes(value); +} + +function typeOf(value: unknown): string { + if (value === null) return "null"; + if (Array.isArray(value)) return "array"; + return typeof value; +} + +function requireRecord( + value: unknown, + label: string, +): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${label}: expected an object, got ${typeOf(value)}`); + } + return value as Record; +} + +function requireString(value: unknown, label: string): string { + if (typeof value !== "string") { + throw new Error(`${label} must be a string, got ${typeOf(value)}`); + } + return value; +} + +function optionalString(value: unknown, label: string): string | undefined { + if (value === undefined) return undefined; + return requireString(value, label); +} + +function requireNumber(value: unknown, label: string): number { + if (typeof value !== "number") { + throw new Error(`${label} must be a number, got ${typeOf(value)}`); + } + return value; +} + +function optionalNumber(value: unknown, label: string): number | undefined { + if (value === undefined) return undefined; + return requireNumber(value, label); +} + +function requireBoolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") { + throw new Error(`${label} must be a boolean, got ${typeOf(value)}`); + } + return value; +} + +function optionalBoolean(value: unknown, label: string): boolean | undefined { + if (value === undefined) return undefined; + return requireBoolean(value, label); +} + +function optionalArray( + value: unknown, + label: string, + parseItem: (item: unknown) => T, +): T[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + throw new Error(`${label} must be an array, got ${typeOf(value)}`); + } + return value.map(parseItem); +} diff --git a/pkg/database/caller_tool_store.go b/pkg/database/caller_tool_store.go index 737d043f..de147466 100644 --- a/pkg/database/caller_tool_store.go +++ b/pkg/database/caller_tool_store.go @@ -4,23 +4,18 @@ import ( "context" "errors" "fmt" - "reflect" "strings" "time" "github.com/flanksource/captain/pkg/api" "github.com/google/uuid" "gorm.io/gorm" - "gorm.io/gorm/clause" ) var ( ErrCallerToolCredentialInvalid = errors.New("invalid caller-tool credential") ErrCallerToolCredentialNotFound = errors.New("caller-tool credential not found") ErrCallerToolCredentialInactive = errors.New("caller-tool credential is inactive") - ErrTurnRequestInvalid = errors.New("invalid Captain turn request") - ErrTurnRequestNotFound = errors.New("captain turn request not found") - ErrTurnRequestConflict = errors.New("captain turn request conflict") ) type CallerToolCredential struct { @@ -163,293 +158,6 @@ func (db *DB) RevokeCallerToolCredential(ctx context.Context, id uuid.UUID, reas return fmt.Errorf("%w: credential %s was not revoked", ErrCallerToolCredentialInactive, id) } -type TurnRequestState string - -const ( - TurnRequestStatePending TurnRequestState = "pending" - TurnRequestStateApproved TurnRequestState = "approved" - TurnRequestStateDenied TurnRequestState = "denied" - TurnRequestStateCancelled TurnRequestState = "cancelled" - TurnRequestStateExpired TurnRequestState = "expired" -) - -type TurnRequest struct { - ID uuid.UUID `json:"id"` - SessionID uuid.UUID `json:"sessionId"` - TurnID *uuid.UUID `json:"turnId,omitempty"` - PromptRunID *uuid.UUID `json:"promptRunId,omitempty"` - ModelCallID *uuid.UUID `json:"modelCallId,omitempty"` - CredentialID *uuid.UUID `json:"-"` - ToolCallID string `json:"toolCallId,omitempty"` - Kind string `json:"kind"` - State TurnRequestState `json:"state"` - Request map[string]any `json:"request"` - Response map[string]any `json:"response,omitempty"` - IdempotencyKey string `json:"idempotencyKey,omitempty"` - RequestedBy string `json:"requestedBy,omitempty"` - ResolvedBy string `json:"resolvedBy,omitempty"` - Reason string `json:"reason,omitempty"` - Version int64 `json:"version"` - ExpiresAt *time.Time `json:"expiresAt,omitempty"` - CreatedAt time.Time `json:"createdAt"` - ResolvedAt *time.Time `json:"resolvedAt,omitempty"` -} - -type turnRequestRecord struct { - ID uuid.UUID `gorm:"column:id;type:uuid;primaryKey"` - SessionID uuid.UUID `gorm:"column:session_id;type:uuid"` - TurnID *uuid.UUID `gorm:"column:turn_id;type:uuid"` - PromptRunID *uuid.UUID `gorm:"column:prompt_run_id;type:uuid"` - ModelCallID *uuid.UUID `gorm:"column:model_call_id;type:uuid"` - CredentialID *uuid.UUID `gorm:"column:credential_id;type:uuid"` - ToolCallID *string `gorm:"column:tool_call_id"` - Kind string `gorm:"column:kind"` - State TurnRequestState `gorm:"column:state"` - Request map[string]any `gorm:"column:request;serializer:json;type:jsonb"` - Response map[string]any `gorm:"column:response;serializer:json;type:jsonb"` - IdempotencyKey *string `gorm:"column:idempotency_key"` - RequestedBy *string `gorm:"column:requested_by"` - ResolvedBy *string `gorm:"column:resolved_by"` - Reason *string `gorm:"column:reason"` - Version int64 `gorm:"column:version"` - ExpiresAt *time.Time `gorm:"column:expires_at"` - CreatedAt time.Time `gorm:"column:created_at"` - ResolvedAt *time.Time `gorm:"column:resolved_at"` -} - -func (turnRequestRecord) TableName() string { return "captain_turn_requests" } - -type CreateToolApprovalRequestInput struct { - CredentialID uuid.UUID - SessionID uuid.UUID - TurnID uuid.UUID - PromptRunID uuid.UUID - ModelCallID uuid.UUID - ToolCallID string - Tool string - Input map[string]any - RequestedBy string - ExpiresAt time.Time -} - -func (db *DB) CreateToolApprovalRequest( - ctx context.Context, - input CreateToolApprovalRequestInput, -) (*TurnRequest, error) { - var credential *CallerToolCredential - if input.CredentialID != uuid.Nil { - if err := db.ValidateCallerToolCredential(ctx, input.CredentialID); err != nil { - return nil, err - } - var err error - credential, err = db.GetCallerToolCredential(ctx, input.CredentialID) - if err != nil { - return nil, err - } - if credential.SessionID != input.SessionID || credential.PromptRunID != input.PromptRunID { - return nil, fmt.Errorf("%w: credential does not belong to the supplied session and run", ErrTurnRequestInvalid) - } - } - input.ToolCallID = strings.TrimSpace(input.ToolCallID) - input.Tool = strings.TrimSpace(input.Tool) - if input.SessionID == uuid.Nil || input.TurnID == uuid.Nil || input.PromptRunID == uuid.Nil || input.ModelCallID == uuid.Nil || - input.ToolCallID == "" || input.Tool == "" || !input.ExpiresAt.After(time.Now()) { - return nil, fmt.Errorf("%w: session, turn, prompt run, model call, tool call, tool, and future expiry are required", ErrTurnRequestInvalid) - } - if credential != nil && credential.Policy[input.Tool] != api.ToolPolicyAsk { - return nil, fmt.Errorf("%w: tool %q is not approved by ask policy", ErrTurnRequestInvalid, input.Tool) - } - if credential != nil && credential.ExpiresAt != nil && input.ExpiresAt.After(*credential.ExpiresAt) { - input.ExpiresAt = *credential.ExpiresAt - } - idempotencyKey := "provider:" + input.PromptRunID.String() + ":" + input.ToolCallID - if credential != nil { - idempotencyKey = "mcp:" + input.CredentialID.String() + ":" + input.ToolCallID - } - request := map[string]any{ - "tool": input.Tool, "input": input.Input, - } - var credentialID *uuid.UUID - if credential != nil { - credentialID = &input.CredentialID - } - record := turnRequestRecord{ - ID: uuid.New(), SessionID: input.SessionID, TurnID: &input.TurnID, PromptRunID: &input.PromptRunID, - ModelCallID: &input.ModelCallID, CredentialID: credentialID, ToolCallID: &input.ToolCallID, - Kind: "tool_approval", State: TurnRequestStatePending, Request: request, - IdempotencyKey: &idempotencyKey, RequestedBy: nullableTrimmed(input.RequestedBy), ExpiresAt: &input.ExpiresAt, - } - result := db.gorm.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(&record) - if result.Error != nil { - return nil, fmt.Errorf("create tool approval request: %w", result.Error) - } - if result.RowsAffected == 1 { - if err := db.touchChatSession(ctx, input.SessionID); err != nil { - return nil, err - } - return db.GetTurnRequest(ctx, record.ID) - } - var existing turnRequestRecord - if err := db.gorm.WithContext(ctx). - Where("session_id = ? AND idempotency_key = ?", input.SessionID, idempotencyKey). - First(&existing).Error; err != nil { - return nil, fmt.Errorf("read existing tool approval request: %w", err) - } - if !reflect.DeepEqual(existing.Request, request) { - return nil, fmt.Errorf("%w: tool call %q was retried with different input", ErrTurnRequestConflict, input.ToolCallID) - } - out := turnRequestFromRecord(existing) - return &out, nil -} - -type ResolveToolApprovalRequestInput struct { - SessionID uuid.UUID - RequestID uuid.UUID - ExpectedTurnID *uuid.UUID - Approved bool - UpdatedInput map[string]any - ResolvedBy string - Reason string -} - -func (db *DB) ResolveToolApprovalRequest( - ctx context.Context, - input ResolveToolApprovalRequestInput, -) (*TurnRequest, error) { - if input.SessionID == uuid.Nil || input.RequestID == uuid.Nil { - return nil, fmt.Errorf("%w: session and approval request IDs are required", ErrTurnRequestInvalid) - } - state := TurnRequestStateDenied - var response map[string]any - if input.Approved { - state = TurnRequestStateApproved - if input.UpdatedInput != nil { - response = map[string]any{"updatedInput": input.UpdatedInput} - } - } - var pending turnRequestRecord - if err := db.gorm.WithContext(ctx). - Where("id = ? AND session_id = ? AND kind = 'tool_approval'", input.RequestID, input.SessionID). - First(&pending).Error; err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, fmt.Errorf("%w: session %s approval %s", ErrTurnRequestNotFound, input.SessionID, input.RequestID) - } - return nil, fmt.Errorf("read tool approval request: %w", err) - } - if pending.State != TurnRequestStatePending { - if pending.State == state && reflect.DeepEqual(pending.Response, response) && - optionalString(pending.Reason) == strings.TrimSpace(input.Reason) { - out := turnRequestFromRecord(pending) - return &out, nil - } - return nil, fmt.Errorf("%w: approval %s already has a different %s decision", ErrTurnRequestConflict, pending.ID, pending.State) - } - if input.ExpectedTurnID != nil && (pending.TurnID == nil || *pending.TurnID != *input.ExpectedTurnID) { - return nil, fmt.Errorf("%w: approval %s does not belong to active turn %s", ErrTurnRequestConflict, pending.ID, *input.ExpectedTurnID) - } - now := time.Now().UTC() - result := db.gorm.WithContext(ctx).Model(&turnRequestRecord{}). - Where("id = ? AND state = 'pending'", pending.ID). - Where(`credential_id IS NOT NULL OR EXISTS ( - SELECT 1 FROM captain_prompt_runs run - WHERE run.id = captain_turn_requests.prompt_run_id AND run.state = 'waiting' - )`). - Where(`credential_id IS NULL OR EXISTS ( - SELECT 1 FROM captain_session_mcp_credentials credential - WHERE credential.id = captain_turn_requests.credential_id - AND credential.revoked_at IS NULL - AND (credential.expires_at IS NULL OR credential.expires_at > ?) - )`, now). - Updates(map[string]any{ - "state": state, "response": response, "resolved_by": nullableTrimmed(input.ResolvedBy), - "reason": nullableTrimmed(input.Reason), "resolved_at": now, - }) - if result.Error != nil { - return nil, fmt.Errorf("resolve tool approval request: %w", result.Error) - } - if result.RowsAffected == 0 { - if pending.CredentialID == nil { - return nil, fmt.Errorf("%w: approval %s cannot be resolved before its prompt run is waiting", ErrTurnRequestConflict, pending.ID) - } - if pending.CredentialID != nil { - if err := db.ValidateCallerToolCredential(ctx, *pending.CredentialID); err != nil { - return nil, err - } - } - return nil, fmt.Errorf("%w: session %s approval %s", ErrTurnRequestNotFound, input.SessionID, input.RequestID) - } - if err := db.touchChatSession(ctx, input.SessionID); err != nil { - return nil, err - } - return db.GetTurnRequest(ctx, pending.ID) -} - -func (db *DB) GetTurnRequest(ctx context.Context, id uuid.UUID) (*TurnRequest, error) { - var record turnRequestRecord - if err := db.gorm.WithContext(ctx).First(&record, "id = ?", id).Error; err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, fmt.Errorf("%w: %s", ErrTurnRequestNotFound, id) - } - return nil, fmt.Errorf("get Captain turn request: %w", err) - } - out := turnRequestFromRecord(record) - return &out, nil -} - -type TurnRequestFilter struct { - SessionID uuid.UUID - PromptRunID *uuid.UUID -} - -func (db *DB) ListTurnRequests(ctx context.Context, filter TurnRequestFilter) ([]TurnRequest, error) { - if filter.SessionID == uuid.Nil { - return nil, fmt.Errorf("%w: session ID is required", ErrTurnRequestInvalid) - } - query := db.gorm.WithContext(ctx).Where("session_id = ?", filter.SessionID).Order("created_at, id") - if filter.PromptRunID != nil { - query = query.Where("prompt_run_id = ?", *filter.PromptRunID) - } - var records []turnRequestRecord - if err := query.Find(&records).Error; err != nil { - return nil, fmt.Errorf("list Captain turn requests: %w", err) - } - requests := make([]TurnRequest, len(records)) - for i := range records { - requests[i] = turnRequestFromRecord(records[i]) - } - return requests, nil -} - -func (db *DB) ExpireToolApprovalRequest(ctx context.Context, id uuid.UUID, state TurnRequestState, reason string) error { - if state != TurnRequestStateExpired && state != TurnRequestStateCancelled { - return fmt.Errorf("%w: terminal state %q is invalid", ErrTurnRequestInvalid, state) - } - now := time.Now().UTC() - result := db.gorm.WithContext(ctx).Model(&turnRequestRecord{}). - Where("id = ? AND state = 'pending'", id). - Updates(map[string]any{"state": state, "reason": nullableTrimmed(reason), "resolved_at": now}) - if result.Error != nil { - return fmt.Errorf("expire tool approval request: %w", result.Error) - } - return nil -} - -func (db *DB) CancelPendingTurnRequests(ctx context.Context, sessionID, promptRunID uuid.UUID, reason string) error { - if sessionID == uuid.Nil || promptRunID == uuid.Nil { - return fmt.Errorf("%w: session and prompt run IDs are required", ErrTurnRequestInvalid) - } - now := time.Now().UTC() - result := db.gorm.WithContext(ctx).Model(&turnRequestRecord{}). - Where("session_id = ? AND prompt_run_id = ? AND state = 'pending'", sessionID, promptRunID). - Updates(map[string]any{ - "state": TurnRequestStateCancelled, "reason": nullableTrimmed(reason), "resolved_at": now, - }) - if result.Error != nil { - return fmt.Errorf("cancel pending Captain turn requests: %w", result.Error) - } - return nil -} - func callerToolCredentialFromRecord(record callerToolCredentialRecord) CallerToolCredential { return CallerToolCredential{ ID: record.ID, SessionID: record.SessionID, PromptRunID: record.PromptRunID, @@ -461,17 +169,6 @@ func callerToolCredentialFromRecord(record callerToolCredentialRecord) CallerToo } } -func turnRequestFromRecord(record turnRequestRecord) TurnRequest { - return TurnRequest{ - ID: record.ID, SessionID: record.SessionID, TurnID: record.TurnID, PromptRunID: record.PromptRunID, ModelCallID: record.ModelCallID, - CredentialID: record.CredentialID, ToolCallID: optionalString(record.ToolCallID), - Kind: record.Kind, State: record.State, Request: record.Request, Response: record.Response, - IdempotencyKey: optionalString(record.IdempotencyKey), RequestedBy: optionalString(record.RequestedBy), ResolvedBy: optionalString(record.ResolvedBy), - Reason: optionalString(record.Reason), Version: record.Version, ExpiresAt: record.ExpiresAt, - CreatedAt: record.CreatedAt, ResolvedAt: record.ResolvedAt, - } -} - // cloneToolPolicy copies the policy map, normalizing the legacy spelling rows // written before the tool vocabulary was unified. // diff --git a/pkg/database/prompt_run_iteration_store.go b/pkg/database/prompt_run_iteration_store.go new file mode 100644 index 00000000..be558105 --- /dev/null +++ b/pkg/database/prompt_run_iteration_store.go @@ -0,0 +1,294 @@ +package database + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/flanksource/captain/pkg/api" + "github.com/google/uuid" + "gorm.io/gorm/clause" +) + +// PromptRunIterationState mirrors the captain_prompt_run_iteration_state enum. +type PromptRunIterationState string + +const ( + PromptRunIterationStatePending PromptRunIterationState = "pending" + PromptRunIterationStateRunning PromptRunIterationState = "running" + PromptRunIterationStateSucceeded PromptRunIterationState = "succeeded" + PromptRunIterationStateFailed PromptRunIterationState = "failed" + PromptRunIterationStateCancelled PromptRunIterationState = "cancelled" +) + +func validPromptRunIterationState(value PromptRunIterationState) bool { + switch value { + case PromptRunIterationStatePending, PromptRunIterationStateRunning, PromptRunIterationStateSucceeded, + PromptRunIterationStateFailed, PromptRunIterationStateCancelled: + return true + default: + return false + } +} + +// PromptRunIteration is one attempt of a prompt run: what was asked, what the +// verifier concluded, and the feedback carried into the next attempt. +type PromptRunIteration struct { + ID uuid.UUID `json:"id"` + PromptRunID uuid.UUID `json:"promptRunId"` + Iteration int `json:"iteration"` + State PromptRunIterationState `json:"state"` + Request map[string]any `json:"request,omitempty"` + Feedback string `json:"feedback,omitempty"` + VerificationResult *api.VerifyReport `json:"verificationResult,omitempty"` + Error string `json:"error,omitempty"` + StartedAt *time.Time `json:"startedAt,omitempty"` + FinishedAt *time.Time `json:"finishedAt,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// UpsertPromptRunIterationInput is the mutable state of one iteration. State, +// feedback, verification result and error belong to the newest write, which is +// what makes a crashed loop's replay converge instead of accumulating stale +// verdicts. Request and the timestamps are written only when supplied: a replay +// that omits them leaves what the row (and the state trigger) already holds. +type UpsertPromptRunIterationInput struct { + PromptRunID uuid.UUID + // Iteration is the 1-based loop turn ("iteration 1 of 3"), matching + // captain_prompt_runs.current_iteration. A loop that indexes its turns from + // zero converts before calling; the store rejects anything below 1. + Iteration int + State PromptRunIterationState + Request map[string]any + // Feedback, VerificationResult and Error are last-write-wins: a replay is a + // full statement of the iteration, so passing a nil VerificationResult (or an + // empty Feedback/Error) CLEARS what the row already holds rather than leaving + // it. Restate the verdict on every replay that still stands behind it. + Feedback string + VerificationResult *api.VerifyReport + Error string + StartedAt *time.Time + FinishedAt *time.Time +} + +type promptRunIterationRecord struct { + ID uuid.UUID `gorm:"column:id;type:uuid;primaryKey"` + PromptRunID uuid.UUID `gorm:"column:prompt_run_id;type:uuid"` + Iteration int `gorm:"column:iteration"` + State PromptRunIterationState `gorm:"column:state"` + Request map[string]any `gorm:"column:request;serializer:json;type:jsonb"` + Feedback *string `gorm:"column:feedback"` + VerificationResult *api.VerifyReport `gorm:"column:verification_result;serializer:json;type:jsonb"` + Error *string `gorm:"column:error"` + StartedAt *time.Time `gorm:"column:started_at"` + FinishedAt *time.Time `gorm:"column:finished_at"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` +} + +func (promptRunIterationRecord) TableName() string { return "captain_prompt_run_iterations" } + +// UpsertPromptRunIteration writes one iteration, keyed on (prompt_run_id, +// iteration) so a retried or resumed loop converges on a single row. +func (db *DB) UpsertPromptRunIteration(ctx context.Context, input UpsertPromptRunIterationInput) (PromptRunIteration, error) { + if err := db.requireGorm(); err != nil { + return PromptRunIteration{}, err + } + if err := input.validate(); err != nil { + return PromptRunIteration{}, err + } + record := promptRunIterationRecord{ + ID: uuid.New(), PromptRunID: input.PromptRunID, Iteration: input.Iteration, State: input.State, + Request: input.Request, Feedback: nullableTrimmed(input.Feedback), VerificationResult: input.VerificationResult, + Error: nullableTrimmed(input.Error), StartedAt: input.StartedAt, FinishedAt: input.FinishedAt, + } + if record.Request == nil { + record.Request = map[string]any{} + } + err := db.gorm.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "prompt_run_id"}, {Name: "iteration"}}, + DoUpdates: clause.Assignments(promptRunIterationConflictAssignments(input)), + }).Create(&record).Error + if err != nil { + return PromptRunIteration{}, fmt.Errorf("upsert iteration %d of Captain prompt run %s: %w", + input.Iteration, input.PromptRunID, err) + } + var stored promptRunIterationRecord + err = db.gorm.WithContext(ctx). + First(&stored, "prompt_run_id = ? AND iteration = ?", input.PromptRunID, input.Iteration).Error + if err != nil { + return PromptRunIteration{}, fmt.Errorf("read iteration %d of Captain prompt run %s: %w", + input.Iteration, input.PromptRunID, err) + } + return promptRunIterationFromRecord(stored), nil +} + +func (input UpsertPromptRunIterationInput) validate() error { + if input.PromptRunID == uuid.Nil { + return fmt.Errorf("%w: iteration requires a prompt run ID", ErrInvalidPromptRun) + } + if input.Iteration < 1 { + return fmt.Errorf("%w: iteration %d is out of range; iterations are 1-based", + ErrInvalidPromptRun, input.Iteration) + } + if !validPromptRunIterationState(input.State) { + return fmt.Errorf("%w: unknown iteration state %q", ErrInvalidPromptRun, input.State) + } + if input.VerificationResult != nil { + // A report carries the turn it judged. An unstamped (zero) report inherits + // this row; one stamped for another turn is not this row's verdict. + if stamped := input.VerificationResult.Iteration; stamped != 0 && stamped != input.Iteration { + return fmt.Errorf("%w: verification result is stamped for iteration %d, cannot store it on iteration %d", + ErrInvalidPromptRun, stamped, input.Iteration) + } + if err := input.VerificationResult.Validate(); err != nil { + return fmt.Errorf("%w: iteration %d verification result: %v", ErrInvalidPromptRun, input.Iteration, err) + } + } + if input.StartedAt != nil && input.FinishedAt != nil && input.FinishedAt.Before(*input.StartedAt) { + return fmt.Errorf("%w: iteration %d finished at %s, before it started at %s", + ErrInvalidPromptRun, input.Iteration, input.FinishedAt.UTC(), input.StartedAt.UTC()) + } + return nil +} + +// promptRunIterationConflictAssignments is what a replay of an iteration +// overwrites on the row already there. +func promptRunIterationConflictAssignments(input UpsertPromptRunIterationInput) map[string]any { + assignments := map[string]any{ + "state": excludedValue("state"), + "feedback": excludedValue("feedback"), + "verification_result": excludedValue("verification_result"), + "error": excludedValue("error"), + "updated_at": clause.Expr{SQL: "now()"}, + } + // captain_set_prompt_iteration_state fires BEFORE INSERT OR UPDATE on the + // proposed row and derives started_at/finished_at from its state, so `excluded` + // never carries a NULL timestamp to fall back from — and it is the UPDATE half + // that back-fills finished_at when a terminal replay omits it. Assign only what + // the caller supplied and let the existing row plus that trigger own the rest. + if input.Request != nil { + assignments["request"] = excludedValue("request") + } + if input.StartedAt != nil { + assignments["started_at"] = excludedValue("started_at") + } + if input.FinishedAt != nil { + assignments["finished_at"] = excludedValue("finished_at") + } + return assignments +} + +func excludedValue(column string) clause.Expr { + return clause.Expr{SQL: "excluded." + column} +} + +// ListPromptRunIterations returns a run's attempts in attempt order. +func (db *DB) ListPromptRunIterations(ctx context.Context, runID uuid.UUID) ([]PromptRunIteration, error) { + if err := db.requireGorm(); err != nil { + return nil, err + } + if runID == uuid.Nil { + return nil, fmt.Errorf("%w: prompt run ID is required", ErrInvalidPromptRun) + } + var records []promptRunIterationRecord + err := db.gorm.WithContext(ctx).Where("prompt_run_id = ?", runID).Order("iteration ASC").Find(&records).Error + if err != nil { + return nil, fmt.Errorf("list iterations of Captain prompt run %s: %w", runID, err) + } + iterations := make([]PromptRunIteration, len(records)) + for i := range records { + iterations[i] = promptRunIterationFromRecord(records[i]) + } + return iterations, nil +} + +// LatestPromptRunVerification returns the newest report a run actually produced +// and the iteration that produced it. The newest iteration is often still +// running and carries no verdict, so the newest *report* is not the newest row. +// A run that has never been verified yields (nil, 0, nil): iterations are +// 1-based, so iteration 0 is never a real turn and the zero is unambiguous. +func (db *DB) LatestPromptRunVerification(ctx context.Context, runID uuid.UUID) (*api.VerifyReport, int, error) { + if runID == uuid.Nil { + return nil, 0, fmt.Errorf("%w: prompt run ID is required", ErrInvalidPromptRun) + } + latest, err := db.latestPromptRunVerifications(ctx, []uuid.UUID{runID}) + if err != nil { + return nil, 0, err + } + found, ok := latest[runID] + if !ok { + return nil, 0, nil + } + return found.Report, found.Iteration, nil +} + +// PromptRunVerification is the newest report a run produced and the iteration +// that produced it. +type PromptRunVerification struct { + Iteration int + Report *api.VerifyReport +} + +// LatestPromptRunVerifications is LatestPromptRunVerification for a batch of +// runs, resolved in one query so an attempt listing never fans out per row. A +// run that was never verified is absent from the result. +func (db *DB) LatestPromptRunVerifications(ctx context.Context, runIDs []uuid.UUID) (map[uuid.UUID]PromptRunVerification, error) { + return db.latestPromptRunVerifications(ctx, runIDs) +} + +type promptRunVerification = PromptRunVerification + +type promptRunVerificationRecord struct { + PromptRunID uuid.UUID `gorm:"column:prompt_run_id"` + Iteration int `gorm:"column:iteration"` + VerificationResult json.RawMessage `gorm:"column:verification_result"` +} + +// latestPromptRunVerifications resolves the newest report per run for a whole +// batch of runs in one DISTINCT ON query, so a listing never fans out per row. +func (db *DB) latestPromptRunVerifications(ctx context.Context, runIDs []uuid.UUID) (map[uuid.UUID]promptRunVerification, error) { + if err := db.requireGorm(); err != nil { + return nil, err + } + if len(runIDs) == 0 { + return map[uuid.UUID]promptRunVerification{}, nil + } + var records []promptRunVerificationRecord + // jsonb_typeof excludes a stored JSON `null`, which passes IS NOT NULL and + // would otherwise decode into a zero-valued report that reads as a blank pass. + err := db.gorm.WithContext(ctx).Model(&promptRunIterationRecord{}). + Select("DISTINCT ON (prompt_run_id) prompt_run_id, iteration, verification_result"). + Where("prompt_run_id IN ? AND verification_result IS NOT NULL AND jsonb_typeof(verification_result) <> 'null'", runIDs). + Order("prompt_run_id, iteration DESC"). + Scan(&records).Error + if err != nil { + return nil, fmt.Errorf("read latest verification of %d Captain prompt run(s): %w", len(runIDs), err) + } + latest := make(map[uuid.UUID]promptRunVerification, len(records)) + for _, record := range records { + var report api.VerifyReport + if err := json.Unmarshal(record.VerificationResult, &report); err != nil { + return nil, fmt.Errorf("decode verification result of Captain prompt run %s iteration %d: %w", + record.PromptRunID, record.Iteration, err) + } + if err := report.Validate(); err != nil { + return nil, fmt.Errorf("verification result of Captain prompt run %s iteration %d is corrupt: %w", + record.PromptRunID, record.Iteration, err) + } + latest[record.PromptRunID] = promptRunVerification{Iteration: record.Iteration, Report: &report} + } + return latest, nil +} + +func promptRunIterationFromRecord(record promptRunIterationRecord) PromptRunIteration { + return PromptRunIteration{ + ID: record.ID, PromptRunID: record.PromptRunID, Iteration: record.Iteration, State: record.State, + Request: record.Request, Feedback: optionalString(record.Feedback), + VerificationResult: record.VerificationResult, Error: optionalString(record.Error), + StartedAt: record.StartedAt, FinishedAt: record.FinishedAt, + CreatedAt: record.CreatedAt, UpdatedAt: record.UpdatedAt, + } +} diff --git a/pkg/database/prompt_run_iteration_store_integration_test.go b/pkg/database/prompt_run_iteration_store_integration_test.go new file mode 100644 index 00000000..d9c3d5e7 --- /dev/null +++ b/pkg/database/prompt_run_iteration_store_integration_test.go @@ -0,0 +1,199 @@ +package database + +import ( + "testing" + "time" + + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/commons-db/dbtest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPromptRunIterationUpsertAndLatestVerification(t *testing.T) { + db := openPromptRunIterationDB(t, "captain_prompt_run_iterations") + run := newPromptRunForIterations(t, db) + + report, verified, err := db.LatestPromptRunVerification(t.Context(), run.ID) + require.NoError(t, err) + assert.Nil(t, report, "a run with no iteration has no verification") + assert.Zero(t, verified) + + startedAt := time.Date(2026, time.September, 3, 9, 0, 0, 0, time.UTC) + first, err := db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 1, State: PromptRunIterationStateRunning, + Request: map[string]any{"prompt": "implement the store"}, StartedAt: &startedAt, + }) + require.NoError(t, err) + assert.Equal(t, PromptRunIterationStateRunning, first.State) + + finishedAt := startedAt.Add(90 * time.Second) + replayed, err := db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 1, State: PromptRunIterationStateFailed, + Feedback: "tests failed; retry with the failing case", Error: "exit status 1", FinishedAt: &finishedAt, + }) + require.NoError(t, err) + assert.Equal(t, first.ID, replayed.ID, "(prompt_run_id, iteration) identifies the row; a replay must update it") + assert.Equal(t, PromptRunIterationStateFailed, replayed.State) + assert.Equal(t, "tests failed; retry with the failing case", replayed.Feedback) + assert.Equal(t, "exit status 1", replayed.Error) + assert.Equal(t, map[string]any{"prompt": "implement the store"}, replayed.Request, + "a replay that omits the request must not erase it") + require.NotNil(t, replayed.StartedAt) + assert.Equal(t, startedAt, replayed.StartedAt.UTC(), "a replay that omits started_at must not erase it") + require.NotNil(t, replayed.FinishedAt) + assert.Equal(t, finishedAt, replayed.FinishedAt.UTC()) + + _, err = db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 2, State: PromptRunIterationStateFailed, + VerificationResult: ptr(verifyReportFixture("iteration 2 verification", 2)), + }) + require.NoError(t, err) + + latest := verifyReportFixture("iteration 3 verification", 3) + _, err = db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 3, State: PromptRunIterationStateSucceeded, + VerificationResult: &latest, + }) + require.NoError(t, err) + _, err = db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 4, State: PromptRunIterationStateRunning, + }) + require.NoError(t, err) + + report, verified, err = db.LatestPromptRunVerification(t.Context(), run.ID) + require.NoError(t, err) + require.NotNil(t, report) + assert.Equal(t, 3, verified, "the newest iteration carrying a report wins, not the newest iteration") + assert.Equal(t, latest, *report, "the whole report round-trips, nested children and checklist included") + + var stored string + require.NoError(t, db.Gorm().Raw( + `SELECT verification_result::text FROM captain_prompt_run_iterations WHERE prompt_run_id = ? AND iteration = ?`, + run.ID, 3).Scan(&stored).Error) + assert.Contains(t, stored, `"cel_expression"`, "the stored report keeps the snake_case wire shape") + assert.Contains(t, stored, `"exit_code"`) + + iterations, err := db.ListPromptRunIterations(t.Context(), run.ID) + require.NoError(t, err) + require.Len(t, iterations, 4, "a replay must not insert a second row for the same iteration") + for i, iteration := range iterations { + assert.Equal(t, i+1, iteration.Iteration, "iterations list in ascending 1-based iteration order") + } + + overview, err := db.GetPromptRunOverview(t.Context(), run.ID) + require.NoError(t, err) + require.NotNil(t, overview.LatestVerification) + assert.Equal(t, latest, *overview.LatestVerification) + assert.Equal(t, 4, overview.CurrentIteration, + "captain_sync_prompt_run_iteration tracks the highest 1-based iteration written") +} + +// TestPromptRunIterationTerminalReplayBackfillsFinishedAt pins the UPDATE half of +// captain_prompt_run_iterations_state_before: a replay that reports a terminal +// state without a finished_at still gets one, and does not lose started_at. +func TestPromptRunIterationTerminalReplayBackfillsFinishedAt(t *testing.T) { + db := openPromptRunIterationDB(t, "captain_prompt_run_iteration_backfill") + run := newPromptRunForIterations(t, db) + + // The trigger back-fills finished_at from clock_timestamp(), and + // captain_prompt_run_iterations_time_order rejects finished_at < started_at, so + // the iteration must have started in the past for the back-fill to be legal. + startedAt := time.Now().UTC().Add(-90 * time.Second).Truncate(time.Microsecond) + running, err := db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 1, State: PromptRunIterationStateRunning, StartedAt: &startedAt, + }) + require.NoError(t, err) + require.Nil(t, running.FinishedAt, "a running iteration has not finished") + + finished, err := db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 1, State: PromptRunIterationStateSucceeded, + }) + require.NoError(t, err) + require.NotNil(t, finished.FinishedAt, "the state trigger back-fills finished_at on a terminal UPDATE") + assert.False(t, finished.FinishedAt.Before(startedAt)) + require.NotNil(t, finished.StartedAt) + assert.Equal(t, startedAt, finished.StartedAt.UTC(), "back-filling finished_at must not move started_at") +} + +// TestPromptRunIterationReplayClearsVerificationResult documents the last-write-wins +// contract: a replay is a full statement of the iteration, so dropping the report +// clears it and the latest-verification reader falls back to the earlier turn. +func TestPromptRunIterationReplayClearsVerificationResult(t *testing.T) { + db := openPromptRunIterationDB(t, "captain_prompt_run_iteration_replay") + run := newPromptRunForIterations(t, db) + + firstReport := verifyReportFixture("iteration 1 verification", 1) + _, err := db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 1, State: PromptRunIterationStateFailed, + VerificationResult: &firstReport, Feedback: "retry", Error: "exit status 1", + }) + require.NoError(t, err) + _, err = db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 2, State: PromptRunIterationStateFailed, + VerificationResult: ptr(verifyReportFixture("iteration 2 verification", 2)), + }) + require.NoError(t, err) + + replayed, err := db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 2, State: PromptRunIterationStateRunning, + }) + require.NoError(t, err) + assert.Nil(t, replayed.VerificationResult, "a replay without a report clears the stored one") + + report, verified, err := db.LatestPromptRunVerification(t.Context(), run.ID) + require.NoError(t, err) + require.NotNil(t, report) + assert.Equal(t, 1, verified, "with iteration 2's report cleared the reader falls back to iteration 1") + assert.Equal(t, firstReport, *report) +} + +func openPromptRunIterationDB(t *testing.T, name string) *DB { + t.Helper() + handle := dbtest.ForT(t, dbtest.Options{Name: name}) + db, err := Open(t.Context(), WithDSN(handle.DSN()), WithMigrations()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db +} + +func newPromptRunForIterations(t *testing.T, db *DB) *PromptRun { + t.Helper() + session, err := db.CreateOrGetSession(t.Context(), CreateSessionInput{Source: "codex", Provider: "openai"}) + require.NoError(t, err) + run, err := db.CreatePromptRun(t.Context(), CreatePromptRunInput{SessionID: session.ID}) + require.NoError(t, err) + require.NotNil(t, run) + return run +} + +// verifyReportFixture is a failing two-leaf report: one group node with a passed +// and a failed child, plus a checklist item, so a round-trip exercises nesting, +// pointer fields, and the snake_case context keys. +func verifyReportFixture(name string, iteration int) api.VerifyReport { + itemPassed := false + report := api.VerifyReport{ + Kind: api.VerifyKindCmd, Name: name, Ran: true, Passed: false, + Reason: "1 of 2 tests failed", Feedback: "fix the failing assertion", Iteration: iteration, + State: api.VerifyStateFailed, + Tests: []api.VerifyNode{{ + Name: "pkg/database", + Framework: "go", + Children: []api.VerifyNode{ + {Name: "upsert is idempotent", Passed: true, Duration: 1500 * time.Millisecond}, + {Name: "latest verification wins", Failed: true, Message: "expected 2, got 3", + Context: &api.VerifyNodeContext{ + Command: "go test ./pkg/database/...", ExitCode: 1, + CELExpression: "verify.summary.failed == 0", Expected: float64(0), Actual: float64(1), + }}, + }, + }}, + Checklist: []api.VerifyChecklistItem{ + {Item: "iterations persist their verification", Passed: &itemPassed, Message: "not yet"}, + }, + } + report.Summary = api.SummarizeNodes(report.Tests) + return report +} + +func ptr[T any](value T) *T { return &value } diff --git a/pkg/database/prompt_run_iteration_store_validation_integration_test.go b/pkg/database/prompt_run_iteration_store_validation_integration_test.go new file mode 100644 index 00000000..a74bd516 --- /dev/null +++ b/pkg/database/prompt_run_iteration_store_validation_integration_test.go @@ -0,0 +1,98 @@ +package database + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPromptRunIterationRejectsInvalidInputBeforeWriting(t *testing.T) { + db := openPromptRunIterationDB(t, "captain_prompt_run_iteration_validation") + run := newPromptRunForIterations(t, db) + + inconsistent := verifyReportFixture("inconsistent", 7) + inconsistent.Passed = true // the tree still carries a failure, so State stays "failed" + rejected := []struct { + name string + input UpsertPromptRunIterationInput + }{ + {"negative iteration", UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: -1, State: PromptRunIterationStateRunning}}, + {"zero iteration", UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 0, State: PromptRunIterationStateRunning}}, + {"unknown state", UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 7, State: PromptRunIterationState("verifying")}}, + {"empty prompt run", UpsertPromptRunIterationInput{ + Iteration: 7, State: PromptRunIterationStateRunning}}, + {"self-inconsistent verification report", UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 7, State: PromptRunIterationStateSucceeded, + VerificationResult: &inconsistent}}, + {"report stamped for another iteration", UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 2, State: PromptRunIterationStateSucceeded, + VerificationResult: ptr(verifyReportFixture("iteration 3 verification", 3))}}, + {"finished before started", UpsertPromptRunIterationInput{ + PromptRunID: run.ID, Iteration: 7, State: PromptRunIterationStateSucceeded, + StartedAt: ptr(time.Date(2026, time.September, 3, 9, 0, 0, 0, time.UTC)), + FinishedAt: ptr(time.Date(2026, time.September, 3, 8, 0, 0, 0, time.UTC))}}, + } + for _, tc := range rejected { + t.Run(tc.name, func(t *testing.T) { + _, err := db.UpsertPromptRunIteration(t.Context(), tc.input) + assert.ErrorIs(t, err, ErrInvalidPromptRun) + }) + } + + iterations, err := db.ListPromptRunIterations(t.Context(), run.ID) + require.NoError(t, err) + assert.Empty(t, iterations, "a rejected upsert must not have written a row") + + _, err = db.ListPromptRunIterations(t.Context(), uuid.Nil) + assert.ErrorIs(t, err, ErrInvalidPromptRun) + _, _, err = db.LatestPromptRunVerification(t.Context(), uuid.Nil) + assert.ErrorIs(t, err, ErrInvalidPromptRun) +} + +// TestPromptRunVerificationReaderRejectsCorruptStoredReports covers rows the store +// itself would never write: a JSONB `null` (which passes `IS NOT NULL`) must not +// decode into a blank passing report, and a report that fails Validate must surface +// as an error rather than render as an empty verdict. +func TestPromptRunVerificationReaderRejectsCorruptStoredReports(t *testing.T) { + db := openPromptRunIterationDB(t, "captain_prompt_run_iteration_corrupt") + + jsonNullRun := newPromptRunForIterations(t, db) + insertRawIterationVerification(t, db, jsonNullRun.ID, 1, `null`) + report, verified, err := db.LatestPromptRunVerification(t.Context(), jsonNullRun.ID) + require.NoError(t, err) + assert.Nil(t, report, "a JSONB null is the absence of a report, not a blank passing one") + assert.Zero(t, verified) + + validRun := newPromptRunForIterations(t, db) + valid := verifyReportFixture("iteration 1 verification", 1) + _, err = db.UpsertPromptRunIteration(t.Context(), UpsertPromptRunIterationInput{ + PromptRunID: validRun.ID, Iteration: 1, State: PromptRunIterationStateFailed, VerificationResult: &valid, + }) + require.NoError(t, err) + insertRawIterationVerification(t, db, validRun.ID, 2, `null`) + report, verified, err = db.LatestPromptRunVerification(t.Context(), validRun.ID) + require.NoError(t, err) + require.NotNil(t, report) + assert.Equal(t, 1, verified, "a JSONB-null row is skipped, so the last real report wins") + assert.Equal(t, valid, *report) + + corruptRun := newPromptRunForIterations(t, db) + insertRawIterationVerification(t, db, corruptRun.ID, 1, + `{"kind":"cmd","name":"corrupt","ran":true,"passed":true,"state":"failed","summary":{}}`) + _, _, err = db.LatestPromptRunVerification(t.Context(), corruptRun.ID) + require.Error(t, err, "a stored report that fails Validate must surface, not render as a blank pass") + assert.Contains(t, err.Error(), "passed=true with state") +} + +func insertRawIterationVerification(t *testing.T, db *DB, runID uuid.UUID, iteration int, verification string) { + t.Helper() + require.NoError(t, db.Gorm().Exec( + `INSERT INTO captain_prompt_run_iterations (prompt_run_id, iteration, state, verification_result) + VALUES (?, ?, 'failed', ?::jsonb)`, runID, iteration, verification).Error) +} diff --git a/pkg/database/prompt_run_overview.go b/pkg/database/prompt_run_overview.go index 3e76d651..dd0b06a3 100644 --- a/pkg/database/prompt_run_overview.go +++ b/pkg/database/prompt_run_overview.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/flanksource/captain/pkg/api" "github.com/google/uuid" ) @@ -56,6 +57,10 @@ type PromptRunOverview struct { PID *int64 `json:"pid,omitempty"` DurationMS *int64 `json:"durationMs,omitempty"` ProcessActive bool `json:"processActive"` + // LatestVerification is the newest report the run's iterations produced. The + // overview view projects the newest iteration's result, which is null while + // that iteration is still running; this carries the last actual verdict. + LatestVerification *api.VerifyReport `json:"latestVerification,omitempty"` } type PromptRunOverviewFilter struct { @@ -160,9 +165,34 @@ func (db *DB) ListPromptRunOverviews(ctx context.Context, filter PromptRunOvervi } rows[i] = row } + if err := db.attachLatestVerifications(ctx, rows); err != nil { + return nil, err + } return rows, nil } +// attachLatestVerifications resolves every listed run's newest verification in a +// single batched query rather than one per row. +func (db *DB) attachLatestVerifications(ctx context.Context, rows []PromptRunOverview) error { + if len(rows) == 0 { + return nil + } + ids := make([]uuid.UUID, len(rows)) + for i := range rows { + ids[i] = rows[i].ID + } + latest, err := db.latestPromptRunVerifications(ctx, ids) + if err != nil { + return err + } + for i := range rows { + if found, ok := latest[rows[i].ID]; ok { + rows[i].LatestVerification = found.Report + } + } + return nil +} + func promptRunOverviewFromRecord(record promptRunOverviewRecord) (PromptRunOverview, error) { var runtime PromptRunRuntime if len(record.Runtime) > 0 && string(record.Runtime) != "null" { diff --git a/pkg/database/session_notice_store.go b/pkg/database/session_notice_store.go index 4a0cf1bc..65bc0daf 100644 --- a/pkg/database/session_notice_store.go +++ b/pkg/database/session_notice_store.go @@ -17,6 +17,20 @@ import ( // reader can tell "the harness committed this" from anything the model said. const noticeRole = "system" +// roleForNotice keeps a notice's own kind in the transcript. A verify verdict is +// the run's outcome rather than narration of it, so it is selectable by role — +// "which runs failed verification, and on what" is a query, not a text search. +func roleForNotice(notice api.Notice) string { + switch notice.Kind { + case api.EventVerified: + return session.RoleVerified + case api.EventVerifyFailed: + return session.RoleVerifyFailed + default: + return noticeRole + } +} + // PutSessionNotices records what a run's lifecycle hooks did as transcript // messages, so a commit cut between two turns is readable from the transcript // rather than only from whatever scrolled past in the terminal. @@ -45,7 +59,11 @@ func (db *DB) PutSessionNotices(ctx context.Context, sessionID uuid.UUID, notice if text == "" { continue } - parts, err := json.Marshal([]session.Part{{Type: session.PartText, Text: text}}) + content, err := noticeParts(notice, text) + if err != nil { + return fmt.Errorf("encode notice %d: %w", i, err) + } + parts, err := json.Marshal(content) if err != nil { return fmt.Errorf("encode notice %d: %w", i, err) } @@ -54,7 +72,7 @@ func (db *DB) PutSessionNotices(ctx context.Context, sessionID uuid.UUID, notice at = time.Now().UTC() } records = append(records, messageRecord{ - ID: uuid.New(), SessionID: sessionID, Role: noticeRole, + ID: uuid.New(), SessionID: sessionID, Role: roleForNotice(notice), Parts: parts, OccurredAt: &at, // Stable across replays of the same run, so re-flushing the same // workspace (a resumed run, a retried write) updates rather than @@ -93,6 +111,26 @@ func (db *DB) PutSessionNotices(ctx context.Context, sessionID uuid.UUID, notice return nil } +// noticeParts is the notice as transcript content: its prose, and — for a +// verdict — the typed report beside it under the AI SDK's data-part convention, +// exactly as the live stream carries the two. +// +// Storing only the text made a stored verdict strictly poorer than a live one: a +// reader got "failed in 4ms — verify:go test" and nothing else, while the tree, +// the checklist and the counters the verification panel is built to draw had all +// been thrown away at the point they were written down. +func noticeParts(notice api.Notice, text string) ([]session.Part, error) { + parts := []session.Part{{Type: session.PartText, Text: text}} + if notice.Report == nil { + return parts, nil + } + encoded, err := json.Marshal(notice.Report) + if err != nil { + return nil, fmt.Errorf("encode the verify report of %q: %w", notice.Report.Name, err) + } + return append(parts, session.Part{Type: session.PartVerify, Data: encoded}), nil +} + // noticeMessageID identifies a notice across replays of the same run. The phase // and text alone are not unique — a run commits at PhaseTurn repeatedly with the // same wording — so the index within the flush disambiguates. diff --git a/pkg/database/session_notice_store_integration_test.go b/pkg/database/session_notice_store_integration_test.go index b448a317..4739f429 100644 --- a/pkg/database/session_notice_store_integration_test.go +++ b/pkg/database/session_notice_store_integration_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/session" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -126,3 +127,83 @@ func TestEmptyNoticesWriteNothing(t *testing.T) { require.NoError(t, err) assert.Len(t, messages, len(testIngestBatch(modTime, 2048).Messages)) } + +// A verify verdict is a tree, not a sentence. The notice recorded only its +// headline, so a run whose verdict was read back from the transcript lost the +// checklist, the counters and every failing row — the whole reason the panel +// exists — while the live stream had them all along. +func TestNoticeCarriesItsVerifyReport(t *testing.T) { + db := openIngestTestDB(t) + modTime := time.Now().UTC().Truncate(time.Second) + sess, err := db.IngestTranscript(t.Context(), testIngestBatch(modTime, 2048)) + require.NoError(t, err) + + report := api.NewNodeReport(api.VerifyKindCmd, "verify:go test ./...", api.VerifyNode{ + Name: "go test ./...", Failed: true, Message: "TestFoo failed", + }) + report.Ran, report.Reason, report.Iteration = true, "go test ./... failed", 2 + + require.NoError(t, db.PutSessionNotices(t.Context(), sess.ID, []api.Notice{{ + At: modTime, Phase: "turn", Kind: api.EventVerifyFailed, + Text: "failed in 4ms: go test ./... failed — verify:go test ./...", Report: &report, + }})) + + messages, err := db.ListTranscriptMessages(t.Context(), TranscriptPage{SessionID: sess.ID}) + require.NoError(t, err) + + var verdict *TranscriptMessage + for i := range messages { + if messages[i].Role == session.RoleVerifyFailed { + verdict = &messages[i] + } + } + require.NotNil(t, verdict, "the verdict is stored under its own role") + + var parts []struct { + Type string `json:"type"` + Text string `json:"text"` + Data json.RawMessage `json:"data"` + } + require.NoError(t, json.Unmarshal(verdict.Parts, &parts)) + require.Len(t, parts, 2, "the prose and the report travel together") + assert.Equal(t, "text", parts[0].Type) + assert.Equal(t, "data-verify", parts[1].Type) + + var stored api.VerifyReport + require.NoError(t, json.Unmarshal(parts[1].Data, &stored)) + require.NoError(t, stored.Validate()) + assert.Equal(t, report.Summary, stored.Summary) + assert.Equal(t, 2, stored.Iteration) + assert.Equal(t, api.VerifyStateFailed, stored.State) +} + +// A notice with no report is one text part, exactly as before: the second part +// exists only when there is a report to put in it. +func TestNoticeWithoutAReportStaysOnePart(t *testing.T) { + db := openIngestTestDB(t) + modTime := time.Now().UTC().Truncate(time.Second) + session, err := db.IngestTranscript(t.Context(), testIngestBatch(modTime, 2048)) + require.NoError(t, err) + + require.NoError(t, db.PutSessionNotices(t.Context(), session.ID, []api.Notice{ + {At: modTime, Phase: "turn", Text: "[post-turn] committed abc1234"}, + })) + + messages, err := db.ListTranscriptMessages(t.Context(), TranscriptPage{SessionID: session.ID}) + require.NoError(t, err) + + var found int + for _, message := range messages { + var parts []struct { + Type string `json:"type"` + Text string `json:"text"` + } + require.NoError(t, json.Unmarshal(message.Parts, &parts)) + if len(parts) == 0 || parts[0].Text != "[post-turn] committed abc1234" { + continue + } + found++ + assert.Len(t, parts, 1, "a notice with no report is one text part") + } + assert.Equal(t, 1, found, "the notice is in the transcript") +} diff --git a/pkg/database/tool_approval_store.go b/pkg/database/tool_approval_store.go new file mode 100644 index 00000000..acb6e915 --- /dev/null +++ b/pkg/database/tool_approval_store.go @@ -0,0 +1,334 @@ +package database + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "time" + + "github.com/flanksource/captain/pkg/api" + "github.com/google/uuid" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +var ( + ErrTurnRequestInvalid = errors.New("invalid Captain turn request") + ErrTurnRequestNotFound = errors.New("captain turn request not found") + ErrTurnRequestConflict = errors.New("captain turn request conflict") +) + +type TurnRequestState string + +const ( + TurnRequestStatePending TurnRequestState = "pending" + TurnRequestStateApproved TurnRequestState = "approved" + TurnRequestStateDenied TurnRequestState = "denied" + TurnRequestStateCancelled TurnRequestState = "cancelled" + TurnRequestStateExpired TurnRequestState = "expired" +) + +type TurnRequest struct { + ID uuid.UUID `json:"id"` + SessionID uuid.UUID `json:"sessionId"` + TurnID *uuid.UUID `json:"turnId,omitempty"` + PromptRunID *uuid.UUID `json:"promptRunId,omitempty"` + ModelCallID *uuid.UUID `json:"modelCallId,omitempty"` + CredentialID *uuid.UUID `json:"-"` + ToolCallID string `json:"toolCallId,omitempty"` + Kind string `json:"kind"` + State TurnRequestState `json:"state"` + Request map[string]any `json:"request"` + Response map[string]any `json:"response,omitempty"` + IdempotencyKey string `json:"idempotencyKey,omitempty"` + RequestedBy string `json:"requestedBy,omitempty"` + ResolvedBy string `json:"resolvedBy,omitempty"` + Reason string `json:"reason,omitempty"` + Version int64 `json:"version"` + ExpiresAt *time.Time `json:"expiresAt,omitempty"` + CreatedAt time.Time `json:"createdAt"` + ResolvedAt *time.Time `json:"resolvedAt,omitempty"` +} + +type turnRequestRecord struct { + ID uuid.UUID `gorm:"column:id;type:uuid;primaryKey"` + SessionID uuid.UUID `gorm:"column:session_id;type:uuid"` + TurnID *uuid.UUID `gorm:"column:turn_id;type:uuid"` + PromptRunID *uuid.UUID `gorm:"column:prompt_run_id;type:uuid"` + ModelCallID *uuid.UUID `gorm:"column:model_call_id;type:uuid"` + CredentialID *uuid.UUID `gorm:"column:credential_id;type:uuid"` + ToolCallID *string `gorm:"column:tool_call_id"` + Kind string `gorm:"column:kind"` + State TurnRequestState `gorm:"column:state"` + Request map[string]any `gorm:"column:request;serializer:json;type:jsonb"` + Response map[string]any `gorm:"column:response;serializer:json;type:jsonb"` + IdempotencyKey *string `gorm:"column:idempotency_key"` + RequestedBy *string `gorm:"column:requested_by"` + ResolvedBy *string `gorm:"column:resolved_by"` + Reason *string `gorm:"column:reason"` + Version int64 `gorm:"column:version"` + ExpiresAt *time.Time `gorm:"column:expires_at"` + CreatedAt time.Time `gorm:"column:created_at"` + ResolvedAt *time.Time `gorm:"column:resolved_at"` +} + +func (turnRequestRecord) TableName() string { return "captain_turn_requests" } + +// CreateToolApprovalRequestInput describes one durable tool approval. TurnID and +// ModelCallID are required on the caller-tool path (CredentialID set) and +// optional on the credential-less provider path, where a streaming provider or +// an external host has a session and a prompt run but never opens a turn or a +// model call. uuid.Nil writes NULL. +type CreateToolApprovalRequestInput struct { + CredentialID uuid.UUID + SessionID uuid.UUID + TurnID uuid.UUID + PromptRunID uuid.UUID + ModelCallID uuid.UUID + ToolCallID string + Tool string + Input map[string]any + RequestedBy string + ExpiresAt time.Time +} + +func (db *DB) CreateToolApprovalRequest( + ctx context.Context, + input CreateToolApprovalRequestInput, +) (*TurnRequest, error) { + var credential *CallerToolCredential + if input.CredentialID != uuid.Nil { + if err := db.ValidateCallerToolCredential(ctx, input.CredentialID); err != nil { + return nil, err + } + var err error + credential, err = db.GetCallerToolCredential(ctx, input.CredentialID) + if err != nil { + return nil, err + } + if credential.SessionID != input.SessionID || credential.PromptRunID != input.PromptRunID { + return nil, fmt.Errorf("%w: credential does not belong to the supplied session and run", ErrTurnRequestInvalid) + } + } + input.ToolCallID = strings.TrimSpace(input.ToolCallID) + input.Tool = strings.TrimSpace(input.Tool) + if input.SessionID == uuid.Nil || input.PromptRunID == uuid.Nil || + input.ToolCallID == "" || input.Tool == "" || !input.ExpiresAt.After(time.Now()) { + return nil, fmt.Errorf("%w: session, prompt run, tool call, tool, and future expiry are required", ErrTurnRequestInvalid) + } + if credential != nil && (input.TurnID == uuid.Nil || input.ModelCallID == uuid.Nil) { + return nil, fmt.Errorf("%w: a caller-tool approval requires its turn and model call", ErrTurnRequestInvalid) + } + if credential != nil && credential.Policy[input.Tool] != api.ToolPolicyAsk { + return nil, fmt.Errorf("%w: tool %q is not approved by ask policy", ErrTurnRequestInvalid, input.Tool) + } + if credential != nil && credential.ExpiresAt != nil && input.ExpiresAt.After(*credential.ExpiresAt) { + input.ExpiresAt = *credential.ExpiresAt + } + idempotencyKey := "provider:" + input.PromptRunID.String() + ":" + input.ToolCallID + if credential != nil { + idempotencyKey = "mcp:" + input.CredentialID.String() + ":" + input.ToolCallID + } + request := map[string]any{ + "tool": input.Tool, "input": input.Input, + } + var credentialID *uuid.UUID + if credential != nil { + credentialID = &input.CredentialID + } + record := turnRequestRecord{ + ID: uuid.New(), SessionID: input.SessionID, TurnID: nullableUUID(input.TurnID), PromptRunID: &input.PromptRunID, + ModelCallID: nullableUUID(input.ModelCallID), CredentialID: credentialID, ToolCallID: &input.ToolCallID, + Kind: "tool_approval", State: TurnRequestStatePending, Request: request, + IdempotencyKey: &idempotencyKey, RequestedBy: nullableTrimmed(input.RequestedBy), ExpiresAt: &input.ExpiresAt, + } + result := db.gorm.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(&record) + if result.Error != nil { + return nil, fmt.Errorf("create tool approval request: %w", result.Error) + } + if result.RowsAffected == 1 { + if err := db.touchChatSession(ctx, input.SessionID); err != nil { + return nil, err + } + return db.GetTurnRequest(ctx, record.ID) + } + var existing turnRequestRecord + if err := db.gorm.WithContext(ctx). + Where("session_id = ? AND idempotency_key = ?", input.SessionID, idempotencyKey). + First(&existing).Error; err != nil { + return nil, fmt.Errorf("read existing tool approval request: %w", err) + } + if !reflect.DeepEqual(existing.Request, request) { + return nil, fmt.Errorf("%w: tool call %q was retried with different input", ErrTurnRequestConflict, input.ToolCallID) + } + out := turnRequestFromRecord(existing) + return &out, nil +} + +type ResolveToolApprovalRequestInput struct { + SessionID uuid.UUID + RequestID uuid.UUID + ExpectedTurnID *uuid.UUID + Approved bool + UpdatedInput map[string]any + ResolvedBy string + Reason string +} + +func (db *DB) ResolveToolApprovalRequest( + ctx context.Context, + input ResolveToolApprovalRequestInput, +) (*TurnRequest, error) { + if input.SessionID == uuid.Nil || input.RequestID == uuid.Nil { + return nil, fmt.Errorf("%w: session and approval request IDs are required", ErrTurnRequestInvalid) + } + state := TurnRequestStateDenied + var response map[string]any + if input.Approved { + state = TurnRequestStateApproved + if input.UpdatedInput != nil { + response = map[string]any{"updatedInput": input.UpdatedInput} + } + } + var pending turnRequestRecord + if err := db.gorm.WithContext(ctx). + Where("id = ? AND session_id = ? AND kind = 'tool_approval'", input.RequestID, input.SessionID). + First(&pending).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, fmt.Errorf("%w: session %s approval %s", ErrTurnRequestNotFound, input.SessionID, input.RequestID) + } + return nil, fmt.Errorf("read tool approval request: %w", err) + } + if pending.State != TurnRequestStatePending { + if pending.State == state && reflect.DeepEqual(pending.Response, response) && + optionalString(pending.Reason) == strings.TrimSpace(input.Reason) { + out := turnRequestFromRecord(pending) + return &out, nil + } + return nil, fmt.Errorf("%w: approval %s already has a different %s decision", ErrTurnRequestConflict, pending.ID, pending.State) + } + if input.ExpectedTurnID != nil && (pending.TurnID == nil || *pending.TurnID != *input.ExpectedTurnID) { + return nil, fmt.Errorf("%w: approval %s does not belong to active turn %s", ErrTurnRequestConflict, pending.ID, *input.ExpectedTurnID) + } + now := time.Now().UTC() + result := db.gorm.WithContext(ctx).Model(&turnRequestRecord{}). + Where("id = ? AND state = 'pending'", pending.ID). + Where(`credential_id IS NOT NULL OR EXISTS ( + SELECT 1 FROM captain_prompt_runs run + WHERE run.id = captain_turn_requests.prompt_run_id AND run.state = 'waiting' + )`). + Where(`credential_id IS NULL OR EXISTS ( + SELECT 1 FROM captain_session_mcp_credentials credential + WHERE credential.id = captain_turn_requests.credential_id + AND credential.revoked_at IS NULL + AND (credential.expires_at IS NULL OR credential.expires_at > ?) + )`, now). + Updates(map[string]any{ + "state": state, "response": response, "resolved_by": nullableTrimmed(input.ResolvedBy), + "reason": nullableTrimmed(input.Reason), "resolved_at": now, + }) + if result.Error != nil { + return nil, fmt.Errorf("resolve tool approval request: %w", result.Error) + } + if result.RowsAffected == 0 { + if pending.CredentialID == nil { + return nil, fmt.Errorf("%w: approval %s cannot be resolved before its prompt run is waiting", ErrTurnRequestConflict, pending.ID) + } + if pending.CredentialID != nil { + if err := db.ValidateCallerToolCredential(ctx, *pending.CredentialID); err != nil { + return nil, err + } + } + return nil, fmt.Errorf("%w: session %s approval %s", ErrTurnRequestNotFound, input.SessionID, input.RequestID) + } + if err := db.touchChatSession(ctx, input.SessionID); err != nil { + return nil, err + } + return db.GetTurnRequest(ctx, pending.ID) +} + +func (db *DB) GetTurnRequest(ctx context.Context, id uuid.UUID) (*TurnRequest, error) { + var record turnRequestRecord + if err := db.gorm.WithContext(ctx).First(&record, "id = ?", id).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, fmt.Errorf("%w: %s", ErrTurnRequestNotFound, id) + } + return nil, fmt.Errorf("get Captain turn request: %w", err) + } + out := turnRequestFromRecord(record) + return &out, nil +} + +type TurnRequestFilter struct { + SessionID uuid.UUID + PromptRunID *uuid.UUID +} + +func (db *DB) ListTurnRequests(ctx context.Context, filter TurnRequestFilter) ([]TurnRequest, error) { + if filter.SessionID == uuid.Nil { + return nil, fmt.Errorf("%w: session ID is required", ErrTurnRequestInvalid) + } + query := db.gorm.WithContext(ctx).Where("session_id = ?", filter.SessionID).Order("created_at, id") + if filter.PromptRunID != nil { + query = query.Where("prompt_run_id = ?", *filter.PromptRunID) + } + var records []turnRequestRecord + if err := query.Find(&records).Error; err != nil { + return nil, fmt.Errorf("list Captain turn requests: %w", err) + } + requests := make([]TurnRequest, len(records)) + for i := range records { + requests[i] = turnRequestFromRecord(records[i]) + } + return requests, nil +} + +func (db *DB) ExpireToolApprovalRequest(ctx context.Context, id uuid.UUID, state TurnRequestState, reason string) error { + if state != TurnRequestStateExpired && state != TurnRequestStateCancelled { + return fmt.Errorf("%w: terminal state %q is invalid", ErrTurnRequestInvalid, state) + } + now := time.Now().UTC() + result := db.gorm.WithContext(ctx).Model(&turnRequestRecord{}). + Where("id = ? AND state = 'pending'", id). + Updates(map[string]any{"state": state, "reason": nullableTrimmed(reason), "resolved_at": now}) + if result.Error != nil { + return fmt.Errorf("expire tool approval request: %w", result.Error) + } + return nil +} + +func (db *DB) CancelPendingTurnRequests(ctx context.Context, sessionID, promptRunID uuid.UUID, reason string) error { + if sessionID == uuid.Nil || promptRunID == uuid.Nil { + return fmt.Errorf("%w: session and prompt run IDs are required", ErrTurnRequestInvalid) + } + now := time.Now().UTC() + result := db.gorm.WithContext(ctx).Model(&turnRequestRecord{}). + Where("session_id = ? AND prompt_run_id = ? AND state = 'pending'", sessionID, promptRunID). + Updates(map[string]any{ + "state": TurnRequestStateCancelled, "reason": nullableTrimmed(reason), "resolved_at": now, + }) + if result.Error != nil { + return fmt.Errorf("cancel pending Captain turn requests: %w", result.Error) + } + return nil +} + +func nullableUUID(id uuid.UUID) *uuid.UUID { + if id == uuid.Nil { + return nil + } + return &id +} + +func turnRequestFromRecord(record turnRequestRecord) TurnRequest { + return TurnRequest{ + ID: record.ID, SessionID: record.SessionID, TurnID: record.TurnID, PromptRunID: record.PromptRunID, ModelCallID: record.ModelCallID, + CredentialID: record.CredentialID, ToolCallID: optionalString(record.ToolCallID), + Kind: record.Kind, State: record.State, Request: record.Request, Response: record.Response, + IdempotencyKey: optionalString(record.IdempotencyKey), RequestedBy: optionalString(record.RequestedBy), ResolvedBy: optionalString(record.ResolvedBy), + Reason: optionalString(record.Reason), Version: record.Version, ExpiresAt: record.ExpiresAt, + CreatedAt: record.CreatedAt, ResolvedAt: record.ResolvedAt, + } +} diff --git a/pkg/dod/cache.go b/pkg/dod/cache.go deleted file mode 100644 index db109c56..00000000 --- a/pkg/dod/cache.go +++ /dev/null @@ -1,83 +0,0 @@ -package dod - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/flanksource/captain/pkg/claude" -) - -type CommandResult struct { - Command string `json:"command"` - ExitCode int `json:"exit_code"` - Passed bool `json:"passed"` - Stdout string `json:"stdout,omitempty"` - Stderr string `json:"stderr,omitempty"` -} - -type LastRun struct { - At time.Time `json:"at"` - Results []CommandResult `json:"results"` -} - -type DodFile struct { - Commands []string `json:"commands"` - Workdir string `json:"workdir"` - Timeout int `json:"timeout"` - CreatedAt time.Time `json:"created_at"` - LastRun *LastRun `json:"last_run,omitempty"` -} - -func GetDodDir() string { - return filepath.Join(claude.GetClaudeHome(), "dod") -} - -func CachePath(sessionID string) string { - return filepath.Join(GetDodDir(), sessionID+".json") -} - -func Read(sessionID string) (*DodFile, error) { - data, err := os.ReadFile(CachePath(sessionID)) - if err != nil { - return nil, err - } - var dod DodFile - if err := json.Unmarshal(data, &dod); err != nil { - return nil, fmt.Errorf("corrupt dod file: %w", err) - } - return &dod, nil -} - -func Write(sessionID string, dod *DodFile) error { - dir := GetDodDir() - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("creating dod dir: %w", err) - } - - data, err := json.MarshalIndent(dod, "", " ") - if err != nil { - return err - } - - tmp := CachePath(sessionID) + ".tmp" - if err := os.WriteFile(tmp, data, 0644); err != nil { - return err - } - return os.Rename(tmp, CachePath(sessionID)) -} - -func Delete(sessionID string) error { - path := CachePath(sessionID) - if err := os.Remove(path); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -func Exists(sessionID string) bool { - _, err := os.Stat(CachePath(sessionID)) - return err == nil -} diff --git a/pkg/dod/cache_test.go b/pkg/dod/cache_test.go deleted file mode 100644 index 9fd1ee18..00000000 --- a/pkg/dod/cache_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package dod - -import ( - "os" - "path/filepath" - "testing" - "time" -) - -func TestWriteReadDelete(t *testing.T) { - tmpDir := t.TempDir() - t.Setenv("HOME", tmpDir) - - sessionID := "test-session-123" - - dod := &DodFile{ - Commands: []string{"make test", "make lint"}, - Workdir: "/tmp/project", - Timeout: 300, - CreatedAt: time.Now().UTC().Truncate(time.Second), - } - - if err := Write(sessionID, dod); err != nil { - t.Fatalf("Write: %v", err) - } - - if _, err := os.Stat(filepath.Join(tmpDir, ".claude", "dod", sessionID+".json")); err != nil { - t.Fatalf("cache file not created: %v", err) - } - - got, err := Read(sessionID) - if err != nil { - t.Fatalf("Read: %v", err) - } - - if len(got.Commands) != 2 || got.Commands[0] != "make test" || got.Commands[1] != "make lint" { - t.Errorf("commands mismatch: %v", got.Commands) - } - if got.Workdir != "/tmp/project" { - t.Errorf("workdir = %q, want /tmp/project", got.Workdir) - } - if got.Timeout != 300 { - t.Errorf("timeout = %d, want 300", got.Timeout) - } - if !got.CreatedAt.Equal(dod.CreatedAt) { - t.Errorf("created_at = %v, want %v", got.CreatedAt, dod.CreatedAt) - } - - if !Exists(sessionID) { - t.Error("Exists returned false for existing file") - } - - if err := Delete(sessionID); err != nil { - t.Fatalf("Delete: %v", err) - } - - if Exists(sessionID) { - t.Error("Exists returned true after deletion") - } -} - -func TestReadNonExistent(t *testing.T) { - tmpDir := t.TempDir() - t.Setenv("HOME", tmpDir) - - _, err := Read("nonexistent") - if err == nil { - t.Error("expected error for nonexistent session") - } -} - -func TestWriteOverwrites(t *testing.T) { - tmpDir := t.TempDir() - t.Setenv("HOME", tmpDir) - - sessionID := "overwrite-test" - - first := &DodFile{Commands: []string{"make test"}, Workdir: "/a", Timeout: 60, CreatedAt: time.Now().UTC()} - if err := Write(sessionID, first); err != nil { - t.Fatalf("Write first: %v", err) - } - - second := &DodFile{Commands: []string{"make lint"}, Workdir: "/b", Timeout: 120, CreatedAt: time.Now().UTC()} - if err := Write(sessionID, second); err != nil { - t.Fatalf("Write second: %v", err) - } - - got, err := Read(sessionID) - if err != nil { - t.Fatalf("Read: %v", err) - } - if len(got.Commands) != 1 || got.Commands[0] != "make lint" { - t.Errorf("expected overwritten commands, got %v", got.Commands) - } - if got.Workdir != "/b" { - t.Errorf("workdir = %q, want /b", got.Workdir) - } -} - -func TestDeleteNonExistent(t *testing.T) { - tmpDir := t.TempDir() - t.Setenv("HOME", tmpDir) - - if err := Delete("does-not-exist"); err != nil { - t.Errorf("Delete nonexistent should not error: %v", err) - } -} - -func TestWriteWithLastRun(t *testing.T) { - tmpDir := t.TempDir() - t.Setenv("HOME", tmpDir) - - sessionID := "lastrun-test" - now := time.Now().UTC().Truncate(time.Second) - - dod := &DodFile{ - Commands: []string{"make test"}, - Workdir: "/tmp", - Timeout: 300, - CreatedAt: now, - LastRun: &LastRun{ - At: now, - Results: []CommandResult{ - {Command: "make test", ExitCode: 1, Passed: false, Stderr: "FAIL"}, - }, - }, - } - - if err := Write(sessionID, dod); err != nil { - t.Fatalf("Write: %v", err) - } - - got, err := Read(sessionID) - if err != nil { - t.Fatalf("Read: %v", err) - } - if got.LastRun == nil { - t.Fatal("LastRun is nil") - } - if len(got.LastRun.Results) != 1 { - t.Fatalf("expected 1 result, got %d", len(got.LastRun.Results)) - } - if got.LastRun.Results[0].ExitCode != 1 || got.LastRun.Results[0].Passed { - t.Errorf("unexpected result: %+v", got.LastRun.Results[0]) - } -} diff --git a/pkg/dod/run.go b/pkg/dod/run.go deleted file mode 100644 index 09b84c0e..00000000 --- a/pkg/dod/run.go +++ /dev/null @@ -1,83 +0,0 @@ -package dod - -import ( - "bytes" - "context" - "fmt" - "os/exec" - "strings" - "time" -) - -const maxOutputLines = 100 - -func RunCommands(dod *DodFile) *LastRun { - run := &LastRun{At: time.Now().UTC()} - for _, cmd := range dod.Commands { - result := runCommand(cmd, dod.Workdir, time.Duration(dod.Timeout)*time.Second) - run.Results = append(run.Results, result) - if !result.Passed { - break // fail-fast - } - } - return run -} - -func runCommand(command, workdir string, timeout time.Duration) CommandResult { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - cmd := exec.CommandContext(ctx, "sh", "-c", command) - cmd.Dir = workdir - - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err := cmd.Run() - exitCode := 0 - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - exitCode = exitErr.ExitCode() - } else if ctx.Err() == context.DeadlineExceeded { - exitCode = -1 - } else { - exitCode = -1 - } - } - - return CommandResult{ - Command: command, - ExitCode: exitCode, - Passed: exitCode == 0, - Stdout: truncateOutput(stdout.String()), - Stderr: truncateOutput(stderr.String()), - } -} - -func truncateOutput(s string) string { - lines := strings.Split(s, "\n") - if len(lines) <= maxOutputLines { - return s - } - return fmt.Sprintf("... (%d lines truncated)\n%s", len(lines)-maxOutputLines, strings.Join(lines[len(lines)-maxOutputLines:], "\n")) -} - -func FormatFailureMessage(run *LastRun) string { - var b strings.Builder - b.WriteString("DoD checks failed. Fix the issues and try again.\n\n") - for _, r := range run.Results { - if r.Passed { - fmt.Fprintf(&b, "PASS: %s\n", r.Command) - continue - } - fmt.Fprintf(&b, "FAIL: %s (exit code %d)\n", r.Command, r.ExitCode) - if r.Stdout != "" { - fmt.Fprintf(&b, "stdout:\n%s\n", r.Stdout) - } - if r.Stderr != "" { - fmt.Fprintf(&b, "stderr:\n%s\n", r.Stderr) - } - } - return b.String() -} diff --git a/pkg/dod/run_test.go b/pkg/dod/run_test.go deleted file mode 100644 index 148c5bb5..00000000 --- a/pkg/dod/run_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package dod - -import ( - "strings" - "testing" -) - -func TestRunCommandsPass(t *testing.T) { - dod := &DodFile{ - Commands: []string{"echo hello", "echo world"}, - Workdir: t.TempDir(), - Timeout: 10, - } - - run := RunCommands(dod) - if len(run.Results) != 2 { - t.Fatalf("expected 2 results, got %d", len(run.Results)) - } - for i, r := range run.Results { - if !r.Passed { - t.Errorf("result[%d] %q failed: exit=%d stderr=%q", i, r.Command, r.ExitCode, r.Stderr) - } - } -} - -func TestRunCommandsFailFast(t *testing.T) { - dod := &DodFile{ - Commands: []string{"false", "echo should-not-run"}, - Workdir: t.TempDir(), - Timeout: 10, - } - - run := RunCommands(dod) - if len(run.Results) != 1 { - t.Fatalf("expected 1 result (fail-fast), got %d", len(run.Results)) - } - if run.Results[0].Passed { - t.Error("expected first command to fail") - } -} - -func TestRunCommandsCapturesOutput(t *testing.T) { - dod := &DodFile{ - Commands: []string{"echo out-text && echo err-text >&2 && exit 1"}, - Workdir: t.TempDir(), - Timeout: 10, - } - - run := RunCommands(dod) - if len(run.Results) != 1 { - t.Fatalf("expected 1 result, got %d", len(run.Results)) - } - r := run.Results[0] - if !strings.Contains(r.Stdout, "out-text") { - t.Errorf("stdout = %q, want containing 'out-text'", r.Stdout) - } - if !strings.Contains(r.Stderr, "err-text") { - t.Errorf("stderr = %q, want containing 'err-text'", r.Stderr) - } -} - -func TestRunCommandsTimeout(t *testing.T) { - dod := &DodFile{ - Commands: []string{"sleep 60"}, - Workdir: t.TempDir(), - Timeout: 1, - } - - run := RunCommands(dod) - if len(run.Results) != 1 { - t.Fatalf("expected 1 result, got %d", len(run.Results)) - } - if run.Results[0].Passed { - t.Error("expected timeout to cause failure") - } -} - -func TestTruncateOutput(t *testing.T) { - lines := make([]string, 200) - for i := range lines { - lines[i] = "line" - } - input := strings.Join(lines, "\n") - result := truncateOutput(input) - if !strings.Contains(result, "truncated") { - t.Error("expected truncation notice") - } - - short := "just a few lines" - if truncateOutput(short) != short { - t.Error("short output should not be truncated") - } -} - -func TestFormatFailureMessage(t *testing.T) { - run := &LastRun{ - Results: []CommandResult{ - {Command: "make test", ExitCode: 0, Passed: true}, - {Command: "make lint", ExitCode: 1, Passed: false, Stderr: "lint error"}, - }, - } - msg := FormatFailureMessage(run) - if !strings.Contains(msg, "PASS: make test") { - t.Error("missing pass line") - } - if !strings.Contains(msg, "FAIL: make lint") { - t.Error("missing fail line") - } - if !strings.Contains(msg, "lint error") { - t.Error("missing stderr content") - } -} diff --git a/pkg/dod/skill.go b/pkg/dod/skill.go deleted file mode 100644 index c9aafe26..00000000 --- a/pkg/dod/skill.go +++ /dev/null @@ -1,15 +0,0 @@ -package dod - -import _ "embed" - -//go:embed skills/dod.md -var SkillDod string - -//go:embed skills/dod-clear.md -var SkillDodClear string - -//go:embed skills/dod-status.md -var SkillDodStatus string - -//go:embed skills/dod-run.md -var SkillDodRun string diff --git a/pkg/dod/skills/dod-clear.md b/pkg/dod/skills/dod-clear.md deleted file mode 100644 index d67ee03d..00000000 --- a/pkg/dod/skills/dod-clear.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: dod-clear -description: "Clear the Definition of Done for the current session" -allowed-tools: [Bash] ---- - -# Clear Definition of Done - -Clear the DoD so the stop hook is no longer enforced. - -## What to do - -Run: -```bash -captain dod clear --session-id "" -``` - -Confirm to the user that the DoD has been cleared. diff --git a/pkg/dod/skills/dod-run.md b/pkg/dod/skills/dod-run.md deleted file mode 100644 index 64013e8e..00000000 --- a/pkg/dod/skills/dod-run.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: dod-run -description: "Manually run DoD checks to see current pass/fail state" -allowed-tools: [Bash] ---- - -# Run DoD Checks - -Manually run all DoD commands to see their current status without triggering a stop. - -## What to do - -Run: -```bash -captain dod run --session-id "" -``` - -Display the results showing pass/fail for each command with output. diff --git a/pkg/dod/skills/dod-status.md b/pkg/dod/skills/dod-status.md deleted file mode 100644 index f117eaa2..00000000 --- a/pkg/dod/skills/dod-status.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: dod-status -description: "Show the current Definition of Done commands and last run results" -allowed-tools: [Bash] ---- - -# DoD Status - -Show the current DoD state. - -## What to do - -Run: -```bash -captain dod status --session-id "" -``` - -Display the results showing the registered commands, when they were set, and last run results if available. diff --git a/pkg/dod/skills/dod.md b/pkg/dod/skills/dod.md deleted file mode 100644 index 6c7c7e3f..00000000 --- a/pkg/dod/skills/dod.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: dod -description: "Set Definition of Done commands that must pass before Claude can stop. Usage: /dod make test lint" -allowed-tools: [Bash] ---- - -# Definition of Done (DoD) - -You are setting up a Definition of Done gate. The user has provided commands that must pass before you (Claude) can stop working. - -## What to do - -The user typed `/dod $ARGUMENTS`. Parse the arguments as shell command(s) and register them using `captain dod set`. - -**Steps:** - -1. Extract the session ID from the environment. The session ID is available from the transcript path or can be derived. Use the hook input's `session_id` field if available, otherwise find it from `~/.claude/projects/` session files for the current directory. - -2. Run the following command to register the DoD: - -```bash -captain dod set --session-id "" --workdir "$(pwd)" $ARGUMENTS -``` - -Where `$ARGUMENTS` are the commands the user provided (e.g., `make test lint`). - -3. Confirm to the user what DoD has been set. - -**Important:** -- If `$ARGUMENTS` is empty, run `captain dod status --session-id ""` instead to show current DoD. -- The session ID should be extracted from the `CLAUDE_SESSION_ID` environment variable, or from the transcript path, or by listing recent session files. - -## Example - -User types: `/dod make test lint` - -You run: -```bash -captain dod set --session-id "abc-123" --workdir "/path/to/project" "make test lint" -``` - -Then confirm: "DoD set: `make test lint` must pass before I can stop." diff --git a/pkg/gitagent/hookmain.go b/pkg/gitagent/hookmain.go index 235b6962..b4db526a 100644 --- a/pkg/gitagent/hookmain.go +++ b/pkg/gitagent/hookmain.go @@ -414,7 +414,7 @@ func vetTree(ctx context.Context, repo string, req vetRequest) TierVerdict { // was computed for any other directory would confine the wrong thing. A // factory failure is fail-closed (R5.2): status error, and error rejects. var wrap verify.CommandWrapFunc - if req.host.WrapFor != nil && len(verify.HooksForWorkflow(req.workflow)) > 0 { + if req.host.WrapFor != nil && verify.DeclaresExec(req.workflow) { wrapped, closeWrap, err := req.host.WrapFor(ctx, dir) if err != nil { verdict.Findings = append(verdict.Findings, Finding{Hook: "hookset", Kind: "exec", diff --git a/pkg/gitagent/hookset.go b/pkg/gitagent/hookset.go index 9d1d4ca4..bd5e6913 100644 --- a/pkg/gitagent/hookset.go +++ b/pkg/gitagent/hookset.go @@ -60,7 +60,7 @@ func RunHookSet(ctx context.Context, ws HookWorkspace, opts HookSetOptions) Tier verdict.Findings = append(verdict.Findings, finding) return verdict } - plugins, errFinding := buildHookPlugins(wf, opts) + plugins, errFinding := buildHookPlugins(ctx, wf, opts) if errFinding != nil { verdict.Status = StatusError verdict.Findings = append(verdict.Findings, *errFinding) @@ -106,48 +106,44 @@ func runCommitGates(ws HookWorkspace, wf *api.Workflow) (Finding, bool) { return Finding{}, false } -// buildHookPlugins assembles exec and prompt verifiers via the same builders -// the local run path uses (A5.1: one hook machinery), confining every exec -// hook and bounding prompt-hook recursion. -func buildHookPlugins(wf *api.Workflow, opts HookSetOptions) ([]*verify.Plugin, *Finding) { - var plugins []*verify.Plugin - execHooks := verify.HooksForWorkflow(wf) - if len(execHooks) > 0 && opts.Wrap == nil { +// buildHookPlugins assembles every declared verifier through the registry the +// local run path uses (A5.1: one hook machinery), confining every process the +// checks start and bounding prompt-hook recursion. +func buildHookPlugins(ctx context.Context, wf *api.Workflow, opts HookSetOptions) ([]*verify.Plugin, *Finding) { + if verify.DeclaresExec(wf) && opts.Wrap == nil { return nil, &Finding{ Hook: "hookset", Kind: "exec", Message: "exec hooks require a wrap-command sandbox; refusing to run agent-authored commands on the host (R5.2)", } } - timeout := opts.Timeout - if timeout <= 0 { - timeout = DefaultHookTimeout - } - for _, h := range execHooks { - p, ok := h.(*verify.Plugin) - if !ok { - return nil, &Finding{Hook: "hookset", Kind: "exec", Message: fmt.Sprintf("unexpected hook type %T", h)} - } - if cv, ok := p.Verifier().(*verify.CmdVerifier); ok { - cv.Env = opts.Env - cv.Wrap = opts.Wrap - cv.Timeout = timeout - } - plugins = append(plugins, p) - } if wf.Verify != nil && len(wf.Verify.Prompts) > 0 && opts.Depth+1 > MaxHookDepth { return nil, &Finding{ Hook: "hookset", Kind: "prompt", Message: fmt.Sprintf("prompt hooks at depth %d exceed the recursion bound %d (R5.4/H15)", opts.Depth+1, MaxHookDepth), } } - judgeHooks, err := verify.PromptHooksForWorkflow(wf, opts.Judge) + timeout := opts.Timeout + if timeout <= 0 { + timeout = DefaultHookTimeout + } + hooks, err := verify.HooksFor(ctx, wf, verify.Options{ + Provider: opts.Judge, Env: opts.Env, Wrap: opts.Wrap, Timeout: timeout, + }) if err != nil { - return nil, &Finding{Hook: "hookset", Kind: "prompt", Message: err.Error()} + // The only factories that refuse to build are the judge (a prompt that + // will not load, or no provider to judge it) and the fixture guard, so a + // workflow declaring prompts names them as the finding's kind. + kind := "exec" + if wf.Verify != nil && len(wf.Verify.Prompts) > 0 { + kind = "prompt" + } + return nil, &Finding{Hook: "hookset", Kind: kind, Message: err.Error()} } - for _, h := range judgeHooks { + var plugins []*verify.Plugin + for _, h := range hooks { p, ok := h.(*verify.Plugin) if !ok { - return nil, &Finding{Hook: "hookset", Kind: "prompt", Message: fmt.Sprintf("unexpected hook type %T", h)} + return nil, &Finding{Hook: "hookset", Kind: "exec", Message: fmt.Sprintf("unexpected hook type %T", h)} } plugins = append(plugins, p) } diff --git a/pkg/gitagent/hookset_ginkgo_test.go b/pkg/gitagent/hookset_ginkgo_test.go index 524efde3..40393499 100644 --- a/pkg/gitagent/hookset_ginkgo_test.go +++ b/pkg/gitagent/hookset_ginkgo_test.go @@ -127,6 +127,25 @@ var _ = Describe("hook sets", func() { Expect(v.Findings[0].Message).To(ContainSubstring("R5.2")) }) + It("refuses a fixture the same way, and never accepts one it cannot run", func() { + wf := &api.Workflow{Verify: &api.Verify{Fixture: "# acceptance\n- [ ] it works\n"}} + + v := gitagent.RunHookSet(ctx, ws(), gitagent.HookSetOptions{ + Task: "t-1", Attempt: 1, Tier: "sidecar", Workflow: wf, + }) + Expect(v.Status).To(Equal(gitagent.StatusError)) + Expect(v.Findings[0].Message).To(ContainSubstring("R5.2"), + "a fixture runs a process too, so it needs the same confinement") + + v = gitagent.RunHookSet(ctx, ws(), gitagent.HookSetOptions{ + Task: "t-1", Attempt: 1, Tier: "sidecar", Workflow: wf, Wrap: identityWrap, + }) + Expect(v.Status).To(Equal(gitagent.StatusError)) + Expect(v.Rejects()).To(BeTrue(), "an indeterminate verdict rejects (R7.5)") + Expect(v.Findings[0].Message).To(ContainSubstring("no fixture verifier is registered"), + "a declared definition of done that cannot run must never pass vacuously") + }) + It("kills a hook that overruns its timeout and reports error status", func() { v := gitagent.RunHookSet(ctx, ws(), gitagent.HookSetOptions{ Task: "t-1", Attempt: 1, Tier: "sidecar", diff --git a/pkg/promptrun/hooks.go b/pkg/promptrun/hooks.go new file mode 100644 index 00000000..6b3f5551 --- /dev/null +++ b/pkg/promptrun/hooks.go @@ -0,0 +1,59 @@ +package promptrun + +import ( + "context" + "fmt" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent/commit" + "github.com/flanksource/captain/pkg/ai/agent/setup" + "github.com/flanksource/captain/pkg/ai/agent/verify" +) + +// Hooks assembles the run's hook list in the order the runner dispatches them: +// +// commit → cmd → prompt → fixture → caller → setup +// +// The order is the run's safety contract, because agent.Runner dispatches Post +// hooks in list order at every phase. Commit hooks lead so a PhaseRun squash is +// cut before any teardown can take the tree it commits from. The workflow's +// checks run cheap to expensive (verify.HooksFor's own order), so a run that is +// going to fail fails on the fast check. The caller's hooks sit between the +// checks and setup: a host commit pipeline at PhaseRun still sees a live +// worktree, and a host PreRun runs before the tree is relocated. Setup trails so +// that its teardown is the last thing to happen. +// +// It is exported so a host can assert the list it will run — and so a caller +// that drives verifiers out of loop can still ask what a spec declares. +// +// Input.CallerOwnsCommits drops the leading commit hooks: the host commits, and +// its own hooks keep their position between the checks and setup. +func Hooks(ctx context.Context, in Input, provider ai.Provider) ([]any, error) { + if in.CallerOwnsCommits && len(in.Hooks) == 0 { + return nil, fmt.Errorf("promptrun: CallerOwnsCommits is set but Input.Hooks is empty: nothing would commit") + } + opts := in.Verify + if opts.Provider == nil { + opts.Provider = provider + } + if opts.RunSpec == nil { + // A verifier that runs an agent of its own inherits the run's model, + // permissions and budget from here. It is the resolved request — the same + // one the runner executes — and read-only to a factory. + opts.RunSpec = &in.Request + } + verifyHooks, err := verify.HooksFor(ctx, in.Request.Workflow, opts) + if err != nil { + return nil, err + } + var hooks []any + if !in.CallerOwnsCommits { + hooks = append(hooks, commit.HooksForWorkflow(in.Request.Workflow)...) + } + hooks = append(hooks, verifyHooks...) + hooks = append(hooks, in.Hooks...) + if in.Provider == nil && in.Request.Setup != nil { + hooks = append(hooks, &setup.Plugin{}) + } + return hooks, nil +} diff --git a/pkg/promptrun/iterations.go b/pkg/promptrun/iterations.go new file mode 100644 index 00000000..9bb7998f --- /dev/null +++ b/pkg/promptrun/iterations.go @@ -0,0 +1,117 @@ +package promptrun + +import ( + "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/database" + "github.com/flanksource/commons/logger" +) + +// IterationRecords turns one finished run into the rows +// captain_prompt_run_iterations holds: what each turn was asked, how it ended, +// and the verdict that judged it. It is the one place a host — captain's own +// CLI or an embedding one — derives those rows from, so every host files the +// same account of a run. +// +// It reads the runner's own records — ai.LoopResult.Iterations for the turns +// that executed and agent.VerifyResult for the verdicts — rather than keeping a +// parallel tally that could disagree with them. The two are joined on the turn +// number, which both sides already carry: the loop indexes from 0 and a verdict +// names its turn from 1, the same numbering the store is keyed on. +// +// A turn is judged by every verifier the workflow declares, so a round holds one +// verdict per verifier. All of them are rolled into the turn's single stored +// report (FinalReport → api.MergeReports); keeping only the last one filed a +// `commands` + `fixture` round as the fixture alone. +// +// A verify-only run has no loop: nothing was generated, the workflow's +// verifiers judged the tree as it stood. That is still one iteration — the +// first — and its verdict is the run's whole account, so it is filed as +// iteration 1. A run with neither turns nor verdicts has nothing to file. +// +// stopped says the run was interrupted — its context ended the loop, whether +// because the user pressed stop or because the run's deadline fired. Its last +// turn was cut off rather than judged, and calling that "failed" would blame the +// work for the interruption. +func IterationRecords(result Result, stopped bool) []database.UpsertPromptRunIterationInput { + if result.Loop == nil { + return verifyOnlyRecords(result.Verdicts, stopped) + } + byIteration := make(map[int][]agent.VerifyResult, len(result.Verdicts)) + for _, verdict := range result.Verdicts { + byIteration[verdict.Iteration] = append(byIteration[verdict.Iteration], verdict) + } + + records := make([]database.UpsertPromptRunIterationInput, 0, len(result.Loop.Iterations)) + for i, turn := range result.Loop.Iterations { + iteration := turn.Iteration + 1 + record := database.UpsertPromptRunIterationInput{ + Iteration: iteration, + Request: map[string]any{"prompt": turn.Request.Prompt.User}, + State: database.PromptRunIterationStateSucceeded, + } + if !turn.StartedAt.IsZero() { + started := turn.StartedAt + record.StartedAt = &started + } + if !turn.FinishedAt.IsZero() { + // Both timestamps or neither: the store's state trigger back-fills a + // missing finished_at from its own clock, which is long after the turn. + finished := turn.FinishedAt + record.FinishedAt = &finished + } + if round := byIteration[iteration]; len(round) > 0 { + judge(&record, round) + } + if turn.Err != nil { + // A turn the provider could not complete was never judged; the error is + // the whole account of it. + record.State = database.PromptRunIterationStateFailed + record.Error = turn.Err.Error() + } + if stopped && i == len(result.Loop.Iterations)-1 { + record.State = database.PromptRunIterationStateCancelled + } + records = append(records, record) + } + return records +} + +// verifyOnlyRecords is the single row of a run that generated nothing: the +// verifiers' round, filed as iteration 1, bracketed by the report's own clock +// since there was no provider call to time. +func verifyOnlyRecords(verdicts []agent.VerifyResult, stopped bool) []database.UpsertPromptRunIterationInput { + if len(verdicts) == 0 { + return nil + } + record := database.UpsertPromptRunIterationInput{ + Iteration: 1, + Request: map[string]any{"verify_only": true}, + State: database.PromptRunIterationStateSucceeded, + } + judge(&record, verdicts) + if report := record.VerificationResult; report != nil { + record.StartedAt, record.FinishedAt = report.StartedAt, report.FinishedAt + } + if stopped { + record.State = database.PromptRunIterationStateCancelled + } + return []database.UpsertPromptRunIterationInput{record} +} + +// judge stamps a round's rolled-up verdict onto the turn's record. A round that +// cannot be rolled up still happened and the row must say so; what goes +// missing is the verdict, and loudly, in both the log and the row. +func judge(record *database.UpsertPromptRunIterationInput, round []agent.VerifyResult) { + report, err := FinalReport(round) + if err != nil { + logger.Errorf("prompt run iteration %d: rolling up its %d verdict(s) failed: %v", record.Iteration, len(round), err) + record.Error = err.Error() + } + record.VerificationResult = report + if report != nil { + record.Feedback = report.Feedback + } + if !Passed(round) { + record.State = database.PromptRunIterationStateFailed + } +} diff --git a/pkg/promptrun/iterations_ginkgo_test.go b/pkg/promptrun/iterations_ginkgo_test.go new file mode 100644 index 00000000..f02e1d5b --- /dev/null +++ b/pkg/promptrun/iterations_ginkgo_test.go @@ -0,0 +1,194 @@ +package promptrun_test + +import ( + "errors" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/database" + "github.com/flanksource/captain/pkg/promptrun" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// verdictReport is the report one turn's verifier produced, stamped for that +// 1-based turn exactly as verify.Plugin stamps it. +func verdictReport(iteration int, passed bool) *api.VerifyReport { + node := api.VerifyNode{Name: "go test ./...", Passed: passed, Failed: !passed} + report := api.NewNodeReport(api.VerifyKindCmd, "verify:go test ./...", node) + report.Iteration = iteration + return &report +} + +func loopWith(turns int, base time.Time, err error) *ai.LoopResult { + loop := &ai.LoopResult{StopReason: "condition-met"} + for i := 0; i < turns; i++ { + started := base.Add(time.Duration(i) * time.Minute) + iteration := &ai.LoopIteration{ + Iteration: i, + Request: ai.Request{Prompt: api.Prompt{User: "attempt " + string(rune('A'+i))}}, + StartedAt: started, + FinishedAt: started.Add(30 * time.Second), + Success: true, + } + if i == turns-1 { + iteration.Err = err + } + loop.Iterations = append(loop.Iterations, iteration) + } + return loop +} + +var _ = Describe("IterationRecords", func() { + base := time.Date(2026, 9, 3, 9, 0, 0, 0, time.UTC) + + It("files a failing then a passing turn as two rows carrying their verdicts", func() { + loop := loopWith(2, base, nil) + verdicts := []agent.VerifyResult{ + {Valid: false, Iteration: 1, Report: verdictReport(1, false), Retry: &ai.Request{}}, + {Valid: true, Iteration: 2, Report: verdictReport(2, true)}, + } + verdicts[0].Report.Feedback = "TestFoo failed" + + records := promptrun.IterationRecords(promptrun.Result{Loop: loop, Verdicts: verdicts}, false) + + Expect(records).To(HaveLen(2)) + Expect(records[0].Iteration).To(Equal(1)) + Expect(records[0].State).To(Equal(database.PromptRunIterationStateFailed)) + Expect(records[0].Feedback).To(Equal("TestFoo failed")) + Expect(records[0].VerificationResult).To(Equal(verdicts[0].Report)) + Expect(records[0].Request).To(Equal(map[string]any{"prompt": "attempt A"})) + Expect(records[0].StartedAt).To(HaveValue(Equal(base))) + Expect(records[0].FinishedAt).To(HaveValue(Equal(base.Add(30 * time.Second)))) + Expect(records[1].Iteration).To(Equal(2)) + Expect(records[1].State).To(Equal(database.PromptRunIterationStateSucceeded)) + Expect(records[1].Feedback).To(BeEmpty()) + Expect(records[1].VerificationResult).To(Equal(verdicts[1].Report)) + }) + + // A turn the provider could not complete is a failure of the turn, not a + // verdict: nothing judged it, and recording it as succeeded would make a + // crashed run read as a clean one. + It("files a provider error as a failed, unjudged turn", func() { + records := promptrun.IterationRecords(promptrun.Result{Loop: loopWith(1, base, errors.New("upstream 529"))}, false) + + Expect(records).To(HaveLen(1)) + Expect(records[0].State).To(Equal(database.PromptRunIterationStateFailed)) + Expect(records[0].Error).To(Equal("upstream 529")) + Expect(records[0].VerificationResult).To(BeNil()) + }) + + // A stopped run's last turn was interrupted, not judged; "cancelled" is the + // only state that says so. + It("cancels the last turn of a stopped run", func() { + verdicts := []agent.VerifyResult{{Valid: false, Iteration: 1, Report: verdictReport(1, false)}} + + records := promptrun.IterationRecords(promptrun.Result{Loop: loopWith(2, base, nil), Verdicts: verdicts}, true) + + Expect(records).To(HaveLen(2)) + Expect(records[0].State).To(Equal(database.PromptRunIterationStateFailed)) + Expect(records[1].State).To(Equal(database.PromptRunIterationStateCancelled)) + }) + + // A round runs every verifier the workflow declares — `commands` and + // `fixture` both vote on the same turn. Keeping the last verdict per + // iteration stored the fixture's tree and threw the command's away. + It("rolls a whole round into one report", func() { + cmd := api.NewNodeReport(api.VerifyKindCmd, "verify:go test ./...", api.VerifyNode{Name: "go test ./...", Passed: true}) + cmd.Ran, cmd.Iteration = true, 1 + fixture := api.NewNodeReport(api.VerifyKindFixture, "acceptance", api.VerifyNode{Name: "TestFoo", Failed: true}) + fixture.Ran, fixture.Iteration, fixture.Feedback = true, 1, "TestFoo: want 3, got 4" + + records := promptrun.IterationRecords(promptrun.Result{Loop: loopWith(1, base, nil), Verdicts: []agent.VerifyResult{ + {Valid: true, Iteration: 1, Report: &cmd}, + {Valid: false, Iteration: 1, Report: &fixture}, + }}, false) + + Expect(records).To(HaveLen(1)) + Expect(records[0].State).To(Equal(database.PromptRunIterationStateFailed)) + Expect(records[0].Feedback).To(Equal("TestFoo: want 3, got 4")) + report := records[0].VerificationResult + Expect(report).NotTo(BeNil()) + Expect(report.Tests).To(HaveLen(2), "both verifiers keep their own group node") + Expect(report.Tests[0].Name).To(Equal("verify:go test ./...")) + Expect(report.Tests[1].Name).To(Equal("acceptance")) + Expect(report.Summary).To(Equal(api.VerifySummary{Total: 2, Passed: 1, Failed: 1})) + Expect(report.Passed).To(BeFalse()) + Expect(report.Validate()).To(Succeed()) + }) + + // A single-verdict round is that verdict's report, unwrapped: nesting a lone + // check under a group node would change every stored row for no gain. + It("stores a single verdict as is", func() { + verdicts := []agent.VerifyResult{{Valid: true, Iteration: 1, Report: verdictReport(1, true)}} + + records := promptrun.IterationRecords(promptrun.Result{Loop: loopWith(1, base, nil), Verdicts: verdicts}, false) + + Expect(records).To(HaveLen(1)) + Expect(records[0].VerificationResult).To(BeIdenticalTo(verdicts[0].Report)) + }) + + // A run with no Verify hooks at all has nothing to fail: every completed turn + // stands on its own. + It("files unverified turns as succeeded", func() { + records := promptrun.IterationRecords(promptrun.Result{Loop: loopWith(1, base, nil)}, false) + + Expect(records).To(HaveLen(1)) + Expect(records[0].State).To(Equal(database.PromptRunIterationStateSucceeded)) + Expect(records[0].VerificationResult).To(BeNil()) + }) + + // A verify-only run generated nothing, so the loop never ran — but the + // verifiers judged the tree, and that verdict is the run's whole account. + // Filing no row for it left every such run without a verification report: + // an embedding host's dashboard read "never verified" for a check that passed. + It("files a verify-only run as iteration 1 carrying the round's report", func() { + started, finished := base, base.Add(45*time.Second) + fixture := api.NewNodeReport(api.VerifyKindFixture, "fixture", api.VerifyNode{Name: "echo ok", Passed: true}) + fixture.Ran, fixture.Iteration, fixture.StartedAt, fixture.FinishedAt = true, 1, &started, &finished + + records := promptrun.IterationRecords(promptrun.Result{Verdicts: []agent.VerifyResult{ + {Valid: true, Iteration: 1, Report: &fixture}, + }}, false) + + Expect(records).To(HaveLen(1)) + Expect(records[0].Iteration).To(Equal(1)) + Expect(records[0].State).To(Equal(database.PromptRunIterationStateSucceeded)) + Expect(records[0].VerificationResult).To(BeIdenticalTo(&fixture)) + Expect(records[0].Request).To(Equal(map[string]any{"verify_only": true})) + Expect(records[0].StartedAt).To(HaveValue(Equal(started))) + Expect(records[0].FinishedAt).To(HaveValue(Equal(finished))) + }) + + It("files a failing verify-only run as a failed iteration 1 with its feedback", func() { + fixture := api.NewNodeReport(api.VerifyKindFixture, "fixture", api.VerifyNode{Name: "echo ok", Failed: true}) + fixture.Ran, fixture.Iteration, fixture.Feedback = true, 1, "echo ok: exit 1" + + records := promptrun.IterationRecords(promptrun.Result{Verdicts: []agent.VerifyResult{ + {Valid: false, Iteration: 1, Report: &fixture}, + }}, false) + + Expect(records).To(HaveLen(1)) + Expect(records[0].State).To(Equal(database.PromptRunIterationStateFailed)) + Expect(records[0].Feedback).To(Equal("echo ok: exit 1")) + Expect(records[0].StartedAt).To(BeNil(), "a report without a clock stamps nothing") + }) + + It("cancels an interrupted verify-only run", func() { + fixture := api.NewNodeReport(api.VerifyKindFixture, "fixture", api.VerifyNode{Name: "echo ok", Passed: true}) + fixture.Ran, fixture.Iteration = true, 1 + + records := promptrun.IterationRecords(promptrun.Result{Verdicts: []agent.VerifyResult{ + {Valid: true, Iteration: 1, Report: &fixture}, + }}, true) + + Expect(records).To(HaveLen(1)) + Expect(records[0].State).To(Equal(database.PromptRunIterationStateCancelled)) + }) + + It("files nothing for a run with neither turns nor verdicts", func() { + Expect(promptrun.IterationRecords(promptrun.Result{}, false)).To(BeNil()) + }) +}) diff --git a/pkg/promptrun/promptrun_ginkgo_test.go b/pkg/promptrun/promptrun_ginkgo_test.go new file mode 100644 index 00000000..5858fb06 --- /dev/null +++ b/pkg/promptrun/promptrun_ginkgo_test.go @@ -0,0 +1,453 @@ +package promptrun_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/promptrun" + "github.com/flanksource/commons-db/shell" +) + +// testTimeout is the deadline every spec declares explicitly. promptrun.Run has +// no default of its own: a run whose spec declares no budget.timeout and whose +// host passes none is a run nobody bounded, and silently capping it at two +// minutes is how a 45-minute job died with no explanation. +const testTimeout = 2 * time.Minute + +// scriptedProvider is a streaming provider that emits one scripted turn and +// counts how often it was asked to, so a spec can prove it was never called. +type scriptedProvider struct { + mu sync.Mutex + calls int + model string +} + +func (p *scriptedProvider) GetModel() string { return p.model } +func (p *scriptedProvider) GetRuntime() api.Runtime { + return api.RuntimeOf(api.Anthropic, api.ModeAgent) +} +func (p *scriptedProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { + p.mu.Lock() + p.calls++ + p.mu.Unlock() + return &ai.Response{Text: "done", Model: p.model}, nil +} +func (p *scriptedProvider) ExecuteStream(context.Context, ai.Request) (<-chan ai.Event, error) { + p.mu.Lock() + p.calls++ + p.mu.Unlock() + ch := make(chan ai.Event, 3) + ch <- ai.Event{Kind: ai.EventSystem, SessionID: "sess-1", Model: p.model} + ch <- ai.Event{Kind: ai.EventText, Text: "done", Model: p.model} + ch <- ai.Event{Kind: ai.EventResult, Success: true, Model: p.model, Usage: &ai.Usage{InputTokens: 7, OutputTokens: 3}, CostUSD: 0.25} + close(ch) + return ch, nil +} +func (p *scriptedProvider) Calls() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.calls +} + +// recordingHook is a caller hook that notes every phase it is dispatched at. +type recordingHook struct { + name string + log *[]string +} + +func (h *recordingHook) Name() string { return h.name } +func (h *recordingHook) PreRun(*agent.HookContext) error { + *h.log = append(*h.log, h.name+":prerun") + return nil +} +func (h *recordingHook) Phases() []agent.Phase { return []agent.Phase{agent.PhaseRun} } +func (h *recordingHook) Post(_ *agent.HookContext, phase agent.Phase) error { + *h.log = append(*h.log, h.name+":"+string(phase)) + return nil +} + +// progressVerifier reports three in-flight snapshots and then passes; it is +// what a fixture runner looks like to the loop. +type progressVerifier struct{ progress func(api.VerifyReport) } + +func (v *progressVerifier) SetProgress(fn func(api.VerifyReport)) { v.progress = fn } +func (v *progressVerifier) Verify(context.Context, string, []string) (verify.Verdict, error) { + for i := 1; i <= 3; i++ { + v.progress(api.NewNodeReport(api.VerifyKindFixture, "fixture", api.VerifyNode{Name: fmt.Sprintf("check %d", i), Running: true})) + } + final := api.NewNodeReport(api.VerifyKindFixture, "fixture", api.VerifyNode{Name: "check 3", Passed: true}) + return verify.Verdict{OK: true, Report: &final}, nil +} + +func hookNames(hooks []any) []string { + names := make([]string, 0, len(hooks)) + for _, h := range hooks { + names = append(names, h.(interface{ Name() string }).Name()) + } + return names +} + +func writeJudgePrompt(dir string) string { + path := filepath.Join(dir, "review.prompt") + Expect(os.WriteFile(path, []byte("{{role \"user\"}}\nJudge the work in {{cwd}}."), 0o644)).To(Succeed()) + return path +} + +var _ = Describe("promptrun.Run", func() { + var ( + cwd string + provider *scriptedProvider + request ai.Request + ) + + BeforeEach(func() { + cwd = GinkgoT().TempDir() + provider = &scriptedProvider{model: "fake-model"} + request = ai.Request{ + Model: api.Model{Name: "claude-sonnet-5", Provider: api.Anthropic, Mode: api.ModeAgent}, + Prompt: api.Prompt{User: "fix it"}, + } + request.SetCwd(cwd) + }) + + AfterEach(func() { + verify.Unregister(verify.KindFixture) + }) + + Describe("hook order", func() { + // The order is the run's safety contract: commit hooks lead so a PhaseRun + // squash lands before any teardown; the workflow's own checks run cheap to + // expensive; the caller's hooks sit between the checks and setup so a host + // commit pipeline still sees a live worktree; setup trails so its teardown + // is the last Post to fire. + It("assembles commit → cmd → prompt → fixture → caller → setup", func() { + verify.Register(verify.KindFixture, func(_ context.Context, spec api.Verify, _ verify.Options) ([]*verify.Plugin, error) { + return []*verify.Plugin{verify.New("fixture:"+spec.Fixture, &progressVerifier{})}, nil + }) + judge := writeJudgePrompt(cwd) + request.Workflow = &api.Workflow{ + Verify: &api.Verify{Commands: []string{"true"}, Prompts: []string{judge}, Fixture: "acceptance"}, + Commits: []api.Commit{{On: api.CommitOnRun}}, + } + request.Setup = &shell.Setup{} + var log []string + + hooks, err := promptrun.Hooks(context.Background(), promptrun.Input{ + Request: request, + Hooks: []any{&recordingHook{name: "host", log: &log}}, + Verify: verify.Options{Provider: provider}, + }, provider) + Expect(err).NotTo(HaveOccurred()) + Expect(hookNames(hooks)).To(Equal([]string{ + "commit:run", "verify:true", "judge:" + judge, "fixture:acceptance", "host", "setup", + })) + }) + + It("adds no setup hook when the caller supplies the provider that owns the workspace", func() { + request.Setup = &shell.Setup{} + hooks, err := promptrun.Hooks(context.Background(), promptrun.Input{Request: request, Provider: provider}, provider) + Expect(err).NotTo(HaveOccurred()) + Expect(hookNames(hooks)).To(BeEmpty()) + }) + + It("refuses a fixture the process has no runner for", func() { + request.Workflow = &api.Workflow{Verify: &api.Verify{Fixture: "acceptance"}} + _, err := promptrun.Hooks(context.Background(), promptrun.Input{Request: request}, provider) + Expect(err).To(MatchError(ContainSubstring("no fixture verifier is registered"))) + }) + }) + + Describe("commit ownership", func() { + BeforeEach(func() { + request.Workflow = &api.Workflow{ + Verify: &api.Verify{Commands: []string{"true"}}, + Commits: []api.Commit{{On: api.CommitOnRun}}, + } + }) + + It("builds captain's own commit hook from the workflow by default", func() { + hooks, err := promptrun.Hooks(context.Background(), promptrun.Input{Request: request}, provider) + Expect(err).NotTo(HaveOccurred()) + Expect(hookNames(hooks)).To(Equal([]string{"commit:run", "verify:true", "setup"})) + }) + + // A host with its own commit pipeline — gavel's pre-commit gates and + // trailers — would otherwise commit the same tree twice. It used to strip + // Workflow.Commits off the request to prevent that, which also took the + // declaration out of the spec that gets recorded and validated. + It("adds no commit hook when the caller owns commits, and leaves the declaration on the request", func() { + var log []string + in := promptrun.Input{ + Request: request, + Hooks: []any{&recordingHook{name: "host", log: &log}}, + CallerOwnsCommits: true, + } + + hooks, err := promptrun.Hooks(context.Background(), in, provider) + Expect(err).NotTo(HaveOccurred()) + Expect(hookNames(hooks)).To(Equal([]string{"verify:true", "host", "setup"}), + "the caller's hooks keep their position: after the checks, before setup") + Expect(request.Workflow.Commits).To(Equal([]api.Commit{{On: api.CommitOnRun}}), + "the resolved spec still declares what the host commits") + }) + + It("refuses caller-owned commits with no caller hook to do the committing", func() { + _, err := promptrun.Hooks(context.Background(), + promptrun.Input{Request: request, CallerOwnsCommits: true}, provider) + Expect(err).To(MatchError(ContainSubstring("CallerOwnsCommits"))) + }) + }) + + Describe("the spec a verifier factory is given", func() { + // A factory that runs its own agent — a fixture grader judging a + // document's acceptance criteria — has no other way to inherit the run's + // model, permissions and budget, and one that invents its own runs the + // grading outside the posture the run was started under. + It("hands the run's resolved spec to every registered factory", func() { + var captured *api.Spec + verify.Register(verify.KindFixture, func(_ context.Context, _ api.Verify, opts verify.Options) ([]*verify.Plugin, error) { + captured = opts.RunSpec + return []*verify.Plugin{verify.New("fixture", &progressVerifier{})}, nil + }) + request.Permissions.Mode = api.PermissionAcceptEdits + request.Workflow = &api.Workflow{Verify: &api.Verify{Fixture: "acceptance"}} + + res, err := promptrun.Run(context.Background(), promptrun.Input{ + Request: request, Provider: provider, Timeout: testTimeout, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Passed).To(BeTrue()) + Expect(captured).NotTo(BeNil(), "a factory with no spec cannot inherit the run's model or permissions") + Expect(captured.Model.Name).To(Equal("claude-sonnet-5")) + Expect(captured.Permissions.Mode).To(Equal(api.PermissionAcceptEdits)) + }) + }) + + Describe("verify-only", func() { + // An empty prompt body means "judge the tree as it is". The provider is + // never asked anything — and never even constructed when nothing needs + // one, so a verify-only run works without a model to hand. + It("runs the checks once, never calls the provider, and returns their report", func() { + request.Prompt = api.Prompt{} + request.Workflow = &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}}} + + res, err := promptrun.Run(context.Background(), promptrun.Input{Request: request, Provider: provider, Timeout: testTimeout}) + Expect(err).NotTo(HaveOccurred()) + Expect(provider.Calls()).To(BeZero()) + Expect(res.Passed).To(BeTrue()) + Expect(res.Report).NotTo(BeNil()) + Expect(res.Report.Passed).To(BeTrue()) + Expect(res.Report.Iteration).To(Equal(1)) + Expect(res.Verdicts).To(HaveLen(1)) + Expect(res.Loop).To(BeNil()) + }) + + It("does not construct a provider when no check needs one", func() { + request.Prompt = api.Prompt{} + request.Workflow = &api.Workflow{Verify: &api.Verify{Commands: []string{"exit 3"}}} + + // A Config no provider can be built from: were one constructed, this + // would fail there rather than at the verdict. + res, err := promptrun.Run(context.Background(), promptrun.Input{Request: request, Config: ai.Config{}, Timeout: testTimeout}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Passed).To(BeFalse()) + Expect(res.Report.State).To(Equal(api.VerifyStateFailed)) + Expect(promptrun.FailureReason(res.Verdicts)).NotTo(BeEmpty()) + }) + + // The runner used to call an empty prompt verify-only on its own, so a + // request with attachments or a message history and nothing to verify built + // a provider, generated nothing, and came back passed. There is one rule + // (api.Spec.IsVerifyOnly) and anything outside it is an error. + DescribeTable("refuses a request that neither generates nor verifies", + func(mutate func(*ai.Request)) { + request.Prompt = api.Prompt{} + request.Workflow = nil + mutate(&request) + + _, err := promptrun.Run(context.Background(), promptrun.Input{ + Request: request, Provider: provider, Timeout: testTimeout, + }) + Expect(err).To(MatchError(ContainSubstring("workflow.verify"))) + Expect(provider.Calls()).To(BeZero()) + }, + Entry("nothing at all", func(*ai.Request) {}), + Entry("attachments only", func(r *ai.Request) { + r.Prompt.Attachments = []api.AttachmentRef{preparedAttachment("image/png")} + }), + Entry("messages only", func(r *ai.Request) { + r.Messages = []api.Message{{Role: api.RoleUser, Parts: []api.Part{{Type: api.PartText, Text: "hi"}}}} + }), + ) + }) + + Describe("the deadline", func() { + // DefaultTimeout silently capped a host run at two minutes. A run nobody + // bounded is a configuration error the host can fix, not a limit to invent. + It("refuses a run with no budget.timeout and no caller timeout", func() { + _, err := promptrun.Run(context.Background(), promptrun.Input{Request: request, Provider: provider}) + Expect(err).To(MatchError(ContainSubstring("no timeout"))) + Expect(err).To(MatchError(ContainSubstring("budget.timeout"))) + Expect(provider.Calls()).To(BeZero()) + }) + + It("prefers the spec's own budget.timeout over the caller's default", func() { + request.Budget.Timeout = "45m" + request.Workflow = &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}}} + _, err := promptrun.Run(context.Background(), promptrun.Input{ + Request: request, Provider: provider, Timeout: time.Millisecond, + }) + Expect(err).NotTo(HaveOccurred(), "a one-millisecond host default must not outrank the declared 45m") + }) + }) + + Describe("the executing model", func() { + // Config.Model is what middleware.NewProvider actually runs. Checking the + // policy against one model and the attachments against another let a run + // start on a runtime that could enforce neither. + It("checks the tool policy against the model the provider will run", func() { + request.Model = api.Model{Name: "claude-sonnet-5", Provider: api.Anthropic, Mode: api.ModeAgent} + request.Permissions.Tools = api.Tools{"Bash": api.ToolPolicyDeny} + + _, err := promptrun.Run(context.Background(), promptrun.Input{ + Request: request, Provider: provider, Timeout: testTimeout, + Config: ai.Config{Model: api.Model{Name: "gpt-5", Provider: api.OpenAI, Mode: api.ModeAPI}}, + }) + Expect(err).To(MatchError(ContainSubstring(api.RuntimeOf(api.OpenAI, api.ModeAPI).String()))) + Expect(provider.Calls()).To(BeZero()) + }) + + It("checks attachment compatibility against that same model", func() { + request.Model = api.Model{Name: "claude-sonnet-5", Provider: api.Anthropic, Mode: api.ModeAgent} + request.Prompt.Attachments = []api.AttachmentRef{preparedAttachment("image/png")} + + _, err := promptrun.Run(context.Background(), promptrun.Input{ + Request: request, Provider: provider, Timeout: testTimeout, + Config: ai.Config{Model: api.Model{Name: "claude-sonnet-5", Provider: api.Anthropic, Mode: api.ModeCLI}}, + }) + Expect(err).To(MatchError(ContainSubstring("image/png"))) + Expect(provider.Calls()).To(BeZero()) + }) + }) + + Describe("tool policy", func() { + // No Input.Provider and a Config whose model cannot be resolved: were the + // check to happen after provider construction, the unknown model would be + // the error, not the policy. + It("fails on an unenforceable per-tool policy before any provider is built", func() { + unbuildable := promptrun.Input{ + Request: request, Timeout: testTimeout, + Config: ai.Config{Model: api.Model{Name: "no-such-model-at-all", Provider: api.Anthropic, Mode: api.ModeAgent}}, + } + // The premise: this config cannot produce a provider, so reaching + // construction is visible as a different error than the policy's. + _, err := promptrun.Run(context.Background(), unbuildable) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("per-tool policy")) + + unbuildable.Request.Permissions.Tools = api.Tools{"Bash": api.ToolPolicyAsk} + _, err = promptrun.Run(context.Background(), unbuildable) + Expect(err).To(MatchError(ContainSubstring(`per-tool policy "ask" (Bash)`))) + }) + }) + + Describe("a generate → verify run", func() { + It("returns the loop, verdicts, final report, and the run's identity", func() { + request.Workflow = &api.Workflow{Verify: &api.Verify{Commands: []string{"true"}}} + var events []ai.Event + + var iterations []int + res, err := promptrun.Run(context.Background(), promptrun.Input{ + Request: request, Provider: provider, Timeout: testTimeout, + OnEvent: func(iter int, ev ai.Event) { + events = append(events, ev) + iterations = append(iterations, iter) + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(iterations).NotTo(BeEmpty(), "OnEvent carries the runner's turn index, which a renderer needs") + Expect(provider.Calls()).To(Equal(1)) + Expect(res.Passed).To(BeTrue()) + Expect(res.SessionID).To(Equal("sess-1")) + Expect(res.Model).To(Equal("fake-model")) + Expect(res.Usage).To(Equal(api.Usage{InputTokens: 7, OutputTokens: 3})) + Expect(res.CostUSD).To(Equal(0.25)) + Expect(res.Response.Text).To(Equal("done")) + Expect(res.Loop).NotTo(BeNil()) + Expect(res.Loop.Iterations).To(HaveLen(1)) + Expect(res.Report).To(BeIdenticalTo(res.Verdicts[0].Report)) + Expect(res.Duration).To(BeNumerically(">", 0)) + + var kinds []api.EventKind + for _, ev := range events { + kinds = append(kinds, ev.Kind) + } + Expect(kinds).To(ContainElements(ai.EventSystem, ai.EventText, ai.EventResult, ai.EventVerified)) + }) + + It("delivers in-flight progress to Options.Progress and to OnEvent", func() { + verify.Register(verify.KindFixture, func(_ context.Context, _ api.Verify, _ verify.Options) ([]*verify.Plugin, error) { + return []*verify.Plugin{verify.New("fixture", &progressVerifier{})}, nil + }) + request.Workflow = &api.Workflow{Verify: &api.Verify{Fixture: "acceptance"}} + var snapshots []api.VerifyReport + var progressEvents []ai.Event + + res, err := promptrun.Run(context.Background(), promptrun.Input{ + Request: request, Provider: provider, Timeout: testTimeout, + Verify: verify.Options{Progress: func(r api.VerifyReport) { snapshots = append(snapshots, r) }}, + OnEvent: func(_ int, ev ai.Event) { + if ev.Kind == ai.EventVerifyProgress { + progressEvents = append(progressEvents, ev) + } + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Passed).To(BeTrue()) + Expect(len(snapshots)).To(BeNumerically(">=", 1)) + Expect(snapshots[len(snapshots)-1].Tests[0].Name).To(Equal("check 3"), "the last snapshot is always flushed") + Expect(len(progressEvents)).To(BeNumerically(">=", 1)) + _, ok := progressEvents[0].Raw.(*api.VerifyReport) + Expect(ok).To(BeTrue(), "Raw carries the *api.VerifyReport") + Expect(res.Response.Workspace.Notices).To(HaveLen(1), "progress leaves no notice; the verdict does") + }) + + It("reports the failing check's reason", func() { + request.Workflow = &api.Workflow{Verify: &api.Verify{Commands: []string{"echo nope; exit 1"}}} + + res, err := promptrun.Run(context.Background(), promptrun.Input{Request: request, Provider: provider, Timeout: testTimeout}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Passed).To(BeFalse()) + Expect(promptrun.FailureReason(res.Verdicts)).To(Equal(res.Report.Reason)) + Expect(res.Report.Reason).To(ContainSubstring("failed")) + }) + }) + + Describe("attachments", func() { + It("refuses an attachment the caller did not resolve", func() { + request.Prompt.Attachments = []api.AttachmentRef{{Path: "notes.txt"}} + _, err := promptrun.Run(context.Background(), promptrun.Input{Request: request, Provider: provider, Timeout: testTimeout}) + Expect(err).To(MatchError(ContainSubstring("attachment"))) + Expect(provider.Calls()).To(BeZero()) + }) + }) +}) + +// preparedAttachment is an attachment a store has already resolved, which is +// the only kind promptrun will run with. +func preparedAttachment(mediaType string) api.AttachmentRef { + ref := api.AttachmentRef{ID: api.AttachmentIDPrefix + strings.Repeat("a", 64), MediaType: mediaType} + return ref.WithPreparedContent(api.AttachmentContent{Bytes: []byte("payload")}) +} diff --git a/pkg/promptrun/promptrun_suite_test.go b/pkg/promptrun/promptrun_suite_test.go new file mode 100644 index 00000000..d54299b9 --- /dev/null +++ b/pkg/promptrun/promptrun_suite_test.go @@ -0,0 +1,13 @@ +package promptrun_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPromptRun(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "promptrun") +} diff --git a/pkg/promptrun/provider.go b/pkg/promptrun/provider.go new file mode 100644 index 00000000..6953cf6d --- /dev/null +++ b/pkg/promptrun/provider.go @@ -0,0 +1,83 @@ +package promptrun + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/flanksource/captain/pkg/ai" +) + +// runnerProvider is the streaming face the runner drives. A streaming provider +// is used as-is; a buffered-only one — and every provider under NoStream — is +// wrapped so its completed response is replayed as the events the runner +// expects. A verify-only run has no provider to wrap and needs none. +func runnerProvider(provider ai.Provider, noStream, verifyOnly bool) (ai.StreamingProvider, error) { + if provider == nil { + if verifyOnly { + return nil, nil + } + return nil, errors.New("promptrun: a generating run needs a provider") + } + if noStream { + return bufferedProvider{Provider: provider}, nil + } + if streamer, ok := provider.(ai.StreamingProvider); ok { + return streamer, nil + } + return bufferedProvider{Provider: provider}, nil +} + +// bufferedProvider preserves the runner's event contract while forcing +// generation through Provider.Execute. It emits only completed-response events, +// so NoStream never invokes an underlying ExecuteStream. +type bufferedProvider struct { + ai.Provider +} + +func (p bufferedProvider) ExecuteStream(ctx context.Context, req ai.Request) (<-chan ai.Event, error) { + resp, err := p.Execute(ctx, req) + if err != nil { + return nil, err + } + if resp == nil { + return nil, errors.New("buffered provider returned a nil response") + } + structured, err := bufferedStructuredData(resp.StructuredData) + if err != nil { + return nil, err + } + + events := make(chan ai.Event, 3) + if resp.Workspace != nil && resp.Workspace.SessionID != "" { + events <- ai.Event{Kind: ai.EventSystem, SessionID: resp.Workspace.SessionID, Model: resp.Model} + } + if resp.Text != "" { + events <- ai.Event{Kind: ai.EventText, Text: resp.Text, Model: resp.Model} + } + usage := resp.Usage + events <- ai.Event{ + Kind: ai.EventResult, Success: true, Model: resp.Model, Usage: &usage, + CostUSD: resp.CostUSD, StructuredData: structured, ToolApproval: resp.ToolApproval, + } + close(events) + return events, nil +} + +func bufferedStructuredData(value any) (json.RawMessage, error) { + if value == nil { + return nil, nil + } + if raw, ok := value.(json.RawMessage); ok { + return raw, nil + } + raw, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("encode buffered structured output: %w", err) + } + if string(raw) == "null" { + return nil, nil + } + return raw, nil +} diff --git a/pkg/promptrun/provider_ginkgo_test.go b/pkg/promptrun/provider_ginkgo_test.go new file mode 100644 index 00000000..e230e64b --- /dev/null +++ b/pkg/promptrun/provider_ginkgo_test.go @@ -0,0 +1,103 @@ +package promptrun + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/api" +) + +type bufferedOnlyProvider struct{ executeCalls int } + +func (p *bufferedOnlyProvider) GetModel() string { return "buffered-model" } +func (p *bufferedOnlyProvider) GetRuntime() ai.Runtime { + return api.RuntimeOf(api.DeepSeek, api.ModeAPI) +} +func (p *bufferedOnlyProvider) Execute(context.Context, ai.Request) (*ai.Response, error) { + p.executeCalls++ + return &ai.Response{ + Text: "done", + StructuredData: map[string]any{"status": "ok"}, + Model: "buffered-model", + Runtime: api.RuntimeOf(api.DeepSeek, api.ModeAPI), + Usage: ai.Usage{InputTokens: 2, OutputTokens: 3}, + CostUSD: 0.01, + }, nil +} + +type streamingProvider struct { + bufferedOnlyProvider + streamCalls int +} + +func (p *streamingProvider) ExecuteStream(context.Context, ai.Request) (<-chan ai.Event, error) { + p.streamCalls++ + events := make(chan ai.Event, 1) + events <- ai.Event{Kind: ai.EventResult, Success: true} + close(events) + return events, nil +} + +func drain(events <-chan ai.Event) []ai.Event { + var got []ai.Event + for event := range events { + got = append(got, event) + } + return got +} + +var _ = Describe("runnerProvider", func() { + It("replays a buffered-only provider's response as completed events", func() { + provider := &bufferedOnlyProvider{} + runner, err := runnerProvider(provider, false, false) + Expect(err).NotTo(HaveOccurred()) + + events, err := runner.ExecuteStream(context.Background(), ai.Request{}) + Expect(err).NotTo(HaveOccurred()) + got := drain(events) + Expect(provider.executeCalls).To(Equal(1)) + Expect(got).To(HaveLen(2)) + Expect(got[0].Kind).To(Equal(ai.EventText)) + Expect(got[0].Text).To(Equal("done")) + Expect(got[1].Kind).To(Equal(ai.EventResult)) + Expect(string(got[1].StructuredData)).To(Equal(`{"status":"ok"}`)) + Expect(got[1].Usage.InputTokens).To(Equal(2)) + Expect(got[1].CostUSD).To(Equal(0.01)) + }) + + It("forces a streaming provider through Execute under NoStream", func() { + provider := &streamingProvider{} + runner, err := runnerProvider(provider, true, false) + Expect(err).NotTo(HaveOccurred()) + + events, err := runner.ExecuteStream(context.Background(), ai.Request{}) + Expect(err).NotTo(HaveOccurred()) + drain(events) + Expect(provider.executeCalls).To(Equal(1)) + Expect(provider.streamCalls).To(BeZero()) + }) + + It("leaves a streaming provider alone otherwise", func() { + provider := &streamingProvider{} + runner, err := runnerProvider(provider, false, false) + Expect(err).NotTo(HaveOccurred()) + + events, err := runner.ExecuteStream(context.Background(), ai.Request{}) + Expect(err).NotTo(HaveOccurred()) + drain(events) + Expect(provider.streamCalls).To(Equal(1)) + Expect(provider.executeCalls).To(BeZero()) + }) + + It("needs no provider for a verify-only run and refuses to generate without one", func() { + runner, err := runnerProvider(nil, false, true) + Expect(err).NotTo(HaveOccurred()) + Expect(runner).To(BeNil()) + + _, err = runnerProvider(nil, false, false) + Expect(err).To(MatchError(ContainSubstring("needs a provider"))) + }) +}) diff --git a/pkg/promptrun/result.go b/pkg/promptrun/result.go new file mode 100644 index 00000000..2046ac9a --- /dev/null +++ b/pkg/promptrun/result.go @@ -0,0 +1,177 @@ +package promptrun + +import ( + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/api" +) + +// Result is one run's outcome. +type Result struct { + // Response is the runner's accumulated response: final text, structured + // data, terminal outcome, and the Workspace (cwd, changed files, notices). + Response *ai.Response + // Verdicts is every verify verdict in the order it was reached; Report is + // the last one that carries a report — the run's answer. + Verdicts []agent.VerifyResult + Report *api.VerifyReport + // Passed is Passed(Verdicts): the final round's verdict, or true when the + // run declared nothing to verify. + Passed bool + // Loop is the generate loop's own record — nil for a verify-only run. + Loop *ai.LoopResult + // StructuredData and TerminalOutcome are lifted from Response for callers + // that read only the answer. + StructuredData any + TerminalOutcome *api.TerminalOutcome + // SessionID is the provider's session for this run; Model the one that + // answered; Usage and CostUSD the whole run's, summed across iterations. + SessionID string + Model string + Usage api.Usage + CostUSD float64 + Duration time.Duration +} + +// Passed reports whether the run's last verify verdict passed, or trivially +// true when no Verify hooks ran. It is agent.VerifyPassed — the rule the runner +// itself sets HookContext.Verified from — rather than a second copy of it, so a +// caller reading a Result and a Post hook reading its context can never disagree +// about whether the run verified. +func Passed(verdicts []agent.VerifyResult) bool { + return agent.VerifyPassed(verdicts) +} + +// FailureReason is the last failing verdict's reason, for a run summary; empty +// when the run passed. +func FailureReason(verdicts []agent.VerifyResult) string { + if len(verdicts) == 0 { + return "" + } + last := verdicts[len(verdicts)-1] + if last.Valid { + return "" + } + if last.Report != nil && last.Report.Reason != "" { + return last.Report.Reason + } + return "verification failed" +} + +// FinalReport is the run's verdict: everything the last round judged, as one +// report. +// +// A round runs every verifier the workflow declares, so it produces one report +// per verifier. Taking the last of them threw the rest away — a round of +// `commands` + `fixture` came back as the fixture's tree alone, and the run's +// summary counted half of what had actually run. A round of one report is that +// report, unwrapped; a round of several is api.MergeReports, which nests each +// under its own group node. A verdict carrying no report contributes nothing +// rather than blanking the round. +func FinalReport(verdicts []agent.VerifyResult) (*api.VerifyReport, error) { + reports := lastRoundReports(verdicts) + switch len(reports) { + case 0: + return nil, nil + case 1: + return reports[0], nil + } + round := make([]api.VerifyReport, 0, len(reports)) + for _, r := range reports { + round = append(round, *r) + } + merged, err := api.MergeReports(RoundName, round...) + if err != nil { + return nil, err + } + return &merged, nil +} + +// RoundName names a merged round wherever one is built, so the row a host +// persists and the report the webapp renders agree on what to call it. +const RoundName = "verify" + +// lastRoundReports is every report the highest-numbered round produced, in the +// order the verifiers voted. Rounds are identified by the turn they judged, +// which each verdict already carries. +func lastRoundReports(verdicts []agent.VerifyResult) []*api.VerifyReport { + last, found := 0, false + for _, v := range verdicts { + if v.Report != nil && (!found || v.Iteration > last) { + last, found = v.Iteration, true + } + } + if !found { + return nil + } + var reports []*api.VerifyReport + for _, v := range verdicts { + if v.Report != nil && v.Iteration == last { + reports = append(reports, v.Report) + } + } + return reports +} + +// runIdentity is what the event stream says about the run that the runner's +// response does not carry: the model that answered and the session it ran in. +type runIdentity struct { + sessionID string + model string +} + +func (r *runIdentity) observe(ev ai.Event) { + if ev.Model != "" { + r.model = ev.Model + } + if ev.Kind == ai.EventSystem && ev.SessionID != "" { + r.sessionID = ev.SessionID + } +} + +func newResult(out agent.Result[string], identity runIdentity, duration time.Duration) (Result, error) { + report, err := FinalReport(out.Verdicts) + result := Result{ + Response: out.Response, + Verdicts: out.Verdicts, + Report: report, + Passed: Passed(out.Verdicts), + Loop: out.Loop, + Model: identity.model, + Duration: duration, + } + if out.Response != nil { + result.StructuredData = out.Response.StructuredData + result.TerminalOutcome = out.Response.TerminalOutcome + if out.Response.Workspace != nil { + result.SessionID = out.Response.Workspace.SessionID + } + if result.Model == "" { + result.Model = out.Response.Model + } + } + if result.SessionID == "" { + result.SessionID = identity.sessionID + } + if out.Loop != nil { + result.CostUSD = out.Loop.TotalCost + for _, turn := range out.Loop.Iterations { + if result.SessionID == "" { + result.SessionID = turn.SessionID + } + result.Usage = addUsage(result.Usage, turn.Usage) + } + } + return result, err +} + +func addUsage(into, from api.Usage) api.Usage { + into.InputTokens += from.InputTokens + into.OutputTokens += from.OutputTokens + into.ReasoningTokens += from.ReasoningTokens + into.CacheReadTokens += from.CacheReadTokens + into.CacheWriteTokens += from.CacheWriteTokens + return into +} diff --git a/pkg/promptrun/result_ginkgo_test.go b/pkg/promptrun/result_ginkgo_test.go new file mode 100644 index 00000000..20a9cbde --- /dev/null +++ b/pkg/promptrun/result_ginkgo_test.go @@ -0,0 +1,80 @@ +package promptrun_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/promptrun" +) + +var _ = Describe("verdict helpers", func() { + failed := api.NewNodeReport(api.VerifyKindCmd, "verify:go test", api.VerifyNode{Name: "go test", Failed: true, Message: "TestFoo failed"}) + failed.Reason = "go test failed" + passed := api.NewNodeReport(api.VerifyKindCmd, "verify:go test", api.VerifyNode{Name: "go test", Passed: true}) + + Describe("Passed", func() { + // The runner stops each round at its first failure, so the last verdict + // is always the final round's outcome; earlier failures are the history + // of retries, not the answer. + It("reads the last verdict, and passes trivially with none", func() { + Expect(promptrun.Passed(nil)).To(BeTrue()) + Expect(promptrun.Passed([]agent.VerifyResult{{Valid: false}, {Valid: true}})).To(BeTrue()) + Expect(promptrun.Passed([]agent.VerifyResult{{Valid: true}, {Valid: false}})).To(BeFalse()) + }) + }) + + Describe("FailureReason", func() { + It("is the last failing report's reason, and empty for a pass", func() { + Expect(promptrun.FailureReason(nil)).To(BeEmpty()) + Expect(promptrun.FailureReason([]agent.VerifyResult{{Valid: true, Report: &passed}})).To(BeEmpty()) + Expect(promptrun.FailureReason([]agent.VerifyResult{{Valid: false, Report: &failed}})).To(Equal("go test failed")) + Expect(promptrun.FailureReason([]agent.VerifyResult{{Valid: false}})).To(Equal("verification failed")) + }) + }) + + Describe("FinalReport", func() { + It("is the run's last round, and nothing at all with no verdicts", func() { + report, err := promptrun.FinalReport(nil) + Expect(err).NotTo(HaveOccurred()) + Expect(report).To(BeNil()) + + report, err = promptrun.FinalReport([]agent.VerifyResult{ + {Iteration: 1, Report: &failed}, {Iteration: 2, Report: &passed}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(report).To(BeIdenticalTo(&passed), "a single-verdict round is its own report, unwrapped") + }) + + It("does not let a verdict without a report blank the one the round has", func() { + report, err := promptrun.FinalReport([]agent.VerifyResult{{Iteration: 1, Report: &passed}, {Iteration: 1}}) + Expect(err).NotTo(HaveOccurred()) + Expect(report).To(BeIdenticalTo(&passed)) + }) + + // A round runs every declared verifier. Reading only the last verdict + // persisted the fixture's tree and threw the command's away, so the run's + // record showed half of what had actually been checked. + It("rolls a round of several verdicts into one report, each under its own group node", func() { + cmd := api.NewNodeReport(api.VerifyKindCmd, "verify:go test", api.VerifyNode{Name: "go test", Passed: true}) + cmd.Ran, cmd.Iteration = true, 1 + fixture := api.NewNodeReport(api.VerifyKindFixture, "acceptance", api.VerifyNode{Name: "TestFoo", Failed: true}) + fixture.Ran, fixture.Iteration, fixture.Reason = true, 1, "TestFoo failed" + + report, err := promptrun.FinalReport([]agent.VerifyResult{ + {Iteration: 1, Valid: true, Report: &cmd}, + {Iteration: 1, Valid: false, Report: &fixture}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(report).NotTo(BeNil()) + Expect(report.Tests).To(HaveLen(2)) + Expect(report.Tests[0].Name).To(Equal("verify:go test")) + Expect(report.Tests[1].Name).To(Equal("acceptance")) + Expect(report.Summary).To(Equal(api.VerifySummary{Total: 2, Passed: 1, Failed: 1})) + Expect(report.Passed).To(BeFalse()) + Expect(report.Iteration).To(Equal(1)) + Expect(report.Validate()).To(Succeed()) + }) + }) +}) diff --git a/pkg/promptrun/run.go b/pkg/promptrun/run.go new file mode 100644 index 00000000..db532d61 --- /dev/null +++ b/pkg/promptrun/run.go @@ -0,0 +1,268 @@ +// Package promptrun runs one resolved prompt spec through captain's +// generate→verify loop: attachment checks, provider construction, tool-policy +// enforcement, the workflow's commit and verify hooks, the caller's own hooks, +// the setup plugin, and finally the agent.Runner. +// +// It is the seam `captain prompt run` and an embedding host (gavel's todo +// lifecycle) share. Both used to assemble the same pieces by hand and drifted: +// one applied the budget timeout and the other did not, one refused an +// unenforceable tool policy before the first model call and the other let the +// provider discover it mid-run. One function means one definition of a run. +// +// What stays with the caller: rendering the prompt and resolving its spec, +// resolving attachments against a store, persisting the run, and streaming +// events to whoever is watching (OnEvent is the tap for that). +package promptrun + +import ( + "context" + "fmt" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/agent" + "github.com/flanksource/captain/pkg/ai/agent/verify" + "github.com/flanksource/captain/pkg/ai/middleware" + "github.com/flanksource/captain/pkg/api" +) + +// Input is everything one run needs. +type Input struct { + // Request is the rendered prompt and its resolved spec. Attachments must + // already be resolved (see api.AttachmentRef.IsPrepared): the store that + // resolves them belongs to the caller. + Request ai.Request + // Config builds the provider: model, credentials, sandbox, cache, and the + // CanUseTool broker. Ignored for construction when Provider is set. + Config ai.Config + // Provider, when set, is used as-is and is taken to own the workspace — a + // remote-executing sandbox that materialises the checkout on its own side, + // or a test double. Run then adds no setup hook. Nil means Run constructs + // the provider from Config and prepares Request.Setup through the setup + // plugin, in-process. + Provider ai.Provider + // Hooks are the caller's own agent hooks — a host's commit pipeline, an + // environment stamp. They sit after the workflow's checks and before setup, + // so a Post hook of theirs at PhaseRun still sees a live worktree. + Hooks []any + // CallerOwnsCommits hands committing to Hooks: Run builds no commit hook of + // its own from Workflow.Commits, and leaves the declaration on the request so + // the recorded spec still says what the run commits and validation of that + // declaration still runs. It is for a host whose commit pipeline is its own + // (gavel's pre-commit gates and trailers) — without it that host commits the + // same tree twice, and stripping Workflow.Commits to avoid it drops the + // declaration from the spec. Setting it with no Hooks is an error: nothing + // would commit. + CallerOwnsCommits bool + // Verify configures the workflow's verifiers. Provider is filled from the + // run's provider when nil, and RunSpec from the resolved request; Progress + // receives in-flight snapshots. + Verify verify.Options + // OnEvent taps the live event stream: the model's events and the hooks'. It + // takes the runner's own signature — a renderer needs the turn an event + // belongs to, and a single-turn caller ignores it. + OnEvent func(iter int, ev ai.Event) + // Timeout bounds the whole run when the spec's budget.timeout is empty. There + // is no default: a run nobody bounded is the host's configuration to fix, and + // inventing a ceiling here killed long runs with nothing to point at. + Timeout time.Duration + // MaxIterations overrides the workflow's verify.maxIterations for a caller + // that names the loop bound itself (`captain ai agent --max-iterations`); + // zero means the workflow decides. + MaxIterations int + // Scope overrides the workflow's verify.scope; empty means the workflow + // decides. + Scope agent.Scope + // NoStream forces buffered execution even on a streaming provider. + NoStream bool + // Repo is the root of the tree the run's changed files are recorded relative + // to; empty means the request's cwd. + Repo string +} + +// Run executes one prompt run and returns its outcome. A failing verdict is a +// Result with Passed=false, not an error; an error means the run itself could +// not complete — a hook failed, the provider failed, the policy is unenforceable. +func Run(ctx context.Context, in Input) (Result, error) { + start := time.Now() + // One classification, the same one the runner makes: a run generates, or it + // verifies what is already there. Anything else — attachments or a message + // history with no prompt and nothing declared to verify — used to build a + // provider and then quietly report a pass having done neither. + if err := in.Request.ValidateRunnable(); err != nil { + return Result{}, fmt.Errorf("promptrun: %w", err) + } + if err := validateAttachments(in); err != nil { + return Result{}, err + } + timeout, err := runTimeout(in) + if err != nil { + return Result{}, err + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + if err := requireToolPolicy(in); err != nil { + return Result{}, err + } + if err := verify.ValidatePromptDeclarations(in.Request.Workflow); err != nil { + return Result{}, err + } + provider, release, err := buildProvider(in) + if err != nil { + return Result{}, err + } + defer release() + + hooks, err := Hooks(ctx, in, provider) + if err != nil { + return Result{}, err + } + streamer, err := runnerProvider(provider, in.NoStream, in.Request.IsVerifyOnly()) + if err != nil { + return Result{}, err + } + + var identity runIdentity + runner := &agent.Runner[string]{ + Provider: streamer, + Request: in.Request, + Hooks: hooks, + MaxIterations: maxIterations(in), + Repo: repoOf(in), + Cwd: in.Request.Cwd(), + Scope: scopeOf(in), + OnEvent: func(iter int, ev ai.Event) { + identity.observe(ev) + if in.OnEvent != nil { + in.OnEvent(iter, ev) + } + }, + } + out, runErr := runner.Run(ctx) + result, resultErr := newResult(out, identity, time.Since(start)) + if runErr != nil { + return result, runErr + } + return result, resultErr +} + +func maxIterations(in Input) int { + if in.MaxIterations > 0 { + return in.MaxIterations + } + return verify.MaxIterationsForWorkflow(in.Request.Workflow) +} + +func scopeOf(in Input) agent.Scope { + if in.Scope != "" { + return in.Scope + } + return verify.ScopeForWorkflow(in.Request.Workflow) +} + +// validateAttachments refuses a request whose attachments were never resolved +// — a path or URL the provider would have to fetch itself — and one whose +// resolved attachments the selected models cannot accept. +func validateAttachments(in Input) error { + refs := in.Request.Prompt.Attachments + if len(refs) == 0 { + return nil + } + for i, ref := range refs { + if !ref.IsPrepared() { + return fmt.Errorf("promptrun: attachment %d (%s) is not resolved; resolve attachments against a store before running", i, ref.Path+ref.URL+ref.ID) + } + } + model := executingModel(in) + models := append([]api.Model{model}, model.Fallbacks...) + return ai.ValidateAttachmentCompatibility(models, refs) +} + +// executingModel is the model this run will actually be answered by: the one +// middleware.NewProvider is handed, which is Config.Model whenever the config +// names one, and the request's own model otherwise (a caller that supplied its +// provider, or a test). +// +// It exists because the two pre-flight checks resolved it differently — the +// attachment check preferred the request's model and the tool-policy check the +// config's — so a prompt whose frontmatter named a different model than the +// config had its attachments validated against a runtime that would never see +// them, and its policy against one that could not enforce it. +func executingModel(in Input) api.Model { + if in.Config.Model.Name != "" || in.Config.Model.Provider != nil { + return in.Config.Model + } + return in.Request.Model +} + +// runTimeout is the spec's budget.timeout when declared, else the caller's. The +// spec wins because it is what the run's author declared; the caller's value is +// a host default. +// +// There is no third fallback. A compiled-in ceiling capped a run that had +// declared nothing, so a job that legitimately needed an hour died at two +// minutes with a deadline nobody had chosen and nothing naming it. +func runTimeout(in Input) (time.Duration, error) { + timeout, err := in.Request.Budget.ParseTimeout() + if err != nil { + return 0, fmt.Errorf("promptrun: %w", err) + } + if timeout > 0 { + return timeout, nil + } + if in.Timeout > 0 { + return in.Timeout, nil + } + return 0, fmt.Errorf("promptrun: no timeout: declare budget.timeout on the spec or set Input.Timeout") +} + +// requireToolPolicy refuses a per-tool policy the selected runtime cannot +// enforce before anything runs. Every provider repeats the check at execution +// time, but by then setup has materialised a checkout and a host has recorded +// a run that was never going to start. +func requireToolPolicy(in Input) error { + model := executingModel(in) + return api.RequireToolPolicySupport(model.Provider, model.Mode, in.Request.Permissions) +} + +// buildProvider returns the caller's provider, or constructs one from Config +// when the run will call a model: a generating run always does, and a +// verify-only run does when it declares judge prompts. A verify-only run of +// commands and fixtures needs none, so none is built — it must work without +// credentials to hand. +func buildProvider(in Input) (ai.Provider, func(), error) { + release := func() {} + if in.Provider != nil { + return in.Provider, release, nil + } + if in.Request.IsVerifyOnly() && !declaresPrompts(in.Request.Workflow) { + return nil, release, nil + } + cfg := in.Config + if in.Request.NoCache { + cfg.NoCache = true + } + provider, err := middleware.NewProvider(cfg) + if err != nil { + return nil, release, err + } + return provider, func() { closeProvider(provider) }, nil +} + +func declaresPrompts(wf *api.Workflow) bool { + return wf != nil && wf.Verify != nil && len(wf.Verify.Prompts) > 0 +} + +func closeProvider(provider ai.Provider) { + if closer, ok := api.ProviderAs[api.CloseableProvider](provider); ok { + _ = closer.Close() + } +} + +func repoOf(in Input) string { + if in.Repo != "" { + return in.Repo + } + return in.Request.Cwd() +} diff --git a/pkg/session/message.go b/pkg/session/message.go index 9f6fc53d..b9714e78 100644 --- a/pkg/session/message.go +++ b/pkg/session/message.go @@ -18,6 +18,11 @@ const ( PartReasoning = "reasoning" PartFile = "file" PartTool = "dynamic-tool" + // PartVerify carries an api.VerifyReport in Data, alongside the human-readable + // text part of the same verdict message. It follows the AI SDK's `data-` + // convention for a typed payload a renderer draws itself: a viewer that knows + // the verification tree draws it, and one that does not still has the text. + PartVerify = "data-verify" ) // Tool-part state machine values (AI SDK v6). @@ -78,6 +83,16 @@ type Part struct { Approval *Approval `json:"approval,omitempty"` } +// Roles the harness writes itself, alongside the provider's own user/assistant +// /system. A verify verdict gets its own role rather than sharing "system" with +// lifecycle narration, because it is the run's outcome: a reader filtering a +// stored session for "did this pass, and why not" should select on the role +// instead of matching on the text. +const ( + RoleVerified = "verified" + RoleVerifyFailed = "verify_failed" +) + // Message is one message in a session, matching aichat's UIMessage plus an // optional provenance extension. Raw retains the original JSONL line for // internal source-aware processing; explicit raw history output is handled by diff --git a/pkg/session/transcript_pretty.go b/pkg/session/transcript_pretty.go index d9aaf681..e05b79da 100644 --- a/pkg/session/transcript_pretty.go +++ b/pkg/session/transcript_pretty.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/flanksource/captain/pkg/api" "github.com/flanksource/captain/pkg/claude/tools" "github.com/flanksource/clicky" clickyapi "github.com/flanksource/clicky/api" @@ -172,6 +173,12 @@ func partTool(m Message, p Part, agent *Agent) tools.Tool { return newPrettyTool("File", map[string]any{ "filename": p.Filename, "url": p.URL, "mediaType": p.MediaType, }, m.Provenance, agent) + case PartVerify: + input := verifyPartInput(p) + if input == nil { + return nil + } + return newPrettyTool("Verify", input, m.Provenance, agent) default: if strings.HasPrefix(p.Type, "tool-") { name := strings.TrimPrefix(p.Type, "tool-") @@ -188,6 +195,34 @@ func partTool(m Message, p Part, agent *Agent) tools.Tool { } } +// verifyPartInput renders a stored verdict's typed report as the fields a +// transcript row shows: what was checked, how it ended, and the tally. +// +// Without this case a data-verify part fell through to the default branch, which +// renders an unknown part type from its Text — and a verdict's structure lives +// in Data, not Text. Every stored verdict therefore printed one blank row. A +// part with no data at all still prints nothing, which is the row this removes. +func verifyPartInput(p Part) map[string]any { + if len(p.Data) == 0 { + return nil + } + var report api.VerifyReport + if err := json.Unmarshal(p.Data, &report); err != nil { + return map[string]any{"verify": compactWhitespace(string(p.Data))} + } + input := map[string]any{"name": report.Name, "state": string(report.State)} + if report.Reason != "" { + input["reason"] = report.Reason + } + if s := report.Summary; s.Total > 0 { + input["tests"] = fmt.Sprintf("%d passed, %d failed of %d", s.Passed, s.Failed, s.Total) + } + if report.Iteration > 0 { + input["iteration"] = report.Iteration + } + return cleanToolInput(input) +} + func prettyRole(role string) string { switch strings.ToLower(strings.TrimSpace(role)) { case "user": @@ -196,6 +231,10 @@ func prettyRole(role string) string { return "Assistant" case "system": return "System" + case RoleVerified: + return "Verified" + case RoleVerifyFailed: + return "Verify failed" default: if role == "" { return "Message" diff --git a/pkg/session/transcript_verify_ginkgo_test.go b/pkg/session/transcript_verify_ginkgo_test.go new file mode 100644 index 00000000..b8d86f13 --- /dev/null +++ b/pkg/session/transcript_verify_ginkgo_test.go @@ -0,0 +1,51 @@ +package session + +import ( + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/captain/pkg/api" + "github.com/segmentio/encoding/json" +) + +var _ = ginkgo.Describe("a stored verify verdict in the transcript", func() { + report := func() []byte { + r := api.NewNodeReport(api.VerifyKindCmd, "verify:go test ./...", api.VerifyNode{ + Name: "go test ./...", Failed: true, Message: "TestFoo failed", + }) + r.Ran, r.Reason = true, "go test ./... failed" + raw, err := json.Marshal(r) + Expect(err).NotTo(HaveOccurred()) + return raw + } + + // The verdict message carries the prose and the report side by side. The + // data-verify part fell through to the default branch, which renders a part + // type it does not know from its (empty) Text — one blank row per verdict. + ginkgo.It("renders the report rather than an empty row", func() { + s := &Session{Messages: []Message{{ + Role: RoleVerifyFailed, + Parts: []Part{ + {Type: PartText, Text: "failed in 4ms — verify:go test ./..."}, + {Type: PartVerify, Data: report()}, + }, + }}} + + rows := s.TranscriptRows() + Expect(rows).To(HaveLen(2)) + rendered := rows[1].Pretty().String() + Expect(rendered).NotTo(BeEmpty()) + Expect(rendered).To(ContainSubstring("verify:go test ./...")) + Expect(rendered).To(ContainSubstring("failed")) + }) + + // A part with no data at all is a row with nothing to say; emitting it is + // the blank line this case exists to remove. + ginkgo.It("emits no row for a verdict part with no report", func() { + s := &Session{Messages: []Message{{ + Role: RoleVerified, + Parts: []Part{{Type: PartVerify}}, + }}} + Expect(s.TranscriptRows()).To(BeEmpty()) + }) +})