From 377af82a5e522f41bc7d469de6e5475833951035 Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Mon, 9 Nov 2026 22:19:18 +0000 Subject: [PATCH] thinking: note thinking models and add a profile switch Closes #24 --- CHANGELOG.md | 7 + catalogue/engines/README.md | 1 + catalogue/engines/vllm.json | 3 +- catalogue/model-types/README.md | 1 + catalogue/model-types/deepseek_v3.json | 11 ++ catalogue/model-types/deepseek_v4.json | 6 + catalogue/model-types/glm4_moe.json | 11 ++ catalogue/model-types/glm4_moe_lite.json | 11 ++ catalogue/model-types/glm4v_moe.json | 11 ++ catalogue/model-types/glm_moe_dsa.json | 11 ++ catalogue/model-types/qwen3.json | 11 ++ catalogue/model-types/qwen3_5.json | 5 + catalogue/model-types/qwen3_5_moe.json | 5 + catalogue/model-types/qwen3_moe.json | 11 ++ catalogue/model-types/smollm3.json | 11 ++ docs/model-types.md | 17 +- docs/models.md | 13 ++ docs/panel.md | 1 + docs/profiles.md | 7 + docs/providers.md | 1 + docs/reference/cli/new.md | 8 + internal/cli/editset.go | 1 + internal/cli/modeltypescmd.go | 19 +- internal/cli/profiles.go | 39 +++- internal/cli/run_test.go | 51 +++++ internal/cli/search.go | 9 +- internal/engine/engine.go | 7 + internal/engine/engine_test.go | 1 + internal/hf/hf.go | 38 ++++ internal/hf/hf_internal_test.go | 39 ++++ internal/modeltype/modeltype.go | 56 +++++- internal/modeltype/modeltype_internal_test.go | 65 ++++++ internal/profile/profile.go | 20 +- internal/profile/profile_test.go | 28 +++ internal/resolve/resolve.go | 27 ++- internal/resolve/thinking.go | 177 +++++++++++++++++ internal/resolve/thinking_internal_test.go | 187 ++++++++++++++++++ internal/vllm/validate.go | 2 +- internal/web/models.go | 38 +++- internal/web/server_internal_test.go | 28 +++ internal/web/static/app.css | 12 ++ internal/web/static/models.html | 12 ++ internal/web/static/models.js | 34 ++++ 43 files changed, 1026 insertions(+), 27 deletions(-) create mode 100644 catalogue/model-types/deepseek_v3.json create mode 100644 catalogue/model-types/glm4_moe.json create mode 100644 catalogue/model-types/glm4_moe_lite.json create mode 100644 catalogue/model-types/glm4v_moe.json create mode 100644 catalogue/model-types/glm_moe_dsa.json create mode 100644 catalogue/model-types/qwen3.json create mode 100644 catalogue/model-types/qwen3_moe.json create mode 100644 catalogue/model-types/smollm3.json create mode 100644 internal/resolve/thinking.go create mode 100644 internal/resolve/thinking_internal_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 63bf0d4..f80a506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,13 @@ the fact. `--tokenizer-mode deepseek_v4` and the `deepseek_v4` tool-call and reasoning parsers. The repos ship no chat template, so a profile without the tokenizer mode could not serve chat requests. +- Thinking models: `inspect` says when a model's chat template has a thinking + switch and whether it is on by default, `new` records it as `THINKING` in the + profile, and `list` and `status` show it. `new --thinking on|off` and the + panel's Thinking control set the default through vLLM's + `--default-chat-template-kwargs`. Covers Qwen3, Qwen3.5, GLM-4.5 and later, + DeepSeek V3.1 and V4, and SmolLM3; the switch lives in each model type's + catalogue file. - `glm-5.3-flash-tf-ablit-2spark` recipe: the GLM-5.3 Flash TensorFold launch across two DGX Sparks with the abliterated checkpoint, thinking kept on. diff --git a/catalogue/engines/README.md b/catalogue/engines/README.md index 8e5ff01..8057ff1 100644 --- a/catalogue/engines/README.md +++ b/catalogue/engines/README.md @@ -5,6 +5,7 @@ Each file here is one inference server llm-helper can launch, named after the en To add an engine, copy the closest file to `.json` and change it. No Go code is needed: shared code acts on the engine's `capabilities`, never on its name. +`chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; leave it out if the engine has none. The file is checked when it loads, and the tests in `internal/engine` render every file here for every rank. The fields are described in "Contributing an engine" in [docs/providers.md](../../docs/providers.md). diff --git a/catalogue/engines/vllm.json b/catalogue/engines/vllm.json index de8742a..5caf3ae 100644 --- a/catalogue/engines/vllm.json +++ b/catalogue/engines/vllm.json @@ -110,6 +110,7 @@ }, "capabilities": { "vllm_args": true, - "gguf_quant": true + "gguf_quant": true, + "chat_template_kwargs_flag": "--default-chat-template-kwargs" } } diff --git a/catalogue/model-types/README.md b/catalogue/model-types/README.md index 51fafcb..e8f9f7c 100644 --- a/catalogue/model-types/README.md +++ b/catalogue/model-types/README.md @@ -6,6 +6,7 @@ It says what every repo of that type needs on each engine, so the finetunes and To add one, copy the closest file to `.json`. Under `engines`, keyed by an engine id from [`catalogue/engines/`](../engines), it can list files to fetch (`files`), flags to add when the card did not name them (`args`), and chat template fixes (`chat_template_fixes`). Give each a `why`. +A type whose chat template has a switch for reasoning before the answer, such as Qwen3's `enable_thinking`, also gets a top-level `thinking` block; a file can have only that. A configuration for one specific repo that was run and measured belongs in [`catalogue/recipes/`](../recipes) instead. The fields are described in [docs/model-types.md](../../docs/model-types.md). diff --git a/catalogue/model-types/deepseek_v3.json b/catalogue/model-types/deepseek_v3.json new file mode 100644 index 0000000..9358498 --- /dev/null +++ b/catalogue/model-types/deepseek_v3.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "id": "deepseek_v3", + "display_name": "DeepSeek V3", + "why": "DeepSeek V3.1 and later V3 releases can think before they answer. V3-0324 and R1 have no switch in their chat template and are not flagged.", + "thinking": { + "kwarg": "thinking", + "default": false, + "why": "The chat template answers directly unless the thinking kwarg is true, which makes it reason first." + } +} diff --git a/catalogue/model-types/deepseek_v4.json b/catalogue/model-types/deepseek_v4.json index d789925..f40fa23 100644 --- a/catalogue/model-types/deepseek_v4.json +++ b/catalogue/model-types/deepseek_v4.json @@ -3,6 +3,12 @@ "id": "deepseek_v4", "display_name": "DeepSeek V4", "why": "DeepSeek V4 (Flash, Pro and the Flash Vision preview) ships no chat template. It encodes chats in Python, which vLLM carries as its deepseek_v4 tokenizer mode, and writes tool calls and thinking in its own format.", + "thinking": { + "kwarg": "thinking", + "default": true, + "detect": "always", + "why": "vLLM's deepseek_v4 tokenizer mode starts a thinking block before each reply unless the thinking kwarg is false." + }, "engines": { "vllm": { "args": [ diff --git a/catalogue/model-types/glm4_moe.json b/catalogue/model-types/glm4_moe.json new file mode 100644 index 0000000..f4c4e83 --- /dev/null +++ b/catalogue/model-types/glm4_moe.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "id": "glm4_moe", + "display_name": "GLM-4.5 / 4.6 / 4.7", + "why": "GLM-4.5, 4.5-Air, 4.6 and 4.7 think before they answer by default.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "The chat template reasons before answering unless enable_thinking is false." + } +} diff --git a/catalogue/model-types/glm4_moe_lite.json b/catalogue/model-types/glm4_moe_lite.json new file mode 100644 index 0000000..7e4b112 --- /dev/null +++ b/catalogue/model-types/glm4_moe_lite.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "id": "glm4_moe_lite", + "display_name": "GLM-4.7-Flash", + "why": "GLM-4.7-Flash thinks before it answers by default.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "The chat template reasons before answering unless enable_thinking is false." + } +} diff --git a/catalogue/model-types/glm4v_moe.json b/catalogue/model-types/glm4v_moe.json new file mode 100644 index 0000000..66ba817 --- /dev/null +++ b/catalogue/model-types/glm4v_moe.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "id": "glm4v_moe", + "display_name": "GLM-4.5V", + "why": "GLM-4.5V thinks before it answers by default.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "The chat template reasons before answering unless enable_thinking is false." + } +} diff --git a/catalogue/model-types/glm_moe_dsa.json b/catalogue/model-types/glm_moe_dsa.json new file mode 100644 index 0000000..7521e02 --- /dev/null +++ b/catalogue/model-types/glm_moe_dsa.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "id": "glm_moe_dsa", + "display_name": "GLM-5", + "why": "GLM-5 and models built on it think before they answer by default.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "The chat template reasons before answering unless enable_thinking is false." + } +} diff --git a/catalogue/model-types/qwen3.json b/catalogue/model-types/qwen3.json new file mode 100644 index 0000000..f624e17 --- /dev/null +++ b/catalogue/model-types/qwen3.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "id": "qwen3", + "display_name": "Qwen3", + "why": "Qwen3 dense models whose chat template has a thinking switch. The 2507 Instruct and Thinking releases have no switch and are not flagged.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "The chat template reasons inside before answering unless enable_thinking is false, which on short prompts can be most of the reply time." + } +} diff --git a/catalogue/model-types/qwen3_5.json b/catalogue/model-types/qwen3_5.json index bede09b..7f34727 100644 --- a/catalogue/model-types/qwen3_5.json +++ b/catalogue/model-types/qwen3_5.json @@ -3,6 +3,11 @@ "id": "qwen3_5", "display_name": "Qwen3.5", "why": "Qwen3.5 and models built on it (Ornith among them) ship a chat template that raises an error on a system message after the first turn.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "The chat template reasons inside before answering unless enable_thinking is false, which on short prompts can be most of the reply time." + }, "engines": { "vllm": { "chat_template_fixes": [ diff --git a/catalogue/model-types/qwen3_5_moe.json b/catalogue/model-types/qwen3_5_moe.json index 2a48bc8..cd1178c 100644 --- a/catalogue/model-types/qwen3_5_moe.json +++ b/catalogue/model-types/qwen3_5_moe.json @@ -3,6 +3,11 @@ "id": "qwen3_5_moe", "display_name": "Qwen3.5 MoE", "why": "Qwen3.5 and models built on it (Ornith among them) ship a chat template that raises an error on a system message after the first turn.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "The chat template reasons inside before answering unless enable_thinking is false, which on short prompts can be most of the reply time." + }, "engines": { "vllm": { "chat_template_fixes": [ diff --git a/catalogue/model-types/qwen3_moe.json b/catalogue/model-types/qwen3_moe.json new file mode 100644 index 0000000..a0a77a9 --- /dev/null +++ b/catalogue/model-types/qwen3_moe.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "id": "qwen3_moe", + "display_name": "Qwen3 MoE", + "why": "Qwen3 MoE models (30B-A3B, 235B-A22B) whose chat template has a thinking switch. The 2507 Instruct and Thinking releases have no switch and are not flagged.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "The chat template reasons inside before answering unless enable_thinking is false, which on short prompts can be most of the reply time." + } +} diff --git a/catalogue/model-types/smollm3.json b/catalogue/model-types/smollm3.json new file mode 100644 index 0000000..913f9a0 --- /dev/null +++ b/catalogue/model-types/smollm3.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "id": "smollm3", + "display_name": "SmolLM3", + "why": "SmolLM3 thinks before it answers by default.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "The chat template reasons inside before answering unless enable_thinking is false (or the system prompt says /no_think)." + } +} diff --git a/docs/model-types.md b/docs/model-types.md index 85263ef..bbe94d3 100644 --- a/docs/model-types.md +++ b/docs/model-types.md @@ -22,6 +22,11 @@ A launch configuration for one specific repo, run and measured on named hardware "id": "gpt_oss", "display_name": "gpt-oss", "why": "Why this architecture needs anything at all.", + "thinking": { + "kwarg": "enable_thinking", + "default": true, + "why": "What the switch does to replies." + }, "engines": { "vllm": { "files": [ @@ -55,12 +60,22 @@ A launch configuration for one specific repo, run and measured on named hardware ``` - `id` is the `model_type` value, and the file is named after it. +- `thinking` describes a chat template switch for reasoning before the answer. `kwarg` is the template variable, and `default` is whether the model thinks when nothing sets it. With `detect` left out or set to `template`, it applies only to repos whose chat template names `kwarg`, so a release of the same architecture without the switch is not flagged. `detect: always` applies it to every repo of the type, for types whose chat formatting lives in the engine rather than a template. - `engines` is keyed by engine id. An engine with no entry gets nothing added. `files` apply on any engine. `args` and `chat_template_fixes` are in vLLM's flag syntax, so they are only accepted for engines with the `vllm_args` capability. - `files` are fetched into `dir` under the data directory when a profile is created, unless already there. With `env` set, the profile gets that variable pointing at the directory. - `args` are added to the profile's `EXTRA_ARGS` unless the card already named the flag. `values` are tried in order and the first the installed engine registers is used; if none is registered, the flag is left out. `with` flags are added alongside. - `chat_template_fixes` apply when the repo's `chat_template.jinja` contains `contains`. The patched copy is saved under the data directory and passed with `--chat-template`. If the text is there but `match` does not match, nothing is patched and `new` says so. -Each file is checked when it loads: an unknown field, an unknown engine, `args` or `chat_template_fixes` for an engine without `vllm_args`, a bad regular expression, a `dir` outside the data directory or a `base_url` that is not https is refused. +A file needs `engines`, `thinking`, or both. +Each file is checked when it loads: an unknown field, an unknown engine, `args` or `chat_template_fixes` for an engine without `vllm_args`, a bad regular expression, a `dir` outside the data directory, a `base_url` that is not https, a `kwarg` that is not a template variable name or an unknown `detect` is refused. + +## Thinking models + +When a repo matches a `thinking` entry, `inspect` says whether the model thinks by default, `new` records it as `THINKING` in the profile, and `list` and `status` show it. +`new --thinking on|off` and the panel's Thinking control set it for the new profile. +They pass the kwarg through the engine's `chat_template_kwargs_flag` (`--default-chat-template-kwargs` on vLLM), so a client that sends its own `chat_template_kwargs` still overrides it per request. +An engine without that flag cannot switch thinking, and `new` says so. +Models that always think and have no switch are not flagged. ## Changing one locally diff --git a/docs/models.md b/docs/models.md index 842ffe2..bbbc169 100644 --- a/docs/models.md +++ b/docs/models.md @@ -17,6 +17,19 @@ Qwen3's cards are the case this covers: their `vllm serve` line names `deepseek_ `search` marks what is too big for this machine and what cannot load, and preselects the biggest quant that fits. `new` refuses a quant that cannot load rather than serving it — activating a profile restarts the service, so a model that cannot load would take the running one down with it. +## Thinking models + +Some models reason before they answer, and whether they do is a switch in their chat template. +`inspect` says when a repo has that switch and whether it is on by default, and `list` and `status` mark the profile. +A thinking pass can run to thousands of tokens before the first word of the answer, so turn it off for a model you want fast replies from: + +```sh +llm-helper inspect Qwen/Qwen3-0.6B # thinking: on by default +llm-helper new Qwen/Qwen3-0.6B --thinking off +``` + +Clients can still turn it back on per request; see [Model types](./model-types.md). + ## Choosing a quantization Reach for a quantized release when a model is large: roughly 1 byte per parameter for FP8/INT8 and 0.5 for INT4/AWQ/GPTQ, against the budget `doctor` reports. diff --git a/docs/panel.md b/docs/panel.md index 019b13f..671dfce 100644 --- a/docs/panel.md +++ b/docs/panel.md @@ -49,6 +49,7 @@ Picking a result checks its weight size against this machine's budget and its mo The weights themselves are downloaded by vLLM as it starts, so a big model spends a while in the panel's loading state before it serves. Saved models can be removed from the same list they are switched from — the active one can't, and the weights stay in your HuggingFace cache either way. +When the model is a thinking model, the page shows a **Thinking** control set to the model's default; turning it off saves a profile that answers without the reasoning pass. When the installed vLLM cannot load GGUF, the page offers to install the plugin; see [Models and quantizations](./models.md). ## Docs diff --git a/docs/profiles.md b/docs/profiles.md index f68a56d..087f364 100644 --- a/docs/profiles.md +++ b/docs/profiles.md @@ -25,6 +25,7 @@ EXTRA_ARGS="--enable-auto-tool-choice --tool-call-parser qwen3_coder" | `KV_CACHE_MEMORY_BYTES` | Size of the KV cache in bytes, computed when the profile is written. | | `EXTRA_ARGS` | Extra flags passed to the profile's server verbatim. | | `PROVIDER` | The server that runs this profile, `vllm` when absent. See [Providers](./providers.md). | +| `THINKING` | `on` or `off` for a [thinking model](./model-types.md), written by `new`. Shown by `list` and `status`. | Any other `KEY=VALUE` in the file is exported into the server's environment as-is. @@ -33,6 +34,12 @@ Any other `KEY=VALUE` in the file is exported into the server's environment as-i `EXTRA_ARGS` is parsed like a shell line, so keep it quoted when it carries more than one flag. A flag value with spaces in it — a JSON payload, for example — can be quoted inside. +### THINKING + +`THINKING` records what `new` set; the server is switched by the `--default-chat-template-kwargs` flag in `EXTRA_ARGS`. +To turn thinking off in an existing profile, add `--default-chat-template-kwargs '{"enable_thinking":false}'` to its `EXTRA_ARGS` (with the model's own kwarg, which `model-types show ` prints) and set `THINKING=off`. +Profiles written before this key existed have no `THINKING` line, and `list` shows nothing for them until they are re-created. + ### KV cache sizing `KV_CACHE_MEMORY_BYTES` is sized when the profile is written, from the model's own attention shape and `KV_CONTEXT_TARGET` in `config.env` — four full contexts by default. diff --git a/docs/providers.md b/docs/providers.md index ad22ab2..df2f0cd 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -181,6 +181,7 @@ A definition holds `schema_version` and the engine's fields, and says: - `serve` and `optional_args` for a process, or `container.roles` for a container: the command, with placeholders such as `{model}`, `{port}` and the engine's own params. An optional group is left out when any of its placeholders is empty. A container's `launch_args` take only what every rank shares: `{model}`, `{port}`, `{nodes}`, the params and the pinned models' paths. - `health`: the path that answers 200, and whether it does so only once the model is loaded (`ready`) or as soon as the process listens (`live`). - `capabilities`: the few differences shared code acts on, such as `vllm_args` for vLLM's sizing and flag checks. Launching and health checks act on a capability, never on an engine's name. + `chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; see [Model types](./model-types.md). The definition is checked when it loads. An unknown field, an unknown placeholder, a key that would land on the command line, or a container with no role that serves HTTP is refused with the reason. diff --git a/docs/reference/cli/new.md b/docs/reference/cli/new.md index 410ffb8..5d3115f 100644 --- a/docs/reference/cli/new.md +++ b/docs/reference/cli/new.md @@ -30,6 +30,14 @@ return as soon as the server is started instead of watching the boot which quantization to serve, for repos that ship several (see 'llm-helper inspect') +### --thinking + +| | | +| ---- | -------- | +| Type | `string` | + +on or off: whether a thinking model reasons before it answers (default: the model's own) + ### --timeout | | | diff --git a/internal/cli/editset.go b/internal/cli/editset.go index 7f4b6e4..6c3f014 100644 --- a/internal/cli/editset.go +++ b/internal/cli/editset.go @@ -24,6 +24,7 @@ var standardProfileKeys = map[string]bool{ "KV_CACHE_MEMORY_BYTES": true, "EXTRA_ARGS": true, profile.RecipeKey: true, + profile.ThinkingKey: true, engine.Key: true, } diff --git a/internal/cli/modeltypescmd.go b/internal/cli/modeltypescmd.go index e736d98..9bcbb4b 100644 --- a/internal/cli/modeltypescmd.go +++ b/internal/cli/modeltypescmd.go @@ -98,7 +98,11 @@ func runModelTypesList(out, errOut io.Writer, jsonOut bool) error { tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) fmt.Fprintln(tw, "ID\tENGINES\tNAME\tSOURCE") for _, r := range rows { - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", r.ID, strings.Join(engineIDs(r.Entry), ","), r.DisplayName, r.Source) + engines := strings.Join(engineIDs(r.Entry), ",") + if engines == "" { + engines = "-" + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", r.ID, engines, r.DisplayName, r.Source) } if err := tw.Flush(); err != nil { return err @@ -122,6 +126,19 @@ func runModelTypesShow(out io.Writer, id string, jsonOut bool) error { fmt.Fprintf(out, "%s (%s)\n\n", e.DisplayName, e.ID) fmt.Fprintf(out, " %-14s %s\n\n", "source", modelTypeSource(e)) fmt.Fprintln(out, e.Why) + if t := e.Thinking; t != nil { + detect := "when the chat template names " + t.Kwarg + if t.Detect == modeltype.DetectAlways { + detect = "on every repo of this type" + } + def := "off" + if t.Default { + def = "on" + } + fmt.Fprintf(out, "\nThinking (%s)\n", detect) + fmt.Fprintf(out, " chat template kwarg %s, %s unless a profile sets it\n", t.Kwarg, def) + fmt.Fprintf(out, " %s\n", t.Why) + } for _, id := range engineIDs(e) { r := e.Engines[id] fmt.Fprintf(out, "\nOn %s\n", id) diff --git a/internal/cli/profiles.go b/internal/cli/profiles.go index 4e1063c..70ff815 100644 --- a/internal/cli/profiles.go +++ b/internal/cli/profiles.go @@ -27,6 +27,7 @@ type profileInfo struct { GPUMemUtil string `json:"gpu_memory_utilization"` ExtraArgs string `json:"extra_args"` Status string `json:"status"` + Thinking string `json:"thinking,omitempty"` Active bool `json:"active"` } @@ -60,6 +61,7 @@ func runList(jsonOut bool) error { GPUMemUtil: p.File.Get("GPU_MEMORY_UTILIZATION"), ExtraArgs: p.ExtraArgs(), Status: p.Status(), + Thinking: p.Thinking(), Active: p.Name == active, }) } @@ -79,8 +81,11 @@ func printProfiles(cfg *config.Config) { active := cfg.ActiveProfile() for _, p := range profiles { marker := "" + if p.Thinking() == resolve.ThinkingOn { + marker = " [thinking]" + } if p.Name == active { - marker = " [active]" + marker += " [active]" } // A recipe profile with edits is no longer the thing we measured, so // say so wherever it is listed. @@ -160,17 +165,21 @@ func runSwitch(args []string, opts watchOpts) error { func newNewCmd() *cobra.Command { var w waitFlags - var quant string + var quant, thinking string c := &cobra.Command{ Use: "new ", Short: "Pull a new HuggingFace model, save it as a profile, and switch to it", GroupID: groupModels, Args: cobra.MaximumNArgs(1), Example: " llm-helper new Qwen/Qwen3-32B\n" + - " llm-helper new bartowski/Qwen_Qwen3-32B-GGUF --quant Q8_0", + " llm-helper new bartowski/Qwen_Qwen3-32B-GGUF --quant Q8_0\n" + + " llm-helper new Qwen/Qwen3-0.6B --thinking off", RunE: func(cmd *cobra.Command, args []string) error { if len(args) < 1 { - return errors.New("usage: llm-helper new [--quant NAME] [--wait|--no-follow] [--timeout 30m]") + return errors.New("usage: llm-helper new [--quant NAME] [--thinking on|off] [--wait|--no-follow] [--timeout 30m]") + } + if thinking != "" && thinking != resolve.ThinkingOn && thinking != resolve.ThinkingOff { + return fmt.Errorf("--thinking must be %s or %s, not %q", resolve.ThinkingOn, resolve.ThinkingOff, thinking) } cfg, err := loadConfig() if err != nil { @@ -180,11 +189,13 @@ func newNewCmd() *cobra.Command { if err != nil { return err } - return pullAndSwitch(cmd.Context(), cfg, args[0], choice, w.opts()) + return pullAndSwitch(cmd.Context(), cfg, args[0], choice, thinking, w.opts()) }, } c.Flags().StringVar(&quant, "quant", "", "which quantization to serve, for repos that ship several (see 'llm-helper inspect')") + c.Flags().StringVar(&thinking, "thinking", "", + "on or off: whether a thinking model reasons before it answers (default: the model's own)") w.register(c) return c } @@ -247,7 +258,7 @@ func resolveQuant(ctx context.Context, cfg *config.Config, repo, want string) (* // pullAndSwitch is `new` minus the flag parsing, so `search` can funnel // into it with its own options. -func pullAndSwitch(ctx context.Context, cfg *config.Config, repo string, choice *resolve.Choice, opts watchOpts) error { +func pullAndSwitch(ctx context.Context, cfg *config.Config, repo string, choice *resolve.Choice, thinking string, opts watchOpts) error { name := profile.NameFromRepo(repo) if choice != nil { if _, ok := choice.Variant.Selector(); ok { @@ -257,6 +268,9 @@ func pullAndSwitch(ctx context.Context, cfg *config.Config, repo string, choice p, err := profile.Load(cfg.Dirs, name) if err == nil { + if thinking != "" { + return fmt.Errorf("profile '%s' already exists, so --thinking has nothing to set; remove it first, or edit its EXTRA_ARGS", name) + } fmt.Printf("Profile '%s' already exists for %s -- using its saved settings.\n", name, repo) fmt.Printf("(change it with 'llm-helper edit %s' if it needs different flags)\n", name) return activateProfile(cfg, p, opts) @@ -275,6 +289,9 @@ func pullAndSwitch(ctx context.Context, cfg *config.Config, repo string, choice caps = vllmCaps(cfg) } plan := resolve.PrepareFor(ctx, cfg.Dirs, prov, repo, choice, caps, cfg.KVContextTarget()) + if err := plan.SetThinking(thinking, prov, caps); err != nil { + return err + } for _, n := range plan.Notes { fmt.Println(n) } @@ -405,6 +422,7 @@ type statusReport struct { Restarts int `json:"restarts"` FailReason string `json:"fail_reason,omitempty"` LastLog string `json:"last_log,omitempty"` + Thinking string `json:"thinking,omitempty"` } func gatherStatus(cfg *config.Config) statusReport { @@ -427,6 +445,9 @@ func gatherStatus(cfg *config.Config) statusReport { r.Model = m } } + if p, err := profile.Load(cfg.Dirs, r.Profile); err == nil { + r.Thinking = p.Thinking() + } if c, ok := run.(*clusterRunner); ok { r.HTTPStatus, r.Healthy = c.probeHealth() } else { @@ -464,6 +485,12 @@ func runStatus(jsonOut bool) error { fmt.Printf("%-9s %s\n", svc(cfg).Label()+":", r.Service) fmt.Printf("profile: %s\n", r.Profile) fmt.Printf("model: %s\n", r.Model) + switch r.Thinking { + case resolve.ThinkingOn: + fmt.Println("thinking: on -- replies begin after a reasoning pass") + case resolve.ThinkingOff: + fmt.Println("thinking: off") + } if r.Healthy { fmt.Printf("endpoint: healthy (http %d on port %s)\n", r.HTTPStatus, r.Port) return nil diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 317a176..8db57d3 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -330,6 +330,57 @@ func TestRunStatusJSON(t *testing.T) { } } +func TestRunListAndStatusShowThinking(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})) + t.Cleanup(srv.Close) + + configDir, profilePath := newTree(t) + config := "PORT=" + portOf(t, srv.URL) + "\nSERVICE_NAME=vllm-test\nSERVICE_SCOPE=user\n" + if err := os.WriteFile(filepath.Join(configDir, "config.env"), []byte(config), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(profilePath, []byte(sampleProfile+"THINKING=on\n"), 0o644); err != nil { + t.Fatal(err) + } + state := os.Getenv("VLLM_HELPER_STATE_DIR") + if err := os.MkdirAll(state, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(state, "active-profile"), []byte("qwen\n"), 0o600); err != nil { + t.Fatal(err) + } + + if _, stdout, _ := runCLI(t, "list"); !strings.Contains(stdout, "[thinking]") { + t.Errorf("list = %q, want the profile marked [thinking]", stdout) + } + _, stdout, _ := runCLI(t, "list", "--json") + var list []struct { + Thinking string `json:"thinking"` + } + if err := json.Unmarshal([]byte(stdout), &list); err != nil || len(list) != 1 || list[0].Thinking != "on" { + t.Errorf("list --json = %s (%v), want thinking on", stdout, err) + } + + if _, stdout, _ := runCLI(t, "status"); !strings.Contains(stdout, "thinking: on") { + t.Errorf("status = %q, want a thinking line", stdout) + } + _, stdout, _ = runCLI(t, "status", "--json") + var status struct { + Thinking string `json:"thinking"` + } + if err := json.Unmarshal([]byte(stdout), &status); err != nil || status.Thinking != "on" { + t.Errorf("status --json = %s (%v), want thinking on", stdout, err) + } +} + +func TestRunNewRefusesABadThinking(t *testing.T) { + newTree(t) + code, _, stderr := runCLI(t, "new", "Qwen/Qwen3-8B", "--thinking", "maybe") + if code == 0 || !strings.Contains(stderr, "--thinking must be") { + t.Errorf("exit %d, stderr %q, want --thinking refused", code, stderr) + } +} + func TestRunSet(t *testing.T) { tests := []struct { name string diff --git a/internal/cli/search.go b/internal/cli/search.go index 44f5978..6af61c8 100644 --- a/internal/cli/search.go +++ b/internal/cli/search.go @@ -107,7 +107,7 @@ func runSearch(ctx context.Context, args []string, jsonOut bool) error { label += " (" + choice.Variant.Label + ")" } if confirm(fmt.Sprintf("Pull %s and switch to it?", label), false) { - return pullAndSwitch(ctx, cfg, m.ID, choice, watchOpts{follow: stdoutIsTTY(), wait: true}) + return pullAndSwitch(ctx, cfg, m.ID, choice, "", watchOpts{follow: stdoutIsTTY(), wait: true}) } fmt.Println() } @@ -255,6 +255,13 @@ func printModelDetail(d resolve.Detail, cfg *config.Config) { fmt.Printf(" context: ~%s tokens in %.1f GB of KV cache%s\n", thousands(d.ContextTokens), d.KVCacheGB, full) } + if t := d.Thinking; t != nil { + if t.Default { + fmt.Println(" thinking: on by default -- replies begin after a reasoning pass ('new --thinking off' skips it)") + } else { + fmt.Println(" thinking: off by default ('new --thinking on' turns it on)") + } + } if d.DetectedFlags != "" { fmt.Printf(" vllm flags: %s (auto-detected, will be pre-filled)\n", d.DetectedFlags) } else { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 2030457..fd06da9 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -198,6 +198,9 @@ type Capabilities struct { // repo that holds several, named by appending its quant to the model: // "org/model-GGUF:Q8_0". Only then does choosing a quant mean anything. GGUFQuant bool `json:"gguf_quant,omitempty"` + // ChatTemplateKwargsFlag is the server flag that sets default chat + // template kwargs from a JSON object, such as a model's thinking switch. + ChatTemplateKwargsFlag string `json:"chat_template_kwargs_flag,omitempty"` } // Placeholders the templates may use, besides an engine's own params. @@ -237,6 +240,7 @@ var ( paramPattern = regexp.MustCompile(`^[a-z][a-z0-9_]*$`) envNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) placeholderRE = regexp.MustCompile(`\{([a-z][a-z0-9_]*)\}`) + capFlagPattern = regexp.MustCompile(`^--?[A-Za-z0-9][A-Za-z0-9-]*$`) ) // keyMode is a parsed api_key. @@ -785,6 +789,9 @@ func validate(e Engine, hw *hardware.Registry) []error { if !strings.HasPrefix(e.Health.Path, "/") { add("health.path %q must start with /", e.Health.Path) } + if f := e.Caps.ChatTemplateKwargsFlag; f != "" && !capFlagPattern.MatchString(f) { + add("capabilities.chat_template_kwargs_flag %q is not a flag", f) + } params := map[string]bool{} for _, p := range e.Params { diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 6b3ca70..8ecd8cb 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -174,6 +174,7 @@ func TestParseIsStrict(t *testing.T) { {"mount outside home", toyContainer, `"launch_order"`, `"mounts": [{"host": "/etc", "container": "/x"}], "launch_order"`, "under the home"}, {"process engine without install hint", toyProcess, `"install": { "any": "pip install toy" },`, ``, "install hint"}, {"trailing data", toyContainer, ``, ``, "after the top-level"}, + {"kwargs flag that is not a flag", toyContainer, `"capabilities": {}`, `"capabilities": {"chat_template_kwargs_flag": "kwargs"}`, "chat_template_kwargs_flag"}, } { data := strings.Replace(tc.base, tc.old, tc.new, 1) if tc.name == "trailing data" { diff --git a/internal/hf/hf.go b/internal/hf/hf.go index 95a6545..a2b0e3b 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -794,3 +794,41 @@ func ChatTemplate(ctx context.Context, repo string) (text string, found bool, er } return string(body), true, nil } + +// TemplateText returns a repo's chat template for reading: chat_template.jinja, +// or else the chat_template in tokenizer_config.json, with a list of named +// templates joined. It is "" when the repo has neither. +func TemplateText(ctx context.Context, repo string) (string, error) { + if text, found, err := ChatTemplate(ctx, repo); err != nil || found { + return text, err + } + resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/tokenizer_config.json") + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", nil + } + var cfg struct { + ChatTemplate json.RawMessage `json:"chat_template"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&cfg); err != nil { + return "", nil + } + var one string + if json.Unmarshal(cfg.ChatTemplate, &one) == nil { + return one, nil + } + var named []struct { + Template string `json:"template"` + } + if json.Unmarshal(cfg.ChatTemplate, &named) != nil { + return "", nil + } + parts := make([]string, 0, len(named)) + for _, n := range named { + parts = append(parts, n.Template) + } + return strings.Join(parts, "\n"), nil +} diff --git a/internal/hf/hf_internal_test.go b/internal/hf/hf_internal_test.go index a581ed2..9bc4f3d 100644 --- a/internal/hf/hf_internal_test.go +++ b/internal/hf/hf_internal_test.go @@ -743,6 +743,45 @@ func TestChatTemplate(t *testing.T) { } } +func TestTemplateText(t *testing.T) { + tests := []struct { + name string + jinja string + config string + want string + }{ + {name: "template file wins", jinja: "{{ enable_thinking }}", config: `{"chat_template":"other"}`, want: "{{ enable_thinking }}"}, + {name: "tokenizer config string", config: `{"chat_template":"{{ thinking }}"}`, want: "{{ thinking }}"}, + {name: "named templates are joined", config: `{"chat_template":[{"name":"default","template":"a"},{"name":"tool_use","template":"b"}]}`, want: "a\nb"}, + {name: "no template anywhere", want: ""}, + {name: "config without a template", config: `{"bos_token":""}`, want: ""}, + {name: "unreadable config", config: `not json`, want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + if tt.jinja != "" { + mux.HandleFunc("/"+repo+"/raw/main/chat_template.jinja", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(tt.jinja)) + }) + } + if tt.config != "" { + mux.HandleFunc("/"+repo+"/raw/main/tokenizer_config.json", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(tt.config)) + }) + } + newHub(t, mux) + got, err := TemplateText(context.Background(), repo) + if err != nil { + t.Fatalf("TemplateText: %v", err) + } + if got != tt.want { + t.Errorf("TemplateText = %q, want %q", got, tt.want) + } + }) + } +} + func TestClampContext(t *testing.T) { t.Parallel() diff --git a/internal/modeltype/modeltype.go b/internal/modeltype/modeltype.go index c5a4a26..4cb9c09 100644 --- a/internal/modeltype/modeltype.go +++ b/internal/modeltype/modeltype.go @@ -30,7 +30,10 @@ type Entry struct { DisplayName string `json:"display_name"` Why string `json:"why"` // Engines holds what this model type needs on each engine, by engine id. - Engines map[string]Rules `json:"engines"` + Engines map[string]Rules `json:"engines,omitempty"` + // Thinking, when set, describes the model's chat template switch for + // reasoning before it answers. + Thinking *Thinking `json:"thinking,omitempty"` // Source is the model-types.d file a local entry came from, and "" for // a shipped one. Source string `json:"-"` @@ -54,6 +57,34 @@ type FileSet struct { Why string `json:"why"` } +// Thinking is a chat template kwarg that turns reasoning on or off. +type Thinking struct { + Kwarg string `json:"kwarg"` + // Default is whether the model thinks when the kwarg is not passed. + Default bool `json:"default"` + // Detect is DetectTemplate or DetectAlways. + Detect string `json:"detect,omitempty"` + Why string `json:"why"` +} + +const ( + // DetectTemplate applies the switch only to repos whose chat template + // names the kwarg; it is the default. + DetectTemplate = "template" + // DetectAlways applies it to every repo of the model type, for types + // whose chat formatting lives in the engine rather than a template. + DetectAlways = "always" +) + +// Applies reports whether the switch is in template, a repo's chat template +// ("" when it has none). +func (t Thinking) Applies(template string) bool { + if t.Detect == DetectAlways { + return true + } + return regexp.MustCompile(`\b` + regexp.QuoteMeta(t.Kwarg) + `\b`).MatchString(template) +} + // Arg is a flag added to a new profile unless it already names it. Values // are tried in order and the first the installed engine accepts is used; with // none the flag is added bare. With is added alongside it. @@ -156,10 +187,11 @@ func Load(fsys fs.FS) (*Registry, error) { } var ( - idPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) - envPattern = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*$`) - flagPattern = regexp.MustCompile(`^--?[A-Za-z0-9][A-Za-z0-9-]*$`) - namePattern = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + idPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) + envPattern = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*$`) + flagPattern = regexp.MustCompile(`^--?[A-Za-z0-9][A-Za-z0-9-]*$`) + namePattern = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + kwargPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) ) // parse decodes one file strictly and checks it, so a typo fails when the @@ -190,8 +222,18 @@ func parse(name string, data []byte) (Entry, error) { if e.Why == "" { return Entry{}, errors.New("why is empty") } - if len(e.Engines) == 0 { - return Entry{}, errors.New("engines is empty") + if len(e.Engines) == 0 && e.Thinking == nil { + return Entry{}, errors.New("engines and thinking are both empty") + } + if t := e.Thinking; t != nil { + switch { + case !kwargPattern.MatchString(t.Kwarg): + return Entry{}, fmt.Errorf("thinking.kwarg %q is not a template variable name", t.Kwarg) + case t.Detect != "" && t.Detect != DetectTemplate && t.Detect != DetectAlways: + return Entry{}, fmt.Errorf("thinking.detect %q must be %s or %s", t.Detect, DetectTemplate, DetectAlways) + case t.Why == "": + return Entry{}, errors.New("thinking.why is empty") + } } for id, rules := range e.Engines { prov, err := engine.Lookup(id) diff --git a/internal/modeltype/modeltype_internal_test.go b/internal/modeltype/modeltype_internal_test.go index a57a1c8..b84cd5a 100644 --- a/internal/modeltype/modeltype_internal_test.go +++ b/internal/modeltype/modeltype_internal_test.go @@ -75,6 +75,10 @@ func TestBadFilesAreRefused(t *testing.T) { "http_url": {`{"schema_version":1,"id":"http_url","why":"w","engines":{"vllm":{"files":[{"dir":"x","base_url":"http://a/","names":["f"],"why":"w"}]}}}`, "https URL"}, "no_why": {`{"schema_version":1,"id":"no_why","engines":{"vllm":{}}}`, "why is empty"}, "wrong_version": {`{"schema_version":2,"id":"wrong_version","why":"w","engines":{"vllm":{}}}`, "schema_version"}, + "empty": {`{"schema_version":1,"id":"empty","why":"w"}`, "engines and thinking are both empty"}, + "bad_kwarg": {`{"schema_version":1,"id":"bad_kwarg","why":"w","thinking":{"kwarg":"enable thinking","why":"w"}}`, "thinking.kwarg"}, + "bad_detect": {`{"schema_version":1,"id":"bad_detect","why":"w","thinking":{"kwarg":"t","detect":"sometimes","why":"w"}}`, "thinking.detect"}, + "thinking_why": {`{"schema_version":1,"id":"thinking_why","why":"w","thinking":{"kwarg":"t"}}`, "thinking.why is empty"}, } { fsys := fstest.MapFS{name + ".json": {Data: []byte(c.body)}} if _, err := Load(fsys); err == nil || !strings.Contains(err.Error(), c.want) { @@ -83,6 +87,67 @@ func TestBadFilesAreRefused(t *testing.T) { } } +func TestAThinkingOnlyEntryLoads(t *testing.T) { + body := `{"schema_version":1,"id":"toy","why":"w","thinking":{"kwarg":"enable_thinking","default":true,"why":"w"}}` + reg, err := Load(fstest.MapFS{"toy.json": {Data: []byte(body)}}) + if err != nil { + t.Fatal(err) + } + e, ok := reg.Lookup("toy") + if !ok || e.Thinking == nil || !e.Thinking.Default || len(e.Engines) != 0 { + t.Errorf("toy = %+v, %v, want a thinking switch and no engines", e, ok) + } +} + +func TestThinkingApplies(t *testing.T) { + for _, c := range []struct { + name string + t Thinking + template string + want bool + }{ + {"kwarg in the template", Thinking{Kwarg: "enable_thinking"}, "{%- if enable_thinking is defined %}", true}, + {"kwarg absent", Thinking{Kwarg: "enable_thinking"}, "{{ messages }}", false}, + {"only part of a longer name", Thinking{Kwarg: "thinking"}, "{% if enable_thinking %}", false}, + {"no template", Thinking{Kwarg: "thinking"}, "", false}, + {"always, with no template", Thinking{Kwarg: "thinking", Detect: DetectAlways}, "", true}, + } { + if got := c.t.Applies(c.template); got != c.want { + t.Errorf("%s: Applies = %v, want %v", c.name, got, c.want) + } + } +} + +func TestTheShippedThinkingSwitches(t *testing.T) { + for id, want := range map[string]Thinking{ + "qwen3": {Kwarg: "enable_thinking", Default: true}, + "qwen3_moe": {Kwarg: "enable_thinking", Default: true}, + "qwen3_5": {Kwarg: "enable_thinking", Default: true}, + "qwen3_5_moe": {Kwarg: "enable_thinking", Default: true}, + "glm4_moe": {Kwarg: "enable_thinking", Default: true}, + "glm4_moe_lite": {Kwarg: "enable_thinking", Default: true}, + "glm_moe_dsa": {Kwarg: "enable_thinking", Default: true}, + "glm4v_moe": {Kwarg: "enable_thinking", Default: true}, + "smollm3": {Kwarg: "enable_thinking", Default: true}, + "deepseek_v3": {Kwarg: "thinking", Default: false}, + "deepseek_v4": {Kwarg: "thinking", Default: true, Detect: DetectAlways}, + } { + e, ok := Default().Lookup(id) + if !ok || e.Thinking == nil { + t.Errorf("%s: no thinking switch shipped", id) + continue + } + got := *e.Thinking + got.Why = "" + if got.Detect == DetectTemplate { + got.Detect = "" + } + if got != want { + t.Errorf("%s: thinking = %+v, want %+v", id, got, want) + } + } +} + func TestALocalFileAddsOrReplaces(t *testing.T) { dir := t.TempDir() replaced := `{"schema_version":1,"id":"gpt_oss","why":"local","engines":{"vllm":{}}}` diff --git a/internal/profile/profile.go b/internal/profile/profile.go index f03d723..c871c65 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -69,6 +69,13 @@ func (p *Profile) MasterPort() int { return n } +// ThinkingKey records whether the model thinks before it answers under this +// profile, "on" or "off". Absent when the model has no switch llm-helper knows. +const ThinkingKey = "THINKING" + +// Thinking returns "on", "off" or "". +func (p *Profile) Thinking() string { return strings.TrimSpace(p.File.Get(ThinkingKey)) } + // Recipe returns the recipe this profile was generated from, or "". func (p *Profile) Recipe() string { return strings.TrimSpace(p.File.Get(RecipeKey)) } @@ -170,6 +177,8 @@ type CreateOpts struct { // Env is KEY=value lines added to the profile, which reach the server's // environment. Env []string + // Thinking is written as THINKING when set. + Thinking string } // Create writes a new profile file. Fails if it already exists. @@ -195,7 +204,7 @@ func Create(dirs config.Dirs, opts CreateOpts) (*Profile, error) { content := fmt.Sprintf("# Status: %s (added via %s on %s)\nPROVIDER=%s\nMODEL=%s\nEXTRA_ARGS=%s\n", opts.Status, opts.Source, time.Now().Format("2006-01-02"), opts.Provider, opts.Repo, config.Quote(opts.ExtraArgs)) - content += envLines(opts.Env) + content += envLines(opts.Env) + thinkingLine(opts.Thinking) if err := os.WriteFile(path, []byte(content), 0o644); err != nil { return nil, err } @@ -212,7 +221,7 @@ EXTRA_ARGS=%s `, opts.Status, opts.Source, time.Now().Format("2006-01-02"), opts.Repo, opts.MaxModelLen, opts.MemUtilization, kvCacheValue(opts.KVCacheBytes), config.Quote(opts.ExtraArgs)) - content += envLines(opts.Env) + content += envLines(opts.Env) + thinkingLine(opts.Thinking) if err := os.WriteFile(path, []byte(content), 0o644); err != nil { return nil, err } @@ -228,6 +237,13 @@ func envLines(env []string) string { return b.String() } +func thinkingLine(v string) string { + if v == "" { + return "" + } + return ThinkingKey + "=" + config.Quote(v) + "\n" +} + // DefaultMaxModelLen is the context a profile gets when the model card did not // say. Long enough to be useful, short of the 1M-token configs that would make // the KV cache the dominant cost on any machine. diff --git a/internal/profile/profile_test.go b/internal/profile/profile_test.go index 98378c7..71ea0f4 100644 --- a/internal/profile/profile_test.go +++ b/internal/profile/profile_test.go @@ -264,6 +264,34 @@ func TestCreateForAProvider(t *testing.T) { } } +func TestThinkingIsWrittenOnlyWhenKnown(t *testing.T) { + t.Parallel() + for _, opts := range []profile.CreateOpts{ + {Repo: "Qwen/Qwen3-8B", Thinking: "off"}, + {Repo: "mlx-community/Qwen3-4B-4bit", Provider: "mlx", Thinking: "on"}, + {Repo: "meta-llama/Llama-3.1-8B"}, + } { + t.Run(opts.Repo, func(t *testing.T) { + t.Parallel() + dirs := newDirs(t) + p, err := profile.Create(dirs, opts) + if err != nil { + t.Fatalf("Create: %v", err) + } + loaded, err := profile.Load(dirs, p.Name) + if err != nil { + t.Fatalf("Load: %v", err) + } + if got := loaded.Thinking(); got != opts.Thinking { + t.Errorf("Thinking() = %q, want %q", got, opts.Thinking) + } + if got, want := loaded.File.Has(profile.ThinkingKey), opts.Thinking != ""; got != want { + t.Errorf("THINKING written = %v, want %v", got, want) + } + }) + } +} + func TestCreateRefusesToOverwrite(t *testing.T) { t.Parallel() diff --git a/internal/resolve/resolve.go b/internal/resolve/resolve.go index ecd9780..b3e7de2 100644 --- a/internal/resolve/resolve.go +++ b/internal/resolve/resolve.go @@ -113,6 +113,9 @@ type Detail struct { // once weights and overhead are paid for, and KVCacheGB what that costs. ContextTokens int64 `json:"context_tokens,omitempty"` KVCacheGB float64 `json:"kv_cache_gb,omitempty"` + // Thinking is the model's switch for reasoning before it answers, when + // its chat template has one. + Thinking *Thinking `json:"thinking,omitempty"` } // ToResult converts a Hub model into a search hit. @@ -172,6 +175,8 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps // A failed measurement is not the same as a repo with nothing to measure: // the pagination guards report an error precisely so a short listing is // not mistaken for a small model, and swallowing it here would undo that. + var arch hf.Arch + archRead := false if w, err := hf.WeightsSize(ctx, m.ID); err != nil { d.SizeError = err.Error() } else if w.SizeGB > 0 { @@ -184,7 +189,7 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps // a quant that cannot load is neither recommended nor selectable. // The alternative is finding out from a crash loop, by which point // the model that was serving has already been stopped. - arch := <-archCh + arch, archRead = <-archCh, true d.MaxContext = arch.MaxContext() d.KVBytesPerToken = arch.KVBytesPerToken(d.MaxContext) // Only a repo that publishes GGUF can be blocked by the plugin's @@ -229,6 +234,10 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps d.KVCacheGB = float64(plan.KVCacheBytes) / 1e9 } } + if !archRead { + arch = <-archCh + } + d.Thinking = detectThinking(ctx, arch.ModelType, m.ID, d.BaseModel) if flags, err := hf.DetectFlags(ctx, m.ID); err == nil { // Validate here too: seeing that a card's parser won't work on this // machine is most useful before pulling tens of gigabytes. @@ -303,6 +312,8 @@ func ForProvider(d Detail, prov engine.Provider) Detail { type Plan struct { Opts profile.CreateOpts Notes []string + // Thinking is the model's thinking switch, nil when it has none. + Thinking *Thinking } // PrepareFor is PrepareProfile for whichever provider will serve the model. @@ -311,8 +322,11 @@ type Plan struct { // provider names the model, gets its model type's files, and leaves the rest // to that server's defaults. func PrepareFor(ctx context.Context, dirs config.Dirs, prov engine.Provider, repo string, choice *Choice, caps vllm.Caps, contextTarget int) Plan { + flag := prov.Capabilities().ChatTemplateKwargsFlag if prov.Capabilities().VLLMArgs { - return PrepareProfile(ctx, dirs, prov.Name(), repo, choice, caps, contextTarget) + p := PrepareProfile(ctx, dirs, prov.Name(), repo, choice, caps, contextTarget) + p.noteThinking(flag) + return p } p := Plan{ Opts: profile.CreateOpts{ @@ -324,9 +338,15 @@ func PrepareFor(ctx context.Context, dirs config.Dirs, prov engine.Provider, rep Notes: []string{"Served by " + prov.Name() + ". The flags on a model card are vLLM's, so none were copied; " + "add any this server needs to EXTRA_ARGS."}, } - env, notes := otherEngineFiles(ctx, dirs.Data, prov.Name(), configRepoFor(repo, choice)) + cfgRepo := configRepoFor(repo, choice) + env, notes := otherEngineFiles(ctx, dirs.Data, prov.Name(), cfgRepo) p.Opts.Env = env p.Notes = append(p.Notes, notes...) + if flag != "" { + arch, _ := hf.Config(ctx, cfgRepo) + p.Thinking = detectThinking(ctx, arch.ModelType, repo, cfgRepo) + p.noteThinking(flag) + } // Named the way PrepareProfile names a quant, so `new` and the panel // find the profile again under the name they derive. if choice != nil && prov.Capabilities().GGUFQuant { @@ -388,6 +408,7 @@ func PrepareProfile(ctx context.Context, dirs config.Dirs, engineID, repo string arch, _ := hf.Config(ctx, configRepoFor(repo, choice)) rules := modeltype.Default().RulesFor(arch.ModelType, engineID) + p.Thinking = detectThinking(ctx, arch.ModelType, repo, configRepoFor(repo, choice)) if patched, why, err := fixChatTemplate(ctx, repo, rules.ChatTemplateFixes); err != nil { p.Notes = append(p.Notes, "Chat template check failed, continuing without it: "+err.Error()) diff --git a/internal/resolve/thinking.go b/internal/resolve/thinking.go new file mode 100644 index 0000000..294903b --- /dev/null +++ b/internal/resolve/thinking.go @@ -0,0 +1,177 @@ +package resolve + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/modeltype" + "github.com/coder/llm-helper/internal/vllm" +) + +// Thinking is a model's chat template switch for reasoning before it answers. +type Thinking struct { + Kwarg string `json:"kwarg"` + // Default is whether the model thinks when nothing sets the kwarg. + Default bool `json:"default"` + Why string `json:"why"` +} + +// What a new profile can ask for. +const ( + ThinkingOn = "on" + ThinkingOff = "off" +) + +// detectThinking returns the switch modelType's catalogue entry describes, +// when the repo's chat template has it. fallbackRepo is read when repo has no +// template, as GGUF repos often do. +func detectThinking(ctx context.Context, modelType, repo, fallbackRepo string) *Thinking { + e, ok := modeltype.Default().Lookup(modelType) + if !ok || e.Thinking == nil { + return nil + } + t := *e.Thinking + var text string + if t.Detect != modeltype.DetectAlways { + text, _ = hf.TemplateText(ctx, repo) + if text == "" && fallbackRepo != "" && fallbackRepo != repo { + text, _ = hf.TemplateText(ctx, fallbackRepo) + } + } + if !t.Applies(text) { + return nil + } + return &Thinking{Kwarg: t.Kwarg, Default: t.Default, Why: t.Why} +} + +// thinkingIn reports whether args turn thinking on through flag, and def +// when they do not set kwarg. The last occurrence wins, as it does for the +// server. +func thinkingIn(args, flag, kwarg string, def bool) bool { + words, err := config.SplitArgs(args) + if err != nil || flag == "" { + return def + } + for i, w := range words { + val, ok := strings.CutPrefix(w, flag+"=") + if !ok && w == flag && i+1 < len(words) { + val, ok = words[i+1], true + } + if !ok { + continue + } + var m map[string]any + if json.Unmarshal([]byte(val), &m) == nil { + if b, isBool := m[kwarg].(bool); isBool { + def = b + } + } + } + return def +} + +// withThinking returns args with kwarg set to on in flag's JSON object, +// merged into the object args already pass. +func withThinking(args, flag, kwarg string, on bool) (string, error) { + words, err := config.SplitArgs(args) + if err != nil { + return "", err + } + kwargs := map[string]any{} + var kept []string + for i := 0; i < len(words); i++ { + w := words[i] + val, ok := strings.CutPrefix(w, flag+"=") + if !ok && w == flag && i+1 < len(words) { + val, ok = words[i+1], true + i++ + } + if !ok { + kept = append(kept, w) + continue + } + if err := json.Unmarshal([]byte(val), &kwargs); err != nil { + return "", fmt.Errorf("%s %s is not a JSON object", flag, val) + } + } + kwargs[kwarg] = on + obj, err := json.Marshal(kwargs) + if err != nil { + return "", err + } + out := make([]string, 0, len(kept)+2) + for _, w := range kept { + out = append(out, config.Quote(w)) + } + out = append(out, flag, config.Quote(string(obj))) + return strings.Join(out, " "), nil +} + +// onOff renders a thinking state as the profile records it. +func onOff(on bool) string { + if on { + return ThinkingOn + } + return ThinkingOff +} + +// note is the plan note for a thinking state. +func (t *Thinking) note(on bool) string { + if on { + return "Thinking model: replies begin after a reasoning pass. " + t.Why + } + return "Thinking is off for this profile. " + t.Why +} + +// noteThinking records what the plan's args leave thinking at. +func (p *Plan) noteThinking(flag string) { + if p.Thinking == nil { + return + } + on := thinkingIn(p.Opts.ExtraArgs, flag, p.Thinking.Kwarg, p.Thinking.Default) + p.Opts.Thinking = onOff(on) + p.Notes = append(p.Notes, p.Thinking.note(on)) +} + +// SetThinking turns the model's thinking on or off in the profile, through +// the engine's flag for chat template kwargs. mode "" leaves the plan as it +// is. +func (p *Plan) SetThinking(mode string, prov engine.Provider, caps vllm.Caps) error { + if mode == "" { + return nil + } + if mode != ThinkingOn && mode != ThinkingOff { + return fmt.Errorf("thinking must be %s or %s, not %q", ThinkingOn, ThinkingOff, mode) + } + if p.Thinking == nil { + return fmt.Errorf("%s has no thinking switch llm-helper knows of", p.Opts.Repo) + } + flag := prov.Capabilities().ChatTemplateKwargsFlag + if flag == "" { + return fmt.Errorf("%s has no flag for chat template kwargs, so a profile cannot switch thinking", prov.Name()) + } + if prov.Capabilities().VLLMArgs && caps.KnowsFlags() && !caps.HasFlag(flag) { + return fmt.Errorf("the installed vLLM %s does not take %s, so a profile cannot switch thinking", caps.Version, flag) + } + on := mode == ThinkingOn + if thinkingIn(p.Opts.ExtraArgs, flag, p.Thinking.Kwarg, p.Thinking.Default) == on { + p.Opts.Thinking = mode + return nil + } + args, err := withThinking(p.Opts.ExtraArgs, flag, p.Thinking.Kwarg, on) + if err != nil { + return err + } + p.Opts.ExtraArgs = args + p.Opts.Thinking = mode + stale := p.Thinking.note(!on) + p.Notes = slices.DeleteFunc(p.Notes, func(n string) bool { return n == stale }) + p.Notes = append(p.Notes, p.Thinking.note(on), fmt.Sprintf("Thinking set %s with %s.", mode, flag)) + return nil +} diff --git a/internal/resolve/thinking_internal_test.go b/internal/resolve/thinking_internal_test.go new file mode 100644 index 0000000..c4c2df8 --- /dev/null +++ b/internal/resolve/thinking_internal_test.go @@ -0,0 +1,187 @@ +package resolve + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/vllm" +) + +const kwargsFlag = "--default-chat-template-kwargs" + +// serveThinking answers config.json with modelType and the chat template +// with template, or 404 when template is "". +func serveThinking(t *testing.T, modelType, template string) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/config.json"): + fmt.Fprintf(w, `{"model_type":%q}`, modelType) + case strings.HasSuffix(r.URL.Path, "chat_template.jinja") && template != "": + fmt.Fprint(w, template) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + t.Cleanup(hf.SetBaseURLForTest(srv.URL)) +} + +func vllmProvider(t *testing.T) engine.Provider { + t.Helper() + prov, err := engine.Lookup(engine.VLLM) + if err != nil { + t.Fatal(err) + } + return prov +} + +func TestAThinkingModelIsNotedOnItsProfile(t *testing.T) { + for _, c := range []struct { + name, modelType, template string + want string + }{ + {"switch in the template", "qwen3", "{%- if enable_thinking is defined and enable_thinking is false %}", ThinkingOn}, + {"no switch in the template", "qwen3", "{{ messages }}", ""}, + {"model type without a switch", "llama", "{{ enable_thinking }}", ""}, + {"switch off by default", "deepseek_v3", "{% if thinking %}{% endif %}", ThinkingOff}, + {"engine-side formatting with no template", "deepseek_v4", "", ThinkingOn}, + } { + t.Run(c.name, func(t *testing.T) { + serveThinking(t, c.modelType, c.template) + p := PrepareFor(context.Background(), config.Dirs{Data: t.TempDir()}, vllmProvider(t), "org/model", nil, vllm.Caps{}, 0) + if p.Opts.Thinking != c.want { + t.Errorf("Thinking = %q, want %q", p.Opts.Thinking, c.want) + } + if (p.Thinking != nil) != (c.want != "") { + t.Errorf("plan Thinking = %+v, want it set only for a thinking model", p.Thinking) + } + noted := slices.ContainsFunc(p.Notes, func(n string) bool { return strings.Contains(n, "Thinking") }) + if noted != (c.want != "") { + t.Errorf("Notes = %q, want a thinking note only for a thinking model", p.Notes) + } + }) + } +} + +func TestSetThinkingWritesTheKwarg(t *testing.T) { + serveThinking(t, "qwen3", "{{ enable_thinking }}") + p := PrepareFor(context.Background(), config.Dirs{Data: t.TempDir()}, vllmProvider(t), "org/model", nil, vllm.Caps{}, 0) + before := p.Opts.ExtraArgs + if err := p.SetThinking(ThinkingOff, vllmProvider(t), vllm.Caps{}); err != nil { + t.Fatal(err) + } + if p.Opts.Thinking != ThinkingOff { + t.Errorf("Thinking = %q, want off", p.Opts.Thinking) + } + words, err := config.SplitArgs(p.Opts.ExtraArgs) + if err != nil { + t.Fatal(err) + } + n := len(words) + if n < 2 || words[n-2] != kwargsFlag || words[n-1] != `{"enable_thinking":false}` { + t.Errorf("ExtraArgs = %q, want it to end with %s and the kwarg", p.Opts.ExtraArgs, kwargsFlag) + } + if !strings.HasPrefix(p.Opts.ExtraArgs, before) { + t.Errorf("ExtraArgs = %q, want %q kept", p.Opts.ExtraArgs, before) + } + if slices.ContainsFunc(p.Notes, func(n string) bool { return strings.HasPrefix(n, "Thinking model:") }) { + t.Errorf("Notes = %q, want the note for thinking on replaced", p.Notes) + } + + // Asking for what the model already does adds nothing. + q := PrepareFor(context.Background(), config.Dirs{Data: t.TempDir()}, vllmProvider(t), "org/model", nil, vllm.Caps{}, 0) + args := q.Opts.ExtraArgs + if err := q.SetThinking(ThinkingOn, vllmProvider(t), vllm.Caps{}); err != nil { + t.Fatal(err) + } + if q.Opts.ExtraArgs != args || q.Opts.Thinking != ThinkingOn { + t.Errorf("ExtraArgs = %q, Thinking = %q, want the args unchanged and thinking on", q.Opts.ExtraArgs, q.Opts.Thinking) + } +} + +// The value goes through the env file, so it must read back as the JSON it +// was written as. +func TestTheKwargSurvivesTheEnvFile(t *testing.T) { + args, err := withThinking(`--max-model-len 8192 --default-chat-template-kwargs '{"custom":"a b"}'`, kwargsFlag, "enable_thinking", false) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "p.env") + if err := os.WriteFile(path, []byte("EXTRA_ARGS="+config.Quote(args)+"\n"), 0o644); err != nil { + t.Fatal(err) + } + f, err := config.ParseEnvFile(path) + if err != nil { + t.Fatal(err) + } + words, err := config.SplitArgs(f.Get("EXTRA_ARGS")) + if err != nil { + t.Fatal(err) + } + want := []string{"--max-model-len", "8192", kwargsFlag, `{"custom":"a b","enable_thinking":false}`} + if !slices.Equal(words, want) { + t.Errorf("words = %q, want %q", words, want) + } +} + +func TestThinkingInTakesTheLastKwarg(t *testing.T) { + for _, c := range []struct { + args string + def bool + want bool + }{ + {"", true, true}, + {"--max-model-len 8192", false, false}, + {`--default-chat-template-kwargs {"enable_thinking":false}`, true, false}, + {`--default-chat-template-kwargs='{"enable_thinking":true}'`, false, true}, + {`--default-chat-template-kwargs {"enable_thinking":true} --default-chat-template-kwargs {"enable_thinking":false}`, true, false}, + {`--default-chat-template-kwargs {"other":1}`, true, true}, + } { + if got := thinkingIn(c.args, kwargsFlag, "enable_thinking", c.def); got != c.want { + t.Errorf("thinkingIn(%q, default %v) = %v, want %v", c.args, c.def, got, c.want) + } + } +} + +func TestSetThinkingRefuses(t *testing.T) { + serveThinking(t, "qwen3", "{{ enable_thinking }}") + dirs := config.Dirs{Data: t.TempDir()} + thinking := PrepareFor(context.Background(), dirs, vllmProvider(t), "org/model", nil, vllm.Caps{}, 0) + mlx, err := engine.Lookup(engine.MLX) + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + name string + plan Plan + mode string + prov engine.Provider + caps vllm.Caps + want string + }{ + {"a mode that is not on or off", thinking, "maybe", vllmProvider(t), vllm.Caps{}, "on or off"}, + {"a model with no switch", Plan{}, ThinkingOff, vllmProvider(t), vllm.Caps{}, "no thinking switch"}, + {"an engine with no kwargs flag", thinking, ThinkingOff, mlx, vllm.Caps{}, "no flag for chat template kwargs"}, + {"a vLLM without the flag", thinking, ThinkingOff, vllmProvider(t), vllm.Caps{Version: "0.8.0", Flags: []string{"--port"}}, "does not take"}, + } { + p := c.plan + if err := p.SetThinking(c.mode, c.prov, c.caps); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: err = %v, want one mentioning %q", c.name, err, c.want) + } + } + p := thinking + if err := p.SetThinking("", vllmProvider(t), vllm.Caps{}); err != nil || p.Opts.ExtraArgs != thinking.Opts.ExtraArgs { + t.Errorf("an empty mode: err %v, ExtraArgs %q, want the plan left alone", err, p.Opts.ExtraArgs) + } +} diff --git a/internal/vllm/validate.go b/internal/vllm/validate.go index 6564482..848d79d 100644 --- a/internal/vllm/validate.go +++ b/internal/vllm/validate.go @@ -131,7 +131,7 @@ func unknownParserMessage(flag, value, pluginFile string, caps Caps) string { // nearest finds a registered name plausibly meant by an unknown one. It is // deliberately conservative: a shared leading or trailing word, not an edit -// distance, because "deepseek_r1" and "deepseek_v3" are one character apart +// distance, because deepseek_r1 and deepseek_v3 are one character apart // and completely different parsers. func nearest(unknown string, registry []string) string { parts := strings.FieldsFunc(strings.ToLower(unknown), func(r rune) bool { return r == '_' || r == '-' }) diff --git a/internal/web/models.go b/internal/web/models.go index ae68251..eeaefe7 100644 --- a/internal/web/models.go +++ b/internal/web/models.go @@ -82,6 +82,18 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { type inspectResponse struct { resolve.Detail ExistingProfile string `json:"existing_profile,omitempty"` + // ThinkingSwitch is whether a new profile here can turn the model's + // thinking on or off. + ThinkingSwitch bool `json:"thinking_switch,omitempty"` +} + +// thinkingSwitchable reports whether prov can set a thinking model's default. +func thinkingSwitchable(d resolve.Detail, prov engine.Provider, caps vllm.Caps) bool { + flag := prov.Capabilities().ChatTemplateKwargsFlag + if d.Thinking == nil || flag == "" { + return false + } + return !prov.Capabilities().VLLMArgs || !caps.KnowsFlags() || caps.HasFlag(flag) } // profileHolds reports whether a derived profile name is taken, and whether @@ -115,8 +127,9 @@ func (s *Server) handleInspect(w http.ResponseWriter, r *http.Request) { } type result struct { - d resolve.Detail - err error + d resolve.Detail + canSwitch bool + err error } ch := make(chan result, 1) go func() { @@ -131,7 +144,8 @@ func (s *Server) handleInspect(w http.ResponseWriter, r *http.Request) { if prov.Capabilities().VLLMArgs { caps = s.caps() } - ch <- result{d: resolve.ForProvider(resolve.Inspect(ctx, m, s.budget(), caps, s.cfg.KVContextTarget()), prov)} + d := resolve.ForProvider(resolve.Inspect(ctx, m, s.budget(), caps, s.cfg.KVContextTarget()), prov) + ch <- result{d: d, canSwitch: thinkingSwitchable(d, prov, caps)} }() select { @@ -146,7 +160,7 @@ func (s *Server) handleInspect(w http.ResponseWriter, r *http.Request) { }) return } - out := inspectResponse{Detail: res.d} + out := inspectResponse{Detail: res.d, ThinkingSwitch: res.canSwitch} name := profile.NameFromRepo(repo) if taken, mine := s.profileHolds(name, repo); taken && mine { out.ExistingProfile = name @@ -184,6 +198,11 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusForbidden, map[string]string{"error": reason}) return } + thinking := r.URL.Query().Get("thinking") + if thinking != "" && thinking != resolve.ThinkingOn && thinking != resolve.ThinkingOff { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "thinking must be on or off"}) + return + } // An explicit quant is resolved against what the repo actually holds, // so a stale page cannot ask for one that is gone -- or one that vLLM @@ -262,6 +281,13 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { }) return case taken: + if thinking != "" { + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "profile '" + name + "' already exists, so there is no new profile to set thinking on -- " + + "remove it first, or edit its EXTRA_ARGS", + }) + return + } out.Existing = true out.Notes = append(out.Notes, "Profile '"+name+"' already existed -- switching to it with its saved settings.") @@ -271,6 +297,10 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { caps = s.caps() } plan := resolve.PrepareFor(r.Context(), s.cfg.Dirs, prov, repo, choice, caps, s.cfg.KVContextTarget()) + if err := plan.SetThinking(thinking, prov, caps); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } out.Notes = append(out.Notes, plan.Notes...) if _, err := profile.Create(s.cfg.Dirs, plan.Opts); err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) diff --git a/internal/web/server_internal_test.go b/internal/web/server_internal_test.go index fa3dd39..892bd6d 100644 --- a/internal/web/server_internal_test.go +++ b/internal/web/server_internal_test.go @@ -533,6 +533,34 @@ func TestPullRefusesAQuantItCannotResolve(t *testing.T) { } } +func TestPullChecksThinking(t *testing.T) { + s := newTestServer(t) + h := s.Handler(nil) + + for _, c := range []struct { + query string + want int + }{ + {"repo=org/demo&thinking=maybe", http.StatusBadRequest}, + {"repo=org/demo&thinking=off", http.StatusConflict}, + } { + req := httptest.NewRequest("POST", "/api/pull?"+c.query, nil) + req.Header.Set(tokenHeader, s.token) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != c.want { + t.Errorf("POST /api/pull?%s = %d, want %d: %s", c.query, rec.Code, c.want, rec.Body) + } + } + b, err := os.ReadFile(s.cfg.Dirs.ProfileFile("demo")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(b), "THINKING") { + t.Error("a refused pull changed the existing profile") + } +} + // The panel offers the same install the CLI does, so the endpoints behind it // need the same guard as every other mutation -- this one runs pip against the // machine's vLLM environment. diff --git a/internal/web/static/app.css b/internal/web/static/app.css index f60de6d..8bc2a40 100644 --- a/internal/web/static/app.css +++ b/internal/web/static/app.css @@ -892,6 +892,18 @@ input:focus-visible { font-size: 0.6875rem; } +/* ---------- thinking ---------- */ + +.thinking { + margin: 0 0 1rem; +} + +.thinking-note { + margin: 0 0 0.375rem; + color: var(--content-secondary); + font-size: 0.75rem; +} + /* ---------- gguf offer ---------- */ .gguf { diff --git a/internal/web/static/models.html b/internal/web/static/models.html index aaaa05e..26343b7 100644 --- a/internal/web/static/models.html +++ b/internal/web/static/models.html @@ -98,6 +98,18 @@

+