diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f604f0..16ec590 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,12 @@ the fact. ### Fixed +- `inspect`, `new` and the panel sized the KV cache of GLM-5.3-Flash about + 50 times too large: its 34 linear-attention layers counted as full + attention, and its 11 latent-attention layers as 64 full heads. A + `linear_attention` layer now holds no cache, and a latent-attention layer + (`kv_lora_rank`, as in DeepSeek V2 and V3 and GLM-5) holds its latent, rotary + key and indexer key. DeepSeek V3's estimate falls about 25 times with it. - Profiles for Qwen3 hybrid models got `--reasoning-parser deepseek_r1` from the card's `vllm serve` line. With thinking turned off (`enable_thinking: false`), a non-streaming reply then came back with diff --git a/docs/providers.md b/docs/providers.md index 0bab677..163b959 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -151,6 +151,25 @@ llm-helper set deepseek-v4-flash-vision-exp-gguf-ud-q8_k_xl -- \ - It writes about 33 tokens/sec, reads prompts at about 825 tokens/sec, and uses 161.8 of the 240 GiB of GPU memory. - Images, tool calls and thinking work through the chat template in the GGUF. +### Example: GLM-5.3-Flash on a Mac Studio + +This ran on the same Mac, with the same llama.cpp 0.8.0. +llama.cpp runs GLM-5.3-Flash since ggml-org/llama.cpp#27773, and its MTP head since #29928. +The model is Unsloth's `UD-Q4_K_XL` GGUF (186 GiB in 6 files), plus `mmproj-F16.gguf` for images. + +```sh +llm-helper new unsloth/GLM-5.3-Flash-GGUF --quant UD-Q4_K_XL +llm-helper set glm-5.3-flash-gguf-ud-q4_k_xl -- \ + EXTRA_ARGS="-c 1048576 -np 4 -ngl all --fit off -fa on -b 4096 -ub 2048 --spec-type draft-mtp --temp 1.0 --top-p 0.95 --min-p 0.0 --metrics --offline" +``` + +- `--spec-type draft-mtp` drafts tokens with the model's own MTP head. No second model is needed. 46 to 68 percent of drafts were accepted. +- One request writes about 58 tokens/sec. Four at once write about 16 tokens/sec each, 63 in total. These numbers come from Huihui's abliterated copy of the same quant (`huihui-ai/Huihui-GLM-5.3-Flash-abliterated-GGUF`), which has the same shapes. +- It reads prompts at about 585 tokens/sec, slower than DeepSeek V4 Flash on this Mac. +- `-c 1048576 -np 4` gives 4 slots of 262,144 tokens. Most of its layers are linear attention, so the KV cache is small (`inspect` gives about 14 KB a token). +- Images and tool calls work through the chat template in the GGUF. +- It always thinks, and the default `reasoning_effort` is `max`. A short question can use thousands of tokens. A client can send `"reasoning_effort": "low"` or `"high"`. + A machine can hold profiles for different providers. On the same Mac, stopping this profile and starting a vLLM (vllm-metal) Qwen3-Coder profile in its place took about 20 seconds, and the same back. diff --git a/internal/hf/arch.go b/internal/hf/arch.go index a457d41..3b1b850 100644 --- a/internal/hf/arch.go +++ b/internal/hf/arch.go @@ -30,6 +30,14 @@ type Arch struct { HiddenSize int `json:"hidden_size"` MaxPositionEmbed int `json:"max_position_embeddings"` TorchDtype string `json:"torch_dtype"` + // KVLoraRank and QKRopeHeadDim are set on multi-head latent attention + // (DeepSeek V2/V3, GLM-5), which caches one compressed latent and the + // rotary key per token and layer instead of K and V for every head. + KVLoraRank int `json:"kv_lora_rank"` + QKRopeHeadDim int `json:"qk_rope_head_dim"` + // IndexHeadDim is the key width of a DeepSeek sparse attention indexer, + // which caches one indexer key per token beside the latent. + IndexHeadDim int `json:"index_head_dim"` // LayerTypes names each layer for hybrids, where only some attend. LayerTypes []string `json:"layer_types"` // SlidingWindow caps how far a sliding-attention layer looks back, so @@ -48,12 +56,15 @@ func (a Arch) text() Arch { } // attentionLayers counts the layers holding a KV cache, split by how much each -// holds. Hybrids interleave attention with Mamba layers that hold none, and -// sliding layers hold only their window -- gpt-oss is half full, half sliding -// at a window of 128, so treating them alike doubles the estimate. +// holds. Hybrids interleave attention with Mamba or linear-attention layers +// that hold a fixed-size state rather than a cache that grows with the +// context, and sliding layers hold only their window -- gpt-oss is half full, +// half sliding at a window of 128, so treating them alike doubles the estimate. func (a Arch) attentionLayers() (full, sliding int) { for _, t := range a.LayerTypes { switch t := strings.ToLower(t); { + case strings.Contains(t, "linear"): + // GLM-5.3 names its Kimi Delta layers linear_attention. case strings.Contains(t, "sliding"): sliding++ case strings.Contains(t, "attention"): @@ -93,10 +104,22 @@ func (a Arch) KVBytesPerToken(maxLen int) int64 { // Last resort only; deriving over a declared value is the OOM above. dim = t.HiddenSize / t.NumAttentionHeads } + if t.KVLoraRank > 0 { + // Latent attention: vLLM and llama.cpp cache kv_lora_rank + + // qk_rope_head_dim per token, whatever the head count, plus the + // sparse-attention indexer's key where there is one (stored narrower + // than this counts it). Counting heads overstates DeepSeek V3 about + // 25 times. + heads, dim = 1, t.KVLoraRank+t.QKRopeHeadDim+t.IndexHeadDim + } if full+sliding <= 0 || heads <= 0 || dim <= 0 || maxLen <= 0 { return 0 } perLayer := 2 * int64(heads) * int64(dim) * int64(kvDtypeBytes(t.TorchDtype)) + if t.KVLoraRank > 0 { + // One latent per token, not a K and a V. + perLayer /= 2 + } // A sliding layer never holds more than its window, so past that point // it stops costing anything per extra token of context. diff --git a/internal/hf/kv_internal_test.go b/internal/hf/kv_internal_test.go index f0759c1..96154fe 100644 --- a/internal/hf/kv_internal_test.go +++ b/internal/hf/kv_internal_test.go @@ -1,6 +1,10 @@ package hf -import "testing" +import ( + "encoding/json" + "os" + "testing" +) // Every figure below was read off a live config.json and, where noted, // confirmed against what vLLM then allocated on real hardware. @@ -167,3 +171,54 @@ func TestADenseModelIgnoresSlidingWindow(t *testing.T) { t.Errorf("KVBytesPerToken = %d, want %d", got, want) } } + +// GLM-5.3-Flash (config.json trimmed to its shape, from zai-org/GLM-5.3-Flash) +// is 34 Kimi Delta linear-attention layers, which hold a fixed-size state, and +// 11 latent-attention layers that cache a 512-wide latent and a 128-wide +// indexer key per token. Counting every layer as 64 full heads put about +// 380k tokens in 280 GB, so a 256 GB Mac looked unable to hold any context. +func TestGLM53FlashCountsOnlyItsLatentLayers(t *testing.T) { + raw, err := os.ReadFile("testdata/glm-5.3-flash-config.json") + if err != nil { + t.Fatal(err) + } + var a Arch + if err := json.Unmarshal(raw, &a); err != nil { + t.Fatal(err) + } + const maxLen = 1048576 + if got, want := a.KVBytesPerToken(maxLen), int64(11*(512+128)*2); got != want { + t.Errorf("KVBytesPerToken() = %d, want %d (11 latent layers of 45)", got, want) + } + if got := a.MaxContext(); got != maxLen { + t.Errorf("MaxContext() = %d, want %d", got, maxLen) + } +} + +// Latent attention caches kv_lora_rank + qk_rope_head_dim per token and +// layer whatever the head count: 576 for DeepSeek V3, as vLLM allocates it. +func TestLatentAttentionCachesTheLatentNotEveryHead(t *testing.T) { + a := Arch{ + NumHiddenLayers: 61, NumAttentionHeads: 128, NumKeyValueHeads: 128, + HiddenSize: 7168, KVLoraRank: 512, QKRopeHeadDim: 64, TorchDtype: "bfloat16", + } + if got, want := a.KVBytesPerToken(163840), int64(61*(512+64)*2); got != want { + t.Errorf("KVBytesPerToken() = %d, want %d", got, want) + } +} + +// A hybrid that names its layers linear_attention and full_attention: the +// linear name holds "attention", which must not make those layers count as full. +func TestLinearAttentionLayersHoldNoCache(t *testing.T) { + types := make([]string, 48) + for i := range types { + types[i] = "linear_attention" + if i%4 == 3 { + types[i] = "full_attention" + } + } + a := Arch{NumHiddenLayers: 48, LayerTypes: types, NumKeyValueHeads: 2, HeadDim: 256, TorchDtype: "bfloat16"} + if got, want := a.KVBytesPerToken(262144), int64(2*12*2*256*2); got != want { + t.Errorf("KVBytesPerToken() = %d, want %d (12 full layers of 48)", got, want) + } +} diff --git a/internal/hf/testdata/glm-5.3-flash-config.json b/internal/hf/testdata/glm-5.3-flash-config.json new file mode 100644 index 0000000..4648019 --- /dev/null +++ b/internal/hf/testdata/glm-5.3-flash-config.json @@ -0,0 +1,68 @@ +{ + "architectures": [ + "Glm5NextForConditionalGeneration" + ], + "model_type": "glm5_next", + "text_config": { + "model_type": "glm5_next_text", + "num_hidden_layers": 45, + "num_attention_heads": 64, + "num_key_value_heads": 64, + "head_dim": 0, + "hidden_size": 4096, + "max_position_embeddings": 1048576, + "dtype": "bfloat16", + "kv_lora_rank": 512, + "qk_rope_head_dim": 0, + "index_head_dim": 128, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "deepseek_sparse_attention", + "linear_attention" + ], + "n_routed_experts": 288, + "num_experts_per_tok": 8 + } +}