hf: size GLM-5.3-Flash's KV cache from its latent layers only

CHANGELOG.md

@@ -98,6 +98,12 @@ the fact.
 
 ### Fixed
 
+- `inspect`, `new` and the panel sized the KV cache of GLM-5.3-Flash about
+  50 times too large: its 34 linear-attention layers counted as full
+  attention, and its 11 latent-attention layers as 64 full heads. A
+  `linear_attention` layer now holds no cache, and a latent-attention layer
+  (`kv_lora_rank`, as in DeepSeek V2 and V3 and GLM-5) holds its latent, rotary
+  key and indexer key. DeepSeek V3's estimate falls about 25 times with it.
 - Profiles for Qwen3 hybrid models got `--reasoning-parser deepseek_r1` from
   the card's `vllm serve` line. With thinking turned off
   (`enable_thinking: false`), a non-streaming reply then came back with

docs/providers.md

@@ -151,6 +151,25 @@ llm-helper set deepseek-v4-flash-vision-exp-gguf-ud-q8_k_xl -- \
 - It writes about 33 tokens/sec, reads prompts at about 825 tokens/sec, and uses 161.8 of the 240 GiB of GPU memory.
 - Images, tool calls and thinking work through the chat template in the GGUF.
 
+### Example: GLM-5.3-Flash on a Mac Studio
+
+This ran on the same Mac, with the same llama.cpp 0.8.0.
+llama.cpp runs GLM-5.3-Flash since ggml-org/llama.cpp#27773, and its MTP head since #29928.
+The model is Unsloth's `UD-Q4_K_XL` GGUF (186 GiB in 6 files), plus `mmproj-F16.gguf` for images.
+
+```sh
+llm-helper new unsloth/GLM-5.3-Flash-GGUF --quant UD-Q4_K_XL
+llm-helper set glm-5.3-flash-gguf-ud-q4_k_xl -- \
+  EXTRA_ARGS="-c 1048576 -np 4 -ngl all --fit off -fa on -b 4096 -ub 2048 --spec-type draft-mtp --temp 1.0 --top-p 0.95 --min-p 0.0 --metrics --offline"
+```
+
+- `--spec-type draft-mtp` drafts tokens with the model's own MTP head. No second model is needed. 46 to 68 percent of drafts were accepted.
+- One request writes about 58 tokens/sec. Four at once write about 16 tokens/sec each, 63 in total. These numbers come from Huihui's abliterated copy of the same quant (`huihui-ai/Huihui-GLM-5.3-Flash-abliterated-GGUF`), which has the same shapes.
+- It reads prompts at about 585 tokens/sec, slower than DeepSeek V4 Flash on this Mac.
+- `-c 1048576 -np 4` gives 4 slots of 262,144 tokens. Most of its layers are linear attention, so the KV cache is small (`inspect` gives about 14 KB a token).
+- Images and tool calls work through the chat template in the GGUF.
+- It always thinks, and the default `reasoning_effort` is `max`. A short question can use thousands of tokens. A client can send `"reasoning_effort": "low"` or `"high"`.
+
 A machine can hold profiles for different providers.
 On the same Mac, stopping this profile and starting a vLLM (vllm-metal) Qwen3-Coder profile in its place took about 20 seconds, and the same back.
 

internal/hf/arch.go

@@ -30,6 +30,14 @@ type Arch struct {
 	HiddenSize        int    `json:"hidden_size"`
 	MaxPositionEmbed  int    `json:"max_position_embeddings"`
 	TorchDtype        string `json:"torch_dtype"`
+	// KVLoraRank and QKRopeHeadDim are set on multi-head latent attention
+	// (DeepSeek V2/V3, GLM-5), which caches one compressed latent and the
+	// rotary key per token and layer instead of K and V for every head.
+	KVLoraRank    int `json:"kv_lora_rank"`
+	QKRopeHeadDim int `json:"qk_rope_head_dim"`
+	// IndexHeadDim is the key width of a DeepSeek sparse attention indexer,
+	// which caches one indexer key per token beside the latent.
+	IndexHeadDim int `json:"index_head_dim"`
 	// LayerTypes names each layer for hybrids, where only some attend.
 	LayerTypes []string `json:"layer_types"`
 	// SlidingWindow caps how far a sliding-attention layer looks back, so
@@ -48,12 +56,15 @@ func (a Arch) text() Arch {
 }
 
 // attentionLayers counts the layers holding a KV cache, split by how much each
-// holds. Hybrids interleave attention with Mamba layers that hold none, and
-// sliding layers hold only their window -- gpt-oss is half full, half sliding
-// at a window of 128, so treating them alike doubles the estimate.
+// holds. Hybrids interleave attention with Mamba or linear-attention layers
+// that hold a fixed-size state rather than a cache that grows with the
+// context, and sliding layers hold only their window -- gpt-oss is half full,
+// half sliding at a window of 128, so treating them alike doubles the estimate.
 func (a Arch) attentionLayers() (full, sliding int) {
 	for _, t := range a.LayerTypes {
 		switch t := strings.ToLower(t); {
+		case strings.Contains(t, "linear"):
+			// GLM-5.3 names its Kimi Delta layers linear_attention.
 		case strings.Contains(t, "sliding"):
 			sliding++
 		case strings.Contains(t, "attention"):
@@ -93,10 +104,22 @@ func (a Arch) KVBytesPerToken(maxLen int) int64 {
 		// Last resort only; deriving over a declared value is the OOM above.
 		dim = t.HiddenSize / t.NumAttentionHeads
 	}
+	if t.KVLoraRank > 0 {
+		// Latent attention: vLLM and llama.cpp cache kv_lora_rank +
+		// qk_rope_head_dim per token, whatever the head count, plus the
+		// sparse-attention indexer's key where there is one (stored narrower
+		// than this counts it). Counting heads overstates DeepSeek V3 about
+		// 25 times.
+		heads, dim = 1, t.KVLoraRank+t.QKRopeHeadDim+t.IndexHeadDim
+	}
 	if full+sliding <= 0 || heads <= 0 || dim <= 0 || maxLen <= 0 {
 		return 0
 	}
 	perLayer := 2 * int64(heads) * int64(dim) * int64(kvDtypeBytes(t.TorchDtype))
+	if t.KVLoraRank > 0 {
+		// One latent per token, not a K and a V.
+		perLayer /= 2
+	}
 
 	// A sliding layer never holds more than its window, so past that point
 	// it stops costing anything per extra token of context.

internal/hf/kv_internal_test.go

@@ -1,6 +1,10 @@
 package hf
 
-import "testing"
+import (
+	"encoding/json"
+	"os"
+	"testing"
+)
 
 // Every figure below was read off a live config.json and, where noted,
 // confirmed against what vLLM then allocated on real hardware.
@@ -167,3 +171,54 @@ func TestADenseModelIgnoresSlidingWindow(t *testing.T) {
 		t.Errorf("KVBytesPerToken = %d, want %d", got, want)
 	}
 }
+
+// GLM-5.3-Flash (config.json trimmed to its shape, from zai-org/GLM-5.3-Flash)
+// is 34 Kimi Delta linear-attention layers, which hold a fixed-size state, and
+// 11 latent-attention layers that cache a 512-wide latent and a 128-wide
+// indexer key per token. Counting every layer as 64 full heads put about
+// 380k tokens in 280 GB, so a 256 GB Mac looked unable to hold any context.
+func TestGLM53FlashCountsOnlyItsLatentLayers(t *testing.T) {
+	raw, err := os.ReadFile("testdata/glm-5.3-flash-config.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var a Arch
+	if err := json.Unmarshal(raw, &a); err != nil {
+		t.Fatal(err)
+	}
+	const maxLen = 1048576
+	if got, want := a.KVBytesPerToken(maxLen), int64(11*(512+128)*2); got != want {
+		t.Errorf("KVBytesPerToken() = %d, want %d (11 latent layers of 45)", got, want)
+	}
+	if got := a.MaxContext(); got != maxLen {
+		t.Errorf("MaxContext() = %d, want %d", got, maxLen)
+	}
+}
+
+// Latent attention caches kv_lora_rank + qk_rope_head_dim per token and
+// layer whatever the head count: 576 for DeepSeek V3, as vLLM allocates it.
+func TestLatentAttentionCachesTheLatentNotEveryHead(t *testing.T) {
+	a := Arch{
+		NumHiddenLayers: 61, NumAttentionHeads: 128, NumKeyValueHeads: 128,
+		HiddenSize: 7168, KVLoraRank: 512, QKRopeHeadDim: 64, TorchDtype: "bfloat16",
+	}
+	if got, want := a.KVBytesPerToken(163840), int64(61*(512+64)*2); got != want {
+		t.Errorf("KVBytesPerToken() = %d, want %d", got, want)
+	}
+}
+
+// A hybrid that names its layers linear_attention and full_attention: the
+// linear name holds "attention", which must not make those layers count as full.
+func TestLinearAttentionLayersHoldNoCache(t *testing.T) {
+	types := make([]string, 48)
+	for i := range types {
+		types[i] = "linear_attention"
+		if i%4 == 3 {
+			types[i] = "full_attention"
+		}
+	}
+	a := Arch{NumHiddenLayers: 48, LayerTypes: types, NumKeyValueHeads: 2, HeadDim: 256, TorchDtype: "bfloat16"}
+	if got, want := a.KVBytesPerToken(262144), int64(2*12*2*256*2); got != want {
+		t.Errorf("KVBytesPerToken() = %d, want %d (12 full layers of 48)", got, want)
+	}
+}

internal/hf/testdata/glm-5.3-flash-config.json

@@ -0,0 +1,68 @@
+{
+  "architectures": [
+    "Glm5NextForConditionalGeneration"
+  ],
+  "model_type": "glm5_next",
+  "text_config": {
+    "model_type": "glm5_next_text",
+    "num_hidden_layers": 45,
+    "num_attention_heads": 64,
+    "num_key_value_heads": 64,
+    "head_dim": 0,
+    "hidden_size": 4096,
+    "max_position_embeddings": 1048576,
+    "dtype": "bfloat16",
+    "kv_lora_rank": 512,
+    "qk_rope_head_dim": 0,
+    "index_head_dim": 128,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "deepseek_sparse_attention",
+      "linear_attention"
+    ],
+    "n_routed_experts": 288,
+    "num_experts_per_tok": 8
+  }
+}