Repository navigation
Expand file tree
/
Copy pathserver.py
More file actions
2814 lines (2492 loc) · 126 KB
/
Copy pathserver.py
File metadata and controls
2814 lines (2492 loc) · 126 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""
MLX Native Anthropic Server — Claude Code on Apple Silicon.
Single-file server: MLX inference + Anthropic Messages API + tool use support.
Converts Anthropic tool format <-> the model's native function calling format
(Gemma 4's `<|tool_call>call:Name{...}<tool_call|>`, Llama 3.3's raw-JSON
`{"type":"function",...}`, and the common HuggingFace `<tool_call>` JSON form
used by Qwen and others).
Pick a model from the lineup with the MLX_MODEL env var:
MLX_MODEL=divinetribe/gemma-4-31b-it-abliterated-4bit-mlx (THE QUICK ONE — default, our own MLX upload)
MLX_MODEL=divinetribe/Llama-3.3-70B-Instruct-abliterated-8bit-mlx (THE WISE ONE — our own MLX upload)
MLX_MODEL=mlx-community/Qwen3.5-122B-A10B-4bit (THE BEAST)
NOTE FOR CONTRIBUTORS: this file is the source of truth. `setup.sh` installs it
at `~/.local/mlx-native-server/server.py` via a symlink, so edits here take
effect on the running server after a restart — no re-copying needed.
"""
import json
import os
import re
import subprocess
import sys
import threading
import time
import uuid
from http.server import HTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse
# Release version. Keep in step with the VERSION file at the repo root
# (tests/test_version.py checks they match).
__version__ = "1.0.0"
# MLX is the Mac inference backend. Guard the import so this module can also be
# imported on Linux/Windows (which have no MLX) purely to reuse the pure
# Anthropic<->tool-call translation helpers below — portable/portable_server.py
# does exactly that and talks to llama.cpp instead. On a Mac nothing changes:
# the import succeeds and MLX_AVAILABLE is True.
try:
import mlx.core as mx
import mlx.nn as nn
from mlx_lm.utils import load
from mlx_lm.generate import stream_generate
from mlx_lm.sample_utils import make_sampler
from mlx_lm.models.cache import make_prompt_cache
MLX_AVAILABLE = True
except ImportError:
MLX_AVAILABLE = False
# mlx-vlm is optional. With it installed, a model that ships a vision tower
# (Qwen 3.5, Gemma 4 E2B/E4B, ...) is loaded whole, and images Claude Code
# sends (a Read on a .png, a pasted screenshot) reach the model instead of
# being replaced with a marker. Without it, nothing changes.
try:
from mlx_vlm import load as vlm_load
from mlx_vlm import stream_generate as vlm_stream_generate
VLM_AVAILABLE = True
except Exception:
VLM_AVAILABLE = False
# ─── Configuration ───────────────────────────────────────────────────────────
def env_int(name, default):
"""int() an env var, treating unset *and empty* as "use the default".
os.environ.get(name, default) only falls back when the key is absent, not
when it is present-but-empty. The launchers pass values through
ensure_mlx_server() as MLX_KV_BITS="${MLX_KV_BITS:-}", which exports an
empty string when the caller hasn't set one — so int("") raised ValueError
and the server died before it ever bound the port.
"""
raw = os.environ.get(name, "")
return int(raw) if raw.strip() else default
MODEL_PATH = os.environ.get("MLX_MODEL", "divinetribe/gemma-4-31b-it-abliterated-4bit-mlx")
PORT = env_int("MLX_PORT", 4000)
KV_BITS = env_int("MLX_KV_BITS", 0) # Gemma 4 RotatingKVCache doesn't support quantization
# This used to default to 8192, which quietly TRIPLES peak memory on a long
# prompt and is the reason a big Claude Code request could take the machine into
# swap (or straight into the OOM killer on a 32GB Mac) while the model itself
# fits fine.
#
# mlx_lm's prefill loop runs the WHOLE model on each chunk, lm_head included, so
# a single 8192-token chunk allocates an (8192 x vocab) bf16 logits tensor it
# immediately throws away — 4.3GB at Gemma's 262144 vocab. On top of that the
# full attention layers score the entire chunk against every key so far. None of
# it is reused, all of it is live at once.
#
# Measured on gemma-4-31b-it-abliterated-4bit-mlx (16.7GB of weights resident),
# same prompt each time, via mx.get_peak_memory():
#
# prompt prefill 8192 prefill 1024 prefill 512
# 21.4k 34.2GB / 49.3s 20.9GB / 37.7s - / 39.9s
# 38.5k - / - 23.5GB / 83.1s 21.9GB / 71.0s
#
# 17.5GB of transients on a 21k prompt, against 4.2GB for the same work at 512.
# Small chunks are not slower either — the GPU saturates far below 8192 tokens —
# so there was never anything being traded away for that memory.
PREFILL_SIZE = env_int("MLX_PREFILL_SIZE", 512)
# Ceiling on MLX's buffer recycle pool, which counts toward this process's
# memory. Measured at 0.0GB on the model above either way, so treat it as a
# backstop for larger ones (Llama 70B 8-bit) rather than a fix on its own.
# MLX_CACHE_LIMIT_GB=0 disables the cap.
MEM_CACHE_LIMIT_GB = float(os.environ.get("MLX_CACHE_LIMIT_GB") or 3)
# Pre-fill an empty thinking block to skip Gemma 4 reasoning chains entirely.
# Set MLX_SUPPRESS_THINKING=0 to disable (e.g. when you want reasoning output).
SUPPRESS_THINKING = os.environ.get("MLX_SUPPRESS_THINKING", "1") == "1"
DEFAULT_MAX_TOKENS = env_int("MLX_MAX_TOKENS", 8192)
KV_QUANT_START = env_int("MLX_KV_QUANT_START", 256)
MAX_TOOL_RETRIES = env_int("MLX_TOOL_RETRIES", 2)
# Browser mode: strip Claude Code bloat, keep only MCP tools
BROWSER_MODE = os.environ.get("MLX_BROWSER_MODE", "0") == "1"
# Code mode: auto-detect Claude Code coding sessions and replace the huge harness
# system prompt with a Llama-tuned one. Set MLX_CODE_MODE=0 to disable.
CODE_MODE_ENABLED = os.environ.get("MLX_CODE_MODE", "1") != "0"
# Vision: "auto" (default) turns it on when mlx-vlm is installed AND the model's
# config.json has a vision_config. "1" requires it, "0" keeps the text-only path.
VISION_SETTING = (os.environ.get("MLX_VISION") or "auto").strip().lower()
# Hard ceiling on prompt + answer tokens. A prompt over it (or one that leaves
# no room to answer) gets the same 400 the real API sends for an over-long
# prompt, so Claude Code compacts instead of the server allocating KV cache the
# Mac does not have, and max_tokens is trimmed to the room left. 0 = no
# ceiling. This is the only thing that enforces a small window: Claude Code
# 2.1.306 treats any CLAUDE_CODE_AUTO_COMPACT_WINDOW under 100000 as 100000, so
# scripts/mac-tier.sh no longer sets it for the 16K tiers.
MAX_CONTEXT = env_int("MLX_MAX_CONTEXT", 0)
# Longest side, in pixels, an image is shrunk to before the vision tower sees
# it. A 1568x980 screenshot raised the server's peak by about 0.4 GB on
# Qwen 3.5 2B bf16; fewer pixels is fewer patches to encode. 0 = send as is.
MAX_IMAGE_SIDE = env_int("MLX_MAX_IMAGE_SIDE", 0)
# ─── Globals ─────────────────────────────────────────────────────────────────
model = None
tokenizer = None
processor = None # mlx-vlm processor; set only when VISION is on
VISION = False # True once a vision model is loaded through mlx-vlm
generate_lock = threading.Lock()
# Prompt cache: reuse KV state across requests to avoid re-prefilling system+tools
_prompt_cache = None
_cached_token_prefix = None # token IDs we've already prefilled
# ─── Logging ─────────────────────────────────────────────────────────────────
def log(msg):
print(f"[{time.strftime('%H:%M:%S')}] {msg}", file=sys.stderr, flush=True)
# ─── Model Loading ───────────────────────────────────────────────────────────
GEMMA4_CHAT_TEMPLATE = (
"{{ bos_token }}"
"{% set ns = namespace(system='') %}"
"{% for message in messages %}{% if message['role'] == 'system' %}{% set ns.system = message['content'] %}{% endif %}{% endfor %}"
"{% for message in messages %}"
"{% if message['role'] == 'user' %}"
"<|turn>user\n{% if ns.system and loop.first %}{{ ns.system }}\n\n{% endif %}{{ message['content'] }}<turn|>"
"{% elif message['role'] == 'assistant' %}"
"<|turn>model\n{{ message['content'] }}<turn|>"
"{% elif message['role'] == 'tool' %}"
"<|turn>tool_response\n{{ message['content'] }}<turn|>"
"{% endif %}"
"{% endfor %}"
"{% if add_generation_prompt %}<|turn>model\n{% endif %}"
)
GB = 1024 ** 3
def mem_snapshot():
"""What this process is actually holding, in GB.
Watch mx_peak, not rss. `ps rss` cannot see Metal buffers — it reported a
flat 16.7GB straight through a prefill that MLX itself measured at 34GB, so
a server can be moments from swapping while every RSS reading looks calm.
Activity Monitor's "Memory" column (phys_footprint) does see them.
"""
try:
rss = psutil_rss_gb()
except Exception:
rss = 0.0
return {
"rss_gb": round(rss, 1),
"mx_active_gb": round(mx.get_active_memory() / GB, 1),
"mx_cache_gb": round(mx.get_cache_memory() / GB, 1),
"mx_peak_gb": round(mx.get_peak_memory() / GB, 1),
}
def psutil_rss_gb():
"""RSS without a psutil dependency — the mlx venv doesn't ship one."""
out = subprocess.run(["ps", "-o", "rss=", "-p", str(os.getpid())],
capture_output=True, text=True).stdout.strip()
return int(out) / 1024 / 1024 if out else 0.0
def release_transients(tag=""):
"""Hand MLX's recycle pool back after every generation, and log the peak.
The log line is half the point. When a memory death happens there is no
traceback and usually no crash report, because the process is killed from
outside — so a per-request peak, written down while the server was still
alive, is the only thing that distinguishes "that prompt was too big" from
any other silent exit. On a 31B 4-bit the pool itself measures 0.0GB, so the
clear_cache is insurance for models that do accumulate one.
"""
before = mx.get_cache_memory() / GB
peak = mx.get_peak_memory() / GB
mx.clear_cache()
log(f" Memory{tag}: peak {peak:.1f}GB this request, "
f"released {before:.1f}GB of buffer cache, rss now {psutil_rss_gb():.1f}GB")
mx.reset_peak_memory()
def model_config(path):
"""The model's config.json as a dict, from a local folder or the HF cache.
Returns {} when it cannot be read (no network, private repo, ...)."""
try:
if os.path.isdir(path):
cfg = os.path.join(path, "config.json")
else:
from huggingface_hub import hf_hub_download
cfg = hf_hub_download(path, "config.json")
with open(cfg) as f:
return json.load(f)
except Exception:
return {}
def wants_vision(setting, config, vlm_available):
"""Decide whether to load through mlx-vlm. Pure, so it is unit-tested.
"0"/"off" never; "1"/"on" whenever mlx-vlm is there (load fails loudly if
the model has no vision tower); "auto" only when both mlx-vlm is installed
and config.json declares a vision tower.
"""
if setting in ("0", "off", "false", "no"):
return False
if not vlm_available:
return False
if setting in ("1", "on", "true", "yes"):
return True
return bool(config.get("vision_config"))
def load_model():
global model, tokenizer, processor, VISION, KV_BITS
if MEM_CACHE_LIMIT_GB > 0:
mx.set_cache_limit(int(MEM_CACHE_LIMIT_GB * GB))
log(f"MLX buffer cache capped at {MEM_CACHE_LIMIT_GB:g}GB")
log(f"Loading model: {MODEL_PATH}")
t0 = time.time()
if wants_vision(VISION_SETTING, model_config(MODEL_PATH), VLM_AVAILABLE):
# One copy of the weights serves both text and image turns.
model, processor = vlm_load(MODEL_PATH)
tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor
VISION = True
mx.eval(model.parameters())
log(f"Vision on (mlx-vlm): images in the newest message reach the model")
log(f"Model loaded in {time.time() - t0:.1f}s")
return
if VISION_SETTING in ("1", "on", "true", "yes") and not VLM_AVAILABLE:
log("MLX_VISION=1 but mlx-vlm is not installed (pip install mlx-vlm); "
"loading text-only")
model, tokenizer = load(MODEL_PATH)
mx.eval(model.parameters())
# Fallback chat template if model doesn't provide one (Llama 3.3 has its own)
if not getattr(tokenizer, 'chat_template', None):
tokenizer.chat_template = GEMMA4_CHAT_TEMPLATE
log("Injected Gemma 4 chat template")
# Some quant repos ship a generation_config.json whose eos_token_id
# disagrees with tokenizer_config.json's eos_token. mlx-lm seeds its stop
# set (tokenizer.eos_token_ids) from generation_config, so when the chat
# template actually terminates turns with the *other* token, generation
# never stops: the real stop token is decoded as ordinary text and every
# request runs to max_tokens.
#
# Seen on divinetribe/Hermes-4-14B-abliterated-4bit-mlx, whose
# generation_config says 151643 (<|endoftext|>) while its template emits
# 151645 (<|im_end|>) — 503s and 4096 tokens for what should be a 20-token
# tool call. Unioning the two is safe: a token that is EOS in either config
# is never a legitimate mid-sequence token.
_tok_eos = getattr(tokenizer, "eos_token_id", None)
if _tok_eos is not None:
_stop_set = set(getattr(tokenizer, "eos_token_ids", None) or set())
if _tok_eos not in _stop_set:
tokenizer.eos_token_ids = _stop_set | {_tok_eos}
log(f"Added eos_token_id {_tok_eos} to stop set (was {_stop_set or set()})")
elapsed = time.time() - t0
log(f"Model loaded in {elapsed:.1f}s")
# Safety net: Gemma uses sliding-window attention → RotatingKVCache, which
# mlx-lm can't quantize yet ("RotatingKVCache Quantization NYI"). The
# default for MLX_KV_BITS is already 0, but if a user explicitly sets it to
# 8 and happens to be running Gemma, auto-disable it so inference doesn't
# 500 on the first call. (Credit: asdmoment, PR #7.)
if KV_BITS and "gemma" in MODEL_PATH.lower():
log("Gemma detected: disabling KV cache quantization (RotatingKVCache NYI)")
KV_BITS = 0
log(f"KV cache quantization: {KV_BITS}-bit" if KV_BITS else "KV cache: full precision")
# ─── Think Tag Stripping ────────────────────────────────────────────────────
def strip_think_tags(text):
"""Remove thinking blocks from model reasoning output."""
# Standard <think>...</think>
cleaned = re.sub(r'<think>.*?</think>', '', text, flags=re.DOTALL).strip()
# A lone </think>: the chat template opened the block in the prompt
# (Qwen 3.5 ends its generation prompt with "<think>\n"), so the output
# starts mid-thought. Everything before the close is reasoning; it used to
# go out as the answer, followed by the real answer a second time.
if '</think>' in cleaned:
cleaned = cleaned.rsplit('</think>', 1)[1].strip()
# Gemma 4 thinking: <|channel>thought\n...<channel|>
cleaned = re.sub(r'<\|channel>thought\n.*?<channel\|>', '', cleaned, flags=re.DOTALL).strip()
# A lone <channel|>, the Gemma 4 version of the lone </think> above: the
# thought channel was opened (and closed empty) in the prompt to suppress
# thinking, and Gemma 4 E4B reasons anyway, then closes the channel again.
# Everything before the close is reasoning, not the answer.
if '<channel|>' in cleaned:
cleaned = cleaned.rsplit('<channel|>', 1)[1].strip()
# Empty tool_call blocks
cleaned = re.sub(r'<tool_call>\s*</tool_call>', '', cleaned).strip()
return cleaned if cleaned else text
class ThinkingFilter:
"""Real-time filter that removes Gemma 4 thinking blocks from the mlx_lm token stream.
Applied token-by-token inside the stream_generate loop so thinking content is
discarded as it arrives rather than accumulated and regex-stripped after the fact.
Works independently of HTTP SSE — the SSE layer (send_anthropic_stream) operates
on the already-cleaned result returned by generate_response.
"""
THINK_START = "<|channel>thought\n"
THINK_END = "<channel|>"
def __init__(self):
self.in_thinking = False
self.buf = ""
def feed(self, chunk):
self.buf += chunk
output = ""
while True:
if self.in_thinking:
idx = self.buf.find(self.THINK_END)
if idx == -1:
safe = max(0, len(self.buf) - len(self.THINK_END) + 1)
self.buf = self.buf[safe:]
break
self.in_thinking = False
self.buf = self.buf[idx + len(self.THINK_END):]
else:
idx = self.buf.find(self.THINK_START)
if idx == -1:
safe = max(0, len(self.buf) - len(self.THINK_START) + 1)
output += self.buf[:safe]
self.buf = self.buf[safe:]
break
output += self.buf[:idx]
self.in_thinking = True
self.buf = self.buf[idx + len(self.THINK_START):]
return output
def flush(self):
return "" if self.in_thinking else self.buf
def clean_response(text):
"""Strip think tags, stop tokens, and reasoning artifacts (but preserve tool_call tags)."""
text = strip_think_tags(text)
# Llama 3.x: strip function-call prefix token
text = text.replace('<|python_tag|>', '').strip()
# Gemma 4 + Qwen + ChatML: truncate at end-of-turn or start of a new turn,
# whichever marker comes first (the live stream cuts the same way).
cuts = [text.index(m) for m in ['<turn|>', '<|turn>', '<|im_end|>', '<|endoftext|>',
'<|im_start|>', '<|end_of_text|>', '<|eot_id|>'] if m in text]
if cuts:
text = text[:min(cuts)].strip()
# Remove reasoning preamble if present
if text.lstrip().startswith("Thinking"):
lines = text.split('\n')
for i, line in enumerate(lines):
s = line.strip()
if any(s.startswith(p) for p in ['```', 'def ', 'class ', 'function ', 'import ', '#', '//', '<tool_call>']):
return '\n'.join(lines[i:])
return text
# ─── Tool Conversion ────────────────────────────────────────────────────────
def convert_tools_for_llm(anthropic_tools):
"""Convert Anthropic tool definitions to HuggingFace/Llama format."""
if not anthropic_tools:
return None
llm_tools = []
for tool in anthropic_tools:
llm_tools.append({
"type": "function",
"function": {
"name": tool.get("name", ""),
"description": tool.get("description", ""),
"parameters": tool.get("input_schema", {"type": "object", "properties": {}}),
}
})
return llm_tools
def format_tools_as_text(tools):
"""Format tools as text for system prompt (fallback if chat template doesn't support tools param)."""
lines = ["# Available Tools\n"]
lines.append("CRITICAL: You MUST call tools using EXACTLY this JSON format inside <tool_call> tags:")
lines.append("")
lines.append('<tool_call>')
lines.append('{"name": "Bash", "arguments": {"command": "ls -la"}}')
lines.append('</tool_call>')
lines.append("")
lines.append("RULES:")
lines.append('- The content inside <tool_call> MUST be valid JSON with "name" and "arguments" keys')
lines.append('- Do NOT use <parameter=...> tags inside <tool_call> — use the "arguments" JSON object')
lines.append('- Do NOT mix XML and JSON — use ONLY pure JSON inside the tags')
lines.append("- You may call multiple tools by using multiple <tool_call> blocks")
lines.append("- Output any reasoning text BEFORE the tool calls, not inside them")
lines.append("")
for tool in tools:
func = tool.get("function", tool)
name = func.get("name", "")
desc = func.get("description", "")
params = func.get("parameters", {})
lines.append(f"## {name}")
if desc:
lines.append(f"{desc}")
props = params.get("properties", {})
required = params.get("required", [])
if props:
for pname, pdef in props.items():
req = " (required)" if pname in required else ""
ptype = pdef.get("type", "any")
pdesc = pdef.get("description", "")
lines.append(f" - {pname}: {ptype}{req} — {pdesc}")
lines.append("")
return "\n".join(lines)
def recover_garbled_tool_json(content, original_text=""):
"""Attempt to recover tool name and arguments from garbled JSON inside <tool_call> tags.
Models sometimes produce hybrid XML/JSON like:
{"name": "Bash", "parameter=command>cd ~/Desktop && rm -rf ...
{"name": "Bash", "<parameter_commands>["rm -rf ...
{"name": "Edit", "parameter=file_path>/some/path</parameter...
"""
# Extract tool name
name_match = re.search(r'"name"\s*:\s*"([^"]+)"', content)
if not name_match:
return None
tool_name = name_match.group(1)
arguments = {}
# Pattern A: "parameter=key>value" (most common garble)
# Matches: "parameter=command>cd ~/Desktop..." or parameter=command>value</parameter>
param_a = re.finditer(r'["\s,]?parameter=(\w+)>\s*(.*?)(?:</parameter>|$)', content, re.DOTALL)
for m in param_a:
key = m.group(1)
val = m.group(2).strip().rstrip('"}\n')
if val:
arguments[key] = val
# Pattern B: "<parameter_key>value" or "<parameter_key>["value"]"
if not arguments:
param_b = re.finditer(r'<parameter[_=](\w+)>\s*(.*?)(?:</parameter|<|$)', content, re.DOTALL)
for m in param_b:
key = m.group(1)
val = m.group(2).strip().strip('[]"')
if val:
arguments[key] = val
# Pattern C: "arguments" key exists but is malformed — try to extract the value after it
if not arguments:
args_match = re.search(r'"arguments"\s*:\s*\{(.*)', content, re.DOTALL)
if args_match:
raw = args_match.group(1)
# Try to find key-value pairs
kv_matches = re.finditer(r'"(\w+)"\s*:\s*"((?:[^"\\]|\\.)*)"', raw)
for m in kv_matches:
arguments[m.group(1)] = m.group(2)
# Pattern D: single-argument tools — if we have a tool name and leftover text, use it
# Common for Bash (command), Read (file_path), etc.
if not arguments:
single_arg_tools = {
"Bash": "command", "Read": "file_path", "Write": "file_path",
"Glob": "pattern", "Grep": "pattern",
}
if tool_name in single_arg_tools:
# Everything after the tool name declaration is likely the argument value
after_name = content[name_match.end():]
# Strip JSON noise
val = re.sub(r'^[\s,":{}]+', '', after_name)
val = re.sub(r'[\s"}]+$', '', val)
# Remove parameter= prefix if present
val = re.sub(r'^parameter=\w+>\s*', '', val)
val = re.sub(r'^<parameter[_=]\w+>\s*', '', val)
if val and len(val) > 2:
arguments[single_arg_tools[tool_name]] = val
if arguments:
log(f" Recovered garbled tool call: {tool_name} with {list(arguments.keys())}")
return {"name": tool_name, "arguments": arguments}
return None
def _decode_string_args(args):
"""Some models send arguments as a JSON-encoded string, OpenAI style:
"arguments": "{\"command\": \"ls\"}". Decode it so the call doesn't go
out with empty input."""
if isinstance(args, str):
try:
decoded = json.loads(args)
except ValueError:
return args
if isinstance(decoded, dict):
return decoded
return args
def _gemma4_args(args_str):
"""Parse Gemma 4's argument list (the part inside call:Name{...}) as JSON.
Gemma 4 writes strings as <|"|>text<|"|>, everything else as bare JSON
(true, 50, [...], {...}), and object keys unquoted, sometimes with a dash
(Grep's -i). Swap the strings out, quote the keys, swap the strings back as
JSON strings, and parse. Returns None when that is not valid JSON, so the
caller can fall back to the older regex matching.
"""
strings = []
def stash(m):
strings.append(m.group(1))
return f"\x00{len(strings) - 1}\x00"
s = re.sub(r'<\|"\|>(.*?)<\|"\|>', stash, args_str, flags=re.DOTALL)
s = re.sub(r'(^|[{,])\s*([\w-]+)\s*:', r'\1"\2":', s)
s = re.sub(r'\x00(\d+)\x00', lambda m: json.dumps(strings[int(m.group(1))]), s)
try:
parsed = json.loads("{" + s + "}")
except ValueError:
return None
return parsed if isinstance(parsed, dict) else None
def _tool_schemas(tools):
"""{name: JSON schema} from Anthropic tools ({name, input_schema}) or the
HuggingFace/OpenAI shape convert_tools_for_llm produces."""
schemas = {}
for t in tools or []:
if not isinstance(t, dict):
continue
fn = t.get("function") if isinstance(t.get("function"), dict) else t
name = fn.get("name")
if name:
schema = fn.get("input_schema") or fn.get("parameters") or {}
schemas[name] = schema if isinstance(schema, dict) else {}
return schemas
def _fits_schema(args, schema):
"""True when args is a dict that has every required key and no key the
schema does not declare. Used to accept a recovered tool call only when it
is unambiguous."""
if not isinstance(args, dict) or not args:
return False
props = schema.get("properties") or {}
required = schema.get("required") or []
if any(k not in args for k in required):
return False
return not props or all(k in props for k in args)
def _recover_gemma4_missing_opener(text, tools):
"""Gemma 4 tool calls that lost their opening marker.
On long calls (a Write whose content is a whole HTML page) Gemma 4 31B
8-bit sometimes drops the <|tool_call>call:Write{content:<|"|> head and
starts straight into the value, so the output ends like
...</html>\\n<|"|>,file_path:<|"|>budget.html<|"|>}<tool_call|>
Format 0 never sees it (no opener), the file is never written, and the
raw page is shown to the user as text. 4 of 4 tries on 10/29.
The closer <tool_call|> and the <|"|> delimiters are Gemma special tokens
that never appear in prose, so they anchor the recovery. A call is only
accepted when it names (or schema-matches exactly one) declared tool and
its arguments fit that tool's schema. Three shapes, most explicit first:
a) the name survived: call:Write{...}<tool_call|> / Write{...}
b) the name is gone too: {file_path:<|"|>..<|"|>,content:..}<tool_call|>
c) the first key is gone too: <page>..<|"|>,file_path:<|"|>..<|"|>}<tool_call|>
Returns (tool_calls, remaining_text) or ([], text).
"""
schemas = _tool_schemas(tools)
if not schemas or "<tool_call|>" not in text:
return [], text
Q = '<|"|>'
calls, kept, start = [], [], 0
for closer in re.finditer(r'<tool_call\|>', text):
segment = text[start:closer.start()]
start = closer.end()
body_end = segment.rstrip().rstrip(">").rstrip()
if not body_end.endswith("}"):
kept.append(segment)
continue
body_end = body_end[:-1]
found = None
# a) a declared tool name followed by its argument list.
names = "|".join(re.escape(n) for n in sorted(schemas, key=len, reverse=True))
for m in re.finditer(rf'(?:<\|tool_call>|<tool_call:|call:)?(?<![\w.-])({names})\s*\{{', body_end):
args = _gemma4_args(body_end[m.end():])
if _fits_schema(args, schemas[m.group(1)]):
found = (m.start(), m.group(1), args)
break
# b) a bare argument list; the schema has to pick exactly one tool.
if not found:
brace = body_end.find("{")
# A name right before the brace is a tool the client never
# declared (else (a) had it); don't re-route it to another tool.
named = re.search(r'[\w.-]\s*$', body_end[:brace]) if brace >= 0 else None
if brace >= 0 and not named and Q in body_end[brace:]:
args = _gemma4_args(body_end[brace + 1:])
fits = [n for n, s in schemas.items() if _fits_schema(args, s)]
if len(fits) == 1:
found = (brace, fits[0], args)
# c) the value of the first argument starts the segment; the one
# required string key the rest of the call lacks is its name.
if not found and Q in body_end:
first_q = body_end.find(Q)
tail = body_end[first_q:]
if re.match(r'<\|"\|>\s*,\s*[\w-]+\s*:', tail):
rest = _gemma4_args(tail[len(Q):].lstrip().lstrip(","))
if isinstance(rest, dict):
options = []
for n, s in schemas.items():
props = s.get("properties") or {}
missing = [k for k in s.get("required") or [] if k not in rest]
if (len(missing) == 1
and (props.get(missing[0]) or {}).get("type", "string") == "string"):
args = dict(rest, **{missing[0]: body_end[:first_q].strip("\n")})
if _fits_schema(args, s):
options.append((n, args))
if len(options) == 1:
found = (0, options[0][0], options[0][1])
if found:
pos, name, args = found
kept.append(segment[:pos])
calls.append({"name": name, "arguments": args})
log(f" Gemma4 tool call with no opening marker recovered: {name}({list(args.keys())})")
else:
kept.append(segment)
kept.append(closer.group(0))
if not calls:
return [], text
kept.append(text[start:])
return calls, "".join(kept).strip()
def _element_params(fname, params_text, tools):
"""<function=Write> followed by <file_path>..</file_path><content>..</content>.
Qwen 3.5 0.8B (llama.cpp, phone test 10/30) writes each argument as an
element named after the parameter instead of <parameter=file_path>, so the
call went out with no arguments and Claude Code stopped after one turn.
Only element names that are parameters of the declared tool count, and the
result has to fit that tool's schema, so HTML or XML inside a value (a
<div> in content) is never read as an argument. Returns {} when it is not
that shape.
"""
schema = _tool_schemas(tools).get(fname)
if not schema:
return {}
props = schema.get("properties") or {}
arguments = {}
pos = 0
while True:
m = re.compile(r'<(' + "|".join(map(re.escape, props)) + r')>').search(params_text, pos) if props else None
if not m:
break
close = params_text.find(f"</{m.group(1)}>", m.end())
if close < 0:
break
arguments.setdefault(m.group(1), params_text[m.end():close].strip("\n"))
pos = close + len(m.group(1)) + 3
return arguments if _fits_schema(arguments, schema) else {}
def parse_tool_calls(text, tools=None):
"""Parse tool calls from generated text. Handles multiple formats including
Gemma 4 native format. Returns (list of tool calls, remaining text).
tools: the tools the request declared (Anthropic or HF shape). Only used to
recover calls whose opening marker the model dropped; without it that
recovery is off, so a guess can never name a tool the client didn't offer.
"""
tool_calls = []
# Format 0: Gemma 4 native — <|tool_call>call:Name{key:<|"|>val<|"|>}<tool_call|>
# Parse BEFORE replacing escape tokens — use <|"|> as reliable value delimiters
#
# Three spellings of the opener turn up in practice. The documented one is
# <|tool_call>call:Name{...}. Some models drop the call: prefix. And
# abliterated Gemma 4 31B collapses the whole marker to <tool_call:Name{...},
# which is the SAME grammar — same <|"|> value delimiters, same <tool_call|>
# closer, only the opening marker differs — so it is one alternation here
# rather than a new format further down.
#
# That third spelling cost 6 of 14 on scripts/test_mlx_server.py with
# divinetribe/gemma-4-31b-it-abliterated-4bit-mlx. Every call was well
# formed, every call was invisible, and the retry path can't rescue it
# because it asks for <tool_call> tags the model has already decided against.
# Some also emit a stray > after the closing brace; the body regex tolerates
# it because re.match doesn't have to consume the whole string.
gemma4_pattern = r'(?:<\|tool_call>|<tool_call:)(.*?)<tool_call\|>'
gemma4_matches = list(re.finditer(gemma4_pattern, text, re.DOTALL))
if gemma4_matches:
remaining = text
for match in gemma4_matches:
remaining = remaining.replace(match.group(0), "", 1)
content = match.group(1).strip()
# [\w.-]: MCP tool names carry the server name, which often has a
# dash (mcp__chrome-devtools__click), same as Formats 1 and 2 allow.
name_m = re.match(r'(?:call:)?([\w.-]+)\s*\{(.*)\}', content, re.DOTALL)
if not name_m:
log(f" Gemma4 no name match: {content[:80]}")
continue
name = name_m.group(1)
args_str = name_m.group(2)
# Primary: read the whole argument list as Gemma's JSON dialect, which
# covers nested arrays/objects and keys like -i that the regexes miss.
arguments = _gemma4_args(args_str)
# call:list_pages{} parses to {}: a real call with no arguments,
# not a failed parse, so it has to go out as one.
no_args_call = arguments == {}
if arguments is None:
arguments = {}
# Fallback: extract key:<|"|>value<|"|> pairs (handles embedded quotes)
for km in re.finditer(r'(\w+):<\|"\|>(.*?)<\|"\|>', args_str, re.DOTALL):
arguments[km.group(1)] = km.group(2)
# Gemma 4 writes non-string values bare (replace_all:true, limit:50)
# in the same call as <|"|> strings, and they were dropped. Pick them
# up from what is left once the strings are removed, so a colon
# inside a string value can't be read as a key.
if arguments:
bare = re.sub(r'\w+:<\|"\|>.*?<\|"\|>', '', args_str, flags=re.DOTALL)
for km in re.finditer(r'(\w+):([^,}]+)', bare):
try:
arguments.setdefault(km.group(1), json.loads(km.group(2).strip()))
except ValueError:
pass
# Fallback: unquoted values (numbers, simple strings)
if not arguments:
for km in re.finditer(r'(\w+):([^,}]+)', args_str):
val = km.group(2).strip().strip('"\'')
arguments[km.group(1)] = val
if arguments or no_args_call:
tool_calls.append({"name": name, "arguments": arguments})
log(f" Gemma4 tool call: {name}({list(arguments.keys())})")
else:
log(f" Gemma4 no args parsed: {content[:80]}")
if tool_calls:
return tool_calls, remaining.strip()
# Format 0b: Gemma 4 with the opening marker missing (long Write calls).
# Needs the declared tools; see _recover_gemma4_missing_opener.
if tools:
recovered, rest = _recover_gemma4_missing_opener(text, tools)
if recovered:
return recovered, rest
remaining = text
# Format 0.5: Llama 3.3 raw JSON — {"type":"function","name":"...","parameters":{...}}
# Use json.JSONDecoder.raw_decode for robust nested JSON parsing
_decoder = json.JSONDecoder()
_pos = 0
while _pos < len(text):
idx = text.find('{"type"', _pos)
if idx == -1:
break
try:
obj, end_pos = _decoder.raw_decode(text, idx)
if obj.get("type") == "function" and "name" in obj:
name = obj["name"]
arguments = _decode_string_args(obj.get("parameters", {}))
tool_calls.append({"name": name, "arguments": arguments})
remaining = remaining.replace(text[idx:end_pos], "", 1)
log(f" Llama tool call: {name}({list(arguments.keys())})")
_pos = end_pos
except json.JSONDecodeError:
_pos = idx + 1
if tool_calls:
return tool_calls, remaining.strip()
# Format 1: <tool_call>{"name": "x", "arguments": {...}}</tool_call>
pattern1 = r'<tool_call>\s*(.*?)\s*</tool_call>'
for match in re.finditer(pattern1, text, re.DOTALL):
content = match.group(1).strip()
remaining = remaining.replace(match.group(0), "", 1)
if not content:
continue
try:
call_data = json.loads(content)
# Some models wrap several calls in one tag as a JSON array; a
# list (or any non-object) here used to raise AttributeError.
for cd in call_data if isinstance(call_data, list) else [call_data]:
if isinstance(cd, dict):
tool_calls.append({
"name": cd.get("name", ""),
"arguments": cd.get("arguments", {}),
})
except json.JSONDecodeError:
# The model often puts Format 2 (<function=X><parameter=Y>...</parameter></function>)
# inside <tool_call> tags. Handle that first.
func_in_tag = re.search(r'<function=([\w.-]+)>(.*)', content, re.DOTALL)
if func_in_tag:
fname = func_in_tag.group(1)
params_text = func_in_tag.group(2)
arguments = {}
for pmatch in re.finditer(r'<parameter=(\w+)>\s*(.*?)\s*(?:</parameter>|$)', params_text, re.DOTALL):
arguments[pmatch.group(1)] = pmatch.group(2).strip()
if not arguments:
arguments = _element_params(fname, params_text, tools)
if arguments:
tool_calls.append({"name": fname, "arguments": arguments})
log(f" Recovered function-in-tag: {fname}")
else:
log(f" Warning: function-in-tag but no params: {content[:100]}")
else:
# Try general garbled recovery
recovered = recover_garbled_tool_json(content, text)
if recovered:
tool_calls.append(recovered)
else:
log(f" Warning: unrecoverable tool_call JSON: {content[:100]}")
# Format 2: <function=name><parameter=key>value</parameter>...</function>
if not tool_calls:
pattern2 = r'<function=([\w.-]+)>(.*?)</function>'
for match in re.finditer(pattern2, text, re.DOTALL):
func_name = match.group(1)
params_text = match.group(2)
arguments = {}
for pmatch in re.finditer(r'<parameter=(\w+)>\s*(.*?)\s*</parameter>', params_text, re.DOTALL):
arguments[pmatch.group(1)] = pmatch.group(2)
if not arguments:
arguments = _element_params(func_name, params_text, tools)
tool_calls.append({"name": func_name, "arguments": arguments})
remaining = remaining.replace(match.group(0), "", 1)
# Format 3: <|tool_call|>...<|/tool_call|> (some model versions)
if not tool_calls:
pattern3 = r'<\|tool_call\|>\s*(.*?)\s*<\|/tool_call\|>'
for match in re.finditer(pattern3, text, re.DOTALL):
remaining = remaining.replace(match.group(0), "", 1)
try:
call_data = json.loads(match.group(1))
for cd in call_data if isinstance(call_data, list) else [call_data]:
if isinstance(cd, dict):
tool_calls.append({
"name": cd.get("name", ""),
"arguments": cd.get("arguments", {}),
})
except json.JSONDecodeError:
recovered = recover_garbled_tool_json(match.group(1))
if recovered:
tool_calls.append(recovered)
# Format 3.5: Qwen 2.5 — <tools>{"name": "x", "arguments": {...}}</tools>
# Some Qwen 2.5 fine-tunes emit a <tools> wrapper instead of <tool_call>.
if not tool_calls:
pattern_qwen25 = r'<tools>\s*(.*?)\s*</tools>'
for match in re.finditer(pattern_qwen25, text, re.DOTALL):
content = match.group(1).strip()
remaining = remaining.replace(match.group(0), "", 1)
if not content:
continue
try:
call_data = json.loads(content)
if isinstance(call_data, list):
for cd in call_data:
if isinstance(cd, dict) and "name" in cd:
tool_calls.append({
"name": cd.get("name", ""),
"arguments": cd.get("arguments", cd.get("parameters", {})),
})
elif isinstance(call_data, dict) and "name" in call_data:
tool_calls.append({
"name": call_data.get("name", ""),
"arguments": call_data.get("arguments", call_data.get("parameters", {})),
})
except json.JSONDecodeError:
recovered = recover_garbled_tool_json(content, text)
if recovered:
tool_calls.append(recovered)
else:
log(f" Warning: unrecoverable <tools> JSON: {content[:100]}")
# Format 3.6: JSON inside markdown code block — ```json\n{"name":..,"arguments":..}\n```
# Some Qwen 2.5 fine-tunes emit tool calls as a fenced code block when the
# system prompt / chat template doesn't explicitly tell them to use the
# native <tool_call> wrapper. Three sub-cases supported:
# (a) single JSON object inside the fence
# (b) multiple JSON objects back-to-back inside the same fence (no
# array, no commas) — the model expressing several sequential
# tool calls in one go. Use raw_decode in a loop to extract them.
# (c) a single JSON array of call objects — Hermes 4 14B emits this
# shape instead of (b); without this branch the whole fence was
# skipped (array doesn't start with "{") and fell through to the
# garbled-tool-call retry path even though the JSON was valid.
def _as_call(obj):
if not isinstance(obj, dict):
return None
name = obj.get("name") or obj.get("tool")
args = (
obj.get("arguments")
or obj.get("parameters")
or obj.get("args")
or obj.get("input")
)
return {"name": name, "arguments": args} if name and isinstance(args, dict) else None
if not tool_calls:
pattern_md = r'```(?:json|tool[_ ]?call)?\s*\n?(.*?)\s*\n?```'
decoder_md = json.JSONDecoder()
for match in re.finditer(pattern_md, text, re.DOTALL):
content = match.group(1).strip()
stripped = content.lstrip()
if not stripped.startswith("{") and not stripped.startswith("["):
continue
extracted_in_block = []
if stripped.startswith("["):
try:
items = json.loads(content)
except json.JSONDecodeError:
items = []
for obj in items if isinstance(items, list) else []:
call = _as_call(obj)
if call:
extracted_in_block.append(call)
else:
pos = 0
while pos < len(content):
# Skip whitespace and stray commas between objects
while pos < len(content) and content[pos] in " \t\n\r,":
pos += 1
if pos >= len(content):
break
if content[pos] != "{":
break
try:
obj, end_pos = decoder_md.raw_decode(content, pos)
except json.JSONDecodeError:
break
pos = end_pos
call = _as_call(obj)
if call:
extracted_in_block.append(call)
if extracted_in_block:
tool_calls.extend(extracted_in_block)
remaining = remaining.replace(match.group(0), "", 1)
log(
f" Markdown-fenced tool calls: "
f"{', '.join(t['name'] for t in extracted_in_block)}"
)
# Format 3.7: bare top-level JSON with no wrapper at all —
# {"name": "Bash", "arguments": {"command": "ls /tmp"}}
# Qwen 2.5 Coder emits exactly this: no <tool_call> tags, no <tools>, no
# markdown fence, just the object. Formats 3.5/3.6 already cover the
# wrapped and fenced variants of the same shape, so this is the last one.
# Without it Qwen 2.5 Coder scores 0/12 on scripts/test_mlx_server.py —
# every call is well-formed and every call is invisible to the parser.
# Re-prompting doesn't rescue it either: the retry explicitly asks for
# <tool_call> tags and the model declines, and recover_garbled_tool_json
# never fires because the JSON was never garbled.
#
# Same back-to-back handling as 3.6 (raw_decode in a loop) for models that