diff --git a/convert/ollaya_convert/catalog.py b/convert/ollaya_convert/catalog.py index 6b3b77ef..61381a54 100644 --- a/convert/ollaya_convert/catalog.py +++ b/convert/ollaya_convert/catalog.py @@ -53,9 +53,10 @@ def _laya(sub, description, params, ctx, languages, mlx=False, repo=LAYA_REPO, c } -def _gguf(export, repo, commit, gguf, description, params, languages, notice=None): +def _gguf(export, repo, commit, gguf, description, params, languages, notice=None, tokenizer=None): """A GGUF model run by llama.cpp: the author's GGUF, unmodified, plus the decision.json and - calibration.json that `llm_common/export_llama.py` derived from it (`out/`).""" + calibration.json that `llm_common/export_llama.py` derived from it (`out/`). `tokenizer`: the + author's tokenizer.json in `repo`, for a layout that renders with it instead of the GGUF's vocabulary.""" return { "kind": "gguf", "repo": repo, @@ -66,6 +67,7 @@ def _gguf(export, repo, commit, gguf, description, params, languages, notice=Non "languages": languages, "description": description, "notice": notice, + "tokenizer": tokenizer, } @@ -1069,6 +1071,44 @@ def _kev_license(repo, base): "CUDA (RTX 4090): 593 questions, every decision the same, option logits within 7.7e-6 and " "probabilities within 1.6e-6. The prompts are byte-identical to the author's jevk5.prompt.", }, + "jev-style": { + "namespace": "library", + "model": "jev-style", + "family": "jev-style", + "author": "chaoliangUNSW", + "license": "Apache-2.0", + "license_text": "Jev-Style by chaoliangUNSW (https://huggingface.co/chaoliangUNSW), a fine-tune of " + "Qwen3.5-2B by the Qwen team (Apache-2.0). Its render v2 input and verdict readout " + "are the author's.\n" + "Licensed under the Apache License, Version 2.0.\n\n" + LICENSE_APACHE, + "tags": { + # Jev-Style-2B-Decision-v3. The GGUF holds the weights and the readout tokens (' yes' 9542, + # ' no' 874, ' ->' 1411); the render uses the author's tokenizer.json, which the runtime the + # author measured uses (llama.cpp's own pre-tokenizer splits some scripts, e.g. Devanagari and + # Arabic, differently). Q8_0 is the default: the card puts Q4_K_M at 95.7 % top-1 agreement with + # fp32 and calls it noisier on long inputs. + "2b": _gguf("jev-style-2b-q8_0", "chaoliangUNSW/Jev-Style-2B-Decision-v3-GGUF", + "283b9eb7903aeb184b16d49ab8f38b256f4afcb3", + "Jev-Style-2B-Decision-v3-Q8_0.gguf", + "Jev-Style-2B-Decision-v3 (Qwen3.5-2B fine-tune), Q8_0 GGUF on llama.cpp: a " + "non-causal block-attention decision model, the verdict readout logit(' yes') - " + "logit(' no') at each option's ' ->' slot, with the author's global temperature 0.828. " + "No option cap; up to 25,600 input tokens.", + "2B", ["en"], tokenizer="tokenizer/tokenizer.json"), + "2b-q4_k_m": _gguf("jev-style-2b-q4_k_m", "chaoliangUNSW/Jev-Style-2B-Decision-v3-GGUF", + "283b9eb7903aeb184b16d49ab8f38b256f4afcb3", + "Jev-Style-2B-Decision-v3-Q4_K_M.gguf", + "Jev-Style-2B-Decision-v3, Q4_K_M GGUF (1.3 GB): the smaller download, 95.7 % " + "top-1 agreement with fp32 by the author's measurement and noisier on long inputs.", + "2B", ["en"], tokenizer="tokenizer/tokenizer.json"), + }, + "aliases": {"latest": "2b"}, + "parity": "Ollaya's runner matches the author's own jev-score-v2 (unchanged, on the pinned llama.cpp build " + "b11146) on the same GGUF and device: 582 questions per file, identical token ids, verdict slots " + "and attention blocks, every decision the same and verdict scores within 4.4e-16 (Q8_0 and " + "Q4_K_M; x86-64 CPU on a community machine, and CUDA on an RTX 5090). The render is identical " + "to the author's jev_style_decision_gguf.py on the author's tokenizer.", + }, "decima": { "namespace": "library", "model": "decima", diff --git a/convert/ollaya_convert/families/jev_style/__init__.py b/convert/ollaya_convert/families/jev_style/__init__.py new file mode 100644 index 00000000..a3af0348 --- /dev/null +++ b/convert/ollaya_convert/families/jev_style/__init__.py @@ -0,0 +1 @@ +"""Jev-Style (chaoliangUNSW/Jev-Style-2B-Decision-v3, GGUF) on Ollaya's llama.cpp runner.""" diff --git a/convert/ollaya_convert/families/jev_style/goldens.py b/convert/ollaya_convert/families/jev_style/goldens.py new file mode 100644 index 00000000..e4ec0225 --- /dev/null +++ b/convert/ollaya_convert/families/jev_style/goldens.py @@ -0,0 +1,243 @@ +"""Goldens for `jev-style-v1` from the author's own runtime: the render of `jev_style_decision_gguf.py` (its +`Renderer` on the pinned `tokenizer/tokenizer.json`) and the scores of `jev-score-v2` (`jev_score_v2.cpp`, +unchanged, built against the llama.cpp build Ollaya ships, b11146). ADR 0003: a `seq_cp` split needs its own +reference, and this is it. + + python -m ollaya_convert.families.jev_style.goldens out/jev-style-2b-q8_0 \ + --upstream --scorer /jev-score-v2 \ + --gguf --requests ollaya-shared-requests.jsonl [--device cpu|CUDA0] + +Build the scorer with the author's `build_jev_score.sh` against a b11146 checkout, or compile it against the +b11146 headers and link the stock release libraries (`scripts/llama-cpp.sh linux-amd64`): the same libllama +the runner loads. Needs the author's pinned `tokenizers` (requirements.txt) and numpy. + +The requests are Ollaya's shared set (the wire form, as an HTTP client sends it) plus this layout's own cases: +a state of several attention blocks, a catalogue overflow of 150 options (several catalogue blocks), an input of +exactly 25,600 tokens and one of 25,601 (rejected). + +A wire question becomes the author's `{"t", "ins", "crit"}` with Ollaya's request semantics (laya's), which the +author's `make_question` shares: a choice given as a list of labels is `{label: None}` (duplicates collapse onto +their first position), and noul keys are lower-cased (the last spelling wins). Everything else, including the +author's limits (2..10 score levels, a non-empty string instruction), is the author's code. + +Writes /goldens-.jsonl, one line per request: + {"id", "state", "questions", "error": "invalid", "message"} the author's runtime rejects it + {"id", "state", "questions", "state_tokens", + "rows": [{"qid", "names", "ids", "slots", "blocks", "catalogue_overflow", + "logits": [[yes, no], ...], "scores": [...], "probabilities": [...]}]} +A rejected request is followed by "#valid" with the questions the author's runtime accepts on their own. +Every accepted render is also checked against this family's `ref.py` before it is written. +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import subprocess +import sys +import time + +import numpy as np + +from . import ref + +OUT = os.path.join(os.path.dirname(__file__), "..", "..", "..", "out") + + +def load_upstream(path): + spec = importlib.util.spec_from_file_location("jev_style_decision_gguf", + os.path.join(path, "jev_style_decision_gguf.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def author_question(q): + """A wire question as the author's {"t", "ins", "crit"}, with Ollaya's (laya's) request semantics.""" + if not isinstance(q, dict): + return q + t, crit = q.get("type"), q.get("criteria") + if t == "choice" and isinstance(crit, list) and crit and all(isinstance(c, str) for c in crit): + crit = dict.fromkeys(crit) + elif t == "noul" and isinstance(crit, dict): + crit = {str(k).lower(): v for k, v in crit.items()} + return {"t": t, "ins": q.get("instructions"), "crit": crit} + + +def device_class(device): + return "cpu" if device == "cpu" else "cuda" if device.lower().startswith("cuda") else device.lower() + + +class Scorer: + """The author's jev-score-v2, a JSON-lines server on stdin/stdout.""" + + def __init__(self, binary, gguf, device, threads): + args = [binary, "--model", gguf, "--ngl", "0" if device == "cpu" else "999", "--threads", str(threads)] + self.p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1) + self.info = json.loads(self.p.stdout.readline()) + if not self.info.get("ready"): + raise SystemExit("jev-score-v2 did not start: %s" % self.info) + + def ask(self, req): + self.p.stdin.write(json.dumps(req) + "\n") + self.p.stdin.flush() + resp = json.loads(self.p.stdout.readline()) + if "error" in resp: + raise RuntimeError("jev-score-v2: %s" % resp["error"]) + return resp + + def close(self): + try: + self.p.stdin.write(json.dumps({"cmd": "quit"}) + "\n") + self.p.stdin.flush() + except BrokenPipeError: + pass + self.p.wait(timeout=60) + + +def softmax(scores, t): + z = np.asarray(scores, dtype=np.float64) / t + e = np.exp(z - z.max()) + return (e / e.sum()).tolist() + + +def layout_cases(renderer): + """This layout's own requests: several state blocks, catalogue overflow, the 25,600-token edge.""" + noul = {"type": "noul", "instructions": "Did the deploy cause the checkout failures?"} + sentence = ("The customer reports intermittent failures on the checkout page after the latest deploy; " + "payments time out, carts empty themselves and the support queue keeps growing. ") + out = [("jev/multi_block_state", sentence * 220, { + "caused": noul, + "team": {"type": "choice", "instructions": "Which team should take it?", + "criteria": {"payments": "card processing and refunds", "web": "the checkout front end", + "infra": None}}, + "severity": {"type": "score", "instructions": "How severe is it?", + "criteria": ["minor", "degraded", "major", "outage"]}})] + options = {"intent_%03d" % i: "the customer asks about item %d of the catalogue: billing, shipping, returns, " + "warranty or account changes for that item" % i for i in range(150)} + out.append(("jev/overflow_150", "I want to return the blender I bought last week and get my money back.", + {"intent": {"type": "choice", "instructions": "Which intent matches the message?", + "criteria": options}})) + # Digits are one token each in this tokenizer (and never merge), so the state can hit an exact length. + q = author_question(noul) + for want in (ref.CONTEXT_LIMIT, ref.CONTEXT_LIMIT + 1): + m = 0 + for _ in range(8): + state = "Ticket log follows. " + "7" * m + p = renderer.pieces(state, q) + n = len(p["prefix"]) + len(p["head"]) + sum(len(renderer.dash) + len(o) + 1 + len(renderer.newline) + for o in p["opts"]) # the short form's length + if n == want: + break + m += want - n + else: + raise SystemExit("could not build a %d-token input" % want) + out.append(("jev/budget_%d" % want, state, {"q": noul})) + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("export_dir", help="out/jev-style-, with the decision.json export_llama wrote") + ap.add_argument("--upstream", required=True, help="the GGUF repo at the pinned commit (jev_style_decision_gguf.py, " + "readout_config.json, tokenizer/tokenizer.json)") + ap.add_argument("--scorer", required=True, help="jev-score-v2 built against llama.cpp b11146") + ap.add_argument("--gguf", required=True) + ap.add_argument("--requests", required=True, help="Ollaya's shared request set (JSON lines)") + ap.add_argument("--device", default="cpu", help="cpu, or CUDA0 (all layers offloaded)") + ap.add_argument("--threads", type=int, default=8) + a = ap.parse_args() + + up = load_upstream(a.upstream) + readout = json.load(open(os.path.join(a.upstream, "readout_config.json"), encoding="utf-8")) + enc = up.TextEncoder(os.path.join(a.upstream, "tokenizer", "tokenizer.json")) + renderer = up.Renderer(enc, readout) + temperature = float(readout["temperatures"]["global"]) + decision = json.load(open(os.path.join(a.export_dir, "decision.json"), encoding="utf-8")) + yes, no, arrow = decision["yes"], decision["no"], decision["arrow"] + assert (yes, no, arrow) == (renderer.yes, renderer.no, renderer.arrow[0]), "readout ids differ from decision.json" + + cases = [(r["id"], r["state"], r["questions"]) for r in map(json.loads, open(a.requests, encoding="utf-8")) + if r.get("id")] + cases += layout_cases(renderer) + + scorer = Scorer(a.scorer, a.gguf, a.device, a.threads) + name = "goldens-%s" % device_class(a.device) + with open(os.path.join(a.export_dir, name + ".meta.json"), "w", encoding="utf-8") as f: + json.dump({"reference": "jev_style_decision_gguf.py Renderer + jev-score-v2 (jev_score_v2.cpp, unchanged)", + "scorer": {k: scorer.info.get(k) for k in ("protocol", "flash_attn", "gpu_offload", "n_ctx", + "n_ubatch", "n_seq_max", "n_outputs_max", "cache_type", + "n_threads", "desc", "system_info")}, + "device": a.device, "gguf": os.path.basename(a.gguf), "temperature": temperature, + "requests": os.path.basename(a.requests)}, f, indent=1) + + def render_all(state, questions): + """The author's render of every question (all before any scoring, as score_many), checked against ref.py.""" + if not isinstance(questions, dict) or not questions: + raise up.QuestionError("questions must be a non-empty object") + prefix = renderer.prefix_ids(state) + out = [] + for qid, q in questions.items(): + r = renderer.render(state, up.make_question(author_question(q)), prefix=prefix) + mine = ref.render(state, q, enc, arrow, prefix=prefix) + assert (mine.ids, mine.slots, mine.blocks, mine.names) == (r.ids, r.slots, [tuple(b) for b in r.blocks], + r.names), "ref.py differs on %s" % qid + out.append((qid, r)) + return prefix, out + + def record(cid, state, questions): + prefix, rendered = render_all(state, questions) + p = len(prefix) + resp = scorer.ask({"prefix": prefix, "prefix_blocks": rendered[0][1].state_blocks, + "questions": [{"ids": r.ids[p:], "blocks": r.question_blocks, "slots": r.slots, + "rows": [yes, no]} for _, r in rendered], + "keep_prefix": False, "return_logits": True}) + rows = [] + for (qid, r), res in zip(rendered, resp["results"]): + if not res["finite"]: + raise RuntimeError("%s/%s: non-finite score from jev-score-v2" % (cid, qid)) + rows.append({"qid": qid, "names": r.names, "ids": r.ids, "slots": r.slots, + "blocks": [list(b) for b in r.blocks], "catalogue_overflow": r.catalogue_overflow, + "logits": res["logits"], "scores": res["scores"], + "probabilities": softmax(res["scores"], temperature)}) + return {"id": cid, "state": state, "questions": questions, "state_tokens": p, "rows": rows} + + n, t0 = 0, time.time() + try: + with open(os.path.join(a.export_dir, name + ".jsonl"), "w", encoding="utf-8") as f: + for cid, state, questions in cases: + try: + rec = record(cid, state, questions) + except (up.QuestionError, up.InputBudgetError) as e: + for qid, q in (questions.items() if isinstance(questions, dict) else []): + try: # the port must reject what the author's runtime rejects + ref.render(state, q, enc, arrow) + except (ref.QuestionError, ref.InputBudgetError): + break + else: + raise AssertionError("%s: the author's runtime rejects it (%s), ref.py accepts" % (cid, e)) + f.write(json.dumps({"id": cid, "state": state, "questions": questions, "error": "invalid", + "message": str(e)}, ensure_ascii=False) + "\n") + valid = {} + for qid, q in (questions.items() if isinstance(questions, dict) else []): + try: + render_all(state, {qid: q}) + valid[qid] = q + except (up.QuestionError, up.InputBudgetError): + pass + if not valid: + continue + cid, rec = cid + "#valid", record(cid + "#valid", state, valid) + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + f.flush() + n += 1 + print("%4d %-48s %6d tokens %6.0fs" % (n, cid, max(len(r["ids"]) for r in rec["rows"]), + time.time() - t0), flush=True) + finally: + scorer.close() + print("wrote %d scored requests to %s" % (n, os.path.join(a.export_dir, name + ".jsonl"))) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/convert/ollaya_convert/families/jev_style/load_backends.c b/convert/ollaya_convert/families/jev_style/load_backends.c new file mode 100644 index 00000000..3d266862 --- /dev/null +++ b/convert/ollaya_convert/families/jev_style/load_backends.c @@ -0,0 +1,20 @@ +/* Linked next to the author's jev_score_v2.cpp (unchanged) when it is built against ggml-org's stock release + * libraries of llama.cpp b11146, the ones the runner loads. Those builds load the CPU backend variants at run + * time (GGML_BACKEND_DL), and jev-score-v2 calls only llama_backend_init(), so the backends are loaded here, + * before main(), from the directory that holds the libraries (JEV_GGML_BACKEND_DIR). + * + * c++ -std=c++17 -O2 -I/include -I/ggml/include -I/vendor \ + * jev_score_v2.cpp -x c load_backends.c -o jev-score-v2 -L -lllama -lggml -lggml-base -Wl,-rpath, + */ +#include + +#include "ggml-backend.h" + +__attribute__((constructor)) static void jev_load_backends(void) { + const char * dir = getenv("JEV_GGML_BACKEND_DIR"); + if (dir && *dir) { + ggml_backend_load_all_from_path(dir); + } else { + ggml_backend_load_all(); + } +} diff --git a/convert/ollaya_convert/families/jev_style/ref.py b/convert/ollaya_convert/families/jev_style/ref.py new file mode 100644 index 00000000..c9079583 --- /dev/null +++ b/convert/ollaya_convert/families/jev_style/ref.py @@ -0,0 +1,149 @@ +"""Reference implementation of `jev-style-v1`: chaoliangUNSW/Jev-Style-2B-Decision-v3's render v2 +(`macjev-render-v2-long-options`, layout `sb`) and verdict readout, ported from the model's own +self-contained runtime `jev_style_decision_gguf.py` (the "Shared core", Apache-2.0). + +Input layout (token segments are encoded separately so the ` ->` slots are exact): + + state prefix State:\n\n\n + short form Question []: \nOptions:\n + (q+opts+slots -