diff --git a/README.md b/README.md index a9c95ed..5fbadeb 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,15 @@ SkillForge treats skills as procedural memory. It can detect reusable workflows, write skill files, track execution feedback, and evolve instructions when they stop working. +### 6. A Harness That Evolves Itself + +`raven.evolver` runs measured self-evolution against a benchmark: it diagnoses +failing trajectories, designs candidate harness patches as real git commits, +and promotes only what passes statistical gates — with a sealed test set for +an honest generalisation number. One command +(`python -m raven.evolver run --config `), fully resumable. Start at +[raven/evolver/README.md](raven/evolver/README.md). +
@@ -293,6 +302,7 @@ official endorsement unless explicitly approved by EverMind. | Memory and plugin architecture | [docs/memory-plugin-architecture.md](docs/memory-plugin-architecture.md) | | Sandbox usage and debugging | [docs/sandbox/usage.md](docs/sandbox/usage.md) | | Proactivity design | [docs/Proactivity-Plan.md](docs/Proactivity-Plan.md) | +| Benchmark self-evolution | [raven/evolver/README.md](raven/evolver/README.md) | | Detailed design notes | [docs/README.md](docs/README.md) |
diff --git a/benchmarks/README.md b/benchmarks/README.md index 8529c3f..490753d 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -12,6 +12,12 @@ part of the end-user CLI package. ``` benchmarks/ +├── appworld/ AppWorld agent benchmark + evolver plugin +│ ├── agent_cli.py One-task subject agent (drives AgentLoop) +│ ├── batch.py Batch scorer: N tasks x K trials, resumable +│ └── evolve/ raven.evolver BenchBundle plugin (entry.py) +│ + designer/diagnosis/sandbox/precheck glue +│ ├── pinchbench/ Context / AgentLoop capability benchmark │ ├── tasks/ 23 task_*.md cards (YAML frontmatter + sections) │ ├── direct/ Drives AgentLoop.process_direct() per task @@ -135,9 +141,15 @@ uv run python benchmarks/proactivity_eval/runners/run.py \ ## Relation to runtime -The runtime (`raven/`) **never imports from `benchmarks/`** — this is the -"independent eval track" principle. The reverse is allowed and expected: -benchmarks import `raven.agent`, `raven.providers`, etc. directly. +The runtime (`raven/`) **never statically imports from `benchmarks/`** — this +is the "independent eval track" principle. The reverse is allowed and +expected: benchmarks import `raven.agent`, `raven.providers`, etc. directly. + +One scoped exception: `raven.evolver` loads its bench *plugins* from here by +registry name at launch (`benchmarks.appworld.evolve.entry:build`), inserting +the subject repo root on `sys.path` first. It is lazy, opt-in, and only works +from a repo checkout — evolution needs the git repo as its subject anyway, so +nothing in the installed wheel depends on this directory. Two unit tests under `tests/` reach into `benchmarks/proactivity_eval/runners/` via `sys.path` injection because they exercise the synthesizer / prompt-loader diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..1344a28 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1,8 @@ +"""Repo-root benchmark harnesses (see README.md in this directory). + +This __init__ exists so evolvable benches (e.g. ``benchmarks.appworld``) are +importable with the repo checkout root on ``sys.path`` — the evolver inserts +the subject repo root before loading a bench plugin, and worktree evals rely +on cwd-first ``python -m`` resolution. The directory still ships in neither +the wheel nor the sdist. +""" diff --git a/benchmarks/appworld/__init__.py b/benchmarks/appworld/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/appworld/agent_cli.py b/benchmarks/appworld/agent_cli.py new file mode 100644 index 0000000..b344c4f --- /dev/null +++ b/benchmarks/appworld/agent_cli.py @@ -0,0 +1,210 @@ +"""Run ONE AppWorld task through a minimal Raven AgentLoop, then grade. + +Cross-venv design (see tool.py): AppWorld (pydantic v1) runs as an HTTP +``environment`` server; this runner (Raven, pydantic v2) never imports +appworld — it drives the task purely over HTTP: + + POST {env}/initialize {task_id} -> instruction + supervisor (loads world) + agent's `execute` tool -> POST {env}/execute {task_id, code} + POST {env}/evaluate {task_id} -> TestTracker dict (success = pass@1) + POST {env}/task_completed, /close + +One env server holds one world at a time, so the batch runner gives each +concurrent task its own server/port (--env-url). The harness is plain vanilla: +the agent gets only the `execute` tool and the base AppWorld prompt. + +Usage:: + + python -m benchmarks.appworld.agent_cli \ + --task-id 82e2fac_1 --env-url http://127.0.0.1:8100 \ + --config --out result.json [--experiment vanilla] +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import sys +import time + +from benchmarks.appworld.evolve import grade + +APPWORLD_PROMPT = """You are an autonomous agent that completes a digital task for your supervisor by writing and running Python code. + +You act ONLY through the `execute` tool: you write Python, it runs in a stateful REPL (variables, imports and logins persist across calls, like a Jupyter notebook), and you get back stdout. You MUST print() anything you want to observe. + +How to work: +- See the apps: print(apis.api_docs.show_app_descriptions()) +- List an app's APIs: print(apis.api_docs.show_api_descriptions(app_name='spotify')) +- Read one API's doc: print(apis.api_docs.show_api_doc(app_name='spotify', api_name='login')) +- Your supervisor's data: apis.supervisor.show_profile(), show_account_passwords(), show_addresses(), show_payment_cards() +- Most APIs need an access_token. Log in first, e.g.: + pwds = apis.supervisor.show_account_passwords() + token = apis.spotify.login(username=..., password=...)["access_token"] +- Go step by step: inspect first, then act, then verify before finalizing. + +When the task is complete, call exactly once: +- A question -> apis.supervisor.complete_task(answer=) +- An action -> apis.supervisor.complete_task() + +Your supervisor: {supervisor} + +Your task: +{instruction} +""" + + +def _build_agent(args): + from benchmarks.appworld.tool import AppWorldExecuteTool + from raven.agent.loop import AgentLoop + from raven.cli._helpers import load_runtime_config, make_provider + from raven.config.raven import load_raven_config + from raven.session.manager import SessionManager + + config = load_runtime_config(args.config, args.workspace) + ec_config = load_raven_config() + provider = make_provider(config) + + # AppWorld is pure API/code: disable every default tool, give only `execute`. + disabled = [ + "read_file", + "write_file", + "edit_file", + "list_dir", + "exec", + "web_search", + "web_fetch", + "message", + "spawn", + "cron", + ] + agent = AgentLoop( + provider=provider, + workspace=config.workspace_path, + model=args.model or config.agents.defaults.model, + max_iterations=config.agents.defaults.max_tool_iterations, + context_window_tokens=config.agents.defaults.context_window_tokens, + exec_config=config.tools.exec, + restrict_to_workspace=config.tools.restrict_to_workspace, + session_manager=SessionManager(config.workspace_path), + context_config=ec_config.context, + hooks=None, + disabled_tools=disabled, + ) + exec_tool = AppWorldExecuteTool(args.env_url, args.task_id) + agent.tools.register(exec_tool) + return agent, exec_tool + + +async def _run(args) -> dict: + # Grading, infra classification, and the result write live in + # benchmarks.appworld.evolve.grade (the evolver-immutable scorer); + # this file is the editable agent surface and must never grade itself. + t0 = time.time() + result: dict = {"task_id": args.task_id, "experiment": args.experiment} + try: + init = grade.post( + args.env_url, + "/initialize", + # unique experiment_name per attempt -> isolated experiments/outputs/<...>/tasks// + # dir, so concurrent K-trials of the same task never collide on model_hashes.json (Errno 22). + {"task_id": args.task_id, "experiment_name": (args.session or args.experiment)}, + ) + prompt = APPWORLD_PROMPT.format(supervisor=init.get("supervisor"), instruction=init.get("instruction")) + agent, exec_tool = _build_agent(args) + skey = args.session or args.task_id + try: + # Raven's AgentLoop is spine-driven (no process_direct): drive one headless + # turn through run_turn with no-op emit/drain and capture the final reply via + # text_sink. AppWorld success is judged by the env oracle (/evaluate), not this + # text; it is kept only for transport-error detection and the result record. + from raven.spine import ChatType, Origin, Source, TurnRequest + + async def _emit(_event): + return None + + def _drain(): + return [] + + sink: dict = {} + req = TurnRequest( + origin=Origin.USER, + source=Source(channel="cli", chat_id="direct", sender_id="user", chat_type=ChatType.DM), + # Raven's SessionManager splits the conversation key on ':' into + # /.jsonl. Prefix a fixed channel so the per- + # attempt transcript lands at a clean flat path the evolver reads: + # ws/sessions/appworld/__k.jsonl. + text=prompt, + conversation=f"appworld:{skey}", + ) + await agent.run_turn(req, _emit, _drain, stream=False, text_sink=sink) + response = sink.get("text") or "" + finally: + await agent.close_executor() + client = getattr(getattr(agent, "provider", None), "_client", None) + closer = getattr(client, "close", None) + if closer is not None: + try: + await closer() + except Exception: + pass + grade.grade_and_record( + result, + env_url=args.env_url, + task_id=args.task_id, + response=response, + config_path=args.config, + t0=t0, + ) + except BaseException as e: + grade.record_infra(result, e, t0=t0) + finally: + try: + grade.post(args.env_url, "/close", {"task_id": args.task_id}) + except Exception: + pass + return result + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="appworld-agent") + p.add_argument("--task-id", required=True) + p.add_argument("--env-url", required=True, help="AppWorld environment server base URL.") + p.add_argument("--config", required=True, help="Raven runtime config JSON.") + p.add_argument("--out", required=True, help="Where to write the result JSON.") + p.add_argument("--workspace", default=os.path.expanduser("~/workspace/appworld-run/ws")) + p.add_argument("--model", default=None) + p.add_argument("--experiment", default="vanilla") + p.add_argument( + "--session", + default=None, + help="Session key (jsonl stem). Default task_id; pass a per-attempt " + "key to retain all K trajectories instead of overwriting.", + ) + args = p.parse_args(argv) + + import contextlib + + try: + import litellm + + litellm.suppress_debug_info = True + except Exception: + pass + + with contextlib.redirect_stdout(sys.stderr): + result = asyncio.run(_run(args)) + + grade.write_result(args.out, result) + sys.stderr.write( + f"[appworld] {args.task_id} success={result.get('success')} " + f"infra={result.get('infra_error')} t={result.get('elapsed_s')}s\n" + ) + return 0 + + +if __name__ == "__main__": + import os + + os._exit(main()) diff --git a/benchmarks/appworld/batch.py b/benchmarks/appworld/batch.py new file mode 100644 index 0000000..b120675 --- /dev/null +++ b/benchmarks/appworld/batch.py @@ -0,0 +1,277 @@ +"""Batch runner for AppWorld via N single-world env servers (multi-port). + +Each ``appworld serve environment`` holds ONE world at a time, so concurrency = +N server processes on N ports. We pin one port per worker thread and stream +tasks through; per task we spawn the agent_cli subprocess (this raven venv) +pointed at that worker's port. + + raven venv : this orchestrator + agent_cli subprocesses (pydantic v2) + appworld venv : N env servers (pydantic v1), started here as subprocesses + +Usage:: + + python -m benchmarks.appworld.batch \ + --split train --n 20 --k 1 --conc 8 \ + --config /path/to/subject_config.json \ + --out-dir /private/tmp/appworld-eval/runs/floor \ + --experiment vanilla +""" + +from __future__ import annotations + +import argparse +import json +import os +import queue +import subprocess +import sys +import threading +import time +import urllib.request + +from raven.evolver.activation.ledger import ( + WORKSPACE_ENV, + beacon_workspace, + mark_beacons_enabled, +) + +# Dev-box defaults, overridable per machine without a code edit. +APPWORLD_ROOT = os.environ.get("APPWORLD_ROOT", os.path.expanduser("~/workspace/appworld-run")) +APPWORLD_BIN = os.environ.get("APPWORLD_BIN", os.path.join(APPWORLD_ROOT, "appworld-venv/bin/appworld")) +APPWORLD_PY = os.environ.get("APPWORLD_PY", os.path.join(APPWORLD_ROOT, "appworld-venv/bin/python")) + + +def _task_ids(split: str, n: int | None) -> list[str]: + out = subprocess.check_output( + [APPWORLD_PY, "-c", f"from appworld import load_task_ids; print('\\n'.join(load_task_ids('{split}')))"], + cwd=APPWORLD_ROOT, + text=True, + timeout=120, + ) + ids = [x.strip() for x in out.splitlines() if x.strip()] + return ids[:n] if n else ids + + +def _start_server(port: int, log_dir: str) -> subprocess.Popen: + logf = open(os.path.join(log_dir, f"envserver-{port}.log"), "w") + return subprocess.Popen( + [APPWORLD_BIN, "serve", "environment", "--port", str(port)], + cwd=APPWORLD_ROOT, + stdout=logf, + stderr=subprocess.STDOUT, + ) + + +def _wait_up(port: int, timeout: float = 60.0) -> bool: + t0 = time.time() + while time.time() - t0 < timeout: + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=3) as r: + if r.status == 200: + return True + except Exception: + time.sleep(1) + return False + + +def _run_one(task_id: str, k: int, port: int, args, out_dir: str) -> dict: + out = os.path.join(out_dir, f"{task_id}_k{k}.json") + # Trial-level resume: a parseable result file is the proof this trial ran; + # skip it so re-invocation only fills the gaps. A half-written file (crash + # mid-dump) fails to parse and is re-run. Infra-marked results are kept: + # their retry belongs to the infra-rerun ladder's separate out-dirs. + try: + with open(out) as f: + return json.load(f) + except (OSError, ValueError): + pass + cmd = [ + sys.executable, + "-m", + "benchmarks.appworld.agent_cli", + "--task-id", + task_id, + "--env-url", + f"http://127.0.0.1:{port}", + "--config", + args.config, + "--out", + out, + "--workspace", + args.workspace, + "--experiment", + args.experiment, + "--session", + f"{task_id}_{args.experiment}_k{k}", + ] # per-attempt: retain all K trajectories + if args.model: + cmd += ["--model", args.model] + env = dict(os.environ) + # Per-attempt beacon workspace (Gate-b): each agent_cli subprocess writes + # its activation beacons under its own dir, pre-split by task. Beacon-less + # code never writes anything, so this is behavior-neutral for vanilla. + beacon_ws = beacon_workspace(out_dir, task_id, k) + try: + beacon_ws.mkdir(parents=True, exist_ok=True) + env[WORKSPACE_ENV] = str(beacon_ws) + except OSError: + pass + for kv in (args.env or "").split(","): + if "=" in kv: + kk, vv = kv.split("=", 1) + env[kk.strip()] = vv.strip() + logf = os.path.join(out_dir, f"{task_id}_k{k}.stderr") + run_err = None + try: + with open(logf, "w") as lf: + subprocess.run(cmd, stderr=lf, stdout=lf, timeout=args.task_timeout, env=env) + except Exception as e: # timeout / spawn failure: agent_cli never wrote --out + run_err = f"runner: {type(e).__name__}: {e}" + try: + return json.load(open(out)) + except Exception as e: + # The trial MUST leave a result file: scoring counts attempts from + # files, and the infra-rerun ladder only reruns tasks whose eval shows + # infra trials. An unwritten timeout would otherwise be invisible + # (task scored over fewer attempts, never re-run). + rec = { + "task_id": task_id, + "success": False, + "task_completed": False, + "infra_error": run_err or f"no-result: {e}", + } + try: + with open(out, "w") as f: + json.dump(rec, f) + except OSError: + pass + return rec + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="appworld-batch") + p.add_argument("--split", default="train") + p.add_argument("--n", type=int, default=20) + p.add_argument("--k", type=int, default=1) + p.add_argument("--conc", type=int, default=8) + p.add_argument("--config", required=True) + p.add_argument("--out-dir", required=True) + p.add_argument("--workspace", default=os.path.join(APPWORLD_ROOT, "ws")) + p.add_argument("--model", default=None) + p.add_argument("--experiment", default="vanilla") + p.add_argument("--env", default="", help="Candidate env vars, e.g. 'VERIFY_FINALIZE=1,C3=1'") + p.add_argument("--tasklist", default="", help="Explicit task-id file (overrides --split/--n).") + p.add_argument("--base-port", type=int, default=8100) + p.add_argument("--task-timeout", type=int, default=900) + args = p.parse_args(argv) + + os.makedirs(args.out_dir, exist_ok=True) + os.makedirs(args.workspace, exist_ok=True) + mark_beacons_enabled(args.out_dir) + if args.tasklist: + tasks = [x.strip() for x in open(args.tasklist) if x.strip()] + else: + tasks = _task_ids(args.split, args.n) + print(f"[batch] {len(tasks)} tasks x K={args.k}, conc={args.conc}, exp={args.experiment} env={args.env}") + + ports = list(range(args.base_port, args.base_port + args.conc)) + servers: dict[int, subprocess.Popen] = {} + + def _shutdown_servers(): + for proc in servers.values(): + proc.terminate() + for proc in servers.values(): + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + try: + for pt in ports: + servers[pt] = _start_server(pt, args.out_dir) + failed_ports = [] + for pt in ports: + ok = _wait_up(pt) + print(f"[batch] env server :{pt} {'UP' if ok else 'FAILED'}") + if not ok: + failed_ports.append(pt) + if failed_ports: + # A dead server would burn 1/N of every task as infra errors; + # refusing to score is the only honest option. + print( + f"[batch] aborting: env servers failed to start on ports " + f"{failed_ports} (see envserver-*.log in {args.out_dir})", + file=sys.stderr, + ) + return 3 + + # work queue of (task_id, k); worker per port + work: "queue.Queue" = queue.Queue() + for t in tasks: + for k in range(args.k): + work.put((t, k)) + results: list[dict] = [] + lock = threading.Lock() + done = [0] + total = work.qsize() + + def worker(port: int): + while True: + try: + task_id, k = work.get_nowait() + except queue.Empty: + return + try: + res = _run_one(task_id, k, port, args, args.out_dir) + except Exception as e: + res = {"task_id": task_id, "success": False, "infra_error": f"runner: {e}"} + with lock: + results.append(res) + done[0] += 1 + print( + f"[batch] {done[0]}/{total} :{port} {task_id} " + f"success={res.get('success')} done={res.get('task_completed')} " + f"infra={res.get('infra_error')}" + ) + work.task_done() + + threads = [threading.Thread(target=worker, args=(pt,)) for pt in ports] + for th in threads: + th.start() + for th in threads: + th.join() + finally: + _shutdown_servers() + + # ---- summarize ---- + def mode(r: dict) -> str: + if r.get("success"): + return "PASS" + if r.get("infra_error"): + return "INFRA" + if r.get("task_completed"): + return "LEGIT_FAIL" # tried + completed but wrong + return "INCOMPLETE" # stopped early / empty response + + from collections import Counter + + by_mode = Counter(mode(r) for r in results) + npass = by_mode.get("PASS", 0) + summary = { + "n_tasks": len(tasks), + "k": args.k, + "n_trials": len(results), + "pass_at_1": round(npass / len(results), 4) if results else 0, + "modes": dict(by_mode), + "experiment": args.experiment, + } + tmp_path = os.path.join(args.out_dir, "summary.json.tmp") + with open(tmp_path, "w") as f: + json.dump({"summary": summary, "results": results}, f, indent=2) + os.replace(tmp_path, os.path.join(args.out_dir, "summary.json")) + print("\n[batch] SUMMARY:", json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/appworld/evolve/__init__.py b/benchmarks/appworld/evolve/__init__.py new file mode 100644 index 0000000..f74ea26 --- /dev/null +++ b/benchmarks/appworld/evolve/__init__.py @@ -0,0 +1,6 @@ +"""In-package AppWorld self-evolution: the domain brain (W1-W7 diagnosis, +trajectory rendering, bash-editor design) + wiring into the generic evolver +orchestrator. Replaces the external ``appworld-run/evo_scripts/appworld_bash.py`` +loop reimplementation — here the orchestrator/gate/baseline come from +``raven.evolver.orchestrator`` and only the AppWorld-specific steps live here. +""" diff --git a/benchmarks/appworld/evolve/adapter.py b/benchmarks/appworld/evolve/adapter.py new file mode 100644 index 0000000..756b4af --- /dev/null +++ b/benchmarks/appworld/evolve/adapter.py @@ -0,0 +1,363 @@ +"""AppWorld scorer adapter — the fast bench for a real self-evolution round. + +AppWorld (StonyBrookNLP) is an interactive benchmark: the agent writes Python +that calls app APIs (venmo / spotify / gmail / ...) to complete a task, scored +``pass@1`` over K attempts. Splits: train 90 / dev 57 / test_normal 168. It runs +much faster than SWE-bench, which makes a full evolution round tractable. + +The scorer is the batch orchestrator ``benchmarks.appworld.batch`` (in +the appworld worktree; talks to appworld over HTTP — two venvs, appworld pins +pydantic v1, raven v2). CLI (from the real ``batch.py``):: + + python -m benchmarks.appworld.batch \ + --split train --n 90 --k 3 --conc 8 \ + --config \ + --out-dir --experiment --env "VERIFY_FINALIZE=1" + +A task subset (anchor screen) is passed as ``--tasklist `` (one id per +line), not a comma list. The primary eval path checks a candidate's real +commit out into a worktree and runs against that checkout (``cwd`` override +in :func:`run_eval`); the ``--env`` activation string remains supported for +env-gated experiments but the shipped harness carries no env-gated levers. + +Output layout (what ``batch.py`` writes): the out-dir +holds one ``{task_id}_k{k}.json`` per attempt plus a ``summary.json``. Each +attempt dict carries ``success`` (pass), ``task_completed``, and — on infra +failure — ``infra_error``. :func:`read_out_dir` replicates +``paired_verdict.load_passcounts`` exactly: attempts counts every trial +(including infra, which scores as a non-pass), passes counts ``success``, and +infra attempts are surfaced separately for Gate-f re-run accounting. Emitting +the bench-neutral :class:`TaskEval` contract keeps the orchestrator +bench-agnostic above this seam. +""" + +from __future__ import annotations + +import glob +import json +import os +import re +import subprocess +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any, Callable, Optional + +from raven.evolver.analysis.stability_bucket import ( + TaskStability, + _bucket_for, +) +from raven.evolver.orchestrator.scoring import ( + EvalBackend, + TaskEval, + with_infra_rerun, +) +from raven.evolver.scheduler.anchor_selection import simple_anchor +from raven.evolver.tree.node import HarnessNode + +ActivationOf = Callable[[HarnessNode], Any] + +# Batch mode classifications (from batch.py's mode()). INFRA is an +# infrastructure failure (env/proxy/timeout) — a Gate-f re-run candidate. +MODE_PASS = "PASS" +MODE_LEGIT_FAIL = "LEGIT_FAIL" # completed but wrong answer +MODE_INCOMPLETE = "INCOMPLETE" # stopped early / empty response +MODE_INFRA = "INFRA" # env/proxy/timeout + +_TRIAL_SUFFIX_RE = re.compile(r"_k\d+\.json$") + + +@dataclass(frozen=True) +class AppWorldConfig: + """Locate and parameterise the AppWorld batch scorer.""" + + appworld_root: Path # raven checkout/worktree (holds the batch module) + python_exe: str # the raven (pydantic v2) venv python + config_path: Path # subject runtime config JSON + out_dir_root: Path + split: str = "train" + n: int = 90 + conc: int = 8 + base_port: int | None = None + batch_module: str = "benchmarks.appworld.batch" + # Agent workspace (sessions land under /...). Must equal the + # ws_root diagnosis reads trajectories from; None = batch.py's default. + workspace: Path | None = None + extra_args: tuple[str, ...] = () + + def __post_init__(self) -> None: + for name in ("appworld_root", "config_path", "out_dir_root"): + object.__setattr__(self, name, Path(getattr(self, name))) + if self.workspace is not None: + object.__setattr__(self, "workspace", Path(self.workspace)) + + +def _activation_arg(activation_env: str | dict[str, str] | None) -> str: + """Coerce a candidate's activation into the ``--env "A=1,B=1"`` string.""" + if not activation_env: + return "" + if isinstance(activation_env, dict): + return ",".join(f"{k}={v}" for k, v in activation_env.items()) + return str(activation_env) + + +def _task_id_of(record: dict, path: str) -> str: + """Task id from the record, falling back to the filename (as paired_verdict does).""" + return record.get("task_id") or _TRIAL_SUFFIX_RE.sub("", os.path.basename(path)) + + +def build_argv( + aw: AppWorldConfig, + K: int, + experiment: str, + out_dir: str | Path, + *, + activation_env: str | dict[str, str] | None = None, + tasklist_path: str | Path | None = None, +) -> list[str]: + """Assemble the ``batch.py`` command line for one screen/confirm run. + + A subset run passes ``--tasklist`` (a file of ids); a full-split run passes + ``--n``. ``batch.py`` has no comma ``--tasks`` flag. + """ + if K < 1: + raise ValueError(f"K must be >= 1, got {K}") + argv = [ + aw.python_exe, + "-m", + aw.batch_module, + "--split", + aw.split, + "--k", + str(K), + "--conc", + str(aw.conc), + "--config", + str(aw.config_path), + "--out-dir", + str(out_dir), + "--experiment", + experiment, + ] + if tasklist_path is not None: + argv += ["--tasklist", str(tasklist_path)] + else: + argv += ["--n", str(aw.n)] + if aw.workspace is not None: + argv += ["--workspace", str(aw.workspace)] + env_arg = _activation_arg(activation_env) + if env_arg: + argv += ["--env", env_arg] + if aw.base_port is not None: + argv += ["--base-port", str(aw.base_port)] + argv += list(aw.extra_args) + return argv + + +def read_out_dir(out_dir: str | Path) -> dict[str, TaskEval]: + """Aggregate a batch out-dir into per-task ``TaskEval``. + + Reads the per-attempt ``{task_id}_k{k}.json`` files exactly as + ``paired_verdict.load_passcounts``: ``attempts`` counts every trial + (infra included, as a non-pass), ``passes`` counts ``success``, and + ``infra_attempts`` counts trials carrying ``infra_error``. + """ + out_dir = Path(out_dir) + files = sorted(glob.glob(str(out_dir / "*_k*.json"))) + if not files: + raise FileNotFoundError(f"no per-attempt *_k*.json files in AppWorld out-dir: {out_dir}") + + passes: dict[str, int] = {} + attempts: dict[str, int] = {} + infra: dict[str, int] = {} + for path in files: + try: + rec = json.load(open(path)) + except Exception: # noqa: BLE001 — a corrupt trial file is skipped + continue + tid = _task_id_of(rec, path) + attempts[tid] = attempts.get(tid, 0) + 1 + if rec.get("infra_error"): + infra[tid] = infra.get(tid, 0) + 1 + if rec.get("success"): + passes[tid] = passes.get(tid, 0) + 1 + + return { + t: TaskEval( + task_id=t, + passes=passes.get(t, 0), + attempts=attempts[t], + infra_attempts=infra.get(t, 0), + ) + for t in attempts + } + + +def ladder_out_dirs(out_dir: str | Path) -> list[Path]: + """The base out-dir plus its SOP §0 infra-rerun ladder siblings that exist + (``_infra_rerun{1,2}``, written by ``eval_with_infra_rerun``).""" + out_dir = Path(out_dir) + dirs = [] + for name in (out_dir.name, f"{out_dir.name}_infra_rerun1", f"{out_dir.name}_infra_rerun2"): + d = out_dir.parent / name + if d.exists() and list(d.glob("*_k*.json")): + dirs.append(d) + return dirs + + +def read_kept_out_dir(out_dir: str | Path) -> dict[str, TaskEval]: + """Per-task KEPT measurement across the infra-rerun ladder. + + Mirrors ``eval_with_infra_rerun``'s keep rule exactly — fewest infra trials + wins, earlier dir wins ties — so a control arm read from disk sees the same + measurement the candidate arm was scored with. Reading only the base dir + would score a rerun-salvaged task as its contaminated first attempt, + deflating the control and handing every candidate a free lift (SOP §0). + """ + dirs = ladder_out_dirs(out_dir) + if not dirs: + raise FileNotFoundError(f"no per-attempt *_k*.json files in AppWorld out-dir: {out_dir}") + kept: dict[str, TaskEval] = {} + for d in dirs: + for tid, ev in read_out_dir(d).items(): + cur = kept.get(tid) + if cur is None or ev.infra_attempts < cur.infra_attempts: + kept[tid] = ev + return kept + + +def run_eval( + aw: AppWorldConfig, + K: int, + experiment: str, + *, + activation_env: str | dict[str, str] | None = None, + task_ids: list[str] | None = None, + timeout: float | None = None, + cwd: str | Path | None = None, +) -> dict[str, TaskEval]: + """Run the AppWorld batch scorer and read per-task results back. + + ``experiment`` names the out-dir under ``out_dir_root`` and the session tag. + Use a distinct ``experiment`` (and ``base_port``) per concurrent run so two + batches never share single-world env servers (onboarding §6.5). A + ``task_ids`` subset is written to ``/tasklist.txt`` and passed via + ``--tasklist``. + + ``cwd`` overrides the subprocess working directory (default + ``aw.appworld_root``). Passing a candidate commit's worktree here makes + ``python -m raven...`` import the candidate's harness from that checkout + (cwd is first on ``sys.path`` for ``-m``) — the zero-contamination eval that + replaces writing candidate files into the live repo. Activation env is then + unnecessary: the committed code is already the candidate. + """ + out_dir = aw.out_dir_root / experiment + out_dir.mkdir(parents=True, exist_ok=True) + tasklist_path: Path | None = None + if task_ids: + tasklist_path = out_dir / "tasklist.txt" + tasklist_path.write_text("\n".join(task_ids) + "\n") + argv = build_argv( + aw, + K, + experiment, + out_dir, + activation_env=activation_env, + tasklist_path=tasklist_path, + ) + subprocess.run(argv, cwd=str(cwd or aw.appworld_root), env=dict(os.environ), check=True, timeout=timeout) + return read_out_dir(out_dir) + + +def stability_from_out_dir(out_dir: str | Path) -> dict[str, TaskStability]: + """Build a ``{task_id: TaskStability}`` from an AppWorld vanilla out-dir. + + The orchestrator's cold start normally reads legacy trial-dirs via + ``compute_stability``; AppWorld's ``{task}_k{k}.json`` format needs this + equivalent so a vanilla AppWorld run can seed ``vanilla_stability`` and the + anchor. Buckets use the same ``_bucket_for`` as the legacy path. Reads the + KEPT measurement across the infra-rerun ladder (:func:`read_kept_out_dir`) + so the baseline reflects the same salvage rule candidate evals get. + """ + evals = read_kept_out_dir(out_dir) + return { + t: TaskStability( + task_id=t, + attempts=ev.attempts, + passes=ev.passes, + bucket=_bucket_for(ev.passes, ev.attempts), + ) + for t, ev in evals.items() + } + + +def make_appworld_backend( + aw: AppWorldConfig, + *, + vanilla_out_dir: str | Path, + train_task_ids: list[str], + test_task_ids: list[str] = (), + activation_of: Optional[ActivationOf] = None, + cull_sigma_mult: float = 1.5, + trajectory_source=None, + eval_fn=None, + infra_max_reruns: int = 2, + precheck=None, + vanilla_node: Optional[HarnessNode] = None, + cold_start_k: int = 3, +) -> EvalBackend: + """AppWorld backend: batch scorer + cold-start over a vanilla out-dir. + + AppWorld surfaces per-trial ``infra_error``, so the SOP §0 rerun ladder is + active here: infra-contaminated tasks are re-scored up to ``infra_max_reruns`` + times before any survivor is left to score 0 in the denominator. + + Cold start is **read-or-run** (SOP §1, idempotent): if ``vanilla_out_dir`` + already holds a ledger it is reused; otherwise, given ``vanilla_node``, the + vanilla harness is scored on train x ``cold_start_k`` into that dir first + (through the same infra-rerun eval). ``vanilla_out_dir`` must be + ``aw.out_dir_root / `` so the run lands where the read looks. + """ + act = activation_of or (lambda _node: None) + vdir = Path(vanilla_out_dir) + + def default_eval(node, task_ids, k, job_name, *, split="train"): + cfg = aw if split == aw.split else replace(aw, split=split) + return run_eval(cfg, K=k, experiment=job_name, task_ids=task_ids, activation_env=act(node)) + + wrapped_eval = with_infra_rerun(eval_fn or default_eval, infra_max_reruns) + + def cold_start() -> dict[str, TaskStability]: + if not (vdir.exists() and list(vdir.glob("*_k*.json"))): + if vanilla_node is None: + raise FileNotFoundError(f"vanilla ledger missing at {vdir} and no vanilla_node given to run it") + wrapped_eval(vanilla_node, list(train_task_ids), cold_start_k, vdir.name, split="train") + return stability_from_out_dir(vdir) + + def anchor(affinity=None): + return simple_anchor(stability_from_out_dir(vdir), cull_sigma_mult=cull_sigma_mult) + + return EvalBackend( + train_task_ids=list(train_task_ids), + test_task_ids=list(test_task_ids), + eval=wrapped_eval, + cold_start=cold_start, + anchor=anchor, + trajectories=trajectory_source, + precheck=precheck, + ) + + +__all__ = [ + "AppWorldConfig", + "MODE_PASS", + "MODE_LEGIT_FAIL", + "MODE_INCOMPLETE", + "MODE_INFRA", + "build_argv", + "read_out_dir", + "read_kept_out_dir", + "ladder_out_dirs", + "run_eval", + "stability_from_out_dir", + "make_appworld_backend", +] diff --git a/benchmarks/appworld/evolve/agentic.py b/benchmarks/appworld/evolve/agentic.py new file mode 100644 index 0000000..8822596 --- /dev/null +++ b/benchmarks/appworld/evolve/agentic.py @@ -0,0 +1,278 @@ +"""AppWorld agentic analysis front-end (analysis_mode="agentic"). + +Replaces the map-reduce diagnosis (one shallow LLM read per failing +trajectory) with ONE read-only Claude Code session that investigates the run +like an engineer — pre-aggregated ledger first, then deep-reads of +representative transcripts per failure signature, then the harness source — +and emits a taxonomy-constrained diagnosis. The output converts into the same +``failure_map`` shape as :func:`classify_failures`, so WHY selection, briefs, +history, and gates downstream are untouched (SOP funnel unchanged; only HOW +the analyst reads changed). + +The session workspace is assembled per round under the orchestrator work dir: + + ledger_digest.md pre-aggregated outcomes (spares the session Bash-less + aggregation of 270 result files) + runs/ symlink -> the parent baseline out-dir (raw result JSONs) + sessions/, att/ symlinks -> the agent transcript trees + harness/ git worktree PINNED AT THE PARENT COMMIT (the exact code + a fix would patch — the live repo may have drifted) +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import Callable, Optional + +from benchmarks.appworld.evolve.diagnose import ( + APPWORLD_BENCH_INTRO, + APPWORLD_DIAGNOSIS_RULES, +) +from raven.evolver.orchestrator.nodes.taxonomy import ( + TaxonomySpec, + add_failure_mode, + coerce_mode, + empty_failure_map, + strip_code_fence, +) +from raven.evolver.orchestrator.providers.claude_agentic import run_agentic_session +from raven.evolver.tree import git_ops +from raven.evolver.tree.node import HarnessNode + + +def _task_states(runs_root: Path, exp: str, k: int) -> dict[str, dict]: + """Per-task attempt outcomes + first oracle signature from the ledger.""" + out: dict[str, dict] = {} + for p in sorted((runs_root / exp).glob("*_k*.json")): + r = json.loads(p.read_text()) + tid = r.get("task_id") + if not tid: + continue + st = out.setdefault(tid, {"marks": [], "sig": ""}) + if r.get("infra_error"): + st["marks"].append("I") + continue + st["marks"].append("P" if r.get("success") else "F") + if not r.get("success") and not st["sig"]: + fails = (r.get("evaluation") or {}).get("failures") or [] + if fails: + req = str(fails[0].get("requirement", ""))[:80] + # The trace tail carries the class-defining detail the + # requirement lacks (e.g. '<>' = nothing submitted) + trace = str(fails[0].get("trace", "")).strip().splitlines() + tail = trace[-1][:80] if trace else "" + st["sig"] = f"{req} | {tail}" if tail else req + return out + + +def build_analysis_workspace( + work_dir: Path, + runs_root: Path, + ws_root: Path, + exp: str, + k: int, +) -> tuple[Path, list[str]]: + """Assemble the read-only workspace; returns ``(ws, failing_task_ids)``.""" + ws = work_dir / "agentic_analysis" / exp + ws.mkdir(parents=True, exist_ok=True) + states = _task_states(runs_root, exp, k) + failing = sorted(t for t, s in states.items() if "F" in s["marks"]) + sig_hist = Counter(s["sig"] for t, s in states.items() if t in set(failing) and s["sig"]) + + lines = [ + "# Ledger digest (pre-aggregated — start here)", + "", + f"experiment: {exp} | tasks: {len(states)} | failing (need labels): {len(failing)}", + "", + "## Failure signature histogram (oracle first-failed requirement)", + ] + lines += [f"- {c:3d}x {s}" for s, c in sig_hist.most_common()] + lines += ["", "## Per-task outcomes (P=pass F=fail I=infra, one mark per attempt)"] + for tid in sorted(states): + s = states[tid] + sig = f' sig="{s["sig"]}"' if s["sig"] else "" + lines.append(f"- {tid}: {''.join(s['marks'])}{sig}") + lines += [ + "", + "## Where to look deeper", + "- raw results: runs/_k.json (evaluation.failures = oracle)", + f"- transcripts: sessions/appworld/_{exp}_k.jsonl (Raven layout) or att/_{exp}_k/sessions/", + "- harness source (the code a fix would patch): harness/benchmarks/appworld/, harness/raven/agent/", + ] + (ws / "ledger_digest.md").write_text("\n".join(lines)) + + for name, target in ( + ("runs", runs_root / exp), + ("sessions", ws_root / "sessions"), + ("att", ws_root / "att"), + ): + link = ws / name + if link.is_symlink() or link.exists(): + continue + if Path(target).exists(): + link.symlink_to(Path(target).resolve()) + return ws, failing + + +def _salvage_truncated(blob: str): + """Parse a truncation-damaged JSON blob by cutting back to the last + complete object and balance-closing the open brackets. Long agentic + sessions can hit the final-message length cap mid-array; the assignments + generated before the cut are still good data.""" + ends = [i for i, ch in enumerate(blob) if ch == "}"] + for end in reversed(ends[-60:]): + cand = blob[: end + 1] + stack = [] + in_str = esc = False + for ch in cand: + if esc: + esc = False + continue + if ch == "\\": + esc = True + elif ch == '"': + in_str = not in_str + elif not in_str and ch in "[{": + stack.append("]" if ch == "[" else "}") + elif not in_str and ch in "]}": + if stack and stack[-1] == ch: + stack.pop() + if in_str: + continue + try: + return json.loads(cand + "".join(reversed(stack))) + except json.JSONDecodeError: + continue + return None + + +def _agentic_system(taxonomy: TaxonomySpec) -> str: + why = "\n".join(f" - {k}: {v}" for k, v in taxonomy.why_classes.items()) + where = "\n".join(f" - {k}: {v}" for k, v in taxonomy.where_classes.items()) + return ( + f"{APPWORLD_BENCH_INTRO} You are the ANALYST for one evolution round: " + "investigate WHY the parent harness fails, agentically, then emit one " + "structured diagnosis.\n\n" + "Method (funnel — cheap first, deep second):\n" + "1) Read ledger_digest.md; group failing tasks by oracle signature.\n" + "2) Deep-read at least ONE representative transcript per signature group " + "(cross-reference turns: e.g. compare an argument the agent used against " + "the data it had retrieved — transcription slips, missed pages, wrong " + "tokens hide there).\n" + "3) Read the harness source under harness/ BEFORE naming a WHERE.\n" + "Coverage duty: EVERY failing task gets >=1 label; state in 'coverage' " + "which tasks you deep-read and which you labeled by signature analogy.\n\n" + f"WHY classes:\n{why}\n\nWHERE classes:\n{where}\n\n" + f"Rules: {APPWORLD_DIAGNOSIS_RULES} Mark EXACTLY ONE mode per trajectory " + '"dominant": true — the failure that directly explains the verdict.\n\n' + "Final message: ONLY this JSON object, no prose around it. GROUP tasks " + "sharing the same mode into one entry via trajectory_ids (keeps the " + "output compact — it must not truncate); keep reasoning/fix_hint terse:\n" + '{"assignments":[{"trajectory_ids":["id1","id2"],"why":"",' + '"where":"","dominant":true|false,"reasoning":"<=12 words",' + '"fix_hint":"<=12 words"}],"coverage":""}' + ) + + +def make_agentic_diagnose_fn( + *, + repo_root: str | Path, + runs_root: str | Path, + ws_root: str | Path, + exp_of: Callable[[HarnessNode], str], + work_dir: str | Path, + taxonomy: TaxonomySpec, + k: int = 3, + model: str = "claude-opus-4-8", + claude_bin: str = "claude", + timeout: float = 1800.0, + run: Optional[Callable] = None, +) -> Callable[[int, HarnessNode], dict]: + """Build the loop's ``diagnose_fn`` backed by one agentic session per round.""" + repo_root = Path(repo_root) + runs_root = Path(runs_root) + ws_root = Path(ws_root) + work_dir = Path(work_dir) + + def diagnose_fn(round_index: int, parent: HarnessNode) -> dict: + exp = exp_of(parent) + ws, failing = build_analysis_workspace(work_dir, runs_root, ws_root, exp, k) + if not failing: + return empty_failure_map() + + harness = ws / "harness" + made_worktree = False + if not harness.exists() and parent.git_commit_sha not in ("", "unknown"): + try: + git_ops.create_worktree(repo_root, harness, parent.git_commit_sha) + made_worktree = True + except Exception: # noqa: BLE001 — analysis degrades, doesn't die + pass + try: + user = ( + f"Round {round_index}, parent {parent.node_id}. " + f"{len(failing)} failing tasks need labels: {failing}\n" + "Start with ledger_digest.md. Produce the diagnosis JSON." + ) + raw = run_agentic_session( + user, + system_prompt=_agentic_system(taxonomy), + cwd=ws, + model=model, + claude_bin=claude_bin, + timeout=timeout, + run=run, + add_dirs=(runs_root, ws_root), + ) + finally: + if made_worktree: + try: + git_ops.remove_worktree(repo_root, harness, force=True) + except Exception: # noqa: BLE001 + pass + + (ws / "last_response.txt").write_text(raw) + s = strip_code_fence(raw) + i, j = s.find("{"), s.rfind("}") + if i < 0 or j <= i: + raise RuntimeError(f"agentic diagnosis returned no JSON: {raw[:300]}") + blob = s[i : j + 1] + try: + obj = json.loads(blob) + except json.JSONDecodeError: + # Two observed damage modes: a Python-style dict (single quotes), + # and a final message cut mid-array by the output length cap — + # salvage what was generated rather than discarding the session. + import ast + + try: + obj = ast.literal_eval(blob) + except (ValueError, SyntaxError): + obj = _salvage_truncated(blob) + if obj is None: + raise RuntimeError(f"agentic diagnosis unparseable (saved to last_response.txt): {blob[:200]}") + fm = empty_failure_map() + failing_set = set(failing) + seen: set[str] = set() + for a in obj.get("assignments", []): + if not isinstance(a, dict): + continue + tids = a.get("trajectory_ids") or ([a["trajectory_id"]] if a.get("trajectory_id") else []) + for tid in tids: + if tid not in failing_set: + continue + seen.add(tid) + add_failure_mode(fm, tid, coerce_mode(a, taxonomy)) + fm["_n_judged"] = len(seen) + fm["_coverage"] = str(obj.get("coverage", ""))[:500] + missed = failing_set - seen + if missed: + print(f"[agentic] {len(missed)} failing tasks unlabeled: {sorted(missed)[:5]}", flush=True) + return fm + + return diagnose_fn + + +__all__ = ["make_agentic_diagnose_fn", "build_analysis_workspace"] diff --git a/benchmarks/appworld/evolve/diagnose.py b/benchmarks/appworld/evolve/diagnose.py new file mode 100644 index 0000000..ccf45e5 --- /dev/null +++ b/benchmarks/appworld/evolve/diagnose.py @@ -0,0 +1,184 @@ +"""AppWorld failure diagnosis into a WHY/WHERE taxonomy (in-package). + +Two taxonomy sources, toggled by the caller (default = hardcoded): + +- **hardcoded** (default): the hand-derived 7 AppWorld WHY classes (W1-W7), derived once by + hand from real vanilla trajectories. Frozen constant :data:`DEFAULT_APPWORLD_TAXONOMY`. +- **induce**: the bench-neutral open-ended map-reduce in + :mod:`raven.evolver.orchestrator.nodes.taxonomy` discovers a taxonomy from + vanilla failures; :func:`ensure_taxonomy` here is the AppWorld-bound wrapper + (AppWorld bench description, W1-W7 as the hardcoded default). Induction + failure raises — it never silently substitutes the hardcoded table. + +Diagnosis is **multi-label**: one trajectory can exhibit several failure modes, +so it may land in several ``WHERE::WHY`` cells (each hit increments that WHY's +count). Output is a failure_map dict shaped exactly like +``failure_map_builder.build_failure_map`` so the orchestrator's +``select_target_whys`` / design step are unchanged: + + {"why_distribution": {why: count}, + "cells": {"::": {"candidates": [ + {"trajectory_id", "reasoning", "components": [{"summary": fix_hint}]}]}}, + "_n_judged": n} +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Callable, Optional + +from raven.evolver.orchestrator.nodes.taxonomy import ( + TaxonomySpec, + classify_failures, +) +from raven.evolver.orchestrator.nodes.taxonomy import ( + ensure_taxonomy as _generic_ensure_taxonomy, +) +from raven.evolver.orchestrator.nodes.taxonomy import ( + induce_taxonomy as _generic_induce_taxonomy, +) +from raven.evolver.tree.node import HarnessNode + +# The hand-derived 7 AppWorld WHY classes (verbatim) + one escape hatch. +WHY_CLASSES = { + "W1_empty_response_stall": "Agent emits an empty / no-tool-call turn and stops early — no real work done.", + "W2_no_finalize": "Agent does work but NEVER calls apis.supervisor.complete_task (or only says 'done' in prose) — nothing submitted.", + "W3_api_contract_misuse": "Agent misuses the API contract: wrong/missing access_token, blocked modules, guessing signatures/params by trial-and-error, login->token dance failing.", + "W4_temporal_grounding": "Agent treats 'today'/'this year'/'recent' as real-world time (e.g. 2026) instead of the environment's simulated date (~2023) -> wrong date filtering.", + "W5_premature_answer": "Agent answers/finalizes before gathering or computing enough — hallucinated value, or accepts a 0-result filtered query as the answer.", + "W6_action_state_mismatch": "Agent's actions leave the DB in the wrong state (edited wrong records / missed some / did extra) — the mutation doesn't match the required outcome. Often a capability/logic limit, not harness-fixable.", + "W7_borderline_flaky": "No stable pathology — the task flips pass/fail run-to-run; this is noise, not a fixable failure mode.", + "other": "None of the above — provide a short sub-name.", +} + +# AppWorld patch surface (WHERE a fix would go). +WHERE_CLASSES = { + "appworld_prompt": "benchmarks/appworld/agent_cli.py APPWORLD_PROMPT (the agent instruction text).", + "exec_tool": "benchmarks/appworld/tool.py AppWorldExecuteTool (execution / error-recovery behaviour).", + "agent_hook_new": "a NEW lifecycle hook raven/agent/hook/.py (runtime intervention: finalize/recovery/verify).", + "agent_hook_wire": "benchmarks/appworld/agent_cli.py hook wiring (construct the hook and pass it to AgentLoop, unconditionally).", + "agent_loop": "raven/agent/loop/ (loop-level change — use sparingly).", + "none": "capability ceiling — no harness fix can help (typical for W6/W7); do not target.", +} + +DEFAULT_APPWORLD_TAXONOMY = TaxonomySpec(dict(WHY_CLASSES), dict(WHERE_CLASSES)) + +APPWORLD_BENCH_DESC = ( + "an AppWorld agent harness (the agent writes Python calling " + "apis..(...) and must finish with apis.supervisor.complete_task)" +) + + +APPWORLD_BENCH_INTRO = ( + "You are a failure analyst for an AppWorld agent harness. AppWorld tasks: an agent writes " + "Python calling apis..(...) to fulfill a supervisor request and must finish with " + "apis.supervisor.complete_task(answer=...)." +) + +APPWORLD_DIAGNOSIS_RULES = ( + "If the agent clearly could not do the task correctly even " + "with more turns (wrong multi-step data logic), that is W6 with WHERE=none (capability ceiling). " + "If pass/fail looks random with no clear cause, use W7 / none. " + "The ATTEMPTS line (when present) summarizes ALL runs of this task: pass/fail flips across " + "attempts are the W7 signature; repeated failures on the SAME check indicate a stable pathology " + "(not W7)." +) + + +def diagnose_appworld( + call_fn: Callable[[list], str], + trajectories, + *, + taxonomy: TaxonomySpec = DEFAULT_APPWORLD_TAXONOMY, + max_workers: int = 8, + retries: int = 2, +) -> dict: + """Judge failing trajectories into a multi-label failure_map over ``taxonomy``. + + ``trajectories`` = list of ``(trajectory_id, task_description, transcript)``. + Each trajectory can contribute several modes; every hit increments its WHY. + Thin AppWorld binding over the bench-neutral :func:`classify_failures`. + """ + return classify_failures( + call_fn, + trajectories, + taxonomy, + bench_intro=APPWORLD_BENCH_INTRO, + extra_rules=APPWORLD_DIAGNOSIS_RULES, + max_workers=max_workers, + retries=retries, + ) + + +def induce_taxonomy( + call_fn: Callable[[list], str], + trajectories, + *, + max_workers: int = 8, + retries: int = 2, + target_min: int = 5, + target_max: int = 9, +) -> tuple[TaxonomySpec, dict]: + """AppWorld-bound wrapper over the generic induction (see nodes/taxonomy).""" + return _generic_induce_taxonomy( + call_fn, + trajectories, + bench_desc=APPWORLD_BENCH_DESC, + max_workers=max_workers, + retries=retries, + target_min=target_min, + target_max=target_max, + ) + + +def ensure_taxonomy( + call_fn: Callable[[list], str], + trajectories, + path: str | Path, + *, + mode: str = "hardcoded", + default: Optional[TaxonomySpec] = DEFAULT_APPWORLD_TAXONOMY, + max_workers: int = 8, + seed_path: Optional[str | Path] = None, +) -> TaxonomySpec: + """AppWorld-bound wrapper: W1-W7 as the hardcoded default, AppWorld bench + description for induction. Induction failure raises (no silent fallback).""" + return _generic_ensure_taxonomy( + call_fn, + trajectories, + path, + mode=mode, + default=default, + bench_desc=APPWORLD_BENCH_DESC, + max_workers=max_workers, + seed_path=seed_path, + ) + + +def make_appworld_diagnose_fn( + call_fn: Callable[[list], str], + trajectory_source: Callable[[int, HarnessNode], list], + *, + taxonomy: TaxonomySpec = DEFAULT_APPWORLD_TAXONOMY, + max_workers: int = 8, +) -> Callable[[int, HarnessNode], dict]: + """Bind the driver + a trajectory source into the loop's ``diagnose_fn``.""" + + def diagnose_fn(round_index: int, parent: HarnessNode) -> dict: + trajs = trajectory_source(round_index, parent) + return diagnose_appworld(call_fn, trajs, taxonomy=taxonomy, max_workers=max_workers) + + return diagnose_fn + + +__all__ = [ + "diagnose_appworld", + "make_appworld_diagnose_fn", + "TaxonomySpec", + "DEFAULT_APPWORLD_TAXONOMY", + "APPWORLD_BENCH_DESC", + "induce_taxonomy", + "ensure_taxonomy", + "WHY_CLASSES", + "WHERE_CLASSES", +] diff --git a/benchmarks/appworld/evolve/editor.py b/benchmarks/appworld/evolve/editor.py new file mode 100644 index 0000000..c7cf8d2 --- /dev/null +++ b/benchmarks/appworld/evolve/editor.py @@ -0,0 +1,658 @@ +"""Bash-editor candidate design: WHY selection + an agentic file-edit loop. + +The design step (SOP §2 ②) for AppWorld. Deterministic WHY selection off the +failure map (count x fixability x history-decay, so the budget flows to fixable +under-explored levers, not the ever-common W6/W7 capability ceilings), then a +per-candidate agentic loop where the driver runs ``bash`` in a :class:`Sandbox` +(a worktree at the parent commit) to edit the harness. Output is a +:class:`Candidate` (the edited file bytes vs the parent) that +``make_git_commit_apply_fn`` commits. +""" + +from __future__ import annotations + +import ast +import json +import re +from pathlib import Path +from typing import Callable, Optional + +from benchmarks.appworld.evolve.eval import Candidate +from benchmarks.appworld.evolve.sandbox import Sandbox +from raven.evolver.orchestrator.config import Budget +from raven.evolver.tree.node import HarnessNode + +# WHY fixability weight: the taxonomy itself flags W6/W7 as capability-ceiling / +# noise, yet they're usually the MOST common failures, so a raw-count WHY +# selection wastes every round on them. Down-weight so the budget flows to the +# fixable levers (W2/W3/W4/W5) once the obvious ones are explored. +WHY_FIX_WEIGHT = { + "W6_action_state_mismatch": 0.25, + "W7_borderline_flaky": 0.1, + "W1_empty_response_stall": 0.7, +} + +_BASH_SYSTEM = """You improve an agent harness to fix a diagnosed failure mode. You are in a git \ +worktree of the repo (your cwd). The harness that runs the benchmark is \ +benchmarks/appworld/agent_cli.py (it builds the agent prompt APPWORLD_PROMPT and \ +runs one AppWorld task through a minimal AgentLoop). You MAY edit or \ +create files ONLY under: benchmarks/appworld/ and raven/agent/ (prompts, \ +tools, the agent loop, and hooks under raven/agent/hook/) — EXCEPT \ +benchmarks/appworld/evolve/ (the evolution machinery that scores you: any edit \ +there is rejected at commit time). Anything you change outside the allowed trees is \ +reverted. Work ONLY inside your cwd: this worktree holds the exact code version \ +you are patching. Never cd elsewhere or touch other checkouts of this repo — they are \ +different versions and out of bounds. + +Work in a loop, ONE JSON object per message, no prose, no code fences: + inspect a real run: {"action":"read_trajectory","task_id":""} + run a shell command: {"action":"bash","command":""} (read files with cat/grep/sed; \ +fine for small sed edits) + write/replace a whole file: {"action":"write_file","path":"","content":""} (PREFERRED for multi-line edits and new files — no shell quoting to get wrong) + finish: {"action":"done","summary":""} + +You MUST make at least one concrete file edit before you say done — reading files only \ +is not enough; the point is to CHANGE the harness. Rules: make a SMALL, targeted change \ +that fixes the failure mode. + +ACTIVATION (critical): your change MUST take effect with NO special environment variable \ +set — the candidate's code is committed and run as-is, so any fix hidden behind \ +`if os.environ.get(...)` is DEAD CODE that scores identical to vanilla. Do NOT gate your \ +fix behind an env var / feature flag. If you add a hook under raven/agent/hook/, wire \ +it in UNCONDITIONALLY (agent_cli.py builds the AgentLoop with hooks=None — construct \ +your hook there and pass it in). A prompt-string change in APPWORLD_PROMPT is \ +unconditional by nature. Verify edits compile before done; keep vanilla behavior intact \ +except your fix. + +GENERALIZATION (critical): the TRAIN failures shown are only to understand the general \ +MECHANISM. The harness is scored on a HELD-OUT test set you never see, so a change that \ +only helps these specific train tasks is worthless. Do NOT hardcode any task id, expected \ +answer, or task-specific value, or special-case the inspected tasks. Make a PRINCIPLED, \ +general change that would help the whole distribution.""" + + +def _loose_json(raw: str) -> Optional[dict]: + s = raw.strip() + if s.startswith("```"): + s = s.split("```", 2)[1] + if s[:4].lower() == "json": + s = s[4:] + i, j = s.find("{"), s.rfind("}") + if i < 0 or j < 0: + return None + try: + return json.loads(s[i : j + 1]) + except json.JSONDecodeError: + return None + + +def why_all_tids(failure_map: dict, why: str) -> list[str]: + """All train task_ids diagnosed with this WHY -> the focused eval subset.""" + tids: list[str] = [] + for key, cell in (failure_map.get("cells") or {}).items(): + if key.split("::", 1)[-1] != why: + continue + for c in cell.get("candidates", []): + t = c.get("trajectory_id") + if t and t not in tids: + tids.append(t) + return tids + + +def why_brief(failure_map: dict, why: str) -> tuple[str, list[str]]: + """A concrete diagnosis brief for one WHY (WHERE-hints + judge reasoning + + fix-hints + evidence task_ids) — feeds the driver the diagnosis, not a label.""" + cells = failure_map.get("cells", {}) + lines: list[str] = [] + wheres: set[str] = set() + tids: list[str] = [] + for key, cell in cells.items(): + w, _, cw = key.partition("::") + if cw != why: + continue + wheres.add(w) + for c in cell.get("candidates", []): + tid = c.get("trajectory_id") + if tid: + tids.append(tid) + r = (c.get("reasoning") or "").strip()[:300] + if r: + lines.append(f"- [{tid}] {r}") + for comp in (c.get("components") or [])[:1]: + s = (comp.get("summary") or "").strip()[:160] + if s: + lines.append(f" judge fix-hint: {s}") + brief = ( + f"DIAGNOSIS\nfailure mode (WHY): {why}\n" + f"suggested levers (WHERE-hints): {sorted(wheres)}\n" + f"judge's per-trajectory reasoning (sample):\n" + "\n".join(lines[:8]) + ) + return brief, [t for t in tids if t][:8] + + +def _attempt_counts(hist: list[dict]) -> tuple[int, int, int]: + """(n_fail, n_inert, n_win) for one WHY's attempt history. + + Inert deaths (preflight: trigger never fired on any historical trajectory) + are split out of ``n_fail``: they indict the trigger's reachability, not + the WHY's fixability, so the decay must not punish the WHY as if the + mechanism had been tried and measured. + """ + n_inert = sum(1 for h in hist if h.get("outcome") == "pruned_inert") + n_fail = sum(1 for h in hist if h.get("promoted") is False) - n_inert + n_win = sum(1 for h in hist if h.get("promoted") is True) + return n_fail, n_inert, n_win + + +def rerank_whys( + failure_map: dict, + k: int, + history: dict[str, list[dict]], + *, + why_fix_weight: dict[str, float] = WHY_FIX_WEIGHT, +) -> list[str]: + """Top-k WHYs by count x fixability x history-decay (not raw count). + + Decay WHYs whose prior attempts were all pruned (0.55^n_fail), reset on a win, + so exhausted / capability-ceiling WHYs stop soaking the budget. Inert deaths + decay gently (0.85^n_inert): the mechanism was never exercised, but repeated + unreachable triggers still argue for spending the budget elsewhere. + """ + why_dist = failure_map.get("why_distribution", {}) + if not why_dist: + return [] + scored = [] + for why, count in why_dist.items(): + fw = why_fix_weight.get(why, 1.0) + n_fail, n_inert, n_win = _attempt_counts(history.get(why, [])) + penalty = (0.55**n_fail) * (0.85**n_inert) if n_win == 0 else 1.0 + scored.append((why, count * fw * penalty)) + scored.sort(key=lambda x: (-x[1], x[0])) + return [w for w, s in scored[:k] if s > 0] + + +def _strip_hints(brief: str) -> str: + """Drop the judge fix-hint lines from a diagnosis brief. + + On attempt >=1 the hints are withheld: attempt 0 already followed them, and + an anchor that strong makes "take a DISTINCT approach" a dead letter — the + driver must re-derive the mechanism from the trajectories instead.""" + return "\n".join(line for line in brief.splitlines() if "judge fix-hint:" not in line) + + +_WHY_SELECT_SYS = ( + "You allocate one evolution round's design budget across the diagnosed " + "failure modes of an agent harness. Each mode comes with its definition, " + "weighted diagnosis count, evidence task count, the judge's WHERE-hint " + "distribution (which harness surface a fix would target; WHERE=none means " + "the judge saw NO harness lever for that trajectory — a capability " + "ceiling), and prior attempt history. Pick the K modes MOST worth " + "attacking NOW: prefer modes with concrete WHERE-hints where one " + "principled mechanism could plausibly rescue many tasks; avoid modes " + "dominated by WHERE=none, run-to-run noise, or exhausted by repeated " + "failed attempts. A large count does NOT outrank fixability. Respond ONLY " + 'a JSON array of exactly K items: [{"why":"","reason":"<=1 line"}]' +) + + +def _why_where_hints(failure_map: dict, why: str) -> dict[str, int]: + """Per-WHY WHERE-hint counts from the ``::`` cells.""" + hints: dict[str, int] = {} + for key, cell in (failure_map.get("cells") or {}).items(): + w, _, cw = key.partition("::") + if cw == why: + hints[w] = hints.get(w, 0) + len(cell.get("candidates") or []) + return hints + + +def driver_select_whys( + call_fn: Callable[[list], str], + failure_map: dict, + k: int, + history: dict[str, list[dict]], + *, + why_defs: Optional[dict[str, str]] = None, +) -> tuple[list[str], dict[str, str]]: + """Let the driver pick the round's target WHYs (selection is a PROPOSAL, + the gates still judge every candidate). Returns ``([], {})`` on any parse + or validity failure — the caller falls back to the formula ranking. + + ``why_defs`` (taxonomy key -> definition text) grounds each mode; the + WHERE-hint distribution is computed from the map's cells so the driver + sees the judge's per-instance fixability evidence (WHERE=none share), + not just bucket sizes. + """ + dist = failure_map.get("why_distribution", {}) + if not dist: + return [], {} + lines = [] + for why, cnt in sorted(dist.items(), key=lambda x: (-x[1], x[0])): + _, tids = why_brief(failure_map, why) + n_fail, n_inert, n_win = _attempt_counts(history.get(why, [])) + hints = _why_where_hints(failure_map, why) + total = sum(hints.values()) + none_pct = round(100 * hints.get("none", 0) / total) if total else 0 + hint_str = ", ".join(f"{w}={c}" for w, c in sorted(hints.items(), key=lambda x: -x[1])) or "(no WHERE data)" + attempts = f"prior attempts: {n_fail} failed / {n_win} promoted" + if n_inert: + attempts += f" / {n_inert} never-fired" + lines.append( + f"- {why}: weighted_count={cnt:.1f}, evidence_tasks={len(tids)}, " + f"where_hints: {hint_str} (judged unfixable: {none_pct}%), " + f"{attempts}" + ) + d = (why_defs or {}).get(why, "").strip() + if d: + lines.append(f" def: {d[:200]}") + user = ( + f"K={k}\nDiagnosed failure modes this round:\n" + + "\n".join(lines) + + "\n\nPick the K modes to attack. JSON array only." + ) + try: + raw = call_fn( + [ + {"role": "system", "content": _WHY_SELECT_SYS}, + {"role": "user", "content": user}, + ] + ) + except Exception: # noqa: BLE001 — selection must never kill the round + return [], {} + s = raw.strip() + if "```" in s: + s = s.split("```")[1] if s.count("```") >= 2 else s + s = s.split("\n", 1)[-1] if s.lstrip().startswith(("json", "JSON")) else s + i, j = s.find("["), s.rfind("]") + if i < 0 or j <= i: + return [], {} + try: + arr = json.loads(s[i : j + 1]) + except json.JSONDecodeError: + return [], {} + whys, reasons = [], {} + for item in arr: + w = item.get("why") if isinstance(item, dict) else None + if w in dist and w not in whys: + whys.append(w) + reasons[w] = str(item.get("reason", ""))[:200] + return whys[:k], reasons + + +def _fmt_history(history: dict[str, list[dict]], why: str) -> str: + hist = history.get(why, []) + if not hist: + return "" + lines = [ + "PRIOR ATTEMPTS for this failure mode (LEARN — do NOT repeat a failed approach; if one " + "helped its focused subset but REGRESSED full-train, that lever OVER-TRIGGERS on healthy " + "tasks, so narrow it sharply or try a different lever; a pruned_inert attempt never " + "fired at all — its TRIGGER was unreachable, so redesign the trigger/reachability, " + "not the mechanism body):" + ] + for h in hist[-8:]: + flip = "" + if "rescued" in h: + ids = ",".join(h.get("regressed_ids", [])) + flip = f" (rescued {h['rescued']}, regressed {h['regressed']}" + (f": {ids}" if ids else "") + ")" + reason = f" ({h['reason']})" if h.get("reason") else "" + lines.append( + f"- {h['node_id']} [{','.join(h.get('files', []))}]: {h.get('summary', '')[:160]} " + f"-> {h['outcome']}{flip}{reason}" + ) + if h.get("harm"): + lines.append( + " HOW it broke a healthy task (that candidate's own run — " + "your fix must not reproduce this):\n | " + h["harm"][:450].replace("\n", "\n | ") + ) + return "\n".join(lines) + + +_BEACON_REQUIREMENT = ( + "\nINSTRUMENTATION (required for python edits): the new code path MUST call\n" + " from raven.evolver.activation.ledger import activation_beacon\n" + " activation_beacon('', '')\n" + "at the exact place your mechanism fires (INSIDE its trigger condition, NOT at " + "import/module level — a beacon that fires on every task carries no attribution " + "signal). It no-ops outside evaluation, so it never changes behavior. A python " + "CODE edit without a beacon is REJECTED; edits that only change prompt/string " + "constants (e.g. APPWORLD_PROMPT text) are exempt — do not add dead code just to " + "carry a beacon. Optionally end your done-summary with a line " + "'TRIGGER_REGEX: ' matching the trajectory text your mechanism " + "reacts to (used to pre-skip candidates whose trigger never occurs).\n" +) + +_TRIGGER_RE = re.compile(r"TRIGGER_REGEX:\s*(.+?)\s*$", re.M) + + +def _has_beacon(changed: dict[str, bytes]) -> bool: + return any(p.endswith(".py") and b"activation_beacon(" in b for p, b in changed.items()) + + +def _code_changed(old: Optional[bytes], new: bytes) -> bool: + """True when a .py edit changes executable structure, not just string + constants. AppWorld's agent prompt lives INSIDE agent_cli.py, so a + prompt-only candidate is a .py edit with an identical code shape — it has + no execution point to carry a beacon and must not be held to one (its + Gate-b attribution runs on trajectory presence, like .md/.yaml edits). + A new file or unparseable content counts as a code change (fail-closed; + the compile check screens unparseable content anyway).""" + if old is None: + return True + + def norm(src: bytes) -> str: + tree = ast.parse(src.decode()) + for n in ast.walk(tree): + if isinstance(n, ast.Constant) and isinstance(n.value, str): + n.value = "_" + return ast.dump(tree) + + try: + return norm(old) != norm(new) + except (SyntaxError, UnicodeDecodeError): + return True + + +def _parse_trigger_spec(summary: str) -> Optional[dict]: + """The driver's optional self-declared trigger predicate, or None. + + Malformed regexes are dropped (fail-open): a bad predicate must cost the + candidate nothing — preflight simply has no signal to prune on. + """ + m = _TRIGGER_RE.search(summary or "") + if not m: + return None + pattern = m.group(1).strip() + try: + re.compile(pattern) + except re.error: + return None + return {"kind": "trajectory_regex", "pattern": pattern} + + +def bash_edit_candidate( + call_fn: Callable[[list], str], + sandbox: Sandbox, + diagnosis: str, + ev_ids: list[str], + tag: str, + *, + render_failed: Optional[Callable[[str], str]] = None, + passing_ids: Optional[list[str]] = None, + max_turns: int = 22, + force_after_readonly: int = 4, + attempt: int = 0, + history_text: str = "", + guard_text: str = "", + import_smoke_module: Optional[str] = "benchmarks.appworld.agent_cli", + import_smoke_python: Optional[str] = None, + system_prompt: Optional[str] = None, + require_beacon: bool = False, +) -> tuple[Optional[dict[str, bytes]], list[str], str]: + """Run the agentic bash edit loop for one candidate in ``sandbox``. + + Returns ``(changed_files | None, deleted_paths, summary)``: changed_files + is the whitelist bytes the driver changed vs the parent commit, + deleted_paths the whitelist files it removed (both halves of the edit — + a rename is one change + one deletion). ``None`` changed_files = no edit / + compile failure / import-smoke failure -> skip this candidate. + + ``import_smoke_module`` is the SOP §2 ③ free prune: after the AST compile + check, import the harness entry module inside the sandbox worktree (with + ``import_smoke_python``, ideally the same interpreter the eval's batch uses). + AST parsing passes bad imports / module-level NameErrors that would turn the + candidate's whole eval into infra noise; the import smoke drops those + candidates before any eval budget is spent. ``None`` disables it. + + ``system_prompt`` overrides the bash-editor system prompt (default = the + AppWorld one); the editable-paths line in the user prompt always follows + ``sandbox.whitelist``, so another bench only supplies its prompt + whitelist. + """ + user = ( + f"{diagnosis}\n\n" + + (history_text + "\n\n" if history_text else "") + + (guard_text + "\n\n" if guard_text else "") + + f"Real failed task_ids you can inspect (read_trajectory): {ev_ids[:20]}\n" + + ( + f"PASSING task_ids for CONTRAST (read_trajectory works on these too): " + f"{passing_ids[:4]} — healthy runs your fix must NOT change.\n" + if passing_ids + else "" + ) + + f"Editable files: under {' and '.join(sandbox.whitelist)}.\n\n" + "PROTOCOL — do NOT skip straight to editing:\n" + "1) Read the DIAGNOSIS above.\n" + "2) Inspect 1-2 evidence trajectories (read_trajectory) to CONFIRM the concrete " + "mechanism — what exactly the agent does wrong and at which step.\n" + "3) Read the relevant harness code (agent_cli.py / the loop / hooks).\n" + "4) Then make ONE minimal edit targeting THAT mechanism, grounded in what you saw. " + "Over-broad prompt nagging regresses passing tasks, so be surgical.\n" + + ( + "5) BEFORE done: check your trigger condition against BOTH sides — it must fire " + "on the failing evidence and NOT on a passing trajectory (read one, or test the " + "condition with bash/python on its text). Report hits vs false-positives in your " + "done summary.\n" + if passing_ids + else "5) BEFORE done: sanity-check your trigger condition against the failing evidence " + "text (test it with bash/python) and say in your done summary when it fires.\n" + ) + + ( + f"\nThis is attempt #{attempt + 1} for this failure mode — take a DISTINCT approach " + "from earlier attempts: pick a DIFFERENT lever (prompt vs hook vs loop vs tool). " + "Judge fix-hints are withheld this attempt on purpose: derive the mechanism yourself " + "from the evidence trajectories.\n" + if attempt + else "" + ) + + (_BEACON_REQUIREMENT if require_beacon else "") + + "Begin." + ) + convo = [ + {"role": "system", "content": system_prompt or _BASH_SYSTEM}, + {"role": "user", "content": user}, + ] + _FORCE = ( + "\n\nSTOP READING — you have changed NOTHING so far. Your NEXT message MUST be a bash " + "action that WRITES an edit (not another read). Make the concrete edit now." + ) + readonly = 0 + last_summary = "" + + def _has_edit() -> bool: + return bool(sandbox.changed_whitelist() or sandbox.deleted_whitelist()) + + for _turn in range(max_turns): + turns_left = max_turns - _turn - 1 + try: + raw = call_fn(convo) + except Exception as exc: # noqa: BLE001 — a driver error skips this candidate + # Surface WHY in the summary slot: a silent empty return made a + # dead driver indistinguishable from "designed nothing" (an entire + # smoke round produced zero candidates with no trace, 2026-07-09). + return None, [], f"driver error: {exc}"[:300] + obj = _loose_json(raw) + act = (obj or {}).get("action") + if act == "done": + if _has_edit(): + last_summary = str((obj or {}).get("summary", ""))[:300] + break + fb = ( + "You have NOT edited any file — reading alone does not fix it. Run a bash " + "command that WRITES a change now, then reply done." + ) + elif act == "read_trajectory": + rendered = render_failed(str(obj.get("task_id", ""))) if render_failed else "(unavailable)" + fb = f"Trajectory:\n{rendered}\n\nContinue (bash) or done." + elif act == "bash": + fb = f"$ output:\n{sandbox.bash(str(obj.get('command', '')))}\n\nContinue (bash) or done." + elif act == "write_file": + fb = ( + sandbox.write_text(str(obj.get("path", "")), str(obj.get("content", ""))) + + "\n\nContinue (bash) or done." + ) + else: + fb = 'Respond with ONE JSON action: {"action":"bash"|"write_file"|"read_trajectory"|"done", ...}' + edited = _has_edit() + # Only bash read-only turns count toward the force: read_trajectory is + # evidence-gathering the PROTOCOL itself mandates (2-3 reads), so + # counting it punished drivers for following the protocol. + if edited: + readonly = 0 + elif act == "bash": + readonly += 1 + if not edited and (readonly >= force_after_readonly or turns_left <= 2): + fb += _FORCE + if edited and turns_left <= 2: + fb += ( + "\n\nAlmost out of turns — if your change is complete, reply " + '{"action":"done","summary":"..."} NOW (an unfinished loop loses your summary).' + ) + fb += f"\n[turns left: {turns_left}]" + convo += [{"role": "assistant", "content": raw}, {"role": "user", "content": fb}] + + sandbox.scope_restore() + changed = sandbox.changed_whitelist() + deleted = sandbox.deleted_whitelist() + if not changed and not deleted: + return None, [], "" + ok, err = sandbox.compile_check(changed) + if not ok: + return None, [], f"compile FAIL: {err}" + if import_smoke_module: + ok, err = sandbox.import_check(import_smoke_module, python_exe=import_smoke_python) + if not ok: + return None, [], f"import smoke FAIL: {err}" + if ( + require_beacon + and not _has_beacon(changed) + and any(p.endswith(".py") and _code_changed(sandbox.original(p), b) for p, b in changed.items()) + ): + # A code edit without a beacon is unmeasurable by Gate-b (its firing + # leaves no per-task record), so it is rejected before any eval spend. + # Prompt/config edits are exempt — .md/.yaml by extension, and .py + # edits that only touch string constants (the AppWorld prompt lives + # inside agent_cli.py); their attribution runs on trajectory presence. + return None, [], "missing activation_beacon in python edit" + default = f"edited {sorted(changed)}" + (f", deleted {deleted}" if deleted else "") + return changed or {}, deleted, (last_summary or default) + + +def make_bash_editor_design_fn( + call_fn: Callable[[list], str], + *, + repo_root: str | Path, + worktree_root: str | Path, + sha_of: Callable[[HarnessNode], str], + budget: Budget, + render_failed: Optional[Callable[[str], str]] = None, + render_failed_of: Optional[Callable[[HarnessNode], Callable[[str], str]]] = None, + passing_ids_of: Optional[Callable[[HarnessNode], list[str]]] = None, + history: Optional[dict[str, list[dict]]] = None, + archive_summary_of: Optional[Callable[[], str]] = None, + guard_text_of: Optional[Callable[[], str]] = None, + max_turns: int = 22, + force_after_readonly: int = 4, + import_smoke_module: Optional[str] = "benchmarks.appworld.agent_cli", + import_smoke_python: Optional[str] = None, + system_prompt: Optional[str] = None, + whitelist_prefixes: Optional[tuple[str, ...]] = None, + require_beacon: bool = False, + why_selection: str = "driver", + why_defs_of: Optional[Callable[[], Optional[dict[str, str]]]] = None, +) -> Callable[[int, dict, HarnessNode], list[Candidate]]: + """Build the loop's ``design_fn``: select WHYs, run the bash-editor per + candidate off the parent commit, return :class:`Candidate` objects. + + ``render_failed_of(parent)`` (preferred) binds the trajectory renderer to + the round's parent, so ``read_trajectory`` shows the CURRENT parent's + failures; ``render_failed`` is the parent-agnostic fallback. + ``archive_summary_of`` renders the GSME elite bank into the editor's + context so the driver neither re-invents an already-banked mechanism nor + proposes one already tried and rejected in another cell. + """ + repo_root = Path(repo_root) + worktree_root = Path(worktree_root) + history = history if history is not None else {} + + def design_fn(round_index: int, failure_map: dict, parent: HarnessNode) -> list[Candidate]: + base_sha = sha_of(parent) + rf = render_failed_of(parent) if render_failed_of else render_failed + p_ids = passing_ids_of(parent) if passing_ids_of else [] + arch_text = archive_summary_of() if archive_summary_of else "" + formula = rerank_whys(failure_map, budget.max_why_per_round, history) + why_reasons: dict[str, str] = {} + if why_selection == "driver": + whys, why_reasons = driver_select_whys( + call_fn, + failure_map, + budget.max_why_per_round, + history, + why_defs=why_defs_of() if why_defs_of else None, + ) + if not whys: + whys = formula + # Shadow log: driver choice vs formula choice, so a few rounds of + # data can arbitrate which selector earns the default. + print(f"[why-select] driver={whys} formula={formula} reasons={why_reasons}", flush=True) + else: + whys = formula + cands: list[Candidate] = [] + for why in whys: + brief, ev_ids = why_brief(failure_map, why) + if why_reasons.get(why): + brief += f"\nTARGET RATIONALE (why this mode was picked): {why_reasons[why]}" + focused = why_all_tids(failure_map, why) + for attempt in range(budget.candidates_per_why): + tag = f"r{round_index}-{why.split('_')[0]}-{attempt}" + sb = ( + Sandbox(repo_root, worktree_root / tag, base_sha, whitelist_prefixes=whitelist_prefixes) + if whitelist_prefixes is not None + else Sandbox(repo_root, worktree_root / tag, base_sha) + ) + try: + changed, deleted, summary = bash_edit_candidate( + call_fn, + sb, + brief if attempt == 0 else _strip_hints(brief), + ev_ids, + tag, + render_failed=rf, + passing_ids=p_ids, + max_turns=max_turns, + force_after_readonly=force_after_readonly, + attempt=attempt, + history_text="\n\n".join(s for s in (arch_text, _fmt_history(history, why)) if s), + guard_text=guard_text_of() if guard_text_of else "", + import_smoke_module=import_smoke_module, + import_smoke_python=import_smoke_python, + system_prompt=system_prompt, + require_beacon=require_beacon, + ) + finally: + sb.close() + if changed is not None: + cands.append( + Candidate( + files=changed, + why=why, + focused_task_ids=focused, + summary=summary, + deletions=deleted, + has_beacon=_has_beacon(changed), + activation_spec=_parse_trigger_spec(summary), + ) + ) + return cands + + return design_fn + + +__all__ = [ + "make_bash_editor_design_fn", + "bash_edit_candidate", + "driver_select_whys", + "rerank_whys", + "why_brief", + "why_all_tids", + "WHY_FIX_WEIGHT", +] diff --git a/benchmarks/appworld/evolve/entry.py b/benchmarks/appworld/evolve/entry.py new file mode 100644 index 0000000..5bf11f3 --- /dev/null +++ b/benchmarks/appworld/evolve/entry.py @@ -0,0 +1,340 @@ +"""AppWorld bench plugin for the unified launcher (built-in scorer line). + +``bench_config`` schema (YAML, under the run spec): + + bench_config: + config_path: /path/subject_runtime.json # required: agent runtime config + appworld_data_root: /path/appworld # AppWorld install (holds data/); + # exported as APPWORLD_ROOT + train_task_file: /path/train.txt # or train_task_ids: [...] + test_task_file: /path/test.txt # optional (enables sealed test) + n: 90 # optional cap on train tasks + conc: 8 + base_port: 8600 + python_exe: # default: current interpreter + vanilla_experiment: vanilla # cold-start ledger name + extra_args: ["--task-timeout", "400"] # passed through to batch.py + whitelist: ["raven/agent/", ...] # default: sandbox whitelist + min_confirm_lift: 0.0 + taxonomy_mode: hardcoded # or induce + why_selection: driver + analysis_mode: mapreduce # or agentic + agentic_model: claude-opus-4-8 + require_beacon: true + zero_hit_preflight: false + baseline_mode: frozen # or same_session (~2x eval cost, + # immune to endpoint drift) +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from benchmarks.appworld.evolve import adapter as aw_adapter +from raven.evolver.launch.contract import BenchBundle, LaunchContext, validate_whitelist +from raven.evolver.orchestrator.scoring import eval_with_infra_rerun + +_KNOWN_KEYS = { + "config_path", + "train_task_file", + "train_task_ids", + "test_task_file", + "test_task_ids", + "n", + "conc", + "base_port", + "python_exe", + "vanilla_experiment", + "extra_args", + "whitelist", + "min_confirm_lift", + "taxonomy_mode", + "taxonomy_path", + "why_selection", + "analysis_mode", + "agentic_model", + "require_beacon", + "zero_hit_preflight", + "appworld_data_root", + "precheck", + "precheck_min_tok_s", + "baseline_mode", +} + + +def _wait_ports_free(base: int, count: int, timeout: float = 20.0) -> None: + """Wait for this run's own env-server ports to free after a batch. + + ``batch.py`` terminates its servers with SIGTERM; a server can outlive the + batch by a few seconds. The Gate0 precheck (correctly) treats a bound port + as an orphan, so give our just-finished phase a grace window instead of + failing the round on our own shutdown race. + """ + import socket + import time + + def bound(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(("127.0.0.1", port)) == 0 + + deadline = time.time() + timeout + while time.time() < deadline: + if not any(bound(p) for p in range(base, base + count)): + return + time.sleep(1.0) + + +def _abs_against(base_dir: Path, value: str) -> Path: + # bench_config paths resolve like the top-level keys do: against the + # config file's directory, so a resume from another CWD reads the same + # inputs the run started with. + p = Path(value).expanduser() + return p if p.is_absolute() else (base_dir / p).resolve() + + +def _task_ids(bc: dict, prefix: str, base_dir: Path) -> list[str]: + ids = bc.get(f"{prefix}_task_ids") + if ids: + ids = [str(t) for t in ids] + else: + file_key = f"{prefix}_task_file" + if not bc.get(file_key): + return [] + path = _abs_against(base_dir, bc[file_key]) + if not path.is_file(): + raise ValueError(f"bench_config.{file_key}: not found: {path}") + ids = [line.strip() for line in path.read_text().splitlines() if line.strip()] + placeholders = [t for t in ids if "<" in t or ">" in t] + if placeholders: + raise ValueError( + f"bench_config {prefix} task ids contain placeholders " + f"{placeholders[:3]} — replace them with real AppWorld task ids " + "(see raven/evolver/README.md, Bootstrap)" + ) + return ids + + +def build(ctx: LaunchContext) -> BenchBundle: + spec = ctx.spec + bc = dict(spec.bench_config) + unknown = set(bc) - _KNOWN_KEYS + if unknown: + raise ValueError(f"bench_config: unknown keys {sorted(unknown)}") + if not bc.get("config_path"): + raise ValueError("bench_config.config_path is required (subject runtime config)") + config_path = _abs_against(spec.config_dir, bc["config_path"]) + if not config_path.is_file(): + raise ValueError( + f"bench_config.config_path not found: {config_path} — this is the " + "subject agent's runtime config JSON (model endpoint etc.; start " + "from docs/examples/subject_runtime.json)" + ) + # Validate against the runtime-config schema now, not at trial time: an + # empty or malformed subject config otherwise passes `check` and fails + # 270 trials deep into the cold start. + try: + import contextlib + import io + + from raven.config.loader import load_config as _load_subject_config + + with contextlib.redirect_stdout(io.StringIO()): + _load_subject_config(config_path) + except ValueError as exc: + raise ValueError( + f"bench_config.config_path {config_path} is not a valid Raven " + f"runtime config: {exc}\nStart from docs/examples/subject_runtime.json" + ) from exc + + # The batch scorer locates the AppWorld install/data via APPWORLD_ROOT; + # surface a missing install at build time (check catches it), not as a + # mid-run stack trace. Falls back to the APPWORLD_ROOT env var, then the + # batch scorer's default, but is validated unconditionally either way. + data_root = _abs_against( + spec.config_dir, + bc.get("appworld_data_root") or os.environ.get("APPWORLD_ROOT") or "~/workspace/appworld-run", + ) + if not (data_root / "data").is_dir(): + raise ValueError( + f"no AppWorld install found at {data_root} (no data/ under it) — " + "install AppWorld there or point bench_config.appworld_data_root " + "at your install (see raven/evolver/README.md, Bootstrap)" + ) + if not any((data_root / "data").iterdir()): + raise ValueError( + f"AppWorld data dir is empty: {data_root / 'data'} — the download " + "did not finish; run `appworld download data` in that install" + ) + appworld_bin = Path(os.environ.get("APPWORLD_BIN") or data_root / "appworld-venv/bin/appworld") + if not appworld_bin.is_file(): + raise ValueError( + f"appworld binary not found at {appworld_bin} — create the venv " + "per raven/evolver/README.md Bootstrap step 1, or set APPWORLD_BIN" + ) + os.environ["APPWORLD_ROOT"] = str(data_root) + + train_ids = _task_ids(bc, "train", spec.config_dir) + if not train_ids: + raise ValueError("bench_config: train_task_ids or train_task_file is required") + if bc.get("n") is not None: + n = int(bc["n"]) + if n <= 0: + raise ValueError(f"bench_config.n must be > 0, got {n}") + train_ids = train_ids[:n] + conc = int(bc.get("conc") or 8) + if conc <= 0: + raise ValueError(f"bench_config.conc must be > 0, got {conc}") + test_ids = _task_ids(bc, "test", spec.config_dir) + overlap = set(train_ids) & set(test_ids) + if overlap: + raise ValueError(f"train/test task sets overlap: {sorted(overlap)[:5]} …") + + from benchmarks.appworld.evolve.sandbox import WHITELIST_PREFIXES + + whitelist = tuple(bc.get("whitelist") or WHITELIST_PREFIXES) + validate_whitelist(spec.repo_root, spec.base_sha, whitelist) + + work = Path(spec.work_dir) + runs_root = work / "runs" + ws_root = work / "ws" + worktree_root = work / "wt" + van_exp = bc.get("vanilla_experiment", "vanilla") + vanilla_out_dir = runs_root / van_exp + k_confirm = spec.funnel.k_confirm + + cfg = aw_adapter.AppWorldConfig( + appworld_root=spec.repo_root, + python_exe=bc.get("python_exe") or sys.executable, + config_path=config_path, + out_dir_root=runs_root, + split="train", + n=len(train_ids), + conc=conc, + base_port=int(bc.get("base_port") or 8100), + workspace=ws_root, + extra_args=tuple(str(a) for a in bc.get("extra_args", ())), + ) + + def cold_start_done() -> int: + if not vanilla_out_dir.is_dir(): + return 0 + return sum(1 for tid in train_ids for k in range(k_confirm) if (vanilla_out_dir / f"{tid}_k{k}.json").is_file()) + + def make_precheck(): + if not bc.get("precheck", True): + return lambda: None + from benchmarks.appworld.evolve.precheck import make_appworld_precheck + + if bc.get("precheck_min_tok_s") is not None: + return make_appworld_precheck(cfg, min_tok_per_s=float(bc["precheck_min_tok_s"])) + return make_appworld_precheck(cfg) + + def run_cold_start() -> None: + # Fill missing trials, then the SOP infra-rerun ladder: a vanilla + # baseline scored with infra failures kept as zeros hands every + # candidate a free lift, so salvageable infra must be re-scored + # (into vanilla_infra_rerun{1,2}; the KEPT readers pick them up). + runs_root.mkdir(parents=True, exist_ok=True) + needs_fill = cold_start_done() < len(train_ids) * k_confirm + needs_salvage = False + if not needs_fill: + try: + kept = aw_adapter.read_kept_out_dir(vanilla_out_dir) + needs_salvage = any(ev.infra_attempts > 0 for ev in kept.values()) or len(kept) < len(train_ids) + except FileNotFoundError: + needs_fill = True + if needs_fill or needs_salvage: + # Gate0 before spending: neither the initial fill nor the + # infra-rerun ladder may burn trials against a dead endpoint + # (SOP §0). A clean, complete cold start skips the probe — + # rounds have their own per-round Gate0. + make_precheck()() + + def base_eval(_node, task_ids, k, job_name, *, split="train"): + return aw_adapter.run_eval(cfg, K=k, experiment=job_name, task_ids=task_ids) + + eval_with_infra_rerun(base_eval, None, train_ids, k_confirm, van_exp) + + def build_orchestrator(): + from benchmarks.appworld.evolve.run import build_appworld_orchestrator + + if cfg.base_port is not None: + _wait_ports_free(cfg.base_port, cfg.conc) + return build_appworld_orchestrator( + config=spec.funnel, + aw_cfg=cfg, + repo_root=spec.repo_root, + base_sha=spec.base_sha, + driver_call_fn=ctx.models.get("driver"), + design_call_fn=ctx.models.get("design") or ctx.models.get("driver"), + verdict_call_fn=ctx.models.get("verdict"), + vanilla_out_dir=vanilla_out_dir, + train_task_ids=train_ids, + test_task_ids=test_ids, + runs_root=runs_root, + ws_root=ws_root, + worktree_root=worktree_root, + min_confirm_lift=float(bc.get("min_confirm_lift", 0.0)), + taxonomy_mode=bc.get("taxonomy_mode", "hardcoded"), + taxonomy_path=bc.get("taxonomy_path"), + require_beacon=bool(bc.get("require_beacon", True)), + zero_hit_preflight=bool(bc.get("zero_hit_preflight", False)), + why_selection=bc.get("why_selection", "driver"), + analysis_mode=bc.get("analysis_mode", "mapreduce"), + agentic_model=bc.get("agentic_model", "claude-opus-4-8"), + whitelist_prefixes=whitelist, + precheck=make_precheck(), + baseline_mode=bc.get("baseline_mode", "frozen"), + ) + + from raven.evolver.tree.node import HarnessNode + + root_node = HarnessNode( + node_id="C0", + parent_id=None, + git_commit_sha=spec.base_sha, + git_branch="", + created_at=HarnessNode.utc_now(), + created_at_iter=0, + ) + + unseal = None + if test_ids: + + def unseal(records: list[dict], orch) -> dict: + import dataclasses + + from benchmarks.appworld.evolve.run import build_appworld_sealed_runner + from raven.evolver.orchestrator.sealed.runner import unseal_retention + + runner = build_appworld_sealed_runner( + aw_cfg=cfg, + repo_root=spec.repo_root, + test_task_ids=test_ids, + sealed_dir=spec.funnel.sealed_output_dir or work / "sealed", + k=k_confirm, + ) + report = unseal_retention( + runner, + records, + vanilla_node=root_node, + vanilla_train=orch.vanilla_train_mean, + ) + return dataclasses.asdict(report) if dataclasses.is_dataclass(report) else dict(report) + + return BenchBundle( + root_node_id="C0", + root_node=root_node, + journal_path=work / "journal" / "rounds.jsonl", + cold_start_total=len(train_ids) * k_confirm, + cold_start_done=cold_start_done, + run_cold_start=run_cold_start, + build_orchestrator=build_orchestrator, + unseal=unseal, + precheck=make_precheck() if bc.get("precheck", True) else None, + ) + + +__all__ = ["build"] diff --git a/benchmarks/appworld/evolve/eval.py b/benchmarks/appworld/evolve/eval.py new file mode 100644 index 0000000..f837e3e --- /dev/null +++ b/benchmarks/appworld/evolve/eval.py @@ -0,0 +1,68 @@ +"""AppWorld eval + candidate representation for the in-package evolution. + +A candidate is the driver's edited harness files (``Candidate.files`` = +``{repo-rel path: bytes}``); ``make_git_commit_apply_fn`` (with ``files_of``) +turns them into a real child commit. Eval then checks that commit out into an +ephemeral worktree and runs ``batch.py`` with ``cwd=worktree`` so the candidate's +committed harness is what imports — no writing into the live repo (the +zero-contamination replacement for RealPathSync). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from benchmarks.appworld.evolve import adapter as aw_adapter +from raven.evolver.tree import git_ops + + +@dataclass +class Candidate: + """One candidate harness edit produced by the bash-editor design step. + + ``has_beacon`` marks a code edit carrying an ``activation_beacon`` call — + only those get Gate-b per-task attribution (prompt/config edits have no + code execution point and fail open). ``activation_spec`` is the driver's + optional self-declared trigger predicate, consumed by the zero-hit + preflight. + """ + + files: dict[str, bytes] # full new bytes for each edited repo-rel path + why: str # the WHY this candidate targets + focused_task_ids: list[str] = field(default_factory=list) # WHY's evidence subset + summary: str = "" # the editor's one-line "what I changed" + deletions: list[str] = field(default_factory=list) # repo-rel paths removed + has_beacon: bool = False + activation_spec: dict | None = None + + +def files_of(cand: Candidate) -> dict[str, bytes]: + """Extract the edited file bytes for ``make_git_commit_apply_fn``.""" + return cand.files + + +def deletions_of(cand: Candidate) -> list[str]: + """Extract the deleted paths for ``make_git_commit_apply_fn``.""" + return cand.deletions + + +def make_appworld_eval_fn(aw: "aw_adapter.AppWorldConfig", repo_root: str | Path): + """Eval a node by checking its commit out into a worktree and running + ``batch.py`` there (``cwd=worktree``). No activation env, no live-repo writes. + """ + root = Path(repo_root) + + def eval_fn(node, task_ids, k, job_name, *, split="train"): + cfg = aw + if split != aw.split: + from dataclasses import replace + + cfg = replace(aw, split=split) + with git_ops.worktree_at(root, node.git_commit_sha) as wt: + return aw_adapter.run_eval(cfg, K=k, experiment=job_name, task_ids=task_ids, cwd=wt) + + return eval_fn + + +__all__ = ["Candidate", "files_of", "deletions_of", "make_appworld_eval_fn"] diff --git a/benchmarks/appworld/evolve/grade.py b/benchmarks/appworld/evolve/grade.py new file mode 100644 index 0000000..9078328 --- /dev/null +++ b/benchmarks/appworld/evolve/grade.py @@ -0,0 +1,128 @@ +"""Grading and result recording for AppWorld trials — the fixed scorer. + +This module is the measurement half of ``agent_cli.py``: the ``/evaluate`` +oracle call, the success/infra classification, and the result-file write. +It lives under ``evolve/`` deliberately — that prefix is in the evolver's +IMMUTABLE_PATTERNS, so a candidate may rewrite the agent surface +(prompt, loop wiring, tools) but never the code that grades it. Keep any +logic that decides *what counts as a pass or an infra failure* here, not +in the editable files. +""" + +from __future__ import annotations + +import json +import time +import traceback + +import requests + + +def post(env_url: str, path: str, body: dict, timeout: float = 120.0) -> dict: + r = requests.post(f"{env_url.rstrip('/')}{path}", json=body, timeout=timeout) + r.raise_for_status() + return r.json().get("output", {}) + + +# When the LLM endpoint drops/times-out/rate-limits, the agent loop catches the +# provider error and returns it as the final content (finish_reason=="error"), +# so it never raises up to the HTTP layer and would otherwise be graded as a +# normal wrong answer. That is infra, not the agent's fault -- detect it and +# raise so the run is recorded as infra_error and excluded from scoring. The +# signatures are short one-liners; the length guard avoids flagging a long +# genuine answer that merely mentions one of these words. +_LLM_TRANSPORT_SIGNS = ( + "error: connection error", + "sorry, i encountered an error calling the ai model", + "apiconnectionerror", + "apitimeouterror", + "api timeout error", + "ratelimiterror", + "rate limit", + "internal server error", + "service unavailable", + "502 bad gateway", + "503 service", + "bad gateway", + # litellm renders provider failures without the spaced/colon forms above + # (observed leaking through as INCOMPLETE: "Error calling LLM: + # litellm.InternalServerError: OpenAIException - Connection error.") + "error calling llm", + "connection error", + "internalservererror", +) + + +def is_llm_transport_error(resp: str | None) -> bool: + if not resp: + return False + s = resp.strip().lower() + if len(s) > 300: + return False + return any(sig in s for sig in _LLM_TRANSPORT_SIGNS) + + +def endpoint_dead(config_path: str) -> bool: + """True only when the subject endpoint is transport-unreachable. + + Disambiguates an EMPTY final response: a dead endpoint (DNS gone, + connection refused) yields empty completions on every task, which would + otherwise be graded INCOMPLETE — real-looking fails that poison whole + evals (observed: a mid-run endpoint death scored 270/270 INCOMPLETE). + But an empty response with a HEALTHY endpoint is the agent's own W1 + stall and must stay a legit fail, so only a transport-level probe + failure counts as dead; any HTTP status means alive. + """ + try: + cfg = json.load(open(config_path)) + defaults = cfg.get("agents", {}).get("defaults", {}) + base = (cfg.get("providers", {}).get(defaults.get("provider")) or {}).get("api_base") + if not base: + return False + requests.get(base.rstrip("/") + "/models", timeout=10) + return False + except requests.RequestException: + return True + except Exception: + return False + + +def grade_and_record(result: dict, *, env_url: str, task_id: str, response: str, config_path: str, t0: float) -> None: + """Score one finished attempt via the env oracle and fill ``result``. + + Raises ``RuntimeError`` on a transport-shaped final response so the + caller records the attempt as infra (Gate-f), not as a legit fail. + """ + if is_llm_transport_error(response): + raise RuntimeError(f"llm_transport_error: {response.strip()[:160]}") + if not response.strip() and endpoint_dead(config_path): + raise RuntimeError("llm_transport_error: empty response and subject endpoint unreachable") + ev = post(env_url, "/evaluate", {"task_id": task_id, "suppress_errors": True}) + done = post(env_url, "/task_completed", {"task_id": task_id}) + # Capture the full evaluation oracle (passes/failures) for the evolver's + # diagnose step; derive pass_count when the env omits it. + _passes = ev.get("passes") or [] + _failures = ev.get("failures") or [] + result.update( + success=bool(ev.get("success")), + num_tests=ev.get("num_tests"), + pass_count=ev.get("pass_count") if ev.get("pass_count") is not None else len(_passes), + evaluation={"passes": _passes, "failures": _failures}, + task_completed=bool(done) if not isinstance(done, dict) else done, + response=(response or "")[:2000], + elapsed_s=round(time.time() - t0, 1), + ) + + +def record_infra(result: dict, exc: BaseException, *, t0: float) -> None: + result.update( + success=False, + infra_error=f"{type(exc).__name__}: {exc}", + traceback=traceback.format_exc()[-2000:], + elapsed_s=round(time.time() - t0, 1), + ) + + +def write_result(out_path: str, result: dict) -> None: + with open(out_path, "w") as f: + json.dump(result, f, indent=2) diff --git a/benchmarks/appworld/evolve/precheck.py b/benchmarks/appworld/evolve/precheck.py new file mode 100644 index 0000000..228a3bc --- /dev/null +++ b/benchmarks/appworld/evolve/precheck.py @@ -0,0 +1,142 @@ +"""Gate0 pre-scoring environment health check for the AppWorld line (SOP §0). + +Wired as ``EvalBackend.precheck``, the loop runs it at the start of every round +BEFORE any scoring. A dirty environment makes every score of the round invalid +(the TB2 debian-apt / proxy-hijack lesson), so this raises with everything it +found wrong — fix the box, then resume — instead of letting a batch run against +a broken setup and record garbage. + +Checks, cheapest first: + +- the appworld install ``batch.py`` will spawn env servers from (venv binary + + data dir) is present; +- the subject config exists and names a provider endpoint; +- no orphan env servers hold the ports this run's batches will bind (the + classic killed-run leftover that 500s every task); +- the subject endpoint sustains a REAL generation (~300 tokens) at healthy + throughput. A tiny ping is not enough: a congested shared backend answers a + 16-token probe in seconds while real tasks blow the runner timeout (observed: + 3.5s ping alongside 68% of trials timing out at 900s), so the probe must + measure decode throughput, not reachability. +""" + +from __future__ import annotations + +import json +import socket +from pathlib import Path +from typing import Optional + +from benchmarks.appworld import batch as batch_mod +from benchmarks.appworld.evolve.adapter import AppWorldConfig +from raven.evolver.orchestrator.scoring import PrecheckFn + + +def _port_bound(port: int, timeout: float = 0.3) -> bool: + try: + with socket.create_connection(("127.0.0.1", port), timeout=timeout): + return True + except OSError: + return False + + +def _subject_endpoint(config_path: Path) -> tuple[Optional[str], Optional[str], Optional[str]]: + """``(api_base, model, problem)`` from the subject config; problem set on failure.""" + try: + cfg = json.loads(config_path.read_text()) + defaults = cfg.get("agents", {}).get("defaults", {}) + model = defaults.get("model") + provider = defaults.get("provider") + api_base = (cfg.get("providers", {}).get(provider) or {}).get("api_base") + except (OSError, ValueError, AttributeError) as e: + return None, None, f"subject config unreadable ({config_path}): {e}" + if not (api_base and model): + return None, None, f"subject config missing provider api_base/model: {config_path}" + return api_base, model, None + + +def _endpoint_problem(api_base: str, model: str, timeout: float, min_tok_per_s: float) -> Optional[str]: + import time + + import httpx # lazy, same as the driver transport + + url = api_base.rstrip("/") + "/chat/completions" + # Real-generation probe: ask for a ~300-token completion and judge decode + # throughput. On reasoning models the budget fills with thinking tokens, + # which is fine — usage.completion_tokens still measures decode speed. + body = { + "model": model, + "messages": [{"role": "user", "content": "Explain how TCP congestion control works, in about 300 words."}], + "max_tokens": 300, + "temperature": 0, + } + t0 = time.monotonic() + try: + resp = httpx.post(url, json=body, timeout=timeout) + except httpx.TimeoutException: + return f"subject endpoint degraded ({url}): no 300-token completion within {timeout:.0f}s" + except httpx.HTTPError as e: + return f"subject endpoint unreachable ({url}): {type(e).__name__}: {e}" + elapsed = max(time.monotonic() - t0, 1e-6) + if resp.status_code != 200: + return f"subject endpoint unhealthy ({url}): HTTP {resp.status_code}: {resp.text[:200]}" + try: + ntok = int(resp.json().get("usage", {}).get("completion_tokens") or 0) + except ValueError: + return f"subject endpoint unhealthy ({url}): non-JSON response: {resp.text[:200]}" + if ntok == 0: + return f"subject endpoint unhealthy ({url}): empty generation (0 completion tokens)" + tps = ntok / elapsed + if tps < min_tok_per_s: + return ( + f"subject endpoint degraded ({url}): {ntok} tokens in {elapsed:.1f}s " + f"= {tps:.1f} tok/s (< {min_tok_per_s:.1f} tok/s floor)" + ) + return None + + +def make_appworld_precheck( + aw: AppWorldConfig, + *, + check_endpoint: bool = True, + # 300 tokens in <25s (the SOP health bar) is 12 tok/s; the healthy band + # observed on the shared subject backend is 12-33 tok/s. + endpoint_timeout: float = 60.0, + min_tok_per_s: float = 12.0, +) -> PrecheckFn: + """Build the per-round Gate0 precheck for one AppWorld configuration.""" + + def precheck() -> None: + problems: list[str] = [] + if not Path(batch_mod.APPWORLD_BIN).exists(): + problems.append(f"appworld venv binary missing: {batch_mod.APPWORLD_BIN}") + data_dir = Path(batch_mod.APPWORLD_ROOT) / "data" + if not data_dir.exists(): + problems.append(f"appworld data dir missing: {data_dir}") + + base = aw.base_port if aw.base_port is not None else 8100 + busy = [p for p in range(base, base + aw.conc) if _port_bound(p)] + if busy: + problems.append( + f"env-server ports already bound (orphan servers from a killed run?): {busy}" + " — pkill -9 -f 'serve environment' and re-run" + ) + + if not aw.config_path.exists(): + problems.append(f"subject config missing: {aw.config_path}") + elif check_endpoint: + api_base, model, cfg_problem = _subject_endpoint(aw.config_path) + if cfg_problem: + problems.append(cfg_problem) + else: + ep_problem = _endpoint_problem(api_base, model, endpoint_timeout, min_tok_per_s) + if ep_problem: + problems.append(ep_problem) + + if problems: + raise RuntimeError("Gate0 precheck failed: " + " | ".join(problems)) + + return precheck + + +__all__ = ["make_appworld_precheck"] diff --git a/benchmarks/appworld/evolve/run.py b/benchmarks/appworld/evolve/run.py new file mode 100644 index 0000000..975d009 --- /dev/null +++ b/benchmarks/appworld/evolve/run.py @@ -0,0 +1,339 @@ +"""Entrypoint: assemble the in-package AppWorld evolution orchestrator. + +Everything generic (round loop, focused-Fisher gate, per-parent frozen baseline, +edit-then-commit apply, termination, journal/resume) comes from +``raven.evolver.orchestrator``; only the AppWorld brain is wired here: + +- diagnose_fn = W1-W7 judge over the parent's failing trajectories +- design_fn = bash-editor producing candidate file edits off the parent commit +- apply_fn = commit those edits as a real child commit (edit-then-commit) +- eval = check the candidate commit out into a worktree, run batch.py there + (cwd=worktree, zero live-repo mutation) +- baseline = per-parent frozen, seeded with the vanilla van0 out-dir; on + resume a missing parent's baseline is rebuilt from its confirm + out-dir on disk +- focused_source / outcome_hook = the WHY's evidence subset / cross-round history +- verdict_fn = the driver drafts each round's findings-log narrative + +The scorer subprocess + driver endpoints are external, so an end-to-end run is +validated only in the real AppWorld environment; this module is the wiring. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Callable, Optional + +from benchmarks.appworld.evolve import adapter as aw_adapter +from benchmarks.appworld.evolve.adapter import make_appworld_backend +from benchmarks.appworld.evolve.diagnose import ( + DEFAULT_APPWORLD_TAXONOMY, + make_appworld_diagnose_fn, +) +from benchmarks.appworld.evolve.editor import make_bash_editor_design_fn +from benchmarks.appworld.evolve.eval import ( + deletions_of, + files_of, + make_appworld_eval_fn, +) +from benchmarks.appworld.evolve.precheck import make_appworld_precheck +from benchmarks.appworld.evolve.trajectories import ( + build_failed_attempt_renderer, + build_out_dir_trajectory_source, + build_passing_ids_source, + render_candidate_failure, +) +from raven.evolver.orchestrator.config import Budget, OrchestratorConfig +from raven.evolver.orchestrator.gates.policy import make_frozen_baseline +from raven.evolver.orchestrator.gates.strategies import ( + FocusedFisherGate, + confirm_job_name, +) +from raven.evolver.orchestrator.loop import EvolutionOrchestrator +from raven.evolver.orchestrator.nodes.taxonomy import resolve_taxonomy +from raven.evolver.orchestrator.production import build_evolution_orchestrator +from raven.evolver.tree.node import HarnessNode + + +def build_appworld_orchestrator( + *, + config: OrchestratorConfig, + aw_cfg: "aw_adapter.AppWorldConfig", + repo_root: str | Path, + base_sha: str, + driver_call_fn: Callable[[list], str], + design_call_fn: Callable[[list], str], + verdict_call_fn: Optional[Callable[[list], str]] = None, + vanilla_out_dir: str | Path, + train_task_ids: list[str], + runs_root: str | Path, + ws_root: str | Path, + worktree_root: str | Path, + root_node_id: str = "C0", + test_task_ids: list[str] = (), + budget: Optional[Budget] = None, + min_confirm_lift: float = 0.0, + exp_of: Optional[Callable[[HarnessNode], str]] = None, + render_failed: Optional[Callable[[str], str]] = None, + taxonomy_mode: str = "hardcoded", + taxonomy_path: Optional[str | Path] = None, + precheck: Optional[Callable[[], None]] = None, + require_beacon: bool = True, + zero_hit_preflight: bool = False, + whitelist_prefixes: Optional[tuple[str, ...]] = None, + why_selection: str = "driver", + analysis_mode: str = "mapreduce", + agentic_model: str = "claude-opus-4-8", + baseline_mode: str = "frozen", +) -> EvolutionOrchestrator: + """Wire the full AppWorld evolution into one :class:`EvolutionOrchestrator`. + + ``taxonomy_mode`` selects the WHY/WHERE taxonomy: ``"hardcoded"`` (default, + the hand-derived W1-W7) or ``"induce"`` (discover it once from vanilla failures and cache + to ``taxonomy_path`` / ``work_dir/taxonomy.json``). + + Promotion is the SOP navigator condition (full-train mean beats the parent + baseline; the credited paired-2σ label is reported alongside on the + outcome). ``min_confirm_lift`` optionally demands a minimum lift on top. + + ``precheck`` is the per-round Gate0 env health check; default = + :func:`make_appworld_precheck` over ``aw_cfg`` (appworld install present, no + orphan env servers on the batch ports, subject endpoint answering). Pass + ``lambda: None`` to disable. + + ``zero_hit_preflight`` (default off) enables the SOP §2 ③ free prune: a + candidate whose self-declared TRIGGER_REGEX matches none of the parent's + failing trajectories is culled as ``pruned_inert`` before any eval spend. + Off by default because Gate-b already denies credit to never-fired + mechanisms; the preflight only saves budget, at a small false-prune risk. + """ + from dataclasses import replace as _dc_replace + + if analysis_mode not in ("mapreduce", "agentic"): + raise ValueError(f"unknown analysis_mode {analysis_mode!r}") + if analysis_mode == "agentic": + # Fail fast at build time: agentic analysis only runs on Claude models + # via a present, logged-in claude CLI — never mid-run, never on other + # drivers (their arms use mapreduce). + from raven.evolver.orchestrator.providers.claude_agentic import ( + require_claude_for_agentic, + ) + + require_claude_for_agentic(agentic_model) + + budget = budget or config.budget + vanilla_out_dir = Path(vanilla_out_dir) + runs_root = Path(runs_root) + ws_root = Path(ws_root) + if runs_root != aw_cfg.out_dir_root: + # Diagnosis reads a promoted parent's confirm out-dir under runs_root; + # the gate writes it under aw_cfg.out_dir_root. Split roots would make + # every round-2+ diagnosis silently come up empty. + raise ValueError(f"runs_root ({runs_root}) must equal aw_cfg.out_dir_root ({aw_cfg.out_dir_root})") + # Same contract on the session side: diagnosis looks for session jsonls + # under ws_root, so the batch runner must actually write them there. + if aw_cfg.workspace is None: + aw_cfg = _dc_replace(aw_cfg, workspace=ws_root) + elif Path(aw_cfg.workspace) != ws_root: + raise ValueError(f"ws_root ({ws_root}) must equal aw_cfg.workspace ({aw_cfg.workspace})") + + # ① diagnose (W1-W7) over the parent's baseline failing trajectories: the + # root diagnoses the vanilla out-dir (by ITS name); a promoted parent + # diagnoses its OWN confirm out-dir (confirm_job_name — the naming contract + # shared with the gate policies). + exp_of = exp_of or ( + lambda parent: vanilla_out_dir.name if parent.node_id == root_node_id else confirm_job_name(parent.node_id) + ) + trajectory_source = build_out_dir_trajectory_source( + runs_root=runs_root, ws_root=ws_root, exp_of=exp_of, k=config.k_confirm + ) + + # ⑤ eval: worktree checkout of the candidate commit, batch.py with cwd=worktree. + # vanilla_node lets cold_start run the vanilla ledger if it is missing (SOP §1). + backend = make_appworld_backend( + aw_cfg, + vanilla_out_dir=vanilla_out_dir, + train_task_ids=train_task_ids, + test_task_ids=list(test_task_ids), + eval_fn=make_appworld_eval_fn(aw_cfg, repo_root), + vanilla_node=HarnessNode( + node_id=root_node_id, + parent_id=None, + git_commit_sha=base_sha, + git_branch="evolver/orchestrator", + created_at=HarnessNode.utc_now(), + created_at_iter=0, + ), + cold_start_k=config.k_confirm, + precheck=precheck or make_appworld_precheck(aw_cfg), + ) + + # Captured by diagnose_of for the verdict's next_target constraint and the + # driver's WHY selection — with induction the keys/definitions only exist + # after the first diagnose resolves them. + taxonomy_keys: list[str] = [] + taxonomy_why_defs: dict[str, str] = {} + + def diagnose_of(vanilla_node): + taxonomy, seed = resolve_taxonomy( + driver_call_fn, + trajectory_source, + vanilla_node, + mode=taxonomy_mode, + work_dir=config.work_dir, + hardcoded=DEFAULT_APPWORLD_TAXONOMY, + taxonomy_path=taxonomy_path, + ) + taxonomy_keys[:] = list(taxonomy.why_classes) + taxonomy_why_defs.clear() + taxonomy_why_defs.update(taxonomy.why_classes) + if analysis_mode == "agentic": + from benchmarks.appworld.evolve.agentic import ( + make_agentic_diagnose_fn, + ) + + return ( + make_agentic_diagnose_fn( + repo_root=repo_root, + runs_root=runs_root, + ws_root=ws_root, + exp_of=exp_of, + work_dir=config.work_dir, + taxonomy=taxonomy, + k=config.k_confirm, + model=agentic_model, + ), + None, + ) + return ( + make_appworld_diagnose_fn(driver_call_fn, trajectory_source, taxonomy=taxonomy), + seed, + ) + + # ② design (bash-editor) off the parent commit. The designer's + # read_trajectory action renders the CURRENT parent's failing attempts by + # default; sha_of (owned by the assembler) resolves the parent commit. + def design_of(sha_of, history, archive_summary_of): + return make_bash_editor_design_fn( + design_call_fn, + repo_root=repo_root, + worktree_root=worktree_root, + sha_of=sha_of, + budget=budget, + history=history, + archive_summary_of=archive_summary_of, + require_beacon=require_beacon, + whitelist_prefixes=whitelist_prefixes, + import_smoke_python=aw_cfg.python_exe, + render_failed=render_failed, + render_failed_of=( + None + if render_failed is not None + else build_failed_attempt_renderer( + runs_root=runs_root, ws_root=ws_root, exp_of=exp_of, k=config.k_confirm + ) + ), + passing_ids_of=build_passing_ids_source( + runs_root=runs_root, ws_root=ws_root, exp_of=exp_of, k=config.k_confirm + ), + why_selection=why_selection, + why_defs_of=lambda: dict(taxonomy_why_defs) or None, + ) + + # baseline: default "frozen" = per-parent frozen, seeded with the vanilla + # ledger through the infra-rerun KEPT overlay (control sees the same + # salvage rule candidate evals get, SOP §0); resume fallback re-reads a + # parent's confirm out-dir. Frozen is the cost-bound choice and is + # cross-time-shift blind (see gates.policy); "same_session" re-measures + # the parent every round (~2x eval cost, drift-immune). + if baseline_mode not in ("frozen", "same_session"): + raise ValueError(f"baseline_mode must be 'frozen' or 'same_session', got {baseline_mode!r}") + + def baseline_of(): + if baseline_mode == "same_session": + from raven.evolver.orchestrator.gates.policy import ( + SameSessionPairedBaseline, + ) + + return SameSessionPairedBaseline(k=config.k_confirm) + return make_frozen_baseline( + root_node_id=root_node_id, + vanilla_dir=vanilla_out_dir, + kept_reader=aw_adapter.read_kept_out_dir, + confirm_dir_of=lambda p: runs_root / confirm_job_name(p.node_id), + train_task_ids=train_task_ids, + seed_label="van0", + ) + + # Gate-b read-back: which train tasks a beacon-carrying candidate actually + # fired on, unioned over its confirm out-dir + infra-rerun ladder siblings. + from raven.evolver.activation.ledger import read_fired_tasks + + def fired_source_of(node: HarnessNode, task_ids: list[str]): + dirs = aw_adapter.ladder_out_dirs(runs_root / confirm_job_name(node.node_id)) + return read_fired_tasks(dirs, task_ids) + + preflight_fn = None + if zero_hit_preflight: + from raven.evolver.orchestrator.production import make_zero_hit_preflight + + preflight_fn = make_zero_hit_preflight(trajectory_source) + + return build_evolution_orchestrator( + config, + repo_root=repo_root, + base_sha=base_sha, + root_node_id=root_node_id, + backend=backend, + gate_policy=FocusedFisherGate(k=config.k_confirm, min_confirm_lift=min_confirm_lift), + diagnose_of=diagnose_of, + design_of=design_of, + baseline_of=baseline_of, + files_of=files_of, + deletions_of=deletions_of, + driver_call_fn=driver_call_fn, + verdict_call_fn=verdict_call_fn, + verdict_why_keys_of=lambda: taxonomy_keys or None, + harm_excerpt_of=lambda node_id, tid: render_candidate_failure( + runs_root, ws_root, confirm_job_name(node_id), tid, k=config.k_confirm + ), + preflight_fn=preflight_fn, + fired_source_of=fired_source_of, + ) + + +def build_appworld_sealed_runner( + *, + aw_cfg: "aw_adapter.AppWorldConfig", + repo_root: str | Path, + test_task_ids: list[str], + sealed_dir: str | Path, + k: int = 3, + infra_max_reruns: int = 2, +): + """The C3 sealed test runner for AppWorld (approach B, post-hoc). + + Reuses the same worktree-checkout eval as the loop (a candidate commit is + checked out and ``batch.py`` runs against it), invoked with ``split="test"`` + and the infra rerun ladder, so test is scored exactly like train. Never + called during evolution — feed the journal records to + :func:`raven.evolver.orchestrator.sealed.runner.unseal_retention` after the + loop finishes. + """ + from raven.evolver.orchestrator.scoring import eval_with_infra_rerun + from raven.evolver.orchestrator.sealed.runner import SealedTestRunner + + raw = make_appworld_eval_fn(aw_cfg, repo_root) + + def sealed_eval(node, task_ids, k_, job_name, *, split="test"): + return eval_with_infra_rerun(raw, node, task_ids, k_, job_name, split=split, max_reruns=infra_max_reruns) + + return SealedTestRunner( + eval_fn=sealed_eval, + test_task_ids=list(test_task_ids), + sealed_dir=Path(sealed_dir), + k=k, + ) + + +__all__ = ["build_appworld_orchestrator", "build_appworld_sealed_runner"] diff --git a/benchmarks/appworld/evolve/sandbox.py b/benchmarks/appworld/evolve/sandbox.py new file mode 100644 index 0000000..1f5310e --- /dev/null +++ b/benchmarks/appworld/evolve/sandbox.py @@ -0,0 +1,205 @@ +"""Isolated git-worktree sandbox the driver bash-edits (edit-then-commit design). + +The evolution model (SOP §3.1) is "the model edits code freely; robustness via +snapshot/restore + versioning", not diff-application — qwen is unreliable at +writing applyable unified diffs. So a candidate is produced by letting the driver +run ``bash`` in a detached worktree checked out at the PARENT commit, editing any +whitelisted file; we then capture exactly what it changed vs the parent as +``{repo-rel: bytes}``. Those bytes go to ``make_git_commit_apply_fn`` which +commits them onto the parent — giving a real child commit (no RealPathSync, no +live-repo writes; eval checks the commit out into its own worktree). +""" + +from __future__ import annotations + +import ast +import subprocess +from pathlib import Path +from typing import Optional + +from raven.evolver.tree import git_ops + +# The harness surface the driver may edit; anything else it touches is reverted. +WHITELIST_PREFIXES = ( + "benchmarks/appworld/", + "raven/agent/", +) +# Build artifacts only — every OTHER whitelist file is captured regardless of +# suffix, because a suffix filter silently halves candidates whose fix includes +# a .toml/.cfg/extensionless file: the driver verified the full change in the +# sandbox but the commit (and thus the eval) carried only part of it. +_IGNORED_SUFFIXES = (".pyc", ".pyo") +_IGNORED_DIR_PARTS = ("__pycache__",) + + +def _in_whitelist(rel: str) -> bool: + return any(rel.startswith(p) for p in WHITELIST_PREFIXES) + + +def _capturable(rel_parts: tuple[str, ...], suffix: str) -> bool: + if suffix in _IGNORED_SUFFIXES: + return False + return not any(part in _IGNORED_DIR_PARTS for part in rel_parts) + + +class Sandbox: + """A detached worktree at ``base_sha`` the driver bash-edits; captures the + whitelist files it changed vs that base.""" + + def __init__( + self, + repo_root: str | Path, + worktree_path: str | Path, + base_sha: str, + *, + whitelist_prefixes: tuple[str, ...] = WHITELIST_PREFIXES, + ): + self.repo_root = Path(repo_root) + self.root = Path(worktree_path) + self.whitelist = whitelist_prefixes + git_ops.remove_worktree(self.repo_root, self.root, force=True) if self.root.exists() else None + git_ops.create_worktree(self.repo_root, self.root, base_sha) + self._orig = self._snapshot() + + def _iter_whitelist(self): + for prefix in self.whitelist: + d = self.root / prefix + if d.exists(): + for p in d.rglob("*"): + if p.is_file(): + rel = p.relative_to(self.root) + if _capturable(rel.parts, p.suffix): + yield rel.as_posix(), p + + def _snapshot(self) -> dict[str, bytes]: + return {rel: p.read_bytes() for rel, p in self._iter_whitelist()} + + def bash(self, command: str, timeout: float = 60.0) -> str: + # The candidate is the diff of THIS worktree vs base; a command that + # names the origin repo escapes the experiment — reads see the wrong + # version (the worktree is pinned at the parent commit), writes poison + # every later measurement AND are never captured into the candidate. + # The path is discoverable in-sandbox (the worktree's .git file names + # the origin gitdir), so an in-context prohibition alone is not enough. + if str(self.repo_root.resolve()) in command: + return ( + f"[refused: command references the origin repo " + f"({self.repo_root}). Work ONLY inside your cwd (this worktree " + "holds the exact code version you are patching); the origin is " + "a different checkout and out of bounds.]" + ) + try: + r = subprocess.run( + ["bash", "-c", command], + cwd=str(self.root), + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return ( + f"[TIMEOUT after {timeout:.0f}s — command too slow, aborted. Do NOT scan the " + "whole filesystem (no `find /`); you are at the repo root, use relative paths " + "like benchmarks/appworld/agent_cli.py.]" + ) + except Exception as e: # noqa: BLE001 — a bad command must not crash the run + return f"[bash error: {type(e).__name__}: {str(e)[:150]}]" + out = (r.stdout or "")[-4000:] + err = (r.stderr or "")[-1500:] + return f"[exit {r.returncode}]\n{out}" + (f"\n[stderr]\n{err}" if err else "") + + def write_text(self, rel: str, content: str) -> str: + """Write a whole file (the editor's ``write_file`` action) — bypasses + shell quoting entirely. Out-of-whitelist paths are refused up front + (``scope_restore`` would revert them anyway; failing fast saves turns).""" + rel = rel.lstrip("/") + if ".." in Path(rel).parts: + return f"[refused: path escapes the worktree: {rel}]" + if not any(rel.startswith(p) for p in self.whitelist): + return ( + f"[refused: {rel} is outside the editable whitelist " + f"({', '.join(self.whitelist)}) — it would be reverted]" + ) + p = self.root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + return f"[wrote {rel} ({len(content)} chars)]" + + def scope_restore(self) -> list[str]: + """Revert any edit OUTSIDE the whitelist (the worktree is a full checkout).""" + r = subprocess.run( + ["git", "status", "--porcelain", "-uall"], + cwd=str(self.root), + capture_output=True, + text=True, + ) + reverted = [] + for line in r.stdout.splitlines(): + rel = line[3:].strip() + if rel and not any(rel.startswith(p) for p in self.whitelist): + subprocess.run(["git", "checkout", "--", rel], cwd=str(self.root), capture_output=True) + subprocess.run(["git", "clean", "-fdq", "--", rel], cwd=str(self.root), capture_output=True) + reverted.append(rel) + return reverted + + def original(self, rel: str) -> Optional[bytes]: + """The base-commit bytes of one whitelist file (None = didn't exist).""" + return self._orig.get(rel) + + def changed_whitelist(self) -> dict[str, bytes]: + """Whitelist files the driver changed vs the base — this candidate's edits.""" + now = self._snapshot() + return {rel: data for rel, data in now.items() if self._orig.get(rel) != data} + + def deleted_whitelist(self) -> list[str]: + """Whitelist files the driver deleted vs the base (a rename shows up as + one changed file + one deletion; without this the commit keeps the old + file and the candidate diverges from what the driver verified).""" + now = self._snapshot() + return sorted(rel for rel in self._orig if rel not in now) + + def compile_check(self, files: dict[str, bytes]) -> tuple[bool, str]: + for rel, data in files.items(): + if rel.endswith(".py"): + try: + ast.parse(data.decode("utf-8", "replace")) + except SyntaxError as e: + return False, f"{rel}: {e}" + return True, "" + + def import_check( + self, + module: str, + *, + python_exe: str | None = None, + timeout: float = 120.0, + ) -> tuple[bool, str]: + """Import ``module`` in a subprocess with cwd = this worktree, so the + candidate's edited tree is what resolves — the same cwd-first import the + eval's batch subprocess relies on. Catches what ``compile_check``'s AST + parse cannot (bad imports, module-level NameError, a deleted file + breaking the package) before a full eval is burned on a dead candidate.""" + import sys + + exe = python_exe or sys.executable + try: + r = subprocess.run( + [exe, "-c", f"import {module}"], + cwd=str(self.root), + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return False, f"import {module} timed out after {timeout:.0f}s" + except OSError as e: + return False, f"import check could not run ({exe}): {e}" + if r.returncode != 0: + return False, (r.stderr or r.stdout or "")[-800:] + return True, "" + + def close(self): + git_ops.remove_worktree(self.repo_root, self.root, force=True) + + +__all__ = ["Sandbox", "WHITELIST_PREFIXES"] diff --git a/benchmarks/appworld/evolve/trajectories.py b/benchmarks/appworld/evolve/trajectories.py new file mode 100644 index 0000000..2d6f0de --- /dev/null +++ b/benchmarks/appworld/evolve/trajectories.py @@ -0,0 +1,375 @@ +"""AppWorld trajectory rendering + the diagnosis trajectory source. + +``render_trajectory`` turns a real session.jsonl + result.json into transcript +text for the judge (task prompt, each assistant turn's narrative + Python, tool +stdout, final response, and the benchmark's own verdict/oracle) — no operator +interpretation added. ``build_out_dir_trajectory_source`` wraps it into the +loop's trajectory source: for a parent's baseline out-dir, pick each failing +task's first non-infra attempt and render it (infra attempts are skipped — SOP +§0: infra failure != agent failure, don't diagnose it). + +The infra rerun ladder (SOP §0) re-scores infra-contaminated tasks into sibling +``_infra_rerun{i}`` out-dirs, and the measurement with the fewest infra +trials wins (see ``scoring.eval_with_infra_rerun``). Diagnosis mirrors that +choice here: for each task the dir that provided the kept measurement is the +one whose failing attempt gets rendered — a task salvaged by a rerun is +diagnosed from its rerun trajectory, not invisible because the base dir only +holds its infra trials. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Callable, Optional + +from benchmarks.appworld.evolve.adapter import read_out_dir +from raven.evolver.orchestrator.scoring import TaskEval +from raven.evolver.tree.node import HarnessNode + + +def _tool_code(assistant_msg: dict) -> str: + out = [] + for tc in assistant_msg.get("tool_calls") or []: + fn = tc.get("function", {}) + args = fn.get("arguments") + if isinstance(args, str): + try: + args = json.loads(args) + except Exception: + pass + code = args.get("code") if isinstance(args, dict) else args + out.append(str(code)) + return "\n".join(out) + + +def _instruction_of(first_user_msg: str) -> str: + """The real task instruction from AppWorld's first user message. + + That message is the full APPWORLD_PROMPT: ~1.3k chars of static boilerplate + (API rules, formatting) with the actual instruction appended after the + ``"Your task:"`` marker. A prefix cap here used to keep the boilerplate and + cut the instruction (review round-2 P1-12); slicing at the bench's own + marker is the fix — trimming is bench knowledge, so it lives here, and the + generic layers apply no cap at all. + """ + marker = "Your task:" + i = first_user_msg.find(marker) + return first_user_msg[i + len(marker) :].strip() if i >= 0 else first_user_msg + + +def _clip(s: str, cap: int) -> str: + """Head+tail clip: keep both ends of an over-long block. + + A head-only cap amputates exactly where the evidence usually is — a long + stdout ends with the traceback's last frame / the assertion message — so + over-cap text keeps ~60% head + ~40% tail around a snip marker. + """ + if len(s) <= cap: + return s + head = int(cap * 0.6) + tail = cap - head + return s[:head] + " …[snip]… " + s[-tail:] + + +def render_trajectory(session_path: str | Path, result: dict, *, cap: int = 600) -> tuple[str, str]: + """Return ``(task_description, transcript_text)`` from a real session + result.""" + lines = [ln for ln in Path(session_path).read_text().splitlines() if ln.strip()] + msgs = [json.loads(ln) for ln in lines if json.loads(ln).get("_type") != "metadata"] + task_desc = "" + turns = [] + for m in msgs: + role = m.get("role") + content = m.get("content") + if role == "user" and not task_desc: + task_desc = _instruction_of(content or "") + continue + if role == "assistant": + narr = (m.get("content") or "").strip() + code = _tool_code(m).strip() + block = "" + if narr: + block += f"[assistant] {_clip(narr, cap)}\n" + if code: + block += f"[ran python]\n{_clip(code, cap)}\n" + if block: + turns.append(block) + elif role == "tool": + turns.append(f"[stdout] {_clip(content or '', cap)}") + ev = result.get("evaluation") or {} + passes = ev.get("passes") or [] + failures = ev.get("failures") or [] + verdict = ( + f"[benchmark verdict] success={result.get('success')} " + f"tests_passed={len(passes)}/{result.get('num_tests')} " + f"final_response={_clip(result.get('response') or '', cap)!r}" + ) + oracle = "" + if failures: + oracle = "\n[benchmark oracle] failed checks: " + _clip(json.dumps(failures), 2000) + return task_desc, "\n".join(turns) + "\n" + verdict + oracle + + +def default_sess_path(runs_root: Path, ws_root: Path, tid: str, exp: str, k: int) -> Path | None: + """Locate a session.jsonl for one attempt (per-attempt isolated workspace, + with the shared-sessions dir as a legacy fallback). Returns None if absent.""" + p = ws_root / f"att/{tid}_{exp}_k{k}/sessions/{tid}_{exp}_k{k}.jsonl" + if p.exists(): + return p + legacy = ws_root / "sessions" / f"{tid}_{exp}_k{k}.jsonl" + if legacy.exists(): + return legacy + # Raven SessionManager layout: conversation key "appworld:__k" + # persists to sessions//.jsonl. + raven = ws_root / "sessions" / "appworld" / f"{tid}_{exp}_k{k}.jsonl" + return raven if raven.exists() else None + + +def _ladder_exps(exp: str) -> list[str]: + """The base experiment plus its SOP §0 infra-rerun ladder dirs, in the + order ``eval_with_infra_rerun`` produced them.""" + return [exp, f"{exp}_infra_rerun1", f"{exp}_infra_rerun2"] + + +def _ladder_evals(runs_root: Path, exp: str) -> list[tuple[str, dict[str, TaskEval]]]: + """``(experiment, evals)`` for every existing dir in the rerun ladder.""" + out = [] + for name in _ladder_exps(exp): + d = runs_root / name + if d.exists() and list(d.glob("*_k*.json")): + out.append((name, read_out_dir(d))) + return out + + +def _kept_measurement( + ladder: list[tuple[str, dict[str, TaskEval]]], +) -> dict[str, tuple[str, TaskEval]]: + """Per task, the (experiment, eval) whose measurement was kept — fewest + infra trials wins, earlier dir wins ties (mirrors ``eval_with_infra_rerun``).""" + chosen: dict[str, tuple[str, TaskEval]] = {} + for name, evals in ladder: + for tid, ev in evals.items(): + cur = chosen.get(tid) + if cur is None or ev.infra_attempts < cur[1].infra_attempts: + chosen[tid] = (name, ev) + return chosen + + +def _failing_attempt(runs_root: Path, ws_root: Path, exp: str, tid: str, k: int) -> Optional[tuple[Path, dict]]: + """First non-infra FAILING attempt of ``tid`` in ``exp``'s out-dir, as + ``(session_path, result)``; None when the task has none there.""" + bdir = runs_root / exp + for kk in range(k): + rp = bdir / f"{tid}_k{kk}.json" + sp = default_sess_path(runs_root, ws_root, tid, exp, kk) + if sp is None or not rp.exists(): + continue + r = json.load(open(rp)) + if r.get("infra_error"): # infra != agent failure — don't diagnose + continue + if r.get("success"): # want a FAILING attempt (borderline tasks have both) + continue + return sp, r + return None + + +def _attempts_line(runs_root: Path, exp: str, tid: str, k: int) -> str: + """One-line per-attempt outcome summary prepended to the diagnosed + transcript. Cross-attempt patterns are diagnostic signal the single + rendered attempt can't carry: pass/fail flips are the noise-class + signature, same-check repeat failures indicate a stable pathology.""" + parts = [] + for kk in range(k): + rp = runs_root / exp / f"{tid}_k{kk}.json" + if not rp.exists(): + continue + r = json.load(open(rp)) + if r.get("infra_error"): + parts.append(f"k{kk}=INFRA") + elif r.get("success"): + parts.append(f"k{kk}=PASS") + else: + failures = (r.get("evaluation") or {}).get("failures") or [] + req = str((failures[0] or {}).get("requirement", ""))[:60] if failures else "" + parts.append(f'k{kk}=FAIL("{req}")' if req else f"k{kk}=FAIL") + return f"ATTEMPTS (k={k}): " + ", ".join(parts) if parts else "" + + +def _passing_attempt(runs_root: Path, ws_root: Path, exp: str, tid: str, k: int) -> Optional[tuple[Path, dict]]: + """First non-infra PASSING attempt of ``tid`` in ``exp``'s out-dir — the + contrast material for the design step's over-trigger self-check.""" + bdir = runs_root / exp + for kk in range(k): + rp = bdir / f"{tid}_k{kk}.json" + sp = default_sess_path(runs_root, ws_root, tid, exp, kk) + if sp is None or not rp.exists(): + continue + r = json.load(open(rp)) + if r.get("infra_error") or not r.get("success"): + continue + return sp, r + return None + + +def build_out_dir_trajectory_source( + *, + runs_root: str | Path, + ws_root: str | Path, + exp_of: Callable[[HarnessNode], str], + k: int = 3, + cap: int = 600, +) -> Callable[[int, HarnessNode], list]: + """Build the loop's trajectory source over a parent's baseline out-dir. + + ``exp_of(parent)`` returns the experiment name whose out-dir + sessions hold + that parent's baseline run (e.g. ``"van0"`` for the root). Diagnoses every + task the baseline does NOT fully pass — both all-fail (passes==0) and + **borderline** (0 < passes < attempts, the most fixable) — by rendering its + first non-infra **failing** attempt (SOP §2 step 2: read every failing task, borderline and all-fail), + read from whichever infra-rerun ladder dir provided the kept measurement. + """ + runs_root = Path(runs_root) + ws_root = Path(ws_root) + + def trajectory_source(round_index: int, parent: HarnessNode) -> list: + exp = exp_of(parent) + ladder = _ladder_evals(runs_root, exp) + if not ladder: + return [] + chosen = _kept_measurement(ladder) + trajs = [] + n_failing = 0 + for tid in sorted(chosen): + src_exp, ev = chosen[tid] + if ev.passes >= ev.attempts: # fully passing -> not a failure, skip + continue + n_failing += 1 + picked = _failing_attempt(runs_root, ws_root, src_exp, tid, k) + if picked is None and src_exp != exp: + picked = _failing_attempt(runs_root, ws_root, exp, tid, k) + if picked is None: + continue + desc, transcript = render_trajectory(picked[0], picked[1], cap=cap) + attempts = _attempts_line(runs_root, src_exp, tid, k) + if attempts: + transcript = f"{attempts}\n\n{transcript}" + trajs.append((tid, desc, transcript)) + if n_failing and not trajs: + # Results exist but not one failing trajectory could be rendered: + # the session transcripts are gone (e.g. a results-only cold start + # where runs/ was copied but ws/sessions/ was not). Diagnosis would + # otherwise silently judge 0 failures and every round spins empty. + raise RuntimeError( + f"diagnosis found {n_failing} failing task(s) in '{exp}' but " + f"could render 0 trajectories — the session transcripts under " + f"{ws_root}/ are missing. Re-run cold start so trajectories are " + "written (or restore them); results alone cannot be diagnosed." + ) + return trajs + + return trajectory_source + + +def build_failed_attempt_renderer( + *, + runs_root: str | Path, + ws_root: str | Path, + exp_of: Callable[[HarnessNode], str], + k: int = 3, + cap: int = 600, +) -> Callable[[HarnessNode], Callable[[str], str]]: + """Per-parent renderer of one failing attempt — the design step's + ``read_trajectory`` backend. ``renderer_of(parent)`` binds the parent's + out-dir (rerun-ladder aware); the returned callable renders ``task_id``'s + first non-infra failing attempt as evidence text.""" + runs_root = Path(runs_root) + ws_root = Path(ws_root) + + def renderer_of(parent: HarnessNode) -> Callable[[str], str]: + exp = exp_of(parent) + + def render(task_id: str) -> str: + for name in _ladder_exps(exp): + picked = _failing_attempt(runs_root, ws_root, name, task_id, k) + if picked is not None: + desc, transcript = render_trajectory(picked[0], picked[1], cap=cap) + return f"TASK: {desc}\n\n{transcript}" + # No failing attempt -> a fully-passing task: render it as contrast + # material, so the driver can check its trigger does NOT fire here. + for name in _ladder_exps(exp): + picked = _passing_attempt(runs_root, ws_root, name, task_id, k) + if picked is not None: + desc, transcript = render_trajectory(picked[0], picked[1], cap=cap) + return ( + f"PASSING TRAJECTORY (healthy run, for contrast — your fix " + f"must not change its behavior):\nTASK: {desc}\n\n{transcript}" + ) + return "(no non-infra attempt on record for this task)" + + return render + + return renderer_of + + +def render_candidate_failure( + runs_root: str | Path, + ws_root: str | Path, + exp: str, + tid: str, + *, + k: int = 3, + tail: int = 450, +) -> Optional[str]: + """Compact excerpt of HOW a candidate broke a task (harm replay). + + Reads the candidate's own out-dir (``exp`` = its confirm job) for ``tid``'s + failing attempt and returns the trajectory tail + verdict. Fed back into + the next round's PRIOR ATTEMPTS: designers predict fixes well but harms + poorly, so 'narrow it sharply' needs the actual wound, not just a count.""" + picked = _failing_attempt(Path(runs_root), Path(ws_root), exp, tid, k) + if picked is None: + return None + _, transcript = render_trajectory(picked[0], picked[1], cap=300) + t = transcript.strip() + return t[-tail:] if len(t) > tail else t + + +def build_passing_ids_source( + *, + runs_root: str | Path, + ws_root: str | Path, + exp_of: Callable[[HarnessNode], str], + k: int = 3, + limit: int = 6, +) -> Callable[[HarnessNode], list[str]]: + """Per-parent list of fully-passing train tasks with a renderable session — + the design step's contrast set for the over-trigger self-check.""" + runs_root = Path(runs_root) + ws_root = Path(ws_root) + + def passing_ids_of(parent: HarnessNode) -> list[str]: + exp = exp_of(parent) + ladder = _ladder_evals(runs_root, exp) + if not ladder: + return [] + out: list[str] = [] + for tid, (src_exp, ev) in sorted(_kept_measurement(ladder).items()): + if ev.passes < ev.attempts: + continue + if _passing_attempt(runs_root, ws_root, src_exp, tid, k) is not None: + out.append(tid) + if len(out) >= limit: + break + return out + + return passing_ids_of + + +__all__ = [ + "render_trajectory", + "default_sess_path", + "build_out_dir_trajectory_source", + "build_failed_attempt_renderer", + "build_passing_ids_source", + "render_candidate_failure", +] diff --git a/benchmarks/appworld/tool.py b/benchmarks/appworld/tool.py new file mode 100644 index 0000000..7bcc117 --- /dev/null +++ b/benchmarks/appworld/tool.py @@ -0,0 +1,78 @@ +"""The single agent tool for AppWorld: execute Python in the AppWorld REPL. + +AppWorld is pinned to pydantic v1 (via sqlmodel) while Raven needs pydantic +v2, so the two cannot share a venv. We therefore run AppWorld as its own HTTP +``environment`` server (``appworld serve environment``, in the AppWorld venv) and +talk to it over HTTP from here — this module never imports ``appworld``. + +This tool routes the agent's code to ``POST {env_url}/execute`` (the stateful +in-server world REPL): variables, imports and logins persist across calls. One +env server holds ONE world at a time, so the batch runner gives each concurrent +task its own server/port. + +This file is deliberately plain vanilla: it adds nothing to the model's inputs +or outputs beyond the REPL round-trip. It sits inside the evolution whitelist, +so improvements to it are the evolver's job — shipping pre-built assists here +would let a candidate "win" by switching them on. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import requests + +from raven.agent.tools.base import Tool + + +class AppWorldExecuteTool(Tool): + def __init__(self, env_url: str, task_id: str, timeout: float = 180.0) -> None: + self._env_url = env_url.rstrip("/") + self._task_id = task_id + self._timeout = timeout + self.saw_complete_task = False + + @property + def name(self) -> str: + return "execute" + + @property + def description(self) -> str: + return ( + "Execute Python code in the AppWorld environment and return its stdout. " + "Call app APIs via apis..(...). State (variables, imports, logins) " + "persists across calls like a Jupyter notebook, so build up the solution " + "incrementally. You MUST print() anything you want to see — only stdout is " + "returned. Discover APIs with apis.api_docs.show_api_descriptions(app_name=...) " + "and apis.api_docs.show_api_doc(app_name=..., api_name=...). When the task is " + "done call apis.supervisor.complete_task(answer=...) exactly once." + ) + + @property + def parameters(self) -> dict[str, Any]: + return { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "Python code to run in the stateful AppWorld REPL.", + } + }, + "required": ["code"], + } + + def _post_execute(self, code: str) -> str: + if "complete_task" in code: + self.saw_complete_task = True + resp = requests.post( + f"{self._env_url}/execute", + json={"task_id": self._task_id, "code": code}, + timeout=self._timeout, + ) + resp.raise_for_status() + return resp.json().get("output", "") + + async def execute(self, code: str) -> str: + # Blocking HTTP -> run off the event loop so the loop stays responsive. + return await asyncio.to_thread(self._post_execute, code) diff --git a/docs/examples/evolve_appworld.yaml b/docs/examples/evolve_appworld.yaml new file mode 100644 index 0000000..4e72dc2 --- /dev/null +++ b/docs/examples/evolve_appworld.yaml @@ -0,0 +1,62 @@ +# Example: AppWorld self-evolution run spec (built-in scorer line). +# +# uv run python -m raven.evolver check --config # validate setup +# uv run python -m raven.evolver run --config --smoke # tiny wiring run +# uv run python -m raven.evolver run --config # the real run +# uv run python -m raven.evolver status --config # progress (sealed-safe) +# +# One command does everything: cold-start baseline (if missing) -> evolution +# rounds -> terminate -> unseal + retention. Interrupt freely; the same +# command resumes. `finalize` ends a run early and unseals (one-way). +# +# Prerequisites (see raven/evolver/README.md, "Bootstrap: AppWorld"): +# an AppWorld install with data/, a subject runtime config JSON, and task +# id lists. Replace every /path/to/... below. + +bench: appworld +repo_root: /path/to/Raven # the repo being evolved (subject) +# base_sha: # root node; omit to use repo_root's + # current HEAD (resolved once at launch + # and pinned for the whole run) +work_dir: /path/to/evolution/work + +# Omit `models` entirely to use Raven's own configured model for all roles. +# These are the evolution loop's brains — distinct from the subject agent's +# model, which is pinned inside bench_config.config_path for the whole run. +models: + driver: {provider: claude_cli, model: claude-haiku-4-5, max_concurrency: 4} + design: {provider: claude_cli, model: claude-opus-4-8} + verdict: {provider: claude_cli, model: claude-opus-4-8} + # or: {provider: openai_compat, base_url: https://host/v1, model: , + # api_key_env: EVOLVER_DRIVER_API_KEY} + +funnel: # all optional; example values for a compact run + # (omitted -> defaults: k_screen 1, k_confirm 3, + # budget 2x3, patience 10, max_rounds 20) + k_screen: 1 + k_confirm: 3 + budget: {max_why_per_round: 2, candidates_per_why: 2} + termination: {patience: 5, max_rounds: 8} + +bench_config: + # Copy docs/examples/subject_runtime.json and fill in your endpoint. + config_path: /path/to/subject_runtime.json + appworld_data_root: /path/to/appworld # holds data/; exported as APPWORLD_ROOT + train_task_file: /path/to/train.txt + # test_task_file: /path/to/test.txt # enables the sealed test + retention + n: 90 + conc: 8 + base_port: 8600 + extra_args: ["--task-timeout", "400"] + # whitelist: omit to use the repo default (raven/agent/, benchmarks/appworld/); + # every prefix is validated against base_sha at startup — a dead prefix + # refuses to start instead of silently dropping designer edits. + +# Deep-merged over the config when --smoke is passed (on top of the built-in +# shrink: 1 WHY x 1 candidate x 1 round, K=1). work_dir gets a _smoke suffix. +smoke: + bench_config: + train_task_ids: ["", "", ""] + n: 3 + conc: 3 + base_port: 8700 diff --git a/docs/examples/subject_runtime.json b/docs/examples/subject_runtime.json new file mode 100644 index 0000000..6b67f0a --- /dev/null +++ b/docs/examples/subject_runtime.json @@ -0,0 +1,21 @@ +{ + "skill_forge": { + "enabled": false + }, + "providers": { + "custom": { + "api_base": "http://your-inference-host:8000/v1", + "api_key": "sk-..." + } + }, + "agents": { + "defaults": { + "provider": "custom", + "model": "your-model-name", + "max_tokens": 8192, + "max_tool_iterations": 40, + "context_window_tokens": 65536, + "temperature": 0.0 + } + } +} diff --git a/docs/specs/evolve-bench-contract.md b/docs/specs/evolve-bench-contract.md new file mode 100644 index 0000000..52d5c1f --- /dev/null +++ b/docs/specs/evolve-bench-contract.md @@ -0,0 +1,84 @@ +# Adding a benchmark: the evolution plugin contract + +The unified entry `python -m raven.evolver run --config ` runs the SOP +self-evolution loop on any registered benchmark (methodology: +[`self-evolution-loop-sop.md`](self-evolution-loop-sop.md); implementation +mapping: [`self-evolution-loop-raven-mapping.md`](self-evolution-loop-raven-mapping.md)). +This document answers: what do you build to make *your* benchmark evolvable? +Reference implementation: `benchmarks/appworld/evolve/entry.py` +(built-in scorer line). User-facing docs live in `raven/evolver/README.md`. + +## What you bring (bench side) + +| # | Deliverable | Shape | Hard requirement | +|---|---|---|---| +| 1 | **Scorer** | subprocess-invokable: given a task list, K attempts, and a checkout of the candidate code, write **one result file per trial** | Result files must distinguish "task failed" from "infrastructure failed" (an infra marker) — this feeds Gate-f and the fixed-denominator rule | +| 2 | **Result reader** | result files -> `TaskEval(task_id, passes, attempts, infra_attempts)` | Idempotent: re-reading the same directory yields the same evals | +| 3 | **Trajectories** | one renderable execution record per attempt, plus a "failing attempt -> diagnosis text" renderer | Diagnosis only ever reads train-side trajectories | +| 4 | **Task split** | train / test id lists (files or inline) | A test list enables the sealed flow; train∩test must be empty (checked at startup) | +| 5 | **Editable-path whitelist** | which path prefixes of the subject repo the designer may edit | Every prefix must match files at `base_sha` or the run refuses to start (prevents silently-dropped edits). **The scorer must not be inside it**: the grading *implementation* (oracle call, infra classification, result-write helper) must be covered by `path_guard.IMMUTABLE_PATTERNS` — a candidate that can edit its own judge is not being measured (AppWorld: `evolve/grade.py` + `batch.py` are immutable; `agent_cli.py`/`tool.py` are the editable agent surface). Note the limit: the editable agent process is still the *writer of record* — it runs candidate code in the same process as the immutable grader and owns the result file, so it can in principle discard the oracle verdict. Immutability stops accidental drift and casual gaming, not a determined adversary; see the README threat model (diff-audit promoted candidates) | + +Optional: an environment precheck (pre-run health gate), a WHY taxonomy +(omitted -> induced automatically from vanilla failures), per-WHY focused +subsets (targeted probing). + +**Two different models, do not conflate:** the *subject's* model (what the +benchmarked agent runs on — pinned inside your scorer config for the whole +run, same-regime rule) and the loop's driver/design/verdict models (the yaml +`models:` section; omitted -> Raven's own configured model). + +## Implementation shape: one `build()` function + +In your package (e.g. `benchmarks//evolve/entry.py`): + +```python +from raven.evolver.launch.contract import BenchBundle, LaunchContext, validate_whitelist + +def build(ctx: LaunchContext) -> BenchBundle: + spec = ctx.spec # bench/repo_root/base_sha/work_dir/funnel/bench_config + bc = spec.bench_config # your own yaml section; you own its schema + validate_whitelist(spec.repo_root, spec.base_sha, whitelist) + return BenchBundle( + root_node_id="C0", + root_node=..., # HarnessNode anchored at base_sha + journal_path=spec.work_dir / "journal" / "rounds.jsonl", + cold_start_total=len(train_ids) * spec.funnel.k_confirm, + cold_start_done=..., # count existing vanilla trial result files + run_cold_start=..., # idempotent; invoked every run: fill missing + # trials + any infra-rerun ladder salvage + build_orchestrator=..., # assemble EvolutionOrchestrator (EvalBackend + gate) + unseal=..., # only with a test set; journal records -> report dict + precheck=..., # optional Gate0 probe: raise RuntimeError (actionable + # message) on a dead endpoint / bound ports / missing + # install; `check` runs it before anything is spent + ) +``` + +Register it: `raven.evolver.launch.registry.BENCHES[""] = "your.module:build"`. + +Framework pieces you reuse instead of rewriting: + +- `make_worktree_eval_fn` — candidate commit -> ephemeral worktree checkout; + you only implement "score this directory on these tasks"; +- `eval_with_infra_rerun` — the SOP §0 infra rerun ladder (<=2 reruns, KEPT rule); +- `compute_stability` / `simple_anchor` / `select_anchor` — cold-start + bucketing and anchor selection; +- two gate policies: `PairedTwoSigmaGate` (generic anchor screen) and + `FocusedFisherGate` (per-WHY targeted probe); +- `SealedTestRunner` + `unseal_retention` — the sealed test / retention suite. + +## Idempotency requirements (what resume stands on) + +The resume model is "artifacts are the state": `run_cold_start` and your eval +path must, on re-invocation, only fill missing trials (an existing parseable +result file == that trial is done; a half-written file must be re-run). Round +granularity is covered by the journal replay. If your scorer skips existing +result files, interruption/resume works with no further effort. + +## Acceptance checklist + +- [ ] `run --smoke` completes one round (pin 2-3 known-failing tasks in the + yaml `smoke:` section) +- [ ] Ctrl-C mid-run, re-run the same command: completed trials are not re-run +- [ ] `status` at each of the three phases; no test numbers ever appear +- [ ] Put a wrong prefix in the whitelist: the run must refuse to start diff --git a/docs/specs/self-evolution-loop-raven-mapping.md b/docs/specs/self-evolution-loop-raven-mapping.md new file mode 100644 index 0000000..92a992c --- /dev/null +++ b/docs/specs/self-evolution-loop-raven-mapping.md @@ -0,0 +1,168 @@ +# Self-evolution SOP <-> Raven implementation map + +**Companion:** [`self-evolution-loop-sop.md`](self-evolution-loop-sop.md) — an +English translation of the upstream project's SOP, the authoritative +methodology spec. This document answers: which code +implements each SOP clause in Raven, which deviations are deliberate, and +which parts are present but unwired. + +**How to use:** before changing `raven/evolver/**` or +`benchmarks/appworld/evolve/**`, locate the SOP clause here; if your +change alters a correspondence, update this document in the same PR. + +--- + +## 0. The fundamental architectural difference (read first) + +The SOP's loop (§3 / §8.3) is **Claude-driven**: no driver program — a human +opens Claude and walks the seven-step funnel by hand, state persists in three +file layers, and the parts are a set of CLI scripts. Raven implements the same +methodology as a **program-driven** loop: +`raven/evolver/orchestrator/loop.py::EvolutionOrchestrator` is a deterministic +driver for the funnel, and the SOP's CLI parts became functions embedded in +the loop. + +The SOP §8.0 division of labor (semantics to the model, deterministic +arithmetic to code) is preserved unchanged: diagnosis / design / verdict are +still LLM calls, wrapped in `orchestrator/nodes/semantic.py::SemanticNode` +(parse failures are fed back for bounded repair retries). Only "who presses +the next-step button" changed. + +The SOP itself (§8.3) judges this route: "packaging into one-click tools is a +large-benchmark task". Raven chose to build the integration now, buying +cross-window hand-off freedom and mechanized methodology (see §5, "Where we +exceed the SOP"). + +--- + +## 1. SOP §0 general rules -> implementation + +| SOP clause | Raven implementation | Evidence (file :: symbol) | +|---|---|---| +| Gate0 pre-run environment health check | precheck injected per round, forced once before cold start | `orchestrator/production.py::build_evolution_orchestrator` (`run_gate0`); `benchmarks/appworld/evolve/precheck.py::make_appworld_precheck` | +| Infra-failure ladder: detect -> rerun <=2 -> score 0 if still broken | `eval_with_infra_rerun`, `max_reruns=2`, produces the `_infra_rerun{1,2}` directory ladder; the KEPT rule takes the measurement with the fewest infra trials | `orchestrator/scoring.py::eval_with_infra_rerun`; `benchmarks/appworld/evolve/adapter.py::ladder_out_dirs / read_kept_out_dir` | +| ★ Denominator = total tasks; infra tasks never excluded, scored 0 in place | Gate-f only reports the contamination list, never shrinks the denominator | `orchestrator/gates/pipeline.py::run_gates` (comment cites SOP §0 directly) | +| Division of labor: semantics = model, determinism = code | see §0 above | `orchestrator/nodes/semantic.py::SemanticNode` | +| Two verdicts: navigation = K3 mean > vanilla (bank); credited = paired 2σ (paper) | `PairedResult` keeps them as two independent fields: `promoted` (navigator) and `credited_2sigma` (label); promotion reads only the former | `orchestrator/gates/paired.py::PairedResult / paired_lift` | +| Paired σ: std of per-task paired diffs, removing between-task difficulty | `d_i = rate_c,i − rate_v,i`, `se = stdev(d)/√n`, `z = lift/se` | `orchestrator/gates/paired.py::paired_lift` | +| ★ Sealed test: blind-run each round, invisible to decisions, unseal at the end for retention | `SealedTestRunner.score` writes to a driver-invisible directory and **returns None** (no test number can physically enter the decision path); `unseal` only after the loop ends | `orchestrator/sealed/runner.py::SealedTestRunner / unseal_retention` | +| Test never enters anchor/train | stronger than the SOP's discipline: a mechanized assertion — leakage raises at startup | `orchestrator/sealed/runner.py::assert_no_test_leak`, wired in `loop.py` construction | +| Discipline: diagnose from train trajectories only | the diagnosis corpus source hangs off train only; test trajectories have no read path (the sealed runner stores scores only) | `orchestrator/scoring.py::EvalBackend.trajectories` | +| Discipline: configuration identical throughout | mechanized at the launcher level: `run_meta.json` records a config fingerprint; a changed config refuses to resume | `evolver/launch/state.py::RunMeta.check_config` | + +## 2. SOP §1 cold start -> implementation + +| SOP clause | Raven implementation | Evidence | +|---|---|---| +| Vanilla train full set x K=3 thick ledger | `backend.cold_start()` must return a non-empty stability ledger; baseline frozen-seeded from the vanilla out-dir | `orchestrator/loop.py` (construction); `benchmarks/appworld/evolve/run.py` (`seed_label="van0"`, `cold_start_k`) | +| Failure map covers >= 7 WHY classes | `diagnose_round(min_why_classes=7)` | `orchestrator/nodes/diagnose.py::diagnose_round` | +| WHY x WHERE taxonomy; inducible for a new bench | `TaxonomySpec` + two-stage `induce_taxonomy`; induction failure raises loudly, never silently borrowing another bench's table | `orchestrator/nodes/taxonomy.py` | + +## 3. SOP §2 seven-step funnel -> implementation + +| SOP step | Raven implementation | Evidence | +|---|---|---| +| ① Round 1 uses the cold-start map; round 2+ re-diagnoses the child (fired / flipped / still-failing), appending to the living map | `_diagnosed_parents` prevents duplicate diagnosis; `merge_failure_maps` accumulates cross-round and persists; flips recorded in `_flips` with a harm-replay excerpt (how the regressed task actually broke) | `orchestrator/loop.py` (round body); `orchestrator/production.py::outcome_hook` | +| ② 1-2 WHYs x 2-3 candidates, budget-capped; env-gated default-off | WHY selection defaults to driver mode (model picks; formula fallback + shadow log); `Budget(max_why_per_round x candidates_per_why)` enforced in code; the appworld line passes per-node `activation_env` | `benchmarks/appworld/evolve/editor.py::driver_select_whys / rerank_whys`; `orchestrator/config.py::Budget`; `benchmarks/appworld/evolve/adapter.py` | +| ③ Free pruning: beacon_guard + preflight | beacon: the editor rejects python edits without `activation_beacon` (hard); preflight: `make_zero_hit_preflight` zero-hit prune, **off by default** (see §6 ①) | `benchmarks/appworld/evolve/editor.py`; `orchestrator/production.py::make_zero_hit_preflight`; `benchmarks/appworld/evolve/run.py` (`zero_hit_preflight=False`) | +| ④ Apply, create child node; path_guard shields the kernel; git persistence | edit-then-commit: edits land as a real git child commit of the parent, the live tree untouched, mechanical changed-paths returned | `evolver/tree/git_ops.py::commit_files_as_child`; `evolver/applier/path_guard.py` | +| ⑤a K=1 anchor generous-pass screen; σ_screen computed; cull at 1.5σ; three tiers | three buckets clear_win / within_band / cull, only cull is blocked; σ formula identical to the SOP, emitted by `select_anchor` from the ledger; the AppWorld line uses a focused-subset Fisher probe variant (also generous-pass: culls only significantly-worse) | `orchestrator/nodes/screen.py::screen_candidate`; `evolver/scheduler/anchor_selection.py::select_anchor / simple_anchor`; `orchestrator/gates/strategies.py::FocusedFisherGate` | +| ⑤a anchor composition: affinity majority + icebreakers + sentinels, ⊂ train | all three roles implemented; the sentinel guard adds stratification the SOP does not have (stable = mean guard, fragile = Fisher, avoiding noise kills); the affinity data source is unwired (§6 ③) | `evolver/scheduler/anchor_selection.py`; `orchestrator/gates/strategies.py` (sentinel guard) | +| ⑤ Borrowing (SOP tags [defer]) | module present (byte-identical to upstream), unwired (§6 ②) | `evolver/scheduler/tree_aware_bandit.py` | +| ⑤b Survivors: full set x K=3 confirm | `k_confirm=3`, confirm runs the full train set | `orchestrator/gates/strategies.py` | +| ⑥ Three shields in order Gate-f -> Gate-b -> Gate2 | `run_gates` is exactly this order; Gate-b fails OPEN without instrumentation data (never condemns an uninstrumented-but-honest candidate); the reported score always uses the full-set fixed denominator, so a Gate-b subset mean can never masquerade as the score | `orchestrator/gates/pipeline.py::run_gates`; `orchestrator/production.py` (beacon-aware `fired_source`) | +| Gate-b data chain | write side: the editor forces inline beacons -> per-attempt beacon directories; read side: union over the confirm dir + infra-ladder siblings. Honesty note: attribution is presence-level — Gate-b proves *a* beacon in the candidate's code executed on a task, not that the beacon sat inside the mechanism's trigger condition; an unconditionally-placed beacon degrades Gate-b to a no-op (promotion still requires the full-train win) | `evolver/activation/ledger.py::beacon_workspace / mark_beacons_enabled` (called from `benchmarks/appworld/batch.py`); `evolver/activation/ledger.py::read_fired_tasks` | +| ⑦ Best of bank becomes parent; all dead -> keep the old parent + fresh diagnosis | `beat_vanilla` patience signal + greedy parent selection | `orchestrator/loop.py` | +| Termination: 10 rounds with nobody above vanilla (vs vanilla, not the previous parent) or 20-round cap; never consult test | `TerminationTracker(patience=10, max_rounds=20)`, the signal defined as beating the FIXED vanilla; plus a protection the SOP lacks: errored rounds do not burn patience (`max_consecutive_errors` is a separate backstop) | `orchestrator/termination.py` | +| Four node statuses | superset: `pruned_inert / pruned_at_screen / pruned_at_confirm / promoted_to_baseline / errored / blocked_l1 / archived-methodology-failure`; inert deaths additionally feed the per-WHY history so the designer learns from them | `evolver/tree/node.py::NodeStatus`; `orchestrator/production.py::inert_hook` | +| WHERE bound mechanically from the artifact, never by self-declaration | `bind_where` derives the lever from the actually-touched files; the self-declared `patch_where` stays in the ledger for audit and never decides the archive coordinate (4-tier granularity, see §6 ④) | `orchestrator/archive.py::bind_where / cell_of` | + +## 4. SOP §3 persistence -> implementation + +| SOP layer | Raven counterpart | +|---|---| +| findings work log | `/findings.md` (one section per round, driver verdict) | +| Cross-session state | journal (`orchestrator/state/journal.py`, crash-resume replays completed rounds) + `history.json` (per-WHY attempt history) | +| Box durable artifacts | `failure_map.json` (cross-round living map) / `nodes/.json` (node ledger: identity + git anchor + final status + gate stats) / per-round out-dirs (per-task results) | + +The git + node-ledger dual source of truth (SOP §3.1) carries over +isomorphically: code state lives in git commits (`commit_files_as_child` +produces real SHAs), tree lineage in `nodes/*.json`, aligned via +`git_commit_sha`. + +## 5. Where we exceed the SOP + +- **The sealed-test runner is mechanized.** SOP §8.2/§9.2 books its own + sharpest debt: "not building yet; interim relies on discipline; reviewers + will challenge it". Raven's `SealedTestRunner` + `assert_no_test_leak` is + exactly the mechanism isolation the SOP demands — the upstream's sharpest + methodological debt does not exist here. +- **A single candidate's crash cannot sink a round:** the `errored` status + + errored rounds not burning patience (`max_consecutive_errors` as its own + backstop); not covered by the SOP. +- **QD archive and recombination:** a (WHERE x WHY) per-cell elite bank + + cross-cell recombinant candidates (`orchestrator/archive.py`) — an + exploration mechanism beyond the SOP; gates and calibers unchanged. +- **The inert-death feedback loop** (2026-07): a distinct `pruned_inert` + status + inert deaths recorded into history + the design prompt + distinguishing "the trigger never fired" from "the mechanism was rejected" + + gentle WHY decay for inert deaths (`0.55^n_fail x 0.85^n_inert`). +- **Harm replay:** when a candidate breaks a task, the regressed task's actual + trajectory excerpt under that candidate is fed to the next design attempt; + the SOP only requires flip counts. + +## 6. Deliberate deviations and unwired parts (honest list) + +1. **zero-hit preflight is off by default** (`zero_hit_preflight=False`, + decided 2026-07). Rationale: Gate-b already denies credit to never-fired + mechanisms (no correctness hole); preflight only saves budget, at a small + false-prune risk; TRIGGER_REGEX declaration is opt-in and the actual prune + rate is unknown. Gather data first (enable on a run, inspect + `pruned_inert` entries in history), then decide the default. SOP §2 tags ③ + as [now]; this is a deliberate deviation. +2. **Borrowing unwired** (SOP §2 ⑤, tagged [defer]): `tree_aware_bandit` is + present, the orchestrator does not call it. The AppWorld train set is + affordable to run in full — consistent with the SOP's "small benchmarks + don't need it"; wire it for large task sets. +3. **Affinity anchor lacks a data source:** `select_anchor(affinity)` accepts + it; the appworld side has no trigger-density source (upstream's + `affinity_picker.py` was not ported). Anchors are currently icebreakers + + sentinels + borderline; screening still works, information efficiency is + slightly lower. +4. **WHERE mechanical binding has 4 tiers** (prompt/runtime/mixed/edit), + coarser than the judge schema's 14 classes. Sufficient for QD cells; + refine `_lever_of_path` for fine-grained lever statistics. Self-declared + and mechanical values are both in the ledger; no reconciliation alarm + (an optional observability add-on). +5. **`pruned_inert` matches the SOP's status semantically** but differs in + implementation: the SOP judges it via a preflight CLI, we via the embedded + loop preflight (default off, see ①). + +## 6.5 The unified entry (added 2026-07) + +SOP §8.3's "manual orchestration" is superseded by +`python -m raven.evolver run --config `: a single-command state machine +running cold start -> rounds -> termination -> unseal, resumable after any +interruption (artifacts are the state: trial files / journal / meta stamps, +three tiers of truth). Config drift and unseal one-wayness are mechanized in +`run_meta.json` (the codification of SOP §0's same-regime discipline). +Benches plug in via the contract in +[`evolve-bench-contract.md`](evolve-bench-contract.md); implementation in +`raven/evolver/launch/` + `raven/evolver/cli.py`. + +## 7. SOP parts <-> Raven parts quick reference + +| Part cited in SOP §8.1 | Raven counterpart | Shape difference | +|---|---|---| +| `analysis/proxy_features.py` | `evolver/analysis/proxy_features.py` | byte-identical | +| `analysis/failure_map_builder.py` | `evolver/analysis/failure_map_builder.py` | byte-identical | +| `activation/preflight.py` (CLI) | `orchestrator/production.py::make_zero_hit_preflight` | CLI -> embedded; regex variant, default off | +| `analysis/stability_bucket.py` | `evolver/analysis/stability_bucket.py` | byte-identical | +| `scheduler/bandit_tasks.py` | `evolver/scheduler/bandit_tasks.py` | byte-identical | +| `scheduler/anchor_selection.py` | `evolver/scheduler/anchor_selection.py` | same lineage + our `simple_anchor` | +| `scheduler/affinity_picker.py` | **no counterpart** (§6 ③) | — | +| `scheduler/tree_aware_bandit.py` | `evolver/scheduler/tree_aware_bandit.py` | same lineage, unwired (§6 ②) | +| `activation/gate_audit.py` (CLI) | `orchestrator/gates/pipeline.py::run_gates` (Gate-b embedded) | CLI -> embedded | +| `analysis/paired_significance.py` (CLI) | `orchestrator/gates/paired.py::paired_lift`; retention in `sealed/runner.py::unseal_retention` | CLI -> embedded | +| `aggregate_keq3` / `gate0_ctrf_audit` (external eval engine) | `benchmarks/appworld/evolve/eval.py / adapter.py` (K=3 aggregation + infra ladder) | cross-repo contract -> same-repo module | +| `tree/*` | `evolver/tree/` (node/store/git_ops) | same lineage + our `commit_files_as_child` / `read_file_at` | diff --git a/docs/specs/self-evolution-loop-sop.md b/docs/specs/self-evolution-loop-sop.md new file mode 100644 index 0000000..67c962d --- /dev/null +++ b/docs/specs/self-evolution-loop-sop.md @@ -0,0 +1,328 @@ +# Self-Evolution Main Loop — Standard Operating Procedure (SOP) + +> **Provenance:** English translation of the upstream project's internal SOP +> document. Internal infrastructure details (box addresses, share paths, +> endpoint/tooling names) are redacted in this translation. The two +> evaluation lines it references are **TB2** (an internal terminal-agent +> benchmark) and **EvoAgentBench/LiveCode** (a coding benchmark); Raven's +> shipped example benchmark is AppWorld — see the mapping doc for how each +> clause lands in this repo. Raven's +> implementation mapping lives in +> [`self-evolution-loop-raven-mapping.md`](self-evolution-loop-raven-mapping.md). + +**Date:** 2026-06-24 | **Purpose:** TB2 / EvoAgentBench scoring runs follow this to produce paper-composable, uniform results. +**Nature:** operating manual (humans / Claude follow it), not theory. Theory/design sources are listed under "Finalized sources" at the end. +**Companions:** upstream working documents (design-gap notes, three-tier sampling design, paper positioning), internal to the upstream project and not shipped here. +> ⚠ **Cross-repo:** this SOP stays in the upstream harness repo (operational runbook); the paper design docs above moved to the **paper repo `docs/specs/`** on 2026-06-26 (same for every paper doc cited in §6). + +--- + +## 0. General rules (apply to every round) + +**★ Gate0 / environment & measurement validity (foremost; two checks, before and after scoring — violation voids that trial's data):** +- **Before scoring — environment health precheck (intra-env / sandbox check):** before running any task, verify the environment itself is sane — sandbox boots, docker networking works, dependencies installed, verifier can produce ctrf. Scores from a dirty environment (e.g. TB2's broken apt mirror in the sandbox image producing false-negative infra_err 38-45, or proxy x docker route hijacking producing connErr) are **all void**; fix the environment first (an infra overlay patch), then run. +- **After scoring — infra-failure handling ladder (detect -> rerun <=2 -> fix infra -> [still broken after 2] record as failure):** + - **Detect:** ctrf missing / sandbox crashed / verifier timed out = infra failure (L1), **not "the agent could not solve it"**; + - **★ Rerun (cap 2)** (user 2026-06-25): a trial that died on infra is **re-run once the environment is healthy**, topping K back up — measure the task's true ability; + - **Fix infra:** for a **deterministic** infra failure (crashes every time, e.g. the broken-apt-mirror case above), rerunning is useless — fix the infrastructure first, then rerun (still counted within the 2); + - **[Still broken after 2] record as unsuccessful** (user 2026-06-25): the rare task still broken after 2 reruns + an infra fix is **recorded as unsuccessful (numerator 0)** **but stays in the denominator** (no exclusion, no coverage bookkeeping). Every task ends up 0/1 -> clean denominator = total task count. +- **★ pass@1 denominator = total number of tasks (2026-06-25 user, hard rule):** + - **Denominator = all tasks; never remove infra-failed tasks from the denominator.** Reruns (<=2) turn salvageable tasks into honest measurements; the unsalvageable score 0 and stay in the denominator. + - **Why:** dropping infra-failed tasks = shrinking the denominator = **overestimating pass@1** (dropped tasks skew hard; cf. the v7 uv false-negative lesson: extrapolating a fair-subset to the full set overestimates). + - **Three-way contrast (all with denom = total; they differ in how the numerator is filled):** + - HarnessX A.3: an infra failure **counts as unsolved = 0 on sight** (no rerun) -> also crushes plenty of fixable/transient false negatives (contamination); + - old "timeout-fair exclude-not-zero": infra failures **removed from the denominator too** -> shrunken denominator -> overestimate; + - **ours:** **rerun <=2 first + fix infra** (salvaging TB2's 38-45 -> 7 fixable false negatives into real measurements), **only the truly persistent score 0, never excluded** -> denom = total, conservative and unbiased — the core of C1 measurement integrity (Gate-f); the differentiator is "two reruns before judging", not "zero on sight". + + +**Division of labor:** +- **Semantic operations = Claude (CC SDK):** diagnosis, WHY selection, candidate design, patch writing, verdict drafting. +- **Deterministic code/scripts:** scoring orchestration, gates, bookkeeping, coordinate binding, aggregation. + +**Two verdicts (dual thresholds — do not mix):** +| Use | Criterion | +|---|---| +| Internal navigation (enter bank / choose parent) | **K=3 mean pass@1 > vanilla** (loose). ⚠ use the K=3 mean, never a single run | +| External claims (paper "credited") | **paired 2σ** (per-task pairing on the shared task set, σ = measured std of paired diffs, computed per-task transparently). **Paired 2σ < unpaired 2σ:** pairing removes between-task difficulty variance; the residual is dominated by borderline tasks -> TB2 bar ~3-4pp (child ~47-48%) vs unpaired ~8pp (~52%). The paper still says "2σ (z>=1.96)"; only σ is computed paired. Never argue "paired therefore small" verbally — attach the per-task computation. See the gap doc's significance section | + +**Core discipline (settled after hard lessons; violations void the data):** +1. Diagnose only from train / failing trajectories; **held-out / test yields numbers only — never read its trajectories** (leak/overfit prevention). +2. Configuration identical throughout (timeout-fair same caps); failure fallbacks **never touch max_tokens/temp/thinking**; dirty data is filtered explicitly at the analysis layer. +3. Comparable only within same benchmark / same K / same verifier; single-run pass@1 swings ~5pp — **bank entry uses the K=3 mean**. + +**★ train/test iron law + sealed test set (C3's lifeline; violation voids retention):** +- **train:** scoring + **trajectory reading for diagnosis** -> drives evolution (diagnosis / WHY choice / design / parent selection draw **exclusively** on train). +- **test = sealed test set:** scored for numbers only; never read its trajectories, never modify the harness based on it. +- **Test runs every round, but sealed:** test scoring is done by a **deterministic script and stored where Claude cannot read it**; **during evolution decisions Claude cannot see test numbers** (withheld from the evolution agent). The risk comes from "having seen test, using it" (early stopping / parent choice / subconscious direction leakage), not from the running itself. **Sealing by mechanism (invisible at decision time) is what counts as sealed; "resisting the urge to look" does not** — reviewers will challenge it. +- **Unseal after evolution ends:** pull each round's test numbers -> plot train/test curves (catch the overfitting knee) + pick the highest-test round as the deliverable + compute **retention rate = test lift / train lift** (the C3 metric). +- = sealed-but-logged (run and store each round, sealed from the decision maker): you get the curve at zero leakage; leak risk ≈ fully-sealed (score only at the very end). Mirrors the division of labor: test scoring = deterministic script (run + store, never shown) / evolution decisions = Claude (fed train only). +- **Paper wording:** "We evaluate on a **sealed test set**: scored each round by an automated harness but **withheld from the evolution agent** — no test signal informs diagnosis, candidate design, or parent selection; unsealed only after evolution completes." (Contrast HarnessX §6.1's own admission of same-set, no held-out.) + +--- + +## 1. Cold-start SOP (once per (model x benchmark)) + +> ★ **Cold start also uses train only:** "full set / failing tasks / failure map" below all mean the **train set** (test is sealed and never enters diagnosis / sampling / the thick ledger). Vanilla's test score is blind-run by script and stored; it plays no part in cold start. + +``` +1. Run vanilla (root) on the train full set x K=3 -> thick ledger (for descendants to borrow + the locked anchor baseline) +2. Diagnosis scope (failing tasks): + - train set <= 200 tasks -> read every failing task (borderline + always-fail); sample a few always-pass as controls/sentinels + - train set > 200 tasks -> stratified sampling of failures by "failure signature" for coverage (this work: TB2 89 / LiveCode 97, both <= 200, read all) +3. Claude diagnoses failing trajectories one by one, using only "failure signatures (source A, native to raw trajectories)": + finish_reason=length + empty (thinking-runaway / empty response) / repeated tool_call / iteration-cap hit / docker errors / code never pasted ... + (no affinity — no mechanisms exist yet) +4. Emit a structured failure_map.json along the way (lightweight, CC SDK schema output, not the four-stage pipeline): + each entry = { WHY (failure mode), which tasks (task_id + stability bucket/signature), WHERE-hint (soft lever suggestion, non-binding) } +5. Coverage check: number of WHY classes covered by the failure map (target >= ~7); enough -> cold start complete +``` +Deliverables: `vanilla thick ledger (full set K=3)` + `failure_map.json`. + +--- + +## 2. Per-round evolution SOP (the seven-step funnel) + +> Tags: [now] do at current scale / [defer] only for large benchmarks / [human=Claude] semantic step. +> **Who does which step, calling which code -> see §8** (this section is flow only; §8 is the Claude Code <-> prepared-code division). + +**One round = one "diagnose -> design n candidates -> each candidate takes its own anchor fork -> choose parent".** +"Return after the anchor" = that *candidate* was judged a clear loser at screen, culled on the spot, **never reaching the full set** — not every round walking the full set. + +``` +① Diagnose [human] — round 1 vs later rounds: + - round 1: no "previous round's child" yet -> use the cold-start failure map directly (built from vanilla failures). + - round 2+ (re-diagnosis): read the previous round's CHILD trajectories (not a rescan of all pass/fail), looking at: + tasks where the mechanism fired (did it work / better or worse) + flipped tasks (rescued = confirmed credit / + broken = regression) + still-failing tasks (next target) -> append-update the failure map. + Note: the failure map is a CROSS-ROUND LIVING MAP (cold start builds v1; every round appends new failures + + WHY-distribution drift), never frozen — that is exactly its value: auditable + round-comparable. + +② Choose WHY + design candidates C1..Cn [human]: pick this round's target WHYs from the failure map; design candidates + per WHY, across levers (prompt/knowledge/runtime/config). Patch env-gated, default off = byte-identical to vanilla, + never touching the kernel. + · Candidate count n (decided 2026-06-24, small-benchmark standard tier): 1-2 WHYs per round x 2-3 candidates per WHY + = 3-4 candidates/round. n is driven by "how many WHYs this round's diagnosis deems worth attacking" + capped by + budget, not fixed; the full 3x3=9 only on large budgets [defer]. + +③ Free pruning (zero GPU) [now]: beacon_guard (reject anything without activation_beacon) + preflight (trigger + predicate with zero historical hits = inert, never fielded). + +──── for each candidate Ci (candidate-level fork) ──────────────────────── +④ Apply, create child node [deterministic]: git apply (path_guard shields the kernel) -> create node + bookkeeping (nodes/) + +⑤a Prober screen: anchor subset (~15 tasks = affinity majority + always-fail icebreakers + 2-3 always-pass sentinels) x K=1 + · ★ The anchor draws from the TRAIN pool only, never test (test is sealed; the anchor is part of evolution + decisions, touching test = leakage). Hence anchor ⊂ train. Screen/confirm both compare to vanilla on train. + · Borrowing [defer]: tasks the mechanism does not fire on borrow the parent's thick ledger; the child skips them / K=1 + · Three-tier verdict (★ generous-pass; noise band σ_screen below): + clearly above vanilla -> ⑤b full set + slightly below / inside band -> ⑤b full set (C3: slightly-low is NOT culled! anchor mean != full-set mean) + below vanilla by > ~1.2-1.5x σ_screen -> ✂ cull Ci, skip ⑤b/⑥, record status=pruned_at_screen + (light bookkeeping: Ci / anchor performance / cull reason; those anchor results enter the thick ledger + for borrowing; prevents next round redesigning the same Ci) + · ★ σ_screen source (decided 2026-06-24): from the cold-start thick ledger (vanilla full train x K=3) take the + anchor's ~15 per-task pass rates p_i and compute σ_screen = sqrt( (1/n²)·Σ_i p_i(1−p_i) ) + (= the sampling σ of a K=1 anchor mean). This is the BIG, LOOSE σ (K=1 + small n + deliberately high-variance + borderline tasks) != Gate2's "full-set K=3 paired σ" (small, tight) — same ledger, two calibers. + · Cull threshold = 1.5·σ_screen (never a hardcoded pp); "inside the band" = within ±σ_screen (indistinguishable + from vanilla — still goes to the full set). + · ⚠ The old "−6pp" is void: a too-tight placeholder (≈0.5 σ_screen; it would cull indistinguishable candidates, + violating generous-pass). Rough estimate 9-13pp, far above the 5pp of a single full-set run. + · ★ Who computes it: σ_screen + cull_threshold are emitted by select_anchor() (same ledger, same p_i(1−p_i), + shared with borderline-task selection, see §8.2) — never Claude estimating on the spot. + +⑤b Estimator / full-set confirm (screen survivors only): full set x K=3 + · TB2 89 / small benchmarks: estimator = the full set; large benchmarks that cannot afford it: representative + subset (stratified by share + weighted) [defer] + +⑥ Three shields [deterministic]: + · Gate0/Gate-f (measurement validity, see general rules ★): environment health precheck + the infra-failure ladder + (detect -> rerun -> fix infra -> report coverage on exclusion, never silent zeros). TB2's infra overlay fixed + the apt-mirror false negatives (38-45 -> 7). Contrast HarnessX A.3 "infra counts as failure" = contamination. + · Gate-b: attribution allowed only when the activation ledger recorded the mechanism firing + · Gate2: paired lift; navigation pass@1 (K=3 mean) > vanilla -> enter bank, status=promoted; failed -> archived + as pruned_at_confirm +────────────────────────────────────────────────────── + +⑦ Choose parent [human/deterministic]: best in bank -> next round's parent; if every candidate was culled / none + passed -> no new parent: keep the old parent + fresh diagnosis, proceed to the next round. + -> check termination (below); if not triggered, back to ① +``` + +**Loop termination (decided 2026-06-24 by user; stop on either):** +``` +① 10 consecutive rounds without a harness "better than vanilla on train" <- primary signal (exploration exhausted) +② 20 rounds reached (hard cap, backstop) +``` +- **★ ①'s baseline is vanilla, not the previous round's parent** (user emphasis): each round asks "did any candidate beat **vanilla** on train K=3 mean pass@1"; 10 consecutive rounds with **not one candidate above vanilla** -> stop. A fixed bar is more stable and reproducible than a rising one, and matches the promotion criterion. +- **Never consult test to decide stopping** (sealed iron law): ① uses train-vs-vanilla only. +- **10 / 20 are initial values, tuned on real runs** (a low-ceiling model may hit 10 promotion-free rounds quickly = ceiling signal). +- **Post-termination wrap-up:** (a) unseal test -> plot train/test curves + pick the highest-test round as deliverable + compute retention rate; (b) record the stop reason (10 promotion-free rounds / 20-round cap) — honest logging, never "ran enough, stopping". + +**Node statuses (bookkeeping labels):** `pruned_at_screen` (died at anchor) / `pruned_at_confirm` (failed the full set) / `promoted` (entered bank as parent) / `pruned_inert` (died at ③ preflight). Every candidate leaves a record wherever it dies (auditable + ledger reuse + duplicate-design prevention). + +**WHERE coordinate binding** (diagnosis only emits hints): once the patch is written, WHERE is bound mechanically from the artifact (target_file); modeling error concentrates on the WHY axis. + +--- + +## 3. State persistence standard (uniform across windows/benchmarks — the SOP's core value) + +**The loop is Claude-driven, not program-driven; state lives in three persistence layers (validated on LiveCode; standardized for both):** + +| Layer | Content | Files | +|---|---|---| +| **findings work log** (primary state) | one section per round: what was tried, results, next target | `docs/specs/handoffs/--evolution-log.md` | +| **memory** (cross-session, read directly at cold start) | project state (updated per round) + discipline feedback | `memory/project__*.md` + `MEMORY.md` index | +| **box durable artifacts** (diagnosis raw material + result evidence) | failure_map.json / per-round reports (pass@1 + failure distribution) / session.jsonl (trajectories = next round's read-only input) / node ledger | `nodes/` (per-model node ledger dir) / `reports/` / `jobs/*/session.jsonl` | + +**Uniform requirements (both benchmarks, or the paper cannot compose them):** +- Every round's reports must contain: **per-task results (passes / K)** (for the paired σ) + failure-signature distribution + WHY distribution; +- **pass@1 denominator = total task count** (§0 hard rule): infra-failed tasks rerun (<=2) into valid measurements, never dropped; still broken after 2 -> score 0, stay in the denominator; +- mechanism patches: env-gated; record the flag name + target_file (for WHERE coordinate binding); +- scoring job names carry: benchmark / mechanism / K / split (train|test|full) / date. + +### 3.1 Harness kept in git + node ledger (the physical form of the evolution tree) + +**The evolution tree is not an abstraction; its physical form is exactly two things — git + the node ledger:** +- **Code lives in git:** each candidate harness = one **git branch + commit**; **C0/vanilla = the baseline branch**. Mechanism candidates = branches grown from the baseline/parent (e.g. `evolver/rN-*`). Mechanism patches are env-gated, default off = byte-identical to vanilla. + ⚠ **Evolver infrastructure (select_anchor / gates / sampling) is written to the C0 baseline, never to a mechanism candidate branch** (the evolver operates on the harness; it is not itself a mechanism). +- **The tree structure lives in the node ledger** `nodes/*.json` (one node per file), anchored to git: + +```json +{ "node_id": "R4F", "parent_id": "v7", // tree lineage (parent_id) + git lineage, both recorded + "git_branch": "evolver/r4f-std-v3", "git_commit_sha": "a996daa", + "core_version": "v7", "created_at_iter": 4, + "status": "promoted | pruned_at_screen | pruned_at_confirm | pruned_inert | archived-not-credited", + "activation_spec": { "kind": "...", "threshold": N }, // mechanism trigger predicate (Gate-b / preflight) + "patch": { "patch_where": "loop_override", "patch_why": "...", + "components": [ { "target_file": "...", "diff": "..." } ] } } // WHERE bound from target_file +``` + +- **How it is used:** `select_anchor` / sampling / diagnosis read the vanilla thick ledger -> Claude designs a patch -> new branch commit + one `nodes/.json` anchored to that commit -> evaluate -> backfill `status`. The node ledger = the single source of truth for auditability + ledger borrowing (parent_id tree distance) + duplicate-design prevention; git is the code source of truth; the two align via `git_branch`/`git_commit_sha`. + +--- + +## 4. Two-benchmark adaptation (difference table) + +| Dimension | TB2 | EvoAgentBench (LiveCode) | +|---|---|---| +| Scoring | internal verifier (timeout-fair overlay) | official lcb_runner check_correctness (subprocess sandbox) | +| **split** | **train 53 / held-out 36 (cut, sealed)** `s_tb2_{train53,heldout36}.txt` | **train 97 / test 39 (official)** <- the C3 generalization main experiment lives here | +| Scoring flow | anchor screen (~15 from train53) -> confirm train53 -> held-out36 sealed blind run | currently: train+test both full-run K=3 (97 tasks are cheap enough) | +| Scale | 89 | train 97 / test 39 | +| Infra critical | sandbox infra overlay (docker0 bridge), fixed apt-mirror false negatives (38-45 -> 7) | NO_PROXY=* direct connection (avoids proxy x docker route hijacking) | +| K=3 orchestration | `--attempts 3` single job | conc2x2 interleaved (controls endpoint drift) | + +**Uniform actions (paving C3 for the paper):** +- ~~TB2 should also cut a train/test split~~ **done (53/36, 2026-06-24)**; both benchmarks now have splits and can enter C3. +- Both benchmarks report held-out **retention rate = test lift / train lift** (the C3 metric), never just two absolute numbers. +- ⚠ The anchor currently draws from **train53** (~15/53 ≈ 28%, above §2 B's default 15-20% — normal for a small train set); σ_screen computed from the train53 thick ledger. + +--- + +## 5. One-round checklist (tickable) + +``` +[ ] ① Read the previous round's child trajectories for diagnosis (failures only; never held-out) -> update failure_map +[ ] ② Choose WHY + design candidates (env-gated, across levers, never touching the kernel) +[ ] ③ preflight + beacon_guard prune inert candidates (zero GPU) +[ ] Before scoring: Gate0 environment health precheck (sandbox/docker/deps/verifier); fix a dirty environment first +[ ] Per candidate Ci: ④ apply + bookkeeping -> ⑤a anchor K=1 (three-tier generous-pass) +[ ] ├ hopelessly bad -> cull Ci, record pruned_at_screen, skip the full set ("return after the anchor") +[ ] └ clearly above / slightly low / inside the band -> ⑤b full set K=3 -> ⑥ Gate0/Gate-f (infra rerun -> exclude, + never zeroed) / Gate-b / paired lift +[ ] ⑦ promotion (K=3 mean pass@1 > vanilla) -> best in bank becomes parent; all dead -> keep the old parent; + three persistence layers + node status labels +[ ] test = sealed: script-run and stored where decisions cannot see it (withheld); decisions use train only +[ ] For "credited": paired 2σ (per-task computed σ) +[ ] Termination: 10 consecutive rounds with no candidate train>vanilla, or 20 rounds -> stop (never consult test) +[ ] After evolution, unseal: train/test curves + highest-test round + retention rate = test lift / train lift +``` + +--- + +## 6. Finalized sources (per-step basis) +> ⚠ **All paper design docs below live in the paper repo `docs/specs/`** (moved out of the harness repo 2026-06-26); this SOP stays in the harness repo. +- Cold start / failure signatures / 200 threshold / failure_map: upstream design-gap notes §3b, failure_map section (internal, not shipped here) +- Three tiers / anchor / borrowing / N* / paired 2σ: `2026-06-16-sampling-and-decision-design.md` + the gap doc's significance-caliber section +- Two ablations (anchor / borrowing): gap doc §3c +- Contributions C1/C2/C3 + wording red lines: `2026-06-15-paper-positioning.md` §4c +- Three shields / gate definitions: `2026-06-12-gsme-theorems.md` (Gate <-> (b,f)), `2026-06-15-evolver-code-architecture.md` + +## 7. Current status (honest labeling, 2026-06-24) +- Actual runs = run_round MVP + scripts + Claude-in-loop, **not a fully deterministic funnel**; ② 3x3 breadth / ③ preflight / ⑤ borrowing are mostly missing or deferred. +- Top todo (outside this SOP; the main experiment): turn LiveCode into **GSME vs task-aware two arms + report retention** (the C3 headline result). +- TB2 numbers (van 44% / harness 45.3%) had infra issues at the time of writing and were being re-measured upstream; treat them as illustrative, not final. + +## 8. Claude Code <-> prepared-code contract (who does which step / calls which code) + +### §8.0 Guiding principle: semantics via Claude Code, deterministic arithmetic via prepared code +- **Semantic operations = Claude Code:** reading trajectories for diagnosis, choosing WHY, designing patches, writing code, judging "exploration incomplete vs ceiling". +- **Deterministic / arithmetic = prepared code:** task sampling, borrowing discounts, bookkeeping, σ computation, gate verdicts, aggregation. +- **Why anchor selection / borrowing must be code, never Claude improvising** (the heart of this contract): + 1. **Reproducible / auditable** (the paper's lifeline): which 15 tasks, what discount, what the paired 2σ works out to — must be computed by deterministic scripts and checkable. Claude "picking tasks by feel, eyeballing a discount" = irreproducible, unauditable = exactly the "ad-hoc diagnosis" pit we criticize HarnessX for. + 2. **LLMs cannot compute:** Bernoulli-variance ranking, ancestry-kernel discounts, exact paired σ = precise numerics; mental math will be wrong -> hand it to code. +- **Claude Code's role at these steps = invoke the code + read its output to make the next semantic decision**, never computing it itself. + +### §8.1 Step-by-step division (maps §2's seven steps) +| §2 step | Claude Code does | Prepared code (deterministic) | Hand-off | +|---|---|---|---| +| ① Diagnose | **read trajectories, judge WHY/severity/WHERE-hint** (semantic) | failure-signature pre-extraction `analysis/proxy_features.py`; living map `analysis/failure_map_builder.py` | code emits signature + failure_map.json -> Claude reads, writes the diagnosis | +| ② Choose WHY + design + write code | **all Claude** (target choice, patch design, patch code) | — | Claude produces the patch artifact (WHERE mechanically bound from the artifact) | +| ③ preflight prune | reads the prune verdict, decides to drop or not | **`activation/preflight.py`** (reachability over historical trajectories, zero GPU) | code emits reachable? -> Claude decides | +| ④ apply, create node | triggers | `tree/*` (node creation + git + bookkeeping, deterministic) | code creates the node, returns node_id | +| **⑤a anchor selection** | triggers + reads the subset | **`analysis/stability_bucket.py` (bucketing) + `scheduler/bandit_tasks.py` (variance ranking) + `scheduler/affinity_picker.py` (mechanism-fire tasks)** | code emits the anchor task list (⊂ train) | +| **⑤ borrowing** | triggers | **`scheduler/tree_aware_bandit.py`** (ancestry-kernel discount + weighted posterior, pure math) | code emits "which tasks skip runs / at what discount" | +| ⑤b full-set confirm | triggers + reads results | run the full set + `scripts/aggregate_keq3.py` (K=3 aggregation) | code emits per-task + pass@1 | +| ⑥ Gate0/f/b/2 | reads gate verdicts | **Gate-f `scripts/gate0_ctrf_audit.py` (-> the external eval engine) / Gate-b `activation/gate_audit.py` / Gate2 paired σ `analysis/paired_significance.py` (generic, shared by 6 benchmarks)** | code emits pass/fail + σ -> Claude decides | +| ⑦ choose parent | **threshold verdict = code; "exploration incomplete vs ceiling" = Claude** | bank comparison (deterministic) | code emits the promotion verdict; Claude makes the semantic termination call | + +### §8.2 Code inventory: parts present vs missing +- **✅ Present** (verified, in the repo): preflight, the anchor trio, borrowing, Gate-f/b, paired σ + retention (`analysis/paired_significance.py`, generic, shared by 6 benchmarks), K=3 aggregation, failure_map builder, failure signatures. +- **✅ `select_anchor()` built** (2026-06-24, on a feature branch off the C0 baseline, **unmerged** at the time of writing): `evolver/scheduler/anchor_selection.py` — returns `AnchorSelection(task_ids, σ_screen, cull_threshold=1.5·σ_screen, tasks, shortfalls)` in one call. Pure ledger read (depends only on `stability_bucket.compute_stability`): three tiers = sentinels (STABLE_PASS) + icebreakers (STABLE_FAIL, affinity-preferred optional) + borderline (BORDERLINE_*, ranked by `p(1−p)`); σ_screen dominated by borderline tasks; deterministic (ties by task_id). Unit tests 5/5 + 40 existing subsystem tests unbroken. **Todo:** ① run on the box against the train53 ledger for the real σ_screen ② commit/push/merge into C0. +- **⏸ sealed-test runner — not building yet (user decision 2026-06-24: mechanism design unsettled; interim relies on Claude discipline):** + - **Interim discipline (the concrete content of "discipline", tickable):** during evolution, no decision of any round (diagnosis / WHY choice / design / parent selection / early stop) **ever reads test scores or trajectories**; test scoring runs and is stored only, **never opened in a decision context**; looked at only at the final unseal. + - **Honest consequence (must be addressed at paper time):** this is **discipline-based sealing, not mechanism isolation**; §0 itself says "resisting the urge does not count — reviewers will challenge it". Before submission, one of: ① build the sealed-test runner (mechanism isolation); ② disclose honestly in limitations that test was discipline-sealed + provide auditable evidence (e.g. each round's test-score file timestamps predate that round's decision records). **Currently accepted as ① not built; on the books.** +- **✅ The two items formerly listed as missing are done in the upstream harness repo** (verified 2026-07-01, superseding the old round7_paired): + 1. **retention rate computation** — `evolver/analysis/paired_significance.py` (outputs retention). + 2. **generic paired σ** — same file: per-task paired z = Δ̄/(σ_paired/√n), credited iff |z|>=1.96, + p / 95% CI / bootstrap; shared by 6 benchmarks (moved from the paper repo's e0 into the harness repo). + Note: **the gate ablation (paper §15, comparing Δ>0 / single-run / paired-2σ gates) is not a loop step**; it is a one-off methodological contrast, staying in the upstream analysis scripts, not entering the evolver. + +### §8.3 Interface shape & integration timing (scale-dependent) +- **Current state = manual orchestration:** Claude (a human) runs the scripts above by hand, reads outputs, strings a round together. The parts exist; the "Claude Code auto-invokes code through a whole round" integration does not. +- **Small scale (TB2 89 / LiveCode):** few tasks, full-read/full-run affordable, anchor/borrowing rarely needed -> **manual orchestration + occasional scripts suffice**; no heavy integration. +- **Large scale (SWE-bench 500):** anchor/borrowing become genuinely necessary and frequent -> **then it pays to wrap these as one-click tools for Claude Code** (CLI subcommands / MCP tools) invoked directly at decision time. +- **In one sentence:** anchor/borrowing = deterministic arithmetic, computed by code and invoked by Claude Code (never improvised — unauditable and numerically wrong); packaging into one-click tools is a large-benchmark task; small scale runs fine on manual orchestration. + +--- + +## 9. Pre-run dependencies (what this document does NOT cover — honest boundary) + +**This SOP = a method spec (how to walk / what calibers / why test must not be peeked at / when to stop), not a turnkey runbook.** A fresh window can independently understand and execute the loop logic from it, but actually producing scores needs the external dependencies below. Both lines' benchmark/infra wiring has its own authoritative runbook: + +### §9.1 Benchmark / infra wiring (authoritative runbooks; this SOP only points) +| | TB2 | LiveCode (EvoAgentBench) | +|---|---|---| +| **Runbook to follow** | `` | `` | +| box | `` | same box | +| Deploy | `` | `` | +| Model/endpoint | ``, `` (rotating, must be in no_proxy) | same model, `NO_PROXY=*` direct | +| split | `s_tb2_train53.txt` / `s_tb2_heldout36.txt` (sealed) | `splits/livecode_task_split.json` train97/test39 | +| Invocation | `` ` ` | `framework/.venv/bin/python src/run.py --config --split --parallel N --job ` | +| vanilla anchor | vanilla baseline arm pass@1 0.440 | C0 train 64.6 / test 58.1 | +| Scoring | `result.json` reward==1.0, excluding exception_info (timeout-fair) | official `lcb_runner` check_correctness | + +### §9.2 Code not yet built (= §8.2; affects "methodological cleanliness", not "can it run") +- ★ **Sharpest:** **the sealed-test runner is still unbuilt** — both lines' current sealing **relies on discipline** (memory feedback constrains to train-only reads), **not mechanism isolation**; §0 itself rules "resisting the urge does not count as sealed — reviewers will challenge it". True sealing requires "script runs test + stores where Claude cannot read". +- `select_anchor()` (with σ_screen) **built and merged** (`evolver/scheduler/anchor_selection.py`); retention computation + generic paired σ also merged into `evolver/analysis/paired_significance.py` (the gate ablation e6 is paper analysis, not loop, staying in the paper repo). §9.2's only genuine gap now is the sealed-test runner (mechanism isolation). + +### §9.3 Must-do before running (once per model x benchmark) +- Cold start produces the `vanilla thick ledger (K=3 full train)` + `failure_map.json` (§1). **Current state (2026-06-24):** + - **Thick ledger ✅ both lines** — TB2 vanilla baseline arm (full-89 K=3, train53 = the screening subset) / LiveCode C0 K=3 train97. + - **Structured failure_map.json ❌ neither line** — currently ad-hoc / signature-classified diagnosis, not §1's `{WHY, tasks, WHERE-hint}` json. Cheap fix: **no re-scoring needed** — one CC SDK pass over the existing vanilla failure trajectories emits it (raw material on the box). The ledger itself only re-runs on a model/benchmark change. +- Environment health precheck (§0 Gate0): 1-task smoke to validate the endpoint (~10k tokens = healthy), sandbox/docker/verifier available. + +### §9.4 Readiness verdict (honest) +- **Can score independently** ✅: both benchmarks wired, splits cut, vanilla anchors in place. +- **Can run a methodologically clean C3** ⚠️: sealed-test runner **deferred** (user decision 2026-06-24); the interim relies on Claude discipline; this is discipline-based sealing — before the paper, either build the mechanism or disclose honestly in limitations (on the books, see §8.2 ⏸). +- **Practice confirms** (LiveCode record §8): the loop = **agent-driven, no state-machine code, scripts only for batch scoring** — consistent with §8.3 "small benchmarks run fine on manual orchestration". diff --git a/pyproject.toml b/pyproject.toml index 3fc59f9..0ac5c2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,6 +220,25 @@ ignore = ["E501"] "scripts/check_large_files.py" = ["S607"] "scripts/skill_hub_e2e.py" = ["S101"] +# evolver + appworld bench: git/bash subprocess drivers by design (S607); +# K is SOP domain vocabulary for trials-per-task (N803/N806); *_PASS / +# *_TOKEN constants are outcome labels and a beacon marker, not credentials +# (S105); asserts tracked with the convert-vs-keep batch above (S101). +"raven/evolver/tree/git_ops.py" = ["S607"] +"raven/evolver/launch/config.py" = ["S607"] +"raven/evolver/launch/contract.py" = ["S607"] +"raven/evolver/launch/runner.py" = ["S607"] +"raven/evolver/orchestrator/nodes/semantic.py" = ["S101"] +"raven/evolver/scheduler/cold_start_bandit.py" = ["N803", "N806"] +"raven/evolver/scheduler/tree_aware_bandit.py" = ["N803"] +"raven/evolver/scheduler/tree_loader.py" = ["S101"] +"raven/evolver/analysis/stability_bucket.py" = ["S105"] +"raven/evolver/applier/beacon_guard.py" = ["S105"] +"raven/evolver/judge/prompts.py" = ["S105"] +"benchmarks/appworld/evolve/adapter.py" = ["N803", "S105"] +"benchmarks/appworld/evolve/editor.py" = ["N806"] +"benchmarks/appworld/evolve/sandbox.py" = ["S607"] + # benchmarks/: eval harness; naming, ambiguous names, unused vars, # partial-path subprocess. "benchmarks/clawbench/stream.py" = ["S607"] diff --git a/raven/evolver/README.md b/raven/evolver/README.md new file mode 100644 index 0000000..9b1a0d0 --- /dev/null +++ b/raven/evolver/README.md @@ -0,0 +1,220 @@ +# raven.evolver — harness self-evolution + +A budget-bounded loop that improves an agent harness against a benchmark: +diagnose failing trajectories, design candidate patches, screen them cheaply, +confirm survivors at K=3, and promote only what beats the baseline through +three verification gates — with a sealed test set for an honest +generalisation number. The methodology is specified in +`docs/specs/self-evolution-loop-sop.md` (the SOP) and mapped to this codebase +in `docs/specs/self-evolution-loop-raven-mapping.md`. + +## Two ways to run self-evolution + +The SOP is the methodology; this package is one executor of it. There are two: + +1. **The evolver pipeline (this package)** — the automated loop below. The + orchestrator executes the whole funnel unattended: diagnosis, candidate + design, screening, K=3 confirmation, the three gates, promotion, and the + sealed-test unseal, with resumability and the guardrails (path whitelist, + fixed denominators, config fingerprint) enforced in code. Use this for + long multi-round runs where the numbers have to be trustworthy. +2. **An agent driving the SOP directly** — hand + `docs/specs/self-evolution-loop-sop.md` to a coding agent (e.g. Claude + Code) inside the subject repo and have it walk the steps itself: read + failing trajectories, pick a WHY, write a candidate as a git commit, run + the benchmark scorer for the K evaluations, and apply the gate arithmetic + by hand. Nothing enforces the protocol — sealed-test discipline, fixed + denominators, and honest gate math are only as good as the agent's + adherence — so treat results as exploratory. It is the fastest way to + prototype on a benchmark that has no `BenchBundle` plugin yet, and what + it learns (WHY taxonomy, whitelist, scorer wiring) feeds directly into + writing one (`docs/specs/evolve-bench-contract.md`). + +The rest of this README covers way 1. + +## Glossary (terms used below) + +- **WHY** — a failure-cause class (e.g. "stops before verifying"); each round + targets one or two WHYs. **WHERE** — the code location a patch touches. +- **K** — attempts per task per evaluation; K=3 confirm is the SOP standard. +- **The funnel** — cheap screen (K=1 on a small anchor task set) before the + expensive full-train K=3 confirmation; most candidates die cheap. +- **Three gates** — Gate-f: enough valid (non-infra) measurements; Gate-b: + the candidate's code actually executed (beacon fired); Gate2: paired + statistical significance vs the baseline on the same tasks. +- **Sealed test** — the test split is scored but stored where no decision + step can read it; opened once, at the end (`retention.json`). +- **BenchBundle** — the plugin object a benchmark implements to become + evolvable (see "Adding a benchmark"). + +## Quickstart + +All commands run from the repo checkout root (they invoke the +`raven.evolver` module; from elsewhere, add `--project /path/to/checkout` +to `uv run`). + +```bash +# 1. Write a run spec (copy docs/examples/evolve_appworld.yaml and edit paths) +# 2. Validate everything cheap — config, models, bench setup, plus one small +# probe completion against the subject endpoint; no trials run: +uv run python -m raven.evolver check --config my_run.yaml + +# 3. Tiny wiring run (isolated _smoke; minutes, not hours): +uv run python -m raven.evolver run --config my_run.yaml --smoke + +# 4. The real run — one command does all three phases: +# cold-start baseline (if missing) -> evolution rounds -> unseal + retention +uv run python -m raven.evolver run --config my_run.yaml +``` + +A real run takes hours to days, and the cost is dominated by **subject-agent +evaluations**, not LLM calls: cold start = train x K trials (e.g. 90 x 3), +plus roughly a full train x K evaluation per candidate that survives +screening. The loop's own calls per round are modest — tens of driver calls +(diagnosis), one design call per candidate (2-6, on your `design` model, +typically the expensive one), a handful of verdict calls. Interrupt freely — +**re-running the same command resumes** from the last durable artifact: +completed trials are never re-run (trial-level idempotency), completed +rounds are replayed from the journal. + +```bash +uv run python -m raven.evolver status --config my_run.yaml # progress; never + # reveals test numbers +uv run python -m raven.evolver finalize --config my_run.yaml --yes + # end now + unseal (one-way) +``` + +What you get at the end: the console summary plus, under `work_dir/`, +`retention.json` (the sealed-test verdict: best round, train/test curve, +retention rate, paired significance), `nodes/*.json` (every candidate's +ledger: git commit, final status, gate stats), `findings.md` (human-readable +per-round log), and the promoted harness as a **real git commit** in your +subject repo. + +## Interruption semantics + +| Interrupted during | What exists | `status` shows | +|---|---|---| +| phase 1 (cold start) | partial baseline trials (kept) | trial count, "no results yet" | +| phase 2 (rounds) | commit list: promoted nodes + train scores | rounds, candidates by status — test stays sealed | +| phase 3 (unseal) | everything | the final report | + +While a run is resumable, test numbers are physically withheld (they sit in a +directory nothing in the decision path reads). `finalize` is the explicit +trade: end the run now, see the result, never resume — the one-way stamp in +`run_meta.json` enforces it. The same file records a config fingerprint: a +run refuses to resume under a changed configuration (candidate and control +arms must stay comparable). + +## Config file + +See `docs/examples/evolve_appworld.yaml` for the annotated schema. The +sections: `bench` (registered benchmark name), `repo_root`/`base_sha` (the +subject repo being evolved and its root commit; omit `base_sha` to use the +repo's current HEAD, resolved once at launch and pinned for the whole run — +uncommitted changes are never part of the root), `models` (the loop's +driver/design/verdict brains — omit entirely to use Raven's own configured +model; note this is distinct from the *subject agent's* model, which is +pinned inside the bench config for the whole run), `funnel` (K values, +per-round budget, termination), `bench_config` (bench-owned), and `smoke` +(overlay applied by `--smoke`). + +## Bootstrap: AppWorld + +The shipped example bench. One-time setup: + +1. Install [AppWorld](https://github.com/StonyBrookNLP/appworld) into a + directory of its own — it is the *subject environment*, kept out of this + repo's uv env. In that directory: create a venv at `appworld-venv/`, + `pip install appworld && appworld install`, then `appworld download data` + — it must end up with a `data/` folder. Point + `bench_config.appworld_data_root` at the directory. +2. Write the subject runtime config JSON (the *benchmarked agent's* model + endpoint — not the loop's models). Copy + `docs/examples/subject_runtime.json` and fill in your endpoint; `check` + validates it against the schema. Point `bench_config.config_path` at it. +3. Put train (and optionally test) task ids in text files, one id per line. + Valid ids come from AppWorld's official splits: + + ```bash + APPWORLD_ROOT= /appworld-venv/bin/python -c \ + "from appworld import load_task_ids; print('\n'.join(load_task_ids('train')))" > train.txt + ``` + + (`APPWORLD_ROOT` matters: appworld resolves `data/` relative to the + current directory otherwise. The bench exports it automatically at run + time, but this manual step happens before any bench code runs.) + + (splits: `train`, `dev`, `test_normal`, `test_challenge`; using `dev` as + the sealed test file is the usual setup). Placeholder ids like + `` are refused at `check` time. +4. `check`, then `run --smoke`, then `run`. For the `smoke:` section's + pinned tasks, any 2-3 train ids work for a wiring test; ids your agent + *fails* make the smoke round exercise the full design path (you learn + which ones from the cold start of a real run, or a quick manual batch). + +## Adding a benchmark + +Implement one `build(ctx) -> BenchBundle` function and register it — the +contract (scorer subprocess, result reader with an infra marker, trajectory +renderer, splits, editable-path whitelist) is documented in +`docs/specs/evolve-bench-contract.md`. `benchmarks/appworld/evolve/entry.py` +is the reference implementation. + +## Documentation map + +| Document | What it is | +|---|---| +| this README | user-facing: quickstart, config, bootstrap, security | +| `docs/specs/self-evolution-loop-sop.md` | the methodology spec (SOP; English translation of the upstream document) | +| `docs/specs/self-evolution-loop-raven-mapping.md` | SOP clause -> Raven code, deviations, unwired parts | +| `docs/specs/evolve-bench-contract.md` | what a new benchmark implements | +| `orchestrator/DESIGN.md` | design notes: inversion of control, SOP<->module table, cross-module conventions | + +## Status and known limitations (2026-07) + +- **Verified:** the unit/e2e suite (126 tests: gate math, the infra-rerun + ladder, git commit/worktree primitives, sandbox whitelist capture, driver + transports, config, guards, idempotency, interrupt -> resume -> finalize + with a fake bench) plus a real AppWorld smoke through the unified entry: cold start -> resume -> one full round + (real diagnose/design, a real candidate commit, focused probe + confirm, + promoted with an honest `credited=False` Gate-b verdict) -> termination. +- **Not yet exercised at scale:** a full-size run (90 tasks x K=3, + multi-round, hours) and the sealed-test unseal/retention path on a real + benchmark have only been exercised via tests, not a production run. +- One bench example ships today (AppWorld, built-in scorer line); a + framework-line example (external scorer) is planned. + +## Security notes + +This system has an LLM **edit and execute code**. Understand the boundaries +before running it anywhere sensitive: + +- Candidate edits are constrained to a per-bench **path whitelist** + (`path_guard` / sandbox capture); everything else the designer touches is + reverted, and each whitelist prefix is validated at startup. On top of the + whitelist, an **immutable kernel** (`applier/path_guard.py`) carves out + the measurement surface — the evolver itself, and for AppWorld the scorer + (`evolve/` including `grade.py`, and `batch.py`) — so a candidate can + rewrite the agent (prompt, loop wiring, tools) but never the code that + grades it or records infra failures. +- **Threat model:** these guards stop an LLM designer from taking shortcuts, + not a determined adversary. Candidate code runs in the same process as + the (immutable-on-disk) grader and the benchmark oracle is reachable over + HTTP, so in-memory tampering or oracle probing is possible in principle. + Promoted candidates are a handful of small commits — **diff-audit them** + before citing numbers (`nodes/*.json` records every commit). +- Candidate code **runs during evaluation** with the same privileges as your + benchmark scorer, and the design-step sandbox is not filesystem/network + jailed. Run evolutions in an isolated environment (container/VM, scoped + credentials), not on a workstation with live secrets. +- Python candidates are required to carry an `activation_beacon()` call, and + credit is only assigned when the beacon actually fired (Gate-b) — this is + an attribution mechanism (presence-level, see the mapping doc), not a + sandbox. +- The default per-round baseline is frozen at cold start (cost-bound); if + your subject endpoint's throughput drifts across a long run, set + `bench_config.baseline_mode: same_session` (~2x eval cost) so control and + candidate arms are always measured in the same window. +- The design step sends your failing trajectories to whatever model you + configure; treat trajectory content accordingly. diff --git a/raven/evolver/__init__.py b/raven/evolver/__init__.py new file mode 100644 index 0000000..eac904a --- /dev/null +++ b/raven/evolver/__init__.py @@ -0,0 +1,34 @@ +"""Harness self-evolution subsystem. + +A budget-bounded loop that improves an agent harness against a benchmark: +diagnose failing trajectories, design candidate patches as real git commits, +screen them cheaply, confirm survivors at K=3, and promote only what beats +the baseline through three verification gates — with a sealed test set for +an honest generalisation number. Methodology: +``docs/specs/self-evolution-loop-sop.md``; user-facing entry point and +quickstart: ``raven/evolver/README.md`` (``python -m raven.evolver``). + +Package layout: + +- ``launch`` — the unified entry: run spec, bench plugin contract, + run/check/status/finalize state machine +- ``orchestrator`` — the SOP round loop: diagnose, design, screen, confirm, + gates, sealed test (see ``orchestrator/DESIGN.md``) +- ``tree`` — git-backed harness version tree (commit-per-candidate) +- ``activation`` — beacon ledger: which candidate code actually fired +- ``applier`` — path whitelist + beacon guards on candidate edits +- ``judge`` — LLM judge with L1/L2/L3 + (WHERE, WHY) output +- ``scheduler`` — task/anchor selection and bandit utilities +- ``analysis`` — trial-ledger readers and stability bucketing +- ``compressor`` — trajectory compression for diagnosis prompts + +Provenance notes for readers of module docstrings: citations of the form +``spec §NN`` refer to the upstream project's internal design document (not +shipped; the shipped methodology spec is +``docs/specs/self-evolution-loop-sop.md``) — the numbers are kept as +provenance for the constants and enums they justify. ``TB2`` names the +upstream terminal-agent benchmark line this system was first developed +against; the example benchmark shipped here is AppWorld. ``GSME`` (Gated +Semantic MAP-Elites) is the quality-diversity elite archive +(``orchestrator/archive.py``). +""" diff --git a/raven/evolver/__main__.py b/raven/evolver/__main__.py new file mode 100644 index 0000000..7b6e9fc --- /dev/null +++ b/raven/evolver/__main__.py @@ -0,0 +1,5 @@ +import sys + +from raven.evolver.cli import main + +sys.exit(main()) diff --git a/raven/evolver/activation/__init__.py b/raven/evolver/activation/__init__.py new file mode 100644 index 0000000..0d984ff --- /dev/null +++ b/raven/evolver/activation/__init__.py @@ -0,0 +1,30 @@ +from raven.evolver.activation.audit import audit_trials +from raven.evolver.activation.chamber import ( + ChamberReport, + Corpus, + load_corpus, + run_chamber, +) +from raven.evolver.activation.ledger import ( + WORKSPACE_ENV, + ActivationLedger, + activation_beacon, + set_activation_workspace, +) +from raven.evolver.activation.routing_query import dry_query +from raven.evolver.activation.spec import ActivationSpec, evaluate_spec + +__all__ = [ + "dry_query", + "ActivationLedger", + "WORKSPACE_ENV", + "activation_beacon", + "set_activation_workspace", + "ActivationSpec", + "evaluate_spec", + "Corpus", + "ChamberReport", + "load_corpus", + "run_chamber", + "audit_trials", +] diff --git a/raven/evolver/activation/audit.py b/raven/evolver/activation/audit.py new file mode 100644 index 0000000..ec032ff --- /dev/null +++ b/raven/evolver/activation/audit.py @@ -0,0 +1,62 @@ +"""Gate 1 post-run audit: did the mechanisms under test actually run? + +Merges activation_ledger.jsonl (hook fires + beacons + presence asserts) +with the pre-existing skill_injections.jsonl telemetry so all four +mechanism classes are counted from one call. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + + +def _iter_jsonl(path: Path): + """Iterate over JSON lines in a file, skipping malformed lines.""" + try: + for line in path.open(): + try: + yield json.loads(line) + except json.JSONDecodeError: + continue + except OSError: + return + + +def audit_trials(roots: list[Path], expected_sources: list[str]) -> dict: + """Audit all trials under given roots for mechanism activation.""" + counts: Counter = Counter() + wired: Counter = Counter() + n_ledgers = 0 + + for root in roots: + for ledger in Path(root).glob("**/activation_ledger.jsonl"): + n_ledgers += 1 + for rec in _iter_jsonl(ledger): + src = rec.get("source", "?") + if rec.get("kind") == "hook_active": + wired[src] += 1 + else: + counts[src] += 1 + + # Skills log only to skill_injections.jsonl and hooks only to the + # ledger - no overlap, so merging the two never double-counts. + for inj in Path(root).glob("**/skill_injections.jsonl"): + for rec in _iter_jsonl(inj): + for s in rec.get("skills", []): + name = s.get("name") + if name: + counts[name] += 1 + + inert = [s for s in expected_sources if counts[s] == 0] + inert_but_wired = [s for s in inert if wired[s] > 0] + return { + "verdict": "FAIL" if inert else "PASS", + "inert_sources": inert, + "inert_but_wired": inert_but_wired, + "counts": {s: counts[s] for s in expected_sources}, + "wired": {s: wired[s] for s in expected_sources}, + "all_observed": dict(counts), + "n_ledgers": n_ledgers, + } diff --git a/raven/evolver/activation/chamber.py b/raven/evolver/activation/chamber.py new file mode 100644 index 0000000..2df9ffd --- /dev/null +++ b/raven/evolver/activation/chamber.py @@ -0,0 +1,76 @@ +"""Pre-flight chamber: replay activation specs over recorded trajectories. + +Corpus note (design section 8): agent trajectories from the uv-contaminated +runs are VALID behavioral data — the contamination was verifier-side. Default +corpus = every trial dir handed in via roots; the report records provenance. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +from raven.evolver.activation.spec import ActivationSpec, evaluate_spec + +SESSION_GLOB = "**/sessions/tb2-task.jsonl" + + +@dataclass +class Corpus: + trajectories: list[list[dict]] + provenance: list[str] = field(default_factory=list) + + +@dataclass +class ChamberReport: + node_id: str + spec_kind: str + reachable_count: int + corpus_size: int + provenance: list[str] + + @property + def verdict(self) -> str: + return "PASS" if self.reachable_count > 0 else "BLOCK" + + def to_dict(self) -> dict: + return { + "node_id": self.node_id, + "spec_kind": self.spec_kind, + "reachable_count": self.reachable_count, + "corpus_size": self.corpus_size, + "provenance": self.provenance, + "verdict": self.verdict, + } + + +def load_corpus(roots: list[Path]) -> Corpus: + trajectories: list[list[dict]] = [] + provenance: list[str] = [] + for root in roots: + provenance.append(str(root)) + for session in sorted(Path(root).glob(SESSION_GLOB)): + traj = [] + try: + for line in session.open(): + try: + traj.append(json.loads(line)) + except json.JSONDecodeError: + continue + except OSError: + continue + if traj: + trajectories.append(traj) + return Corpus(trajectories=trajectories, provenance=provenance) + + +def run_chamber(node_id: str, spec: ActivationSpec, corpus: Corpus) -> ChamberReport: + count = evaluate_spec(spec, corpus.trajectories) + return ChamberReport( + node_id=node_id, + spec_kind=spec.kind, + reachable_count=count, + corpus_size=len(corpus.trajectories), + provenance=corpus.provenance, + ) diff --git a/raven/evolver/activation/ledger.py b/raven/evolver/activation/ledger.py new file mode 100644 index 0000000..103bce6 --- /dev/null +++ b/raven/evolver/activation/ledger.py @@ -0,0 +1,124 @@ +"""Per-trial activation ledger + beacon. + +Every runtime activation event (hook fire, skill injection mirror, code +beacon, presence assert) appends one JSON line to +``/activation_ledger.jsonl``. Writes are best-effort: a ledger +failure must never affect the trial. + +``activation_beacon`` is the mandatory one-liner inside every evolved +code path (design section 3, code class). It resolves the workspace from +``RAVEN_ACTIVATION_WORKSPACE`` and silently no-ops when unset, so +product runtime never pays for it. +""" + +from __future__ import annotations + +import json +import os +import time +from contextvars import ContextVar +from pathlib import Path + +LEDGER_FILENAME = "activation_ledger.jsonl" +WORKSPACE_ENV = "RAVEN_ACTIVATION_WORKSPACE" + +_workspace_var: ContextVar[str | None] = ContextVar("raven_activation_workspace", default=None) + + +def set_activation_workspace(workspace: "Path | str"): + """Bind the activation workspace to the current asyncio context. + + Called once per trial by the benchmark harness. Child tasks inherit + the binding, so concurrent trials in one process do not cross-write + (the process-global env var cannot guarantee that). + """ + return _workspace_var.set(str(workspace)) + + +class ActivationLedger: + def __init__(self, workspace: Path | str): + self._path = Path(workspace) / LEDGER_FILENAME + + def record(self, *, kind: str, source: str, detail: dict | None = None) -> None: + try: + with open(self._path, "a") as f: + f.write( + json.dumps( + { + "ts": time.time(), + "kind": kind, + "source": source, + "detail": detail or {}, + } + ) + + "\n" + ) + except Exception: + pass + + +def activation_beacon(node_id: str, site: str = "", **detail: object) -> None: + workspace = _workspace_var.get() or os.environ.get(WORKSPACE_ENV) + if not workspace: + return + d: dict = dict(detail) + if site: + d["site"] = site + ActivationLedger(workspace).record(kind="beacon", source=node_id, detail=d) + + +# ---- per-task collection (the Gate-b read-back side) ------------------------- + +BEACONS_DIRNAME = "beacons" +ENABLED_MARKER = ".enabled" + + +def beacon_workspace(out_dir: "Path | str", task_id: str, k: int) -> Path: + """The canonical per-attempt beacon workspace under an eval out-dir. + + The batch runner points ``WORKSPACE_ENV`` here for each task-attempt + subprocess, so beacon lines land pre-split by task; the reader below + globs the same layout. Keep writer and reader on this one function. + """ + return Path(out_dir) / BEACONS_DIRNAME / f"{task_id}_k{k}" + + +def mark_beacons_enabled(out_dir: "Path | str") -> None: + """Drop the collection marker distinguishing "instrumentation ran, nothing + fired" (an honest zero) from "collection was never wired" (no data).""" + root = Path(out_dir) / BEACONS_DIRNAME + try: + root.mkdir(parents=True, exist_ok=True) + (root / ENABLED_MARKER).touch() + except OSError: + pass + + +def read_fired_tasks(out_dirs: "list[Path | str]", task_ids: "list[str]") -> "set[str] | None": + """Which of ``task_ids`` have at least one beacon line under any of the + given eval out-dirs (the confirm dir plus its infra-rerun ladder siblings). + + Returns None when NO out-dir carries the collection marker — no + instrumentation data means Gate-b must fail OPEN (skip attribution), not + reject everything. With the marker present, an empty set is an honest + "the mechanism never fired anywhere" and Gate-b correctly credits nothing. + """ + roots = [Path(d) / BEACONS_DIRNAME for d in out_dirs] + if not any((r / ENABLED_MARKER).exists() for r in roots): + return None + fired: set[str] = set() + for root in roots: + if not root.is_dir(): + continue + for tid in task_ids: + if tid in fired: + continue + for ws in root.glob(f"{tid}_k*"): + lf = ws / LEDGER_FILENAME + try: + if lf.is_file() and lf.stat().st_size > 0: + fired.add(tid) + break + except OSError: + continue + return fired diff --git a/raven/evolver/activation/predicates.py b/raven/evolver/activation/predicates.py new file mode 100644 index 0000000..e3767c5 --- /dev/null +++ b/raven/evolver/activation/predicates.py @@ -0,0 +1,73 @@ +"""Single source of truth for mechanism trigger predicates. + +Runtime hooks and the evolver's activation-spec evaluators import THESE +functions, so chamber preflight predictions and live hook behavior can +not drift (round-1 incidents C1/C3). Each predicate takes a normalized +record: a dict with 'content' (str), 'tool_calls' (list), optionally +'role'. Hooks normalize live response objects via normalize_response(); +the corpus evaluator feeds logged session records directly (same shape). +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +_THINK_RE = re.compile(r".*?", re.S) + + +def normalize_response(response: Any) -> dict: + if isinstance(response, dict): + return {"content": str(response.get("content") or ""), "tool_calls": response.get("tool_calls") or []} + return { + "content": str(getattr(response, "content", None) or ""), + "tool_calls": getattr(response, "tool_calls", None) or [], + } + + +def is_empty_response(rec: dict) -> bool: + """Truly dead iteration: no visible content AND no tool calls.""" + return not rec.get("content", "").strip() and not rec.get("tool_calls") + + +def visible_reasoning_len(rec: dict) -> int: + """Length of think-stripped, whitespace-collapsed content.""" + stripped = _THINK_RE.sub("", rec.get("content", "")) + return len(" ".join(stripped.split())) + + +def is_short_toolcall_iteration(rec: dict, max_chars: int = 80) -> bool: + """Tool-call iteration whose visible reasoning is below max_chars.""" + return bool(rec.get("tool_calls")) and visible_reasoning_len(rec) < max_chars + + +def command_head(rec: dict) -> str | None: + """Head token of the actual shell command (from tool_calls arguments), + falling back to assistant prose only when no tool call is present. + + The exec-style tool carries the command in tool_calls[0].function.arguments + (a JSON string or dict) under a 'command' key. Reading the command head + here (not the prose head) is what makes repeated_failure_run and the + forced-replan family-detection key on real command families rather than + on prose openers like 'Let' (C3 round-2 improvement).""" + tcs = rec.get("tool_calls") or [] + if tcs: + tc = tcs[0] + fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) + args = None + if isinstance(fn, dict): + args = fn.get("arguments") + elif fn is not None: + args = getattr(fn, "arguments", None) + if isinstance(args, str): + try: + args = json.loads(args) + except (ValueError, TypeError): + args = None + if isinstance(args, dict): + cmd = str(args.get("command") or "").strip() + if cmd: + return cmd.split()[0] + c = rec.get("content", "").strip() + return c.split()[0] if c else None diff --git a/raven/evolver/activation/routing_query.py b/raven/evolver/activation/routing_query.py new file mode 100644 index 0000000..68db68b --- /dev/null +++ b/raven/evolver/activation/routing_query.py @@ -0,0 +1,83 @@ +"""dry_query — offline probe of the real skill discovery + selection path. + +Round-4 forensics found that a custom skill authored on disk under +``skill_library/tb2_gap_fill/`` was never injected: the benchmark agent +constructs ``SkillForgeConfig(enabled=False)`` with empty ``local_dirs``, so +the directory is never mounted as a discovery layer and ``select()`` returns +``[]`` before any retrieval runs. ``dry_query`` answers, without an LLM or the +SR server, "which skill names would routing inject for this task?" by building +a real :class:`LocalSkillCatalog` + :class:`SkillForgeRouter` whose +``local_dirs`` point at ``library_root`` and running the actual BM25 retrieval ++ resolve path. + +No LLM is involved: the LLM gate and query rewriter are disabled, so selection +reduces to filesystem discovery + lexical (BM25) scoring — the deterministic +core of the real path that the benchmark must wire up. +""" + +from __future__ import annotations + +import asyncio +import tempfile +from pathlib import Path + +__all__ = ["dry_query"] + + +def dry_query(task_text: str, *, library_root: Path | None = None) -> list[str]: + """Return the skill names routing would inject for ``task_text``. + + Args: + task_text: The task description fed to skill selection. + library_root: Root dir mounted as an extra discovery layer (the + recursive ``SKILL.md`` scan walks its subtree, so + ``.../skill_library`` surfaces ``tb2_gap_fill//SKILL.md``). + ``None`` exercises the default layers only (workspace + builtin). + + Returns: + Flat list of skill names (``SkillMeta.name``) routing would inject, + in injection order. This mirrors the benchmark's two injection + surfaces (ContextBuilder.build_system_prompt): the ``always: true`` + skills rendered under "Active Skills", followed by the retrieval + ``select()`` hits, deduped by name. + """ + from raven.config.raven import LocalDirConfig, SkillForgeConfig + + # Raven split the old unified SkillService into a discovery catalog + # (always-skills + registry/pool) and a retrieval router over sources. + from raven.memory_engine.skill_forge.catalog import LocalSkillCatalog + from raven.memory_engine.skill_forge.local_source import LocalSkillSource + from raven.memory_engine.skill_forge.router import SkillForgeRouter + + local_dirs: list[LocalDirConfig] = [] + if library_root is not None: + local_dirs.append(LocalDirConfig(path=str(Path(library_root)), name="tb2_gap_fill")) + + config = SkillForgeConfig( + enabled=True, + local_dirs=local_dirs, + llm_gate_enabled=False, + rewrite_enabled=False, + reranker_enabled=False, + disable_always=False, + ) + + with tempfile.TemporaryDirectory() as ws: + catalog = LocalSkillCatalog( + Path(ws), + config=config, + llm_provider=None, + start_watcher=False, + ) + always = catalog.get_always_skills() + router = SkillForgeRouter([LocalSkillSource(catalog.pool, catalog.registry)]) + selected = asyncio.run(router.select(task_text, [])) + + names: list[str] = [] + seen: set[str] = set() + for meta in [*always, *selected]: + if meta.name in seen: + continue + seen.add(meta.name) + names.append(meta.name) + return names diff --git a/raven/evolver/activation/spec.py b/raven/evolver/activation/spec.py new file mode 100644 index 0000000..01b2cd7 --- /dev/null +++ b/raven/evolver/activation/spec.py @@ -0,0 +1,262 @@ +"""activation_spec — a node's machine-checkable "when do I take effect". + +Kinds (one per mechanism class, design section 3): + +- ``trajectory_regex`` code class. Counts trajectories containing a line + matching ``pattern`` within ``scope`` records. + (Delta-3's "pure cd" predicate is this kind.) +- ``consecutive_repeat`` hook class, repetition-trigger family. Counts + trajectories with >= ``threshold`` consecutive + identical ``scope`` contents. Empty/whitespace + contents are skipped by default (``ignore_empty: false`` + to count them). Other hook trigger families express as + trajectory_regex or get a new kind + evaluator here. +- ``short_content_run`` hook class, reasoning-visibility family. Counts + trajectories with >= ``threshold`` consecutive + tool-call iterations whose visible assistant content + is shorter than ``max_chars`` (default 80). Uses the + shared predicate ``is_short_toolcall_iteration`` from + raven.evolver.activation.predicates, the same function the + ReasoningVisibilityHook calls at runtime: only records + carrying tool_calls count; a record WITHOUT tool_calls + (or a long-content one) resets the run. Content is + measured after stripping ``...`` blocks + and collapsing whitespace. Approximation: the hook + reads the live ``response.content``; over recorded + sessions we read the ``content`` field of the assistant + record (Qwen's chain-of-thought lands in a separate + ``reasoning_content`` field the hook does not read). +- ``empty_run`` hook class, response-quality family. Counts + trajectories with >= ``threshold`` consecutive empty + iterations. Uses the shared predicate + ``is_empty_response`` (raven.evolver.activation.predicates), + the same the EmptyRunBreakerHook calls: an iteration is + empty iff its content is blank AND it carries no + tool_calls. A blank-content record that still issues a + tool call is NOT empty (round-1 incident C1: the old + content-only predicate predicted 64.8% reachable while + the tool_calls-respecting hook fired 0). Any non-empty + iteration resets the run. +- ``repeated_failure_run`` hook class, robustness family. Counts trajectories + with >= ``threshold`` consecutive failures of the same + command (same head token). Walks raw records: assistant + content sets a pending head token (first whitespace-split + token); tool record resolves it (failed iff exit code + is nonzero); same head + fail grows the run; success + or head-change resets it. Tool records without an + "Exit code:" line count as success (conservative: + under-counts reachability, never over-counts). + ``scope`` is ignored (the kind inherently walks + assistant and tool records). +- ``min_iterations`` hook class, budget family. Counts trajectories with + >= ``threshold`` assistant iterations — the wrap-up + nudge trigger is an iteration-count crossing, + structurally drift-free (no semantic predicate). +- ``skill_routing`` skill class. Reachability is answered by a routing + dry-query, not by corpus replay — the chamber + delegates; evaluate_spec rejects it. +- ``presence`` always-on class. Answered by offline render/config + assert in the preflight CLI; evaluate_spec rejects + it likewise. + +``evaluate_spec(spec, corpus) -> int`` returns HOW MANY trajectories the +spec is reachable in. 0 = the mechanism would never run = block. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from raven.evolver.activation.predicates import ( + command_head, + is_empty_response, + is_short_toolcall_iteration, +) + +_CORPUS_KINDS = { + "trajectory_regex", + "consecutive_repeat", + "short_content_run", + "empty_run", + "repeated_failure_run", + "min_iterations", +} +_KNOWN_KINDS = _CORPUS_KINDS | {"skill_routing", "presence"} + +_DEFAULT_MAX_CHARS = 80 + + +def _normalize_record(r: dict) -> dict: + """Coerce a logged session record into the predicate record shape + (content as str, tool_calls as list) so the shared predicate functions + see the same view the hooks build via normalize_response().""" + return {"content": str(r.get("content") or ""), "tool_calls": r.get("tool_calls") or []} + + +@dataclass +class ActivationSpec: + kind: str + raw: dict = field(default_factory=dict) + + @classmethod + def from_dict(cls, d: dict) -> "ActivationSpec": + kind = d.get("kind") + if not kind: + raise ValueError("activation_spec requires a 'kind' field") + if kind not in _KNOWN_KINDS: + raise ValueError(f"unknown activation_spec kind: {kind!r}") + if kind == "trajectory_regex" and "pattern" not in d: + raise ValueError("trajectory_regex requires 'pattern'") + if kind == "trajectory_regex": + try: + re.compile(d["pattern"]) + except re.error as exc: + raise ValueError(f"trajectory_regex pattern is not valid regex: {exc}") from exc + if kind == "consecutive_repeat" and "threshold" not in d: + raise ValueError("consecutive_repeat requires 'threshold'") + if kind == "consecutive_repeat": + try: + int(d["threshold"]) + except (TypeError, ValueError) as exc: + raise ValueError(f"consecutive_repeat threshold must be an int: {d['threshold']!r}") from exc + if kind == "short_content_run" and "threshold" not in d: + raise ValueError("short_content_run requires 'threshold'") + if kind == "short_content_run": + try: + int(d["threshold"]) + except (TypeError, ValueError) as exc: + raise ValueError(f"short_content_run threshold must be an int: {d['threshold']!r}") from exc + if "max_chars" in d: + try: + int(d["max_chars"]) + except (TypeError, ValueError) as exc: + raise ValueError(f"short_content_run max_chars must be an int: {d['max_chars']!r}") from exc + if kind == "empty_run" and "threshold" not in d: + raise ValueError("empty_run requires 'threshold'") + if kind == "empty_run": + try: + int(d["threshold"]) + except (TypeError, ValueError) as exc: + raise ValueError(f"empty_run threshold must be an int: {d['threshold']!r}") from exc + if kind == "repeated_failure_run" and "threshold" not in d: + raise ValueError("repeated_failure_run requires 'threshold'") + if kind == "repeated_failure_run": + try: + int(d["threshold"]) + except (TypeError, ValueError) as exc: + raise ValueError(f"repeated_failure_run threshold must be an int: {d['threshold']!r}") from exc + if kind == "min_iterations" and "threshold" not in d: + raise ValueError("min_iterations requires 'threshold'") + if kind == "min_iterations": + try: + int(d["threshold"]) + except (TypeError, ValueError) as exc: + raise ValueError(f"min_iterations threshold must be an int: {d['threshold']!r}") from exc + if kind == "skill_routing" and "skill_name" not in d: + raise ValueError("skill_routing requires 'skill_name'") + if kind == "presence" and "needle" not in d: + raise ValueError("presence requires 'needle'") + raw = {k: v for k, v in d.items() if k != "kind"} + return cls(kind=kind, raw=raw) + + @property + def pattern(self) -> str: + return self.raw["pattern"] + + @property + def threshold(self) -> int: + return int(self.raw["threshold"]) + + @property + def max_chars(self) -> int: + return int(self.raw.get("max_chars", _DEFAULT_MAX_CHARS)) + + +def _scope_contents(traj: list[dict], scope: str) -> list[str]: + return [str(r.get("content") or "") for r in traj if scope in ("any", r.get("role"))] + + +def evaluate_spec(spec: ActivationSpec, corpus: list[list[dict]]) -> int: + if spec.kind not in _CORPUS_KINDS: + raise ValueError(f"{spec.kind} is not corpus-evaluable; use the preflight CLI path") + scope = spec.raw.get("scope", "assistant") + hits = 0 + if spec.kind == "trajectory_regex": + pat = re.compile(spec.pattern, re.M) + for traj in corpus: + if any(pat.search(c) for c in _scope_contents(traj, scope)): + hits += 1 + elif spec.kind == "consecutive_repeat": + threshold = spec.threshold + ignore_empty = spec.raw.get("ignore_empty", True) + for traj in corpus: + run, prev = 0, object() + for c in _scope_contents(traj, scope): + if ignore_empty and not c.strip(): + # Empty responses carry no command; a repetition + # trigger comparing commands never sees them. + continue + run = run + 1 if c == prev else 1 + prev = c + if run >= threshold: + hits += 1 + break + elif spec.kind == "short_content_run": + threshold = spec.threshold + max_chars = spec.max_chars + for traj in corpus: + run = 0 + fired = False + for r in traj: + if scope not in ("any", r.get("role")): + continue + rec = _normalize_record(r) + run = run + 1 if is_short_toolcall_iteration(rec, max_chars) else 0 + if run >= threshold: + fired = True + break + if fired: + hits += 1 + elif spec.kind == "empty_run": + threshold = spec.threshold + for traj in corpus: + run = 0 + for r in traj: + if scope not in ("any", r.get("role")): + continue + rec = _normalize_record(r) + run = run + 1 if is_empty_response(rec) else 0 + if run >= threshold: + hits += 1 + break + elif spec.kind == "repeated_failure_run": + threshold = spec.threshold + exit_re = re.compile(r"Exit code: (\d+)") + for traj in corpus: + run, prev_head = 0, None + pending_head = None + for r in traj: + role, c = r.get("role"), str(r.get("content") or "") + if role == "assistant" and c.strip(): + pending_head = command_head(_normalize_record(r)) + elif role == "tool" and pending_head is not None: + m = exit_re.search(c) + failed = bool(m and m.group(1) != "0") + if failed and pending_head == (prev_head or pending_head): + run += 1 + prev_head = pending_head + else: + run = 1 if failed else 0 + prev_head = pending_head if failed else None + pending_head = None + if run >= threshold: + hits += 1 + break + elif spec.kind == "min_iterations": + threshold = spec.threshold + for traj in corpus: + count = sum(1 for r in traj if r.get("role") == "assistant") + if count >= threshold: + hits += 1 + return hits diff --git a/raven/evolver/analysis/__init__.py b/raven/evolver/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/raven/evolver/analysis/failure_map_builder.py b/raven/evolver/analysis/failure_map_builder.py new file mode 100644 index 0000000..69a47f0 --- /dev/null +++ b/raven/evolver/analysis/failure_map_builder.py @@ -0,0 +1,243 @@ +"""Aggregate judge results into a ``failure_map.json``. + +Consumes a list of :class:`JudgeResult` objects (the output of the +cold-start coverage bandit's ``claude_judge(trial)`` calls) and produces +a single structured aggregation file. This file is the bridge between +spec §14 step ③ (cold-start coverage) and step ④ (first evolution round): + +- **Coverage check** — does the judge output cover at least + ``min_why_classes=7`` of the WHY pathology axes (spec §14 step ③ + acceptance gate)? +- **L1 routing** — surface L1 alerts as ``human_review_needed`` items + (evolver pauses, engineer fixes infrastructure). +- **L2 / L3 cell aggregation** — group patch proposals by + ``(PatchWhere, PatchWhy)`` cell so the evolver can scan a single + dict to find candidate patches for a chosen pathology. +- **WHERE / WHY marginals** — counts per axis, useful for paper + §15 Must-nail #1 diversity plots. + +JSON layout (``schema_version = "1.0"``):: + + { + "schema_version": "1.0", + "n_total_judged": 25, + "n_l1": 3, + "n_l2": 12, + "n_l3": 10, + "covered_why_classes": ["budget_awareness", ...], + "covered_why_count": 7, + "coverage_satisfied": true, + "min_why_classes_target": 7, + "l1_alerts": [ + { + "trajectory_id": "...", + "signal_description": "...", + "reasoning": "...", + "confidence": 0.9 + } + ], + "cells": { + "hook_new::budget_awareness": { + "n_candidates": 3, + "trajectory_ids": ["...", ...], + "candidates": [ + { + "trajectory_id": "...", + "issue_type": "L3", + "confidence": 0.85, + "components": [ + {"component_id": "comp_1", "target_file": "...", + "summary": "...", "depends_on": []} + ], + "reasoning": "..." + } + ] + }, + ... + }, + "where_distribution": {"hook_new": 5, "skill": 3, ...}, + "why_distribution": {"budget_awareness": 4, ...} + } + +Cell keys use ``"::"`` (double-colon separator, no nesting) +so the file remains valid JSON and grep-able from shell. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +from raven.evolver.judge.schema import ( + IssueType, + JudgeResult, + PatchWhere, + PatchWhy, +) + +SCHEMA_VERSION = "1.0" +DEFAULT_MIN_WHY_CLASSES = 7 # spec §14 step ③ acceptance gate + + +def build_failure_map( + judge_results: Iterable[JudgeResult], + *, + min_why_classes: int = DEFAULT_MIN_WHY_CLASSES, +) -> dict[str, Any]: + """Aggregate a list of ``JudgeResult`` into the failure_map dict. + + Parameters + ---------- + judge_results + Iterable of JudgeResult — usually from claude judge call on + cold-start bandit's sampled trials. + min_why_classes + Target WHY class coverage. Default 7 (spec §14 ③). The + resulting ``coverage_satisfied`` reflects whether the judge + output reached this bar. + + Returns + ------- + dict + Structured ``failure_map`` ready for ``json.dump``. + """ + results = list(judge_results) + + cells: dict[str, dict[str, Any]] = defaultdict(lambda: {"n_candidates": 0, "trajectory_ids": [], "candidates": []}) + l1_alerts: list[dict[str, Any]] = [] + where_distribution: dict[str, int] = defaultdict(int) + why_distribution: dict[str, int] = defaultdict(int) + covered_why: set[str] = set() + + n_l1 = n_l2 = n_l3 = 0 + for r in results: + if r.issue_type == IssueType.L1: + n_l1 += 1 + l1_alerts.append( + { + "trajectory_id": r.trajectory_id, + "signal_description": r.signal_description, + "reasoning": r.proposed_action.reasoning, + "confidence": r.confidence, + } + ) + continue + + # L2 / L3: must have patch_proposal (schema invariant enforces this) + if r.issue_type == IssueType.L2: + n_l2 += 1 + else: + n_l3 += 1 + + action = r.proposed_action + where = action.patch_where + why = action.patch_why + if where is None or why is None: + # Defensive: schema __post_init__ should have caught this + continue + + where_key = where.value + # patch_why_extra carries the full sub-name including the + # "other:" prefix per schema convention (see PatchWhy.other docstring). + why_key = why.value if why != PatchWhy.other else (action.patch_why_extra or "other:unknown") + where_distribution[where_key] += 1 + why_distribution[why_key] += 1 + covered_why.add(why_key) + + cell_key = f"{where_key}::{why_key}" + cells[cell_key]["n_candidates"] += 1 + cells[cell_key]["trajectory_ids"].append(r.trajectory_id) + cells[cell_key]["candidates"].append( + { + "trajectory_id": r.trajectory_id, + "issue_type": r.issue_type.value, + "confidence": r.confidence, + "reasoning": action.reasoning, + "components": [c.to_dict() for c in action.components], + } + ) + + coverage_count = len(covered_why) + return { + "schema_version": SCHEMA_VERSION, + "n_total_judged": len(results), + "n_l1": n_l1, + "n_l2": n_l2, + "n_l3": n_l3, + "covered_why_classes": sorted(covered_why), + "covered_why_count": coverage_count, + "min_why_classes_target": min_why_classes, + "coverage_satisfied": coverage_count >= min_why_classes, + "l1_alerts": l1_alerts, + "cells": dict(cells), + "where_distribution": dict(where_distribution), + "why_distribution": dict(why_distribution), + } + + +def write_failure_map( + failure_map: dict[str, Any], + out_path: str | Path, + *, + indent: int = 2, +) -> None: + """Atomically write the failure_map dict to ``out_path``. + + Uses temp file + rename for crash safety (same pattern as + ``evolver/tree/store.py``). + """ + out_path = Path(out_path) + tmp = out_path.with_suffix(out_path.suffix + ".tmp") + tmp.write_text(json.dumps(failure_map, indent=indent, sort_keys=True)) + tmp.replace(out_path) + + +def coverage_gap(failure_map: dict[str, Any]) -> list[str]: + """Return WHY enum values not yet covered by the judge output. + + Useful for diagnostic / decision logic in the loop: if returns + non-empty, the cold-start bandit hasn't satisfied spec §14 step + ③ — either rerun bandit with bigger budget or accept partial + coverage and proceed. + + Note: this checks against the **first-class** ``PatchWhy`` enum + values (excluding ``other``). ``other:*`` sub-names in the judge + output do contribute to coverage_count but don't fill in the + canonical WHY axis — the gap result names which canonical + classes are still missing. + """ + covered = set(failure_map.get("covered_why_classes", [])) + canonical = {w.value for w in PatchWhy if w != PatchWhy.other} + return sorted(canonical - covered) + + +def candidates_for_cell( + failure_map: dict[str, Any], + where: PatchWhere | str, + why: PatchWhy | str, +) -> list[dict[str, Any]]: + """Return the ``candidates`` list for a given ``(WHERE, WHY)`` cell. + + Returns ``[]`` if the cell is empty or absent. Accepts either enum + instances or the underlying string values. + """ + where_key = where.value if isinstance(where, PatchWhere) else where + why_key = why.value if isinstance(why, PatchWhy) else why + cell_key = f"{where_key}::{why_key}" + cell = failure_map.get("cells", {}).get(cell_key) + if cell is None: + return [] + return list(cell.get("candidates", [])) + + +__all__ = [ + "SCHEMA_VERSION", + "DEFAULT_MIN_WHY_CLASSES", + "build_failure_map", + "write_failure_map", + "coverage_gap", + "candidates_for_cell", +] diff --git a/raven/evolver/analysis/proxy_features.py b/raven/evolver/analysis/proxy_features.py new file mode 100644 index 0000000..93703af --- /dev/null +++ b/raven/evolver/analysis/proxy_features.py @@ -0,0 +1,303 @@ +"""Per-trial cheap metadata extraction for downstream stratification. + +Reads a legacy-runner trial dir and emits a :class:`ProxyFeatures` dataclass per +trial. The features are cheap to compute (parse one session.jsonl + one +result.json, no replay, no LLM, no container) and stable enough to feed +into the cold-start bandit's K-means sub-strata on the ``stable_fail`` 0/3 +bucket — where ~70% of tasks land under v7 and where the bandit needs to +slice the population for cohort selection. + +Feature set (all per-trial, all O(session.jsonl)): + +- ``turn_count`` — assistant turns with at least one tool call +- ``final_exit_status`` — categorical (:class:`ExitStatus`) +- ``has_tool_calls_ever`` — bool: did the agent ever invoke a tool +- ``assistant_text_length_avg``— mean length (chars) of assistant.content across all assistant messages +- ``docker_error_count`` — count of docker-error patterns across tool responses + exception traceback + +The docker-error count is a noise indicator (container-side issues that +surface as docker daemon errors during exec). It is intentionally loose +since per-trial occurrences should be rare; spikes indicate infra issues +worth filtering out before bandit stratification. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +__all__ = [ + "ExitStatus", + "ProxyFeatures", + "extract_features", + "extract_trial_dir", +] + + +class ExitStatus(str, Enum): + """Outcome of a trial as observed at the harness level. + + ``PASSED`` / ``FAILED_VERIFIER`` are the normal exit modes; the rest + encode different failure modes the bandit may want to filter out + (e.g. wall-cap-bound tasks shouldn't be in the same K-means cluster + as agent-decision-bound tasks). + """ + + PASSED = "passed" + FAILED_VERIFIER = "failed_verifier" + AGENT_TIMEOUT = "agent_timeout" + VERIFIER_TIMEOUT = "verifier_timeout" + REWARD_FILE_NOT_FOUND = "reward_file_not_found" + RUNTIME_ERROR = "runtime_error" + NO_SESSION = "no_session" + OTHER = "other" + + +@dataclass(frozen=True) +class ProxyFeatures: + trial_id: str + task_id: str + turn_count: int + final_exit_status: ExitStatus + has_tool_calls_ever: bool + assistant_text_length_avg: float + docker_error_count: int + + +# Docker / container daemon errors most commonly seen inside exec tool output +# or in the runner's exception traceback when something blows up at the infra layer. +_DOCKER_ERROR_PATTERNS = re.compile( + r"(?:" + r"docker:\s+error\b" + r"|cannot\s+connect\s+to\s+the\s+docker\s+daemon" + r"|error\s+response\s+from\s+daemon" + r"|no\s+such\s+container" + r"|container\s+not\s+running" + r"|docker\.errors\." + r")", + re.IGNORECASE, +) + + +_EXCEPTION_TO_STATUS = { + "AgentTimeoutError": ExitStatus.AGENT_TIMEOUT, + "VerifierTimeoutError": ExitStatus.VERIFIER_TIMEOUT, + "RewardFileNotFoundError": ExitStatus.REWARD_FILE_NOT_FOUND, + "RuntimeError": ExitStatus.RUNTIME_ERROR, +} + + +def _classify_exit(result_json: dict, reward_passed: bool | None) -> ExitStatus: + """Map result.json's exception_info / reward.txt into an ExitStatus.""" + if reward_passed is True: + return ExitStatus.PASSED + exc = (result_json.get("exception_info") or {}).get("exception_type") + if exc in _EXCEPTION_TO_STATUS: + return _EXCEPTION_TO_STATUS[exc] + if reward_passed is False: + return ExitStatus.FAILED_VERIFIER + # no reward and no recognised exception → bucket as OTHER + if exc: + return ExitStatus.OTHER + return ExitStatus.NO_SESSION + + +def _read_reward(trial_dir: Path) -> bool | None: + rt = trial_dir / "verifier" / "reward.txt" + if not rt.exists(): + return None + try: + return float(rt.read_text().strip()) >= 1.0 + except (ValueError, OSError): + return None + + +def _read_result_json(trial_dir: Path) -> dict: + rj = trial_dir / "result.json" + if not rj.exists(): + return {} + try: + return json.loads(rj.read_text()) + except (json.JSONDecodeError, OSError): + return {} + + +def _session_path(trial_dir: Path) -> Path | None: + """Locate the agent's session.jsonl file under a trial dir.""" + sessions_dir = trial_dir / "agent" / "workspace" / "sessions" + if not sessions_dir.is_dir(): + return None + # the legacy runner writes a single tb2-task.jsonl by convention; fall back to + # any .jsonl if name differs. + preferred = sessions_dir / "tb2-task.jsonl" + if preferred.exists(): + return preferred + candidates = sorted(sessions_dir.glob("*.jsonl")) + return candidates[0] if candidates else None + + +def _scan_session(session_path: Path) -> tuple[int, bool, float, int]: + """Walk session.jsonl once and emit + ``(turn_count, has_tool_calls_ever, assistant_text_length_avg, docker_error_count)``. + + Iteration is a single pass over the file; each line is parsed once + and three counters are updated. Returns 0/False/0.0/0 for an empty + or unreadable session. + """ + turn_count = 0 + has_tool_calls_ever = False + assistant_lengths: list[int] = [] + docker_errors = 0 + + try: + with session_path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + role = r.get("role") + if role == "assistant": + content = r.get("content") or "" + assistant_lengths.append(len(content)) + if r.get("tool_calls"): + turn_count += 1 + has_tool_calls_ever = True + elif role == "tool": + content = r.get("content") or "" + docker_errors += len(_DOCKER_ERROR_PATTERNS.findall(content)) + except OSError: + pass + + avg_len = (sum(assistant_lengths) / len(assistant_lengths)) if assistant_lengths else 0.0 + return turn_count, has_tool_calls_ever, avg_len, docker_errors + + +def _trial_task_id(trial_name: str) -> str: + return trial_name.rsplit("__", 1)[0] if "__" in trial_name else trial_name + + +def extract_features(trial_dir: str | Path) -> ProxyFeatures: + """Extract :class:`ProxyFeatures` from a single trial dir. + + Raises :class:`FileNotFoundError` when ``trial_dir`` does not exist + or is not a directory. + """ + p = Path(trial_dir) + if not p.is_dir(): + raise FileNotFoundError(p) + + result = _read_result_json(p) + reward_passed = _read_reward(p) + + session = _session_path(p) + if session is None: + turn_count = 0 + has_tool_calls_ever = False + avg_len = 0.0 + docker_errors_session = 0 + else: + turn_count, has_tool_calls_ever, avg_len, docker_errors_session = _scan_session(session) + + # also scan exception traceback for docker errors + exc_tb = (result.get("exception_info") or {}).get("exception_traceback") or "" + docker_errors = docker_errors_session + len(_DOCKER_ERROR_PATTERNS.findall(exc_tb)) + + final = _classify_exit(result, reward_passed) + + return ProxyFeatures( + trial_id=p.name, + task_id=_trial_task_id(p.name), + turn_count=turn_count, + final_exit_status=final, + has_tool_calls_ever=has_tool_calls_ever, + assistant_text_length_avg=avg_len, + docker_error_count=docker_errors, + ) + + +def _find_attempt_root(trial_dir: Path) -> Path: + """Same logic as stability_bucket: discriminate by ``verifier/`` subdir.""" + if not trial_dir.is_dir(): + raise NotADirectoryError(trial_dir) + has_trial_children = any(p.is_dir() and "__" in p.name and (p / "verifier").is_dir() for p in trial_dir.iterdir()) + if has_trial_children: + return trial_dir + nested = [p for p in trial_dir.iterdir() if p.is_dir()] + if len(nested) == 1: + return nested[0] + return trial_dir + + +def extract_trial_dir(trial_dir: str | Path) -> dict[str, ProxyFeatures]: + """Extract :class:`ProxyFeatures` for every trial under ``trial_dir``. + + Accepts either the legacy jobs_dir or the dated subdir; returns + ``{trial_id: ProxyFeatures}``. + """ + root = _find_attempt_root(Path(trial_dir)) + out: dict[str, ProxyFeatures] = {} + for d in sorted(root.iterdir()): + if not d.is_dir() or "__" not in d.name: + continue + if not (d / "result.json").exists(): + continue + if not (d / "verifier").is_dir(): + continue + out[d.name] = extract_features(d) + return out + + +def main(argv: list[str] | None = None) -> int: + import argparse + from collections import Counter + + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--trial-dir", required=True, help="legacy jobs_dir or dated subdir") + ap.add_argument("--json", default=None, help="optional JSON dump path") + args = ap.parse_args(argv) + + feats = extract_trial_dir(args.trial_dir) + by_status = Counter(f.final_exit_status.value for f in feats.values()) + + print(f"trial_dir: {args.trial_dir}") + print(f"trials observed: {len(feats)}") + print("\nExit status breakdown:") + for status, n in sorted(by_status.items(), key=lambda x: -x[1]): + print(f" {status:24s} {n}") + + print("\nTurn count quantiles (incl. wall-cap-zero):") + turns = sorted(f.turn_count for f in feats.values()) + if turns: + for q, p in (("min", 0.0), ("p25", 0.25), ("p50", 0.5), ("p75", 0.75), ("max", 1.0)): + idx = min(len(turns) - 1, int(p * len(turns))) + print(f" {q}: {turns[idx]}") + + if args.json: + with open(args.json, "w") as f: + json.dump( + { + tid: { + "task_id": x.task_id, + "turn_count": x.turn_count, + "final_exit_status": x.final_exit_status.value, + "has_tool_calls_ever": x.has_tool_calls_ever, + "assistant_text_length_avg": x.assistant_text_length_avg, + "docker_error_count": x.docker_error_count, + } + for tid, x in sorted(feats.items()) + }, + f, + indent=2, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/raven/evolver/analysis/stability_bucket.py b/raven/evolver/analysis/stability_bucket.py new file mode 100644 index 0000000..b888b0c --- /dev/null +++ b/raven/evolver/analysis/stability_bucket.py @@ -0,0 +1,183 @@ +"""Per-task k-attempts stability bucketing for paired baseline trial dirs. + +Reads a legacy-runner trial dir (e.g. ``data/v7_k3_baseline//``) where each +task has up to ``k`` independent attempts and emits a per-task stability +classification used downstream by the cold-start bandit's task-cohort +stratification. + +Buckets (for k=3): + +- ``STABLE_PASS`` = passed in every attempt (e.g. 3/3) +- ``BORDERLINE_2_3`` = passed in (k-1) of k attempts (e.g. 2/3) +- ``BORDERLINE_1_3`` = passed in 1 attempt +- ``STABLE_FAIL`` = 0 pass in any attempt + +Pass criterion: ``verifier/reward.txt`` exists and reads as ``>= 1.0``. +Trials with no reward (e.g. ``RewardFileNotFoundError`` / wall-clock +``AgentTimeoutError`` / ``VerifierTimeoutError``) count as FAIL. + +The k value is inferred from the data — a task may have < k attempts if +some failed pre-trial; the bucket label reflects fractional pass count +over the attempts actually observed (so a 1/2 task with k=3 nominal still +gets bucketed as ``BORDERLINE_1_3`` in that grouping). +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + + +class StabilityBucket(str, Enum): + STABLE_PASS = "stable_pass" + BORDERLINE_2_3 = "borderline_2_3" + BORDERLINE_1_3 = "borderline_1_3" + STABLE_FAIL = "stable_fail" + + +@dataclass(frozen=True) +class TaskStability: + task_id: str + attempts: int + passes: int + bucket: StabilityBucket + + +def _trial_passed(trial_dir: Path) -> bool: + """Return True iff ``verifier/reward.txt`` exists and reads ``>= 1.0``.""" + reward = trial_dir / "verifier" / "reward.txt" + if not reward.exists(): + return False + try: + return float(reward.read_text().strip()) >= 1.0 + except (ValueError, OSError): + return False + + +def _bucket_for(passes: int, attempts: int) -> StabilityBucket: + """Classify a (passes, attempts) tuple into a StabilityBucket. + + Designed for k in {2, 3}. For arbitrary k, ``passes == 0`` is always + ``STABLE_FAIL`` and ``passes == attempts`` is always ``STABLE_PASS``; + in-between gets the BORDERLINE_2_3 / BORDERLINE_1_3 split by which + side of the midpoint the pass count falls on. + """ + if passes == 0: + return StabilityBucket.STABLE_FAIL + if passes == attempts: + return StabilityBucket.STABLE_PASS + # mixed: split between "mostly pass" and "mostly fail" + if passes * 2 > attempts: + return StabilityBucket.BORDERLINE_2_3 + return StabilityBucket.BORDERLINE_1_3 + + +def _task_id(trial_name: str) -> str: + """Extract canonical task id from a legacy trial dir name. + + Layout: ``{task-id}__{8-char-suffix}``. We strip the suffix. + """ + sep = "__" + if sep in trial_name: + return trial_name.rsplit(sep, 1)[0] + return trial_name + + +def _looks_like_trial_dir(p: Path) -> bool: + """A legacy trial dir always carries a top-level ``result.json`` AND + a ``verifier/`` subdir; the job-level dated dir has ``result.json`` + too but no ``verifier/`` of its own, so the verifier check is what + discriminates a trial dir from the job dir. + """ + return p.is_dir() and "__" in p.name and (p / "result.json").exists() and (p / "verifier").is_dir() + + +def _find_attempt_root(trial_dir: Path) -> Path: + """Locate the directory whose children are the per-trial dirs. + + Accepts either the legacy jobs_dir (e.g. ``data/v7_k3_baseline/`` + which contains a dated subdir) or the dated subdir itself. + Both dated dirs and trial dirs use ``__`` in their names, so we + discriminate by the presence of ``result.json``. + """ + if not trial_dir.is_dir(): + raise NotADirectoryError(trial_dir) + if any(_looks_like_trial_dir(p) for p in trial_dir.iterdir()): + return trial_dir + nested = [p for p in trial_dir.iterdir() if p.is_dir()] + if len(nested) == 1: + return nested[0] + return trial_dir + + +def compute_stability(trial_dir: str | Path) -> dict[str, TaskStability]: + """Aggregate k-attempts pass counts per task and assign a bucket. + + Returns mapping ``{task_id: TaskStability}``. + """ + root = _find_attempt_root(Path(trial_dir)) + per_task_passes: dict[str, list[bool]] = defaultdict(list) + for d in sorted(root.iterdir()): + if not d.is_dir(): + continue + if "__" not in d.name: + continue + task_id = _task_id(d.name) + per_task_passes[task_id].append(_trial_passed(d)) + + result: dict[str, TaskStability] = {} + for task_id, passes in per_task_passes.items(): + n = len(passes) + k = sum(passes) + result[task_id] = TaskStability( + task_id=task_id, + attempts=n, + passes=k, + bucket=_bucket_for(k, n), + ) + return result + + +def bucket_counts(stability: Iterable[TaskStability]) -> dict[StabilityBucket, int]: + """Tally how many tasks fall into each bucket.""" + counts = {b: 0 for b in StabilityBucket} + for ts in stability: + counts[ts.bucket] += 1 + return counts + + +def main(argv: list[str] | None = None) -> int: + import argparse + + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--trial-dir", required=True, help="legacy jobs_dir or dated subdir") + ap.add_argument("--json", default=None, help="optional JSON dump path") + args = ap.parse_args(argv) + + stab = compute_stability(args.trial_dir) + counts = bucket_counts(stab.values()) + + print(f"trial_dir: {args.trial_dir}") + print(f"tasks observed: {len(stab)}") + for b in StabilityBucket: + print(f" {b.value:18s} {counts[b]}") + + if args.json: + with open(args.json, "w") as f: + json.dump( + { + tid: {"attempts": ts.attempts, "passes": ts.passes, "bucket": ts.bucket.value} + for tid, ts in sorted(stab.items()) + }, + f, + indent=2, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/raven/evolver/analysis/trial_pool.py b/raven/evolver/analysis/trial_pool.py new file mode 100644 index 0000000..afc7dca --- /dev/null +++ b/raven/evolver/analysis/trial_pool.py @@ -0,0 +1,120 @@ +"""Trial pool construction — adapter between trial dir + analysis modules. + +Wires together the per-task stability output from +:func:`raven.evolver.analysis.stability_bucket.compute_stability` and +the per-trial features from +:func:`raven.evolver.analysis.proxy_features.extract_trial_dir` +into the unified :class:`Trial` view consumed by +:class:`raven.evolver.scheduler.cold_start_bandit.ColdStartCoverageBandit`. + +A trial dir layout (k=3 paired baseline) maps to **89 task × 3 attempts +≈ 267 Trial objects**. All attempts of a single task share the same +task-level ``stability`` bucket; each trial carries its own per-trial +``proxy_features`` dict, projected to numeric-only entries so the +bandit's K-means sub-strata clustering can compute Euclidean distance +without categorical-encoding overhead. + +Categorical ``ExitStatus`` is mapped through a stable ordinal table +(``_EXIT_STATUS_ORDINAL``) so K-means distance is deterministic across +Python runs (Python's built-in ``hash()`` honours ``PYTHONHASHSEED`` +and isn't reliable for paper reproducibility). +""" + +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path + +from raven.evolver.analysis.proxy_features import ( + ExitStatus, + ProxyFeatures, + extract_trial_dir, +) +from raven.evolver.analysis.stability_bucket import ( + compute_stability, +) +from raven.evolver.scheduler.cold_start_bandit import Trial + +_EXIT_STATUS_ORDINAL: dict[ExitStatus, int] = { + ExitStatus.PASSED: 0, + ExitStatus.FAILED_VERIFIER: 1, + ExitStatus.AGENT_TIMEOUT: 2, + ExitStatus.VERIFIER_TIMEOUT: 3, + ExitStatus.REWARD_FILE_NOT_FOUND: 4, + ExitStatus.RUNTIME_ERROR: 5, + ExitStatus.NO_SESSION: 6, + ExitStatus.OTHER: 7, +} + + +def proxy_features_to_kmeans_dict(pf: ProxyFeatures) -> dict[str, float]: + """Project a :class:`ProxyFeatures` into the numeric dict the bandit + feeds into K-means for stable_fail sub-strata clustering. + + All entries are floats so the K-means impl in + ``cold_start_bandit._kmeans_strata`` can min-max normalise them + uniformly. The ``exit_status_ordinal`` slot maps the categorical + :class:`ExitStatus` through :data:`_EXIT_STATUS_ORDINAL` — distance + isn't semantically meaningful across enum values, but it's stable + and surfaces "this trial behaves like that one" via co-occurrence + in cluster centroids. + """ + return { + "turn_count": float(pf.turn_count), + "has_tool_calls_ever": 1.0 if pf.has_tool_calls_ever else 0.0, + "assistant_text_length_avg": float(pf.assistant_text_length_avg), + "docker_error_count": float(pf.docker_error_count), + "exit_status_ordinal": float(_EXIT_STATUS_ORDINAL.get(pf.final_exit_status, 99)), + } + + +def build_trial_pool(trial_dir: str | Path) -> list[Trial]: + """Construct a list of :class:`Trial` from a legacy-runner trial dir. + + Walks the trial dir once via the existing analysis modules, + groups trials by task to assign per-task stability + deterministic + attempt indices (1..k), then materialises each trial with both + its task-level stability bucket and its per-trial proxy-feature + K-means dict. + + Returns the list sorted by ``(task_id, attempt)`` for + repr-friendliness; ``ColdStartCoverageBandit`` shuffles the + borderline pool internally under its own RNG seed regardless. + """ + trial_dir = Path(trial_dir) + stability_map = compute_stability(trial_dir) + features_map = extract_trial_dir(trial_dir) + + # Group ProxyFeatures by task for attempt numbering + by_task: dict[str, list[ProxyFeatures]] = defaultdict(list) + for pf in features_map.values(): + by_task[pf.task_id].append(pf) + for tid in by_task: + by_task[tid].sort(key=lambda p: p.trial_id) + + trials: list[Trial] = [] + for task_id in sorted(by_task.keys()): + task_stability = stability_map.get(task_id) + if task_stability is None: + # Defensive: shouldn't happen since both modules walk the + # same dir, but skip silently rather than crash on edge case + continue + for attempt_idx, pf in enumerate(by_task[task_id], start=1): + passed = pf.final_exit_status == ExitStatus.PASSED + trials.append( + Trial( + trial_id=pf.trial_id, + task_id=task_id, + attempt=attempt_idx, + passed=passed, + stability=task_stability.bucket, + proxy_features=proxy_features_to_kmeans_dict(pf), + ) + ) + return trials + + +__all__ = [ + "build_trial_pool", + "proxy_features_to_kmeans_dict", +] diff --git a/raven/evolver/applier/__init__.py b/raven/evolver/applier/__init__.py new file mode 100644 index 0000000..93e7f0a --- /dev/null +++ b/raven/evolver/applier/__init__.py @@ -0,0 +1,40 @@ +"""Evolver patch-application machinery. + +Owns the **commit-side** of evolver patches: given a candidate +``AppliedPatch`` produced by the judge, decide whether it can be applied +to the working tree, and if so, write it out. + +Currently exposes: + +- :mod:`path_guard` — first-line immutability check; rejects patches that + target kernel paths (spec §22 + §22.7). +- :mod:`beacon_guard` — code-class patch validation; rejects patches without + activation beacons before eval (design section 3). + +Future C2 will add a Git-backed atomic apply that uses ``path_guard`` +internally to short-circuit before any commit is created. +""" + +from .beacon_guard import ( + MissingBeaconError, + assert_beacon_present, +) +from .path_guard import ( + IMMUTABLE_PATTERNS, + MUTABLE_OVERRIDES, + ImmutablePathError, + assert_patch_allowed, + check_patch_paths, + is_immutable, +) + +__all__ = [ + "IMMUTABLE_PATTERNS", + "MUTABLE_OVERRIDES", + "ImmutablePathError", + "assert_patch_allowed", + "check_patch_paths", + "is_immutable", + "MissingBeaconError", + "assert_beacon_present", +] diff --git a/raven/evolver/applier/beacon_guard.py b/raven/evolver/applier/beacon_guard.py new file mode 100644 index 0000000..a532bab --- /dev/null +++ b/raven/evolver/applier/beacon_guard.py @@ -0,0 +1,51 @@ +"""Reject code-class patches that carry no activation beacon. + +Design section 3 (code class): every evolved code path must call +``activation_beacon(node_id)`` so Gate 1 can prove the path executed. +A diff with no beacon is unmonitorable — rejected before eval. +""" + +from __future__ import annotations + +# WHERE classifies by behavioral target, not file location: a direct edit +# to benchmark agent code (e.g. termination policy) classifies as +# loop_override / tool_override / context_override by what it changes, +# so no separate "benchmark_agent_code" value exists. +CODE_CLASS_WHERES = { + "tool_new", + "loop_override", + "context_override", + "tool_override", +} +BEACON_TOKEN = "activation_beacon(" + + +class MissingBeaconError(ValueError): + """Raised when a code-class patch lacks an activation beacon.""" + + pass + + +def assert_beacon_present( + node_id: str, + *, + patch_where: str, + diff_text: str, +) -> None: + """Validate that a code-class patch contains an activation beacon. + + Args: + node_id: The node ID (used in error messages). + patch_where: The patch location (e.g. "loop_override", "skill"). + diff_text: The unified diff or code text to check. + + Raises: + MissingBeaconError: If patch_where is a code class and diff_text + contains no activation_beacon() call. + """ + if patch_where not in CODE_CLASS_WHERES: + return + if BEACON_TOKEN not in diff_text: + raise MissingBeaconError( + f"{node_id}: {patch_where} patch contains no {BEACON_TOKEN}...) call - unmonitorable, rejected" + ) diff --git a/raven/evolver/applier/path_guard.py b/raven/evolver/applier/path_guard.py new file mode 100644 index 0000000..8bb821a --- /dev/null +++ b/raven/evolver/applier/path_guard.py @@ -0,0 +1,225 @@ +"""Path guard for evolver-generated patches (spec §22 + §22.7). + +The immutable set is **evolver-immutable**, not human-immutable. +Human-driven development (PR review) can still touch these paths +through normal workflow; only auto-evolver patches are blocked. + +Two pattern types are supported in ``IMMUTABLE_PATTERNS``: + +- **Exact file** — e.g. ``"raven/agent/loop/main.py"``. Matches + exactly this repo-relative path. +- **Directory subtree** — trailing-slash, e.g. ``"raven/evolver/"``. + Matches the directory itself and any descendant path. + +``MUTABLE_OVERRIDES`` takes precedence: a path matching an override is +mutable even if it also matches an immutable pattern. Use this to carve +out mutable sub-trees from a broader immutable directory. + +Why repo-relative-string matching instead of full glob: +This module's job is fast, predictable, easy-to-audit gating. Glob +patterns (``**``, ``*.py``) invite surprises (case-sensitivity, slash +behaviour). Exact paths + directory subtrees cover everything in +§22.2 without ambiguity. + +Reference layers (spec §22.1, §22.2): + L1 — Self-reference (evolver/**) + L2 — Evaluation substrate (eval_engine + external grader) + L3 — Capability contract (agent loop, tools framework, providers, + skill loader, sandbox, config schema, ...) + L4 — Audit / data integrity (tool_audit_hook) + L5 — Tests, deps, CI +""" + +from __future__ import annotations + +# --------------------------------------------------------------------------- +# Immutable pattern list (spec §22.2) +# --------------------------------------------------------------------------- + + +# All paths are repo-relative with forward-slash separators. +# A trailing slash means "this directory and everything inside it". +IMMUTABLE_PATTERNS: tuple[str, ...] = ( + # ── L1 — Self-reference ──────────────────────────────────────────────── + "raven/evolver/", + # ── L2 — Evaluation substrate ────────────────────────────────────────── + "raven/eval_engine/engine.py", + "raven/eval_engine/adapter/", + "raven/eval_engine/judge/", + # The AppWorld evolve glue (adapter/eval/diagnose/editor/precheck, plus + # grade.py — the /evaluate call, success/infra classification and result + # write) scores candidates; it sits INSIDE the designer whitelist tree, + # so the guard must carve it out (maps the upstream evaluation/ entry). + "benchmarks/appworld/evolve/", + # The batch scorer orchestrates trials and records runner-level infra; + # a candidate that can edit it can reshape its own denominator. The + # editable agent surface is agent_cli.py (loop/prompt) and tool.py. + "benchmarks/appworld/batch.py", + # ── L3 — Capability contract ─────────────────────────────────────────── + "raven/agent/loop/main.py", + "raven/agent/context/", + "raven/agent/tools/base.py", + "raven/agent/tools/registry.py", + "raven/agent/personalizer/", + "raven/agent/subagent/", + # Raven replaced the upstream agent/api with the spine: the runner is the + # bridge every turn goes through, same contract level. + "raven/agent/spine_runner.py", + "raven/spine/", + "raven/providers/", + "raven/session/", + "raven/memory_engine/", + "raven/skill_hub/", + "raven/context_engine/", + "raven/sandbox/", + "raven/security/", + # config: schema/loader (.py) are immutable; values (.yaml/.json) mutable + "raven/config/__init__.py", + "raven/config/raven.py", + "raven/config/loader.py", + "raven/config/paths.py", + "raven/config/schema.py", + "raven/config/update.py", + "raven/config/update_channels.py", + "raven/config/update_providers.py", + # ── L4 — Audit / data integrity ──────────────────────────────────────── + "raven/eval_engine/hooks/tool_audit_hook.py", + # ── L5 — Tests, deps, CI ─────────────────────────────────────────────── + "tests/", + "pyproject.toml", + "uv.lock", + ".github/", + # ── Core version (spec §22.5) — only humans bump ─────────────────────── + "raven/__init__.py", +) + + +# Carve-outs for paths inside immutable subtrees that should remain +# mutable. Empty at the moment — no carve-outs needed under the current +# §22.2 layout. Add an entry like ``"raven/sandbox/policies.yaml"`` +# here if a specific file inside an immutable directory needs to evolve. +MUTABLE_OVERRIDES: tuple[str, ...] = () + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +class ImmutablePathError(ValueError): + """Raised when an evolver patch targets an immutable kernel path. + + See spec §22 for the evolver-immutable kernel definition. If a + legitimate patch is being blocked, two options: + + 1. Reframe the patch — split off the mutable part, drop the + immutable part. This is what `path_guard` expects. + 2. Walk the human-driven development path: open a PR that + updates ``MUTABLE_OVERRIDES`` (if the path was misclassified) + or that modifies the kernel directly (then bump ``core_version`` + per spec §22.5). + """ + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _normalise(path: str) -> str: + """Normalise a path to repo-relative, forward-slash form. + + - Windows-style backslashes are converted to forward slashes + - A leading ``./`` is stripped (callers sometimes pass it) + + Callers are expected to pass repo-relative paths; absolute paths + will not match any pattern (and therefore be reported as mutable). + Validating that is the caller's responsibility. + """ + p = path.replace("\\", "/") + if p.startswith("./"): + p = p[2:] + return p + + +def _match(path: str, pattern: str) -> bool: + """Match a normalised path against one pattern. + + Two pattern shapes: + + - Trailing slash → directory subtree match (the directory itself + and any descendant). + - Otherwise → exact-string match. + """ + if pattern.endswith("/"): + prefix = pattern.rstrip("/") + return path == prefix or path.startswith(pattern) + return path == pattern + + +# --------------------------------------------------------------------------- +# Predicates +# --------------------------------------------------------------------------- + + +def is_immutable(path: str) -> bool: + """Return True iff ``path`` is in the evolver-immutable kernel. + + Algorithm: + + 1. Normalise the path (handle Windows paths + leading ``./``). + 2. If any pattern in ``MUTABLE_OVERRIDES`` matches → return False + (override wins). + 3. If any pattern in ``IMMUTABLE_PATTERNS`` matches → return True. + 4. Otherwise → return False (path is mutable by default). + """ + norm = _normalise(path) + for pat in MUTABLE_OVERRIDES: + if _match(norm, pat): + return False + for pat in IMMUTABLE_PATTERNS: + if _match(norm, pat): + return True + return False + + +def check_patch_paths(target_files: list[str]) -> list[str]: + """Return the subset of ``target_files`` that hit immutable paths. + + Useful for evolver code that wants to inspect violations without + raising — e.g., to route the patch to a TODO markdown + (spec §21.4.2) instead of attempting to apply it. + """ + return [p for p in target_files if is_immutable(p)] + + +def assert_patch_allowed(target_files: list[str]) -> None: + """Raise :class:`ImmutablePathError` if any target is immutable. + + Use this as the first-line gate in the evolver applier: + + .. code-block:: python + + from raven.evolver.applier import assert_patch_allowed + assert_patch_allowed([c.target_file for c in patch.components]) + # ... proceed to apply patch only if no error + """ + offenders = check_patch_paths(target_files) + if offenders: + head = offenders[:5] + more = f" ... and {len(offenders) - 5} more" if len(offenders) > 5 else "" + raise ImmutablePathError( + f"Patch targets {len(offenders)} evolver-immutable path(s): " + f"{head}{more}. The gates, ledgers, and eval glue may never be " + "edited by a candidate (see IMMUTABLE_PATTERNS in this module)." + ) + + +__all__ = [ + "IMMUTABLE_PATTERNS", + "MUTABLE_OVERRIDES", + "ImmutablePathError", + "assert_patch_allowed", + "check_patch_paths", + "is_immutable", +] diff --git a/raven/evolver/cli.py b/raven/evolver/cli.py new file mode 100644 index 0000000..e58a07b --- /dev/null +++ b/raven/evolver/cli.py @@ -0,0 +1,59 @@ +"""``python -m raven.evolver`` — the unified self-evolution entry point. + +run cold start -> rounds -> unseal, resumable at any interruption +check validate config / models / bench setup without running anything +status inspect progress (never reveals sealed test numbers) +finalize end the run now and unseal (one-way) +""" + +from __future__ import annotations + +import argparse +import sys + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="raven.evolver", + description="Run the SOP self-evolution loop on a registered benchmark.", + ) + sub = p.add_subparsers(dest="command", required=True) + + def common(sp: argparse.ArgumentParser) -> None: + sp.add_argument("--config", required=True, help="run spec YAML") + sp.add_argument("--smoke", action="store_true", help="shrunk verification run in _smoke") + + run = sub.add_parser("run", help="start or resume an evolution run") + common(run) + run.add_argument("--force", action="store_true", help="override the unseal / config-drift guards") + + check = sub.add_parser("check", help="validate config/models/bench setup, run nothing") + common(check) + + status = sub.add_parser("status", help="show run progress (sealed-safe)") + common(status) + + fin = sub.add_parser("finalize", help="terminate now and unseal (one-way)") + common(fin) + fin.add_argument("--yes", action="store_true", help="confirm ending the run") + + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + from raven.evolver.launch import runner + + if args.command == "run": + return runner.cmd_run(args.config, smoke=args.smoke, force=args.force) + if args.command == "check": + return runner.cmd_check(args.config, smoke=args.smoke) + if args.command == "status": + return runner.cmd_status(args.config, smoke=args.smoke) + if args.command == "finalize": + return runner.cmd_finalize(args.config, smoke=args.smoke, yes=args.yes) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/raven/evolver/compressor/__init__.py b/raven/evolver/compressor/__init__.py new file mode 100644 index 0000000..c13a1f2 --- /dev/null +++ b/raven/evolver/compressor/__init__.py @@ -0,0 +1,22 @@ +"""Trajectory compression — session.jsonl → ~10K-token diagnostic summary. + +Used by the judge LLM client (B3) when ``trajectory_format="compressed"`` +is requested. The compressor is rule-based (no LLM call), so it's cheap +and deterministic. +""" + +from .trajectory import ( + CompressorConfig, + Event, + TrajectoryCompressor, + estimate_tokens, + load_session_jsonl, +) + +__all__ = [ + "CompressorConfig", + "Event", + "TrajectoryCompressor", + "estimate_tokens", + "load_session_jsonl", +] diff --git a/raven/evolver/compressor/trajectory.py b/raven/evolver/compressor/trajectory.py new file mode 100644 index 0000000..9874a87 --- /dev/null +++ b/raven/evolver/compressor/trajectory.py @@ -0,0 +1,496 @@ +"""Trajectory compressor — session.jsonl → ~10K-token diagnostic summary. + +The judge LLM (especially the cheap L1-detection backend in mix mode) +can't afford to ingest a full 80-200K-token SWE-bench trajectory per +analysis. This module reduces the raw event stream to a structured +"agent debugger" style overview that preserves the high-signal portions +and elides the bulky tool outputs. + +What we **keep verbatim**: + +- Task description (first user message — `WRAPPER_PATH` + STRICT RULES + + issue text from the external scorer). +- Every assistant text content (the model's reasoning chain). +- Tool call names + truncated arguments (enough to identify what tool + was called and what it operated on). + +What we **truncate**: + +- Tool results — keep first ``head_chars`` + last ``tail_chars``, elide + the middle with a marker. Long file dumps, stack traces, and `cat` + output mostly fall into this bucket. + +What we **collapse**: + +- Runs of identical (tool_name, args-prefix) calls are emitted once + with a "× N times" marker. Highly repetitive trajectories (the + pager-stuck / tmux-poll / re-read pathologies from the 244-paired + analysis) compress dramatically here. + +What we **flag**: + +- Anomaly section appended at the end: empty-content turn count, syntax + errors, docker errors, repetition density. These are exactly the + signals the L1 detector watches for. + +The compression is **purely rule-based** — no LLM call. A v2 could add +an LLM summarisation pass over the rule-compressed output, but the v1 +already typically lands a 150K trajectory at ~5-15K tokens with the +defaults. + +Token counting uses a rough ``chars / 4`` heuristic; we don't need a +tokenizer dependency for budget shaping at this granularity. +""" + +from __future__ import annotations + +import json +import re +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Optional, Union + +# --------------------------------------------------------------------------- +# Event model +# --------------------------------------------------------------------------- + + +@dataclass +class Event: + """One parsed line from an external scorer session.jsonl. + + Fields are kept loose because the scorer emits a few distinct event + shapes (metadata, chat messages, tool results). Callers branch on + ``role`` / ``event_type``. + + ``finish_reason`` is captured for assistant turns when the upstream + row carries it (LiteLLM / OpenAI-shape responses). It is the + primary discriminator between L1 and L2 calibration in the judge + prompt (spec §22.5 + r4 Fix A1): ``stop``/``content_filter`` on + empty content → L1, ``length`` → L2 (max_tokens config issue). + """ + + event_type: str # "metadata" | "system" | "user" | "assistant" | "tool" | "other" + content: Optional[str] = None + tool_calls: list[dict[str, Any]] = field(default_factory=list) + tool_name: Optional[str] = None + finish_reason: Optional[str] = None + raw: dict[str, Any] = field(default_factory=dict) + + +def _parse_event(obj: dict[str, Any]) -> Event: + """Classify one raw JSON object from session.jsonl into an Event. + + The classification rules mirror what we observed in the 244-paired + SWE-bench session files: + + - ``{"_type": "metadata", ...}`` → metadata + - ``{"role": "user" | "assistant" | "system" | "tool", ...}`` → chat + - anything else → "other" (will be ignored downstream) + """ + if "_type" in obj: + return Event(event_type="metadata", raw=obj) + role = obj.get("role") + if role in ("system", "user", "assistant", "tool"): + content = obj.get("content") + # Some scorer/litellm rows put content as a list of blocks + # (multimodal); we flatten to a single string for compression + # purposes (we don't compress images). + if isinstance(content, list): + parts = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + parts.append(part.get("text", "")) + elif isinstance(part, str): + parts.append(part) + content = "\n".join(parts) if parts else None + elif content is not None and not isinstance(content, str): + content = str(content) + # finish_reason only meaningful on assistant rows (per OpenAI/LiteLLM + # shape); for other roles we leave it None even if the field is + # present in raw. We tolerate both a top-level key (scorer + # session.jsonl flattens it) and a nested ``choices[0].finish_reason`` + # (raw LiteLLM response shape some adapters preserve). + finish_reason: Optional[str] = None + if role == "assistant": + fr = obj.get("finish_reason") + if isinstance(fr, str) and fr: + finish_reason = fr + else: + choices = obj.get("choices") + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + nested = first.get("finish_reason") + if isinstance(nested, str) and nested: + finish_reason = nested + return Event( + event_type=role, + content=content, + tool_calls=list(obj.get("tool_calls") or []), + tool_name=obj.get("name"), + finish_reason=finish_reason, + raw=obj, + ) + return Event(event_type="other", raw=obj) + + +def load_session_jsonl(path: Union[str, Path]) -> list[Event]: + """Read a session.jsonl file into a list of :class:`Event`. + + Lines that are blank or fail to parse are silently skipped — scorer + has been seen to emit the occasional malformed line on crash, and we + don't want one bad line to abort a 200-turn trajectory. + """ + p = Path(path) + events: list[Event] = [] + with p.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + events.append(_parse_event(obj)) + return events + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + + +@dataclass +class CompressorConfig: + """Knobs for the rule-based compression pass. + + Defaults are tuned so a typical 80-200K-token SWE-bench trajectory + lands at 5-15K tokens — well within budget for the L1-detection + backend (Qwen-397B has 128K context, Claude Haiku has 200K, so + even the "compressed" output is still small relative to context). + """ + + target_tokens: int = 10000 # not a hard cap, soft target + tool_args_chars: int = 200 # truncate tool call args at this length + tool_result_head_chars: int = 200 # keep first N chars of long tool results + tool_result_tail_chars: int = 200 # keep last N chars of long tool results + user_message_max_chars: int = 4000 # task description rarely needs more + repetition_run_threshold: int = 3 # collapse a run of ≥N identical calls + detect_anomalies: bool = True + + +# --------------------------------------------------------------------------- +# Anomaly markers (used by both compression body + final summary) +# --------------------------------------------------------------------------- + + +_SYNTAX_ERROR_PATTERNS = [ + re.compile(r"\bSyntax(\s+)?Error\b", re.IGNORECASE), + re.compile(r"Unterminated quoted string", re.IGNORECASE), + re.compile(r"unexpected EOF", re.IGNORECASE), +] + +_DOCKER_ERROR_PATTERNS = [ + re.compile(r"docker daemon", re.IGNORECASE), + re.compile(r"connection refused", re.IGNORECASE), + re.compile(r"container .* not running", re.IGNORECASE), + re.compile(r"\bOOM\b", re.IGNORECASE), +] + +_NETWORK_ERROR_PATTERNS = [ + re.compile(r"read timeout", re.IGNORECASE), + re.compile(r"pool exhausted", re.IGNORECASE), + re.compile(r"URLError", re.IGNORECASE), +] + + +def _matches_any(text: str, patterns: list[re.Pattern]) -> bool: + return any(p.search(text) for p in patterns) + + +# --------------------------------------------------------------------------- +# Compressor +# --------------------------------------------------------------------------- + + +class TrajectoryCompressor: + """Rule-based compressor — turns Event list into a single text blob.""" + + def __init__(self, config: Optional[CompressorConfig] = None) -> None: + self._cfg = config or CompressorConfig() + + def compress(self, events: Iterable[Event]) -> str: + """Produce a single compressed text block for one trajectory. + + The result is plain text with a structured layout the judge + prompt is designed to handle: header → task → turns → anomaly + summary. No JSON, no Markdown — just headings and indented + body. Keeps the trajectory format token-efficient. + """ + events_list = list(events) + cfg = self._cfg + + out: list[str] = [] + # Header — we'll fill turn / anomaly counts at the end + header_placeholder_idx = len(out) + out.append("") # placeholder for header + + # Task description: take the first non-metadata user message + task = self._find_task_description(events_list) + if task: + out.append("--- TASK ---") + truncated = task[: cfg.user_message_max_chars] + if len(task) > cfg.user_message_max_chars: + truncated += f"\n... [truncated, {len(task) - cfg.user_message_max_chars} more chars]" + out.append(truncated) + + # Turns (assistant + tool_result pairs) + out.append("\n--- TURNS ---") + turn_idx = 0 + empty_content_count = 0 + i = 0 + while i < len(events_list): + ev = events_list[i] + if ev.event_type != "assistant": + i += 1 + continue + turn_idx += 1 + # Empty content detection — L1 signal. ``finish_reason`` is + # the discriminator between L1 (stop / content_filter) and + # L2 (length budget too small for the model) — see Fix A1 + # in judge prompt. + fr_tag = f", finish_reason={ev.finish_reason}" if ev.finish_reason else "" + asst_text = (ev.content or "").strip() + if not asst_text and not ev.tool_calls: + empty_content_count += 1 + out.append(f"\nTurn {turn_idx} (assistant): [EMPTY content + no tool_calls{fr_tag}]") + i += 1 + continue + header_suffix = f" [finish_reason={ev.finish_reason}]" if ev.finish_reason else "" + out.append(f"\nTurn {turn_idx} (assistant){header_suffix}:") + if asst_text: + out.append(self._indent(asst_text, " ")) + elif ev.tool_calls: + # Tool-using turn without narrative text is NORMAL agent + # behaviour (model chose to act instead of explain), NOT + # an empty-content anomaly. Do NOT increment empty_content + # _count — the L1 trigger must reserve for completely + # silent turns (no content AND no tool_calls), matching + # EmptyResponseAlertHook.is_empty_response. + out.append(f" [tool-only turn, no narrative{fr_tag}]") + # Find immediately-following tool calls + results, with + # collapse of identical runs. + i += 1 + i = self._emit_tool_section(events_list, i, out, turn_idx) + + # Anomaly summary + anomaly_block = [] + if cfg.detect_anomalies: + anomaly_block = self._compute_anomaly_summary(events_list, empty_content_count) + if anomaly_block: + out.append("\n--- ANOMALIES DETECTED ---") + out.extend(anomaly_block) + + # Final summary line up top, now that we know turn count + out[header_placeholder_idx] = ( + f"=== TRAJECTORY SUMMARY ===\n" + f"Total turns: {turn_idx} | Empty-content turns: {empty_content_count} " + f"| Anomalies flagged: {len(anomaly_block)}" + ) + + return "\n".join(out) + + # -- helpers ------------------------------------------------------------ + + def _find_task_description(self, events: list[Event]) -> Optional[str]: + """Pick the first non-empty user-message content as the task text.""" + for ev in events: + if ev.event_type == "user" and ev.content: + return ev.content + return None + + def _emit_tool_section( + self, + events: list[Event], + start_idx: int, + out: list[str], + turn_idx: int, + ) -> int: + """Emit pending tool calls + results following an assistant turn. + + Walks ``events[start_idx:]`` while encountering ``role=tool`` events + (which carry the result of the prior assistant's tool_calls), + collapsing consecutive identical (name, args-prefix) calls. + + Returns the index where the tool section ends (so the outer loop + can continue from there). + + Collapse logic: we look at the PREVIOUS assistant's ``tool_calls`` + list and emit one summary per call. If the calls in this run match + a recent identical call within a small window, we annotate + "× N repetitions". + """ + cfg = self._cfg + i = start_idx + + # Look back at last assistant for its tool_calls (already past it in i-1) + prev_assistant = self._find_last_assistant(events, start_idx) + if prev_assistant is None: + return i + if not prev_assistant.tool_calls: + # No tool calls promised — skip any stray tool events (rare) + while i < len(events) and events[i].event_type == "tool": + i += 1 + return i + + # Emit each tool call with truncated args + for tc in prev_assistant.tool_calls: + fn = tc.get("function") or {} + name = fn.get("name", "?") + args_raw = fn.get("arguments", "") or "" + if not isinstance(args_raw, str): + args_raw = json.dumps(args_raw, ensure_ascii=False) + args_trunc = args_raw[: cfg.tool_args_chars] + if len(args_raw) > cfg.tool_args_chars: + args_trunc += f" [...truncated {len(args_raw) - cfg.tool_args_chars} chars]" + out.append(f" → call {name}({args_trunc})") + + # Now consume the corresponding tool-result events + while i < len(events) and events[i].event_type == "tool": + result = events[i].content or "" + out.append(f" ← result: {self._summarize_tool_result(result)}") + i += 1 + + return i + + def _find_last_assistant(self, events: list[Event], current_idx: int) -> Optional[Event]: + """The most recent assistant event strictly before ``current_idx``.""" + for j in range(current_idx - 1, -1, -1): + if events[j].event_type == "assistant": + return events[j] + return None + + def _summarize_tool_result(self, content: str) -> str: + cfg = self._cfg + if not content: + return "[empty result]" + total = len(content) + head_n = cfg.tool_result_head_chars + tail_n = cfg.tool_result_tail_chars + if total <= head_n + tail_n: + return self._oneline(content) + head = content[:head_n] + tail = content[-tail_n:] + return f"{self._oneline(head)} ... [ELIDED {total - head_n - tail_n} chars] ... {self._oneline(tail)}" + + @staticmethod + def _oneline(text: str) -> str: + """Collapse newlines + repeated whitespace so the result fits one line.""" + return re.sub(r"\s+", " ", text).strip() + + @staticmethod + def _indent(text: str, prefix: str) -> str: + return "\n".join(prefix + line for line in text.splitlines()) + + def _compute_anomaly_summary(self, events: list[Event], empty_content_count: int) -> list[str]: + cfg = self._cfg + out: list[str] = [] + + # 1. Empty content ratio (key L1 vs L2 signal). finish_reason + # breakdown disambiguates: ``stop`` = true L1 (model silent), + # ``length`` = L2 (max_tokens budget too small for this model — + # reasoning model burned budget without producing output). + n_asst = sum(1 for e in events if e.event_type == "assistant") + if n_asst > 0 and empty_content_count > 0: + pct = empty_content_count / n_asst * 100 + severity = "HIGH" if pct > 30 else "MEDIUM" if pct > 10 else "LOW" + fr_breakdown = Counter( + (e.finish_reason or "unknown") + for e in events + if e.event_type == "assistant" and not (e.content or "").strip() and not e.tool_calls + ) + fr_tag = "" + if fr_breakdown: + parts = [f"{k}={v}" for k, v in sorted(fr_breakdown.items())] + fr_tag = f" (by finish_reason: {', '.join(parts)})" + out.append( + f" [{severity}] empty-content assistant turns: {empty_content_count}/{n_asst}" + f" ({pct:.0f}%){fr_tag} — L1 if finish_reason=stop/content_filter dominates," + f" L2 if finish_reason=length dominates (max_tokens budget too small)" + ) + + # 2. Syntax / docker / network errors in tool outputs + syntax_errs = 0 + docker_errs = 0 + net_errs = 0 + for ev in events: + if ev.event_type != "tool" or not ev.content: + continue + if _matches_any(ev.content, _SYNTAX_ERROR_PATTERNS): + syntax_errs += 1 + if _matches_any(ev.content, _DOCKER_ERROR_PATTERNS): + docker_errs += 1 + if _matches_any(ev.content, _NETWORK_ERROR_PATTERNS): + net_errs += 1 + if syntax_errs: + out.append(f" [LOW] {syntax_errs} tool result(s) contained syntax-error markers") + if docker_errs: + out.append(f" [HIGH] {docker_errs} tool result(s) contained docker/container errors — L1 signal") + if net_errs: + out.append(f" [HIGH] {net_errs} tool result(s) contained network errors — L1 signal") + + # 3. Tool call repetition density + tool_call_sigs: list[tuple[str, str]] = [] + for ev in events: + if ev.event_type != "assistant": + continue + for tc in ev.tool_calls: + fn = tc.get("function") or {} + name = fn.get("name", "?") + args = fn.get("arguments", "") or "" + if not isinstance(args, str): + args = json.dumps(args, ensure_ascii=False) + sig = (name, args[: cfg.tool_args_chars]) + tool_call_sigs.append(sig) + if tool_call_sigs: + c = Counter(tool_call_sigs) + most_common_sig, most_common_n = c.most_common(1)[0] + if most_common_n >= cfg.repetition_run_threshold: + args_preview = most_common_sig[1][:80].replace("\n", " ") + out.append( + f" [MEDIUM] tool call repeated {most_common_n}× " + f"({most_common_sig[0]}, args~{args_preview!r}) " + f"— possible repetition_breaker pathology" + ) + + return out + + +# --------------------------------------------------------------------------- +# Quick token-estimate helper +# --------------------------------------------------------------------------- + + +def estimate_tokens(text: str) -> int: + """Rough token count via ``len(text) / 4`` — good enough for budget + shaping at the 1K-100K scale; do NOT use for billing / SLA decisions. + + ``ceil`` semantics so a 5-char string reports 2 tokens not 1. + """ + if not text: + return 0 + return -(-len(text) // 4) + + +__all__ = [ + "CompressorConfig", + "Event", + "TrajectoryCompressor", + "estimate_tokens", + "load_session_jsonl", +] diff --git a/raven/evolver/judge/__init__.py b/raven/evolver/judge/__init__.py new file mode 100644 index 0000000..60da538 --- /dev/null +++ b/raven/evolver/judge/__init__.py @@ -0,0 +1,79 @@ +"""LLM-judge subsystem for the evolver. + +Reads compressed trajectories, classifies failures as L1/L2/L3 +(spec §3), and proposes structured (WHERE, WHY) patches for L2/L3 cases +(spec §12.4-§12.5). + +Public surface: + +- ``IssueType`` / ``PatchWhere`` / ``PatchWhy`` / ``ActionKind`` — enums +- ``JudgeAction`` / ``JudgeResult`` — parsed analysis dataclasses +- ``build_judge_messages`` — assemble (system, user) for one LLM call +- ``parse_judge_output`` — turn LLM raw text into a validated JudgeResult +- ``JudgeParseError`` — raised on malformed output + +The LLM client itself is in ``raven.evolver.judge.llm_client`` (B3, +written separately). +""" + +from .llm_client import ( + JudgeLLM, + JudgeLLMBackend, + JudgeLLMConfig, + LitellmBackend, + MockBackend, + Mode, + OpenRouterBackend, + TrajectoryFormat, + build_backend, + build_judge_llm, +) +from .parser import JudgeParseError, parse_judge_output +from .prompts import ( + JUDGE_SYSTEM_PROMPT, + JUDGE_USER_TEMPLATE, + WHERE_DESCRIPTIONS, + WHY_DESCRIPTIONS, + build_judge_messages, +) +from .schema import ( + ActionKind, + IssueType, + JudgeAction, + JudgeResult, + PatchWhere, + PatchWhy, + ProposedComponent, +) + +__all__ = [ + # Enums + "ActionKind", + "IssueType", + "PatchWhere", + "PatchWhy", + # Dataclasses + "JudgeAction", + "JudgeResult", + "ProposedComponent", + # Prompt building + "JUDGE_SYSTEM_PROMPT", + "JUDGE_USER_TEMPLATE", + "WHERE_DESCRIPTIONS", + "WHY_DESCRIPTIONS", + "build_judge_messages", + # Parsing + "JudgeParseError", + "parse_judge_output", + # LLM client (B3) + "JudgeLLM", + "JudgeLLMBackend", + "JudgeLLMConfig", + "LitellmBackend", + "MockBackend", + "Mode", + "OpenRouterBackend", + "TrajectoryFormat", + "build_backend", + "build_judge_llm", +] diff --git a/raven/evolver/judge/llm_client.py b/raven/evolver/judge/llm_client.py new file mode 100644 index 0000000..5bbcfa8 --- /dev/null +++ b/raven/evolver/judge/llm_client.py @@ -0,0 +1,620 @@ +"""LLM client for the judge — pluggable backends + mix routing. + +The judge has two distinct workloads: + +- **L1 detection** (is this trajectory hosed by an infrastructure bug?) + — pattern recognition, cheap model suffices. +- **L2 / L3 patch proposal** (which file to edit, what change) — needs + stronger reasoning; pay for a larger model. + +We support three operating modes, controllable via :class:`JudgeLLMConfig`: + +- ``"single"`` — one backend handles everything. Pick a model strong + enough for patches; it will also do L1 detection. Useful for ablations. +- ``"two_step"`` *(default mix mode)* — cheap L1 backend runs first; if + it determines L1, we return its output and skip the expensive call. + Otherwise we discard its patch guess and call the strong patch backend. +- ``"pure_"`` — alias for ``single`` with both slots configured to + the same backend (just a convenience for "use only Qwen" or "use only + OpenRouter"). + +Backend implementations: + +- :class:`LitellmBackend` wraps any ``LLMProvider`` from + ``raven.providers`` — works with self-hosted Qwen-397B, plus any + other model the existing provider stack supports. +- :class:`OpenRouterBackend` issues direct HTTPS to + ``openrouter.ai/api/v1`` — gives access to Claude / GPT / Gemini / + Qwen / DeepSeek through one API. +- :class:`MockBackend` returns canned responses; used by every test in + this module and any downstream test that needs a deterministic judge. + +Configuration is data-driven (:func:`load_judge_config`) so the same +code path runs in production (real backends), in tests (MockBackend), +and in ablations (config-flip changes mode). +""" + +from __future__ import annotations + +import logging +import os +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Literal, Optional + +from .parser import JudgeParseError, parse_judge_output +from .prompts import build_judge_messages +from .schema import IssueType, JudgeResult + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Backend protocol & implementations +# --------------------------------------------------------------------------- + + +class JudgeLLMBackend(ABC): + """Minimum surface a backend must expose. + + The judge does not need streaming, tool calls, or function-calling — + just plain chat completion that returns the assistant's text. Keep + the interface narrow so adding a new backend (Vertex / Together / + a local vLLM endpoint) requires <30 lines. + """ + + name: str # set by subclass + + @abstractmethod + async def call( + self, + messages: list[dict[str, str]], + *, + max_tokens: int = 4000, + temperature: float = 0.0, + ) -> str: + """Run one chat completion. Returns the assistant's text body.""" + + def __repr__(self) -> str: + return f"{type(self).__name__}(name={self.name!r})" + + +class LitellmBackend(JudgeLLMBackend): + """Backend that delegates to a raven ``LLMProvider``. + + Used for the self-hosted Qwen-397B path (which raven's litellm + provider already routes). The provider is lazy-imported so this + module doesn't drag the full provider stack into pure-unit-test + environments. + + The wrapped provider must expose ``async def chat(messages, ...)`` + returning an object with a ``.content`` string attribute (the + standard ``LLMResponse``). We strip everything else. + """ + + def __init__( + self, + provider: Any, # raven.providers.base.LLMProvider — lazy-typed + *, + model: Optional[str] = None, + name: str = "litellm", + ) -> None: + self._provider = provider + self._model = model + self.name = name + + async def call( + self, + messages: list[dict[str, str]], + *, + max_tokens: int = 4000, + temperature: float = 0.0, + ) -> str: + # Route through chat_with_retry (base.LLMProvider) so the judge + # inherits the empty-response retry + sync OpenAI fallback that + # the agent loop already relies on. Calling provider.chat() + # directly surfaced provider empty-content errors at the judge + # layer (~3% of calls in an early dry run). + # The provider's chat[_with_retry] accepts model=None to fall + # back to its configured default; we pass our override only when + # explicit. + kwargs: dict[str, Any] = {"max_tokens": max_tokens, "temperature": temperature} + if self._model: + kwargs["model"] = self._model + response = await self._provider.chat_with_retry(messages, **kwargs) + content = getattr(response, "content", None) + if not isinstance(content, str): + raise RuntimeError( + f"LitellmBackend({self.name}): provider returned non-string content" + f" ({type(content).__name__}); cannot parse as judge output." + ) + return content + + +class OpenRouterBackend(JudgeLLMBackend): + """Direct HTTP backend for OpenRouter (``openrouter.ai``). + + Why direct HTTP instead of routing through litellm? OpenRouter API + keys live in a separate env var from raven's main provider stack, + and we want the judge's external-LLM budget to be visible and + accounted independently — easier to enforce a hard cap and to swap + models per ablation. The HTTP shape is OpenAI-compatible, so the + code is small. + + Model strings use OpenRouter's namespaced form, e.g.: + + - ``"anthropic/claude-haiku-4-5"`` + - ``"openai/gpt-4.1-mini"`` + - ``"google/gemini-2.5-flash"`` + - ``"qwen/qwen3-235b"`` + + API key resolution: explicit ``api_key`` arg → env var ``api_key_env`` + → ``OPENROUTER_API_KEY``. Raises at call time if none found. + + ``httpx`` is required at call time but imported lazily so unit tests + that only use ``MockBackend`` don't need it installed. + """ + + DEFAULT_API_BASE = "https://openrouter.ai/api/v1" + DEFAULT_API_KEY_ENV = "OPENROUTER_API_KEY" + # Retry on empty content / transient HTTP errors. OpenRouterBackend + # bypasses the raven provider stack, so it doesn't get + # chat_with_retry for free; we add a minimal internal retry to match. + _RETRY_DELAYS = (1.0, 2.0, 4.0) + + def __init__( + self, + *, + model: str, + api_key: Optional[str] = None, + api_key_env: Optional[str] = None, + api_base: Optional[str] = None, + timeout_seconds: float = 60.0, + name: str = "openrouter", + ) -> None: + self._model = model + self._explicit_key = api_key + self._api_key_env = api_key_env or self.DEFAULT_API_KEY_ENV + self._api_base = (api_base or self.DEFAULT_API_BASE).rstrip("/") + self._timeout = timeout_seconds + self.name = name + + def _resolve_api_key(self) -> str: + if self._explicit_key: + return self._explicit_key + key = os.environ.get(self._api_key_env) + if not key: + raise RuntimeError( + f"OpenRouterBackend({self.name}): no API key — pass api_key, or set env {self._api_key_env}" + ) + return key + + async def call( + self, + messages: list[dict[str, str]], + *, + max_tokens: int = 4000, + temperature: float = 0.0, + ) -> str: + # Lazy import: tests that don't hit OpenRouter don't need httpx + import asyncio # noqa: PLC0415 + + import httpx # noqa: PLC0415 + + api_key = self._resolve_api_key() + url = f"{self._api_base}/chat/completions" + payload: dict[str, Any] = { + "model": self._model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + } + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + # OpenRouter encourages identifying the caller for routing analytics: + "HTTP-Referer": "https://github.com/EverMind-AI/Raven", + "X-Title": "Raven Harness Evolver", + } + + # Retry on empty content / transient HTTP errors. LitellmBackend + # gets retry for free via provider.chat_with_retry; OpenRouter + # bypasses the provider stack so we add a minimal in-place + # retry that mirrors the same empty-detect → backoff → retry + # shape. Total attempts = len(_RETRY_DELAYS) + 1 final = 4. + last_exc: Exception | None = None + last_data: dict[str, Any] | None = None + for attempt, delay in enumerate(self._RETRY_DELAYS, start=1): + try: + async with httpx.AsyncClient(timeout=self._timeout) as client: + resp = await client.post(url, json=payload, headers=headers) + resp.raise_for_status() + data = resp.json() + content = self._extract_content(data) + if content and content.strip(): + return content + # Empty content — retry with backoff. + last_data = data + except (httpx.HTTPError, KeyError, IndexError, TypeError) as exc: + last_exc = exc + await asyncio.sleep(delay) + + # Final attempt: don't suppress exceptions. + async with httpx.AsyncClient(timeout=self._timeout) as client: + resp = await client.post(url, json=payload, headers=headers) + resp.raise_for_status() + data = resp.json() + content = self._extract_content(data) + if content and content.strip(): + return content + # Still empty after all retries — raise so the batch records an error. + if last_exc is not None: + raise RuntimeError( + f"OpenRouterBackend({self.name}): empty content after " + f"{len(self._RETRY_DELAYS) + 1} attempts; last exc: {last_exc!r}" + ) from last_exc + raise RuntimeError( + f"OpenRouterBackend({self.name}): empty content after " + f"{len(self._RETRY_DELAYS) + 1} attempts; last payload: " + f"{(last_data or data)!r}" + ) + + @staticmethod + def _extract_content(data: dict[str, Any]) -> Optional[str]: + """Pull message content from an OpenAI-shaped response, or None.""" + try: + return data["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError): + return None + + +class MockBackend(JudgeLLMBackend): + """In-memory backend that returns scripted responses, for tests. + + Pass a list of strings in ``responses``; each call pops the next one + in order. Raises ``IndexError`` if the test under-scripts responses + — fail loud rather than silently re-using the last. + + ``calls`` records every (messages, max_tokens, temperature) tuple for + assertions: tests can verify the backend was invoked the right number + of times with the expected payload. + """ + + def __init__(self, responses: list[str], *, name: str = "mock") -> None: + self._responses = list(responses) + self.calls: list[dict[str, Any]] = [] + self.name = name + + async def call( + self, + messages: list[dict[str, str]], + *, + max_tokens: int = 4000, + temperature: float = 0.0, + ) -> str: + self.calls.append( + { + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + } + ) + if not self._responses: + raise IndexError( + f"MockBackend({self.name}): no more scripted responses; received {len(self.calls)} call(s) total" + ) + return self._responses.pop(0) + + +# --------------------------------------------------------------------------- +# Config dataclass + facade +# --------------------------------------------------------------------------- + + +Mode = Literal["single", "two_step"] +TrajectoryFormat = Literal["full", "compressed"] + + +@dataclass +class JudgeLLMConfig: + """Configuration for :class:`JudgeLLM`. + + The two backend slots are populated externally — we don't construct + backends from this config, so the config stays serialisation-friendly + (yaml/json) and the test harness can inject a ``MockBackend`` without + going through any factory. + + Trajectory format (full vs compressed) is configured per backend slot: + + - ``l1_trajectory_format``: what the L1-detection backend sees. Default + ``"compressed"`` — L1 signals (empty content / docker errors / + pattern repetition) are visible in a ~10K compression; paying for + 150K full context here is waste. + - ``patch_trajectory_format``: what the patch-proposal backend sees. + Default ``"full"`` — writing a good patch needs concrete tool calls + and detailed reasoning steps that compression can blur. + + Caller supplies both ``trajectory_text`` (full) and optional + ``trajectory_text_compressed`` to :meth:`JudgeLLM.judge`. When a + backend slot's format is ``"compressed"`` but no compressed text was + passed, we fall back to full with a debug log (so misconfiguration + is visible but doesn't crash). + + Use :func:`build_judge_llm` to assemble a ``JudgeLLM`` from a + config-dict that *does* describe backends. That function is the + integration point with yaml/json config files. + """ + + mode: Mode = "two_step" + l1_trajectory_format: TrajectoryFormat = "compressed" + patch_trajectory_format: TrajectoryFormat = "full" + max_tokens: int = 4000 + temperature: float = 0.0 + # Diagnostic: log every call's input length + backend used. + # Off by default to keep stdout quiet in tests. + debug_log: bool = False + + +class JudgeLLM: + """Facade that orchestrates one judge analysis. + + Single-call mode is straightforward: build the (system, user) + messages, send to ``patch_backend``, parse and return. + + Two-step mode is the default mix: ``l1_backend`` makes the first + pass. If it judges L1 (infrastructure bug), we return its result + immediately — saves the expensive patch_backend call. If it judges + L2/L3, we discard its patch guess (cheap models tend to be sloppy + on patch proposals) and call ``patch_backend`` for a higher-quality + rewrite. + + Why discard the cheap model's patch instead of keeping it when good? + Reliability of the patch is downstream-critical (a bad patch wastes + evaluation budget). The cheap model's L1 detection is high-recall; + its patch field is low-precision. Splitting trust along this axis + is what makes the mix worth its complexity. + """ + + def __init__( + self, + l1_backend: JudgeLLMBackend, + patch_backend: JudgeLLMBackend, + config: Optional[JudgeLLMConfig] = None, + ) -> None: + self._l1 = l1_backend + self._patch = patch_backend + self._config = config or JudgeLLMConfig() + + @property + def config(self) -> JudgeLLMConfig: + return self._config + + async def judge( + self, + trajectory_id: str, + task_description: str, + trajectory_text: str, + trajectory_text_compressed: Optional[str] = None, + ) -> JudgeResult: + """Run the full judge pipeline on one trajectory. + + ``trajectory_text`` is the full original trajectory (required). + ``trajectory_text_compressed`` is an optional pre-compressed + summary (~10K tokens, "agent debugger" style). When provided AND + a backend slot's ``*_trajectory_format`` is ``"compressed"``, that + backend receives the compressed text; otherwise it falls back to + full (with a debug log when ``debug_log=True``). + """ + l1_text = self._select_trajectory_text( + self._config.l1_trajectory_format, + trajectory_text, + trajectory_text_compressed, + slot="l1", + ) + patch_text = self._select_trajectory_text( + self._config.patch_trajectory_format, + trajectory_text, + trajectory_text_compressed, + slot="patch", + ) + + patch_messages = build_judge_messages( + trajectory_id=trajectory_id, + task_description=task_description, + trajectory_text=patch_text, + ) + if self._config.mode == "single": + return await self._call_and_parse(self._patch, patch_messages, trajectory_id) + + # two_step + l1_messages = build_judge_messages( + trajectory_id=trajectory_id, + task_description=task_description, + trajectory_text=l1_text, + ) + l1_result = await self._call_and_parse(self._l1, l1_messages, trajectory_id) + if l1_result.issue_type == IssueType.L1: + if self._config.debug_log: + logger.info( + "judge: %s → L1 detected by l1_backend, skipping patch_backend", + trajectory_id, + ) + return l1_result + if self._config.debug_log: + logger.info( + "judge: %s → %s detected, calling patch_backend", + trajectory_id, + l1_result.issue_type.value, + ) + return await self._call_and_parse(self._patch, patch_messages, trajectory_id) + + def _select_trajectory_text( + self, + want: TrajectoryFormat, + full: str, + compressed: Optional[str], + *, + slot: str, + ) -> str: + """Pick which trajectory text to feed a backend, with fallback. + + ``want`` is what the config requests; if it's ``"compressed"`` but + no compressed text was provided, we fall back to ``full`` so the + judge still runs (cost balloons but correctness is preserved). + Logged at debug level so misconfig is visible. + """ + if want == "compressed": + if compressed is None: + if self._config.debug_log: + logger.warning( + "judge[%s]: config wants compressed trajectory but none " + "provided; falling back to full (cost may be higher than " + "expected)", + slot, + ) + return full + return compressed + return full + + async def _call_and_parse( + self, + backend: JudgeLLMBackend, + messages: list[dict[str, str]], + trajectory_id: str, + ) -> JudgeResult: + raw = await backend.call( + messages, + max_tokens=self._config.max_tokens, + temperature=self._config.temperature, + ) + try: + return parse_judge_output(raw, expected_trajectory_id=trajectory_id) + except JudgeParseError: + logger.warning( + "judge.parse failed on backend=%s, traj=%s; raw=%s", + backend.name, + trajectory_id, + raw[:500], + ) + raise + + def __repr__(self) -> str: + return f"JudgeLLM(mode={self._config.mode!r}, l1={self._l1!r}, patch={self._patch!r})" + + +# --------------------------------------------------------------------------- +# Config-driven builder (the integration point with yaml/json) +# --------------------------------------------------------------------------- + + +def build_backend(spec: dict[str, Any]) -> JudgeLLMBackend: + """Build one backend from a config-dict. + + Supported ``type`` values: + + - ``"openrouter"`` — :class:`OpenRouterBackend`. Required: ``model``. + Optional: ``api_key``, ``api_key_env``, ``api_base``, + ``timeout_seconds``, ``name``. + - ``"litellm"`` — :class:`LitellmBackend`. Required: ``provider`` + (an already-instantiated ``LLMProvider``). Optional: ``model``, + ``name``. **Cannot be built from pure dict** because the provider + object must be constructed elsewhere — callers pass it in via + ``spec["provider"]`` (typically the same provider AgentLoop uses). + - ``"mock"`` — :class:`MockBackend`. Required: ``responses`` (list + of strings). For tests only. + + Raises ``ValueError`` on unknown ``type``. + """ + backend_type = spec.get("type") + if backend_type == "openrouter": + return OpenRouterBackend( + model=spec["model"], + api_key=spec.get("api_key"), + api_key_env=spec.get("api_key_env"), + api_base=spec.get("api_base"), + timeout_seconds=spec.get("timeout_seconds", 60.0), + name=spec.get("name", "openrouter"), + ) + if backend_type == "litellm": + if "provider" not in spec: + raise ValueError( + "litellm backend spec requires a pre-built 'provider' object; cannot construct from pure dict" + ) + return LitellmBackend( + provider=spec["provider"], + model=spec.get("model"), + name=spec.get("name", "litellm"), + ) + if backend_type == "mock": + return MockBackend( + responses=list(spec.get("responses", [])), + name=spec.get("name", "mock"), + ) + raise ValueError(f"unknown backend type {backend_type!r}; supported: openrouter, litellm, mock") + + +def build_judge_llm(spec: dict[str, Any]) -> JudgeLLM: + """Assemble a :class:`JudgeLLM` from one dict. + + Expected shape:: + + { + "mode": "two_step", # or "single" + "max_tokens": 4000, + "temperature": 0.0, + "debug_log": false, + "l1_backend": { ... build_backend spec ... }, + "patch_backend": { ... build_backend spec ... }, + } + + For ``mode="single"``, ``l1_backend`` may be omitted (the + ``patch_backend`` handles everything). If both are present in + single mode, ``l1_backend`` is ignored. + + Convenience: setting ``l1_backend`` and ``patch_backend`` to the + same spec is the "pure Qwen" / "pure OpenRouter" pattern — works + without a special mode. + """ + mode = spec.get("mode", "two_step") + config = JudgeLLMConfig( + mode=mode, + l1_trajectory_format=spec.get("l1_trajectory_format", "compressed"), + patch_trajectory_format=spec.get("patch_trajectory_format", "full"), + max_tokens=spec.get("max_tokens", 4000), + temperature=spec.get("temperature", 0.0), + debug_log=spec.get("debug_log", False), + ) + + patch_spec = spec.get("patch_backend") + if patch_spec is None: + raise ValueError("build_judge_llm: 'patch_backend' is required") + patch_backend = build_backend(patch_spec) + + if mode == "two_step": + l1_spec = spec.get("l1_backend") + if l1_spec is None: + raise ValueError("build_judge_llm: mode='two_step' requires 'l1_backend'") + l1_backend = build_backend(l1_spec) + else: + # single mode: reuse patch_backend in the l1 slot, never called + l1_backend = patch_backend + + return JudgeLLM( + l1_backend=l1_backend, + patch_backend=patch_backend, + config=config, + ) + + +__all__ = [ + "JudgeLLMBackend", + "LitellmBackend", + "OpenRouterBackend", + "MockBackend", + "JudgeLLMConfig", + "JudgeLLM", + "Mode", + "TrajectoryFormat", + "build_backend", + "build_judge_llm", +] diff --git a/raven/evolver/judge/parser.py b/raven/evolver/judge/parser.py new file mode 100644 index 0000000..83b64c6 --- /dev/null +++ b/raven/evolver/judge/parser.py @@ -0,0 +1,344 @@ +"""Parse LLM-judge raw text output into a validated ``JudgeResult``. + +The judge is instructed to emit a single JSON object (spec §3.2 schema + +spec §12.4-§12.5 enums), but real LLMs occasionally wrap it in markdown +code fences, add leading "Here is the analysis:" prose, or append a +short summary after the JSON. The parser tolerates these defects: + +- Strips ```json / ``` code fences (any language tag). +- Locates the first ``{`` and matches to the final balanced ``}``. +- Validates enum values against ``IssueType`` / ``PatchWhere`` / ``PatchWhy``. +- Enforces the L1/L2/L3 ↔ action-kind cross-field invariants by handing + off to ``JudgeResult.__post_init__``. + +What it does NOT tolerate: + +- Truncated JSON (no closing brace). +- Multiple top-level JSON objects. +- Missing required fields. +- Invalid enum strings (raises with the offender's name). + +Errors are raised as ``JudgeParseError`` with a short reason; callers +typically log + skip the offending trajectory, or feed it back to the +judge as a "retry, fix this" prompt. +""" + +from __future__ import annotations + +import json +import re +from typing import Any, Optional + +from .schema import ( + ActionKind, + IssueType, + JudgeAction, + JudgeResult, + PassFailResult, + PatchWhere, + PatchWhy, + ProposedComponent, +) + + +class JudgeParseError(ValueError): + """Raised when LLM judge output cannot be parsed into a JudgeResult.""" + + +# --------------------------------------------------------------------------- +# Text → JSON dict +# --------------------------------------------------------------------------- + + +_CODE_FENCE_RE = re.compile(r"^\s*```(?:json|JSON|jsonc)?\s*\n?|\n?\s*```\s*$", re.MULTILINE) + + +def _extract_json_object(raw: str) -> str: + """Locate the first balanced ``{...}`` block in ``raw``. + + Strategy: + 1. Strip Markdown code fences. + 2. Find the first ``{``. + 3. Walk forward tracking brace depth (string-aware: ignores braces + inside double-quoted strings, respecting backslash escapes). + 4. Return the substring [first_brace, matching_close_brace+1]. + + Raises ``JudgeParseError`` if no ``{`` is found or braces never + balance. + + This is more robust than a naive ``raw[raw.find('{'):raw.rfind('}')+1]`` + which fails when the LLM appends another ``{...}`` snippet after its + main JSON (e.g. "Note: see {related_file} for context."). + """ + stripped = _CODE_FENCE_RE.sub("", raw).strip() + start = stripped.find("{") + if start < 0: + raise JudgeParseError("no '{' found in judge output") + + depth = 0 + in_string = False + escaped = False + for i in range(start, len(stripped)): + ch = stripped[i] + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return stripped[start : i + 1] + raise JudgeParseError("braces never balance — JSON likely truncated") + + +def _require(d: dict[str, Any], key: str, what: str) -> Any: + if key not in d: + raise JudgeParseError(f"missing required field '{key}' in {what}") + return d[key] + + +def _coerce_enum(value: Any, enum_cls: type, field_name: str) -> Any: + """Coerce a string into an Enum, raising helpfully on miss. + + Accepts the enum value's ``value`` (string form). Rejects ints, + enum member ``.name``, or any other shape. + """ + if not isinstance(value, str): + raise JudgeParseError(f"field '{field_name}' must be a string, got {type(value).__name__}") + try: + return enum_cls(value) + except ValueError as exc: + valid = [m.value for m in enum_cls] + raise JudgeParseError(f"field '{field_name}'={value!r} not one of {valid}") from exc + + +def _parse_components(action_obj: dict[str, Any]) -> list[ProposedComponent]: + """Parse the ``components`` array from a patch_proposal action. + + Backwards-compat: if the LLM emits a flat ``target_file`` + + ``patch_summary`` pair instead of (or in addition to) ``components``, + synthesize a single component. This keeps already-trained judge + prompts working while we roll out the multi-component schema. + """ + raw_components = action_obj.get("components") + + # Path A: new schema — explicit components list + if raw_components is not None: + if not isinstance(raw_components, list): + raise JudgeParseError(f"proposed_action.components must be a list, got {type(raw_components).__name__}") + if not raw_components: + raise JudgeParseError("proposed_action.components must be non-empty for patch_proposal") + parsed: list[ProposedComponent] = [] + for i, item in enumerate(raw_components): + if not isinstance(item, dict): + raise JudgeParseError(f"proposed_action.components[{i}] must be an object, got {type(item).__name__}") + component_id = item.get("component_id") or f"comp_{i + 1}" + if not isinstance(component_id, str) or not component_id.strip(): + raise JudgeParseError(f"proposed_action.components[{i}].component_id must be a non-empty string") + target_file = item.get("target_file") + if not isinstance(target_file, str) or not target_file.strip(): + raise JudgeParseError(f"proposed_action.components[{i}].target_file must be a non-empty string") + summary = item.get("summary") + if not isinstance(summary, str) or not summary.strip(): + raise JudgeParseError(f"proposed_action.components[{i}].summary must be a non-empty string") + depends_on_raw = item.get("depends_on") or [] + if not isinstance(depends_on_raw, list) or not all(isinstance(x, str) for x in depends_on_raw): + raise JudgeParseError(f"proposed_action.components[{i}].depends_on must be a list of strings") + parsed.append( + ProposedComponent( + component_id=component_id, + target_file=target_file, + summary=summary, + depends_on=list(depends_on_raw), + ) + ) + return parsed + + # Path B: legacy flat schema — synthesize a single component + target_file = action_obj.get("target_file") + patch_summary = action_obj.get("patch_summary") + if target_file is None or patch_summary is None: + raise JudgeParseError( + "proposed_action must contain either 'components' (preferred) or " + "the legacy 'target_file' + 'patch_summary' pair" + ) + if not isinstance(target_file, str) or not target_file.strip(): + raise JudgeParseError("proposed_action.target_file must be a non-empty string") + if not isinstance(patch_summary, str) or not patch_summary.strip(): + raise JudgeParseError("proposed_action.patch_summary must be a non-empty string") + return [ + ProposedComponent( + component_id="comp_1", + target_file=target_file, + summary=patch_summary, + depends_on=[], + ), + ] + + +def _parse_evidence_range(raw: Any) -> Optional[tuple[int, int]]: + """Validate the [start, end] turn-range list.""" + if raw is None: + return None + if not isinstance(raw, (list, tuple)): + raise JudgeParseError(f"evidence_turn_range must be a list/tuple, got {type(raw).__name__}") + if len(raw) != 2: + raise JudgeParseError(f"evidence_turn_range must have exactly 2 elements, got {len(raw)}") + a, b = raw + if not (isinstance(a, int) and isinstance(b, int)): + raise JudgeParseError(f"evidence_turn_range elements must be ints, got {a!r}, {b!r}") + if a > b: + raise JudgeParseError(f"evidence_turn_range start ({a}) > end ({b})") + return (a, b) + + +# --------------------------------------------------------------------------- +# Top-level parser +# --------------------------------------------------------------------------- + + +def parse_judge_output( + raw_text: str, + *, + expected_trajectory_id: Optional[str] = None, +) -> JudgeResult: + """Parse one judge response into a validated ``JudgeResult``. + + ``expected_trajectory_id`` (optional): if provided, the parsed + ``trajectory_id`` field must match — protects against the judge + silently mis-binding output to the wrong trajectory in a batch + pipeline. + + The original ``raw_text`` is stored on ``JudgeResult.raw_response`` + for audit (so post-hoc inspection of misclassified trajectories + can see exactly what the LLM said). + """ + json_text = _extract_json_object(raw_text) + try: + obj = json.loads(json_text) + except json.JSONDecodeError as exc: + raise JudgeParseError(f"JSON decode failed: {exc}") from exc + + if not isinstance(obj, dict): + raise JudgeParseError(f"top-level must be a JSON object, got {type(obj).__name__}") + + trajectory_id = _require(obj, "trajectory_id", "judge output") + if not isinstance(trajectory_id, str): + raise JudgeParseError("trajectory_id must be a string") + if expected_trajectory_id is not None: + # Judges occasionally echo the trajectory_id back with a wrapper- + # path suffix copied from the task description, e.g. + # expected: 'swe-vanilla-500/django__django-11292' + # got: 'swe-vanilla-500/django__django-11292-t1-exec' + # (the '-t1-exec' came from WRAPPER_PATH in the user message). + # Accept the parsed id if it starts with the expected id so the + # batch doesn't lose otherwise-valid records over a cosmetic + # suffix. Strict mismatch (different task entirely) still raises. + if trajectory_id != expected_trajectory_id and not trajectory_id.startswith(expected_trajectory_id): + raise JudgeParseError(f"trajectory_id mismatch: expected {expected_trajectory_id!r}, got {trajectory_id!r}") + + issue_type = _coerce_enum(_require(obj, "issue_type", "judge output"), IssueType, "issue_type") + confidence = _require(obj, "confidence", "judge output") + if not isinstance(confidence, (int, float)): + raise JudgeParseError(f"confidence must be a number, got {type(confidence).__name__}") + confidence = float(confidence) + if not 0.0 <= confidence <= 1.0: + raise JudgeParseError(f"confidence must be in [0.0, 1.0], got {confidence}") + + signal_description = _require(obj, "signal_description", "judge output") + if not isinstance(signal_description, str): + raise JudgeParseError("signal_description must be a string") + + evidence_turn_range = _parse_evidence_range(obj.get("evidence_turn_range")) + + action_obj = _require(obj, "proposed_action", "judge output") + if not isinstance(action_obj, dict): + raise JudgeParseError("proposed_action must be an object") + + kind = _coerce_enum(_require(action_obj, "kind", "proposed_action"), ActionKind, "proposed_action.kind") + reasoning = _require(action_obj, "reasoning", "proposed_action") + if not isinstance(reasoning, str): + raise JudgeParseError("proposed_action.reasoning must be a string") + + # Patch-fields: required iff kind=patch_proposal. + patch_where = None + patch_why = None + patch_why_extra = None + components: list[ProposedComponent] = [] + + if kind == ActionKind.patch_proposal: + patch_where = _coerce_enum( + _require(action_obj, "patch_where", "proposed_action(patch_proposal)"), + PatchWhere, + "proposed_action.patch_where", + ) + patch_why = _coerce_enum( + _require(action_obj, "patch_why", "proposed_action(patch_proposal)"), + PatchWhy, + "proposed_action.patch_why", + ) + components = _parse_components(action_obj) + # patch_why=other requires a sub-name + patch_why_extra_raw = action_obj.get("patch_why_extra") + if patch_why == PatchWhy.other: + if not patch_why_extra_raw or not isinstance(patch_why_extra_raw, str): + raise JudgeParseError("patch_why='other' requires non-empty patch_why_extra string") + patch_why_extra = patch_why_extra_raw + + try: + action = JudgeAction( + kind=kind, + reasoning=reasoning, + patch_where=patch_where, + patch_why=patch_why, + patch_why_extra=patch_why_extra, + components=components, + ) + except ValueError as exc: + raise JudgeParseError(str(exc)) from exc + + # JudgeResult.__post_init__ enforces the cross-field invariants. + return JudgeResult( + trajectory_id=trajectory_id, + issue_type=issue_type, + confidence=confidence, + signal_description=signal_description, + proposed_action=action, + evidence_turn_range=evidence_turn_range, + raw_response=raw_text, + ) + + +def parse_pass_fail(raw: str, *, expected_trajectory_id: str = "") -> PassFailResult: + """Parse a no-benchmark pass/fail verdict; raise on any defect (for retry). + + Reuses the judge's tolerant JSON extractor so the same ``SemanticNode`` + repair loop applies. Requires a boolean-ish ``passed`` field. + """ + obj = json.loads(_extract_json_object(raw)) + if "passed" not in obj: + raise JudgeParseError("pass/fail verdict missing required 'passed' field") + passed = obj["passed"] + if isinstance(passed, str): + passed = passed.strip().lower() in ("true", "pass", "passed", "yes", "1") + return PassFailResult( + trajectory_id=str(obj.get("trajectory_id", expected_trajectory_id)), + passed=bool(passed), + reasoning=str(obj.get("reasoning", "")), + raw_response=raw, + ) + + +__all__ = [ + "JudgeParseError", + "parse_judge_output", + "parse_pass_fail", +] diff --git a/raven/evolver/judge/prompts.py b/raven/evolver/judge/prompts.py new file mode 100644 index 0000000..2977253 --- /dev/null +++ b/raven/evolver/judge/prompts.py @@ -0,0 +1,412 @@ +"""LLM-judge prompt templates. + +Two top-level templates are exposed: + +- ``JUDGE_SYSTEM_PROMPT``: the role + rubric + output schema text. Sent + as the system message of the judge LLM call. +- ``JUDGE_USER_TEMPLATE``: parametrized with the specific trajectory + data; sent as the user message. + +The system prompt teaches the L1/L2/L3 rubric, the (WHERE, WHY) labels, +and the JSON output schema. Decoupling system + user lets us reuse the +heavy rubric across many trajectories without re-paying tokens. + +Building blocks: + +- ``build_judge_messages`` returns the (system, user) pair given a + trajectory record — feed directly to any chat-style LLM provider. +- ``WHERE_DESCRIPTIONS`` / ``WHY_DESCRIPTIONS`` document each enum value + in human terms; they are injected into the prompt so the judge model + has a concrete signpost (not just the enum string). + +All prompt text is intentionally explicit about the cross-field +invariants enforced by ``JudgeResult.__post_init__``: L1 → no patch, +L2/L3 → patch_where + patch_why required, ``other`` → must carry a +sub-name. Saying it both in the prompt and in code reduces wasted +LLM calls that produce invalid JSON. +""" + +from __future__ import annotations + +from .schema import PatchWhere, PatchWhy + +# --- Human-readable descriptions injected into the prompt ---------------- + + +WHERE_DESCRIPTIONS: dict[PatchWhere, str] = { + PatchWhere.system_prompt_template: "Raven's system prompt templates (raven/templates/*.md: SOUL, AGENTS, " + "TOOLS, etc). Use when the agent's identity / behavioural guidelines need " + "wording changes.", + PatchWhere.task_wrapper_prompt: "external scorer task domain prompt (src/domains//prompt.md). " + "This is the user-message wrapper telling agent what the task is and what " + "rules to follow. Common L2 fixes live here.", + PatchWhere.judge_prompt: "Raven's internal turn-judge prompt (eval_engine/prompts/*.py, L-B " + "layer). Affects memory updates, NOT the benchmark verifier; safe to " + "evolve.", + PatchWhere.tool_description: "A tool's description string or safety pattern in agent/tools/*.py. Use " + "when the model misuses a tool because the description is vague / misleading.", + PatchWhere.hook_new: "Add a new lifecycle hook file in agent/hook/.py. Hooks fire at " + "phases (before_iteration / before_execute_tools / after_iteration / " + "after_send) and can inject nudges or short-circuit the loop. Best for " + "runtime behaviour interventions (repetition, test cadence, budget).", + PatchWhere.hook_modify: "Tune an existing hook's parameters in eval_engine/hooks/*.py.", + PatchWhere.skill: "Add / edit / retire a skill in memory_engine/skills/. Skills are " + "retrievable procedural recipes — use when a class of tasks needs a " + "reusable how-to.", + PatchWhere.memory: "Add / edit / retire a memory entry in memory_engine/everos/. Memory " + "holds factual knowledge (e.g. 'Django ≥4 uses async views').", + PatchWhere.tool_new: "Add a new Tool subclass in agent/tools/.py. Rare — only when no " + "existing tool plus prompt nudge can express the needed operation.", + PatchWhere.loop_override: "Scoped override to agent/loop/main.py or subsidiary loop logic (code " + "class). Use for instrumentation, conditional logic, or parameter patches " + "that don't merit a full hook. Must include activation_beacon().", + PatchWhere.context_override: "Scoped override to context management (code class). Patches short-lived " + "state structures that vary per iteration. Must include activation_beacon().", + PatchWhere.tool_override: "Scoped override to tool execution or selection logic (code class). Use for " + "tool-routing or execution-time patches. Must include activation_beacon().", + PatchWhere.config: "Tune a yaml / json default value, threshold, or feature flag.", + PatchWhere.control: "Control arm - no mechanism; measures infra runtime-neutrality. Not a real patch surface; never propose patches here.", +} + + +WHY_DESCRIPTIONS: dict[PatchWhy, str] = { + PatchWhy.repetition_breaker: "Agent loops on near-identical (tool, args) sequences without making " + "progress (observed in 72% of 244-paired failed trajectories).", + PatchWhy.test_starvation_remedy: "Agent spends most turns reading code, runs tests too infrequently to " + "iterate (PASS trajectories run tests at 25% of turns vs FAIL at 12%).", + PatchWhy.budget_awareness: "Agent has no sense of how much budget is left and over-explores until " + "max_iter (100% of max_iter failures hit the ceiling).", + PatchWhy.tool_clarity: "A registered tool is unused or misused because its description / " + "documentation is missing / vague / misleading.", + PatchWhy.env_contract_clarify: "The environment's interface contract is inaccurately described — e.g. " + "a prompt forbids using a tool that the env actually supports, or a tool " + "is documented as host-side when it's container-side.", + PatchWhy.skill_gap_fill: "A recurring task type (e.g. Django pytest setup) has no skill in the " + "library, so the agent re-discovers the same recipe in every trajectory.", + PatchWhy.memory_recall_fix: "Agent repeats the same lookup / verification within a single trajectory " + "(re-reads same file range, re-runs same python -c). Short-term memory " + "is leaking.", + PatchWhy.reasoning_visibility: "Agent does not externalize its reasoning before tool calls or final " + "answer. Trajectory shows long stretches of tool calls without " + "narrative explanation, making it hard to follow intent or diagnose " + "failures. Patch typically adds a prompt nudge to 'explain your " + "reasoning before each significant tool call' or similar visibility-" + "improving instruction. Promoted from B2 dry-run 2026-05-30 as the " + "dominant `other` extra (reasoning_visibility_improvement, " + "communication_traceability, explanatory_text_nudge, ...).", + PatchWhy.empty_response_recovery: "Agent enters a streak of completely empty turns (content=None, no tool " + "calls) and does not self-recover. Patch typically adds a hook or prompt " + "nudge that detects the streak and injects a recovery instruction.", + PatchWhy.method_lock_in_remedy: "Agent commits too early to one approach (e.g., a single file path or " + "algorithm) and stops exploring alternatives even when early attempts fail. " + "Patch introduces a method-diversity nudge or conditional branch check.", + PatchWhy.infra_neutrality_control: "Control-arm bookkeeping: this node carries no real patch; it exists to " + "measure baseline infra runtime-neutrality across rounds. Not a real " + "pathology category — never use for actual patch proposals.", + PatchWhy.other: "None of the above categories fit. You MUST specify a free-form " + "sub-name in `patch_why_extra` (e.g. 'plan_action_disconnect'). Reserve " + "for genuinely novel pathologies; do not abuse to avoid picking an " + "existing category.", +} + + +# --- Main prompt ----------------------------------------------------------- + + +def _render_where_block() -> str: + return "\n".join(f" - `{w.value}`: {WHERE_DESCRIPTIONS[w]}" for w in PatchWhere) + + +def _render_why_block() -> str: + return "\n".join(f" - `{w.value}`: {WHY_DESCRIPTIONS[w]}" for w in PatchWhy) + + +JUDGE_SYSTEM_PROMPT = f"""You are an expert LLM-agent harness analyst. + +You read a (compressed) trajectory of an LLM agent attempting a coding task +(SWE-bench / Terminal-Bench style) and produce a structured diagnosis. Your +output drives an automated harness self-evolution loop, so it must be +precise, machine-readable, and conservative. + +# Step 1 — classify the issue type + +You MUST pick exactly one of: + +- **L1 — Infrastructure bug** (model / network / container / tool-implementation + failure). Triggers (any single one is enough): + * assistant **completely silent** turn (content=None or empty string + AND **no** tool_calls) + finish_reason="stop" appears in >30% of turns. + NOTE: a tool-using turn (content=None but tool_calls present) is the + NORMAL agent pattern (model acts instead of explaining); it is NOT + an L1 signal. Only count turns that have neither text nor tool calls. + * tool errors like `docker daemon error`, `connection refused`, `read + timeout`, `OOM`, `pool exhausted` + * same prompt+input produces wildly different behaviour across trials + (variance suggests transient infrastructure failure) + * agent terminated suddenly at turn<10 with no final assistant text + * trajectory shows the agent was writing code but the run aborted from + a system-level error + Action for L1: kind="human_review_needed". Do NOT propose any patch. + + # L1 vs L2 — finish_reason discriminator (read this before triggering L1 + # on any empty-content turn): + + - empty content + finish_reason="stop" → L1 (model truly silent; + provider returned a + terminated stream + with no output) + - empty content + finish_reason="length" → L2 (max_tokens budget too + small for this model + — config issue, not + infra failure). + Patch: bump max_tokens + for this call path, + or disable thinking + for sub-LLM calls. + - empty content + finish_reason="content_filter" → L1 (safety abort = infra- + side decision) + - non-empty but malformed JSON / partial output → L2 (the model IS producing + tokens; the harness + prompt + parser + combination is too + strict for this + model's capability). + Patch: tighten the + format prompt, add + a JSON5/markdown- + strip parser, or use + structured-output + mode if the backend + supports it. + + NOTE: "graceful fallback" patterns in tool results (e.g. "Failed to parse + rewrite response — defaulting to retrieval") are L2 signals, NOT L1. The + agent is still running fine; the harness is just degraded. Look at the WHY + not the symptom. + +- **L2 — Harness config error** (documents / configs / prompts are inaccurate, + contradictory, or stale). Triggers (any one): + * prompts contradict each other (e.g. one says "use X" another says + "NEVER use X") + * a tool is registered but its documentation tells the agent not to use it + * tool description is misleading; the agent picks the wrong tool + * config defaults are inappropriate for the benchmark + Action for L2: kind="patch_proposal" with full WHERE/WHY/target_file fields. + +- **L3 — Harness capability gap** (system runs cleanly but behaviour can be + smarter). Triggers (any one): + * agent failed to retrieve a relevant skill that exists in the library + * memory is missing a known failure-pattern that would have helped + * a useful nudge is missing from the prompt (e.g. "run tests after every + edit") + * a skill exists but its content is low quality + Action for L3: kind="patch_proposal" with full WHERE/WHY/target_file fields. + +If multiple types could apply, prefer L1 > L2 > L3 (errors block evolution +first). + +# Step 1.5 — L2 vs L3 tiebreaker (when both could apply) + +If you are leaning L3 but the issue could also be framed as L2, ask: + + "Could a static edit to an existing config / prompt / tool description + have prevented this *before* the agent run started?" + + - YES → L2 (something is misconfigured / stale / contradictory) + - NO → L3 (config is fine; agent needs more capability — new skill, + new memory, new prompt nudge) + +Examples: + - Agent picked wrong tool because two tool descriptions overlap → L2 + (description needs rewrite to disambiguate; static fix) + - Agent picked wrong tool because no skill explains when to pick which → L3 + (need new skill; not a description bug) + - Prompt says "use pytest" but env has pytest-xdist conflict → L2 + (prompt is stale; update it) + - Prompt is consistent and tool docs are fine but agent forgot to run + tests after edits → L3 (add a nudge skill / memory entry) + +Default tilt when uncertain: **prefer L2** — fixing config is cheaper and +more general than adding new skills/memory. + +# Step 2 — for L2 / L3, choose WHERE the patch goes + +Pick exactly one structural location: + +{_render_where_block()} + +# Step 3 — for L2 / L3, choose WHY this patch is being made + +Pick exactly one pathology category. If none fit, use `other` and provide +a free-form sub-name in `patch_why_extra`: + +{_render_why_block()} + +# Step 4 — emit JSON + +Output exactly the following JSON object — nothing else, no surrounding +prose, no Markdown code fences (the parser strips fences but emitting +clean JSON is faster and cheaper). + +``` +{{ + "trajectory_id": "", + "issue_type": "L1" | "L2" | "L3", + "confidence": , + "signal_description": "", + "evidence_turn_range": [, ], + "proposed_action": {{ + "kind": "human_review_needed" | "patch_proposal", + "reasoning": "", + + // The fields below are required when kind=patch_proposal, + // and MUST be omitted (null/[] is acceptable) when kind=human_review_needed. + "patch_where": "", + "patch_why": "", + "patch_why_extra": "", + "components": [ + {{ + "component_id": "comp_1", + "target_file": "", + "summary": "", + "depends_on": [] + }} + // ...add additional component objects for structurally-coupled edits + ] + }} +}} +``` + +# CRITICAL — cross-field consistency (the parser will reject your output +# if violated): + + - If issue_type == "L1": + kind MUST be "human_review_needed" + patch_proposal MUST be null + - If issue_type == "L2" or "L3": + kind MUST be "patch_proposal" + patch_proposal MUST be a full object with WHERE / WHY / target_file + +Double-check this constraint BEFORE emitting JSON. If you are tempted to +output L2/L3 with kind="human_review_needed", you are confused — re-read +Step 1 and either downgrade to L1 (if truly infra) or commit to producing +a patch_proposal. + +# Multi-component patches + +`components` is ALWAYS a non-empty list (for L2/L3). For most patches use +exactly ONE component (single-file fix). Use MULTIPLE components only when +the change is structurally coupled — i.e., no single component is useful +on its own. Examples: + +- New hook FILE + register the hook in CONFIG → 2 components, where the + config one `depends_on: ["comp_1"]`. +- Add a new TOOL + reference it in the SYSTEM_PROMPT → 2 components. +- Add a SKILL + update MEMORY index pointing to it → 2 components. + +Rules for `components`: + +- `component_id` must be unique within this `components` array + (e.g. "comp_1", "comp_2", ...). +- `depends_on` is a list of sibling `component_id`s that must be applied + first. It MUST only reference siblings within THIS components array. + Use `[]` (empty) when independent. +- If you find yourself wanting to bundle ≥3 unrelated file edits into one + patch, STOP — emit only the highest-priority one. The evolver will + propose other patches in later iterations. + +# Cross-field rules (will be re-validated downstream) + +Emitting invalid JSON wastes a round-trip: + +1. L1 → kind must be `human_review_needed`; patch_where / patch_why / + patch_why_extra / components all null or `[]`. +2. L2 / L3 → kind must be `patch_proposal`; patch_where + patch_why + populated; `components` is a non-empty list with all required + fields per item. +3. `patch_why=other` → `patch_why_extra` non-empty. +4. `confidence` ∈ [0.0, 1.0]; report your true uncertainty, not 1.0 by default. +5. `evidence_turn_range` is the inclusive interval of turn indices that + justify your judgement. If a single turn suffices, use [n, n]. +""" + + +JUDGE_USER_TEMPLATE = """trajectory_id: {trajectory_id} +task_description: +\"\"\" +{task_description} +\"\"\" + +trajectory (compressed): +\"\"\" +{trajectory_text} +\"\"\" + +Output the JSON analysis now.""" + + +def build_judge_messages( + trajectory_id: str, + task_description: str, + trajectory_text: str, +) -> list[dict[str, str]]: + """Assemble the (system, user) message pair for one judge call. + + Returns the list shape most chat-style LLM SDKs expect: + ``[{"role": "system", "content": ...}, {"role": "user", "content": ...}]``. + + ``trajectory_text`` should already be compressed to the judge's context + budget — typically 5-15K tokens of "Agent Debugger" style event summary + (spec §11.4.5). Raw multi-million-token trajectories are NOT supported; + upstream must compress first. + + No string interpolation happens in the system prompt — its content is + fixed across calls, so most providers will cache it on subsequent calls + (significant cost savings during a long evolution run). + """ + user = JUDGE_USER_TEMPLATE.format( + trajectory_id=trajectory_id, + task_description=task_description.strip(), + trajectory_text=trajectory_text.strip(), + ) + return [ + {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, + {"role": "user", "content": user}, + ] + + +PASS_FAIL_SYSTEM_PROMPT = ( + "You are grading whether an AI agent ACCOMPLISHED a task, from its trajectory. " + "There is no automated verifier — your verdict is the score. Judge only " + "whether the task's goal was actually achieved (correct final answer / the " + "requested action completed and verified), not effort or intent. Be strict: " + "an unfinished, guessed, or unverified result is a fail. Respond with ONLY a " + 'JSON object, no prose, no code fences: {"passed": true|false, "reasoning": ' + '""}.' +) + + +def build_pass_fail_messages( + task_description: str, trajectory_text: str, *, trajectory_id: str = "" +) -> list[dict[str, str]]: + """Assemble the (system, user) pair for a no-benchmark pass/fail verdict.""" + user = ( + f"Task:\n{task_description.strip()}\n\n" + f"Agent trajectory:\n{trajectory_text.strip()}\n\n" + "Did the agent accomplish the task? Return the JSON verdict." + ) + return [ + {"role": "system", "content": PASS_FAIL_SYSTEM_PROMPT}, + {"role": "user", "content": user}, + ] + + +__all__ = [ + "JUDGE_SYSTEM_PROMPT", + "JUDGE_USER_TEMPLATE", + "WHERE_DESCRIPTIONS", + "WHY_DESCRIPTIONS", + "build_judge_messages", + "PASS_FAIL_SYSTEM_PROMPT", + "build_pass_fail_messages", +] diff --git a/raven/evolver/judge/schema.py b/raven/evolver/judge/schema.py new file mode 100644 index 0000000..5a524a8 --- /dev/null +++ b/raven/evolver/judge/schema.py @@ -0,0 +1,284 @@ +"""Schema definitions for LLM judge output. + +The judge reads a trajectory and produces a structured analysis with two +orthogonal axes: + +- **Issue type** (L1 / L2 / L3, per spec §3) — routes the output: + L1 → human review; L2/L3 → evolver patch. +- **Patch location & purpose** ((WHERE, WHY), per spec §12.4-§12.5) — only + populated for L2 / L3, identifies what to edit and which pathology + is being addressed. + +All enums are ``str`` subclasses so they JSON-serialize to their string +values without custom encoders, and so they compare cleanly against +strings parsed from LLM output (``IssueType.L1 == "L1"``). + +PatchWhy.other accepts a free-form sub-name (``"other:"``) +to support the evolving WHY taxonomy described in spec §12.5: judge may +propose a new pathology class, which gets routed for human review before +becoming a first-class enum. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Optional + + +class IssueType(str, Enum): + """Three-state classification per spec §3 — drives downstream routing.""" + + L1 = "L1" # infrastructure bug — block evolver, raise to human + L2 = "L2" # harness config error — evolver patches docs/configs + L3 = "L3" # harness capability gap — evolver patches skills/memory/hooks + + +class PatchWhere(str, Enum): + """Structural location of a proposed patch (spec §12.4, 14 categories). + + Each value corresponds to a class of files in the mutable surface + (spec §2.2). The judge picks one based on the failure signature. + + `control` (2026-06-10): added for evolution-restart round 1 to + represent control-arm nodes that carry no real patch surface. + """ + + system_prompt_template = "system_prompt_template" # templates/*.md + task_wrapper_prompt = "task_wrapper_prompt" # scorer src/domains/*/prompt.md + judge_prompt = "judge_prompt" # eval_engine/prompts/*.py (L-B internal) + tool_description = "tool_description" # agent/tools/*.py description field + hook_new = "hook_new" # new agent/hook/.py + hook_modify = "hook_modify" # eval_engine/hooks/*.py existing + skill = "skill" # memory_engine/skills/* + memory = "memory" # memory_engine/everos/* + tool_new = "tool_new" # new agent/tools/.py + loop_override = "loop_override" # scoped loop.py override (code class) + context_override = "context_override" # scoped context.py override (code class) + tool_override = "tool_override" # scoped tool override (code class) + config = "config" # yaml/json defaults + control = "control" # control arm — no patch surface + + +class PatchWhy(str, Enum): + """Pathology category the patch addresses (spec §12.5, 11 named + other). + + Derived from the 244-paired SWE-bench failure analysis. Evolving: judge + may propose a new class via ``other`` plus a free-form sub-name, which + is reviewed before being promoted to a first-class enum value. + + `reasoning_visibility` (2026-06-01): promoted from B2 dry-run + `patch_why_extra` accumulation — the dominant uncategorized class + (5/10 ``other`` entries: reasoning_visibility_improvement, + communication_traceability, communication verbosity nudge, + explanatory_text_nudge, trajectory_logging_quality). + + `empty_response_recovery`, `method_lock_in_remedy`, + `infra_neutrality_control` (2026-06-10): added for evolution-restart + round 1 — empty-response streak recovery, early method lock-in remedy, + and control-arm bookkeeping respectively. + """ + + repetition_breaker = "repetition_breaker" # 72% trajectory tail repetition + test_starvation_remedy = "test_starvation_remedy" # PASS 25% TEST vs FAIL 12% + budget_awareness = "budget_awareness" # FAIL 100% hits maxIter + tool_clarity = "tool_clarity" # tool docs missing/misleading + env_contract_clarify = "env_contract_clarify" # env rules contradictory (e.g. NEVER prompt) + skill_gap_fill = "skill_gap_fill" # recurring task type, no skill + memory_recall_fix = "memory_recall_fix" # re-reads / re-verifies known facts + reasoning_visibility = "reasoning_visibility" # tool-only stretches w/o narrative explanation + empty_response_recovery = "empty_response_recovery" # repeated empty-response streak recovery + method_lock_in_remedy = "method_lock_in_remedy" # early method lock-in remedy + infra_neutrality_control = "infra_neutrality_control" # control-arm bookkeeping; not a real pathology + other = "other" # judge-proposed; sub-name in patch_why_extra + + +class ActionKind(str, Enum): + """What downstream should do with this judge output.""" + + human_review_needed = "human_review_needed" # L1: evolver pauses, engineer fixes + patch_proposal = "patch_proposal" # L2/L3: evolver applies patch + + +@dataclass +class ProposedComponent: + """One file the judge proposes to edit as part of a multi-file patch. + + Counterpart in the tree layer is + :class:`raven.evolver.tree.node.PatchComponent` (with the actual + diff). At judge time we only have natural-language ``summary`` — the + mutation operator (GEPA library) later turns each ``ProposedComponent`` + into a concrete ``PatchComponent`` with a unified diff. + + Multi-component design rationale (spec §12.2, §18.5.1.x): a single + "patch" may bundle several file edits that together implement one + semantic improvement (e.g., new hook file + register it in a config). + The ``depends_on`` graph lets component-level bisect drop subsets + safely when a regression appears. + + For the simple 1-file fix, ``JudgeAction.components`` has length 1. + """ + + component_id: str # "comp_1" / "comp_2" — unique within one JudgeAction + target_file: str # repo-relative path of the file to edit + summary: str # natural-language description of the intended edit + depends_on: list[str] = field(default_factory=list) # sibling component_ids + + def to_dict(self) -> dict[str, Any]: + return { + "component_id": self.component_id, + "target_file": self.target_file, + "summary": self.summary, + "depends_on": list(self.depends_on), + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "ProposedComponent": + missing = [k for k in ("component_id", "target_file", "summary") if k not in d] + if missing: + raise ValueError(f"ProposedComponent missing required fields: {missing}") + return cls( + component_id=d["component_id"], + target_file=d["target_file"], + summary=d["summary"], + depends_on=list(d.get("depends_on") or []), + ) + + +@dataclass +class JudgeAction: + """The judge's recommended next action. + + For ``kind=human_review_needed`` (L1), only ``reasoning`` is populated + — the evolver should NOT attempt to apply any patch, just surface the + issue to a human operator. + + For ``kind=patch_proposal`` (L2/L3), patch fields are populated: + WHERE / WHY and a non-empty ``components`` list. Each component + names one file to edit plus a natural-language summary; mutation + operators later turn these into unified diffs. + + ``patch_why_extra`` is non-empty only when ``patch_why == other``, and + carries the judge's proposed new pathology sub-name (e.g. + ``"other:plan_action_disconnect"``). Reviewing-and-promoting these + sub-names is the mechanism for evolving the WHY taxonomy (spec §12.5). + """ + + kind: ActionKind + reasoning: str + + # Populated only for patch_proposal: + patch_where: Optional[PatchWhere] = None + patch_why: Optional[PatchWhy] = None + patch_why_extra: Optional[str] = None + components: list[ProposedComponent] = field(default_factory=list) + + def __post_init__(self) -> None: + if self.kind == ActionKind.patch_proposal: + if not self.components: + raise ValueError("patch_proposal JudgeAction requires at least one component") + ids = [c.component_id for c in self.components] + if len(ids) != len(set(ids)): + raise ValueError(f"JudgeAction.components have duplicate component_id: {ids}") + valid_ids = set(ids) + for c in self.components: + for dep in c.depends_on: + if dep not in valid_ids: + raise ValueError( + f"ProposedComponent {c.component_id!r} depends_on {dep!r} which is not a sibling component" + ) + + def is_patch(self) -> bool: + return self.kind == ActionKind.patch_proposal + + def is_human_review(self) -> bool: + return self.kind == ActionKind.human_review_needed + + @property + def target_file(self) -> Optional[str]: + """Primary file = first component's target_file (backwards-compat shim).""" + return self.components[0].target_file if self.components else None + + @property + def patch_summary(self) -> Optional[str]: + """For single-component patches, return that component's summary. + + Multi-component patches don't have a single summary — use + ``components[i].summary`` directly, or ``reasoning`` for the + overall narrative. + """ + if not self.components: + return None + if len(self.components) == 1: + return self.components[0].summary + # Multi-component: concatenate for any legacy caller that still + # asks for a single string, but the canonical access is per-comp. + return " | ".join(c.summary for c in self.components) + + +@dataclass +class JudgeResult: + """Complete judge analysis of one trajectory. + + Fields follow spec §3.2 schema. ``evidence_turn_range`` is the + interval (inclusive) of turn indices the judge cites as + supporting evidence — used by the evolver to anchor patches to + specific failure points in the trajectory. + + ``confidence`` ∈ [0.0, 1.0]. Downstream may reject low-confidence + judgments (e.g. evolver skips patch proposals with confidence < 0.5). + """ + + trajectory_id: str + issue_type: IssueType + confidence: float + signal_description: str + proposed_action: JudgeAction + evidence_turn_range: Optional[tuple[int, int]] = None + raw_response: Optional[str] = None # original LLM text, for audit + + def __post_init__(self) -> None: + if not 0.0 <= self.confidence <= 1.0: + raise ValueError(f"confidence must be in [0.0, 1.0], got {self.confidence!r}") + # Cross-field invariant: L1 must use human_review_needed + if self.issue_type == IssueType.L1 and not self.proposed_action.is_human_review(): + raise ValueError( + f"L1 issues must have proposed_action.kind=human_review_needed; got {self.proposed_action.kind!r}" + ) + # L2/L3 must use patch_proposal with populated where/why + if self.issue_type in (IssueType.L2, IssueType.L3): + if not self.proposed_action.is_patch(): + raise ValueError(f"{self.issue_type.value} issues must have proposed_action.kind=patch_proposal") + if self.proposed_action.patch_where is None: + raise ValueError(f"{self.issue_type.value} patch must have patch_where set") + if self.proposed_action.patch_why is None: + raise ValueError(f"{self.issue_type.value} patch must have patch_why set") + if self.proposed_action.patch_why == PatchWhy.other and not self.proposed_action.patch_why_extra: + raise ValueError("patch_why=other requires patch_why_extra to carry the judge-proposed sub-name") + + +@dataclass(frozen=True) +class PassFailResult: + """A no-benchmark pass/fail verdict for one trajectory. + + Distinct from :class:`JudgeResult` (which diagnoses failure *mode* and never + carries a pass/fail): this is what an LLM scorer returns when there is no + verifier — the orchestrator maps ``passed`` onto ``TaskEval.passes``. + """ + + trajectory_id: str + passed: bool + reasoning: str = "" + raw_response: Optional[str] = None + + +__all__ = [ + "IssueType", + "PatchWhere", + "PatchWhy", + "ActionKind", + "ProposedComponent", + "JudgeAction", + "JudgeResult", + "PassFailResult", +] diff --git a/raven/evolver/launch/__init__.py b/raven/evolver/launch/__init__.py new file mode 100644 index 0000000..dba48fb --- /dev/null +++ b/raven/evolver/launch/__init__.py @@ -0,0 +1,12 @@ +"""Unified launch layer: one config file + one command for any registered bench. + +``python -m raven.evolver run --config `` drives the whole SOP flow as a +resumable state machine: cold-start thick ledger -> evolution rounds -> +terminate -> unseal. Interrupt anywhere; re-running the same command resumes +from the last durable artifact (trial files / round journal / meta stamps). +""" + +from raven.evolver.launch.contract import BenchBundle, LaunchContext, validate_whitelist +from raven.evolver.launch.registry import load_bench + +__all__ = ["BenchBundle", "LaunchContext", "load_bench", "validate_whitelist"] diff --git a/raven/evolver/launch/config.py b/raven/evolver/launch/config.py new file mode 100644 index 0000000..80540ea --- /dev/null +++ b/raven/evolver/launch/config.py @@ -0,0 +1,232 @@ +"""Run-spec loading: YAML -> validated RunSpec (+ --smoke overlay). + +The YAML shape: + + bench: appworld + repo_root: /path/to/subject # the repo being evolved + base_sha: # optional; omitted -> repo_root HEAD at launch + work_dir: ./evo_work + + models: # optional; omitted -> raven's own model + driver: {provider: claude_cli, model: claude-haiku-4-5} + design: {provider: claude_cli, model: claude-opus-4-8} + verdict: {provider: openai_compat, base_url: ..., model: ...} + + funnel: # optional; SOP-aligned defaults + k_screen: 1 + k_confirm: 3 + budget: {max_why_per_round: 2, candidates_per_why: 3} + termination: {patience: 10, max_rounds: 20} + anchor: {n_sentinel: 12, cull_sigma_mult: 1.5} + + bench_config: {...} # schema owned by the bench entry + + smoke: {...} # optional deep-merge overlay for --smoke + +``--smoke`` applies built-in shrink defaults (1 WHY x 1 candidate x 1 round, +K=1) first, then the user's ``smoke:`` section on top, and suffixes work_dir +with ``_smoke`` so a smoke run never touches the real run's state. +""" + +from __future__ import annotations + +import copy +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +from raven.evolver.orchestrator.config import ( + AnchorParams, + Budget, + OrchestratorConfig, + Termination, +) + +SMOKE_BUILTIN: dict = { + "funnel": { + "k_confirm": 1, + "budget": {"max_why_per_round": 1, "candidates_per_why": 1, "recombinations_per_round": 0}, + "termination": {"patience": 1, "max_rounds": 1}, + }, +} + + +class RunSpecError(ValueError): + """A config file problem the user must fix; message says exactly what.""" + + +def _redact_secrets(models: dict) -> dict: + if not isinstance(models, dict): + return models + out = {} + for role, spec in models.items(): + if isinstance(spec, dict): + out[role] = {k: ("" if "key" in k.lower() and k != "api_key_env" else v) for k, v in spec.items()} + else: + out[role] = spec + return out + + +def deep_merge(base: dict, overlay: dict) -> dict: + out = copy.deepcopy(base) + for k, v in overlay.items(): + if isinstance(v, dict) and isinstance(out.get(k), dict): + out[k] = deep_merge(out[k], v) + else: + out[k] = copy.deepcopy(v) + return out + + +@dataclass +class RunSpec: + bench: str + repo_root: Path + base_sha: str + work_dir: Path + funnel: OrchestratorConfig + models: dict = field(default_factory=dict) + bench_config: dict = field(default_factory=dict) + smoke: bool = False + base_sha_defaulted: bool = False + config_dir: Path = field(default_factory=Path.cwd) + raw: dict = field(default_factory=dict) + + def snapshot(self) -> dict: + """The effective configuration recorded in run_meta (drift guard). + + Secrets are redacted before they reach disk — work dirs get shared — + and the redaction is a constant, so rotating a key does not trip the + config-drift guard. + """ + return { + "bench": self.bench, + "repo_root": str(self.repo_root), + "base_sha": self.base_sha, + "models": _redact_secrets(self.raw.get("models", {})), + "funnel": self.raw.get("funnel", {}), + "bench_config": self.raw.get("bench_config", {}), + "smoke": self.smoke, + } + + +def _build_funnel(repo_root: Path, work_dir: Path, funnel: dict) -> OrchestratorConfig: + if not isinstance(funnel, dict): + raise RunSpecError(f"funnel: must be a mapping, got {type(funnel).__name__}") + known = {"k_screen", "k_confirm", "anchor", "budget", "termination", "sealed_test_split"} + unknown = set(funnel) - known + if unknown: + raise RunSpecError(f"funnel: unknown keys {sorted(unknown)}") + try: + cfg = OrchestratorConfig( + repo_root=repo_root, + work_dir=work_dir, + driver_llm_spec={}, + k_screen=int(funnel.get("k_screen", 1)), + k_confirm=int(funnel.get("k_confirm", 3)), + anchor=AnchorParams(**(funnel.get("anchor") or {})), + budget=Budget(**(funnel.get("budget") or {})), + termination=Termination(**(funnel.get("termination") or {})), + sealed_test_split=funnel.get("sealed_test_split", "test"), + sealed_output_dir=work_dir / "sealed", + ) + except (TypeError, ValueError) as exc: + raise RunSpecError(f"funnel: {exc}") from exc + if cfg.k_screen < 1 or cfg.k_confirm < 1: + raise RunSpecError("funnel: k_screen and k_confirm must be >= 1") + if cfg.budget.max_why_per_round < 1 or cfg.budget.candidates_per_why < 1: + raise RunSpecError("funnel: budget.max_why_per_round and budget.candidates_per_why must be >= 1") + if cfg.termination.patience < 1 or cfg.termination.max_rounds < 1: + raise RunSpecError("funnel: termination.patience and termination.max_rounds must be >= 1") + return cfg + + +def load_run_spec(config_path: str | Path, *, smoke: bool = False) -> RunSpec: + path = Path(config_path) + if not path.is_file(): + raise RunSpecError(f"config file not found: {path}") + try: + data = yaml.safe_load(path.read_text()) + except yaml.YAMLError as exc: + raise RunSpecError(f"{path}: invalid YAML: {exc}") from exc + if not isinstance(data, dict): + raise RunSpecError(f"{path}: top level must be a mapping") + + if smoke: + overlay = data.pop("smoke", {}) or {} + data = deep_merge(deep_merge(data, SMOKE_BUILTIN), overlay) + else: + data.pop("smoke", None) + + missing = [k for k in ("bench", "repo_root", "work_dir") if not data.get(k)] + if missing: + raise RunSpecError(f"{path}: missing required keys: {missing}") + + def _abs(value: str) -> Path: + # Relative paths resolve against the config file, not the CWD — a + # resume from another directory must find the same work_dir, not + # silently start a fresh run. + p = Path(value).expanduser() + return p if p.is_absolute() else (path.parent / p).resolve() + + repo_root = _abs(data["repo_root"]) + if not (repo_root / ".git").exists(): + raise RunSpecError(f"repo_root is not a git checkout: {repo_root}") + + base_sha = str(data.get("base_sha") or "").strip() + base_sha_defaulted = not base_sha + if base_sha_defaulted: + base_sha = _resolve_head(repo_root) + data["base_sha"] = base_sha + work_dir = _abs(data["work_dir"]) + if smoke: + work_dir = work_dir.with_name(work_dir.name + "_smoke") + + models = data.get("models") or {} + if not isinstance(models, dict): + raise RunSpecError("models: must be a mapping of role -> provider spec") + unknown_roles = set(models) - {"driver", "design", "verdict"} + if unknown_roles: + raise RunSpecError(f"models: unknown roles {sorted(unknown_roles)}") + for role, spec_val in models.items(): + if not isinstance(spec_val, dict): + raise RunSpecError(f"models.{role}: must be a mapping (provider/model/...), got {type(spec_val).__name__}") + + return RunSpec( + bench=str(data["bench"]), + repo_root=repo_root, + base_sha=base_sha, + work_dir=work_dir, + funnel=_build_funnel(repo_root, work_dir, data.get("funnel") or {}), + models=models, + bench_config=data.get("bench_config") or {}, + smoke=smoke, + base_sha_defaulted=base_sha_defaulted, + config_dir=path.parent.resolve(), + raw=data, + ) + + +def _resolve_head(repo_root: Path) -> str: + """Resolve the subject repo's HEAD when the yaml omits ``base_sha``. + + Resolved to a full sha at load time and recorded in the config snapshot, + so the run stays pinned to the commit HEAD pointed at when it started — + resuming after the repo gained commits trips the drift guard instead of + silently moving the root. + """ + try: + proc = subprocess.run( + ["git", "-C", str(repo_root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + ) + except OSError as exc: + raise RunSpecError(f"git is not runnable ({exc}) — install git") from exc + if proc.returncode != 0: + raise RunSpecError(f"base_sha omitted and resolving HEAD of {repo_root} failed: {proc.stderr.strip()}") + return proc.stdout.strip() + + +__all__ = ["RunSpec", "RunSpecError", "load_run_spec", "deep_merge", "SMOKE_BUILTIN"] diff --git a/raven/evolver/launch/contract.py b/raven/evolver/launch/contract.py new file mode 100644 index 0000000..b565a89 --- /dev/null +++ b/raven/evolver/launch/contract.py @@ -0,0 +1,90 @@ +"""The bench plugin contract: what a benchmark implements to become runnable. + +A bench module exposes ``build(ctx: LaunchContext) -> BenchBundle``. The +bundle is pure wiring — nothing expensive happens until the runner calls the +closures, so ``status`` can build a bundle just to count artifacts. + +What a new bench must bring (see docs/specs/evolve-bench-contract.md): +scorer subprocess + result files with an infra marker, a result->TaskEval +reader, per-attempt trajectory files, a train/test split, and an editable-path +whitelist for its subject repo. Everything else (funnel, gates, sealed test, +resume) is the shared loop. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional + +from raven.evolver.launch.config import RunSpec +from raven.evolver.launch.models import CallFn + + +@dataclass(frozen=True) +class LaunchContext: + spec: RunSpec + models: dict[str, Optional[CallFn]] + + @property + def smoke(self) -> bool: + return self.spec.smoke + + +@dataclass +class BenchBundle: + """Everything the runner's state machine needs, all lazily evaluated. + + ``cold_start_done``/``run_cold_start`` must be idempotent at trial + granularity: re-invocation only fills missing trials. The runner invokes + ``run_cold_start`` on every run — including when all base trials exist — + so any infra-salvage rerun the bench owes (SOP §0 ladder) belongs inside + it, not behind the done-count. ``unseal`` receives + the journal records plus the built orchestrator and returns a plain-dict + report; None means the bench has no sealed test set configured. + ``precheck`` (optional) raises RuntimeError with an actionable message + when the environment cannot support a run (dead subject endpoint, bound + ports, missing install); ``check`` invokes it so environment problems + surface before any trial is paid for, not at cold start. + """ + + root_node_id: str + root_node: Any + journal_path: Path + cold_start_total: int + cold_start_done: Callable[[], int] + run_cold_start: Callable[[], None] + build_orchestrator: Callable[[], Any] + unseal: Optional[Callable[[list[dict], Any], dict]] = None + precheck: Optional[Callable[[], None]] = None + + +def validate_whitelist(repo_root: Path, base_sha: str, prefixes: tuple[str, ...]) -> None: + """Fail loudly on a whitelist prefix matching nothing at ``base_sha``. + + A dead prefix does not error at run time — the designer's edits are + silently reverted as out-of-whitelist and every candidate arrives empty, + which once cost a full run. Refusing to start is the only honest behavior. + """ + if not prefixes: + raise ValueError("whitelist is empty: the designer would have no editable surface") + proc = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", base_sha], + cwd=str(repo_root), + capture_output=True, + text=True, + ) + if proc.returncode != 0: + raise ValueError(f"cannot list {base_sha} in {repo_root}: {proc.stderr.strip()}") + paths = proc.stdout.splitlines() + dead = [p for p in prefixes if not any(f.startswith(p) for f in paths)] + if dead: + raise ValueError( + f"whitelist prefixes match no files at {base_sha[:12]}: {dead} — " + "designer edits would be silently dropped; fix the prefixes " + "(or the base_sha) before running" + ) + + +__all__ = ["BenchBundle", "LaunchContext", "validate_whitelist"] diff --git a/raven/evolver/launch/models.py b/raven/evolver/launch/models.py new file mode 100644 index 0000000..ee9cc0c --- /dev/null +++ b/raven/evolver/launch/models.py @@ -0,0 +1,149 @@ +"""Role call_fn factory: yaml ``models:`` section -> {driver, design, verdict}. + +Provider specs: + +- ``{provider: openai_compat, base_url, model, ...}`` -> one OpenAI-compatible + chat endpoint (:func:`raven.evolver.orchestrator.providers.openai_compat.make_call_fn`). +- ``{provider: claude_cli, model, ...}`` -> ``claude -p`` subprocess per call. +- ``{provider: raven, model?, api_base?, api_key_env?}`` -> raven's own + ``LitellmProvider`` bridged to a sync call_fn; ``model`` omitted falls back + to the raven config's ``agents.defaults.model`` — so a config file with no + ``models:`` section evolves with whatever model Raven itself is running. + +Role fallbacks: ``design`` omitted -> reuse driver; ``verdict`` omitted -> +None (the orchestrator then drafts verdicts with the driver). Note the driver +model and the *subject's* model are different knobs: the subject agent's model +lives in the bench config and is pinned for the whole run (same-regime rule). +""" + +from __future__ import annotations + +import asyncio +from typing import Callable, Optional + +CallFn = Callable[[list], str] + +_DEFAULT_SPEC = {"provider": "raven"} + + +def _raven_default_model() -> str: + from raven.config.loader import load_config + + return load_config().agents.defaults.model + + +def make_raven_call_fn( + model: Optional[str] = None, + *, + api_base: Optional[str] = None, + api_key: Optional[str] = None, + max_tokens: int = 8192, + temperature: float = 0.0, +) -> CallFn: + from raven.providers.litellm_provider import LiteLLMProvider + + provider = LiteLLMProvider( + api_key=api_key, + api_base=api_base, + default_model=model or _raven_default_model(), + ) + + def call(messages: list) -> str: + # Sync bridge: each call owns a private event loop, so this is safe + # from the loop's worker threads (parallel taxonomy induction). + resp = asyncio.run( + provider.chat_with_retry( + messages, + max_tokens=max_tokens, + temperature=temperature, + ) + ) + content = getattr(resp, "content", None) + if not isinstance(content, str) or not content.strip(): + raise RuntimeError("raven provider returned empty content") + return content + + return call + + +def build_call_fn(spec: dict, *, role: str = "?") -> CallFn: + try: + return _build_call_fn(spec) + except TypeError as exc: + # An unknown/missing kwarg in the provider factory is a config typo, + # not a programming error — surface it as the readable kind. + raise ValueError(f"models.{role}: {exc}") from exc + except ValueError as exc: + raise ValueError(f"models.{role}: {exc}") from exc + + +def _build_call_fn(spec: dict) -> CallFn: + if not isinstance(spec, dict): + raise ValueError(f"model spec must be a mapping, got {type(spec).__name__}") + kind = spec.get("provider", "raven") + kwargs = {k: v for k, v in spec.items() if k != "provider"} + if kind == "openai_compat": + from raven.evolver.orchestrator.providers.openai_compat import make_call_fn + + if "retry_delays" in kwargs: + delays = kwargs["retry_delays"] + if not isinstance(delays, (list, tuple)): + raise ValueError("retry_delays must be a list of seconds") + kwargs["retry_delays"] = tuple(delays) + return make_call_fn(**kwargs) + if kind == "claude_cli": + import shutil + + from raven.evolver.orchestrator.providers.claude_cli import make_claude_call_fn + + model = kwargs.pop("model", None) + if not model: + raise ValueError("claude_cli spec requires 'model'") + claude_bin = kwargs.get("claude_bin", "claude") + if shutil.which(claude_bin) is None: + raise ValueError( + f"claude_cli: {claude_bin!r} not found on PATH — install the " + "Claude Code CLI and log in, or switch this role to " + "openai_compat/raven" + ) + return make_claude_call_fn(model, **kwargs) + if kind == "raven": + model = kwargs.pop("model", None) + return make_raven_call_fn(model, **kwargs) + raise ValueError(f"unknown model provider {kind!r} (expected openai_compat / claude_cli / raven)") + + +def build_role_call_fns(models_cfg: dict) -> dict[str, Optional[CallFn]]: + driver = build_call_fn(models_cfg.get("driver", _DEFAULT_SPEC), role="driver") + design = build_call_fn(models_cfg["design"], role="design") if models_cfg.get("design") else driver + verdict = build_call_fn(models_cfg["verdict"], role="verdict") if models_cfg.get("verdict") else None + return {"driver": driver, "design": design, "verdict": verdict} + + +def describe_models(models_cfg: dict) -> dict: + """Resolved model description for the run_meta snapshot (no secrets).""" + out = {} + for role in ("driver", "design", "verdict"): + spec = models_cfg.get(role) + if spec is None: + if role == "driver": + spec = _DEFAULT_SPEC + elif role == "design": + spec = {"inherit": "driver"} + else: + spec = {"omitted": "driver drafts verdicts"} + out[role] = {k: v for k, v in spec.items() if "key" not in k.lower()} + if ( + out[role].get("provider", "raven") == "raven" + and "model" not in out[role] + and "inherit" not in out[role] + and "omitted" not in out[role] + ): + try: + out[role]["model"] = _raven_default_model() + except Exception: # noqa: BLE001 — description is best-effort + out[role]["model"] = "" + return out + + +__all__ = ["CallFn", "build_call_fn", "build_role_call_fns", "describe_models", "make_raven_call_fn"] diff --git a/raven/evolver/launch/registry.py b/raven/evolver/launch/registry.py new file mode 100644 index 0000000..83aab80 --- /dev/null +++ b/raven/evolver/launch/registry.py @@ -0,0 +1,37 @@ +"""Bench registry: name -> ``module:function`` building a BenchBundle.""" + +from __future__ import annotations + +import sys +from importlib import import_module +from pathlib import Path +from typing import Callable, Optional, Union + +BENCHES: dict[str, str] = { + "appworld": "benchmarks.appworld.evolve.entry:build", +} + + +def load_bench(name: str, repo_root: Optional[Union[str, Path]] = None) -> Callable: + """Import the bench plugin registered under ``name``. + + Bench plugins live in the subject repo (repo-root ``benchmarks/``), not in + the installed raven package, so ``repo_root`` — the subject checkout — is + put first on ``sys.path`` before importing. Omitting it only works when + the checkout root is already importable (e.g. cwd is the checkout). + """ + target = BENCHES.get(name) + if target is None: + raise ValueError( + f"unknown bench {name!r}; registered: {sorted(BENCHES)} " + "(add yours to raven.evolver.launch.registry.BENCHES)" + ) + if repo_root is not None: + root = str(Path(repo_root).resolve()) + if root not in sys.path: + sys.path.insert(0, root) + mod_name, _, fn_name = target.partition(":") + return getattr(import_module(mod_name), fn_name) + + +__all__ = ["BENCHES", "load_bench"] diff --git a/raven/evolver/launch/runner.py b/raven/evolver/launch/runner.py new file mode 100644 index 0000000..de3e616 --- /dev/null +++ b/raven/evolver/launch/runner.py @@ -0,0 +1,409 @@ +"""The run/status/finalize state machine over durable artifacts. + +Resume model: work is proven by artifacts, not by process state. + +- phase 1 (cold start): proof = vanilla trial result files. Resume fills + only the missing trials. +- phase 2 (rounds): proof = journal/rounds.jsonl. Resume replays completed + rounds (loop.run's built-in journal replay) and continues. +- phase 3 (unseal): proof = the ``unsealed_at`` stamp in run_meta.json. + Unsealing is one-way — a stamped run refuses to resume (--force overrides, + marking any further rounds as retention-invalid is on the caller). + +``status`` never reads the sealed directory: while a run is resumable, test +numbers must stay invisible (SOP §0) — the only path to them is natural +termination or an explicit ``finalize``. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +from pathlib import Path + +from raven.evolver.launch.config import RunSpec, RunSpecError, load_run_spec +from raven.evolver.launch.contract import BenchBundle, LaunchContext +from raven.evolver.launch.models import build_role_call_fns, describe_models +from raven.evolver.launch.registry import load_bench +from raven.evolver.launch.state import RunMeta, atomic_write_json, load_json_or +from raven.evolver.tree import git_ops + + +def _say(msg: str) -> None: + print(f"[evolve] {msg}", flush=True) + + +def _load_spec(config_path: str, smoke: bool) -> RunSpec: + try: + return load_run_spec(config_path, smoke=smoke) + except RunSpecError as exc: + print(f"config error: {exc}", file=sys.stderr) + raise SystemExit(2) from exc + + +def _build_bundle(spec: RunSpec, *, with_models: bool) -> BenchBundle: + try: + models = ( + build_role_call_fns(spec.models) + if with_models + else { + "driver": None, + "design": None, + "verdict": None, + } + ) + except ValueError as exc: + print(f"models error: {exc}", file=sys.stderr) + raise SystemExit(2) from exc + try: + build = load_bench(spec.bench, repo_root=spec.repo_root) + return build(LaunchContext(spec=spec, models=models)) + except ValueError as exc: + print(f"bench setup error ({spec.bench}): {exc}", file=sys.stderr) + raise SystemExit(2) from exc + + +def _note_defaulted_base(spec: RunSpec) -> None: + if not spec.base_sha_defaulted: + return + _say( + f"base_sha not set — using repo HEAD {spec.base_sha[:12]} " + "(pin base_sha in the yaml to freeze the root explicitly)" + ) + proc = subprocess.run( + ["git", "-C", str(spec.repo_root), "status", "--porcelain", "--untracked-files=no"], + capture_output=True, + text=True, + ) + if proc.stdout.strip(): + _say( + "warning: repo_root has uncommitted changes — they are NOT part " + "of the root node (evaluations check out commits, not the " + "working tree)" + ) + + +def _claim_ephemeral_root(spec: RunSpec) -> None: + """Park ephemeral git worktrees under work_dir; sweep hard-kill leftovers. + + Normal exits (including Ctrl-C) clean them via context managers — only a + SIGKILL'd run leaves them behind, still registered against the subject + repo. work_dir is per-run, so the sweep can only ever touch this run's + own garbage, never a concurrent run's live worktrees. + """ + tmp_root = Path(spec.work_dir) / "tmp" + if tmp_root.exists(): + shutil.rmtree(tmp_root, ignore_errors=True) + subprocess.run( + ["git", "-C", str(spec.repo_root), "worktree", "prune"], + capture_output=True, + text=True, + ) + git_ops.set_ephemeral_root(tmp_root) + + +def _meta_guard(spec: RunSpec, *, force: bool) -> RunMeta: + """Config-drift + one-way-unseal guards; returns the (possibly new) meta.""" + snapshot = spec.snapshot() + meta = RunMeta.load(spec.work_dir) + if meta is None: + return RunMeta.create(spec.work_dir, snapshot) + if meta.unsealed_at and not force: + print( + f"this run was unsealed at {meta.unsealed_at} " + f"(reason: {meta.finalize_reason}); resuming would leak test " + "numbers into decisions. Start a fresh work_dir, or pass --force " + "to continue with retention marked invalid.", + file=sys.stderr, + ) + raise SystemExit(2) + if not meta.check_config(snapshot) and not force: + print( + "config drift: the effective configuration differs from the one " + "this work_dir was started with (same-regime rule, SOP §0). " + "Candidate and control arms would no longer be comparable. " + "Use a fresh work_dir for the new config, or --force to override.", + file=sys.stderr, + ) + if spec.base_sha_defaulted: + print( + "note: base_sha is omitted in the yaml, so it re-resolved to " + f"the current HEAD ({spec.base_sha[:12]}); if the repo gained " + "commits since the run started, pin base_sha to the original " + f"root ({meta.config_snapshot.get('base_sha', '?')[:12]}) to " + "resume.", + file=sys.stderr, + ) + raise SystemExit(2) + return meta + + +def _run_rounds(spec: RunSpec, bundle: BenchBundle): + from raven.evolver.orchestrator.state.journal import RoundJournal + + orch = bundle.build_orchestrator() + journal = RoundJournal(bundle.journal_path) + result = orch.run(bundle.root_node_id, journal=journal, root_node=bundle.root_node) + return orch, journal, result + + +def _unseal_and_report( + spec: RunSpec, bundle: BenchBundle, orch, records: list[dict], meta: RunMeta, reason: str +) -> bool: + """Returns True on success; False when unseal scoring failed (not stamped).""" + if bundle.unseal is None: + _say("no sealed test set configured; skipping unseal") + if not meta.unsealed_at: + meta.stamp_unsealed(reason=f"{reason} (no sealed test)") + return True + _say("unsealing: blind-scoring deliverables on the sealed test set …") + try: + report = bundle.unseal(records, orch) + except (RuntimeError, OSError, subprocess.CalledProcessError) as exc: + print(f"unseal scoring failed: {exc}", file=sys.stderr) + print( + "the run is NOT stamped; fix the environment and re-run the same command to retry the unseal", + file=sys.stderr, + ) + return False + # Stamp before writing the report: the stamp is what stops a resume, so + # no crash window may leave test numbers on disk in a still-resumable + # run. A crash after the stamp leaves a final run with retention.json + # missing — `finalize` detects that and recomputes. + if not meta.unsealed_at: + meta.stamp_unsealed(reason=reason) + atomic_write_json(Path(spec.work_dir) / "retention.json", report) + _say(f"retention report -> {Path(spec.work_dir) / 'retention.json'}") + for key in ( + "best_round", + "best_node_id", + "best_train", + "best_test", + "vanilla_train", + "vanilla_test", + "retention", + "sealed_credited_2sigma", + ): + if isinstance(report, dict) and key in report: + _say(f" {key}: {report[key]}") + return True + + +def cmd_run(config_path: str, *, smoke: bool = False, force: bool = False) -> int: + spec = _load_spec(config_path, smoke) + _note_defaulted_base(spec) + + # Build (validate) the bundle before creating run_meta: a first launch + # that dies on a config mistake must not leave a fingerprint behind, or + # the corrected config would be refused as drift on the next attempt. + bundle = _build_bundle(spec, with_models=True) + + spec.work_dir.mkdir(parents=True, exist_ok=True) + _claim_ephemeral_root(spec) + meta = _meta_guard(spec, force=force) + meta.config_snapshot.setdefault("resolved_models", describe_models(spec.models)) + meta.save() + + done, total = bundle.cold_start_done(), bundle.cold_start_total + if done < total: + _say(f"phase 1/3 cold start: {done}/{total} trials present, running the rest …") + else: + _say(f"phase 1/3 cold start: {total} trials present, verifying the infra-rerun ladder …") + # Always invoked: run_cold_start is idempotent (fills missing trials only) + # and owns the infra-rerun ladder, which may have salvage work to do even + # when every base trial file exists. + try: + bundle.run_cold_start() + except KeyboardInterrupt: + _say( + f"interrupted during cold start " + f"({bundle.cold_start_done()}/{total} trials done and kept); " + "re-run the same command to continue" + ) + return 130 + except (subprocess.CalledProcessError, OSError, RuntimeError) as exc: + print(f"cold start failed: {exc}", file=sys.stderr) + print( + f"completed trials are kept " + f"({bundle.cold_start_done()}/{total} present); fix the " + "environment and re-run the same command to resume", + file=sys.stderr, + ) + return 1 + done = bundle.cold_start_done() + if done < total: + _say(f"cold start still incomplete ({done}/{total}); re-run to retry the missing trials") + return 1 + _say(f"phase 1/3 cold start complete ({total} trials)") + + _say("phase 2/3 evolution rounds (interrupt any time; same command resumes)") + _say( + f" live progress: {spec.work_dir}/findings.md (per-round log), " + f"{spec.work_dir}/journal/rounds.jsonl (checkpoints)" + ) + try: + orch, journal, result = _run_rounds(spec, bundle) + except KeyboardInterrupt: + _say( + "interrupted — completed rounds are journaled; " + "re-run the same command to resume, `status` to inspect, " + "`finalize` to stop here and unseal" + ) + return 130 + except RuntimeError as exc: + # Environment-shaped failures (Gate0 precheck, dead endpoint) are + # actionable messages, not tracebacks; completed work is durable. + print(f"run stopped: {exc}", file=sys.stderr) + print("fix the environment and re-run the same command to resume", file=sys.stderr) + return 1 + + for rr in result.rounds: + _say(f"round {rr.round_index}: parent={rr.parent_id} promoted={rr.promoted} -> {rr.next_parent_id}") + _say(f"stopped: {result.stop_reason}; final parent: {result.final_parent_id}") + + _say("phase 3/3 unseal") + ok = _unseal_and_report(spec, bundle, orch, journal.load(), meta, reason=result.stop_reason or "terminated") + return 0 if ok else 1 + + +def cmd_check(config_path: str, *, smoke: bool = False) -> int: + """Validate everything cheap before spending: config, models, bench setup. + + Builds the model call_fns (catches a missing claude binary / bad spec) and + the bench bundle (catches dead whitelist prefixes, missing task files or + subject config, absent AppWorld install) without running anything. + """ + spec = _load_spec(config_path, smoke) + _note_defaulted_base(spec) + bundle = _build_bundle(spec, with_models=True) + _say(f"bench: {spec.bench} (root {spec.base_sha[:12]} @ {spec.repo_root})") + _say(f"work_dir: {spec.work_dir}") + _say(f"models: {describe_models(spec.models)}") + _say( + f"funnel: k_screen={spec.funnel.k_screen} k_confirm={spec.funnel.k_confirm} " + f"budget={spec.funnel.budget.max_why_per_round}x" + f"{spec.funnel.budget.candidates_per_why} " + f"rounds<={spec.funnel.termination.max_rounds}" + ) + _say(f"cold start: {bundle.cold_start_done()}/{bundle.cold_start_total} trials present") + _say(f"sealed test: {'configured' if bundle.unseal else 'not configured'}") + if bundle.precheck is not None: + _say("bench precheck: probing environment + subject endpoint …") + try: + bundle.precheck() + except RuntimeError as exc: + print(f"bench precheck failed: {exc}", file=sys.stderr) + return 1 + _say("bench precheck: OK") + _say("check OK — ready to run") + return 0 + + +def _node_status_counts(work_dir: Path) -> dict[str, int]: + counts: dict[str, int] = {} + for path in sorted(Path(work_dir).glob("nodes/*.json")): + rec = load_json_or(path, {}) + status = rec.get("status", "?") + counts[status] = counts.get(status, 0) + 1 + return counts + + +def cmd_status(config_path: str, *, smoke: bool = False) -> int: + spec = _load_spec(config_path, smoke) + meta = RunMeta.load(spec.work_dir) + if meta is None: + _say(f"no run state under {spec.work_dir} (phase 0: not started)") + return 0 + _say(f"work_dir: {spec.work_dir}") + _say(f"started: {meta.created_at} config: {meta.config_hash}") + if meta.unsealed_at: + _say(f"UNSEALED at {meta.unsealed_at} ({meta.finalize_reason}) — run is final") + report = load_json_or(Path(spec.work_dir) / "retention.json", None) + if report: + _say(f"retention report: {Path(spec.work_dir) / 'retention.json'}") + return 0 + + bundle = _build_bundle(spec, with_models=False) + done, total = bundle.cold_start_done(), bundle.cold_start_total + if done < total: + _say(f"phase 1: cold start {done}/{total} trials — no results yet") + return 0 + + from raven.evolver.orchestrator.state.journal import RoundJournal + + records = RoundJournal(bundle.journal_path).load() + if not records: + _say("phase 2: cold start done, no completed rounds yet") + return 0 + _say(f"phase 2: {len(records)} completed round(s); test stays sealed until termination or `finalize`") + counts = _node_status_counts(spec.work_dir) + if counts: + _say("candidates by status: " + ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))) + promoted = [] + for rec in records: + _say( + f" round {rec.get('round_index')}: promoted={rec.get('promoted')} " + f"beat_vanilla={rec.get('beat_vanilla')} " + f"parent -> {rec.get('next_parent_id')}" + ) + if rec.get("promoted") and rec.get("next_parent_sha"): + promoted.append((rec.get("next_parent_id"), rec.get("next_parent_sha"), rec.get("next_parent_train"))) + if promoted: + _say("promoted commits (train-side numbers only):") + for node_id, sha, train in promoted: + _say(f" {node_id} @ {sha} train={train}") + return 0 + + +def cmd_finalize(config_path: str, *, smoke: bool = False, yes: bool = False) -> int: + spec = _load_spec(config_path, smoke) + meta = RunMeta.load(spec.work_dir) + if meta is None: + print("nothing to finalize: run was never started", file=sys.stderr) + return 2 + recompute = bool(meta.unsealed_at) + if recompute and (Path(spec.work_dir) / "retention.json").is_file(): + _say(f"already unsealed at {meta.unsealed_at}; see retention.json") + return 0 + + bundle = _build_bundle(spec, with_models=False) + if recompute: + if bundle.unseal is None: + # Finalized without a sealed test: there is no report to rebuild. + _say(f"already finalized at {meta.unsealed_at} ({meta.finalize_reason}); no sealed test was configured") + return 0 + _say( + "unseal stamp present but retention.json is missing (interrupted " + "unseal); recomputing the report — the run stays final" + ) + from raven.evolver.orchestrator.state.journal import RoundJournal + + records = RoundJournal(bundle.journal_path).load() + if not records: + print("nothing to finalize: no completed rounds in the journal", file=sys.stderr) + return 2 + if not yes and not recompute: + what = ( + "and unseal the test set" + if bundle.unseal is not None + else "(no sealed test configured — no test numbers exist)" + ) + print( + f"finalize will END this run after {len(records)} round(s) {what} " + "— it cannot be resumed afterwards. Re-run with --yes.", + file=sys.stderr, + ) + return 2 + + # Unsealing scores nodes on test: the orchestrator (hence models for its + # construction path) is not needed, but the bench's eval is — the bundle + # closures carry it. vanilla_train comes from the built orchestrator, so + # build it without LLM roles (they are only called during rounds). + _claim_ephemeral_root(spec) + orch = bundle.build_orchestrator() + ok = _unseal_and_report(spec, bundle, orch, records, meta, reason="user_finalized") + return 0 if ok else 1 + + +__all__ = ["cmd_run", "cmd_status", "cmd_check", "cmd_finalize"] diff --git a/raven/evolver/launch/state.py b/raven/evolver/launch/state.py new file mode 100644 index 0000000..b40d870 --- /dev/null +++ b/raven/evolver/launch/state.py @@ -0,0 +1,130 @@ +"""Durable run state: atomic JSON writes + the run_meta.json lifecycle. + +Two invariants the resume story depends on: + +- Every JSON this layer persists goes through :func:`atomic_write_json` + (write tmp + rename), so a crash can never leave a half-written file that + poisons the next resume. +- ``run_meta.json`` carries the config snapshot (a run's configuration must + not drift mid-run — SOP §0 discipline, enforced here rather than by memory) + and the one-way ``unsealed_at`` stamp (once test numbers are revealed, + continuing the run would leak them into decisions). +""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +META_FILENAME = "run_meta.json" + + +def atomic_write_json(path: Path, obj: Any) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=path.name, suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + json.dump(obj, f, indent=2, ensure_ascii=False) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def load_json_or(path: Path, default: Any) -> Any: + try: + return json.loads(Path(path).read_text()) + except (OSError, ValueError): + return default + + +def config_fingerprint(snapshot: dict) -> str: + """Stable hash of the effective run configuration (order-insensitive).""" + canon = json.dumps(snapshot, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha256(canon.encode()).hexdigest()[:16] + + +def _utcnow() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +@dataclass +class RunMeta: + """The per-run lifecycle record under ``work_dir/run_meta.json``.""" + + work_dir: Path + created_at: str = "" + config_snapshot: dict = field(default_factory=dict) + config_hash: str = "" + unsealed_at: Optional[str] = None + finalize_reason: Optional[str] = None + + @property + def path(self) -> Path: + return Path(self.work_dir) / META_FILENAME + + @classmethod + def load(cls, work_dir: Path) -> Optional["RunMeta"]: + data = load_json_or(Path(work_dir) / META_FILENAME, None) + if not isinstance(data, dict): + return None + return cls( + work_dir=Path(work_dir), + created_at=data.get("created_at", ""), + config_snapshot=data.get("config_snapshot", {}), + config_hash=data.get("config_hash", ""), + unsealed_at=data.get("unsealed_at"), + finalize_reason=data.get("finalize_reason"), + ) + + @classmethod + def create(cls, work_dir: Path, snapshot: dict) -> "RunMeta": + meta = cls( + work_dir=Path(work_dir), + created_at=_utcnow(), + config_snapshot=snapshot, + config_hash=config_fingerprint(snapshot), + ) + meta.save() + return meta + + def save(self) -> None: + atomic_write_json( + self.path, + { + "created_at": self.created_at, + "config_snapshot": self.config_snapshot, + "config_hash": self.config_hash, + "unsealed_at": self.unsealed_at, + "finalize_reason": self.finalize_reason, + }, + ) + + def check_config(self, snapshot: dict) -> bool: + """True iff ``snapshot`` matches the run's recorded configuration.""" + return config_fingerprint(snapshot) == self.config_hash + + def stamp_unsealed(self, reason: str) -> None: + """One-way: after this, ``run`` refuses to resume without --force.""" + self.unsealed_at = _utcnow() + self.finalize_reason = reason + self.save() + + +__all__ = [ + "RunMeta", + "atomic_write_json", + "config_fingerprint", + "load_json_or", + "META_FILENAME", +] diff --git a/raven/evolver/orchestrator/DESIGN.md b/raven/evolver/orchestrator/DESIGN.md new file mode 100644 index 0000000..9c5c339 --- /dev/null +++ b/raven/evolver/orchestrator/DESIGN.md @@ -0,0 +1,182 @@ +# Self-Evolution Orchestration Harness — design notes + +> Solidify a self-evolution SOP that used to depend heavily on Claude into a +> deterministic harness, so that weaker models (qwen / kimi) can drive the +> whole loop too. + +--- + +## 1. What this SOP does + +**One sentence:** make an agent's harness (the outer structure around the +model — system prompts, tools, hooks, memory) improve itself iteratively and +grow stronger on a benchmark. The means is not swapping models or injecting +skills, but looping: **diagnose failures -> design patches -> verify +rigorously -> keep what works, cull what doesn't**. + +### How it runs (cold start + the seven-step loop) + +- **Cold start:** run the unmodified baseline (vanilla) K times over the + train set to produce a **complete baseline record**. Then analyze the + failing trajectories one by one, classify the failure causes (empty + response never submitted / submitted without verifying / bad tool calls / + time-arithmetic errors ...) into a **failure map**. The baseline record is + also the **fixed comparison bar for the whole run**. + +- **① Diagnose:** read the previous round's failing trajectories and classify + them (round 1 uses the cold-start failure map directly). The failure map + **accumulates across rounds**, making the failure distribution's drift + auditable as evolution proceeds. + +- **② Choose targets + design candidates:** pick 1-2 failure causes worth + attacking this round; design 2-3 candidate patches each, across change + levers (prompt / config / runtime logic). **Key constraint:** every patch + is env-var gated and default-off — byte-identical to the baseline when + off, so the baseline is never contaminated. + +- **③ Free pruning:** at zero GPU cost, cull candidates that cannot possibly + take effect (trigger condition never hit in historical trajectories = + would never activate if fielded). + +- **④ Apply and create a node:** apply the patch to the parent, derive a + child version (git branch + commit), and register it in the node ledger. + +- **⑤a Screen:** probe quickly on a small representative subset (the anchor, + K=1). The verdict is **generous-pass**: only cull candidates clearly below + the baseline; slightly-low or inside the noise band always advance — a + single small-sample run has too much variance to conclude anything. + +- **⑤b Full confirm:** screen survivors run the full train set K times, + evaluated seriously. + +- **⑥ Three gates:** confirm the lift is real, not an artifact — + - *measurement validity:* infrastructure failures (crash/timeout) are not + scored as 0; rerun or handle explicitly; + - *attribution correctness:* credit only on tasks where the patch's + mechanism actually fired; + - *significance:* **paired comparison** against the baseline; the lift must + clear 2σ to count. + +- **⑦ Choose parent + termination check:** the round's best candidate enters + the bank and becomes the next parent; if nobody beat the baseline, keep the + old parent. Then check termination: **10 consecutive rounds without a + candidate above the baseline** (exploration exhausted) or the **20-round + cap**. **Iron law: never decide termination from the test set** — the test + set stays sealed throughout; it is opened only after evolution ends, to + compute the generalization retention rate. + +### The core division of labor + +Judgment work (diagnosis, design, decisions) goes to a model capable of +reasoning; arithmetic work (anchor selection, noise-band computation, gate +verdicts, aggregation) **must** be done by deterministic code — experiments +need to be reproducible and auditable. + +### The problem + +This SOP currently **depends on Claude**: Claude can read long specs, follow +instructions faithfully across a single very long session, and run dozens of +steps without drifting or stopping early. Swap in a weaker model and it +fails — weak instruction following, premature wrap-up, broken long chains. + +--- + +## 2. Design idea: inversion of control + +What used to live in the prompt ("how the loop advances, when to stop, do not +exit early") and rely on model discipline moves **into deterministic code (a +state machine)**; the model only makes one **scope-limited judgment with +schema validation + bounded retries** at each step. A weaker model only has +to answer one small question well at a time; "keep running without breaking" +is guaranteed by code. + +**Two model roles:** + +| Role | Who it is | Run by | +|---|---|---| +| **subject model** (being evolved) | the model that runs benchmark tasks | the external scorer (EvoAgentBench / AppWorld) | +| **driver model** (driving evolution) | the model doing diagnosis / design / decisions | this harness's semantic-node layer | + +The point of solidifying is exactly that **the driver can be swapped for a +weaker model**. + +--- + +## 3. SOP steps <-> code modules + +| SOP step | Code | Role | +|---|---|---| +| Seven-step loop + rounds + "no early exit" | `loop.py` (EvolutionOrchestrator) | deterministic state machine owning all control flow | +| Termination (10 / 20 rounds, vs **vanilla** only) | `termination.py` + `loop.py`'s `beat_vanilla` | patience signal = "some candidate's full-train confirm beat vanilla's fixed mean", independent of which baseline gates use (incl. the per-parent ratchet); errored rounds don't burn patience — separate counter (`errors_exhausted`) | +| ① Diagnose | `nodes/diagnose.py` | reuses the existing judge's prompts and parser + schema validation / repair retries; emits the failure map | +| Cross-round failure-map accumulation | `nodes/diagnose.py` `merge_failure_maps` | merged into the living map and persisted each round | +| ② Choose targets + design | `nodes/design.py` | deterministic WHY selection + driver-designed candidate patches, schema-validated | +| ③ Free pruning | `loop.py`'s `preflight(cand, parent)` hook + `production.make_zero_hit_preflight`; the design step's sandbox also runs an AST compile check + import smoke | catches both kinds of dead candidate: **crashers** (bad imports / module-level NameError — import smoke) and **inert ones** (driver-declared `TRIGGER_REGEX` with zero hits across the parent's failing trajectories = would never fire; culled at zero inference cost and recorded as a preflight pseudo-outcome, never silently dropped). No predicate / bad regex / no corpus all pass (fail-open: prune only on positive evidence of inertness) | +| ④ Apply + create node | reuses `tree/store.py` `create_child_node` | git apply + kernel protection + ledger registration | +| ⑤a Screen + generous-pass | `nodes/screen.py` (SWE anchor line) / `gates/strategies.py` FocusedFisher stage-1 (AppWorld line) | both lines keep generous-pass: the SWE line culls only below baseline by > `1.5×σ`; the AppWorld line culls only when the focused probe is **significantly worse** (reverse Fisher p<α) or sentinel regression exceeds one flaky trial (>1.5/(n·K)); slightly-low / indistinguishable always advances to the full set | +| ⑤b Full confirm | `benchmarks/evoagentbench/evolve/adapter.py` / `benchmarks/appworld/evolve/adapter.py` `run_eval` | the piece that was genuinely missing — submit the candidate to the scorer, read per-task pass counts back | +| ⑥ Three gates | `gates/pipeline.py` (Gate-f -> Gate-b -> `gates/paired.py`), shared by both lines | navigator promotion (mean beats control, the SOP's loose caliber; FocusedFisher can add `min_confirm_lift`) + an independent credited-2σ label. **Gate-b (active on the AppWorld line):** python patches must carry `activation_beacon()` (missing = rejected, checked post-edit); `batch.py` injects a private beacon dir per task-attempt subprocess; `fired_source` reads the per-task firing table back from the confirm out-dir + infra-ladder union; attribution only on fired tasks. Three-state semantics: no instrumentation data (no `.enabled` marker / beacon-less candidate) = None = fail-open skip; marker present with zero firings = an honest zero = no attribution; prompt/config patches are beacon-exempt, attribution covered by the focused probe + sentinels | +| ⑦ Choose parent + verdict | `loop.py` + `nodes/verdict.py` | **argmax parent selection (paper Alg.1 L135):** the round's best gated candidate takes over only if its train score strictly beats the incumbent parent; ties/lower enter the bank without taking over (under the frozen-vanilla baseline a lower scorer could once displace a higher parent — fixed); the driver drafts each round's verdict | +| GSME bank + cross-cell recombination (paper §3.3) | `archive.py` + `production.make_git_recombine_fn` | each (WHERE x WHY) cell keeps one elite that "beat vanilla on full confirm" (**bank bar = the vanilla navigation caliber**, deliberately not the ratchet promotion bar — an independently-effective mechanism that loses to the current champion still banks); each round appends up to `budget.recombinations_per_round` recombinant candidates beyond the designed ones: read the elite commit's changed file bytes, stack onto the current parent, walk the **same** apply->gate pipeline. Same-cell / same-file-conflict / already-tried pairings auto-excluded; pairing outcomes recorded to avoid re-proposing during patience | +| WHERE mechanical binding (paper App. A) | `archive.bind_where` | the cell coordinate's WHERE lever derives from the files the patch **actually touched** (path/suffix -> prompt/knowledge/runtime/config; cross-lever = mixed; no files = edit); the driver's self-declared `patch_where` stays in the ledger for audit and never decides the coordinate — zero modeling noise on the WHERE axis, noise concentrated on WHY | +| ①'s flip analysis (rescued/regressed) | `scoring.flip_summary` + per-candidate ledger writes in the loop | every full-confirm candidate is compared per-task against the round's control: rescued / regressed / still-failing, written into outcome.stats, the node ledger, `failure_map.json`'s `_flips`, and via outcome_hook into design history — next round's designer sees causal feedback (who got cured / who got broken), not just a static failure set | +| Division: judgment=model, arithmetic=code | `nodes/semantic.py` | every semantic call goes through schema enforcement + bounded repair retries (the core backstop for weak drivers) | +| Broad driver support: qwen/kimi/claude | `providers/openai_compat.py` + reuse of `judge/llm_client` | OpenAI-compatible endpoints, reasoning models handled | +| Sealed-test iron law | `sealed/runner.py` | test blind-scored into a driver-invisible directory, returning None; leak guard + retention. **Unseal picks the deliverable by train argmax only (paper Alg.1 L140)**, never max-over-test (max over many measurements = selection effect, systematically inflating Δ/retention); the per-round curve is display-only; the report attaches the sealed paired z + credited-2σ label | +| State persistence / resumability | `state/journal.py` + failure-map persistence + `loop.py` writing node ledgers/findings | per-round checkpoints; a killed process resumes; on resume the parent node is rebuilt from the journal's recorded commit SHA, and the AppWorld baseline falls back to rebuilding from the confirm out-dir | +| Scorer format adaptation | `benchmarks/evoagentbench/evolve/adapter.py` + `benchmarks/appworld/evolve/adapter.py` | both benchmarks unified onto the `TaskEval` contract; the upper layers are benchmark-agnostic | + +--- + +## 3.5 Two cross-module conventions (required reading for wiring a new benchmark) + +**Two candidate-activation paradigms — pick one, never mix:** + +| Paradigm | How a candidate takes effect | Gate-b / preflight | Where used | +|---|---|---|---| +| **env-gated** (SOP §2 ② original) | the patch is committed but env-var gated, default off = byte-identical to vanilla; eval passes `activation_env` | has an activation ledger -> Gate-b attribution and zero-hit preflight available (wire `fired_source`) | SWE / framework line | +| **commit-checkout** (AppWorld line) | the candidate = a real child commit; eval checks the commit out into a worktree and runs there (`cwd=worktree`); the code *is* the candidate, ungated | **has** an activation ledger: python patches must embed `activation_beacon()`, batch injects a beacon dir per attempt, `read_fired_tasks` reads back -> per-task Gate-b attribution; prompt/config patches exempt (focused probe + sentinels cover). The EvoAgentBench line (one subprocess runs the whole job; per-trial injection impossible) is not wired yet — fail-open | AppWorld, and the default for new benches | + +New benches default to commit-checkout (cleaner: git lineage guarantees zero +vanilla contamination, and weak models edit files far more reliably than they +emit valid unified diffs); the `activation_env` path in +`make_appworld_backend` is an SWE leftover — don't use both on a new line. + +**Confirm-artifact naming is a cross-module contract:** the gate's full +confirm job name = `gates/strategies.confirm_job_name(node_id)` +(=`_confirm`); it is simultaneously the dir-type scorer's out-dir +name, and next round's diagnosis finds the parent's failing trajectories by +it. Both reader and writer must use this function — never a hand-written +f-string. + +**work_dir on-disk layout** (maintained by the loop; all best-effort, all +resumable): + +``` +work_dir/ + journal/*.jsonl # per-round checkpoints (resume control state + the sealed unseal's curve material) + nodes/.json # node ledger: identity + git anchor + final status + gate stats + candidate metadata + # (WHY/WHERE/files/beacon/predicate/recombination source — fills the audit fields + # when a bench candidate is not an AppliedPatch and patch is null) + findings.md # human-readable per-round record (factual summary + driver verdict) + failure_map.json # cross-round accumulated living map (incl. _diagnosed_parents: no duplicate diagnosis) + archive.json # GSME bank: per-cell elites + promotion lineage (cells/files) + tried recombination pairs; auto-reloaded on resume + history.json # cross-round per-WHY attempt history (incl. flip counts); auto-reloaded on resume — the design step never forgets + taxonomy.json # WHY/WHERE taxonomy discovered in induce mode (induced once, reused) + taxonomy_seed.json # seed failure map emitted during induction; fed to the loop so round 1 skips re-judging the same trajectories +``` + +--- + +## 4. Validated + +- **Offline alignment:** recomputed on real SWE round-3 data — σ_screen + 7.3pp, cull line 26.3%, confirm 77.2% match exactly; this also surfaced and + fixed a navigator-vs-2σ semantic confusion. +- **One real AppWorld round:** real qwen3.6-27B diagnosed failures -> + autonomously picked VERIFY_FINALIZE (matching the empty-response problem) + -> ran the candidate -> gates ruled it did not beat the baseline -> + correctly not promoted, clean termination. The full closed loop ran on real + components. +- Unit tests: everything orchestrator-related passes (full evolver suite + 545 passed / 16 skipped). diff --git a/raven/evolver/orchestrator/__init__.py b/raven/evolver/orchestrator/__init__.py new file mode 100644 index 0000000..c485ec9 --- /dev/null +++ b/raven/evolver/orchestrator/__init__.py @@ -0,0 +1,26 @@ +"""Self-evolution orchestrator — a deterministic harness over the seven-step funnel. + +The canonical evolver package (``raven.evolver.{scheduler,analysis,judge,tree}``) +ships every deterministic operator the self-evolution SOP needs — anchor +selection, stability bucketing, the tree-aware bandit, the failure-map +builder, the node ledger, git ops, and the LLM judge. What it never had is a +faithful *driver* of the SOP's seven-step funnel. + +This package supplies that missing layer. It is a small finite-state machine +that owns the control flow the SOP used to delegate to a long, high-compliance +Claude session: + +- the round loop and per-candidate fork, +- the "never stop early" discipline (a code counter, not a prompt), +- the termination conditions (10 rounds with no vanilla-beating candidate, + or a hard cap of 20 rounds), +- and state persistence so no model has to hold cross-round context. + +The driver model (diagnose / design / verdict) only ever makes small, +schema-validated calls through the existing judge backends +(``raven.evolver.judge.llm_client``), which already route self-hosted Qwen, +Claude, and OpenRouter models. That is what lets a weaker model drive the loop: +the harness carries the control-flow burden the model cannot. +""" + +from __future__ import annotations diff --git a/raven/evolver/orchestrator/archive.py b/raven/evolver/orchestrator/archive.py new file mode 100644 index 0000000..82fd89f --- /dev/null +++ b/raven/evolver/orchestrator/archive.py @@ -0,0 +1,388 @@ +"""GSME archive — one gated elite per (WHERE x WHY) cell, plus recombination. + +The paper's Gated Semantic MAP-Elites has three parts: a categorical archive +keyed on the pathology a patch addresses (one elite per cell, entered only +after the gates), quality-biased selection (each round extends the current +best harness — the loop's greedy parent selection), and cross-cell +recombination (the best harness is stacked with elites from OTHER pathology +cells). The loop always had the middle part; this module adds the other two: + +- :meth:`GsmeArchive.consider` observes every gated outcome. A candidate whose + full-train confirm beat the FIXED vanilla mean (the paper's navigation bar — + deliberately vanilla, not the possibly-ratcheted promotion baseline, so an + independently-good mechanism is banked even when it loses to the current + champion) enters/replaces its cell's elite. +- :meth:`GsmeArchive.eligible_elites` proposes recombination targets for a + parent: elites from cells NOT already stacked into the parent's lineage, + skipping pairings already tried and pairs whose edits touch the same files + (full-file-bytes stacking would silently clobber one side of a same-file + overlap, mismeasuring the stack — such pairs are skipped, not merged). + +The archive never decides credit: a recombinant goes through the exact same +apply -> screen/confirm -> gate pipeline as a designed candidate. It only +steers which combinations get measured, so label noise in WHY degrades +coverage, never the quality of what is ultimately promoted (same argument as +the paper's descriptor-noise analysis). + +State persists to one JSON (``config.archive_path``) after every round, so a +resumed run keeps its elites, lineage metadata, and attempted pairings. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +from raven.evolver.orchestrator.gates.policy import CandidateOutcome +from raven.evolver.tree.node import AppliedPatch, HarnessNode, PatchWhy + + +@dataclass(frozen=True) +class CellElite: + """The gated elite of one (WHERE x WHY) cell. + + ``files`` / ``deletions`` are the repo-relative paths the elite's edit + changed vs its own parent commit — together with ``git_commit_sha`` they + are all a recombiner needs to re-materialise the mechanism onto a new + parent (the bytes are read back from the commit, so this record is + resume-safe without storing file contents). + """ + + cell: str + node_id: str + git_commit_sha: str + score: float + round_index: int + why: str + where: str + credited: bool = False + files: tuple[str, ...] = () + deletions: tuple[str, ...] = () + focused_task_ids: tuple[str, ...] = () + summary: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "cell": self.cell, + "node_id": self.node_id, + "git_commit_sha": self.git_commit_sha, + "score": self.score, + "round_index": self.round_index, + "why": self.why, + "where": self.where, + "credited": self.credited, + "files": list(self.files), + "deletions": list(self.deletions), + "focused_task_ids": list(self.focused_task_ids), + "summary": self.summary, + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "CellElite": + return cls( + cell=d["cell"], + node_id=d["node_id"], + git_commit_sha=d["git_commit_sha"], + score=float(d["score"]), + round_index=int(d["round_index"]), + why=d["why"], + where=d.get("where", "edit"), + credited=bool(d.get("credited", False)), + files=tuple(d.get("files") or ()), + deletions=tuple(d.get("deletions") or ()), + focused_task_ids=tuple(d.get("focused_task_ids") or ()), + summary=d.get("summary", ""), + ) + + +@dataclass +class RecombinantCandidate: + """A cross-cell recombination candidate: an elite's edit re-materialised + onto the current parent. + + Duck-types the bench candidate contract the wired lines share + (``files`` bytes / ``deletions`` / ``why`` / ``focused_task_ids`` / + ``summary``), so ``files_of`` / ``deletions_of`` / ``focused_source`` / + ``outcome_hook`` all consume it unchanged and it flows through the standard + apply -> gate pipeline like any designed candidate. ``elite_node_id`` marks + it for pairing bookkeeping and the audit trail. + """ + + files: dict[str, bytes] + why: str + cell: str + elite_node_id: str + focused_task_ids: list[str] = field(default_factory=list) + summary: str = "" + deletions: list[str] = field(default_factory=list) + # Inherited from the elite's bytes: a beacon-carrying mechanism keeps its + # Gate-b attribution when re-stacked (the code is byte-identical). + has_beacon: bool = False + + +# Path -> lever heuristics for the mechanical WHERE binding (paper App. A: +# the four levers over the harness's edit surfaces). Module-level so a bench +# with an exotic layout can monkeypatch/extend; unknowns default to runtime +# (a code edit is a control-flow change until proven otherwise). +_KNOWLEDGE_MARKERS = ("skills/", "skill/", "everos/", "memory") +_CONFIG_SUFFIXES = (".yaml", ".yml", ".json", ".toml", ".ini", ".cfg") +_PROMPT_SUFFIXES = (".md", ".txt", ".prompt") + + +def _lever_of_path(path: str) -> str: + p = path.replace("\\", "/").lower() + if any(m in p for m in _KNOWLEDGE_MARKERS): + return "knowledge" + if p.endswith(_CONFIG_SUFFIXES): + return "config" + if p.endswith(_PROMPT_SUFFIXES): + return "prompt" + return "runtime" + + +def bind_where(paths) -> str: + """Mechanically bind the WHERE lever from the files a patch touches. + + The paper's WHERE is read off the patch artifact, never self-declared — + that keeps the archive's WHERE axis noise-free while WHY carries all the + modeling error. A patch spanning several levers is its own ``"mixed"`` + coordinate (stacking it wholesale is still meaningful); no paths at all + (a metadata-only candidate) binds to ``"edit"``, the unknown lever. + """ + levers = {_lever_of_path(p) for p in paths} + if not levers: + return "edit" + if len(levers) == 1: + return levers.pop() + return "mixed" + + +def cell_of(cand: Any) -> Optional[tuple[str, str]]: + """The (WHERE-lever, WHY) coordinate of a candidate, or None when it has + no WHY. WHERE is bound mechanically from the touched files (:func:`bind_where`); + a driver-declared ``patch_where`` stays on the node ledger for audit but + never decides the archive coordinate.""" + if isinstance(cand, AppliedPatch): + why = cand.patch_why_extra if cand.patch_why == PatchWhy.other else cand.patch_why.value + paths = [c.target_file for c in cand.components] + return bind_where(paths), str(why) + why = getattr(cand, "why", None) + if why: + changed, deleted = _touched_paths(cand) + return bind_where(list(changed) + list(deleted)), str(why) + return None + + +def _touched_paths(cand: Any) -> tuple[tuple[str, ...], tuple[str, ...]]: + """(changed, deleted) repo-relative paths of a candidate's edit.""" + if isinstance(cand, AppliedPatch): + return tuple(c.target_file for c in cand.components), () + files = getattr(cand, "files", None) + changed = tuple(sorted(files)) if isinstance(files, dict) else () + deleted = tuple(getattr(cand, "deletions", None) or ()) + return changed, deleted + + +def describe_candidate(cand: Any) -> Optional[dict]: + """JSON-safe candidate metadata for the node ledger. + + The wired bench candidates are not :class:`AppliedPatch`, so a node record + would otherwise carry ``patch: null`` and lose the WHERE/WHY/touched-files/ + activation info the ledger is supposed to hold. Returns None for a + candidate with no coordinate (nothing to record). + """ + coord = cell_of(cand) + if coord is None: + return None + where, why = coord + changed, deleted = _touched_paths(cand) + d: dict[str, Any] = { + "why": why, + "where": where, + "files": list(changed), + "deletions": list(deleted), + "summary": str(getattr(cand, "summary", "") or "")[:300], + "has_beacon": bool(getattr(cand, "has_beacon", False)), + } + spec = getattr(cand, "activation_spec", None) + if isinstance(spec, dict): + d["activation_spec"] = spec + elite_id = getattr(cand, "elite_node_id", None) + if elite_id: + d["recombination_of"] = elite_id + return d + + +class GsmeArchive: + """The persistent GSME state: cell elites, lineage metadata, pairings. + + ``node_meta`` records, for every PROMOTED node, which cells its lineage has + stacked and which files that lineage touched — the exclusion sets + :meth:`eligible_elites` filters recombination proposals with. ``pairings`` + records every (parent, elite) recombination already attempted (whatever + its outcome), so a patience streak does not re-propose the same pair every + round. A parent with no metadata (the root, or a pre-archive journal) + just gets an empty lineage: redundant proposals are then caught by the + pairing record and, failing that, measured and rejected by the gate. + """ + + def __init__(self, path: str | Path): + self._path = Path(path) + self._cells: dict[str, CellElite] = {} + self._node_meta: dict[str, dict[str, list[str]]] = {} + self._pairings: dict[str, dict[str, str]] = {} + self._load() + + # ---- observation ------------------------------------------------------- + + def consider( + self, + *, + parent_id: str, + node: HarnessNode, + cand: Any, + outcome: CandidateOutcome, + vanilla_train_mean: float, + round_index: int, + ) -> None: + """Observe one gated outcome: record pairings, lineage, and elites.""" + elite_id = getattr(cand, "elite_node_id", None) + if elite_id: + self.record_pairing(parent_id, elite_id, outcome.status.value) + + coord = cell_of(cand) + if coord is None: + return + where, why = coord + key = f"{where}::{why}" + changed, deleted = _touched_paths(cand) + + if outcome.promoted: + pmeta = self._node_meta.get(parent_id, {}) + self._node_meta[node.node_id] = { + "cells": sorted(set(pmeta.get("cells", [])) | {key}), + "files": sorted(set(pmeta.get("files", [])) | set(changed) | set(deleted)), + } + + # Navigation bar (paper Alg. 1): full-train confirm beat VANILLA. A + # screen-pruned or vanilla-losing candidate never enters the archive. + if not outcome.confirm_evals or outcome.score <= vanilla_train_mean: + return + prev = self._cells.get(key) + if prev is not None and prev.score >= outcome.score: + return + self._cells[key] = CellElite( + cell=key, + node_id=node.node_id, + git_commit_sha=node.git_commit_sha, + score=outcome.score, + round_index=round_index, + why=why, + where=where, + credited=bool(outcome.paired and outcome.paired.credited_2sigma), + files=changed, + deletions=deleted, + focused_task_ids=tuple(getattr(cand, "focused_task_ids", None) or ()), + summary=str(getattr(cand, "summary", "") or "")[:300], + ) + + def record_pairing(self, parent_id: str, elite_node_id: str, status: str) -> None: + self._pairings.setdefault(parent_id, {})[elite_node_id] = status + + # ---- recombination proposals ------------------------------------------- + + def eligible_elites(self, parent_id: str, *, limit: int = 1) -> list[CellElite]: + """Elites worth stacking onto ``parent_id``, best score first. + + Excluded: cells already in the parent's lineage, the parent itself, + pairings already attempted, and elites whose edit overlaps a file the + lineage already changed (byte-level stacking cannot merge same-file + edits, only replace — an overlapping "stack" would silently drop one + mechanism and measure a lie). + """ + if limit <= 0: + return [] + meta = self._node_meta.get(parent_id, {}) + lineage_cells = set(meta.get("cells", [])) + lineage_files = set(meta.get("files", [])) + tried = self._pairings.get(parent_id, {}) + out: list[CellElite] = [] + ranked = sorted(self._cells.items(), key=lambda kv: (-kv[1].score, kv[0])) + for key, elite in ranked: + if key in lineage_cells: + continue + if elite.node_id == parent_id or elite.node_id in tried: + continue + if lineage_files & (set(elite.files) | set(elite.deletions)): + continue + out.append(elite) + if len(out) >= limit: + break + return out + + # ---- reporting ---------------------------------------------------------- + + def summary_text(self) -> str: + """One line per cell elite — injectable into a design prompt so the + driver knows which mechanisms are already banked.""" + if not self._cells: + return "" + lines = ["ARCHIVE (verified elites, one per failure cell):"] + for key in sorted(self._cells): + e = self._cells[key] + cred = " credited" if e.credited else "" + lines.append( + f"- {key}: {e.node_id} score={e.score:.3f}{cred} r{e.round_index}" + + (f" — {e.summary}" if e.summary else "") + ) + return "\n".join(lines) + + @property + def cells(self) -> dict[str, CellElite]: + return dict(self._cells) + + # ---- persistence --------------------------------------------------------- + + def save(self) -> None: + """Persist to ``path`` (best-effort, atomic tmp+rename).""" + try: + self._path.parent.mkdir(parents=True, exist_ok=True) + payload = json.dumps( + { + "cells": {k: e.to_dict() for k, e in self._cells.items()}, + "node_meta": self._node_meta, + "pairings": self._pairings, + }, + indent=2, + ) + tmp = self._path.with_suffix(".json.tmp") + tmp.write_text(payload) + tmp.replace(self._path) + except OSError: + pass + + def _load(self) -> None: + try: + if not self._path.exists(): + return + d = json.loads(self._path.read_text()) + self._cells = {k: CellElite.from_dict(v) for k, v in (d.get("cells") or {}).items()} + self._node_meta = { + k: {"cells": list(v.get("cells", [])), "files": list(v.get("files", []))} + for k, v in (d.get("node_meta") or {}).items() + } + self._pairings = {k: dict(v) for k, v in (d.get("pairings") or {}).items()} + except (OSError, ValueError, KeyError): + self._cells, self._node_meta, self._pairings = {}, {}, {} + + +__all__ = [ + "CellElite", + "GsmeArchive", + "RecombinantCandidate", + "bind_where", + "cell_of", + "describe_candidate", +] diff --git a/raven/evolver/orchestrator/config.py b/raven/evolver/orchestrator/config.py new file mode 100644 index 0000000..e2a41a4 --- /dev/null +++ b/raven/evolver/orchestrator/config.py @@ -0,0 +1,128 @@ +"""Orchestrator configuration — one object wiring the whole seven-step funnel. + +Everything the FSM needs that is *not* code: where the scorer lives +(``framework``), where the vanilla thick ledger sits (``cold_start_ledger_dir``, +the fixed comparison baseline — the funnel always compares against vanilla, not +the previous parent), the anchor/screen knobs, the per-round design budget, the +termination thresholds, and the on-disk state roots. + +The driver model is *not* constructed here — ``driver_llm_spec`` is the dict +handed to ``raven.evolver.judge.llm_client.build_judge_llm``, so the same +config serialises to yaml and a test can inject a ``MockBackend`` without a +factory. Broad model support (self-hosted Qwen, Claude, OpenRouter) comes for +free from that existing backend stack. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class AnchorParams: + """Slot budget + cull width for the K=1 anchor screen (see ``select_anchor``).""" + + # 12 = 6 stable + 6 borderline controls per candidate (stratified + rotated + # in loop._sentinels_for): regressions concentrate on borderline tasks, and + # 3 stable-only sentinels were observed missing a 58%-regression candidate. + n_sentinel: int = 12 + n_icebreaker: int = 5 + n_borderline: int = 7 + cull_sigma_mult: float = 1.5 + + +@dataclass(frozen=True) +class Budget: + """Per-round design budget (SOP §2 ②: 1-2 WHY x 2-3 candidates). + + ``recombinations_per_round`` caps the deterministic cross-cell GSME + recombinations appended after the designed candidates (0 disables). + """ + + max_why_per_round: int = 2 + candidates_per_why: int = 3 + driver_token_budget: int | None = None + recombinations_per_round: int = 1 + + +@dataclass(frozen=True) +class Termination: + """Loop stop conditions. Compared against vanilla; test is never consulted. + + ``patience`` is the SOP's primary exhaustion signal (consecutive rounds with + no candidate beating vanilla on train); ``max_rounds`` is the hard cap. + ``max_consecutive_errors`` stops a run whose rounds keep erroring out + (driver/infra outage) with an honest ``errors_exhausted`` reason instead of + letting the outage burn patience and masquerade as exploration exhaustion. + """ + + patience: int = 10 + max_rounds: int = 20 + max_consecutive_errors: int = 5 + + +@dataclass(frozen=True) +class OrchestratorConfig: + """Top-level orchestrator configuration — bench-neutral. + + Nothing here names a benchmark: the scorer, the dataset splits, the + cold-start baseline, and the anchor all live behind the injected + :class:`~raven.evolver.orchestrator.scoring.EvalBackend`. This object only + holds the driver model, the funnel's numeric knobs, and the on-disk state + roots. ``anchor`` (AnchorParams) is neutral tuning a backend factory may + consume when it builds a trial-ledger anchor. + """ + + repo_root: Path + work_dir: Path + driver_llm_spec: dict[str, Any] + + k_screen: int = 1 + k_confirm: int = 3 + anchor: AnchorParams = field(default_factory=AnchorParams) + budget: Budget = field(default_factory=Budget) + termination: Termination = field(default_factory=Termination) + + # Sealed test: scored by a script into a dir the driver never reads, so the + # sealed-test rule is enforced by isolation, not by driver discipline. + sealed_test_split: str = "test" + sealed_output_dir: Path | None = None + + def __post_init__(self) -> None: + for name in ("repo_root", "work_dir"): + object.__setattr__(self, name, Path(getattr(self, name))) + if self.sealed_output_dir is not None: + object.__setattr__(self, "sealed_output_dir", Path(self.sealed_output_dir)) + if self.k_screen < 1 or self.k_confirm < 1: + raise ValueError("k_screen and k_confirm must be >= 1") + + # Conventional on-disk state layout under work_dir. + @property + def nodes_dir(self) -> Path: + return self.work_dir / "nodes" + + @property + def failure_map_path(self) -> Path: + return self.work_dir / "failure_map.json" + + @property + def archive_path(self) -> Path: + return self.work_dir / "archive.json" + + @property + def findings_path(self) -> Path: + return self.work_dir / "findings.md" + + @property + def journal_dir(self) -> Path: + return self.work_dir / "journal" + + +__all__ = [ + "AnchorParams", + "Budget", + "Termination", + "OrchestratorConfig", +] diff --git a/raven/evolver/orchestrator/gates/__init__.py b/raven/evolver/orchestrator/gates/__init__.py new file mode 100644 index 0000000..dabb33f --- /dev/null +++ b/raven/evolver/orchestrator/gates/__init__.py @@ -0,0 +1,38 @@ +"""Post-confirm gates (SOP §2 ⑥): infra health, activation, and paired lift. + +Also home to the pluggable decision policy (``policy``/``strategies``) and the +focused-Fisher statistics (``fisher``) that let the two benchmark lines share +one round loop. +""" + +from __future__ import annotations + +from raven.evolver.orchestrator.gates.policy import ( + Baseline, + BaselineProvider, + CandidateOutcome, + DecisionContext, + FrozenColdStartBaseline, + GatePolicy, + PerParentFrozenBaseline, + SameSessionPairedBaseline, +) +from raven.evolver.orchestrator.gates.strategies import ( + FocusedFisherGate, + PairedTwoSigmaGate, + confirm_job_name, +) + +__all__ = [ + "Baseline", + "BaselineProvider", + "CandidateOutcome", + "DecisionContext", + "GatePolicy", + "FrozenColdStartBaseline", + "PerParentFrozenBaseline", + "SameSessionPairedBaseline", + "FocusedFisherGate", + "PairedTwoSigmaGate", + "confirm_job_name", +] diff --git a/raven/evolver/orchestrator/gates/fisher.py b/raven/evolver/orchestrator/gates/fisher.py new file mode 100644 index 0000000..e8ed126 --- /dev/null +++ b/raven/evolver/orchestrator/gates/fisher.py @@ -0,0 +1,91 @@ +"""Focused-subset statistics for the two-stage Fisher gate (SOP §2 ⑤/⑥). + +Ported from the AppWorld evolution driver so the two-stage gate is in-package +and unit-testable. The stage-1 test asks a sharp, cheap question on a candidate's +WHY subset: *is the candidate's pass-rate significantly above the baseline's on +exactly the tasks the pathology occurs?* A one-sided Fisher exact on the 2x2 +trial table answers it without a full-train run. + +Denominator discipline (SOP §0 hard rule): infra-contaminated trials are NOT +dropped from the denominator. Recoverable infra is salvaged upstream by the ≤2 +rerun ladder (:func:`raven.evolver.orchestrator.scoring.eval_with_infra_rerun`); +only genuinely persistent infra reaches here and counts as a non-pass (fail), +kept in the denominator. Dropping infra tasks would shrink the denominator and +overestimate pass@1 (the fair-subset extrapolation trap). +""" + +from __future__ import annotations + +import math + +from raven.evolver.orchestrator.scoring import TaskEval + + +def focused_counts(evals: dict[str, TaskEval], focused_ids: list[str]) -> tuple[int, int]: + """``(passes, fails)`` trial counts over the focused subset. + + Infra-still-fail trials count as fails, not excluded — a task is never + dropped (SOP §0). A task absent from ``evals`` (never launched) has no trials + to count and is skipped here; the full-train denominator that governs pass@1 + lives in :func:`train_mean`. + """ + passes = fails = 0 + for tid in focused_ids: + ev = evals.get(tid) + if ev is None: + continue + passes += ev.passes + fails += ev.attempts - ev.passes + return passes, fails + + +def train_mean(evals: dict[str, TaskEval], task_ids: list[str]) -> float: + """Mean per-task pass@1 over ``task_ids`` with a FIXED denominator (SOP §0). + + Denominator = ``len(task_ids)`` (the task count), always. A task missing or + all-infra contributes 0.0 — never dropped from the denominator, because + dropping it shrinks the denominator and overestimates pass@1. infra trials + count as non-passes via ``TaskEval.pass_rate`` (``passes / attempts``). + """ + if not task_ids: + return 0.0 + total = 0.0 + for t in task_ids: + ev = evals.get(t) + total += ev.pass_rate if ev is not None else 0.0 + return total / len(task_ids) + + +def fisher_one_sided(cp: int, cn: int, vp: int, vn: int) -> float: + """One-sided Fisher exact P(candidate pass-rate > vanilla) on a 2x2. + + Table ``[[cp, cn], [vp, vn]]`` = candidate pass/fail vs vanilla pass/fail + (trial counts). Returns 1.0 (not significant) for degenerate margins. + """ + row1, row2 = cp + cn, vp + vn + col1, tot = cp + vp, cp + cn + vp + vn + if row1 == 0 or row2 == 0 or col1 == 0 or col1 == tot: + return 1.0 + + def _p(a: int) -> float: + b, c, d = row1 - a, col1 - a, tot - col1 - (row1 - a) + if b < 0 or c < 0 or d < 0: + return 0.0 + return math.exp( + math.lgamma(row1 + 1) + + math.lgamma(row2 + 1) + + math.lgamma(col1 + 1) + + math.lgamma(tot - col1 + 1) + - math.lgamma(tot + 1) + - sum(math.lgamma(x + 1) for x in (a, b, c, d)) + ) + + hi = min(row1, col1) + return min(1.0, sum(_p(a) for a in range(cp, hi + 1))) + + +__all__ = [ + "focused_counts", + "train_mean", + "fisher_one_sided", +] diff --git a/raven/evolver/orchestrator/gates/paired.py b/raven/evolver/orchestrator/gates/paired.py new file mode 100644 index 0000000..5f37f10 --- /dev/null +++ b/raven/evolver/orchestrator/gates/paired.py @@ -0,0 +1,105 @@ +"""Gate2 — paired lift + 2σ significance (a generalisation of round7_paired). + +The scratchpad ``round7_paired.py`` hard-coded its anchor/explore task lists and +its two arms. This is the same statistics with the task set and the two arms +passed in, so it works for any round and any domain. + +Pairing is what makes the test sensitive: comparing candidate and vanilla on the +*same* tasks removes between-task difficulty, so the standard error is the +spread of the per-task differences, not the spread of raw pass rates. For each +task we take the per-task pass-rate difference ``d_i = rate_candidate,i - +rate_vanilla,i`` (K=3 rates), and test whether the mean difference is far enough +from zero: + + mean_lift = mean(d_i) + se = stdev(d_i) / sqrt(n) # paired standard error + z = mean_lift / se + +Promotion to bank (SOP §2 ⑥ Gate2) is the *navigator* condition alone: the +candidate's mean pass@1 beats vanilla on the shared train set. The paired 2σ +test is a separate *credited-significance* label (``credited_2sigma``) reported +alongside — it says whether the lift is statistically significant, not whether +the candidate banks. This matches the manual round-3 decision, where +budgetnudge banked on +6.4pp over vanilla even though its paired z (1.71) fell +short of 2σ. A candidate that improves every shared task identically has +``se == 0``; that deterministic win is reported as ``z = inf``. + +Whether a banked candidate becomes the next parent, or is pruned for a +qualitative reason (e.g. anchor/full-set sign-flip signalling interference), is +a semantic step ⑦ decision layered on top of this gate, not part of it. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from statistics import mean, stdev + +from raven.evolver.orchestrator.scoring import TaskEval + + +@dataclass(frozen=True) +class PairedResult: + """Outcome of the paired lift test between a candidate and a control arm.""" + + n_tasks: int + candidate_mean: float + control_mean: float + mean_lift: float + se: float + z: float + z_threshold: float + promoted: bool # navigator: candidate_mean > control_mean -> banks + credited_2sigma: bool # separate significance label: z >= z_threshold + + +def _rate(evals: dict[str, TaskEval], task_id: str) -> float: + ev = evals.get(task_id) + return ev.pass_rate if ev is not None else 0.0 + + +def paired_lift( + *, + candidate_evals: dict[str, TaskEval], + control_evals: dict[str, TaskEval], + task_ids: list[str], + z_threshold: float = 2.0, +) -> PairedResult: + """Paired lift + 2σ test over ``task_ids`` (candidate arm vs control arm). + + ``task_ids`` is the shared task set both arms were evaluated on (the full + train set for a confirm). A task missing from an arm scores 0.0 for that arm + — a candidate that failed to launch is not rewarded for the gap. + """ + if not task_ids: + raise ValueError("paired_lift requires a non-empty task list") + + diffs = [_rate(candidate_evals, t) - _rate(control_evals, t) for t in task_ids] + candidate_mean = mean(_rate(candidate_evals, t) for t in task_ids) + control_mean = mean(_rate(control_evals, t) for t in task_ids) + mean_lift = mean(diffs) + + n = len(task_ids) + se = stdev(diffs) / math.sqrt(n) if n > 1 else 0.0 + if se == 0.0: + z = 0.0 if mean_lift == 0.0 else math.copysign(math.inf, mean_lift) + else: + z = mean_lift / se + + promoted = candidate_mean > control_mean # navigator: banks on beating vanilla + credited_2sigma = z >= z_threshold + + return PairedResult( + n_tasks=n, + candidate_mean=candidate_mean, + control_mean=control_mean, + mean_lift=mean_lift, + se=se, + z=z, + z_threshold=z_threshold, + promoted=promoted, + credited_2sigma=credited_2sigma, + ) + + +__all__ = ["PairedResult", "paired_lift"] diff --git a/raven/evolver/orchestrator/gates/pipeline.py b/raven/evolver/orchestrator/gates/pipeline.py new file mode 100644 index 0000000..b260312 --- /dev/null +++ b/raven/evolver/orchestrator/gates/pipeline.py @@ -0,0 +1,109 @@ +"""The three-shield gate as a composable pipeline (SOP §2 ⑥). + +Order matters and each shield narrows the task set the next one judges: + +1. **Gate-f (measurement validity).** Report tasks whose eval was contaminated by + an infrastructure failure on *either* arm, but do NOT drop them from the + denominator (SOP §0 hard rule: detect -> rerun <=2 -> still-fail counts as 0, + kept in the denominator; dropping shrinks the denominator and overestimates + pass@1). Recoverable infra is salvaged upstream by the rerun ladder + (:func:`raven.evolver.orchestrator.scoring.eval_with_infra_rerun`); by the + time run_gates sees the evals, a still-infra trial is scored as a non-pass via + ``pass_rate``. Infra is read from ``infra_attempts`` when present (AppWorld + surfaces it; benches without it report nothing here). +2. **Gate-b (attribution).** Only credit the candidate on tasks where its + mechanism actually fired — a patch can't get credit for a task it never + touched. The set of fired tasks comes from an injectable source (the + activation ledger / beacon); when none is given this shield is a no-op. +3. **Gate2 (significance).** Paired lift on the surviving tasks: navigator + promotion (mean > vanilla) plus a separate credited-2σ label. + +Keeping this a pure function over eval maps makes it bench-agnostic and unit +testable; the loop calls it once per candidate after the confirm eval. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from raven.evolver.orchestrator.gates.paired import PairedResult, paired_lift +from raven.evolver.orchestrator.scoring import TaskEval + + +@dataclass +class GateResult: + """Combined verdict of the three-shield pipeline for one candidate. + + ``infra_contaminated`` is reported for audit only — those tasks stay in the + scoring denominator (SOP §0), scored by their ``pass_rate`` with infra trials + as non-passes. Only ``unfired_excluded`` (Gate-b attribution) narrows the set. + """ + + promoted: bool + paired: PairedResult | None + eligible_tasks: list[str] + infra_contaminated: list[str] = field(default_factory=list) + unfired_excluded: list[str] = field(default_factory=list) + + +def _infra_attempts(ev: TaskEval | None) -> int: + """Infra-trial count for an eval, or 0 when absent (SWE / no-bench leave 0).""" + return ev.infra_attempts if ev is not None else 0 + + +def run_gates( + *, + candidate_evals: dict[str, TaskEval], + control_evals: dict[str, TaskEval], + task_ids: list[str], + z_threshold: float = 2.0, + fired_tasks: set[str] | None = None, + infra_threshold: int = 1, +) -> GateResult: + """Run Gate-f -> Gate-b -> Gate2 and return the combined verdict. + + ``fired_tasks`` (Gate-b) restricts attribution when provided; ``None`` skips + that shield. ``infra_threshold`` is the per-arm infra-trial count at or above + which a task is *reported* as infra-contaminated (it is NOT dropped — SOP §0 + keeps it in the denominator scored 0). Only Gate-b can leave nothing to + measure; then the candidate is not promoted and ``paired`` is None. + """ + infra_contaminated = [ + t + for t in task_ids + if _infra_attempts(candidate_evals.get(t)) >= infra_threshold + or _infra_attempts(control_evals.get(t)) >= infra_threshold + ] + # Gate-f no longer excludes: infra tasks stay in the denominator (scored 0 + # via pass_rate). Only Gate-b attribution narrows the eligible set. + eligible = list(task_ids) + unfired_excluded: list[str] = [] + if fired_tasks is not None: + unfired_excluded = [t for t in eligible if t not in fired_tasks] + eligible = [t for t in eligible if t in fired_tasks] + + if not eligible: + return GateResult( + promoted=False, + paired=None, + eligible_tasks=[], + infra_contaminated=infra_contaminated, + unfired_excluded=unfired_excluded, + ) + + paired = paired_lift( + candidate_evals=candidate_evals, + control_evals=control_evals, + task_ids=eligible, + z_threshold=z_threshold, + ) + return GateResult( + promoted=paired.promoted, + paired=paired, + eligible_tasks=eligible, + infra_contaminated=infra_contaminated, + unfired_excluded=unfired_excluded, + ) + + +__all__ = ["GateResult", "run_gates"] diff --git a/raven/evolver/orchestrator/gates/policy.py b/raven/evolver/orchestrator/gates/policy.py new file mode 100644 index 0000000..943dc4a --- /dev/null +++ b/raven/evolver/orchestrator/gates/policy.py @@ -0,0 +1,245 @@ +"""Pluggable decision policy — the seam the two benchmark lines diverge on. + +The seven-step funnel's outer loop is identical across benchmarks; only the +per-candidate *decision* (screen -> confirm -> promote) and the *control arm* it +compares against differ. Those two concerns are captured here as two protocols +so ``loop._run_round`` stays a thin driver: + +- :class:`GatePolicy` — given a :class:`DecisionContext`, run whatever + screen/confirm/significance logic the bench uses and return one + :class:`CandidateOutcome`. The policy owns its own ``eval`` calls because the + two lines eval different task sets at different stages (K=1 anchor vs K=3 + focused subset). The loop never scores for itself anymore. +- :class:`BaselineProvider` — supply the control arm (a :class:`Baseline`) for + the round and absorb the ratchet on promotion. Frozen cold-start, per-parent + frozen, and same-session paired are the three implementations; the gate never + knows which produced its control, which is why the two seams are orthogonal. + +The empirical regime-shift finding (a frozen baseline compared across time is +unreliable when a whole run can shift ~12pp) is why +:class:`SameSessionPairedBaseline` exists and is the methodology-correct choice; +the frozen variants are kept as cost-bound fallbacks and labelled as such. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Callable, Optional, Protocol + +from raven.evolver.orchestrator.gates.fisher import train_mean +from raven.evolver.orchestrator.scoring import EvalFn, TaskEval +from raven.evolver.scheduler.anchor_selection import AnchorSelection +from raven.evolver.tree.node import HarnessNode, NodeStatus + +if TYPE_CHECKING: + from raven.evolver.orchestrator.gates.paired import PairedResult + from raven.evolver.orchestrator.gates.pipeline import GateResult + from raven.evolver.orchestrator.nodes.screen import ScreenResult + + +# (node, task_ids) -> fired task set, or None when there is no attribution +# data for this node (uninstrumented candidate / collection not wired) — +# None makes Gate-b fail OPEN (skip), an empty set is an honest "never fired". +FiredSourceFn = Callable[[HarnessNode, list[str]], Optional[set]] +FocusedSourceFn = Callable[[HarnessNode], list[str]] + + +@dataclass +class CandidateOutcome: + """What happened to one candidate as it moved through the funnel. + + ``score``/``confirm_evals``/``stats`` are policy-agnostic so parent + selection and the baseline ratchet stay in the loop, not in the policies. + The optional ``screen``/``paired``/``gate`` are the paired-line detail; + ``stats`` carries the Fisher-line detail (fisher_p, foc_c, full_lift, ...). + """ + + node_id: str + status: NodeStatus + score: float = 0.0 + confirm_evals: dict[str, TaskEval] = field(default_factory=dict) + screen: Optional["ScreenResult"] = None + paired: Optional["PairedResult"] = None + gate: Optional["GateResult"] = None + stats: dict = field(default_factory=dict) + + @property + def promoted(self) -> bool: + return self.status == NodeStatus.promoted_to_baseline + + +@dataclass(frozen=True) +class Baseline: + """The control arm for a round: per-task evals + their train mean + a label.""" + + evals: dict[str, TaskEval] + mean: float + label: str + + +@dataclass +class DecisionContext: + """Everything the loop hands a :class:`GatePolicy` to decide one candidate.""" + + node: HarnessNode + parent_id: str + round_index: int + eval: EvalFn + baseline: Baseline + train_task_ids: list[str] + anchor: Optional[AnchorSelection] = None + focused_task_ids: list[str] = field(default_factory=list) + # Stable-pass control tasks carried into the focused eval as a regression + # guard (SOP §2 5a, "2-3 all-pass sentinels"): a fix that helps its WHY subset must not + # break originally-passing tasks. Empty = no guard. + sentinel_task_ids: list[str] = field(default_factory=list) + fired_source: Optional[FiredSourceFn] = None + + +class GatePolicy(Protocol): + def decide(self, ctx: DecisionContext) -> CandidateOutcome: ... + + +class BaselineProvider(Protocol): + def for_round( + self, + round_index: int, + parent: HarnessNode, + *, + eval: EvalFn, + train_task_ids: list[str], + anchor: Optional[AnchorSelection], + ) -> Baseline: ... + + def on_promote(self, node: HarnessNode, outcome: CandidateOutcome, *, train_task_ids: list[str]) -> None: ... + + +class FrozenColdStartBaseline: + """One vanilla cold-start control, reused every round (SWE default). + + Cross-time-invalid (compares later rounds against a run that may be in a + different regime); a cost-bound fallback, not the methodology-correct choice. + """ + + def __init__(self, control_evals: dict[str, TaskEval], *, label: str = "vanilla"): + self._evals = dict(control_evals) + self._label = label + + def for_round(self, round_index, parent, *, eval, train_task_ids, anchor) -> Baseline: + return Baseline(self._evals, train_mean(self._evals, train_task_ids), self._label) + + def on_promote(self, node, outcome, *, train_task_ids) -> None: # frozen: never moves + return None + + +class PerParentFrozenBaseline: + """Per-parent frozen control; a promoted node's confirm evals become its + children's baseline (the AppWorld ratchet). Also cross-time-invalid. + + ``fallback(parent, train_task_ids)`` rebuilds a missing parent's baseline + from durable artifacts (e.g. its confirm out-dir on disk) — the resume path: + after a process restart only the seed survives in memory, but a promoted + parent's confirm evals are still on disk. Return None when the artifacts are + gone; ``for_round`` then raises rather than silently comparing to nothing. + """ + + def __init__( + self, + seed: dict[str, Baseline], + *, + fallback: Optional[Callable[[HarnessNode, list[str]], Optional[Baseline]]] = None, + ): + self._by_parent: dict[str, Baseline] = dict(seed) + self._fallback = fallback + + def for_round(self, round_index, parent, *, eval, train_task_ids, anchor) -> Baseline: + baseline = self._by_parent.get(parent.node_id) + if baseline is None and self._fallback is not None: + baseline = self._fallback(parent, list(train_task_ids)) + if baseline is not None: + self._by_parent[parent.node_id] = baseline + if baseline is None: + raise KeyError( + f"no baseline for parent {parent.node_id!r} " + f"(resumed without a fallback, or its confirm artifacts are gone)" + ) + return baseline + + def on_promote(self, node, outcome, *, train_task_ids) -> None: + # Mean over the FULL train set (SOP §0 fixed denominator): a confirm that + # missed a task must still count it as 0, so this baseline and next + # round's candidate arm share the same denominator basis. + evals = outcome.confirm_evals + mean = train_mean(evals, train_task_ids) + self._by_parent[node.node_id] = Baseline(evals, mean, f"{node.node_id}_confirm") + + +class SameSessionPairedBaseline: + """Re-run the parent's own harness this round as the control (vanilla at C0). + + Methodology-correct under regime shift — candidate and control are always + measured in the same session/window — at ~2x eval cost. Requires ``eval`` to + reproduce the parent node's harness (the backend's job). + """ + + def __init__(self, k: int, *, label: str = "control"): + self._k = k + self._label = label + + def for_round(self, round_index, parent, *, eval, train_task_ids, anchor) -> Baseline: + evals = eval(parent, train_task_ids, self._k, f"{self._label}_r{round_index}") + return Baseline(evals, train_mean(evals, train_task_ids), f"{parent.node_id}_{self._label}_r{round_index}") + + def on_promote(self, node, outcome, *, train_task_ids) -> None: # re-measured each round + return None + + +def make_frozen_baseline( + *, + root_node_id: str, + vanilla_dir: "Path", + kept_reader: Callable[["Path"], dict[str, TaskEval]], + confirm_dir_of: Callable[[HarnessNode], "Path"], + train_task_ids: list[str], + seed_label: str, +) -> PerParentFrozenBaseline: + """Assemble a :class:`PerParentFrozenBaseline` seeded from a vanilla ledger. + + The bench-neutral shape both benches share: the seed baseline is the vanilla + ledger read through ``kept_reader`` (the infra-rerun KEPT overlay, so the + control arm sees the same salvage rule candidate evals get — SOP §0); the + resume fallback re-reads a promoted parent's confirm dir (``confirm_dir_of``) + the same way. The only bench-specific inputs are the reader (out-dir vs + job-dir format) and the confirm-dir locator; everything else is identical. + Must be called after the vanilla cold start has materialised the ledger. + """ + van_evals = kept_reader(vanilla_dir) + + def fallback(parent: HarnessNode, train_ids: list[str]) -> Optional[Baseline]: + d = vanilla_dir if parent.node_id == root_node_id else confirm_dir_of(parent) + try: + evals = kept_reader(d) + except FileNotFoundError: + return None + return Baseline(evals, train_mean(evals, train_ids), f"{d.name}(resumed)") + + return PerParentFrozenBaseline( + seed={root_node_id: Baseline(van_evals, train_mean(van_evals, list(train_task_ids)), seed_label)}, + fallback=fallback, + ) + + +__all__ = [ + "CandidateOutcome", + "Baseline", + "DecisionContext", + "GatePolicy", + "BaselineProvider", + "FrozenColdStartBaseline", + "PerParentFrozenBaseline", + "SameSessionPairedBaseline", + "make_frozen_baseline", + "FiredSourceFn", + "FocusedSourceFn", +] diff --git a/raven/evolver/orchestrator/gates/strategies.py b/raven/evolver/orchestrator/gates/strategies.py new file mode 100644 index 0000000..142f5a3 --- /dev/null +++ b/raven/evolver/orchestrator/gates/strategies.py @@ -0,0 +1,229 @@ +"""The two concrete gate policies (SWE paired-2sigma, AppWorld focused-Fisher). + +Both implement :class:`GatePolicy.decide` over a :class:`DecisionContext`, +owning their own ``eval`` calls. Neither knows how its control arm was produced +(frozen vs same-session) — that is the :class:`BaselineProvider`'s concern. + +Both stages follow the SOP's two disciplines: + +- **Wide-pass screen (SOP §2 ⑤a).** A candidate advances to the full-train + confirm unless it is *clearly worse* than the baseline on its probe set. A + slightly-low or noise-band probe is NOT evidence — small K=1/K=3 probes have + huge variance and the probe mean does not predict the full-set mean. +- **Two-threshold verdict (SOP §0).** Promotion (banking, parent selection) is + the loose *navigator* condition: candidate full-train mean beats the control. + The paired-2σ significance is a separate *credited* label reported alongside + (``CandidateOutcome.paired.credited_2sigma``), never the promotion bar. + +The confirm job name is defined once here (:func:`confirm_job_name`) because it +doubles as the on-disk out-dir a bench's diagnosis later reads trajectories +from — the naming is a cross-module contract, not a local detail. +""" + +from __future__ import annotations + +from raven.evolver.orchestrator.gates.fisher import ( + fisher_one_sided, + focused_counts, + train_mean, +) +from raven.evolver.orchestrator.gates.pipeline import run_gates +from raven.evolver.orchestrator.gates.policy import CandidateOutcome, DecisionContext +from raven.evolver.orchestrator.nodes.screen import screen_candidate +from raven.evolver.tree.node import NodeStatus + +CONFIRM_JOB_SUFFIX = "_confirm" + + +def confirm_job_name(node_id: str) -> str: + """The job name (= out-dir name for dir-based scorers) of a node's full-train + confirm eval. Bench wiring that reads a promoted parent's confirm artifacts + (e.g. AppWorld diagnosis over the confirm out-dir) must use this, not a + hand-rolled f-string, so the gate and the reader cannot drift apart.""" + return f"{node_id}{CONFIRM_JOB_SUFFIX}" + + +class PairedTwoSigmaGate: + """SWE line: K=1 anchor wide-pass screen -> K=3 full-train three-shield gate. + + Promotion is the navigator condition (candidate mean beats vanilla); the 2sigma + credited label is reported alongside (see ``gates/paired``), not the bar. + """ + + def __init__(self, *, k_screen: int = 1, k_confirm: int = 3, z_threshold: float = 2.0): + self.k_screen = k_screen + self.k_confirm = k_confirm + self.z_threshold = z_threshold + + def decide(self, ctx: DecisionContext) -> CandidateOutcome: + if ctx.anchor is None: + raise ValueError("PairedTwoSigmaGate requires an anchor in the context") + node_id = ctx.node.node_id + screen_evals = ctx.eval(ctx.node, ctx.anchor.task_ids, self.k_screen, f"{node_id}_screen") + screen = screen_candidate(candidate_evals=screen_evals, anchor=ctx.anchor, vanilla_evals=ctx.baseline.evals) + if not screen.passes_to_confirm: + return CandidateOutcome(node_id, NodeStatus.pruned_at_screen, screen=screen) + + confirm = ctx.eval(ctx.node, ctx.train_task_ids, self.k_confirm, confirm_job_name(node_id)) + fired = ctx.fired_source(ctx.node, ctx.train_task_ids) if ctx.fired_source else None + gate = run_gates( + candidate_evals=confirm, + control_evals=ctx.baseline.evals, + task_ids=ctx.train_task_ids, + fired_tasks=fired, + z_threshold=self.z_threshold, + ) + # The reported score is ALWAYS the full-train mean over the fixed + # denominator; the paired stats may be narrowed to Gate-b's fired + # subset, and a subset mean leaking out as "the score" would poison + # beat_vanilla / parent selection / the journal curve with a number + # that has a different denominator (review round-2 P0-4). Promotion is + # the dual condition: the (possibly subset) paired verdict holds AND + # the full-train mean does not regress the control — the fired subset + # may claim credit, but never at the expense of the whole set. + full_mean = train_mean(confirm, ctx.train_task_ids) + control_full_mean = train_mean(ctx.baseline.evals, ctx.train_task_ids) + promoted = gate.promoted and full_mean >= control_full_mean + status = NodeStatus.promoted_to_baseline if promoted else NodeStatus.pruned_at_confirm + return CandidateOutcome( + node_id, + status, + score=full_mean, + confirm_evals=confirm, + screen=screen, + paired=gate.paired, + gate=gate, + stats={"full_mean": full_mean, "control_full_mean": control_full_mean}, + ) + + +class FocusedFisherGate: + """AppWorld line: focused-subset wide-pass probe (stage 1) -> full-train + three-shield gate (stage 2). + + Stage 1 runs the candidate only on its WHY's focused subset (plus the + sentinel controls) and culls it ONLY when the probe shows it clearly worse + than the baseline — a one-sided Fisher test in the *worse* direction at + ``alpha``, or a sentinel regression beyond one flaky trial. Everything else + (better, slightly low, indistinguishable) advances to the full confirm: + wide-pass, SOP §2 ⑤a. The improvement-direction Fisher p is still reported + (``fisher_p``) as evidence, but it is not an advancement bar. + + Stage 2 confirms on the full train set through the same three-shield + pipeline as the SWE line: Gate-f infra report, Gate-b attribution when the + bench wires a ``fired_source``, then the paired gate — navigator promotion + (mean beats the control) with the credited-2σ label reported alongside. + ``min_confirm_lift`` (default 0, the SOP navigator bar) optionally demands a + minimum full-train lift on top of the navigator condition. + """ + + def __init__( + self, + *, + k: int = 3, + alpha: float = 0.05, + min_confirm_lift: float = 0.0, + z_threshold: float = 2.0, + ): + self.k = k + self.alpha = alpha + self.min_confirm_lift = min_confirm_lift + self.z_threshold = z_threshold + + def decide(self, ctx: DecisionContext) -> CandidateOutcome: + node_id = ctx.node.node_id + focused = ctx.focused_task_ids + sentinels = ctx.sentinel_task_ids + # One eval over focused (the WHY subset) + sentinels (stable-pass controls), + # so the regression guard costs no extra run. + probe_ids = list(dict.fromkeys(list(focused) + list(sentinels))) + cand_probe = ctx.eval(ctx.node, probe_ids, self.k, f"{node_id}_focused") if probe_ids else {} + stats: dict = {} + if focused: + cp, cn = focused_counts(cand_probe, focused) + vp, vn = focused_counts(ctx.baseline.evals, focused) + foc_c = cp / (cp + cn) if (cp + cn) else 0.0 + foc_v = vp / (vp + vn) if (vp + vn) else 0.0 + stats.update( + fisher_p=fisher_one_sided(cp, cn, vp, vn), + fisher_p_worse=fisher_one_sided(vp, vn, cp, cn), + foc_c=foc_c, + foc_v=foc_v, + ) + + # Sentinel regression guard (SOP §2 ⑤a), stratified: stable-pass + # controls never flake under the baseline, so any drop beyond one flaky + # trial is signal; borderline controls flip by nature, so their verdict + # uses a trial-level Fisher test (worse direction) instead of the mean + # guard — the mean guard on flaky tasks would fire on noise. + if sentinels: + base = ctx.baseline.evals + + def _bmean(tid: str) -> float | None: + ev = base.get(tid) + good = (ev.attempts - ev.infra_attempts) if ev else 0 + return (ev.passes / good) if ev and good else None + + stable = [t for t in sentinels if _bmean(t) == 1.0] + fragile = [t for t in sentinels if t not in stable and _bmean(t)] + stats.update( + sent_c=train_mean(cand_probe, sentinels), + sent_v=train_mean(base, sentinels), + ) + if stable: + st_c = train_mean(cand_probe, stable) + st_v = train_mean(base, stable) + guard = 1.5 / (len(stable) * self.k) + stats.update(sentinel_guard=guard) + if st_c < st_v - guard: + stats["sentinel_regression"] = True + return CandidateOutcome(node_id, NodeStatus.pruned_at_screen, stats=stats) + if fragile: + fc_p, fc_n = focused_counts(cand_probe, fragile) + fv_p, fv_n = focused_counts(base, fragile) + p_worse = fisher_one_sided(fv_p, fv_n, fc_p, fc_n) + stats.update(sent_fragile_p_worse=p_worse) + frag_c = fc_p / (fc_p + fc_n) if (fc_p + fc_n) else 0.0 + frag_v = fv_p / (fv_p + fv_n) if (fv_p + fv_n) else 0.0 + if frag_c < frag_v and p_worse < self.alpha: + stats["sentinel_regression"] = True + return CandidateOutcome(node_id, NodeStatus.pruned_at_screen, stats=stats) + + # Wide-pass cull: only a probe that is SIGNIFICANTLY worse than the + # baseline on the WHY subset is pruned without a full run. Slightly-low + # or indistinguishable probes advance (SOP: slightly-below does not eliminate). + if focused and stats["foc_c"] < stats["foc_v"] and stats["fisher_p_worse"] < self.alpha: + stats["pruned_significantly_worse"] = True + return CandidateOutcome(node_id, NodeStatus.pruned_at_screen, stats=stats) + + confirm = ctx.eval(ctx.node, ctx.train_task_ids, self.k, confirm_job_name(node_id)) + fired = ctx.fired_source(ctx.node, ctx.train_task_ids) if ctx.fired_source else None + gate = run_gates( + candidate_evals=confirm, + control_evals=ctx.baseline.evals, + task_ids=ctx.train_task_ids, + fired_tasks=fired, + z_threshold=self.z_threshold, + ) + cand_mean = train_mean(confirm, ctx.train_task_ids) + lift = cand_mean - ctx.baseline.mean + stats["full_lift"] = lift + promoted = gate.promoted and lift >= self.min_confirm_lift + status = NodeStatus.promoted_to_baseline if promoted else NodeStatus.pruned_at_confirm + return CandidateOutcome( + node_id, + status, + score=cand_mean, + confirm_evals=confirm, + paired=gate.paired, + gate=gate, + stats=stats, + ) + + +__all__ = [ + "PairedTwoSigmaGate", + "FocusedFisherGate", + "confirm_job_name", + "CONFIRM_JOB_SUFFIX", +] diff --git a/raven/evolver/orchestrator/loop.py b/raven/evolver/orchestrator/loop.py new file mode 100644 index 0000000..70c17bc --- /dev/null +++ b/raven/evolver/orchestrator/loop.py @@ -0,0 +1,724 @@ +"""The seven-step funnel as a finite-state machine. + +This is the layer the SOP used to delegate to a long, high-compliance Claude +session. Here the control flow is code: the round loop, the per-candidate fork, +parent selection, and the stop decision. Everything bench-specific is bundled in +an injected :class:`~raven.evolver.orchestrator.scoring.EvalBackend`; the +per-candidate decision (screen -> confirm -> promote) and the control arm are +injected as a :class:`GatePolicy` and a :class:`BaselineProvider`. So a weaker +driver model (Qwen / Kimi) can run the loop without remembering the funnel's +shape, and SWE-bench / AppWorld / a no-benchmark LLM judge all share one loop — +they differ only in which backend + policy + baseline get wired in. + +Two per-round signals feed termination, and they are NOT the same thing: + +- ``promoted`` — the parent changed: some candidate passed the gate against its + round baseline AND beat the incumbent parent's train score (the Alg.1 L135 + argmax). A gate-passer that loses the argmax banks but does not take over. +- ``beat_vanilla`` — a candidate's full-train confirm beat the FIXED vanilla + cold-start mean. This is the SOP's patience signal (no candidate's train mean + beats vanilla for N consecutive rounds), measured against vanilla for every + benchmark regardless of which + baseline provider gates promotion. A round that erred out entirely sets + ``errored`` instead and burns neither counter (it has its own stop). + +The semantic steps are still injected callables: + +- ``diagnose_fn`` (①): read the last child's trajectories, return a failure map. +- ``design_fn`` (②): pick WHYs and design candidates (:class:`AppliedPatch`). +- ``preflight_fn`` (③, optional): drop inert candidates; default keeps all. +- ``apply_fn`` (④): apply a patch on the parent, persist a child node. +- ``verdict_fn`` (⑦, optional): draft a per-round verdict for the findings log. + +⑤/⑥ (screen/confirm/gate) are the ``gate_policy``'s job; the control arm each +round comes from the ``baseline_provider`` (frozen cold-start by default, or the +methodology-correct same-session provider). ``focused_source`` supplies a +candidate's WHY subset (AppWorld's Fisher gate); ``outcome_hook`` lets a bench +learn across rounds (AppWorld's attempt history), and ``inert_hook`` feeds it +the preflight-pruned candidates that never reach a DecisionContext. All +default off. + +On-disk state under ``config.work_dir`` (all best-effort, all resume-safe): +``failure_map.json`` (the cross-round live map), ``nodes/.json`` (the +node ledger: identity + git anchor + final status + gate stats, one file per +candidate), and ``findings.md`` (a human-readable per-round log with the +driver's verdict). The round journal the caller passes to :meth:`run` is the +loop-progress record those three complement. +""" + +from __future__ import annotations + +import dataclasses +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol + +if TYPE_CHECKING: + from raven.evolver.orchestrator.state.journal import RoundJournal + +from raven.evolver.orchestrator.archive import ( + CellElite, + GsmeArchive, + describe_candidate, +) +from raven.evolver.orchestrator.config import OrchestratorConfig +from raven.evolver.orchestrator.gates.fisher import train_mean +from raven.evolver.orchestrator.gates.policy import ( + BaselineProvider, + CandidateOutcome, + DecisionContext, + FiredSourceFn, + FocusedSourceFn, + FrozenColdStartBaseline, + GatePolicy, +) +from raven.evolver.orchestrator.gates.strategies import PairedTwoSigmaGate +from raven.evolver.orchestrator.nodes.diagnose import merge_failure_maps +from raven.evolver.orchestrator.scoring import EvalBackend, TaskEval, flip_summary +from raven.evolver.orchestrator.sealed.runner import assert_no_test_leak +from raven.evolver.orchestrator.termination import TerminationTracker +from raven.evolver.scheduler.anchor_selection import AnchorSelection +from raven.evolver.tree.node import AppliedPatch, HarnessNode, NodeStatus + + +class DiagnoseFn(Protocol): + def __call__(self, round_index: int, parent: HarnessNode) -> dict: ... + + +class DesignFn(Protocol): + def __call__(self, round_index: int, failure_map: dict, parent: HarnessNode) -> list[AppliedPatch]: ... + + +class ApplyFn(Protocol): + def __call__(self, parent_id: str, patch: AppliedPatch, round_index: int) -> HarnessNode: ... + + +# (candidate, parent) -> keep? The parent gives preflight its historical +# corpus (the trajectories a trigger predicate is checked against). +PreflightFn = Callable[[AppliedPatch, HarnessNode], bool] +VerdictFn = Callable[["RoundResult"], str] +OutcomeHook = Callable[[DecisionContext, CandidateOutcome], None] + +# (candidate, outcome) for a preflight-pruned candidate: it was never applied, +# so there is no node/DecisionContext — the raw candidate is all there is. +InertHook = Callable[[Any, CandidateOutcome], None] +# Materialise a cell elite's edit onto the parent as a bench candidate; None = +# the pairing cannot be built (commit gone / nothing to stack) -> skip it. +RecombineFn = Callable[[HarnessNode, CellElite], Optional[object]] + + +@dataclass +class RoundResult: + round_index: int + parent_id: str + next_parent_id: str + promoted: bool + outcomes: list[CandidateOutcome] = field(default_factory=list) + verdict: Optional[str] = None + # SOP patience signal: some candidate's full-train confirm beat the FIXED + # vanilla mean this round (independent of the gate's own baseline). + beat_vanilla: bool = False + # Every candidate/phase erred — no real decision was made this round. + errored: bool = False + # Recorded for the post-hoc sealed unseal (C3, approach B): the deliverable + # harness's commit + its train pass@1, so its test curve is reconstructable + # after evolution without any decision-time test scoring. + next_parent_sha: Optional[str] = None + next_parent_train: Optional[float] = None + + +@dataclass +class RunResult: + rounds: list[RoundResult] = field(default_factory=list) + stop_reason: Optional[str] = None + final_parent_id: Optional[str] = None + resumed_rounds: int = 0 # rounds replayed from a journal, not re-run + + +def summarize_round(rr: RoundResult) -> str: + """Factual one-round summary for the verdict draft / findings log.""" + lines = [f"round {rr.round_index}: parent={rr.parent_id} promoted={rr.promoted}"] + for o in rr.outcomes: + parts = [f" {o.node_id}: {o.status.value}"] + if o.screen is not None: + parts.append(f"screen={o.screen.candidate_mean:.3f} vs van {o.screen.vanilla_mean:.3f} ({o.screen.bucket})") + if o.paired is not None: + parts.append( + f"confirm={o.paired.candidate_mean:.3f} vs van " + f"{o.paired.control_mean:.3f} z={o.paired.z:.2f} " + f"credited={o.paired.credited_2sigma}" + ) + if o.stats: + parts.append(" ".join(f"{k}={v}" for k, v in o.stats.items())) + lines.append(" ".join(parts)) + return "\n".join(lines) + + +def _sha_or_none(sha: Optional[str]) -> Optional[str]: + """A journal-safe commit SHA: the root shim's ``"unknown"`` placeholder is + recorded as None so the post-hoc unseal never tries to check it out.""" + return None if sha in (None, "", "unknown") else sha + + +def _vanilla_control(vanilla_stability) -> dict[str, TaskEval]: + """The cold-start baseline as an eval map to serve as the control arm.""" + return {tid: TaskEval(task_id=tid, passes=st.passes, attempts=st.attempts) for tid, st in vanilla_stability.items()} + + +class EvolutionOrchestrator: + """Drives the seven-step funnel across rounds until a stop condition fires.""" + + def __init__( + self, + config: OrchestratorConfig, + *, + backend: EvalBackend, + diagnose_fn: DiagnoseFn, + design_fn: DesignFn, + apply_fn: ApplyFn, + gate_policy: Optional[GatePolicy] = None, + baseline_provider: Optional[BaselineProvider] = None, + preflight_fn: Optional[PreflightFn] = None, + verdict_fn: Optional[VerdictFn] = None, + fired_source: Optional[FiredSourceFn] = None, + focused_source: Optional[FocusedSourceFn] = None, + outcome_hook: Optional[OutcomeHook] = None, + inert_hook: Optional[InertHook] = None, + seed_failure_map: Optional[dict] = None, + archive: Optional[GsmeArchive] = None, + recombine_fn: Optional[RecombineFn] = None, + ) -> None: + self._cfg = config + self._backend = backend + self._diagnose = diagnose_fn + self._design = design_fn + self._apply = apply_fn + self._eval = backend.eval + self._preflight = preflight_fn or (lambda _patch, _parent: True) + self._verdict = verdict_fn + self._fired_source = fired_source + self._focused_source = focused_source + self._outcome_hook = outcome_hook + self._inert_hook = inert_hook + # GSME: the per-cell elite bank + the cross-cell recombiner. The archive + # loads its persisted state itself, so a resumed run keeps its elites. + self._archive = archive + self._recombine = recombine_fn + + self._vanilla_stability = backend.cold_start() + if not self._vanilla_stability: + raise ValueError("backend.cold_start() returned an empty baseline") + self._train_task_ids = list(backend.train_task_ids) or sorted(self._vanilla_stability) + self._sentinel_task_ids = self._sample_sentinels(config.anchor.n_sentinel) + # The FIXED comparison anchor for the patience signal (SOP: candidate + # train mean vs VANILLA, never vs the previous round's parent) — the + # same for every benchmark no matter which baseline provider gates + # promotion. + self._vanilla_train_mean = train_mean(_vanilla_control(self._vanilla_stability), self._train_task_ids) + + # Default policy = the SWE paired-2σ line; default baseline = frozen + # cold-start (cost-bound; cross-time-invalid — see gates.policy). AppWorld + # / same-session runs inject their own. + self._gate: GatePolicy = gate_policy or PairedTwoSigmaGate(k_screen=config.k_screen, k_confirm=config.k_confirm) + self._baselines: BaselineProvider = baseline_provider or FrozenColdStartBaseline( + _vanilla_control(self._vanilla_stability) + ) + + # Sealed-test iron law as a mechanism: fail loudly if a held-out id has + # crept into the anchor or train sets (SOP §0). + if backend.test_task_ids: + assert_no_test_leak( + anchor_task_ids=self.select_anchor().task_ids, + train_task_ids=self._train_task_ids, + sealed_test_ids=list(backend.test_task_ids), + ) + + # Cross-round live failure map (SOP §2 ①): accumulated, not frozen. + # ``seed_failure_map`` pre-populates it (taxonomy induction's seed, with + # the root in ``_diagnosed_parents`` so round 1 skips re-judging the very + # trajectories induction judged); a journal resume overrides it from disk. + self._failure_map: dict = dict(seed_failure_map) if seed_failure_map else {} + + # Real applied nodes by id (seeded with the root when run() gets one). + # A promoted parent must resolve to its real node — with its commit SHA + # and patch — not a shim, or a same-session baseline / worktree eval + # would check out the wrong (or an "unknown") commit. + self._node_registry: dict[str, HarnessNode] = {} + # Ledger metadata for non-AppliedPatch candidates (the wired bench + # lines), where node.patch is None and the WHERE/WHY/files/activation + # info would otherwise never reach the node record. + self._cand_meta: dict[str, dict] = {} + + @property + def vanilla_train_mean(self) -> float: + """The fixed vanilla train anchor (benches read it at unseal time).""" + return self._vanilla_train_mean + + def _sample_sentinels(self, n: int) -> list[str]: + """Deterministic default sentinel set (used when no node id is at hand).""" + stable, fragile = self._sentinel_pools() + return stable[: n - n // 2] + fragile[: n // 2] + + def _sentinel_pools(self) -> tuple[list[str], list[str]]: + from raven.evolver.analysis.stability_bucket import StabilityBucket + + train = set(self._train_task_ids) + stable, fragile = [], [] + for tid, st in self._vanilla_stability.items(): + if tid not in train: + continue + if st.bucket == StabilityBucket.STABLE_PASS: + stable.append(tid) + elif st.bucket in (StabilityBucket.BORDERLINE_2_3, StabilityBucket.BORDERLINE_1_3): + fragile.append(tid) + return sorted(stable), sorted(fragile) + + def _sentinels_for(self, node_id: str, n: int) -> list[str]: + """Per-candidate regression controls: half stable-pass, half borderline. + + Stratified because over-trigger regressions concentrate on BORDERLINE + tasks (fragile passes flip first) — a stable-only sentinel set is + systematically blind to them (observed live: a candidate with a 58% + passing-task regression rate sailed through 3 stable sentinels). + Rotated per candidate (hash of node_id) so no fixed control set can be + sailed past twice.""" + import hashlib + + stable, fragile = self._sentinel_pools() + + def pick(pool: list[str], m: int, salt: str) -> list[str]: + if not pool or m <= 0: + return [] + h = int(hashlib.sha256(f"{salt}:{node_id}".encode()).hexdigest(), 16) + start = h % len(pool) + return [pool[(start + i) % len(pool)] for i in range(min(m, len(pool)))] + + return pick(stable, n - n // 2, "stable") + pick(fragile, n // 2, "fragile") + + def select_anchor(self, affinity: dict[str, float] | None = None) -> AnchorSelection: + return self._backend.anchor(affinity) + + def run( + self, + root_node_id: str, + journal: Optional["RoundJournal"] = None, + *, + root_node: Optional[HarnessNode] = None, + ) -> RunResult: + """Run rounds from ``root_node_id`` (vanilla) until termination. + + If ``journal`` is given, previously-completed rounds are replayed to seed + the termination counters, current parent, and round index, and the loop + continues from the next round — a killed run resumes without re-running + the evals it already did. Replay also re-registers each promoted + parent's recorded commit SHA, so a resumed round's design / worktree + eval / baseline fallback sees the real commit, not an "unknown" shim; + and the accumulated failure map is re-read from disk so the cross-round + live map (SOP §2 ①) is not truncated back to empty. + + ``root_node`` (optional) registers the real vanilla node so a round whose + parent is the root can resolve its commit SHA (needed by same-session + baselines / worktree evals); without it the root falls back to a shim. + """ + term = TerminationTracker( + patience=self._cfg.termination.patience, + max_rounds=self._cfg.termination.max_rounds, + max_consecutive_errors=self._cfg.termination.max_consecutive_errors, + ) + result = RunResult() + parent_id = root_node_id + # The incumbent's train score — the argmax bar a challenger must beat to + # take over as parent (Alg.1 L135). The root's score is vanilla's mean. + parent_score = self._vanilla_train_mean + round_index = 0 + if root_node is not None: + self._node_registry[root_node.node_id] = root_node + + if journal is not None: + records = journal.load() + for rec in records: + term.record_round( + promoted=rec.get("beat_vanilla", rec["promoted"]), + errored=rec.get("errored", False), + ) + round_index = rec["round_index"] + parent_id = rec["next_parent_id"] + if rec.get("next_parent_train") is not None: + parent_score = rec["next_parent_train"] + sha = rec.get("next_parent_sha") + if ( + sha + and rec["next_parent_id"] != rec["parent_id"] + and rec["next_parent_id"] not in self._node_registry + ): + self._node_registry[rec["next_parent_id"]] = HarnessNode( + node_id=rec["next_parent_id"], + parent_id=rec["parent_id"], + git_commit_sha=sha, + git_branch="journal-resume", + created_at=HarnessNode.utc_now(), + created_at_iter=rec["round_index"], + ) + result.resumed_rounds += 1 + if records: + self._reload_failure_map() + stop, reason = term.should_stop() + if stop: + result.stop_reason = reason + result.final_parent_id = parent_id + return result + + while True: + round_index += 1 + round_result = self._run_round(round_index, parent_id, parent_score) + result.rounds.append(round_result) + if journal is not None: + journal.append(round_result) + self._persist_node_records(round_result) + self._append_findings(round_result) + if self._archive is not None: + self._archive.save() + parent_id = round_result.next_parent_id + if round_result.promoted and round_result.next_parent_train is not None: + parent_score = round_result.next_parent_train + + term.record_round(promoted=round_result.beat_vanilla, errored=round_result.errored) + stop, reason = term.should_stop() + if stop: + result.stop_reason = reason + break + + result.final_parent_id = parent_id + return result + + def _run_round(self, round_index: int, parent_id: str, parent_score: float) -> RoundResult: + # Gate0 (SOP §0, before any scoring): verify the environment before scoring anything + # this round. A dirty env (sandbox down / network unroutable / verifier + # can't emit results) makes every score invalid, so let it raise — fix + # the box, then resume. Benches with no precheck wired skip this. + if self._backend.precheck is not None: + self._backend.precheck() + parent = self._load_parent(parent_id) + anchor = self.select_anchor() + # The control arm may itself be an eval (same-session pairing) or a disk + # rebuild — a transient failure here is round-scoped like any other eval + # failure: record an errored round and let the error counter decide, + # instead of aborting an unattended run with no journal record. + try: + baseline = self._baselines.for_round( + round_index, + parent, + eval=self._eval, + train_task_ids=self._train_task_ids, + anchor=anchor, + ) + except Exception as exc: # noqa: BLE001 — record + continue, don't abort + outcome = CandidateOutcome( + f"r{round_index}-baseline", + NodeStatus.errored, + stats={"phase": "baseline", "error": repr(exc)}, + ) + return RoundResult( + round_index=round_index, + parent_id=parent_id, + next_parent_id=parent_id, + promoted=False, + outcomes=[outcome], + errored=True, + next_parent_sha=_sha_or_none(parent.git_commit_sha), + ) + + # ① diagnose -> merge into the cross-round live failure map, ② design, + # ③ preflight prune. A parent already diagnosed in an earlier round (no + # promotion since) is NOT re-diagnosed: its trajectories haven't changed, + # re-judging them would only re-spend the driver and double-count the + # same failures in the accumulated map. A diagnose/design failure must + # not abort the whole run (same discipline as the per-candidate catch + # below): record the reason on an errored outcome and finish the round + # with no candidates; the tracker's error counter stops a persistent + # outage with an honest reason instead of burning patience. + outcomes: list[CandidateOutcome] = [] + try: + diagnosed = set(self._failure_map.get("_diagnosed_parents") or []) + if parent_id not in diagnosed: + round_map = self._diagnose(round_index, parent) + self._failure_map = merge_failure_maps(self._failure_map, round_map) + self._failure_map["_diagnosed_parents"] = sorted(diagnosed | {parent_id}) + self._persist_failure_map() + candidates = [] + for i, c in enumerate(self._design(round_index, self._failure_map, parent)): + if self._preflight(c, parent): + candidates.append(c) + else: + # ③ zero-inference prune, recorded (never silently dropped): + # a pruned-inert candidate is a real decision this round. + outcome = CandidateOutcome( + f"r{round_index}-preflight{i}", + NodeStatus.pruned_inert, + stats={ + "phase": "preflight", + "why": str(getattr(c, "why", "")), + "reason": "trigger has zero historical hits", + }, + ) + outcomes.append(outcome) + if self._inert_hook is not None: + try: + self._inert_hook(c, outcome) + except Exception: # noqa: BLE001 — advisory learning hook, non-fatal + pass + except Exception as exc: # noqa: BLE001 — record + continue, don't abort + outcomes.append( + CandidateOutcome( + f"r{round_index}-design", + NodeStatus.errored, + stats={"phase": "diagnose_design", "error": repr(exc)}, + ) + ) + candidates = [] + + # GSME cross-cell recombination: stack elites from cells the parent's + # lineage has not covered onto the parent, as ordinary candidates through + # the same apply -> gate pipeline. Deliberately OUTSIDE the design + # try/except — a driver outage does not stop deterministic recombination. + if self._archive is not None and self._recombine is not None: + try: + elites = self._archive.eligible_elites(parent_id, limit=self._cfg.budget.recombinations_per_round) + for elite in elites: + recomb = self._recombine(parent, elite) + if recomb is None: + self._archive.record_pairing(parent_id, elite.node_id, "recombine_failed") + continue + candidates.append(recomb) + except Exception as exc: # noqa: BLE001 — record + continue + outcomes.append( + CandidateOutcome( + f"r{round_index}-recombine", + NodeStatus.errored, + stats={"phase": "recombine", "error": repr(exc)}, + ) + ) + + best_node_id: Optional[str] = None + best_score = -1.0 + + for idx, patch in enumerate(candidates): + # A single candidate's apply/eval crash must not sink the round: catch + # it, record the reason on an ``errored`` outcome, and move on (C). + elite_id = getattr(patch, "elite_node_id", None) + try: + node = self._apply(parent_id, patch, round_index) # ④ + except Exception as exc: # noqa: BLE001 — record + skip, don't abort + if elite_id and self._archive is not None: + self._archive.record_pairing(parent_id, elite_id, "errored") + outcomes.append( + CandidateOutcome( + f"r{round_index}-cand{idx}", + NodeStatus.errored, + stats={"phase": "apply", "error": repr(exc)}, + ) + ) + continue + self._node_registry[node.node_id] = node + meta = describe_candidate(patch) + if meta: + self._cand_meta[node.node_id] = meta + ctx = DecisionContext( + node=node, + parent_id=parent_id, + round_index=round_index, + eval=self._eval, + baseline=baseline, + train_task_ids=self._train_task_ids, + anchor=anchor, + focused_task_ids=(self._focused_source(node) if self._focused_source else []), + sentinel_task_ids=self._sentinels_for(node.node_id, self._cfg.anchor.n_sentinel), + fired_source=self._fired_source, + ) + try: + outcome = self._gate.decide(ctx) # ⑤⑥ delegated to the policy + except Exception as exc: # noqa: BLE001 — record + skip, don't abort + if elite_id and self._archive is not None: + self._archive.record_pairing(parent_id, elite_id, "errored") + outcomes.append( + CandidateOutcome( + node.node_id, + NodeStatus.errored, + stats={"phase": "decide", "error": repr(exc)}, + ) + ) + continue + if elite_id: + outcome.stats["recombination_of"] = elite_id + # Flip table (SOP §2 ①: which tasks flipped): rescued/regressed vs + # this round's control, recorded on the ledger and the live failure + # map so the next diagnose/design sees CAUSAL feedback, not just the + # static failure set. + if outcome.confirm_evals: + flips = flip_summary(outcome.confirm_evals, baseline.evals, self._train_task_ids) + outcome.stats["flips"] = flips + self._failure_map.setdefault("_flips", {})[node.node_id] = { + "round": round_index, + "vs": baseline.label, + **flips, + } + outcomes.append(outcome) + if self._archive is not None: + try: + self._archive.consider( + parent_id=parent_id, + node=node, + cand=patch, + outcome=outcome, + round_index=round_index, + vanilla_train_mean=self._vanilla_train_mean, + ) + except Exception: # noqa: BLE001 — bank-keeping must not sink a round + pass + if self._outcome_hook is not None: + try: + self._outcome_hook(ctx, outcome) + except Exception: # noqa: BLE001 — advisory learning hook, non-fatal + pass + if outcome.promoted: + self._baselines.on_promote(node, outcome, train_task_ids=self._train_task_ids) + if outcome.score > best_score: + best_score = outcome.score + best_node_id = node.node_id + + if any(o.confirm_evals for o in outcomes): + self._persist_failure_map() # the round's flips joined the live map + + # ⑦ select parent — Alg.1 L135 argmax: the round's best gate-passer takes + # over only when it beats the incumbent's train score; a tie keeps the + # incumbent (no churn without improvement). A gate-passer that loses the + # argmax still banks (status/archive) — it just doesn't become parent. + # Under the ratcheted baseline the gate already implies this; under a + # frozen-vanilla control this keeps the champion chain monotone. + promoted = best_node_id is not None and best_score > parent_score + next_parent_id = best_node_id if promoted else parent_id + next_parent_node = self._node_registry.get(next_parent_id) or parent + beat_vanilla = any(o.confirm_evals and o.score > self._vanilla_train_mean for o in outcomes) + errored = bool(outcomes) and all(o.status is NodeStatus.errored for o in outcomes) + + round_result = RoundResult( + round_index=round_index, + parent_id=parent_id, + next_parent_id=next_parent_id, + promoted=promoted, + outcomes=outcomes, + beat_vanilla=beat_vanilla, + errored=errored, + next_parent_sha=_sha_or_none(next_parent_node.git_commit_sha), + next_parent_train=(best_score if promoted else None), + ) + if self._verdict is not None: + round_result.verdict = self._verdict(round_result) + return round_result + + def _persist_failure_map(self) -> None: + """Write the accumulated failure map for audit/resume (best-effort).""" + try: + path = self._cfg.failure_map_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(self._failure_map, indent=2)) + except OSError: + pass + + def _reload_failure_map(self) -> None: + """Re-read the accumulated failure map from disk (resume path).""" + try: + path = self._cfg.failure_map_path + if path.exists(): + self._failure_map = json.loads(path.read_text()) + except (OSError, ValueError): + pass + + def _persist_node_records(self, rr: RoundResult) -> None: + """Write the node ledger (SOP §3.1): one JSON per candidate under + ``work_dir/nodes/``, carrying identity + git anchor + final status + + gate stats. Best-effort — the ledger is the audit trail, not control + state (resume runs off the journal).""" + try: + ndir = self._cfg.nodes_dir + ndir.mkdir(parents=True, exist_ok=True) + for o in rr.outcomes: + node = self._node_registry.get(o.node_id) + if node is None: # errored pseudo-candidates never got a node + continue + patch = node.patch + patch_to_dict = getattr(patch, "to_dict", None) + rec = { + "node_id": node.node_id, + "parent_id": node.parent_id, + "git_commit_sha": node.git_commit_sha, + "git_branch": node.git_branch, + "created_at": node.created_at, + "created_at_iter": node.created_at_iter, + "patch": ( + patch_to_dict() if callable(patch_to_dict) else (repr(patch) if patch is not None else None) + ), + "status": o.status.value, + "round_index": rr.round_index, + "score": o.score, + } + if o.node_id in self._cand_meta: + rec["candidate"] = self._cand_meta[o.node_id] + if o.screen is not None: + rec["screen"] = dataclasses.asdict(o.screen) + if o.paired is not None: + rec["paired"] = dataclasses.asdict(o.paired) + if o.stats: + rec["stats"] = o.stats + (ndir / f"{o.node_id}.json").write_text(json.dumps(rec, indent=2, default=str)) + except OSError: + pass + + def _append_findings(self, rr: RoundResult) -> None: + """Append the per-round findings-log entry (SOP §3 layer 1) to + ``work_dir/findings.md`` — the human-readable record of what each round + tried, what the gates said, and the driver's verdict. Best-effort.""" + try: + path = self._cfg.findings_path + path.parent.mkdir(parents=True, exist_ok=True) + block = [f"\n## round {rr.round_index}\n", "```", summarize_round(rr), "```"] + if rr.verdict: + block.append(f"\nverdict: {rr.verdict}") + with path.open("a") as f: + f.write("\n".join(block) + "\n") + except OSError: + pass + + def _load_parent(self, parent_id: str) -> HarnessNode: + # A promoted parent was applied in an earlier round (or re-registered + # from the journal on resume), so it is in the registry with its real + # commit SHA; return that. Only the root (never applied) falls back to a + # shim — pass ``root_node`` to run() to give it a real SHA too. + node = self._node_registry.get(parent_id) + if node is not None: + return node + return HarnessNode( + node_id=parent_id, + parent_id=None, + git_commit_sha="unknown", + git_branch="unknown", + created_at=HarnessNode.utc_now(), + created_at_iter=0, + ) + + +__all__ = [ + "EvolutionOrchestrator", + "CandidateOutcome", + "RoundResult", + "RunResult", + "DiagnoseFn", + "DesignFn", + "ApplyFn", + "PreflightFn", + "VerdictFn", + "InertHook", + "OutcomeHook", + "RecombineFn", + "summarize_round", +] diff --git a/raven/evolver/orchestrator/nodes/__init__.py b/raven/evolver/orchestrator/nodes/__init__.py new file mode 100644 index 0000000..51b7938 --- /dev/null +++ b/raven/evolver/orchestrator/nodes/__init__.py @@ -0,0 +1,3 @@ +"""Seven-step funnel nodes — semantic (driver LLM) and deterministic (operators).""" + +from __future__ import annotations diff --git a/raven/evolver/orchestrator/nodes/design.py b/raven/evolver/orchestrator/nodes/design.py new file mode 100644 index 0000000..73e0bc3 --- /dev/null +++ b/raven/evolver/orchestrator/nodes/design.py @@ -0,0 +1,203 @@ +"""Step ② — select WHY + design candidates (semantic, budgeted). + +Two halves, matching the SOP: a *deterministic* WHY selection off the failure +map (pick 1-2 pathologies worth attacking this round), then a *semantic* design +call per candidate that writes an env-gated ``AppliedPatch`` (2-3 per WHY, +across levers). The budget (``max_why_per_round`` x ``candidates_per_why``) is +enforced in code so a chatty driver can't blow up the round's candidate count. + +The design call's output is parsed straight through ``AppliedPatch.from_dict``, +so the dataclass's own invariants (non-empty components, unique component ids, +``patch_why=other`` needs ``patch_why_extra``, resolvable ``depends_on``) do the +schema validation; any violation raises and :class:`SemanticNode` feeds it back +for a bounded repair-retry. + +Writing a *useful* diff needs the target file's current contents; the caller +supplies them via ``file_context`` (empty here keeps the node structural — it +still produces a schema-valid patch, but a production run wires repo context in). +The patch must be env-gated and default-off so the vanilla build stays +byte-identical when the activation flag is unset (SOP §2 ②). +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from raven.evolver.orchestrator.config import Budget +from raven.evolver.orchestrator.nodes.semantic import CallFn, SemanticNode +from raven.evolver.tree.node import AppliedPatch + + +@dataclass(frozen=True) +class WhyTarget: + """One pathology selected for this round, with its supporting evidence.""" + + why: str + where_options: list[str] + n_candidates: int + trajectory_ids: list[str] + + +def select_target_whys(failure_map: dict[str, Any], budget: Budget) -> list[WhyTarget]: + """Pick the top ``budget.max_why_per_round`` WHYs from the failure map. + + Ranks by how many L2/L3 candidates the diagnosis attributed to each WHY + (``why_distribution``), then gathers each WHY's WHERE levers and evidence + trajectory ids from ``cells`` (keyed ``"::"``). Ties break by + WHY name so selection is reproducible. + """ + why_dist: dict[str, int] = failure_map.get("why_distribution", {}) + cells: dict[str, Any] = failure_map.get("cells", {}) + if not why_dist: + return [] + + ranked = sorted(why_dist.items(), key=lambda kv: (-kv[1], kv[0])) + targets: list[WhyTarget] = [] + for why, _count in ranked[: budget.max_why_per_round]: + where_options: list[str] = [] + trajectory_ids: list[str] = [] + n_candidates = 0 + for key, cell in cells.items(): + where, _, cell_why = key.partition("::") + if cell_why != why: + continue + where_options.append(where) + n_candidates += int(cell.get("n_candidates", 0)) + trajectory_ids.extend(cell.get("trajectory_ids", [])) + targets.append( + WhyTarget( + why=why, + where_options=sorted(set(where_options)), + n_candidates=n_candidates, + trajectory_ids=trajectory_ids, + ) + ) + return targets + + +def build_design_messages( + target: WhyTarget, + *, + attempt_index: int, + parent_summary: str, + file_context: str = "", + archive_summary: str = "", +) -> list[dict[str, str]]: + """Assemble the design prompt for one candidate against one WHY. + + ``archive_summary`` (the GSME elite bank, one line per cell) tells the + driver which mechanisms are already verified, so it neither re-invents a + banked win nor re-tries a pruned approach as if it were novel. + """ + system = ( + "You design a single harness patch that fixes one pathology in an agent " + "harness. Output ONLY a JSON object, no prose, no code fences, with keys: " + "patch_where (one of the given WHERE options), patch_why, patch_why_extra " + "(null unless patch_why is 'other'), overall_reasoning, and components: a " + "list of {component_id, target_file, diff, rationale, depends_on}. The diff " + "must be a valid unified diff. The patch MUST be env-gated and default OFF " + "so the harness is byte-identical to vanilla when the flag is unset. Prefer " + "one component for a simple fix." + ) + user = ( + f"Pathology (WHY): {target.why}\n" + f"Allowed WHERE levers: {', '.join(target.where_options) or 'config'}\n" + f"Parent harness: {parent_summary}\n" + f"This is design attempt #{attempt_index}; make it a distinct approach " + f"from other attempts on this WHY (different lever or mechanism).\n" + f"Evidence trajectories: {', '.join(target.trajectory_ids[:8])}\n" + f"{(archive_summary + chr(10)) if archive_summary else ''}" + f"{('Target file context:' + chr(10) + file_context) if file_context else ''}" + ) + return [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + + +def _parse_applied_patch(raw: str) -> AppliedPatch: + s = raw.strip() + if s.startswith("```"): + s = s.split("```", 2)[1] + if s[:4].lower() == "json": + s = s[4:] + start, end = s.find("{"), s.rfind("}") + if start < 0 or end < 0: + raise ValueError("no JSON object found in design output") + return AppliedPatch.from_dict(json.loads(s[start : end + 1])) + + +def design_candidate( + call_fn: CallFn, + target: WhyTarget, + *, + attempt_index: int, + parent_summary: str, + file_context: str = "", + archive_summary: str = "", + max_retries: int = 3, +) -> AppliedPatch: + """Design one env-gated candidate patch for ``target`` (schema-validated).""" + messages = build_design_messages( + target, + attempt_index=attempt_index, + parent_summary=parent_summary, + file_context=file_context, + archive_summary=archive_summary, + ) + node: SemanticNode[AppliedPatch] = SemanticNode( + name=f"design:{target.why}#{attempt_index}", + call_fn=call_fn, + parse_fn=_parse_applied_patch, + parse_error_types=(ValueError, json.JSONDecodeError), + max_retries=max_retries, + ) + return node.run(messages) + + +def design_round( + call_fn: CallFn, + failure_map: dict[str, Any], + budget: Budget, + *, + parent_summary: str = "vanilla", + file_context_for: Any = None, + archive_summary: str = "", + max_retries: int = 3, +) -> list[AppliedPatch]: + """Select WHYs and design up to ``candidates_per_why`` candidates each. + + ``file_context_for`` is an optional ``callable(WhyTarget) -> str`` returning + the target file contents to ground the diff. A candidate whose design never + parses is skipped rather than aborting the round. + """ + patches: list[AppliedPatch] = [] + for target in select_target_whys(failure_map, budget): + file_context = file_context_for(target) if file_context_for else "" + for attempt in range(1, budget.candidates_per_why + 1): + try: + patches.append( + design_candidate( + call_fn, + target, + attempt_index=attempt, + parent_summary=parent_summary, + file_context=file_context, + archive_summary=archive_summary, + max_retries=max_retries, + ) + ) + except Exception: # noqa: BLE001 — skip a candidate that won't parse + continue + return patches + + +__all__ = [ + "WhyTarget", + "select_target_whys", + "build_design_messages", + "design_candidate", + "design_round", +] diff --git a/raven/evolver/orchestrator/nodes/diagnose.py b/raven/evolver/orchestrator/nodes/diagnose.py new file mode 100644 index 0000000..cd91eed --- /dev/null +++ b/raven/evolver/orchestrator/nodes/diagnose.py @@ -0,0 +1,134 @@ +"""Step ① — diagnose failing trajectories into a failure map (semantic). + +This is the first semantic step, and it reuses the canonical judge stack rather +than re-inventing it: the judge already has a system prompt +(:func:`build_judge_messages`), a defect-tolerant parser +(:func:`parse_judge_output`, which strips code fences and enforces the L1/L2/L3 +schema invariants), and a failure-map aggregator +(:func:`build_failure_map`). All this node adds is the bounded repair-retry from +:class:`SemanticNode`, so a weak driver that emits malformed judge JSON gets the +parse error fed back and retries instead of derailing the round. + +``diagnose_trajectory`` judges one trajectory into a :class:`JudgeResult`; +``diagnose_round`` judges a batch and folds them into the cross-round failure +map. The result is a plain dict — the loop persists and re-reads it, so no model +holds diagnosis state across rounds. +""" + +from __future__ import annotations + +from typing import Any, Sequence + +from raven.evolver.analysis.failure_map_builder import build_failure_map +from raven.evolver.judge.parser import JudgeParseError, parse_judge_output +from raven.evolver.judge.prompts import build_judge_messages +from raven.evolver.judge.schema import JudgeResult +from raven.evolver.orchestrator.nodes.semantic import CallFn, SemanticNode + + +def diagnose_trajectory( + call_fn: CallFn, + *, + trajectory_id: str, + task_description: str, + trajectory_text: str, + max_retries: int = 3, +) -> JudgeResult: + """Judge one trajectory into a validated :class:`JudgeResult`.""" + messages = build_judge_messages( + trajectory_id=trajectory_id, + task_description=task_description, + trajectory_text=trajectory_text, + ) + node: SemanticNode[JudgeResult] = SemanticNode( + name=f"diagnose:{trajectory_id}", + call_fn=call_fn, + parse_fn=lambda raw: parse_judge_output(raw, expected_trajectory_id=trajectory_id), + parse_error_types=(JudgeParseError,), + max_retries=max_retries, + ) + return node.run(messages) + + +def diagnose_round( + call_fn: CallFn, + trajectories: Sequence[tuple[str, str, str]], + *, + min_why_classes: int = 7, + max_retries: int = 3, +) -> dict[str, Any]: + """Judge a batch of ``(trajectory_id, task_description, trajectory_text)`` and + aggregate into a failure map. + + A trajectory whose diagnosis fails to parse after all retries is skipped + (its id collected under ``_diagnose_failures``) rather than aborting the + round — one unparseable trajectory should not sink a round's diagnosis. + """ + results: list[JudgeResult] = [] + failures: list[str] = [] + for trajectory_id, task_description, trajectory_text in trajectories: + try: + results.append( + diagnose_trajectory( + call_fn, + trajectory_id=trajectory_id, + task_description=task_description, + trajectory_text=trajectory_text, + max_retries=max_retries, + ) + ) + except Exception: # noqa: BLE001 — record and continue; see docstring + failures.append(trajectory_id) + + failure_map = build_failure_map(results, min_why_classes=min_why_classes) + if failures: + failure_map["_diagnose_failures"] = failures + return failure_map + + +def merge_failure_maps(acc: dict[str, Any], new: dict[str, Any]) -> dict[str, Any]: + """Append ``new`` into the accumulated failure map (cross-round live map). + + The failure map is meant to accumulate across rounds (SOP §2 ①: a live map + that accumulates across rounds), so WHY-distribution shifts are auditable + over the evolution. Cells + are merged by ``WHERE::WHY`` key (trajectory ids / candidates concatenated, + counts summed); distributions and totals are summed; covered WHY classes are + unioned. ``acc`` empty returns a copy of ``new``. + """ + if not acc: + return dict(new) + m = dict(acc) + for k in ("n_total_judged", "n_l1", "n_l2", "n_l3"): + m[k] = acc.get(k, 0) + new.get(k, 0) + if "_n_judged" in acc or "_n_judged" in new: + m["_n_judged"] = acc.get("_n_judged", 0) + new.get("_n_judged", 0) + diag_failures = list(acc.get("_diagnose_failures", [])) + list(new.get("_diagnose_failures", [])) + if diag_failures: + m["_diagnose_failures"] = diag_failures + + cells = {k: dict(v) for k, v in acc.get("cells", {}).items()} + for key, cell in new.get("cells", {}).items(): + if key in cells: + base = cells[key] + base["n_candidates"] = base.get("n_candidates", 0) + cell.get("n_candidates", 0) + base["trajectory_ids"] = list(base.get("trajectory_ids", [])) + list(cell.get("trajectory_ids", [])) + base["candidates"] = list(base.get("candidates", [])) + list(cell.get("candidates", [])) + else: + cells[key] = dict(cell) + m["cells"] = cells + + for dist in ("where_distribution", "why_distribution"): + d = dict(acc.get(dist, {})) + for k, v in new.get(dist, {}).items(): + d[k] = d.get(k, 0) + v + m[dist] = d + + covered = sorted(set(acc.get("covered_why_classes", [])) | set(new.get("covered_why_classes", []))) + m["covered_why_classes"] = covered + m["covered_why_count"] = len(covered) + m["l1_alerts"] = list(acc.get("l1_alerts", [])) + list(new.get("l1_alerts", [])) + return m + + +__all__ = ["diagnose_trajectory", "diagnose_round", "merge_failure_maps"] diff --git a/raven/evolver/orchestrator/nodes/screen.py b/raven/evolver/orchestrator/nodes/screen.py new file mode 100644 index 0000000..e2bc5d9 --- /dev/null +++ b/raven/evolver/orchestrator/nodes/screen.py @@ -0,0 +1,97 @@ +"""Step ⑤a — the K=1 anchor screen with a wide-pass verdict. + +The screen is deliberately wide (wide-pass): a candidate advances to the full-set +confirm unless its anchor-mean pass@1 falls *clearly* below vanilla's. The SOP +describes three buckets, but operationally there is a single cut: + +- clear win (margin >= +cull_threshold) -> confirm +- within the band (|margin| < cull_threshold) -> confirm (a slightly-low + anchor mean is not the + full-set mean; don't cull) +- clear loss (margin <= -cull_threshold) -> pruned_at_screen + +``cull_threshold`` is ``cull_sigma_mult * sigma_screen`` from ``select_anchor``, +i.e. sized off the *same* vanilla thick ledger the anchor was drawn from, so the +band reflects genuine K=1 anchor-mean sampling noise rather than a guess. + +Vanilla's anchor mean is read from the cold-start thick-ledger per-task rates +(K=3, the fixed baseline the funnel always compares against); the candidate's is +the K=1 screen run. Comparing a noisy K=1 mean against a tighter K=3 mean is +exactly why the band is wide. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from raven.evolver.orchestrator.scoring import ( + TaskEval, + anchor_mean_pass_rate, +) +from raven.evolver.scheduler.anchor_selection import AnchorSelection + + +@dataclass(frozen=True) +class ScreenResult: + """Verdict of the anchor screen for one candidate.""" + + candidate_mean: float + vanilla_mean: float + cull_threshold: float + sigma_screen: float + passes_to_confirm: bool + bucket: str # "clear_win" | "within_band" | "cull" — for the log/ledger + + @property + def margin(self) -> float: + """Candidate anchor mean minus vanilla anchor mean (pp as a fraction).""" + return self.candidate_mean - self.vanilla_mean + + +def _vanilla_anchor_mean(vanilla_evals: dict[str, TaskEval], anchor_task_ids: list[str]) -> float: + """Mean vanilla per-task pass rate over the anchor subset (control arm). + + A control arm that is a fresh eval (e.g. a same-session paired baseline) can + legitimately be missing an anchor task that failed to launch, so a missing id + contributes 0.0 — symmetric with the candidate side (``anchor_mean_pass_rate``) + and safe for a wide-pass screen (a lower vanilla mean only advances more + candidates). Frozen ledger baselines have every anchor id, so this is a no-op + there. + """ + total = 0.0 + for task_id in anchor_task_ids: + ev = vanilla_evals.get(task_id) + total += ev.pass_rate if ev is not None else 0.0 + return total / len(anchor_task_ids) + + +def screen_candidate( + *, + candidate_evals: dict[str, TaskEval], + anchor: AnchorSelection, + vanilla_evals: dict[str, TaskEval], +) -> ScreenResult: + """Apply the wide-pass screen cut to one candidate's K=1 anchor eval.""" + candidate_mean = anchor_mean_pass_rate(candidate_evals, anchor.task_ids) + vanilla_mean = _vanilla_anchor_mean(vanilla_evals, anchor.task_ids) + margin = candidate_mean - vanilla_mean + cull = anchor.cull_threshold + + if margin >= cull: + bucket = "clear_win" + elif margin > -cull: + bucket = "within_band" + else: + bucket = "cull" + + return ScreenResult( + candidate_mean=candidate_mean, + vanilla_mean=vanilla_mean, + cull_threshold=cull, + sigma_screen=anchor.sigma_screen, + passes_to_confirm=(bucket != "cull"), + bucket=bucket, + ) + + +__all__ = ["ScreenResult", "screen_candidate"] diff --git a/raven/evolver/orchestrator/nodes/semantic.py b/raven/evolver/orchestrator/nodes/semantic.py new file mode 100644 index 0000000..cc44a89 --- /dev/null +++ b/raven/evolver/orchestrator/nodes/semantic.py @@ -0,0 +1,99 @@ +"""Semantic-node contract — the one place the driver model is trusted, bounded. + +A weak driver (Qwen / Kimi) follows instructions loosely and gives up early, so +every semantic step is reduced to a single call whose output must parse into a +fixed schema. When it doesn't, the node feeds the parse error back and retries a +bounded number of times before giving up — the same "retry, fix this" loop the +judge parser docstring anticipates, made explicit and reusable. + +This is deliberately transport-agnostic: ``call_fn`` takes chat messages and +returns the assistant text. In production it wraps a +``raven.evolver.judge.llm_client`` backend (which already routes self-hosted +Qwen, Claude, and OpenRouter); in tests it is a plain function returning scripted +strings. ``parse_fn`` turns raw text into the target schema object and raises on +any defect; ``SemanticNode`` catches that, appends a repair turn, and retries. + +The node is synchronous to match the orchestrator FSM. An async backend is +adapted by the caller (``asyncio.run`` in the production ``call_fn``); keeping the +retry logic sync avoids threading an event loop through the whole loop. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable, Generic, Sequence, TypeVar + +T = TypeVar("T") + +Messages = list[dict[str, str]] +CallFn = Callable[[Messages], str] +ParseFn = Callable[[str], T] + + +class SemanticNodeError(RuntimeError): + """Raised when a semantic node fails to produce a valid object in budget.""" + + def __init__(self, name: str, attempts: int, last_error: Exception) -> None: + super().__init__( + f"semantic node {name!r} failed to parse after {attempts} attempt(s); last error: {last_error!r}" + ) + self.name = name + self.attempts = attempts + self.last_error = last_error + + +def default_repair_prompt(error: Exception) -> str: + """The user turn appended after a parse failure to steer a retry.""" + return ( + "Your previous response could not be parsed into the required format. " + f"Error: {error}. Respond again with ONLY the valid object, no prose, " + "no code fences." + ) + + +@dataclass +class SemanticNode(Generic[T]): + """One schema-validated driver-model call with bounded repair-retry.""" + + name: str + call_fn: CallFn + parse_fn: ParseFn + max_retries: int = 3 + parse_error_types: tuple[type[Exception], ...] = (Exception,) + repair_prompt: Callable[[Exception], str] = default_repair_prompt + + def run(self, messages: Sequence[dict[str, str]]) -> T: + """Call the driver, parse to schema, repairing up to ``max_retries`` times. + + Returns the parsed object on the first success. Raises + :class:`SemanticNodeError` if every attempt (initial + retries) fails to + parse. The raw text of each attempt is preserved in the conversation so + the model sees its own bad output alongside the error. + """ + convo: Messages = [dict(m) for m in messages] + last_error: Exception | None = None + attempts = self.max_retries + 1 + for attempt in range(attempts): + raw = self.call_fn(convo) + try: + return self.parse_fn(raw) + except self.parse_error_types as exc: + last_error = exc + if attempt == attempts - 1: + break + convo = convo + [ + {"role": "assistant", "content": raw}, + {"role": "user", "content": self.repair_prompt(exc)}, + ] + assert last_error is not None # loop runs at least once + raise SemanticNodeError(self.name, attempts, last_error) + + +__all__ = [ + "SemanticNode", + "SemanticNodeError", + "default_repair_prompt", + "CallFn", + "ParseFn", + "Messages", +] diff --git a/raven/evolver/orchestrator/nodes/taxonomy.py b/raven/evolver/orchestrator/nodes/taxonomy.py new file mode 100644 index 0000000..c36572e --- /dev/null +++ b/raven/evolver/orchestrator/nodes/taxonomy.py @@ -0,0 +1,497 @@ +"""Bench-neutral WHY/WHERE taxonomy: the spec + open-ended induction (map-reduce). + +A benchmark's diagnosis step classifies failing trajectories into a +:class:`TaxonomySpec` — WHY (failure-mode) x WHERE (patch-lever) classes. A +bench that has a hand-derived table (AppWorld's W1-W7) passes it as a frozen +constant; a brand-new bench can *discover* one from its vanilla failures via +:func:`induce_taxonomy`: stage-1 writes one open-ended failure report per +trajectory (parallel, no preset table), stage-2 clusters all reports into +classes and assigns each report to its WHY(s). + +Induction failure raises :class:`TaxonomyInductionError` — there is no silent +fallback here, because the only universally-wrong answer for "a brand-new +benchmark's taxonomy" is some *other* benchmark's table. The bench wiring +decides what its safe default is (or lets the run stop loudly). +""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional + +DEFAULT_BENCH_DESC = "an agent harness benchmark" + + +class TaxonomyInductionError(RuntimeError): + """Raised when open-ended taxonomy induction produced no usable taxonomy.""" + + +@dataclass(frozen=True) +class TaxonomySpec: + """A benchmark's WHY (failure-mode) x WHERE (patch-lever) classification. + + ``why_classes`` / ``where_classes`` map a stable key to a one-line + description. ``other`` (why) and ``none`` (where) are the escape hatches and + are always present (added on construction if the source omitted them). + """ + + why_classes: dict[str, str] + where_classes: dict[str, str] + + def __post_init__(self) -> None: + if "other" not in self.why_classes: + self.why_classes["other"] = "None of the above — provide a short sub-name." + if "none" not in self.where_classes: + self.where_classes["none"] = "No harness lever applies." + + def to_dict(self) -> dict[str, Any]: + return {"why_classes": dict(self.why_classes), "where_classes": dict(self.where_classes)} + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "TaxonomySpec": + return cls(dict(d["why_classes"]), dict(d["where_classes"])) + + +def strip_code_fence(raw: str) -> str: + s = raw.strip() + if "```" in s: + s = s.split("```")[1] if s.count("```") >= 2 else s + s = s.split("\n", 1)[-1] if s.lstrip().startswith(("json", "JSON")) else s + return s + + +def _why_prefix_match(why: str, key: str) -> bool: + """True when ``why`` carries ``key``'s leading code with a clean boundary + (``W1_x`` matches ``W1_...`` but ``W10_x`` must not match ``W1_...``).""" + prefix = key.split("_")[0].upper() + w = str(why).upper() + if not w.startswith(prefix): + return False + rest = w[len(prefix) : len(prefix) + 1] + return not rest.isalnum() + + +def coerce_mode(obj: dict, taxonomy: TaxonomySpec) -> dict: + """Normalise one diagnosed failure mode onto the taxonomy's keys.""" + why = obj.get("why") + if why not in taxonomy.why_classes: + cand = next( + (k for k in taxonomy.why_classes if why and _why_prefix_match(why, k)), + None, + ) + why = cand or "other" + where = obj.get("where") if obj.get("where") in taxonomy.where_classes else "none" + return { + "why": why, + "where": where, + "dominant": bool(obj.get("dominant", False)), + "reasoning": str(obj.get("reasoning", ""))[:400], + "fix_hint": str(obj.get("fix_hint", ""))[:300], + } + + +def empty_failure_map() -> dict: + return {"why_distribution": {}, "cells": {}, "_n_judged": 0} + + +def add_failure_mode(fm: dict, trajectory_id: str, mode: dict) -> None: + why, where = mode["why"], mode["where"] + # Dominant-weighted: co-occurring secondary symptoms (near-universal on + # failing runs) at full weight would drown the causal mode in the + # distribution the WHY selection ranks on. Modes without the flag (legacy + # callers) keep the old full weight. + weight = 1.0 if mode.get("dominant", True) else 0.5 + fm["why_distribution"][why] = fm["why_distribution"].get(why, 0) + weight + cell = fm["cells"].setdefault(f"{where}::{why}", {"candidates": []}) + cell["candidates"].append( + { + "trajectory_id": trajectory_id, + "reasoning": mode["reasoning"], + "components": [{"summary": mode["fix_hint"]}] if mode["fix_hint"] else [], + } + ) + + +def _parse_modes(raw: str, taxonomy: TaxonomySpec) -> list[dict] | None: + """Parse a multi-label diagnosis response into a list of modes. + + Accepts a JSON array (preferred) or a single JSON object (back-compat with + single-label callers) -> normalised to a one-element list. + """ + s = strip_code_fence(raw) + obj = None + i, j = s.find("["), s.rfind("]") + if i >= 0 and j > i: + try: + obj = json.loads(s[i : j + 1]) + except json.JSONDecodeError: + obj = None + if obj is None: + i, j = s.find("{"), s.rfind("}") + if i >= 0 and j > i: + try: + obj = json.loads(s[i : j + 1]) + except json.JSONDecodeError: + obj = None + if obj is None: + return None + items = obj if isinstance(obj, list) else [obj] + modes = [coerce_mode(x, taxonomy) for x in items if isinstance(x, dict)] + if not modes: + return None + # Exactly one dominant mode: keep the first flagged one; when the model + # marked none, the ordering rule ("dominant first") makes index 0 it. + first = next((i for i, m in enumerate(modes) if m["dominant"]), 0) + for i, m in enumerate(modes): + m["dominant"] = i == first + return modes + + +def classify_failures( + call_fn: Callable[[list], str], + trajectories, + taxonomy: TaxonomySpec, + *, + bench_intro: str, + extra_rules: str = "", + max_workers: int = 8, + retries: int = 2, +) -> dict: + """Judge failing trajectories into a multi-label failure_map over ``taxonomy``. + + The bench-neutral diagnosis core: ``bench_intro`` describes the harness and + task shape (one or two sentences); ``extra_rules`` optionally appends + taxonomy-specific guidance (e.g. which WHYs are capability ceilings). + ``trajectories`` = ``(trajectory_id, task_description, transcript)`` tuples; + each trajectory can contribute several modes and every hit increments its + WHY. Output shape matches ``failure_map_builder.build_failure_map`` so + ``select_target_whys`` / the design step consume it unchanged. + """ + sys = ( + f"{bench_intro} You are given ONE failing trajectory. Classify " + "ALL failure modes it exhibits (usually 1-3, occasionally more) — a trajectory can fail in " + "several ways at once. For EACH mode name its WHY class, the best patch location (WHERE), a " + "one-line reasoning and a concrete fix hint.\n\n" + "WHY classes:\n" + "\n".join(f" - {k}: {v}" for k, v in taxonomy.why_classes.items()) + "\n\n" + "WHERE classes:\n" + "\n".join(f" - {k}: {v}" for k, v in taxonomy.where_classes.items()) + "\n\n" + 'Rules: mark EXACTLY ONE mode "dominant": true — the failure that directly explains the ' + "benchmark's verdict — and list it first; other modes are secondary symptoms " + '("dominant": false). ' + + (extra_rules + "\n" if extra_rules else "") + + "Respond with ONLY a JSON ARRAY, no prose, no code fences; each element is one mode:\n" + '[{"why":"","where":"","dominant":true|false,' + '"reasoning":"<=1 line","fix_hint":"<=1 line concrete lever>"}]' + ) + + def _one(t): + tid, desc, transcript = t + user = ( + f"TASK: {desc}\n\nFAILING TRAJECTORY (trajectory_id={tid}):\n{transcript}\n\n" + "Classify ALL failure modes. JSON array only." + ) + msgs = [{"role": "system", "content": sys}, {"role": "user", "content": user}] + for _ in range(retries + 1): + try: + modes = _parse_modes(call_fn(msgs), taxonomy) + except Exception: # noqa: BLE001 + modes = None + if modes: + return tid, modes + msgs = msgs + [{"role": "user", "content": "Return ONLY a JSON array of {why,where,...} with valid keys."}] + return None + + fm = empty_failure_map() + with ThreadPoolExecutor(max_workers=max_workers) as ex: + for r in ex.map(_one, list(trajectories)): + if not r: + continue + tid, modes = r + fm["_n_judged"] += 1 + for mode in modes: + add_failure_mode(fm, tid, mode) + return fm + + +# ---- open-ended taxonomy induction (map-reduce; only for a new benchmark) ---- + + +def _parse_reports(raw: str) -> list[dict] | None: + """Stage-1 report parse: list of {failure_point, evidence, fixes:[...]}.""" + s = strip_code_fence(raw) + i, j = s.find("["), s.rfind("]") + if i < 0 or j <= i: + i, j = s.find("{"), s.rfind("}") + if i < 0 or j <= i: + return None + s = "[" + s[i : j + 1] + "]" + else: + s = s[i : j + 1] + try: + arr = json.loads(s) + except json.JSONDecodeError: + return None + return [x for x in arr if isinstance(x, dict)] or None + + +def _induce_reports(call_fn, trajectories, *, bench_desc, max_workers, retries) -> list[dict]: + """Stage 1 (map): one open-ended failure report per trajectory (parallel).""" + sys = ( + f"You analyse ONE failing agent trajectory from {bench_desc} with NO preset failure " + "taxonomy. Describe every distinct way it failed (usually 1-3). For each: the concrete " + "failure_point (what went wrong, at which step), evidence (quote one line from the " + "trajectory), and harness_fixes (1-2 concrete ways an agent-harness change — prompt / a " + "runtime hook / the exec tool / the loop — could prevent it; NOT model retraining). Keep " + "each field short.\n" + 'Respond ONLY a JSON array: [{"failure_point":"...","evidence":"...","harness_fixes":["...","..."]}]' + ) + + def _one(t): + tid, desc, transcript = t + user = ( + f"TASK: {desc}\n\nFAILING TRAJECTORY ({tid}):\n{transcript}\n\nReport its failure modes. JSON array only." + ) + msgs = [{"role": "system", "content": sys}, {"role": "user", "content": user}] + for _ in range(retries + 1): + try: + reps = _parse_reports(call_fn(msgs)) + except Exception: # noqa: BLE001 + reps = None + if reps: + return {"trajectory_id": tid, "modes": reps} + msgs = msgs + [{"role": "user", "content": "Return ONLY the JSON array."}] + return None + + out = [] + with ThreadPoolExecutor(max_workers=max_workers) as ex: + for r in ex.map(_one, list(trajectories)): + if r: + out.append(r) + return out + + +def _parse_taxonomy(raw: str) -> tuple[TaxonomySpec, list[dict]]: + """Stage-2 reduce parse: TaxonomySpec + per-report assignments (multi-label).""" + s = strip_code_fence(raw) + i, j = s.find("{"), s.rfind("}") + if i < 0 or j <= i: + raise ValueError("no JSON object in taxonomy reduce output") + obj = json.loads(s[i : j + 1]) + why = {c["key"]: str(c.get("desc", "")) for c in obj.get("why_classes", []) if c.get("key")} + where = {c["key"]: str(c.get("desc", "")) for c in obj.get("where_classes", []) if c.get("key")} + if not why: + raise ValueError("reduce produced no why_classes") + assignments = [a for a in obj.get("assignments", []) if isinstance(a, dict)] + return TaxonomySpec(why, where), assignments + + +def _pack_reports(reports: list[dict], *, budget: int) -> str: + """JSON-array payload of WHOLE reports within ``budget`` chars. + + A raw ``json.dumps(reports)[:budget]`` cut mid-record, leaving invalid + JSON and silently biasing the taxonomy toward early trajectories; packing + whole records keeps the payload parseable and makes any drop explicit + (the trailing marker names how many were left out). + """ + parts: list[str] = [] + size = 2 # brackets + dropped = 0 + for r in reports: + s = json.dumps(r) + if parts and size + len(s) + 2 > budget: + dropped += 1 + continue + parts.append(s) + size += len(s) + 2 + payload = "[" + ", ".join(parts) + "]" + if dropped: + payload += f"\n({dropped} more reports omitted for length)" + return payload + + +def induce_taxonomy( + call_fn: Callable[[list], str], + trajectories, + *, + bench_desc: str = DEFAULT_BENCH_DESC, + max_workers: int = 8, + retries: int = 2, + target_min: int = 5, + target_max: int = 9, +) -> tuple[TaxonomySpec, dict]: + """Discover a WHY/WHERE taxonomy from vanilla failures (open-ended map-reduce). + + Stage 1 (map): one compact failure report per trajectory, no preset table. + Stage 2 (reduce): cluster all reports into ``target_min..target_max`` WHY + classes + WHERE lever classes, and assign each report to its WHY(s). Returns + the :class:`TaxonomySpec` plus a seed multi-label failure_map from the + assignments. Raises :class:`TaxonomyInductionError` when no report parses or + the reduce never yields a taxonomy — never silently substitutes another + bench's table. + """ + reports = _induce_reports(call_fn, trajectories, bench_desc=bench_desc, max_workers=max_workers, retries=retries) + if not reports: + raise TaxonomyInductionError("taxonomy induction produced no parseable per-trajectory reports") + + sys = ( + "You are consolidating many per-trajectory failure reports into a REUSABLE failure " + f"taxonomy for {bench_desc}. Cluster the reports into " + f"{target_min}-{target_max} WHY classes (abstract failure MODES, each a stable key like " + "'W1_...' + a one-line description) and a small set of WHERE classes (harness patch LEVERS " + "abstracted from the fixes: prompt / runtime hook / exec tool / loop / config / none). Then " + "assign each report (by trajectory_id) to its WHY key(s) — a report may map to several.\n" + 'Respond ONLY JSON: {"why_classes":[{"key":"W1_...","desc":"..."}],' + '"where_classes":[{"key":"...","desc":"..."}],' + '"assignments":[{"trajectory_id":"...","whys":["W1_..."],"wheres":["..."]}]}' + ) + payload = _pack_reports(reports, budget=60000) + msgs = [ + {"role": "system", "content": sys}, + {"role": "user", "content": f"Reports:\n{payload}\n\nProduce the taxonomy JSON."}, + ] + taxonomy: TaxonomySpec | None = None + assignments: list[dict] = [] + last_exc: Exception | None = None + for _ in range(retries + 1): + try: + taxonomy, assignments = _parse_taxonomy(call_fn(msgs)) + break + except Exception as exc: # noqa: BLE001 + last_exc = exc + msgs = msgs + [{"role": "user", "content": f"Invalid ({exc}). Return ONLY the JSON object."}] + if taxonomy is None: + raise TaxonomyInductionError(f"taxonomy reduce failed after {retries + 1} attempts; last error: {last_exc!r}") + + # The seed carries the stage-1 report content (failure_point / harness_fixes) + # into its cells, so a round-1 design step can consume it directly — the + # trajectories were just judged, re-judging them would only re-spend the driver. + seed = empty_failure_map() + modes_by_tid = {r["trajectory_id"]: r["modes"] for r in reports} + for a in assignments: + tid = a.get("trajectory_id") + if tid not in modes_by_tid: + continue + seed["_n_judged"] += 1 + whys = a.get("whys") or [] + wheres = a.get("wheres") or ["none"] + reps = modes_by_tid[tid] + for i, w in enumerate(whys): + rep = reps[i] if i < len(reps) else reps[0] + fixes = rep.get("harness_fixes") or [] + add_failure_mode( + seed, + tid, + coerce_mode( + { + "why": w, + "where": wheres[0], + "dominant": i == 0, + "reasoning": str(rep.get("failure_point", "")), + "fix_hint": str(fixes[0]) if fixes else "", + }, + taxonomy, + ), + ) + return taxonomy, seed + + +def ensure_taxonomy( + call_fn: Callable[[list], str], + trajectories, + path: str | Path, + *, + mode: str = "hardcoded", + default: Optional[TaxonomySpec] = None, + bench_desc: str = DEFAULT_BENCH_DESC, + max_workers: int = 8, + seed_path: Optional[str | Path] = None, +) -> TaxonomySpec: + """Resolve the taxonomy for a bench: hardcoded default, or induce-and-cache. + + ``mode="hardcoded"`` returns ``default`` (required — the bench's own table). + ``mode="induce"`` loads ``path`` if it exists, else runs + :func:`induce_taxonomy` over ``trajectories`` and persists the result to + ``path`` — induction runs once and every later call reuses the frozen + taxonomy. Induction failure raises; it never falls back to ``default``. + + ``seed_path`` (optional) persists the induction's seed failure map next to + the taxonomy, so the caller can feed it to round 1 instead of re-judging the + very trajectories induction just judged (round-0 is genuinely free). It is + written only when induction actually runs; a cached taxonomy leaves any + previously written seed in place. + """ + if mode == "hardcoded": + if default is None: + raise ValueError('mode="hardcoded" requires a default TaxonomySpec') + return default + p = Path(path) + if p.exists(): + return TaxonomySpec.from_dict(json.loads(p.read_text())) + taxonomy, seed = induce_taxonomy(call_fn, trajectories, bench_desc=bench_desc, max_workers=max_workers) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(taxonomy.to_dict(), indent=2)) + if seed_path is not None: + sp = Path(seed_path) + sp.parent.mkdir(parents=True, exist_ok=True) + sp.write_text(json.dumps(seed, indent=2)) + return taxonomy + + +def resolve_taxonomy( + call_fn: Callable[[list], str], + trajectory_source: Callable[[int, Any], list], + vanilla_node: Any, + *, + mode: str, + work_dir: str | Path, + hardcoded: Optional[TaxonomySpec] = None, + taxonomy_path: Optional[str | Path] = None, +) -> tuple[TaxonomySpec, Optional[dict]]: + """Resolve the round's taxonomy + (for induce) a round-1 seed failure map. + + The shared front half of both benches' diagnose wiring. In ``"hardcoded"`` + mode returns the caller-supplied ``hardcoded`` table and no seed. In + ``"induce"`` mode discovers the table once from the vanilla failures (cached + to ``taxonomy_path`` / ``work_dir/taxonomy.json``) and, because induction + judges those failures already, returns its seed failure map marked with the + root as diagnosed — so round 1 reuses it instead of re-judging the same + trajectories (round-0 free). Requires the vanilla ledger to already exist + (cold start run) so ``trajectory_source`` has trajectories to read. + """ + if mode == "hardcoded": + if hardcoded is None: + raise ValueError('resolve_taxonomy mode="hardcoded" requires a table') + return hardcoded, None + tax_path = Path(taxonomy_path) if taxonomy_path else (Path(work_dir) / "taxonomy.json") + seed_path = tax_path.with_name(tax_path.stem + "_seed.json") + taxonomy = ensure_taxonomy( + call_fn, + trajectory_source(1, vanilla_node), + tax_path, + mode="induce", + seed_path=seed_path, + ) + seed_failure_map: Optional[dict] = None + if seed_path.exists(): + seed = json.loads(seed_path.read_text()) + if seed.get("_n_judged"): + seed["_diagnosed_parents"] = [vanilla_node.node_id] + seed_failure_map = seed + return taxonomy, seed_failure_map + + +__all__ = [ + "TaxonomySpec", + "TaxonomyInductionError", + "DEFAULT_BENCH_DESC", + "strip_code_fence", + "coerce_mode", + "empty_failure_map", + "add_failure_mode", + "classify_failures", + "induce_taxonomy", + "ensure_taxonomy", + "resolve_taxonomy", +] diff --git a/raven/evolver/orchestrator/nodes/verdict.py b/raven/evolver/orchestrator/nodes/verdict.py new file mode 100644 index 0000000..3782ee6 --- /dev/null +++ b/raven/evolver/orchestrator/nodes/verdict.py @@ -0,0 +1,109 @@ +"""Step ⑦ — draft a per-round verdict for the findings log (semantic). + +After the gates decide bank/prune, the driver drafts a short narrative for the +findings log: what this round tried, what the result was, the next target, and +whether the curve looks like a capability ceiling. This is advisory — the loop's +stop decision is the deterministic :class:`TerminationTracker`, never this +verdict — but the ``ceiling_signal`` hint is what a human reads to decide an +early unseal (SOP §154 low-ceiling note). + +Kept schema-light (three fields) so even a weak driver returns something usable; +parse failure is non-fatal — the caller falls back to a plain summary. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Optional + +from raven.evolver.orchestrator.nodes.semantic import CallFn, SemanticNode + + +@dataclass(frozen=True) +class Verdict: + """Driver's narrative verdict on one round.""" + + summary: str + next_target: str + ceiling_signal: bool + + +def _parse_verdict(raw: str) -> Verdict: + s = raw.strip() + if s.startswith("```"): + s = s.split("```", 2)[1] + if s[:4].lower() == "json": + s = s[4:] + start, end = s.find("{"), s.rfind("}") + if start < 0 or end < 0: + raise ValueError("no JSON object in verdict output") + d = json.loads(s[start : end + 1]) + return Verdict( + summary=str(d["summary"]), + next_target=str(d.get("next_target", "")), + ceiling_signal=bool(d.get("ceiling_signal", False)), + ) + + +_FIELD_LEGEND = ( + "Field legend for the results lines: screen/confirm = the candidate patch's " + "pass-rate on the probe subset / the full train set; van = the vanilla (parent " + "baseline) harness on the SAME tasks — not a validation split; full_lift = " + "confirm - van; foc_c/foc_v = candidate/baseline pass-rate on the focused " + "(diagnosed) subset; fisher_p / z = significance of that focused comparison; " + "sent_c/sent_v = sentinel-guard pass-rates on known-healthy tasks; credited = " + "whether the patch's mechanism demonstrably fired on the tasks it changed." +) + + +def draft_verdict( + call_fn: CallFn, + *, + round_index: int, + round_summary: str, + history: str = "", + why_keys: Optional[list[str]] = None, + max_retries: int = 2, +) -> Verdict: + """Draft a verdict for one round from a factual ``round_summary`` string. + + ``history`` (prior rounds' factual summaries) grounds ``ceiling_signal`` — a + curve cannot be judged from one round. ``why_keys`` constrains + ``next_target`` to the taxonomy so the field stays machine-readable. + """ + target_rule = f" — MUST be one of the WHY keys: {', '.join(why_keys)}" if why_keys else "" + messages = [ + { + "role": "system", + "content": ( + "You write a one-round verdict for a self-evolution log: an outer " + "loop patches an agent harness, evaluates each candidate patch " + "against a frozen baseline on a train set, and promotes or prunes " + "it. " + _FIELD_LEGEND + " Output ONLY JSON with keys: summary " + "(what happened this round, using the legend's terms), next_target " + f"(what to attack next{target_rule}), ceiling_signal (true ONLY " + "when the HISTORY shows >=2 consecutive rounds with no promotion " + "and non-increasing lift — never judged from a single round). " + "No prose." + ), + }, + { + "role": "user", + "content": ( + (f"HISTORY (previous rounds):\n{history}\n\n" if history else "") + + f"Round {round_index} results:\n{round_summary}" + ), + }, + ] + node: SemanticNode[Verdict] = SemanticNode( + name=f"verdict:r{round_index}", + call_fn=call_fn, + parse_fn=_parse_verdict, + parse_error_types=(ValueError, KeyError, json.JSONDecodeError), + max_retries=max_retries, + ) + return node.run(messages) + + +__all__ = ["Verdict", "draft_verdict"] diff --git a/raven/evolver/orchestrator/production.py b/raven/evolver/orchestrator/production.py new file mode 100644 index 0000000..059a659 --- /dev/null +++ b/raven/evolver/orchestrator/production.py @@ -0,0 +1,709 @@ +"""Production wiring — turn the injectable FSM into a runnable orchestrator. + +:class:`EvolutionOrchestrator` takes one :class:`EvalBackend` plus injected +semantic steps, so this module supplies the real ones: + +- ``make_diagnose_fn`` / ``make_design_fn`` / ``make_verdict_fn`` bind a driver + ``call_fn`` (from :mod:`.providers.openai_compat`) to the semantic nodes. +- ``make_llm_backend`` builds the one bench-neutral :class:`EvalBackend`: an LLM + judges each trajectory pass/fail with no external verifier at all. The + concrete benchmark backends (AppWorld, EvoAgentBench) are built by their own + ``make_*_backend`` factories under ``benchmarks..evolve`` — the + orchestrator core names no benchmark. All emit the same ``dict[str, TaskEval]`` + contract, so the loop stays bench-agnostic. +- ``make_metadata_apply_fn`` records a child node without touching git (replay / + dry runs); a real run passes ``EvolverTreeStore.create_child_node``. + +``build_orchestrator`` assembles these into an :class:`EvolutionOrchestrator`. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional + +from raven.evolver.analysis.stability_bucket import ( + TaskStability, + _bucket_for, +) +from raven.evolver.judge.llm_client import JudgeLLMBackend +from raven.evolver.judge.parser import JudgeParseError, parse_pass_fail +from raven.evolver.judge.prompts import build_pass_fail_messages +from raven.evolver.orchestrator.config import Budget, OrchestratorConfig +from raven.evolver.orchestrator.loop import ( + EvolutionOrchestrator, + RoundResult, + summarize_round, +) +from raven.evolver.orchestrator.nodes.design import design_round +from raven.evolver.orchestrator.nodes.diagnose import diagnose_round +from raven.evolver.orchestrator.nodes.semantic import CallFn, SemanticNode +from raven.evolver.orchestrator.nodes.verdict import draft_verdict +from raven.evolver.orchestrator.scoring import ( + EvalBackend, + TaskEval, + TrajectorySource, +) +from raven.evolver.scheduler.anchor_selection import simple_anchor +from raven.evolver.tree.node import AppliedPatch, HarnessNode + +# per-task trajectory runner for the no-benchmark scorer: +# (node, task_ids, k) -> [(task_id, trajectory_id, task_description, text), ...] +ScoringTrajectoryRun = Callable[[HarnessNode, list, int], list] + + +@dataclass(frozen=True) +class EndpointConfig: + """One OpenAI-compatible driver endpoint.""" + + base_url: str + model: str + max_tokens: int = 8192 + temperature: float = 0.0 + + +# ---- semantic step adapters ------------------------------------------------- + + +def make_diagnose_fn( + call_fn: CallFn, + trajectory_source: TrajectorySource, + *, + min_why_classes: int = 7, +) -> Callable[[int, HarnessNode], dict]: + """Bind a driver + a trajectory source into the loop's ``diagnose_fn``.""" + + def diagnose_fn(round_index: int, parent: HarnessNode) -> dict: + trajectories = trajectory_source(round_index, parent) + return diagnose_round(call_fn, trajectories, min_why_classes=min_why_classes) + + return diagnose_fn + + +def make_design_fn( + call_fn: CallFn, + budget: Budget, + *, + file_context_for: Any = None, + parent_summary_of: Optional[Callable[[HarnessNode], str]] = None, + archive_summary_of: Optional[Callable[[], str]] = None, +) -> Callable[[int, dict, HarnessNode], list[AppliedPatch]]: + def design_fn(round_index: int, failure_map: dict, parent: HarnessNode) -> list[AppliedPatch]: + parent_summary = parent_summary_of(parent) if parent_summary_of else parent.node_id + return design_round( + call_fn, + failure_map, + budget, + parent_summary=parent_summary, + file_context_for=file_context_for, + archive_summary=archive_summary_of() if archive_summary_of else "", + ) + + return design_fn + + +def make_verdict_fn( + call_fn: CallFn, + *, + why_keys_of: Optional[Callable[[], Optional[list[str]]]] = None, +) -> Callable[[RoundResult], str]: + """``why_keys_of`` is a lazy getter: with taxonomy induction the WHY keys + only exist after the first diagnose, which still precedes round 1's verdict.""" + past: list[str] = [] + + def verdict_fn(rr: RoundResult) -> str: + summary = summarize_round(rr) + history = "\n".join(past[-5:]) + past.append(summary) + try: + v = draft_verdict( + call_fn, + round_index=rr.round_index, + round_summary=summary, + history=history, + why_keys=why_keys_of() if why_keys_of else None, + ) + return f"{v.summary} | next: {v.next_target} | ceiling={v.ceiling_signal}" + except Exception: # noqa: BLE001 — verdict is advisory; fall back to facts + return summary + + return verdict_fn + + +# ---- backend factory (the one bench-neutral EvalBackend) -------------------- + + +def make_llm_backend( + judge_backend: JudgeLLMBackend, + run_trajectories: ScoringTrajectoryRun, + *, + train_task_ids: list[str], + vanilla_node: HarnessNode, + test_task_ids: list[str] = (), + k: int = 3, + max_tokens: int = 1024, + cull_sigma_mult: float = 1.5, +) -> EvalBackend: + """No-benchmark backend: an LLM judges each trajectory pass/fail. + + ``run_trajectories(node, task_ids, k)`` yields ``(task_id, trajectory_id, + task_description, text)`` — the same trajectory machinery that feeds + diagnosis, here also driving scoring. Verdicts aggregate to + ``TaskEval(passes, attempts=K)``; ``infra_attempts`` is 0 so Gate-f is a + no-op and every downstream consumer is unchanged. + """ + + def call_fn(messages): + return asyncio.run(judge_backend.call(messages, max_tokens=max_tokens)) + + verdict_node: SemanticNode = SemanticNode( + name="pass_fail", + call_fn=call_fn, + parse_fn=parse_pass_fail, + parse_error_types=(JudgeParseError,), + ) + + def _score(node, task_ids, k_): + passes: dict[str, int] = {} + attempts: dict[str, int] = {} + for task_id, traj_id, desc, text in run_trajectories(node, list(task_ids), k_): + attempts[task_id] = attempts.get(task_id, 0) + 1 + verdict = verdict_node.run(build_pass_fail_messages(desc, text, trajectory_id=traj_id)) + if verdict.passed: + passes[task_id] = passes.get(task_id, 0) + 1 + return {t: TaskEval(t, passes.get(t, 0), n) for t, n in attempts.items()} + + def eval_fn(node, task_ids, k_, job_name, *, split="train"): + return _score(node, task_ids, k_) + + # One judge-scored vanilla pass, shared by cold_start AND anchor — scoring + # twice would double the judge cost and, worse, derive the screen control + # and the anchor thresholds from two different samples. + _stab: dict[str, TaskStability] = {} + + def cold_start() -> dict[str, TaskStability]: + if not _stab: + evals = _score(vanilla_node, list(train_task_ids), k) + _stab.update( + { + t: TaskStability(t, ev.attempts, ev.passes, _bucket_for(ev.passes, ev.attempts)) + for t, ev in evals.items() + } + ) + return dict(_stab) + + def anchor(affinity=None): + return simple_anchor(cold_start(), cull_sigma_mult=cull_sigma_mult) + + def trajectories(round_index, node): + return [(traj_id, desc, text) for _tid, traj_id, desc, text in run_trajectories(node, list(train_task_ids), 1)] + + return EvalBackend( + train_task_ids=list(train_task_ids), + test_task_ids=list(test_task_ids), + eval=eval_fn, + cold_start=cold_start, + anchor=anchor, + trajectories=trajectories, + ) + + +def make_git_commit_apply_fn( + repo_root: str | Path, + files_of: Callable[[Any], dict[str, bytes]], + *, + root_node_id: str, + base_sha: str, + deletions_of: Optional[Callable[[Any], list[str]]] = None, + guard_immutable: bool = True, + git_branch: str = "evolver/orchestrator", + sha_by_node: Optional[dict[str, str]] = None, + node_id_salt: Optional[str] = None, +) -> Callable[[str, Any, int], HarnessNode]: + """Edit-then-commit apply: turn a candidate's edited files into a REAL child + commit off the parent node's commit. + + ``files_of(patch)`` returns the candidate's full new file bytes (repo-relative + path -> bytes); the driver may have produced them however it likes (a bash + edit loop, a template, a diff already applied). We materialise them onto the + parent's commit and ``commit-tree`` — so the node gets a reproducible SHA and + git ancestry, and multi-round chaining works because each promoted node's SHA + is tracked and its children commit off it (no "sandbox" placeholder, no live + working-tree mutation). Immutable-kernel paths are guarded on the changed set. + + Node ids carry ``node_id_salt`` (a fresh 4-hex token per factory unless + given): ``v{round}-c{n}-{salt}``. A crash-resumed process re-runs the + incomplete round with NEW candidates; without the salt they would reuse the + dead run's ids and inherit its out-dirs, session files, and + ``refs/evolver/*`` refs — contaminating the eval with another candidate's + artifacts. The salt makes ids (and everything derived from them) unique per + process, and refs from concurrent/earlier runs are never repointed. + """ + import uuid + + from raven.evolver.applier import assert_patch_allowed + from raven.evolver.tree import git_ops + + root = Path(repo_root) + # A shared registry lets a design step base its sandbox off a promoted + # parent's real commit (same dict passed to both apply and design). + if sha_by_node is None: + sha_by_node = {} + sha_by_node.setdefault(root_node_id, base_sha) + salt = node_id_salt if node_id_salt is not None else uuid.uuid4().hex[:4] + counter = {"n": 0} + + def apply_fn(parent_id: str, patch: Any, round_index: int) -> HarnessNode: + parent_sha = sha_by_node.get(parent_id) + if parent_sha is None: + raise KeyError(f"unknown parent node {parent_id!r}: no commit recorded (chain broken)") + files = files_of(patch) + deletions = tuple(deletions_of(patch)) if deletions_of else () + if guard_immutable: # guard before committing, so no dangling commit on reject + assert_patch_allowed(list(files) + list(deletions)) + child_sha, _changed = git_ops.commit_files_as_child( + root, + parent_sha, + files, + f"evolver: round {round_index} candidate off {parent_id}", + deletions=deletions, + ) + counter["n"] += 1 + node_id = f"v{round_index}-c{counter['n']}-{salt}" if salt else f"v{round_index}-c{counter['n']}" + sha_by_node[node_id] = child_sha + # Anchor the (otherwise unreferenced) candidate commit against git gc, + # so a late worktree eval / post-hoc sealed unseal still finds it. + git_ops.create_ref(root, f"refs/evolver/{node_id}", child_sha) + return HarnessNode( + node_id=node_id, + parent_id=parent_id, + git_commit_sha=child_sha, + git_branch=git_branch, + created_at=HarnessNode.utc_now(), + created_at_iter=round_index, + patch=patch if isinstance(patch, AppliedPatch) else None, + ) + + return apply_fn + + +def make_zero_hit_preflight(trajectory_source: TrajectorySource): + """SOP §2 ③ zero-hit prune: drop a candidate whose self-declared trigger + regex matches NONE of the parent's failing trajectories — it would never + fire, so screening/confirming it only burns eval budget (the "provably + inert" preflight of the paper). + + Fail-open everywhere: no declared spec, a malformed regex, an unreadable + corpus, or an empty corpus all keep the candidate — preflight may only + prune on positive evidence of inertness. The regex runs over the same + rendered trajectory text the driver read when authoring it + (``read_trajectory``), so the predicate and its corpus are self-consistent. + """ + import re as _re + + corpus_cache: dict[str, list[str]] = {} + + def preflight(cand, parent) -> bool: + spec = getattr(cand, "activation_spec", None) + if not isinstance(spec, dict) or spec.get("kind") != "trajectory_regex": + return True + try: + pat = _re.compile(str(spec.get("pattern", "")), _re.M) + except _re.error: + return True + texts = corpus_cache.get(parent.node_id) + if texts is None: + try: + # description + transcript: the same surface read_trajectory + # showed the driver, so a predicate authored against it matches. + texts = [f"{t[1]}\n{t[2]}" for t in trajectory_source(0, parent)] + except Exception: # noqa: BLE001 — no corpus, no pruning signal + texts = [] + corpus_cache[parent.node_id] = texts + if not texts: + return True + return any(pat.search(x) for x in texts) + + return preflight + + +def make_git_recombine_fn(repo_root: str | Path): + """GSME cross-cell recombination: re-materialise a cell elite's edit onto + the current parent as an ordinary candidate. + + The wired candidates carry FULL new file bytes vs their parent commit, so + stacking = reading the elite's changed paths back from its commit and + handing them to the standard edit-then-commit apply off the new parent — + the path guard, gc ref, node ledger, and gate pipeline all apply unchanged. + Same-file overlaps never reach here (:meth:`GsmeArchive.eligible_elites` + filters them); a missing commit/path returns None and the pairing is + recorded as failed instead of retried every round. + """ + from raven.evolver.orchestrator.archive import CellElite, RecombinantCandidate + from raven.evolver.tree import git_ops + from raven.evolver.tree.git_ops import GitOpError + + root = Path(repo_root) + + def recombine_fn(parent: HarnessNode, elite: "CellElite"): + files: dict[str, bytes] = {} + try: + for rel in elite.files: + files[rel] = git_ops.read_file_at(root, elite.git_commit_sha, rel) + except GitOpError: + return None + if not files and not elite.deletions: + return None + return RecombinantCandidate( + files=files, + why=elite.why, + cell=elite.cell, + elite_node_id=elite.node_id, + focused_task_ids=list(elite.focused_task_ids), + deletions=list(elite.deletions), + summary=( + f"recombination: stack elite {elite.node_id} ({elite.cell}, " + f"score {elite.score:.3f}) onto {parent.node_id}" + ), + has_beacon=any(rel.endswith(".py") and b"activation_beacon(" in data for rel, data in files.items()), + ) + + return recombine_fn + + +def make_worktree_eval_fn( + repo_root: str | Path, + score_worktree: Callable[..., dict[str, TaskEval]], +): + """Eval a node by checking its commit out into an ephemeral worktree and + scoring against that checkout — the counterpart to the edit-then-commit + apply. ``score_worktree(worktree_path, node, task_ids, k, split)`` runs the + bench in the clean checkout, so the live repo is never mutated (no RealPathSync).""" + from raven.evolver.tree import git_ops + + root = Path(repo_root) + + def eval_fn(node, task_ids, k, job_name, *, split="train"): + with git_ops.worktree_at(root, node.git_commit_sha) as wt: + return score_worktree(wt, node, task_ids, k, split) + + return eval_fn + + +def make_sealed_runner( + backend: EvalBackend, + sealed_dir: str | Path, + *, + k: int = 3, + test_task_ids: Optional[list[str]] = None, +): + """Build the C3 sealed test runner from a backend (approach B, post-hoc). + + Reuses ``backend.eval`` itself (invoked with ``split="test"``), so the sealed + scorer is the same worktree-checkout / activation scorer the loop uses — no + bench-specific sealed code. Call :func:`unseal_retention` with the journal + records after the loop finishes; the loop never scores test. + """ + from raven.evolver.orchestrator.sealed.runner import SealedTestRunner + + ids = list(test_task_ids if test_task_ids is not None else backend.test_task_ids) + return SealedTestRunner(eval_fn=backend.eval, test_task_ids=ids, sealed_dir=Path(sealed_dir), k=k) + + +def make_metadata_apply_fn( + *, git_branch: str = "evolver/orchestrator", git_commit_sha: str = "replay" +) -> Callable[[str, AppliedPatch, int], HarnessNode]: + """Apply that records a child node without touching git (replay / dry runs). + + A real run passes ``EvolverTreeStore.create_child_node`` instead, which does + the git apply + immutable-kernel guard + JSON persistence. + """ + counter = {"n": 0} + + def apply_fn(parent_id: str, patch: AppliedPatch, round_index: int) -> HarnessNode: + counter["n"] += 1 + node_id = f"v{round_index}-c{counter['n']}" + return HarnessNode( + node_id=node_id, + parent_id=parent_id, + git_commit_sha=git_commit_sha, + git_branch=git_branch, + created_at=HarnessNode.utc_now(), + created_at_iter=round_index, + patch=patch, + ) + + return apply_fn + + +def build_evolution_orchestrator( + config: OrchestratorConfig, + *, + repo_root: str | Path, + base_sha: str, + root_node_id: str, + backend: EvalBackend, + gate_policy, + diagnose_of: Callable[[HarnessNode], tuple[Any, Optional[dict]]], + design_of: Callable[[Callable[[HarnessNode], str], dict], Any], + baseline_of: Callable[[], Any], + files_of: Callable[[Any], dict[str, bytes]], + deletions_of: Optional[Callable[[Any], list[str]]] = None, + driver_call_fn: Optional[CallFn] = None, + verdict_fn=None, + verdict_call_fn: Optional[CallFn] = None, + verdict_why_keys_of: Optional[Callable[[], Optional[list[str]]]] = None, + harm_excerpt_of: Optional[Callable[[str, str], Optional[str]]] = None, + git_branch: str = "evolver/orchestrator", + run_gate0: bool = True, + preflight_fn=None, + fired_source_of=None, +) -> EvolutionOrchestrator: + """Assemble a full evolution run, owning the wiring every bench shares. + + A benchmark only provides the ~7 things that genuinely differ; everything + else — the vanilla root node, the ``node_id -> commit`` registry and the + ``sha_of`` chain resolver, the edit-then-commit apply, the candidate/history + bookkeeping, the focused-subset and outcome hooks, the Gate0-then-cold-start + ordering, and the final :class:`EvolutionOrchestrator` assembly — lives here + once. The bench-specific inputs are deliberately **deferred callables** so + this function controls their firing order (Gate0 + cold start must run before + taxonomy induction reads trajectories and before the baseline reads the + ledger): + + - ``diagnose_of(vanilla_node) -> (diagnose_fn, seed_failure_map)`` — usually + ``resolve_taxonomy`` + a bench diagnose builder. + - ``design_of(sha_of, history, archive_summary_of) -> design_fn`` — receives + the internally-owned ``sha_of`` chain resolver, the cross-round ``history`` + dict, and a zero-arg callable rendering the GSME elite bank (for the + design prompt, so the driver knows what is already verified). + - ``baseline_of() -> BaselineProvider`` — usually + :func:`~...gates.policy.make_frozen_baseline`; called after cold start so + it reads a materialised ledger. + - ``backend`` (carries ``precheck``), ``gate_policy``, ``files_of`` / + ``deletions_of`` (the candidate -> file-bytes extractors). + """ + import json + + vanilla_node = HarnessNode( + node_id=root_node_id, + parent_id=None, + git_commit_sha=base_sha, + git_branch=git_branch, + created_at=HarnessNode.utc_now(), + created_at_iter=0, + ) + # shared state across steps + sha_by_node: dict[str, str] = {root_node_id: base_sha} + cand_by_node: dict[str, Any] = {} + # Cross-round per-WHY attempt history, persisted so a crash-resume does not + # amnesia the designer (prior-attempt lessons + the editor's WHY decay both + # read it); the archive already survives resume, this keeps them symmetric. + history_path = Path(config.work_dir) / "history.json" + history: dict[str, list[dict]] = {} + try: + if history_path.exists(): + history = json.loads(history_path.read_text()) + except (OSError, ValueError): + history = {} + + # Gate0 BEFORE the (potentially hours-long) vanilla cold start — a dirty env + # would bake contaminated trials into the run's permanent baseline — then + # materialise the vanilla ledger so taxonomy induction and the baseline seed + # have trajectories/scores to read. + if run_gate0 and backend.precheck is not None: + backend.precheck() + backend.cold_start() + + diagnose_fn, seed_failure_map = diagnose_of(vanilla_node) + + def sha_of(parent: HarnessNode) -> str: + sha = sha_by_node.get(parent.node_id) + if sha is None and parent.git_commit_sha not in ("", "unknown"): + sha = parent.git_commit_sha + sha_by_node[parent.node_id] = sha + if sha is None: + raise KeyError(f"unknown parent commit for {parent.node_id!r} (chain broken)") + return sha + + # GSME: the per-cell elite bank (persisted under work_dir, reloaded on + # resume). Created before the design step so the designer can read it. + from raven.evolver.orchestrator.archive import GsmeArchive + + archive = GsmeArchive(config.archive_path) + + design_fn = design_of(sha_of, history, archive.summary_text) + + raw_apply = make_git_commit_apply_fn( + repo_root, + files_of, + root_node_id=root_node_id, + base_sha=base_sha, + sha_by_node=sha_by_node, + deletions_of=deletions_of, + git_branch=git_branch, + ) + + def apply_fn(parent_id, cand, round_index): + node = raw_apply(parent_id, cand, round_index) + cand_by_node[node.node_id] = cand + return node + + def focused_source(node: HarnessNode) -> list[str]: + cand = cand_by_node.get(node.node_id) + return list(getattr(cand, "focused_task_ids", [])) if cand else [] + + # Gate-b attribution, beacon-aware: only a candidate that actually carries + # an activation_beacon call gets per-task firing data; for the rest (prompt/ + # config edits, recombinants of such) return None so the gate fails OPEN + # instead of rejecting an uninstrumented-but-honest candidate on an empty + # firing set. ``fired_source_of`` is the bench's raw ledger reader. + fired_source = None + if fired_source_of is not None: + + def fired_source(node: HarnessNode, task_ids: list[str]): + cand = cand_by_node.get(node.node_id) + if not getattr(cand, "has_beacon", False): + return None + return fired_source_of(node, task_ids) + + def _persist_history(): + try: + history_path.parent.mkdir(parents=True, exist_ok=True) + history_path.write_text(json.dumps(history, indent=2)) + except OSError: + pass + + def inert_hook(cand, outcome): + # A preflight-pruned candidate never got a node, but its death is a + # designer lesson (the TRIGGER was unreachable, not the mechanism) — + # without this entry the next round can redesign the same dead end. + why = str(getattr(cand, "why", "") or "") + if not why: + return + files = getattr(cand, "files", None) + history.setdefault(why, []).append( + { + "node_id": outcome.node_id, + "files": sorted(files) if isinstance(files, dict) else [], + "summary": str(getattr(cand, "summary", "") or ""), + "outcome": outcome.status.value, + "promoted": False, + "reason": (outcome.stats or {}).get("reason", ""), + } + ) + _persist_history() + + def outcome_hook(ctx, outcome): + cand = cand_by_node.get(ctx.node.node_id) + if cand is None: + return + entry = { + "node_id": ctx.node.node_id, + "files": sorted(cand.files), + "summary": cand.summary, + "outcome": outcome.status.value, + "promoted": outcome.promoted, + } + flips = outcome.stats.get("flips") if outcome.stats else None + if flips: + entry["rescued"] = flips["n_rescued"] + entry["regressed"] = flips["n_regressed"] + entry["regressed_ids"] = list(flips["regressed"])[:3] + # Harm replay: attach HOW the first regressed task broke under this + # candidate (its own confirm trajectory), so the next attempt's + # "narrow it sharply" has the wound, not just the count. + if harm_excerpt_of and flips["regressed"]: + try: + harm = harm_excerpt_of(ctx.node.node_id, list(flips["regressed"])[0]) + except Exception: # noqa: BLE001 — replay is best-effort context + harm = None + if harm: + entry["harm"] = harm + history.setdefault(cand.why, []).append(entry) + _persist_history() + + # Verdict rides its own driver when given (role->model splits, e.g. a cheap + # diagnose model + a stronger narrative model), else the shared one. + if verdict_fn is None and (verdict_call_fn or driver_call_fn) is not None: + verdict_fn = make_verdict_fn(verdict_call_fn or driver_call_fn, why_keys_of=verdict_why_keys_of) + + # budget.recombinations_per_round=0 disables recombination proposals while + # still banking elites for audit and the design prompt. + return EvolutionOrchestrator( + config, + backend=backend, + diagnose_fn=diagnose_fn, + design_fn=design_fn, + apply_fn=apply_fn, + gate_policy=gate_policy, + baseline_provider=baseline_of(), + verdict_fn=verdict_fn, + preflight_fn=preflight_fn, + fired_source=fired_source, + focused_source=focused_source, + outcome_hook=outcome_hook, + inert_hook=inert_hook, + seed_failure_map=seed_failure_map, + archive=archive, + recombine_fn=make_git_recombine_fn(repo_root), + ) + + +def build_orchestrator( + config: OrchestratorConfig, + *, + backend: EvalBackend, + diagnose_fn, + design_fn, + apply_fn, + gate_policy=None, + baseline_provider=None, + verdict_fn=None, + preflight_fn=None, + fired_source=None, + focused_source=None, + outcome_hook=None, + archive=None, + recombine_fn=None, +) -> EvolutionOrchestrator: + """Low-level assembler: wrap pre-built steps into an orchestrator with no + shared-wiring conveniences (used by tests and the multibench smoke). Most + real runs use :func:`build_evolution_orchestrator` instead.""" + return EvolutionOrchestrator( + config, + backend=backend, + diagnose_fn=diagnose_fn, + design_fn=design_fn, + apply_fn=apply_fn, + gate_policy=gate_policy, + baseline_provider=baseline_provider, + verdict_fn=verdict_fn, + preflight_fn=preflight_fn, + fired_source=fired_source, + focused_source=focused_source, + outcome_hook=outcome_hook, + archive=archive, + recombine_fn=recombine_fn, + ) + + +__all__ = [ + "EndpointConfig", + "ScoringTrajectoryRun", + "make_diagnose_fn", + "make_design_fn", + "make_verdict_fn", + "summarize_round", + "make_llm_backend", + "make_git_commit_apply_fn", + "make_git_recombine_fn", + "make_zero_hit_preflight", + "make_worktree_eval_fn", + "make_sealed_runner", + "make_metadata_apply_fn", + "build_evolution_orchestrator", + "build_orchestrator", +] diff --git a/raven/evolver/orchestrator/providers/__init__.py b/raven/evolver/orchestrator/providers/__init__.py new file mode 100644 index 0000000..fdb7294 --- /dev/null +++ b/raven/evolver/orchestrator/providers/__init__.py @@ -0,0 +1,5 @@ +"""Driver-model transports for semantic nodes: OpenAI-compatible endpoints +(:mod:`.openai_compat`) and the local Claude Code subscription +(:mod:`.claude_cli`).""" + +from __future__ import annotations diff --git a/raven/evolver/orchestrator/providers/claude_agentic.py b/raven/evolver/orchestrator/providers/claude_agentic.py new file mode 100644 index 0000000..6a60fa4 --- /dev/null +++ b/raven/evolver/orchestrator/providers/claude_agentic.py @@ -0,0 +1,125 @@ +"""Agentic analysis session: full-tool Claude Code, locked to read-only. + +The map-reduce diagnosis reads every failing trajectory once, shallowly. The +``agentic`` analysis mode replaces that front half with one Claude Code session +that investigates the run the way an engineer would — ledger first, then +deep-reads of representative transcripts, then the harness source — and returns +one structured diagnosis (meta-harness style, but constrained to our taxonomy +so the failure_map/history/GSME machinery downstream is unchanged). + +Safety model (the session must not be able to leave the safe zone): + +- The tool whitelist is **hard-coded** to ``Read, Glob, Grep`` — no Bash, no + Write/Edit, no Agent, and callers cannot widen it. In ``-p`` mode any tool + outside ``--allowedTools`` is auto-denied, and ``--dangerously-skip-permissions`` + is never passed, so the session can inspect but cannot modify or execute. +- ``cwd`` is the assembled analysis workspace (digest + run data + a pinned + harness worktree), not the live repo — no CLAUDE.md/project injection and + nothing load-bearing to touch even if a write slipped through. +- Only Claude models may run this mode (enforced here): the analyst rides the + local CLI's logged-in subscription, like :mod:`.claude_cli`. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path +from typing import Callable, Optional + +AGENTIC_TOOLS = ("Read", "Glob", "Grep") + + +def claude_cli_available(claude_bin: str = "claude") -> bool: + """True when the claude CLI is on PATH and answers ``--version``.""" + if shutil.which(claude_bin) is None: + return False + try: + r = subprocess.run([claude_bin, "--version"], capture_output=True, text=True, timeout=20) + except Exception: # noqa: BLE001 — any probe failure means "not usable" + return False + return r.returncode == 0 + + +def require_claude_for_agentic(model: str, claude_bin: str = "claude") -> None: + """Gate for analysis_mode="agentic": claude models + a working CLI only.""" + if not str(model).startswith("claude"): + raise ValueError( + f"analysis_mode='agentic' only runs on Claude models via the local " + f"CLI (got model={model!r}); use analysis_mode='mapreduce' for other drivers" + ) + if not claude_cli_available(claude_bin): + raise RuntimeError( + f"analysis_mode='agentic' requires the '{claude_bin}' CLI on PATH " + "(logged-in Claude Code); it was not found or does not answer --version" + ) + + +def run_agentic_session( + prompt: str, + *, + system_prompt: str, + cwd: str | Path, + model: str = "claude-opus-4-8", + claude_bin: str = "claude", + timeout: float = 1800.0, + add_dirs: tuple = (), + run: Optional[Callable[..., "subprocess.CompletedProcess"]] = None, +) -> str: + """One read-only agentic Claude Code session; returns its final text. + + ``--append-system-prompt`` (not ``--system-prompt``) keeps Claude Code's + native agentic system prompt so Read/Glob/Grep are actually driven well; + our instructions ride on top. + + ``add_dirs`` grants READ access to trees outside ``cwd``: print-mode file + access is confined to the cwd subtree by *resolved* path, so workspace + symlinks pointing at run data are denied without it (observed live: the + analyst reported the data 'absent' and degraded to signature analogy). + The tool whitelist stays read-only, so the grant widens visibility, not + write reach. + """ + require_claude_for_agentic(model, claude_bin) + _run = run or subprocess.run + argv = [ + claude_bin, + "-p", + "--model", + model, + "--output-format", + "json", + "--allowedTools", + ",".join(AGENTIC_TOOLS), + "--disallowedTools", + "Bash,Write,Edit,NotebookEdit,Agent,WebFetch,WebSearch", + "--append-system-prompt", + system_prompt, + ] + for d in add_dirs: + argv += ["--add-dir", str(d)] + proc = _run( + argv, + input=prompt, + capture_output=True, + text=True, + timeout=timeout, + cwd=str(cwd), + ) + if proc.returncode != 0: + raise RuntimeError(f"agentic session exit {proc.returncode}: {(proc.stderr or proc.stdout)[:400]}") + data = json.loads(proc.stdout) + if data.get("is_error"): + raise RuntimeError(f"agentic session is_error: {str(data.get('result'))[:400]}") + result = data.get("result") + if not isinstance(result, str) or not result.strip(): + raise RuntimeError(f"agentic session empty result: {proc.stdout[:400]}") + return result + + +__all__ = [ + "AGENTIC_TOOLS", + "claude_cli_available", + "require_claude_for_agentic", + "run_agentic_session", +] diff --git a/raven/evolver/orchestrator/providers/claude_cli.py b/raven/evolver/orchestrator/providers/claude_cli.py new file mode 100644 index 0000000..74a9159 --- /dev/null +++ b/raven/evolver/orchestrator/providers/claude_cli.py @@ -0,0 +1,145 @@ +"""Claude-via-local-CLI ``call_fn``: drivers on the coding-plan subscription. + +Every evolver driver role — diagnose / taxonomy induction / design (bash-editor) +/ verdict — consumes the same seam, the sync ``CallFn`` from +:mod:`~raven.evolver.orchestrator.nodes.semantic`. :func:`make_claude_call_fn` +returns that CallFn backed by the local ``claude`` CLI in print mode, so driver +calls ride the logged-in Claude Code subscription instead of a per-token API +endpoint. The FSM keeps control: ``--allowedTools ""`` disables Claude's own +tools, making each call a pure chat completion (Claude-as-completion inside the +existing loops, isomorphic to the qwen path in :mod:`.openai_compat`). + +Operational notes baked in: + +- The coding plan rate-limits under fan-out (diagnose runs a thread pool), so + each CallFn carries its own concurrency semaphore (default 4) plus + backoff-and-retry that stretches the delay on rate-limit signatures. +- The prompt goes via stdin, not argv — rendered trajectories overflow argv. +- Multi-turn conversations (the bash-editor accumulates action/observation + turns) are serialized into one role-tagged transcript; print mode accepts a + single prompt and cannot replay assistant turns natively. +""" + +from __future__ import annotations + +import json +import subprocess +import tempfile +import threading +import time +from typing import Callable, Optional, Sequence + +from raven.evolver.orchestrator.nodes.semantic import CallFn, Messages + +_CONTINUE_RULE = ( + "You are the assistant in the conversation transcript below. Reply with the " + "assistant's NEXT message only — no role tag, no commentary about the " + "transcript format." +) + + +def render_messages(messages: Messages) -> tuple[str, str]: + """Split a chat into ``(system_prompt, user_prompt)`` for ``claude -p``. + + A single user message passes through verbatim. A multi-turn history is + serialized with role tags plus a continue-instruction, since print mode + accepts one prompt, not a message array. + """ + system = "\n\n".join(str(m.get("content", "")) for m in messages if m.get("role") == "system") + turns = [m for m in messages if m.get("role") != "system"] + if len(turns) == 1: + return system, str(turns[0].get("content", "")) + lines = [_CONTINUE_RULE, ""] + for m in turns: + lines.append(f"[{str(m.get('role', 'user')).upper()}]") + lines.append(str(m.get("content", ""))) + return system, "\n".join(lines) + + +def _looks_rate_limited(text: str) -> bool: + t = text.lower() + return "rate limit" in t or "429" in t or "overloaded" in t or "usage limit" in t + + +def make_claude_call_fn( + model: str, + *, + claude_bin: str = "claude", + timeout: float = 300.0, + retry_delays: Sequence[float] = (5.0, 15.0, 45.0, 90.0), + rate_limit_delay: float = 120.0, + max_concurrency: int = 4, + extra_args: Sequence[str] = (), + cwd: Optional[str] = None, + run: Optional[Callable[..., "subprocess.CompletedProcess"]] = None, +) -> CallFn: + """Build a sync ``call_fn`` running ``claude -p --model `` per call. + + ``run`` overrides ``subprocess.run`` for tests. The semaphore is + per-CallFn: two roles (e.g. haiku diagnose + opus design) each get their + own lane, but one fan-out caller (``classify_failures``'s thread pool) + cannot exceed ``max_concurrency`` concurrent CLI processes on the shared + plan quota. + + ``cwd`` defaults to a fresh empty temp dir, NEVER the inherited process + cwd: Claude Code injects the cwd's project context (CLAUDE.md, env info) + into the model even under ``--system-prompt``. Inheriting the orchestrator + repo's cwd leaked the real repo path to the design driver, which then + ``cd``-ed out of its sandbox worktree to browse (verified live 2026-07-09). + """ + _run = run or subprocess.run + workdir = cwd or tempfile.mkdtemp(prefix="claude-driver-") + gate = threading.Semaphore(max_concurrency) + + def _once(system: str, prompt: str) -> str: + argv = [ + claude_bin, + "-p", + "--model", + model, + "--output-format", + "json", + "--allowedTools", + "", + *(["--system-prompt", system] if system else []), + *extra_args, + ] + with gate: + proc = _run( + argv, + input=prompt, + capture_output=True, + text=True, + timeout=timeout, + cwd=workdir, + ) + if proc.returncode != 0: + raise RuntimeError(f"claude -p exit {proc.returncode}: {(proc.stderr or proc.stdout)[:400]}") + data = json.loads(proc.stdout) + if data.get("is_error"): + raise RuntimeError(f"claude -p is_error: {str(data.get('result'))[:400]}") + result = data.get("result") + if not isinstance(result, str) or not result.strip(): + raise RuntimeError(f"claude -p empty result: {proc.stdout[:400]}") + return result + + def call_fn(messages: Messages) -> str: + system, prompt = render_messages(messages) + last: Exception | None = None + for attempt in range(len(retry_delays) + 1): + try: + return _once(system, prompt) + except Exception as exc: # noqa: BLE001 — retry, then surface loudly + last = exc + if attempt >= len(retry_delays): + break + delay = retry_delays[attempt] + if _looks_rate_limited(str(exc)): + delay = max(delay, rate_limit_delay) + time.sleep(delay) + raise RuntimeError(f"claude driver ({model}) failed after {len(retry_delays) + 1} attempts") from last + + return call_fn + + +__all__ = ["make_claude_call_fn", "render_messages"] diff --git a/raven/evolver/orchestrator/providers/openai_compat.py b/raven/evolver/orchestrator/providers/openai_compat.py new file mode 100644 index 0000000..435b26b --- /dev/null +++ b/raven/evolver/orchestrator/providers/openai_compat.py @@ -0,0 +1,103 @@ +"""Synchronous OpenAI-compatible ``call_fn`` for the semantic-node layer. + +The driver models are served behind OpenAI-compatible ``/v1`` endpoints +(self-hosted Qwen / Kimi via vLLM). :func:`make_call_fn` returns the sync +``CallFn`` a :class:`~raven.evolver.orchestrator.nodes.semantic.SemanticNode` +expects — messages in, assistant text out. + +Two behaviours matter for these specific models: + +- **Reasoning models.** Qwen3.5/3.6 emit a large ``reasoning`` field before the + real answer lands in ``message.content``. A small ``max_tokens`` gets consumed + entirely by reasoning and returns ``content == null``. So the default token + budget here is deliberately generous, and an empty/None content is retried + rather than parsed. +- **No key required.** vLLM accepts any bearer token; ``api_key`` defaults to + ``"EMPTY"`` and can be overridden from the environment. + +This is a plain sync transport (httpx) so it composes with the synchronous +orchestrator FSM without threading an event loop through it. +""" + +from __future__ import annotations + +import os +import time +from typing import Optional + +from raven.evolver.orchestrator.nodes.semantic import CallFn, Messages + + +class EndpointError(RuntimeError): + """Raised when the endpoint fails to return usable content after retries.""" + + +def _extract_content(data: dict) -> Optional[str]: + try: + return data["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError): + return None + + +def make_call_fn( + *, + base_url: str, + model: str, + api_key: Optional[str] = None, + api_key_env: str = "EVOLVER_DRIVER_API_KEY", + max_tokens: int = 8192, + temperature: float = 0.0, + timeout: float = 180.0, + retry_delays: tuple[float, ...] = (1.0, 2.0, 4.0), +) -> CallFn: + """Build a sync ``call_fn`` bound to one OpenAI-compatible chat endpoint. + + ``base_url`` is the ``/v1`` root; ``/chat/completions`` is appended. Empty or + missing content is retried with backoff (reasoning models occasionally emit + no answer); after the final attempt an :class:`EndpointError` is raised so a + node failure is loud rather than a silent empty string. + """ + key = api_key or os.environ.get(api_key_env) or "EMPTY" + url = base_url.rstrip("/") + "/chat/completions" + + def call_fn(messages: Messages) -> str: + import httpx # lazy: unit tests inject their own call_fn + + payload = { + "model": model, + "messages": list(messages), + "max_tokens": max_tokens, + "temperature": temperature, + } + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + last_exc: Exception | None = None + for delay in retry_delays: + try: + with httpx.Client(timeout=timeout) as client: + resp = client.post(url, json=payload, headers=headers) + resp.raise_for_status() + content = _extract_content(resp.json()) + if content and content.strip(): + return content + except (httpx.HTTPError, ValueError) as exc: + last_exc = exc + time.sleep(delay) + + with httpx.Client(timeout=timeout) as client: + resp = client.post(url, json=payload, headers=headers) + resp.raise_for_status() + content = _extract_content(resp.json()) + if content and content.strip(): + return content + raise EndpointError( + f"endpoint {model!r} returned empty content after " + f"{len(retry_delays) + 1} attempts" + (f"; last exc: {last_exc!r}" if last_exc else "") + ) + + return call_fn + + +__all__ = ["make_call_fn", "EndpointError"] diff --git a/raven/evolver/orchestrator/scoring.py b/raven/evolver/orchestrator/scoring.py new file mode 100644 index 0000000..a95410f --- /dev/null +++ b/raven/evolver/orchestrator/scoring.py @@ -0,0 +1,226 @@ +"""Bench-neutral scoring contract — the one currency the funnel scores in. + +Every consumer in the seven-step funnel (screen, paired gate, sealed ledger) +reads only :attr:`TaskEval.pass_rate`, so a *scorer's* sole job is to turn one +evaluation of a candidate into ``{task_id: TaskEval}``. That makes the whole +benchmark surface collapse to a single value — :class:`EvalBackend` — bundling +the four bench-specific things the orchestrator needs: how to score a task set +(``eval``), the vanilla cold-start baseline over train (``cold_start``), the +screen anchor drawn from train (``anchor``), and the trajectories that feed +diagnosis (``trajectories``). SWE-bench, AppWorld, and the no-benchmark LLM +judge are each one :class:`EvalBackend` instance; nothing above this module +imports a concrete bench. + +Train vs. sealed test is first-class here (SOP's iron law, as a mechanism not a +rule): ``EvalBackend`` carries both ``train_task_ids`` (the evolvable pool — +diagnosis, screen, confirm, gate, parent selection all live here) and +``test_task_ids`` (held-out; blind-scored only, never consulted for any +decision). ``eval`` takes an explicit ``split`` so one scorer serves both: the +funnel passes ``split="train"``, the sealed runner ``split="test"``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable, Optional, Protocol + +from raven.evolver.analysis.stability_bucket import TaskStability +from raven.evolver.scheduler.anchor_selection import AnchorSelection + +if TYPE_CHECKING: + from raven.evolver.tree.node import HarnessNode + + +@dataclass(frozen=True) +class TaskEval: + """Per-task K-trial outcome — the universal currency the funnel scores in. + + ``infra_attempts`` is a subset of ``attempts`` (infra trials still count as + non-passes here); Gate-f uses it to exclude infra-contaminated tasks. Benches + that do not track infra leave it 0, so Gate-f is a no-op for them. + """ + + task_id: str + passes: int + attempts: int + infra_attempts: int = 0 + + @property + def pass_rate(self) -> float: + """Fraction of trials that passed; 0.0 when no trial was observed.""" + if self.attempts == 0: + return 0.0 + return self.passes / self.attempts + + +class EvalFn(Protocol): + """Score ``node`` on ``task_ids`` at K trials; return per-task outcomes. + + ``split`` selects the dataset partition — ``"train"`` for the funnel's + screen/confirm, ``"test"`` for the sealed held-out runner. A task that never + launched is simply absent from the returned mapping (the caller decides + whether that is an infra failure per Gate-f). + """ + + def __call__( + self, + node: "HarnessNode", + task_ids: list[str], + k: int, + job_name: str, + *, + split: str = "train", + ) -> dict[str, TaskEval]: ... + + +# round-scoped trajectory feed for diagnosis: +# (round_index, parent) -> [(trajectory_id, task_description, trajectory_text), ...] +TrajectorySource = Callable[[int, "HarnessNode"], list[tuple[str, str, str]]] + +# Pre-scoring environment health precheck (SOP §0 Gate0, before any scoring). Raises when +# the environment is dirty (sandbox down / network unroutable / verifier can't +# emit results) so a run does not score against a broken box. Bench-specific — +# each bench validates what it needs; left None when a bench has none wired. +PrecheckFn = Callable[[], None] + + +def eval_with_infra_rerun( + eval_fn: "EvalFn", + node: "HarnessNode", + task_ids: list[str], + k: int, + job_name: str, + *, + split: str = "train", + max_reruns: int = 2, +) -> dict[str, TaskEval]: + """Wrap an ``eval_fn`` with the SOP §0 infra rerun ladder (detect -> rerun). + + After the first eval, any task that still carries an infra trial — or that + was requested but came back with NO measurement at all (never launched / + result never written: infra by definition) — is re-scored (fresh, at K) up + to ``max_reruns`` times — re-run once the environment is healthy, topping the task back up to K attempts — and the measurement + with the fewest infra trials is kept. This salvages recoverable (transient) + infra so the task is measured for real; genuinely persistent infra survives + all reruns and is left to score 0 in the denominator (never dropped — that + is the caller's fixed-denominator discipline, SOP §0). A bench with no + infra signal that returns every requested task triggers no rerun, so this + is a transparent no-op there. + """ + evals = dict(eval_fn(node, task_ids, k, job_name, split=split)) + for i in range(1, max_reruns + 1): + infra_tasks = [t for t in task_ids if (ev := evals.get(t)) is None or ev.infra_attempts > 0] + if not infra_tasks: + break + redo = eval_fn(node, infra_tasks, k, f"{job_name}_infra_rerun{i}", split=split) + for t in infra_tasks: + new = redo.get(t) + old = evals.get(t) + if new is not None and (old is None or new.infra_attempts < old.infra_attempts): + evals[t] = new + return evals + + +def with_infra_rerun(inner: "EvalFn", max_reruns: int) -> "EvalFn": + """Wrap an ``EvalFn`` with the SOP §0 infra rerun ladder. + + A no-op when ``max_reruns <= 0`` or the bench emits no infra signal (every + requested task comes back with ``infra_attempts == 0``), so a bench with no + infra concept is unaffected. Benches compose this around their raw scorer + when building an :class:`EvalBackend`. + """ + if max_reruns <= 0: + return inner + + def wrapped(node, task_ids, k, job_name, *, split="train"): + return eval_with_infra_rerun(inner, node, task_ids, k, job_name, split=split, max_reruns=max_reruns) + + return wrapped + + +def flip_summary( + candidate_evals: dict[str, TaskEval], + control_evals: dict[str, TaskEval], + task_ids: list[str], + *, + max_ids: int = 12, +) -> dict: + """Per-task flip table between a candidate and its control (SOP §2 ①). + + ``rescued`` = tasks whose pass rate ROSE under the candidate (including + partial, e.g. 1/3 -> 2/3), ``regressed`` = tasks whose rate fell, + ``still_failing`` = tasks below a perfect rate under the candidate. The + id lists are capped at ``max_ids`` (counts are always exact) — this feeds + prompts and ledgers, not statistics; the paired gate owns the stats. A + task absent from an arm scores 0.0 for that arm, same as the gate. + """ + rescued: list[str] = [] + regressed: list[str] = [] + still_failing: list[str] = [] + for t in task_ids: + c_ev, v_ev = candidate_evals.get(t), control_evals.get(t) + c = c_ev.pass_rate if c_ev is not None else 0.0 + v = v_ev.pass_rate if v_ev is not None else 0.0 + if c > v: + rescued.append(t) + elif c < v: + regressed.append(t) + if c < 1.0: + still_failing.append(t) + return { + "n_rescued": len(rescued), + "n_regressed": len(regressed), + "n_still_failing": len(still_failing), + "rescued": rescued[:max_ids], + "regressed": regressed[:max_ids], + "still_failing": still_failing[:max_ids], + } + + +def anchor_mean_pass_rate(evals: dict[str, TaskEval], anchor_task_ids: list[str]) -> float: + """Mean per-task pass rate over the anchor subset. + + The quantity the screen compares against vanilla's anchor mean. Anchor tasks + with no observed trial contribute 0.0 (a candidate that failed to even launch + on an anchor task is not rewarded for the gap). + """ + if not anchor_task_ids: + raise ValueError("anchor_mean_pass_rate requires a non-empty anchor list") + total = 0.0 + for task_id in anchor_task_ids: + ev = evals.get(task_id) + total += ev.pass_rate if ev is not None else 0.0 + return total / len(anchor_task_ids) + + +@dataclass(frozen=True) +class EvalBackend: + """Everything bench-specific the funnel needs, in one value. + + One instance per benchmark (SWE-bench, AppWorld) or per no-benchmark LLM + judge. The FSM stays bench-agnostic: it screens/confirms via ``eval`` on + ``train_task_ids``, seeds the baseline from ``cold_start``, draws the screen + anchor from ``anchor``, and feeds ``trajectories`` to diagnosis. + ``test_task_ids`` is scored blind only, by the sealed runner. + """ + + train_task_ids: list[str] + test_task_ids: list[str] + eval: EvalFn + cold_start: Callable[[], dict[str, TaskStability]] + anchor: Callable[..., AnchorSelection] + trajectories: Optional[TrajectorySource] = None + precheck: Optional[PrecheckFn] = None + + +__all__ = [ + "TaskEval", + "EvalFn", + "TrajectorySource", + "PrecheckFn", + "EvalBackend", + "anchor_mean_pass_rate", + "eval_with_infra_rerun", + "flip_summary", + "with_infra_rerun", +] diff --git a/raven/evolver/orchestrator/sealed/__init__.py b/raven/evolver/orchestrator/sealed/__init__.py new file mode 100644 index 0000000..986ebd7 --- /dev/null +++ b/raven/evolver/orchestrator/sealed/__init__.py @@ -0,0 +1,3 @@ +"""Sealed test-set handling — enforce the train/test firewall mechanically.""" + +from __future__ import annotations diff --git a/raven/evolver/orchestrator/sealed/runner.py b/raven/evolver/orchestrator/sealed/runner.py new file mode 100644 index 0000000..ee9070f --- /dev/null +++ b/raven/evolver/orchestrator/sealed/runner.py @@ -0,0 +1,283 @@ +"""Sealed test scoring — the train/test firewall as a mechanism, not a rule. + +The SOP's iron law is that the sealed test set never influences an evolution +decision: the loop compares candidates against vanilla on *train* only, and test +is scored blind, its numbers unread until the run ends (retention = test_lift / +train_lift, computed at unseal). Historically this was enforced by discipline. +Here it's enforced by construction: + +- :func:`assert_no_test_leak` fails loudly if any test id has crept into the + anchor or train task sets (an anchor/train that contains test = leakage). +- :class:`SealedTestRunner.score` writes per-task test results to a sealed dir + the driver never reads and **returns nothing** — there is no path for a test + number to reach a gate or a verdict. Only :meth:`unseal`, called after the + loop finishes, reads them back. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +from raven.evolver.orchestrator.gates.paired import paired_lift +from raven.evolver.orchestrator.scoring import EvalFn, TaskEval +from raven.evolver.tree.node import HarnessNode + +# The sealed scorer shares the loop's EvalFn signature exactly — it *is* the same +# bench scorer (worktree checkout / activation), just invoked with split="test". +SealedEvalFn = EvalFn + + +class TestLeakError(RuntimeError): + """Raised when sealed test ids appear in the anchor or train sets.""" + + __test__ = False # not a pytest test class despite the Test* name + + +def assert_no_test_leak( + *, + anchor_task_ids: list[str], + train_task_ids: list[str], + sealed_test_ids: list[str], +) -> None: + """Fail if any sealed test id is present in the anchor or train sets.""" + test = set(sealed_test_ids) + anchor_leak = sorted(test & set(anchor_task_ids)) + train_leak = sorted(test & set(train_task_ids)) + if anchor_leak or train_leak: + raise TestLeakError(f"sealed test ids leaked into decision sets: anchor={anchor_leak} train={train_leak}") + + +@dataclass +class SealedTestRunner: + """Scores nodes on the sealed test set into a driver-invisible directory. + + ``eval_fn`` is the loop's own :class:`~raven.evolver.orchestrator.scoring.EvalFn` + (``(node, task_ids, k, job_name, *, split) -> {task_id: TaskEval}``) — the + same bench scorer, invoked with ``split="test"``. ``score`` takes a real + :class:`HarnessNode` (so a worktree eval can check out its commit), writes to + ``sealed_dir``, and returns None so no test number can enter the decision path. + """ + + eval_fn: SealedEvalFn + test_task_ids: list[str] + sealed_dir: Path + k: int = 3 + split: str = "test" + + def __post_init__(self) -> None: + self.sealed_dir = Path(self.sealed_dir) + + def score(self, node: HarnessNode, round_index: int) -> None: + """Blind-score ``node`` on the test set; persist, return nothing.""" + evals = self.eval_fn( + node, + self.test_task_ids, + self.k, + f"{node.node_id}_sealed_test", + split=self.split, + ) + self.sealed_dir.mkdir(parents=True, exist_ok=True) + # pass@1 over a FIXED denominator = the full test set (SOP §0 hard rule): + # a test task that produced no result scores 0, never dropped from the + # denominator (dropping shrinks it and overestimates generalisation). + n_test = len(self.test_task_ids) + record = { + "round": round_index, + "node_id": node.node_id, + "k": self.k, + "per_task": {t: [ev.passes, ev.attempts] for t, ev in evals.items()}, + "pass_at_1": ( + sum(evals[t].pass_rate if t in evals else 0.0 for t in self.test_task_ids) / n_test if n_test else 0.0 + ), + } + (self.sealed_dir / f"round_{round_index}_{node.node_id}.json").write_text(json.dumps(record, indent=2)) + + def unseal(self) -> list[dict]: + """Read all sealed test results (call only after the loop ends).""" + if not self.sealed_dir.exists(): + return [] + out = [] + for path in sorted(self.sealed_dir.glob("round_*.json")): + out.append(json.loads(path.read_text())) + return out + + +def retention(*, vanilla_train: float, best_train: float, vanilla_test: float, best_test: float) -> float | None: + """Retention = test lift / train lift (generalisation), or None if no train lift.""" + train_lift = best_train - vanilla_train + if train_lift <= 0: + return None + return (best_test - vanilla_test) / train_lift + + +@dataclass(frozen=True) +class CurvePoint: + """One (round, harness) point on the train/test generalisation curve.""" + + round_index: int + node_id: str + train_pass_at_1: float + test_pass_at_1: float + + +@dataclass(frozen=True) +class RetentionReport: + """The C3 deliverable: the train/test curve, the train-selected harness's + sealed score + paired significance, and retention. + + ``best_*`` is the TRAIN-argmax deliverable (Alg.1 L140: selection spends its + degrees of freedom on train; its sealed score is a single unbiased + measurement). The per-round ``curve`` is display/audit only — never pick a + reported number by scanning it for the highest test score: a max over many + noisy sealed measurements is inflated by selection (winner's curse). + ``sealed_z`` / ``sealed_credited_2sigma`` are the paper's credit label, + from per-task paired diffs deliverable-vs-vanilla on the sealed set + (None when the deliverable is vanilla itself).""" + + curve: list[CurvePoint] = field(default_factory=list) + vanilla_train: float = 0.0 + vanilla_test: float = 0.0 + best_round: int = 0 + best_node_id: str = "" + best_train: float = 0.0 + best_test: float = 0.0 + retention: float | None = None + sealed_z: float | None = None + sealed_credited_2sigma: bool | None = None + + +def _shim_node(node_id: str, sha: str) -> HarnessNode: + """A minimal node carrying just the identity + commit an eval needs.""" + return HarnessNode( + node_id=node_id, + parent_id=None, + git_commit_sha=sha, + git_branch="sealed", + created_at=HarnessNode.utc_now(), + created_at_iter=0, + ) + + +def unseal_retention( + runner: SealedTestRunner, + journal_records: list[dict], + *, + vanilla_node: HarnessNode, + vanilla_train: float, +) -> RetentionReport: + """Post-hoc C3 unseal (approach B): blind-score the vanilla node and each + round's deliverable (its ``next_parent``, reconstructed from the journal's + recorded commit SHA) on the sealed test set, build the train/test curve, + and report the TRAIN-argmax deliverable's sealed score, paired + significance, and retention. + + All test scoring happens HERE, after evolution finishes — the loop never saw + a test number (SOP §0 sealed-but-logged, fully-sealed-equivalent: every + round's harness is a durable commit, so the per-round curve is reconstructed + without any decision-time test signal). Each distinct node is scored once; + per-round train pass@1 comes from the journal and is carried forward across + rounds that did not promote. + + Deliverable selection is by TRAIN score (Alg.1 L140: ``argmax`` over train, + ties -> earliest round), never by test: picking the max of many noisy sealed + measurements would inflate the reported gain by selection (winner's curse) + and void the paper's no-multiple-comparison argument. Overfitting shows up + as an honestly-low retention, not as a silently swapped deliverable. + """ + runner.score(vanilla_node, 0) + scored: set[str] = {vanilla_node.node_id} + for rec in journal_records: + nid, sha = rec["next_parent_id"], rec.get("next_parent_sha") + # "unknown" = the root shim's placeholder in journals written before the + # loop recorded None; checking it out would crash the whole unseal. + if nid in scored or not sha or sha == "unknown": + continue + runner.score(_shim_node(nid, sha), rec["round_index"]) + scored.add(nid) + + records = runner.unseal() + test_by_node: dict[str, float] = {d["node_id"]: d["pass_at_1"] for d in records} + per_task_by_node: dict[str, dict] = {d["node_id"]: d.get("per_task", {}) for d in records} + van_test = test_by_node.get(vanilla_node.node_id, 0.0) + + curve: list[CurvePoint] = [] + last_train = vanilla_train + for rec in journal_records: + nid = rec["next_parent_id"] + tr = rec.get("next_parent_train") + if tr is not None: + last_train = tr + curve.append( + CurvePoint( + round_index=rec["round_index"], + node_id=nid, + train_pass_at_1=last_train, + test_pass_at_1=test_by_node.get(nid, van_test), + ) + ) + + # Train-argmax over the promoting rounds whose deliverable was measurable + # (has a recorded train score and a scored commit); ties -> earliest round + # (same train, fewer stacked patches). Falls back to vanilla when nothing + # promoted — then there is no lift to report and no paired test to run. + promoting = [ + rec + for rec in journal_records + if rec.get("next_parent_train") is not None and rec["next_parent_id"] in test_by_node + ] + if promoting: + best_rec = max( + promoting, + key=lambda r: (r["next_parent_train"], -r["round_index"]), + ) + best_round, best_node = best_rec["round_index"], best_rec["next_parent_id"] + best_train, best_test = best_rec["next_parent_train"], test_by_node[best_node] + else: + best_round, best_node = 0, vanilla_node.node_id + best_train, best_test = vanilla_train, van_test + + sealed_z: float | None = None + sealed_credited: bool | None = None + if best_node != vanilla_node.node_id and runner.test_task_ids: + + def _evals(nid: str) -> dict[str, TaskEval]: + return {t: TaskEval(t, int(p), int(a)) for t, (p, a) in per_task_by_node.get(nid, {}).items()} + + paired = paired_lift( + candidate_evals=_evals(best_node), + control_evals=_evals(vanilla_node.node_id), + task_ids=list(runner.test_task_ids), + ) + sealed_z, sealed_credited = paired.z, paired.credited_2sigma + + return RetentionReport( + curve=curve, + vanilla_train=vanilla_train, + vanilla_test=van_test, + best_round=best_round, + best_node_id=best_node, + best_train=best_train, + best_test=best_test, + retention=retention( + vanilla_train=vanilla_train, + best_train=best_train, + vanilla_test=van_test, + best_test=best_test, + ), + sealed_z=sealed_z, + sealed_credited_2sigma=sealed_credited, + ) + + +__all__ = [ + "SealedTestRunner", + "assert_no_test_leak", + "TestLeakError", + "retention", + "SealedEvalFn", + "CurvePoint", + "RetentionReport", + "unseal_retention", +] diff --git a/raven/evolver/orchestrator/state/__init__.py b/raven/evolver/orchestrator/state/__init__.py new file mode 100644 index 0000000..02ae215 --- /dev/null +++ b/raven/evolver/orchestrator/state/__init__.py @@ -0,0 +1,3 @@ +"""Durable orchestrator state — round journal for resume.""" + +from __future__ import annotations diff --git a/raven/evolver/orchestrator/state/journal.py b/raven/evolver/orchestrator/state/journal.py new file mode 100644 index 0000000..d204f9e --- /dev/null +++ b/raven/evolver/orchestrator/state/journal.py @@ -0,0 +1,81 @@ +"""Round journal — append-only checkpoint so a killed run resumes. + +Each completed round appends one JSON line with just enough to reconstruct the +loop's control state on restart: the round index, the parent it evolved to, and +whether it promoted (which drives the termination counter). On resume the loop +replays these records to seed the :class:`TerminationTracker`, the current +parent, and the round counter, then continues from the next round without +re-running the expensive evals it already did. + +The node ledger (``nodes/*.json``) is the durable record of the *nodes*; this +journal is the durable record of the *loop's progress* over them. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # avoid a runtime import cycle with loop + from raven.evolver.orchestrator.loop import RoundResult + + +@dataclass +class RoundJournal: + """Append-only JSONL of completed-round checkpoints.""" + + path: Path + + def __post_init__(self) -> None: + self.path = Path(self.path) + + def append(self, rr: "RoundResult") -> None: + """Persist a compact checkpoint for one completed round.""" + self.path.parent.mkdir(parents=True, exist_ok=True) + record = { + "round_index": rr.round_index, + "parent_id": rr.parent_id, + "next_parent_id": rr.next_parent_id, + "promoted": rr.promoted, + # the two termination signals (SOP: patience compares to VANILLA; + # errored rounds have their own counter) — resume replays these. + "beat_vanilla": rr.beat_vanilla, + "errored": rr.errored, + "verdict": rr.verdict, + # for the post-hoc sealed unseal (C3): the deliverable's commit + + # train pass@1, so its test curve is reconstructable after the run. + "next_parent_sha": rr.next_parent_sha, + "next_parent_train": rr.next_parent_train, + "candidates": [{"node_id": o.node_id, "status": o.status.value} for o in rr.outcomes], + } + with self.path.open("a") as f: + f.write(json.dumps(record) + "\n") + f.flush() + os.fsync(f.fileno()) + + def load(self) -> list[dict]: + """Read completed-round checkpoints in order (empty if none). + + A malformed FINAL line is tolerated and dropped — a kill/disk-full mid + ``append`` leaves a truncated last record, and the journal exists + precisely to survive that; the interrupted round simply re-runs. + Corruption anywhere earlier still raises (the history is not trustable). + """ + if not self.path.exists(): + return [] + lines = [ln.strip() for ln in self.path.read_text().splitlines() if ln.strip()] + out = [] + for i, line in enumerate(lines): + try: + out.append(json.loads(line)) + except ValueError as exc: + if i == len(lines) - 1: + break + raise ValueError(f"corrupt journal record at line {i + 1} of {self.path}: {exc}") from exc + return out + + +__all__ = ["RoundJournal"] diff --git a/raven/evolver/orchestrator/termination.py b/raven/evolver/orchestrator/termination.py new file mode 100644 index 0000000..87c732b --- /dev/null +++ b/raven/evolver/orchestrator/termination.py @@ -0,0 +1,83 @@ +"""Loop termination — the "never stop early" discipline, as code not prompt. + +The SOP stops on the first of these conditions, and the exhaustion signal is +always measured against VANILLA (the fixed cold-start baseline) on train, never +against the previous parent and never against the sealed test set: + +- ``patience`` consecutive rounds in which no candidate beat vanilla on train + (exploration exhausted — the primary signal), or +- ``max_rounds`` reached (a hard cap backstop), or +- ``max_consecutive_errors`` rounds in a row that produced no real decision + (driver/apply/eval outage). An errored round is NOT evidence about + exploration, so it must not burn patience — but an endless outage must not + loop either, so it gets its own counter and an honest ``errors_exhausted`` + stop reason. + +``record_round(promoted=...)`` takes the vanilla-comparison signal: True iff at +least one candidate's full-train confirm beat vanilla this round (regardless of +whether it also beat its ratcheted parent baseline and banked). Keeping this in +a small, unit-tested tracker is exactly what lets a weak driver model run the +loop: the stop decision is the harness's, not something the model has to +remember to check. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class TerminationTracker: + """Track per-round outcomes across rounds and decide when to stop.""" + + patience: int = 10 + max_rounds: int = 20 + max_consecutive_errors: int = 5 + rounds_completed: int = 0 + consecutive_no_promotion: int = 0 + consecutive_errors: int = 0 + + def __post_init__(self) -> None: + if self.patience < 1: + raise ValueError("patience must be >= 1") + if self.max_rounds < 1: + raise ValueError("max_rounds must be >= 1") + if self.max_consecutive_errors < 1: + raise ValueError("max_consecutive_errors must be >= 1") + + def record_round(self, promoted: bool, *, errored: bool = False) -> None: + """Record one completed round's outcome. + + ``promoted`` is the SOP exhaustion signal: True iff at least one + candidate beat VANILLA on the full train set this round. ``errored`` + marks a round that produced no real decision (every candidate/phase + errored); it advances the round counter and the error counter but + leaves patience untouched — an outage says nothing about exploration. + """ + self.rounds_completed += 1 + if errored: + self.consecutive_errors += 1 + return + self.consecutive_errors = 0 + if promoted: + self.consecutive_no_promotion = 0 + else: + self.consecutive_no_promotion += 1 + + def should_stop(self) -> tuple[bool, str | None]: + """Return ``(stop, reason)`` given rounds recorded so far. + + ``reason`` is None while the loop should continue. ``max_rounds`` is + checked first so hitting the cap reports the cap even if patience was + also exhausted on the same round. + """ + if self.rounds_completed >= self.max_rounds: + return True, "max_rounds" + if self.consecutive_errors >= self.max_consecutive_errors: + return True, "errors_exhausted" + if self.consecutive_no_promotion >= self.patience: + return True, "patience_exhausted" + return False, None + + +__all__ = ["TerminationTracker"] diff --git a/raven/evolver/scheduler/__init__.py b/raven/evolver/scheduler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/raven/evolver/scheduler/anchor_selection.py b/raven/evolver/scheduler/anchor_selection.py new file mode 100644 index 0000000..22124d3 --- /dev/null +++ b/raven/evolver/scheduler/anchor_selection.py @@ -0,0 +1,236 @@ +"""Anchor-subset selection for the per-round evolution screen. + +:func:`select_anchor` composes the cold-start thick ledger (vanilla K=3 over +the train pool) into the small anchor subset used for K=1 screening, and emits +the screen cull threshold derived from the *same* ledger. + +Why one call returns both the task list and the threshold: the anchor tasks are +picked by per-task Bernoulli variance ``p*(1-p)``, and the screen noise band +``sigma_screen`` is ``sqrt((1/n^2) * sum p*(1-p))`` over exactly those tasks. +Same ledger, same per-task rates -> selecting the subset and sizing its jitter +are one computation, not two. ``sigma_screen`` is the *K=1 anchor-mean* sampling +sigma (large/loose: K=1 + small n + deliberately high-variance tasks); it is a +different quantity from the full-set K=3 paired sigma used for credited +significance. + +The subset mixes three roles (cold start, round 1): + +- ``sentinel`` : STABLE_PASS tasks; catch a candidate that breaks easy tasks. +- ``icebreaker`` : STABLE_FAIL tasks; room for a mechanism to rescue a hard one. +- ``borderline`` : BORDERLINE_* tasks; the high-variance discriminators that + dominate ``sigma_screen``. + +Round 2+: pass an ``affinity`` map (``{task_id: mechanism-fire density}``, e.g. +from ``affinity_task_picker.per_task_density``) to bias icebreaker slots toward +tasks where the round's mechanism actually fires. Without it the selection is +ledger-only and mechanism-agnostic. + +Selection is deterministic: ties break by ascending ``task_id`` so the same +ledger always yields the same anchor (reproducible paper experiments). +""" + +from __future__ import annotations + +import math +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +from raven.evolver.analysis.stability_bucket import ( + StabilityBucket, + TaskStability, + compute_stability, +) + +_BORDERLINE_BUCKETS = ( + StabilityBucket.BORDERLINE_2_3, + StabilityBucket.BORDERLINE_1_3, +) + + +@dataclass(frozen=True) +class AnchorTask: + """One picked anchor task with the ledger stats that placed it.""" + + task_id: str + role: str # "sentinel" | "icebreaker" | "borderline" + pass_rate: float + variance: float + + +@dataclass(frozen=True) +class AnchorSelection: + """Result of :func:`select_anchor`. + + ``task_ids`` is the screen subset; ``cull_threshold`` is the pp margin + below vanilla beyond which a candidate is dropped at screen (a candidate + whose anchor-mean pass@1 is more than ``cull_threshold`` under vanilla's + anchor-mean is pruned; anything closer goes to the full-set confirm). + """ + + task_ids: list[str] + sigma_screen: float + cull_threshold: float + tasks: list[AnchorTask] + shortfalls: dict[str, int] + + def __len__(self) -> int: + return len(self.task_ids) + + +def _variance(stab: TaskStability) -> float: + """Plug-in Bernoulli variance ``p*(1-p)`` from the ledger pass rate.""" + if stab.attempts == 0: + return 0.0 + p = stab.passes / stab.attempts + return p * (1.0 - p) + + +def _pass_rate(stab: TaskStability) -> float: + if stab.attempts == 0: + return 0.0 + return stab.passes / stab.attempts + + +def select_anchor( + ledger_dir: str | Path, + *, + n_sentinel: int = 3, + n_icebreaker: int = 5, + n_borderline: int = 7, + cull_sigma_mult: float = 1.5, + affinity: dict[str, float] | None = None, +) -> AnchorSelection: + """Pick the anchor screen subset from a vanilla K=3 thick ledger. + + Parameters + ---------- + ledger_dir + Path accepted by + :func:`raven.evolver.analysis.stability_bucket.compute_stability` + (a legacy jobs dir or its dated subdir) holding the vanilla K=3 run + over the train pool. Must be train-only — the anchor is part of the + evolution decision, so a sealed test task entering it is a leak. + n_sentinel, n_icebreaker, n_borderline + Slot budget per role. Defaults give a ~15-task anchor. If a bucket has + fewer tasks than its budget the slot count is recorded in + ``shortfalls`` rather than raising. + cull_sigma_mult + Multiplier on ``sigma_screen`` for the cull threshold (default 1.5; + wide-pass screen — only egregious losers are dropped at screen). + affinity + Optional ``{task_id: density}`` to rank icebreaker (STABLE_FAIL) slots + by mechanism-fire density (round 2+). Tasks absent from the map score + 0. Without it icebreakers are taken in ``task_id`` order. + + Returns + ------- + AnchorSelection + With ``task_ids``, ``sigma_screen``, ``cull_threshold``, the per-task + breakdown, and per-role ``shortfalls``. + """ + stability = compute_stability(ledger_dir) + if not stability: + raise ValueError(f"no tasks found in ledger dir: {ledger_dir}") + + by_bucket: dict[StabilityBucket, list[TaskStability]] = defaultdict(list) + for stab in stability.values(): + by_bucket[stab.bucket].append(stab) + + affinity = affinity or {} + picked: list[AnchorTask] = [] + shortfalls: dict[str, int] = {} + + def _record(role: str, stats: list[TaskStability], budget: int, key) -> None: + chosen = sorted(stats, key=key)[:budget] + for stab in chosen: + picked.append( + AnchorTask( + task_id=stab.task_id, + role=role, + pass_rate=_pass_rate(stab), + variance=_variance(stab), + ) + ) + missing = budget - len(chosen) + if missing > 0: + shortfalls[role] = missing + + # sentinels / icebreakers carry no within-task variance (p in {0, 1}); + # order them by affinity (icebreakers, round 2+) then task_id for determinism. + _record( + "sentinel", + by_bucket[StabilityBucket.STABLE_PASS], + n_sentinel, + key=lambda s: s.task_id, + ) + _record( + "icebreaker", + by_bucket[StabilityBucket.STABLE_FAIL], + n_icebreaker, + key=lambda s: (-affinity.get(s.task_id, 0.0), s.task_id), + ) + # borderline tasks are the discriminators: rank by variance desc, then id. + borderline_pool = [stab for bucket in _BORDERLINE_BUCKETS for stab in by_bucket[bucket]] + _record( + "borderline", + borderline_pool, + n_borderline, + key=lambda s: (-_variance(s), s.task_id), + ) + + n = len(picked) + if n == 0: + raise ValueError(f"anchor selection produced 0 tasks from {ledger_dir} (empty buckets for every role?)") + sigma_screen = math.sqrt(sum(t.variance for t in picked) / (n * n)) + cull_threshold = cull_sigma_mult * sigma_screen + + return AnchorSelection( + task_ids=[t.task_id for t in picked], + sigma_screen=sigma_screen, + cull_threshold=cull_threshold, + tasks=picked, + shortfalls=shortfalls, + ) + + +def simple_anchor( + stability: dict[str, TaskStability], + *, + task_ids: list[str] | None = None, + cull_sigma_mult: float = 1.5, +) -> AnchorSelection: + """Build an ``AnchorSelection`` over ``task_ids`` (or all) from a stability map. + + A lightweight alternative to :func:`select_anchor` for benches where a K=1 + cold start yields no BORDERLINE bucket (so the sentinel/icebreaker/borderline + split is degenerate). ``sigma_screen`` uses the same Bernoulli-variance + formula ``sqrt((1/n^2) * sum p*(1-p))``, so the wide-pass cull threshold is + sized identically to ``select_anchor``. Takes an already-built stability map + (not a ledger dir) so any bench that can produce ``{task_id: TaskStability}`` + reuses it — no bench-specific format knowledge lives here. + """ + ids = sorted(task_ids if task_ids is not None else stability) + if not ids: + raise ValueError("simple_anchor requires at least one task") + tasks: list[AnchorTask] = [] + for tid in ids: + st = stability[tid] + p = st.passes / st.attempts if st.attempts else 0.0 + role = ( + "sentinel" + if st.bucket == StabilityBucket.STABLE_PASS + else "icebreaker" + if st.bucket == StabilityBucket.STABLE_FAIL + else "borderline" + ) + tasks.append(AnchorTask(task_id=tid, role=role, pass_rate=p, variance=p * (1 - p))) + n = len(tasks) + sigma_screen = math.sqrt(sum(t.variance for t in tasks) / (n * n)) + return AnchorSelection( + task_ids=ids, + sigma_screen=sigma_screen, + cull_threshold=cull_sigma_mult * sigma_screen, + tasks=tasks, + shortfalls={}, + ) diff --git a/raven/evolver/scheduler/bandit_tasks.py b/raven/evolver/scheduler/bandit_tasks.py new file mode 100644 index 0000000..bc8eb0a --- /dev/null +++ b/raven/evolver/scheduler/bandit_tasks.py @@ -0,0 +1,256 @@ +"""Bandit-on-tasks scheduler. + +Selects an informative subset of tasks to evaluate each new harness +candidate on, rather than running the full benchmark every time. + +The scheduler maintains per-task state across candidate evaluations: + +- ``n_trials`` — how many candidates have run this task +- ``successes`` — how many of those passed +- ``per_candidate`` — task_id → {candidate_id → pass/fail} so we can + compute per-task variance across candidates (= discrimination power) + +For each new candidate, :meth:`choose` returns ``n`` tasks scored by +**information value × uncertainty bonus**: + +- information value = task's posterior variance across already-seen + candidates (high variance = strong discriminator) +- uncertainty bonus = ``1/sqrt(n_trials+1)`` UCB-style, so under-tried + tasks get a fair shot in early rounds + +This is a quality-diversity flavoured variant of best-arm identification +(Even-Dar et al. 2006, Audibert & Bubeck 2010). It is **not** classical +UCB — UCB would maximize expected reward; we want to maximize +**ranking information**, which is captured by between-candidate variance. + +Algorithmic notes for the paper: + +- Beta(1+s, 1+f) posterior per (task, candidate) cell — Laplace prior +- Successive elimination by ``mark_resolved``: once a task is "decided" + (e.g., always-pass / always-fail across many candidates), it drops + out of future ``choose`` rounds +- ``rng`` is seedable for reproducibility (paper requirement) +""" + +from __future__ import annotations + +import math +import random +from dataclasses import dataclass, field +from typing import Iterable + + +@dataclass +class TaskStats: + """Per-task state accumulated across candidate evaluations. + + The ``per_candidate`` map is the source of truth for variance + estimation: variance is computed across the recorded outcomes, + so a task that always passes / always fails has variance ≈ 0 + and naturally gets de-prioritized. + + ``resolved`` is a sticky drop-out flag, set by + :meth:`BanditTaskScheduler.mark_resolved` when the task is no + longer informative (e.g. trivially-easy or trivially-hard across + every candidate seen so far). Resolved tasks return to play only + if explicitly un-resolved. + """ + + task_id: str + n_trials: int = 0 + successes: int = 0 + per_candidate: dict[str, bool] = field(default_factory=dict) + resolved: bool = False + + @property + def empirical_pass_rate(self) -> float: + """``successes / n_trials``, or 0.5 if never tried (uninformative prior).""" + if self.n_trials == 0: + return 0.5 + return self.successes / self.n_trials + + @property + def variance_across_candidates(self) -> float: + """Variance of pass outcomes across distinct candidates seen. + + Returns 0.25 (max for Bernoulli) when fewer than 2 candidates + have run this task — this default is the "cold start exploration + bonus": new tasks look maximally informative until proven + otherwise. + + Once ≥2 candidates contribute, the variance is computed + empirically: ``p * (1 - p)`` where ``p`` is the fraction of + candidates that passed. A task all candidates pass (or all fail) + gets variance 0 and is naturally deprioritized. + """ + n = len(self.per_candidate) + if n < 2: + return 0.25 + passes = sum(1 for v in self.per_candidate.values() if v) + p = passes / n + return p * (1.0 - p) + + +class BanditTaskScheduler: + """Schedule which tasks to evaluate the next harness candidate on. + + Typical use: + + >>> scheduler = BanditTaskScheduler(["task_001", "task_002", ...]) + >>> # for each new candidate node N: + >>> task_subset = scheduler.choose(n=30, candidate_id=N.node_id) + >>> results = run_evaluation(N, task_subset) # {task_id: bool} + >>> scheduler.update(N.node_id, results) + + The internal posterior is Beta-Bernoulli with Laplace prior; the + selection score combines per-task variance (discrimination power) + with a UCB-style uncertainty bonus over ``n_trials``. + + Parameters + ---------- + all_task_ids + The complete pool of available task ids. ``choose`` returns a + subset of this pool. + prior + Optional ``{task_id: prior_pass_rate}`` to seed initial variance + estimates. Useful when porting between benchmarks or warm-starting + from historical runs. + exploration_weight + Scales the UCB-style uncertainty bonus. Larger value → favours + under-tried tasks more aggressively. Default 1.0 matches standard + UCB1 constant. + rng_seed + Seed for the internal RNG used to break ties and inject small + exploration noise. Set this for reproducible paper experiments. + """ + + def __init__( + self, + all_task_ids: Iterable[str], + *, + prior: dict[str, float] | None = None, + exploration_weight: float = 1.0, + rng_seed: int | None = None, + ) -> None: + self.tasks: dict[str, TaskStats] = {tid: TaskStats(task_id=tid) for tid in all_task_ids} + if not self.tasks: + raise ValueError("BanditTaskScheduler requires at least one task id") + self._exploration_weight = exploration_weight + self._rng = random.Random(rng_seed) + if prior: + self._apply_prior(prior) + + def _apply_prior(self, prior: dict[str, float]) -> None: + """Seed variance estimates from a prior pass-rate map. + + Each prior entry is treated as one "synthetic candidate" outcome + — recorded under candidate id ``"__prior__"`` so the variance + machinery picks it up. A prior entry alone does not produce + variance (variance needs ≥2 distinct candidates), but it shifts + what subsequent real candidates need to deviate from to look + informative. + """ + for task_id, p in prior.items(): + if task_id not in self.tasks: + continue + # Treat prior as a single synthetic observation + self.tasks[task_id].per_candidate["__prior__"] = bool(p >= 0.5) + self.tasks[task_id].n_trials += 1 + if p >= 0.5: + self.tasks[task_id].successes += 1 + + def choose(self, n: int = 30, *, candidate_id: str | None = None) -> list[str]: + """Select ``n`` tasks to evaluate the next candidate on. + + Tasks already seen by ``candidate_id`` (if provided) are excluded + — we don't pay to re-evaluate the same (task, candidate) pair. + + If fewer than ``n`` active (non-resolved, non-already-seen) tasks + exist, returns whatever is available. + + Scoring per task:: + + score = variance_across_candidates + + exploration_weight * 1/sqrt(n_trials + 1) + + tiny_random_jitter + + Top-``n`` by score wins. The random jitter is small enough not + to flip clearly-better tasks but breaks ties deterministically + given the seed. + """ + if n <= 0: + return [] + candidates: list[tuple[float, str]] = [] + for stats in self.tasks.values(): + if stats.resolved: + continue + if candidate_id is not None and candidate_id in stats.per_candidate: + continue + uncertainty = self._exploration_weight / math.sqrt(stats.n_trials + 1) + jitter = self._rng.random() * 1e-6 + score = stats.variance_across_candidates + uncertainty + jitter + candidates.append((score, stats.task_id)) + candidates.sort(reverse=True) + return [task_id for _, task_id in candidates[:n]] + + def update(self, candidate_id: str, results: dict[str, bool]) -> None: + """Record per-task results for one candidate evaluation. + + Updates ``n_trials``, ``successes``, and ``per_candidate``. The + variance estimate is recomputed lazily via the property — no + explicit recalculation needed here. + + Unknown ``task_id`` keys in ``results`` are silently skipped so + the scheduler tolerates results that include tasks added later + (e.g. when the task pool expands between rounds). + """ + for task_id, passed in results.items(): + stats = self.tasks.get(task_id) + if stats is None: + continue + # Re-running the same candidate on the same task is a no-op + # (it would double-count successes). Caller should de-dup, + # but we defend here too. + if candidate_id in stats.per_candidate: + continue + stats.per_candidate[candidate_id] = bool(passed) + stats.n_trials += 1 + if passed: + stats.successes += 1 + + def get_rank_estimate(self) -> list[tuple[str, float]]: + """Return tasks ranked by Beta-Bernoulli posterior mean pass rate. + + Posterior mean is ``(successes + 1) / (n_trials + 2)`` (Laplace + smoothing). Useful for diagnostic / analysis, not for the + ``choose`` decision (which uses variance, not mean). + + Sorted descending — easiest tasks first. + """ + rows: list[tuple[str, float]] = [] + for stats in self.tasks.values(): + mean = (stats.successes + 1) / (stats.n_trials + 2) + rows.append((stats.task_id, mean)) + rows.sort(key=lambda r: -r[1]) + return rows + + def mark_resolved(self, task_id: str, resolved: bool = True) -> None: + """Manually flag a task as resolved (skipped by ``choose``). + + Called by the harness when, e.g., a task has been seen by ≥K + candidates with unanimous outcome — further evaluation wastes + budget. Passing ``resolved=False`` reverts the flag. + """ + stats = self.tasks.get(task_id) + if stats is None: + return + stats.resolved = resolved + + def active_task_count(self) -> int: + """Number of tasks currently eligible for ``choose``.""" + return sum(1 for s in self.tasks.values() if not s.resolved) + + def __repr__(self) -> str: + active = self.active_task_count() + total = len(self.tasks) + return f"BanditTaskScheduler(total={total}, active={active}, exploration_weight={self._exploration_weight})" diff --git a/raven/evolver/scheduler/branching_policy.py b/raven/evolver/scheduler/branching_policy.py new file mode 100644 index 0000000..26ded63 --- /dev/null +++ b/raven/evolver/scheduler/branching_policy.py @@ -0,0 +1,217 @@ +"""Adaptive branching policy (spec §18.7 Option G). + +Decides how many children :math:`B` to spawn per evolution round, as a +function of the current evolution state, rather than carrying a fixed +``B`` hyperparameter through every round. Sister algorithm to Option F +(``tree_aware_bandit.py``): + + - Option F: same B, picks K tasks per child more sample-efficiently + - Option G: adaptive B per round (this module) + +Per spec §18.7.3 the three implementation tiers are: + + 1. **Rule-based** (this MVP) — ``B = f(state)`` with a small closed- + form formula. ~2h engineering. + 2. **Learned rule** — fit a regression of B → final lift on each + round's outcomes. ~3-4h. + 3. **Bandit-on-branching** — treat B as the arm, reward as + ``Δ lift − cost`` per round. ~1 week, paper-worthy. + +This module is tier 1. Future tiers can substitute ``branching_policy`` +at its call sites without changing them. + +State inputs: + + - ``round_idx`` — current round number (1-indexed) + - ``remaining_budget`` — children we can still afford this evolution + - ``n_uncovered_cells`` — failure_map cells with NO candidate yet + - ``archive_coverage`` — fraction of target cells touched (0-1) + - ``parent_lift_posterior`` — most recent parent's measured lift + (Bernoulli p̂_child − p̂_root, optional) + +Output is a :class:`BranchingDecision` carrying ``B`` plus a one-line +``rationale`` and the contributing components for logging — so future +analysis can reconstruct WHY a given round picked ``B=3`` rather than +``B=5``. + +Typical B range per spec: ``[3, 5]``. The MVP keeps ``min_b=1`` so the +policy can land on a single child when state strongly indicates depth +(near-saturation coverage + strong parent lift), but the floor sits at +3 under default exploration pressure. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass(frozen=True) +class BranchingState: + """Snapshot of evolution state at decision time. + + All fields except ``parent_lift_posterior`` are required — the + policy doesn't try to infer missing inputs. ``parent_lift_posterior`` + is None for the very first round (no parent to measure from) and + in that case the policy uses pure coverage-driven branching. + """ + + round_idx: int + remaining_budget: int + n_uncovered_cells: int + archive_coverage: float + parent_lift_posterior: Optional[float] = None + + def __post_init__(self) -> None: + if self.round_idx < 1: + raise ValueError(f"round_idx must be >= 1, got {self.round_idx}") + if self.remaining_budget < 0: + raise ValueError(f"remaining_budget must be >= 0, got {self.remaining_budget}") + if self.n_uncovered_cells < 0: + raise ValueError(f"n_uncovered_cells must be >= 0, got {self.n_uncovered_cells}") + if not 0.0 <= self.archive_coverage <= 1.0: + raise ValueError(f"archive_coverage must be in [0, 1], got {self.archive_coverage}") + + +@dataclass(frozen=True) +class BranchingDecision: + """Output of :func:`branching_policy`.""" + + B: int + rationale: str + components: dict[str, int] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Defaults (spec §18.7.2 typical B ∈ [3, 5]) +# --------------------------------------------------------------------------- + +DEFAULT_MIN_B = 1 +DEFAULT_MAX_B = 5 +DEFAULT_TARGET_COVERAGE = 0.85 + +# Lift posterior thresholds (Bernoulli p̂_child − p̂_root) +LIFT_STRONG_POSITIVE = 0.10 # ≥ +10% pass rate → depth on winner (−1 to B) +LIFT_STRONG_NEGATIVE = -0.05 # ≤ -5% → broaden search (+1 to B) + +# Round-decay onset (≥ this round, B is shaded down by 1) +LATE_ROUND_ONSET = 5 + + +def branching_policy( + state: BranchingState, + *, + min_b: int = DEFAULT_MIN_B, + max_b: int = DEFAULT_MAX_B, + target_coverage: float = DEFAULT_TARGET_COVERAGE, +) -> BranchingDecision: + """Decide branching factor ``B`` for the next evolution round. + + Algorithm (per spec §18.7.2 + §18.7.3 tier 1): + + 1. **Coverage-driven base.** Start from the gap between current + coverage and the target. Larger gap → larger B (explore). + Buckets: gap > 0.5 → 4, > 0.3 → 3, > 0.15 → 2, else 1. + + 2. **Uncovered-cell floor.** If there are still ≥ 5 cells without + any candidate, ensure ``B ≥ 3`` regardless of coverage % — many + open cells call for breadth. + + 3. **Parent-lift adjustment.** If the previous round's parent had + a strong measured lift (≥ +0.10), shade B down by 1 (depth on + the winner). If lift was negative (≤ -0.05), shade B up by 1 + (escape the losing direction). + + 4. **Budget cap.** Cannot exceed ``remaining_budget``. + + 5. **Late-round decay.** Round ≥ 5 shades down by 1 (consolidation). + + 6. **Clamp** to ``[min_b, max_b]``. + + Returns a decision with ``B`` plus a rationale string and per-step + contribution dict for later analysis. + """ + if min_b < 1: + raise ValueError(f"min_b must be >= 1, got {min_b}") + if max_b < min_b: + raise ValueError(f"max_b ({max_b}) must be >= min_b ({min_b})") + if not 0.0 <= target_coverage <= 1.0: + raise ValueError(f"target_coverage must be in [0, 1], got {target_coverage}") + + components: dict[str, int] = {} + rationale_parts: list[str] = [] + + # ── 1. Coverage-driven base ── + gap = target_coverage - state.archive_coverage + if gap > 0.5: + b_base = 4 + elif gap > 0.3: + b_base = 3 + elif gap > 0.15: + b_base = 2 + else: + b_base = 1 + components["coverage_base"] = b_base + rationale_parts.append(f"coverage gap {gap:+.2f} → base {b_base}") + + # ── 2. Uncovered-cell floor ── + if state.n_uncovered_cells >= 5 and b_base < 3: + b_after_floor = 3 + components["uncovered_floor"] = b_after_floor - b_base + rationale_parts.append(f"{state.n_uncovered_cells} uncovered cells → floor to {b_after_floor}") + b_base = b_after_floor + + # ── 3. Parent-lift adjustment ── + if state.parent_lift_posterior is not None: + if state.parent_lift_posterior >= LIFT_STRONG_POSITIVE: + components["lift_adjustment"] = -1 + rationale_parts.append( + f"parent lift {state.parent_lift_posterior:+.3f} ≥ {LIFT_STRONG_POSITIVE:+.2f} → depth (-1)" + ) + b_base -= 1 + elif state.parent_lift_posterior <= LIFT_STRONG_NEGATIVE: + components["lift_adjustment"] = +1 + rationale_parts.append( + f"parent lift {state.parent_lift_posterior:+.3f} ≤ {LIFT_STRONG_NEGATIVE:+.2f} → broaden (+1)" + ) + b_base += 1 + else: + components["lift_adjustment"] = 0 + + # ── 4. Late-round decay ── + if state.round_idx >= LATE_ROUND_ONSET: + components["late_round_decay"] = -1 + rationale_parts.append(f"round {state.round_idx} ≥ {LATE_ROUND_ONSET} → decay (-1)") + b_base -= 1 + + # ── 5. Budget cap ── + if b_base > state.remaining_budget: + components["budget_cap"] = state.remaining_budget - b_base + rationale_parts.append(f"budget {state.remaining_budget} caps from {b_base}") + b_base = state.remaining_budget + + # ── 6. Clamp ── + final_b = max(min_b, min(max_b, b_base)) + if final_b != b_base: + components["clamp"] = final_b - b_base + rationale_parts.append(f"clamped {b_base} → {final_b} (min_b={min_b}, max_b={max_b})") + + components["final"] = final_b + return BranchingDecision( + B=final_b, + rationale=" | ".join(rationale_parts), + components=components, + ) + + +__all__ = [ + "BranchingState", + "BranchingDecision", + "branching_policy", + "LIFT_STRONG_POSITIVE", + "LIFT_STRONG_NEGATIVE", + "LATE_ROUND_ONSET", + "DEFAULT_MIN_B", + "DEFAULT_MAX_B", + "DEFAULT_TARGET_COVERAGE", +] diff --git a/raven/evolver/scheduler/cold_start_bandit.py b/raven/evolver/scheduler/cold_start_bandit.py new file mode 100644 index 0000000..1d91690 --- /dev/null +++ b/raven/evolver/scheduler/cold_start_bandit.py @@ -0,0 +1,366 @@ +"""Cold-start coverage bandit (5th orchestration mechanism). + +Selects which trials a judge (claude in Phase 1 bootstrap, Qwen in +later phases) should label, optimizing for K=7 WHY pathology class +coverage at minimum judge calls. + +v2 design (post v7 k=3 paired baseline, 2026-06-05) — uses k=3 +stability bucket as **dominant proxy feature**: + +- ``STABLE_PASS`` (3/3): skip entirely (no pathology to discover) +- ``BORDERLINE_2_3`` / ``BORDERLINE_1_3``: high signal density — Phase 1 + drains these first (every borderline trial has near-guaranteed + informative content because the task itself is at the discrimination + edge) +- ``STABLE_FAIL`` (0/3): persistent pathology — Phase 2 runs UCB on + K-means sub-strata of cheap metadata (turn count / exit status / + ``has_tool_calls`` / docker_error etc.), reward = "new WHY discovered" + +Stops when ``n_why_classes`` covered or ``budget`` exhausted. + +Theoretical complexity: + +- Phase 1: O(|borderline|), deterministic (no posterior to converge) +- Phase 2: O(K log K) expected for UCB best-arm identification on + Beta-Bernoulli posterior over the sub-strata reward signal +- Total: O(|borderline| + K log K) — typically ~22 trials for K=7 + on TB2 v7 k=3 baseline (11 borderline tasks × 3 attempts max) + +Algorithmic notes for paper §13: + +- "5th mechanism" alongside spec §0.2's four (bandit-on-WHY / + bandit-on-nodes / bandit-on-tasks / prefix-replay) +- Same UCB family as ``BanditTaskScheduler`` (A1, ``bandit_tasks.py``) + — Beta-Bernoulli + UCB uncertainty bonus + deterministic jitter for + tie-break — so code structure mirrors that module +- v2 vs v1 (uniform K-means K'=10 on cheap proxy without stability + bucket): v2 leverages k=3 stability as a strong cheap signal, + reduces typical budget from 30 to ~22 (~25% saving) by skipping + stable_pass entirely and saturating borderline first +- For paper §15 Must-nail #1 ablation, compare: + * uniform random + * stratified random (γ baseline) + * v1 UCB on proxy strata (no stability bucket) + * v2 (this implementation) + * DPP-based diverse subset (optional δ baseline) +""" + +from __future__ import annotations + +import math +import random +from dataclasses import dataclass, field +from typing import Sequence + +from raven.evolver.analysis.stability_bucket import StabilityBucket + + +@dataclass(frozen=True) +class Trial: + """A single (task, attempt) trial — the unit the bandit samples. + + Caller is responsible for constructing this from raw legacy trial + dirs + the outputs of ``stability_bucket.compute_stability`` (task + level) and ``proxy_features.extract`` (trial level). + + ``stability`` is the task-level bucket (all attempts of one task + share the same value). ``proxy_features`` is the per-trial cheap + metadata used for Phase 2 K-means sub-strata. + """ + + trial_id: str + task_id: str + attempt: int + passed: bool + stability: StabilityBucket + proxy_features: dict = field(default_factory=dict) + + @property + def is_borderline(self) -> bool: + return self.stability in ( + StabilityBucket.BORDERLINE_1_3, + StabilityBucket.BORDERLINE_2_3, + ) + + @property + def is_stable_fail(self) -> bool: + return self.stability == StabilityBucket.STABLE_FAIL + + @property + def is_stable_pass(self) -> bool: + return self.stability == StabilityBucket.STABLE_PASS + + +@dataclass +class StratumStats: + """Per-sub-stratum bandit state (Beta-Bernoulli over reward). + + Reward is binary ("did judging a trial from this stratum reveal a + WHY class not previously covered"). We accumulate raw counts; the + UCB score is computed lazily from them. + """ + + stratum_id: str + n_pulls: int = 0 + n_new_why: int = 0 + + @property + def empirical_rate(self) -> float: + if self.n_pulls == 0: + return 0.0 + return self.n_new_why / self.n_pulls + + def ucb_score(self, T: int, exploration_weight: float = 1.0) -> float: + """UCB1 score: empirical rate + exploration bonus. + + Returns +inf for never-pulled strata so each is visited at + least once before exploitation takes over. + """ + if self.n_pulls == 0: + return float("inf") + bonus = exploration_weight * math.sqrt(2 * math.log(max(T, 1)) / self.n_pulls) + return self.empirical_rate + bonus + + +def _kmeans_strata( + trials: Sequence[Trial], + n_strata: int, + rng: random.Random, + max_iter: int = 20, +) -> dict[str, list[Trial]]: + """Cluster trials into ``n_strata`` sub-strata by their proxy_features. + + Simple Lloyd's K-means on numeric features (sorted keys for + determinism, min-max normalized for fair Euclidean distance). Pool + sizes here are tiny (~60 trials, K=5) so a custom impl beats + pulling in scikit-learn. Returns ``{stratum_id: [trials]}``. + + Returns ``{}`` for empty input. Caps ``n_strata`` at len(trials). + """ + if not trials: + return {} + n_strata = min(n_strata, len(trials)) + if n_strata == 0: + return {} + + feature_keys = sorted(trials[0].proxy_features.keys()) + if not feature_keys: + return {"stratum_0": list(trials)} + + vectors = [[float(t.proxy_features.get(k, 0.0)) for k in feature_keys] for t in trials] + mins = [min(v[i] for v in vectors) for i in range(len(feature_keys))] + maxs = [max(v[i] for v in vectors) for i in range(len(feature_keys))] + spans = [(maxs[i] - mins[i]) or 1.0 for i in range(len(feature_keys))] + norm = [[(v[i] - mins[i]) / spans[i] for i in range(len(feature_keys))] for v in vectors] + + centroid_indices = rng.sample(range(len(norm)), n_strata) + centroids = [norm[i][:] for i in centroid_indices] + + labels = [-1] * len(trials) + for _ in range(max_iter): + new_labels = [] + for v in norm: + best_idx = 0 + best_dist = float("inf") + for k, c in enumerate(centroids): + dist = sum((v[i] - c[i]) ** 2 for i in range(len(feature_keys))) + if dist < best_dist: + best_dist = dist + best_idx = k + new_labels.append(best_idx) + if new_labels == labels: + break + labels = new_labels + for k in range(n_strata): + members = [v for v, lab in zip(norm, labels) if lab == k] + if members: + centroids[k] = [sum(v[i] for v in members) / len(members) for i in range(len(feature_keys))] + + result: dict[str, list[Trial]] = {f"stratum_{k}": [] for k in range(n_strata)} + for trial, label in zip(trials, labels): + result[f"stratum_{label}"].append(trial) + return {sid: ts for sid, ts in result.items() if ts} + + +class ColdStartCoverageBandit: + """5th mechanism — cold-start coverage sampling. + + Typical use:: + + bandit = ColdStartCoverageBandit( + trials=all_267_trials_from_v7_k3, + n_why_classes=7, + budget=25, + rng_seed=42, + ) + while not bandit.done(): + trial = bandit.next_trial() + result = claude_judge(trial) # JudgeResult JSON + bandit.update(trial, why=result.why) + sampled = bandit.sampled_trials() + covered = bandit.covered_why_classes() + + Parameters + ---------- + trials + Full pool of trials with ``stability`` and ``proxy_features`` + populated. ``STABLE_PASS`` trials are automatically excluded. + n_why_classes + K — number of pathology classes to cover (spec §12.5: 7). + budget + Max number of trials to judge. Default 25 (v2 design). + n_stable_fail_strata + Number of K-means sub-strata for Phase 2. Default 5. + exploration_weight + UCB bonus scaler. Default 1.0 (matches UCB1 constant). + rng_seed + Seed for shuffles, K-means init, and tie-breaking jitter + (paper reproducibility requirement). + """ + + def __init__( + self, + trials: Sequence[Trial], + *, + n_why_classes: int = 7, + budget: int = 25, + n_stable_fail_strata: int = 5, + exploration_weight: float = 1.0, + rng_seed: int | None = None, + ) -> None: + if not trials: + raise ValueError("ColdStartCoverageBandit requires at least one trial") + if budget <= 0: + raise ValueError("budget must be positive") + if n_why_classes <= 0: + raise ValueError("n_why_classes must be positive") + if n_stable_fail_strata <= 0: + raise ValueError("n_stable_fail_strata must be positive") + + self._n_why_classes = n_why_classes + self._budget = budget + self._exploration_weight = exploration_weight + self._rng = random.Random(rng_seed) + + # Partition by stability; stable_pass is dropped (no pathology) + self._borderline_pool: list[Trial] = [t for t in trials if t.is_borderline] + stable_fail_pool: list[Trial] = [t for t in trials if t.is_stable_fail] + + # Phase 1 ordering: shuffle deterministically (every borderline + # trial is high-signal; order matters only for variety across + # different runs with different seeds) + self._rng.shuffle(self._borderline_pool) + + # Phase 2 K-means strata over stable_fail pool + self._stable_fail_strata: dict[str, list[Trial]] = _kmeans_strata( + stable_fail_pool, + n_stable_fail_strata, + self._rng, + ) + # Record which stratum each trial originated from, so update() + # can find the right StratumStats after a trial has been popped + self._trial_to_stratum: dict[str, str] = {} + for stratum_id, ts in self._stable_fail_strata.items(): + self._rng.shuffle(ts) + for t in ts: + self._trial_to_stratum[t.trial_id] = stratum_id + self._strata_bandit: dict[str, StratumStats] = { + sid: StratumStats(stratum_id=sid) for sid in self._stable_fail_strata + } + + # Mutable state + self._sampled: list[Trial] = [] + self._covered_why: set[str] = set() + self._borderline_idx = 0 + + # ────────────────────── public API ────────────────────── + + def done(self) -> bool: + """Stop conditions: budget hit, coverage met, or pool exhausted.""" + if len(self._sampled) >= self._budget: + return True + if len(self._covered_why) >= self._n_why_classes: + return True + return not self._has_unsampled_trial() + + def next_trial(self) -> Trial: + """Pick next trial to judge. + + Phase 1 (borderline drain) always takes priority. Phase 2 (UCB + on stable_fail strata) kicks in only when borderline is empty. + """ + if self.done(): + raise RuntimeError("ColdStartCoverageBandit is done; cannot sample more") + + # Phase 1: borderline pool, sequential drain after init shuffle + if self._borderline_idx < len(self._borderline_pool): + t = self._borderline_pool[self._borderline_idx] + self._borderline_idx += 1 + return t + + # Phase 2: UCB pick on stable_fail strata + T = len(self._sampled) + scored: list[tuple[float, str]] = [] + for sid, stats in self._strata_bandit.items(): + if not self._stable_fail_strata.get(sid): + continue + score = stats.ucb_score(T, self._exploration_weight) + # tiny jitter so equal scores are broken deterministically + score = score + self._rng.random() * 1e-9 + scored.append((score, sid)) + if not scored: + raise RuntimeError("done() lied: no remaining strata but not stopped") + scored.sort(reverse=True) + chosen_sid = scored[0][1] + return self._stable_fail_strata[chosen_sid].pop(0) + + def update(self, trial: Trial, *, why: str) -> None: + """Record judge result for a sampled trial. + + ``why`` is the WHY class label assigned (e.g. ``"budget_awareness"``). + Reward = "discovered new WHY class" — credits the originating + stratum's bandit posterior when applicable. + """ + is_new = why not in self._covered_why + self._covered_why.add(why) + self._sampled.append(trial) + + if trial.is_stable_fail: + stratum_id = self._trial_to_stratum.get(trial.trial_id) + if stratum_id is not None: + stats = self._strata_bandit[stratum_id] + stats.n_pulls += 1 + if is_new: + stats.n_new_why += 1 + + def sampled_trials(self) -> list[Trial]: + return list(self._sampled) + + def covered_why_classes(self) -> set[str]: + return set(self._covered_why) + + def borderline_pool_size(self) -> int: + """Initial borderline pool size (for diagnostic / paper reporting).""" + return len(self._borderline_pool) + + def stable_fail_strata_sizes(self) -> dict[str, int]: + """Current size of each stable_fail sub-stratum (post-popping).""" + return {sid: len(ts) for sid, ts in self._stable_fail_strata.items()} + + # ────────────────────── helpers ────────────────────── + + def _has_unsampled_trial(self) -> bool: + if self._borderline_idx < len(self._borderline_pool): + return True + return any(self._stable_fail_strata.values()) + + def __repr__(self) -> str: + n_borderline = len(self._borderline_pool) - self._borderline_idx + n_stable_fail = sum(len(ts) for ts in self._stable_fail_strata.values()) + return ( + f"ColdStartCoverageBandit(" + f"sampled={len(self._sampled)}/{self._budget}, " + f"covered={len(self._covered_why)}/{self._n_why_classes} WHY, " + f"borderline_remaining={n_borderline}, " + f"stable_fail_remaining={n_stable_fail})" + ) diff --git a/raven/evolver/scheduler/tree_aware_bandit.py b/raven/evolver/scheduler/tree_aware_bandit.py new file mode 100644 index 0000000..5955ee8 --- /dev/null +++ b/raven/evolver/scheduler/tree_aware_bandit.py @@ -0,0 +1,380 @@ +"""Tree-aware bandit-on-tasks (spec §18 Option F.1). + +Hierarchical Bernoulli model with **single-source ancestry kernel**: + + p̂(t, v_new) = Σ_a w(d(v_new, a)) × outcome(a, t) + ───────────────────────────────────── + Σ_a w(d(v_new, a)) + + w(d) = exp(-λ × d) # ancestry distance kernel + +Where: + +- ``a`` ranges over all observed ancestors of ``v_new`` (the new + candidate whose task subset we're selecting) +- ``d(v_new, a)`` is the ancestry distance in the evolution tree +- ``outcome(a, t)`` is each recorded (a, t) outcome ∈ {0, 1} +- Multi-attempt nodes (root k=3 paired) contribute multiple outcomes; + descendant k=1 contributes one + +Score per task ``t`` for next subset pick: + + score(t) = Var_estimate(t) + γ × exploration_bonus(t) + jitter + + Var_estimate(t) = p̂(t) × (1 − p̂(t)) # weighted Bernoulli + exploration_bonus(t) = 1 / √(n_observations + 1) # standard + jitter = ε ∈ [0, 1e-9) # tie-break for reproducibility + +Multi-source extensions (cell prior, trajectory feature prior, per-task +λ learning) deferred to Option F.2 → F.4 per spec §18.6.3-6. v0.1 keeps +the single ancestry kernel so the core mechanics + sample-complexity +bound (§18.6.6) can land first. + +Algorithm family: this mirrors the Beta-Bernoulli + UCB style of +:class:`raven.evolver.scheduler.bandit_tasks.BanditTaskScheduler` — +same exploration-bonus formula, same jitter strategy — extended with +ancestry-weighted posterior. Code structure parallels intentionally. + +References: +- Hong, Branavan, Mansinghka (2022) "Hierarchical Bayesian Bandits", NeurIPS +- Audibert, Bubeck (2010) — best-arm identification baseline +- Gelman et al., *Bayesian Data Analysis* — hierarchical model foundations +""" + +from __future__ import annotations + +import math +import random +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class _NodeInfo: + """Per-node tree topology + accumulated outcomes. + + ``parent_id`` is None only for the tree root. + ``outcomes`` maps task_id → list[bool] (supports multi-attempt nodes + such as a k=3 paired root anchor). + """ + + node_id: str + parent_id: Optional[str] + outcomes: dict[str, list[bool]] = field(default_factory=lambda: defaultdict(list)) + + +class TreeAwareTaskScheduler: + """Tree-aware bandit-on-tasks scheduler (Option F.1, single-source kernel). + + Typical lifecycle: + + scheduler = TreeAwareTaskScheduler( + all_task_ids=tb2_task_ids, + ancestry_lambda=0.7, + rng_seed=42, + ) + + # Phase 0: register root + record k=3 paired outcomes + scheduler.add_node("v7", parent_id=None) + for attempt in range(3): + for task, passed in v7_attempt_results[attempt].items(): + scheduler.add_outcome("v7", task, passed) + + # Round 1: add A, B, C as v7's children, record their k=1 outcomes + scheduler.add_node("A", parent_id="v7") + for task, passed in A_results.items(): + scheduler.add_outcome("A", task, passed) + # ... B, C ... + + # Round 2: pick K tasks for new grandchild A1 of A + scheduler.add_node("A1", parent_id="A") + tasks_for_A1 = scheduler.choose(v_new_id="A1", K=10) + + Parameters + ---------- + all_task_ids + Complete pool of task ids the scheduler may select from. + ancestry_lambda + Decay rate of the ancestry kernel ``w(d) = exp(-λ × d)``. Larger + λ → only close ancestors matter; smaller → distant ancestors + also contribute. Default 0.7 gives w(1) ≈ 0.50, w(2) ≈ 0.25, + w(3) ≈ 0.12. + exploration_weight + UCB-style exploration bonus scaler. Default 1.0. + rng_seed + Seed for the tie-break jitter so identical scenarios reproduce. + + Notes + ----- + v0.1 limitations (deferred to F.2+): + + - Only strict-ancestor outcomes contribute. Siblings, uncles, and + cousins are NOT used (would require a second kernel — Option F.2 + multi-source extension). + - Single global λ. Per-task learned λ is Option F.3. + - No (WHERE × WHY) cell prior. Option F.2. + - No trajectory-feature prior. Option F.4. + """ + + def __init__( + self, + all_task_ids: Iterable[str], + *, + ancestry_lambda: float = 0.7, + exploration_weight: float = 1.0, + rng_seed: Optional[int] = None, + ) -> None: + task_ids = list(all_task_ids) + if not task_ids: + raise ValueError("TreeAwareTaskScheduler requires at least one task id") + if ancestry_lambda <= 0: + raise ValueError("ancestry_lambda must be positive") + if exploration_weight < 0: + raise ValueError("exploration_weight must be non-negative") + + self._task_ids: list[str] = task_ids + self._task_id_set: set[str] = set(task_ids) + self._lambda: float = ancestry_lambda + self._explore_w: float = exploration_weight + self._rng = random.Random(rng_seed) + self._nodes: dict[str, _NodeInfo] = {} + + # ────────────────────────── tree topology ────────────────────────── + + def add_node(self, node_id: str, parent_id: Optional[str]) -> None: + """Register a node in the evolution tree. + + Root has ``parent_id=None``. Duplicate adds are rejected so the + tree topology stays acyclic and well-defined. + """ + if node_id in self._nodes: + raise ValueError(f"node {node_id!r} already registered") + if parent_id is not None and parent_id not in self._nodes: + raise ValueError(f"parent {parent_id!r} of node {node_id!r} not registered yet") + self._nodes[node_id] = _NodeInfo(node_id=node_id, parent_id=parent_id) + + def add_outcome(self, node_id: str, task_id: str, passed: bool) -> None: + """Record one (node, task) pass/fail observation. + + Multiple calls for the same (node, task) pair are appended — + natural for a k>1 paired root or any node re-evaluated. + """ + if node_id not in self._nodes: + raise ValueError(f"node {node_id!r} not registered") + if task_id not in self._task_id_set: + raise ValueError(f"task {task_id!r} not in the registered task pool") + self._nodes[node_id].outcomes[task_id].append(bool(passed)) + + def ancestry_distance(self, descendant_id: str, ancestor_id: str) -> Optional[int]: + """Distance from ``descendant_id`` up to ``ancestor_id``. + + Returns the number of edges if ``ancestor_id`` is on the + descendant's ancestral chain (including ``descendant_id`` + itself at distance 0), or None otherwise. + """ + if descendant_id not in self._nodes: + raise ValueError(f"node {descendant_id!r} not registered") + if ancestor_id not in self._nodes: + raise ValueError(f"node {ancestor_id!r} not registered") + d = 0 + current: Optional[str] = descendant_id + while current is not None: + if current == ancestor_id: + return d + current = self._nodes[current].parent_id + d += 1 + return None + + def ancestors_with_distance(self, node_id: str, *, include_self: bool = True) -> list[tuple[str, int]]: + """All ancestors of ``node_id`` with their distances. + + Ordered from nearest (self at d=0 if ``include_self``) outward + to the root. + """ + if node_id not in self._nodes: + raise ValueError(f"node {node_id!r} not registered") + result: list[tuple[str, int]] = [] + current: Optional[str] = node_id + d = 0 + while current is not None: + if include_self or d > 0: + result.append((current, d)) + current = self._nodes[current].parent_id + d += 1 + return result + + # ────────────────────────── scoring ────────────────────────── + + def _weight(self, distance: int) -> float: + """Ancestry kernel ``exp(-λ × distance)``.""" + return math.exp(-self._lambda * distance) + + def weighted_posterior(self, task_id: str, v_new_id: str) -> tuple[float, float]: + """Hierarchical Bernoulli posterior for (task_id, v_new_id). + + Returns ``(p_hat, effective_n)``: + + - ``p_hat`` is the ancestry-weighted mean of observed outcomes + on ``task_id`` across all of ``v_new_id``'s registered + ancestors (including itself if it already has outcomes). + - ``effective_n`` is the sum of weights — proxy for posterior + tightness. 0 means no informative observations. + + When no ancestor has any outcome on the task, returns the + uninformative prior ``(0.5, 0.0)`` so cold-start tasks still + rank highly via exploration bonus. + """ + if task_id not in self._task_id_set: + raise ValueError(f"task {task_id!r} not in registered pool") + if v_new_id not in self._nodes: + raise ValueError(f"node {v_new_id!r} not registered") + + numerator = 0.0 + denominator = 0.0 + for ancestor_id, distance in self.ancestors_with_distance(v_new_id): + outcomes = self._nodes[ancestor_id].outcomes.get(task_id, ()) + if not outcomes: + continue + w = self._weight(distance) + for passed in outcomes: + numerator += w * (1.0 if passed else 0.0) + denominator += w + + if denominator == 0: + return 0.5, 0.0 + return numerator / denominator, denominator + + def variance_estimate(self, task_id: str, v_new_id: str) -> float: + """Bernoulli variance ``p̂(1-p̂)`` under the weighted posterior. + + Cold-start (no informative ancestor) returns the Bernoulli + maximum 0.25 so unseen tasks compete for exploration slots. + """ + p, n = self.weighted_posterior(task_id, v_new_id) + if n == 0: + return 0.25 + return p * (1.0 - p) + + def exploration_bonus(self, task_id: str) -> float: + """UCB-style bonus ``1 / √(n + 1)`` for under-observed tasks. + + ``n`` counts ALL observations on the task across the whole + tree (not ancestry-weighted) — exploration is about "have we + looked at this task at all", independent of who's asking. + """ + n = sum(len(info.outcomes.get(task_id, ())) for info in self._nodes.values()) + return 1.0 / math.sqrt(n + 1) + + def score(self, task_id: str, v_new_id: str) -> float: + """Composite task selection score: variance + bonus + jitter. + + Higher score = bandit prefers selecting this task for v_new's + next evaluation subset. + """ + var = self.variance_estimate(task_id, v_new_id) + bonus = self._explore_w * self.exploration_bonus(task_id) + jitter = self._rng.random() * 1e-9 + return var + bonus + jitter + + # ────────────────────────── selection API ────────────────────────── + + def choose( + self, + v_new_id: str, + K: int = 10, + *, + exclude_tasks: Optional[Iterable[str]] = None, + ) -> list[str]: + """Pick top-``K`` tasks for the new candidate ``v_new_id``. + + ``exclude_tasks`` lets the caller skip tasks already scheduled + on ``v_new_id`` (e.g., when batching multiple ``choose`` calls + or recovering from a partial run). + + If fewer than ``K`` tasks remain after exclusion, returns + whatever is available. + """ + if K <= 0: + return [] + if v_new_id not in self._nodes: + raise ValueError(f"node {v_new_id!r} not registered") + + excluded: set[str] = set(exclude_tasks) if exclude_tasks else set() + scored: list[tuple[float, str]] = [] + for task_id in self._task_ids: + if task_id in excluded: + continue + scored.append((self.score(task_id, v_new_id), task_id)) + scored.sort(reverse=True) + return [t for _, t in scored[:K]] + + def choose_from_pool( + self, + v_new_id: str, + pool: Iterable[str], + K: int = 5, + ) -> list[str]: + """Pick top-``K`` tasks from a caller-provided ``pool`` subset. + + Identical scoring to :meth:`choose` (variance + γ × exploration + bonus + jitter) but restricted to ``pool`` rather than the full + registered task pool. + + Use case (β+α task selection per + [[project-task-subset-selection-v2]]): + + anchor_10 = LOCKED_ROUND_2_SET_A + non_anchor = set(scheduler._task_ids) - set(anchor_10) + explore_5 = scheduler.choose_from_pool( + v_new_id="round_N_anchor", + pool=non_anchor, + K=5, + ) + subset = list(anchor_10) + explore_5 # K=15 total + + Behavior on edge cases: + - Empty pool → ``[]`` + - K=0 → ``[]`` + - K > pool size → returns all of pool, sorted by score + - Tasks in pool but not in registered ``all_task_ids`` are + silently skipped (caller should pre-filter for clean semantics) + """ + if K <= 0: + return [] + if v_new_id not in self._nodes: + raise ValueError(f"node {v_new_id!r} not registered") + + pool_set: set[str] = set(pool) + if not pool_set: + return [] + # Restrict to tasks the scheduler actually knows about + registered = set(self._task_ids) + valid_pool = pool_set & registered + if not valid_pool: + return [] + + scored: list[tuple[float, str]] = [] + for task_id in valid_pool: + scored.append((self.score(task_id, v_new_id), task_id)) + scored.sort(reverse=True) + return [t for _, t in scored[:K]] + + # ────────────────────────── introspection ────────────────────────── + + def n_nodes(self) -> int: + return len(self._nodes) + + def n_observations(self) -> int: + return sum(sum(len(outs) for outs in info.outcomes.values()) for info in self._nodes.values()) + + def __repr__(self) -> str: + return ( + f"TreeAwareTaskScheduler(tasks={len(self._task_ids)}, " + f"nodes={self.n_nodes()}, observations={self.n_observations()}, " + f"λ={self._lambda})" + ) + + +__all__ = ["TreeAwareTaskScheduler"] diff --git a/raven/evolver/scheduler/tree_loader.py b/raven/evolver/scheduler/tree_loader.py new file mode 100644 index 0000000..e110379 --- /dev/null +++ b/raven/evolver/scheduler/tree_loader.py @@ -0,0 +1,343 @@ +"""Load an evolver tree (HarnessNode JSON files) into a +:class:`TreeAwareTaskScheduler`. + +Bridge between the durable tree representation ( +:mod:`raven.evolver.tree.store`) and the in-memory bandit +(:mod:`raven.evolver.scheduler.tree_aware_bandit`). This is the +canonical loader used for paper §15 #2 tree-aware bandit experiments +and for any future round's task subset selection. + +Reads three kinds of outcome data from each :class:`HarnessNode`: + +1. **Primary eval** — ``node.eval.per_task_results`` (single pass/fail + per task). +2. **Multi-attempt replay** — when ``node.eval.dense_signals`` carries + ``k_attempt_replay_dir``, the loader walks that trial directory and + replays each per-attempt outcome (used for the v7 root's k=3 paired + baseline so the bandit sees three independent observations per + task instead of a single union). +3. **Secondary evals** — when ``dense_signals`` carries + ``secondary_eval__path``, the loader reads the referenced JSON + file and adds those outcomes (used for round-1 validation runs: + 11-borderline + rescue-retest-k=3). + +Secondary eval formats supported (selected by +``secondary_eval__format``): + +- ``task_dict_passed`` — ``{task_id: {passed: bool, ...}}`` (11-borderline) +- ``rescue_k3_attempts`` — ``{task: ..., attempts: [{passed: bool, ...}, ...]}`` (rescue retest) +- ``per_task_passed_block`` — ``{task: {passed: bool, ...}}`` (round 1 first_round_eval child block) +""" + +from __future__ import annotations + +import json +import logging +from collections import deque +from pathlib import Path +from typing import Optional + +from raven.evolver.scheduler.tree_aware_bandit import TreeAwareTaskScheduler +from raven.evolver.tree.node import HarnessNode +from raven.evolver.tree.store import EvolverTreeStore + +logger = logging.getLogger(__name__) + + +def load_tree_into_scheduler( + nodes_dir: Path, + *, + repo_root: Optional[Path] = None, + all_task_ids: Optional[list[str]] = None, + ancestry_lambda: float = 0.7, + exploration_weight: float = 1.0, + rng_seed: Optional[int] = 42, + skip_multi_attempt_replay: bool = False, +) -> TreeAwareTaskScheduler: + """Build a fully populated :class:`TreeAwareTaskScheduler` by reading + every :class:`HarnessNode` JSON in ``nodes_dir`` + their referenced + outcome data files. + + Parameters + ---------- + nodes_dir + Directory containing ``.json`` files (e.g. + ``REPO_ROOT/evolver/nodes/``). + repo_root + Repository root. Used to resolve relative paths in + ``dense_signals``. Defaults to ``nodes_dir.parent.parent``. + all_task_ids + Task pool the scheduler manages. Defaults to ``bandit_tasks_chosen`` + from the root node (typically all 89 TB2 tasks). + ancestry_lambda / exploration_weight / rng_seed + Forwarded to :class:`TreeAwareTaskScheduler`. + skip_multi_attempt_replay + If True, don't walk ``k_attempt_replay_dir`` even if present. The + primary union outcomes get added instead. Used for fast tests. + + Returns + ------- + A scheduler with the full tree topology registered and all known + outcomes replayed. Ready for ``scheduler.choose(v_new_id, K=...)``. + """ + repo_root = repo_root or nodes_dir.parent.parent + store = EvolverTreeStore(repo_root=repo_root, nodes_dir=nodes_dir) + nodes = store.load_all_nodes() + if not nodes: + raise ValueError(f"No HarnessNode JSONs found in {nodes_dir}") + + archived = [n for n in nodes if n.status.value.startswith("archived")] + if archived: + logger.info( + "Excluded %d archived node(s) from scheduler: %s", + len(archived), + [n.node_id for n in archived], + ) + nodes = [n for n in nodes if not n.status.value.startswith("archived")] + if not nodes: + raise ValueError(f"No non-archived HarnessNode JSONs found in {nodes_dir}") + + # ── Determine the task pool ── + if all_task_ids is None: + # Use the root's bandit_tasks_chosen as the canonical pool + roots = [n for n in nodes if n.parent_id is None] + if len(roots) != 1: + raise ValueError(f"Expected exactly one root node, found {len(roots)}: {[r.node_id for r in roots]}") + root = roots[0] + if root.eval is None: + raise ValueError( + f"Root node {root.node_id!r} must carry an eval to define " + f"the task pool. Pass all_task_ids explicitly if intentional." + ) + all_task_ids = list(root.eval.bandit_tasks_chosen) + + scheduler = TreeAwareTaskScheduler( + all_task_ids=all_task_ids, + ancestry_lambda=ancestry_lambda, + exploration_weight=exploration_weight, + rng_seed=rng_seed, + ) + + # ── Topological order — parents before children ── + for node in _topological_order(nodes): + scheduler.add_node(node.node_id, parent_id=node.parent_id) + _record_outcomes( + scheduler=scheduler, + node=node, + repo_root=repo_root, + skip_multi_attempt_replay=skip_multi_attempt_replay, + ) + + logger.info( + "Loaded tree from %s: %d nodes, %d observations", + nodes_dir, + scheduler.n_nodes(), + scheduler.n_observations(), + ) + return scheduler + + +def _topological_order(nodes: list[HarnessNode]) -> list[HarnessNode]: + """Return nodes ordered so every node's ``parent_id`` appears before + it. Uses BFS from roots; raises on cycles or orphan parents.""" + by_id = {n.node_id: n for n in nodes} + + # Pre-check: every non-root node must reference a parent that exists + # in the node set. (Otherwise BFS would silently skip it and fall + # through to the "possible cycle" branch with a less useful message.) + for n in nodes: + if n.parent_id is not None and n.parent_id not in by_id: + raise ValueError(f"Node {n.node_id!r} references parent_id {n.parent_id!r} which is not in the node set") + + children_of: dict[Optional[str], list[HarnessNode]] = {} + for n in nodes: + children_of.setdefault(n.parent_id, []).append(n) + + roots = children_of.get(None, []) + if not roots: + raise ValueError("No root node (parent_id=None) found") + + seen: set[str] = set() + ordered: list[HarnessNode] = [] + queue: deque[HarnessNode] = deque(roots) + while queue: + n = queue.popleft() + if n.node_id in seen: + continue + seen.add(n.node_id) + ordered.append(n) + for child in children_of.get(n.node_id, []): + queue.append(child) + if len(ordered) != len(nodes): + missing = {n.node_id for n in nodes} - seen + raise ValueError(f"Topological sort incomplete — possible cycle. Missing: {missing}") + return ordered + + +def _record_outcomes( + *, + scheduler: TreeAwareTaskScheduler, + node: HarnessNode, + repo_root: Path, + skip_multi_attempt_replay: bool, +) -> None: + """Replay all known outcomes for ``node`` into ``scheduler``.""" + if node.eval is None: + return + + dense = node.eval.dense_signals or {} + + # ── 1. Multi-attempt replay (root k=3 case) ── + replay_dir_rel = dense.get("k_attempt_replay_dir") + if isinstance(replay_dir_rel, str) and replay_dir_rel and not skip_multi_attempt_replay: + replay_dir = repo_root / replay_dir_rel + if replay_dir.is_dir(): + _replay_k_attempts( + scheduler=scheduler, + node_id=node.node_id, + trial_dir=replay_dir, + task_pool=set(scheduler._task_ids), + ) + # Skip the per_task_results union — we just replayed every attempt + else: + logger.warning( + "node %s: k_attempt_replay_dir does not exist: %s — falling back to per_task_results union", + node.node_id, + replay_dir, + ) + _replay_primary(scheduler, node) + else: + # ── 1b. Primary eval (single attempt per task) ── + _replay_primary(scheduler, node) + + # ── 2. Secondary evals ── + n_secondary = int(dense.get("secondary_eval_count", 0) or 0) + for i in range(n_secondary): + label = dense.get(f"secondary_eval_{i}_label") + path_rel = dense.get(f"secondary_eval_{i}_path") + fmt = dense.get(f"secondary_eval_{i}_format") + if not (isinstance(path_rel, str) and isinstance(fmt, str)): + logger.warning( + "node %s secondary_eval_%d missing path/format — skipping", + node.node_id, + i, + ) + continue + path = repo_root / path_rel + if not path.is_file(): + logger.warning( + "node %s secondary_eval_%d path missing: %s — skipping", + node.node_id, + i, + path, + ) + continue + try: + data = json.loads(path.read_text()) + except json.JSONDecodeError: + logger.warning( + "node %s secondary_eval_%d not valid JSON: %s — skipping", + node.node_id, + i, + path, + ) + continue + _apply_secondary_format(scheduler, node.node_id, label, fmt, data) + + +def _replay_primary(scheduler: TreeAwareTaskScheduler, node: HarnessNode) -> None: + """Add the primary eval's per_task_results as one outcome each.""" + assert node.eval is not None + pool = set(scheduler._task_ids) + for task_id, result in node.eval.per_task_results.items(): + if task_id not in pool: + logger.debug( + "node %s: per_task task %s not in task pool, skipping", + node.node_id, + task_id, + ) + continue + scheduler.add_outcome(node.node_id, task_id, result.pass_outcome) + + +def _replay_k_attempts( + *, + scheduler: TreeAwareTaskScheduler, + node_id: str, + trial_dir: Path, + task_pool: set[str], +) -> None: + """Walk a legacy-runner trial dir (each subdir = one trial = one task × + attempt) and replay each attempt as a separate outcome.""" + n_replayed = 0 + for result_json in trial_dir.glob("*/result.json"): + try: + d = json.loads(result_json.read_text()) + except json.JSONDecodeError: + continue + task_name = d.get("task_name") + if not task_name or task_name not in task_pool: + continue + reward = (d.get("verifier_result") or {}).get("rewards", {}).get("reward", 0.0) + passed = bool(reward) and reward > 0 + scheduler.add_outcome(node_id, task_name, passed) + n_replayed += 1 + logger.info( + "node %s: replayed %d k-attempt outcomes from %s", + node_id, + n_replayed, + trial_dir, + ) + + +def _apply_secondary_format( + scheduler: TreeAwareTaskScheduler, + node_id: str, + label: Optional[str], + fmt: str, + data: dict, +) -> None: + pool = set(scheduler._task_ids) + n_added = 0 + if fmt == "task_dict_passed": + # {task_id: {passed: bool, ...}} + for task_id, v in data.items(): + if task_id not in pool: + continue + if isinstance(v, dict) and "passed" in v: + scheduler.add_outcome(node_id, task_id, bool(v["passed"])) + n_added += 1 + elif fmt == "rescue_k3_attempts": + # {task: , attempts: [{passed: bool, ...}, ...]} + task_id = data.get("task") + if task_id and task_id in pool: + for attempt in data.get("attempts", []): + if isinstance(attempt, dict) and "passed" in attempt: + scheduler.add_outcome(node_id, task_id, bool(attempt["passed"])) + n_added += 1 + elif fmt == "per_task_passed_block": + # {task_id: {passed: bool, ...}} (same shape as task_dict_passed) + for task_id, v in data.items(): + if task_id not in pool: + continue + if isinstance(v, dict) and "passed" in v: + scheduler.add_outcome(node_id, task_id, bool(v["passed"])) + n_added += 1 + else: + logger.warning( + "node %s secondary_eval %r unknown format: %r — skipping", + node_id, + label, + fmt, + ) + return + logger.info( + "node %s secondary_eval %r (%s): added %d outcomes", + node_id, + label, + fmt, + n_added, + ) + + +__all__ = ["load_tree_into_scheduler"] diff --git a/raven/evolver/tree/__init__.py b/raven/evolver/tree/__init__.py new file mode 100644 index 0000000..407f614 --- /dev/null +++ b/raven/evolver/tree/__init__.py @@ -0,0 +1,39 @@ +"""Evolver tree subsystem. + +Phase 1 (C1, this module): node schema + JSON round-trip. +Future C2: Git-backed physical state management. +Future C3: tree topology + traversal helpers. +""" + +from . import git_ops +from .node import ( + SCHEMA_VERSION, + AppliedPatch, + CandidatePatch, + EvalResult, + HarnessNode, + JudgeAnalysis, + NodeStatus, + PatchComponent, + PerTaskResult, + ProposedComponent, + SourceEvidence, +) +from .store import EvolverTreeStore, TreeView + +__all__ = [ + "SCHEMA_VERSION", + "AppliedPatch", + "CandidatePatch", + "EvalResult", + "EvolverTreeStore", + "HarnessNode", + "JudgeAnalysis", + "NodeStatus", + "PatchComponent", + "PerTaskResult", + "ProposedComponent", + "SourceEvidence", + "TreeView", + "git_ops", +] diff --git a/raven/evolver/tree/git_ops.py b/raven/evolver/tree/git_ops.py new file mode 100644 index 0000000..6bfc577 --- /dev/null +++ b/raven/evolver/tree/git_ops.py @@ -0,0 +1,454 @@ +"""Low-level Git command wrappers for the evolver tree (C2.1). + +These functions wrap ``subprocess`` calls to ``git`` so the evolver can +construct commits without depending on a particular Python Git library. +They are intentionally small, side-effect-explicit, and easy to mock in +tests via a custom ``repo_root``. + +Two operational categories: + +1. **Read** (cheap, no state mutation): + :func:`get_current_sha`, :func:`commit_exists`, :func:`get_tree_sha`, + :func:`get_commit_message`. + +2. **Write that does NOT touch the working tree**: + :func:`apply_patch_as_commit` uses Git plumbing (``read-tree`` / + ``apply --cached`` / ``write-tree`` / ``commit-tree``) so the + user's working directory is never modified. This is what + ``EvolverTreeStore.create_child_node`` uses to spawn child nodes. + +3. **Write that DOES create a working tree**: + :func:`create_worktree` / :func:`remove_worktree` use + ``git worktree`` to materialise a checkout of a specific commit + at a separate path — used later for evaluation runs. Not used by + commit construction. + +Why subprocess and not pygit2 / GitPython: +- Zero extra dependency +- Git plumbing semantics are stable and well-documented +- Easy to debug (we just print commands on failure) + +Error model: any non-zero exit code raises :class:`GitOpError` with the +exact command and stderr. Callers should not catch this except at the +top of a logical operation. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator, Optional + +# --------------------------------------------------------------------------- +# Exceptions +# --------------------------------------------------------------------------- + + +_EPHEMERAL_ROOT: Optional[Path] = None + + +def set_ephemeral_root(path: Optional[Path]) -> None: + """Route ephemeral worktree temp dirs under a run-scoped directory. + + By default they land in the system temp dir; the context managers clean + them on any Python exit (including Ctrl-C), but a hard kill (SIGKILL, + power loss) leaves them behind — still registered as worktrees of the + subject repo. Parking them under the run's work_dir makes leftovers + discoverable and lets the next launch sweep exactly its own garbage + without touching concurrent runs. ``None`` restores the system default. + """ + global _EPHEMERAL_ROOT + _EPHEMERAL_ROOT = None if path is None else Path(path) + + +def _ephemeral_dir() -> Optional[str]: + if _EPHEMERAL_ROOT is None: + return None + _EPHEMERAL_ROOT.mkdir(parents=True, exist_ok=True) + return str(_EPHEMERAL_ROOT) + + +class GitOpError(RuntimeError): + """Raised when a git command exits non-zero.""" + + def __init__(self, cmd: list[str], returncode: int, stderr: str, stdout: str = ""): + self.cmd = cmd + self.returncode = returncode + self.stderr = stderr + self.stdout = stdout + super().__init__( + f"git command failed (exit {returncode}): {' '.join(cmd)}\n" + f"stderr: {stderr.strip()}\n" + f"stdout: {stdout.strip()}" + ) + + +# --------------------------------------------------------------------------- +# Core runner +# --------------------------------------------------------------------------- + + +def _run( + repo_root: Path, + *args: str, + env: Optional[dict[str, str]] = None, + input_text: Optional[str] = None, + check: bool = True, +) -> str: + """Run ``git `` in ``repo_root`` and return stdout. + + :param env: extra env vars merged on top of os.environ + :param input_text: passed to stdin (used by ``git apply``) + :param check: raise :class:`GitOpError` on non-zero exit + """ + cmd = ["git", *args] + merged_env = {**os.environ, **(env or {})} + proc = subprocess.run( + cmd, + cwd=str(repo_root), + env=merged_env, + input=input_text, + capture_output=True, + text=True, + ) + if check and proc.returncode != 0: + raise GitOpError( + cmd=cmd, + returncode=proc.returncode, + stderr=proc.stderr, + stdout=proc.stdout, + ) + return proc.stdout + + +# --------------------------------------------------------------------------- +# Read operations +# --------------------------------------------------------------------------- + + +def get_current_sha(repo_root: Path) -> str: + """Return the SHA of the current HEAD.""" + return _run(repo_root, "rev-parse", "HEAD").strip() + + +def get_tree_sha(repo_root: Path, commit_sha: str) -> str: + """Return the tree SHA referenced by ``commit_sha``.""" + return _run(repo_root, "rev-parse", f"{commit_sha}^{{tree}}").strip() + + +def commit_exists(repo_root: Path, sha: str) -> bool: + """Return True iff ``sha`` is a known commit object in this repo.""" + try: + out = _run(repo_root, "cat-file", "-t", sha).strip() + except GitOpError: + return False + return out == "commit" + + +def get_commit_message(repo_root: Path, sha: str) -> str: + """Return the full commit message (subject + body) for ``sha``.""" + return _run(repo_root, "log", "-1", "--format=%B", sha) + + +def read_file_at(repo_root: Path, sha: str, rel_path: str) -> bytes: + """Return the raw bytes of ``rel_path`` as stored in commit ``sha``. + + Binary-safe (``_run`` decodes text; blobs may be arbitrary bytes). Raises + :class:`GitOpError` when the path does not exist in that commit. + """ + cmd = ["git", "cat-file", "blob", f"{sha}:{rel_path}"] + proc = subprocess.run(cmd, cwd=str(repo_root), capture_output=True) + if proc.returncode != 0: + raise GitOpError( + cmd=cmd, + returncode=proc.returncode, + stderr=proc.stderr.decode("utf-8", errors="replace"), + ) + return proc.stdout + + +# --------------------------------------------------------------------------- +# Write — construct commits WITHOUT touching the working tree +# --------------------------------------------------------------------------- + + +@contextmanager +def _temp_index() -> Iterator[Path]: + """Yield a path for a temporary git index file, cleaning up after.""" + fd, name = tempfile.mkstemp(prefix="evolver-idx-", suffix=".tmp") + os.close(fd) + path = Path(name) + # ``git read-tree`` requires the file not to exist yet; remove it. + if path.exists(): + path.unlink() + try: + yield path + finally: + if path.exists(): + path.unlink() + + +def apply_patch_as_commit( + repo_root: Path, + parent_sha: str, + unified_diff: str, + message: str, + *, + author_name: str = "evolver-bot", + author_email: str = "evolver@raven.local", +) -> str: + """Construct a child commit by applying ``unified_diff`` on top of + ``parent_sha``. **Does not touch the working tree or move HEAD.** + + Returns the SHA of the new commit. + + Algorithm (Git plumbing): + + 1. ``git read-tree parent_sha`` into a temp index file + 2. ``git apply --cached --index=`` the patch into that index + 3. ``git write-tree`` from that index → tree_sha + 4. ``git commit-tree tree_sha -p parent_sha -m message`` → child_sha + + The new commit is reachable only via its returned SHA — no branch + ref is updated. Callers (typically :meth:`EvolverTreeStore. + create_child_node`) record the SHA in + :attr:`HarnessNode.git_commit_sha` and may later create a ref + via :func:`create_branch`. + + If the patch fails to apply cleanly, raises :class:`GitOpError` + and no commit is created. + + :raises GitOpError: if any sub-step fails + :raises ValueError: if ``parent_sha`` doesn't exist locally + """ + if not commit_exists(repo_root, parent_sha): + raise ValueError(f"parent_sha {parent_sha!r} is not a known commit in {repo_root}") + + git_dir = (repo_root / ".git").resolve() + if not git_dir.exists(): + # Worktrees have .git as a file pointing to gitdir; fall back to + # asking git itself. + git_dir_str = _run(repo_root, "rev-parse", "--git-dir").strip() + git_dir = (repo_root / git_dir_str).resolve() + + with _temp_index() as idx: + # The custom index file is selected via GIT_INDEX_FILE env var + # for the duration of each plumbing call. + env = {"GIT_INDEX_FILE": str(idx)} + # Step 1: load parent tree into temp index + _run(repo_root, "read-tree", parent_sha, env=env) + # Step 2: apply the diff to the temp index (no working-tree touch) + _run( + repo_root, + "apply", + "--cached", + "--allow-empty", + "-", # read patch from stdin + env=env, + input_text=unified_diff, + ) + # Step 3: write a tree object from the temp index + tree_sha = _run(repo_root, "write-tree", env=env).strip() + + # Step 4: commit-tree to create the actual commit. Use environment + # variables for author / committer so the call doesn't depend on + # the global git config (which might not have user.name/email set + # on a CI box). + commit_env = { + "GIT_AUTHOR_NAME": author_name, + "GIT_AUTHOR_EMAIL": author_email, + "GIT_COMMITTER_NAME": author_name, + "GIT_COMMITTER_EMAIL": author_email, + } + child_sha = _run( + repo_root, + "commit-tree", + tree_sha, + "-p", + parent_sha, + "-m", + message, + env=commit_env, + ).strip() + return child_sha + + +def commit_files_as_child( + repo_root: Path, + parent_sha: str, + files: dict[str, bytes], + message: str, + *, + deletions: tuple[str, ...] = (), + author_name: str = "evolver-bot", + author_email: str = "evolver@raven.local", +) -> tuple[str, list[str]]: + """Commit a set of edited files as a child of ``parent_sha``. + + The "edit-then-commit" apply path: the driver edits files directly (weak + models do this far more reliably than emitting a valid unified diff), and we + capture the result as a real git commit off the parent — giving the node a + reproducible SHA, git ancestry, and an immutable-guardable changed-file list, + instead of a "sandbox" placeholder + live-tree mutation. + + ``files`` maps repo-relative paths to their full new bytes; ``deletions`` + lists repo-relative paths to remove. Materialises them into a detached + worktree at ``parent_sha``, stages everything, and ``commit-tree``s onto + ``parent_sha``. **The main working tree is never touched.** Returns + ``(child_sha, changed_paths)``. + """ + if not commit_exists(repo_root, parent_sha): + raise ValueError(f"parent_sha {parent_sha!r} is not a known commit in {repo_root}") + + with tempfile.TemporaryDirectory(prefix="evolver-edit-", dir=_ephemeral_dir()) as tmp: + wt = Path(tmp) / "wt" + create_worktree(repo_root, wt, parent_sha) + try: + for rel, data in files.items(): + dest = wt / rel + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(data if isinstance(data, bytes) else str(data).encode()) + for rel in deletions: + fp = wt / rel + if fp.exists(): + fp.unlink() + _run(wt, "add", "-A") + changed = [ + line for line in _run(wt, "diff", "--cached", "--name-only", parent_sha).splitlines() if line.strip() + ] + tree_sha = _run(wt, "write-tree").strip() + commit_env = { + "GIT_AUTHOR_NAME": author_name, + "GIT_AUTHOR_EMAIL": author_email, + "GIT_COMMITTER_NAME": author_name, + "GIT_COMMITTER_EMAIL": author_email, + } + child_sha = _run( + wt, + "commit-tree", + tree_sha, + "-p", + parent_sha, + "-m", + message, + env=commit_env, + ).strip() + finally: + remove_worktree(repo_root, wt) + return child_sha, changed + + +@contextmanager +def worktree_at(repo_root: Path, sha: str) -> Iterator[Path]: + """Yield an ephemeral detached worktree checked out at ``sha``, cleaned up + after. The eval-time counterpart of :func:`commit_files_as_child` — a scorer + runs against this checkout instead of the mutated live repo.""" + with tempfile.TemporaryDirectory(prefix="evolver-wt-", dir=_ephemeral_dir()) as tmp: + wt = Path(tmp) / "wt" + create_worktree(repo_root, wt, sha) + try: + yield wt + finally: + remove_worktree(repo_root, wt) + + +# --------------------------------------------------------------------------- +# Branch / ref management (lightweight; tag-style use only) +# --------------------------------------------------------------------------- + + +def create_ref(repo_root: Path, ref_name: str, sha: str) -> None: + """Point ``ref_name`` (e.g. ``refs/evolver/``) at ``sha``. + + Evolver-constructed commits are otherwise reachable only through SHAs + recorded in JSON (journal / node ledger), which ``git gc`` does not see — + an unreferenced candidate commit would be pruned after the default grace + period, breaking late worktree evals and the post-hoc sealed unseal. A + plain ref (not a branch: never checked out, invisible to ``git branch``) + anchors it. Overwrites if the ref already exists. + """ + _run(repo_root, "update-ref", ref_name, sha) + + +def delete_ref(repo_root: Path, ref_name: str) -> None: + """Remove a ref created by :func:`create_ref` (no-op if absent).""" + _run(repo_root, "update-ref", "-d", ref_name, check=False) + + +def create_branch(repo_root: Path, branch_name: str, sha: str) -> None: + """Create a branch named ``branch_name`` pointing at ``sha``. + + Used to give an evolver-constructed commit a human-readable ref so + it isn't garbage-collected. Does **not** check out the branch — + HEAD remains where it was. + """ + _run(repo_root, "branch", branch_name, sha) + + +def branch_exists(repo_root: Path, branch_name: str) -> bool: + """True iff a local branch with this name exists.""" + proc = subprocess.run( + ["git", "show-ref", "--verify", "--quiet", f"refs/heads/{branch_name}"], + cwd=str(repo_root), + capture_output=True, + text=True, + ) + return proc.returncode == 0 + + +def delete_branch(repo_root: Path, branch_name: str, *, force: bool = True) -> None: + """Delete a local branch. Force-delete by default since evolver + branches are throwaway anchors, not main-line work.""" + flag = "-D" if force else "-d" + _run(repo_root, "branch", flag, branch_name) + + +# --------------------------------------------------------------------------- +# Worktree management (for future eval runs; not used by commit construction) +# --------------------------------------------------------------------------- + + +def create_worktree(repo_root: Path, target_path: Path, sha: str) -> None: + """Create a temporary worktree at ``target_path`` checked out at + ``sha``. Caller must clean up with :func:`remove_worktree`. + + Used by the evaluation driver (later C-series tasks) to get a clean + checkout of a specific harness version for running tests, without + disturbing the main working tree. + """ + _run(repo_root, "worktree", "add", "--detach", str(target_path), sha) + + +def remove_worktree(repo_root: Path, target_path: Path, *, force: bool = True) -> None: + """Tear down a worktree previously created by :func:`create_worktree`.""" + args = ["worktree", "remove", str(target_path)] + if force: + args.insert(2, "--force") + _run(repo_root, *args) + # On rare interrupt cases git may leave the directory; clean up. + if target_path.exists(): + shutil.rmtree(target_path, ignore_errors=True) + + +__all__ = [ + "GitOpError", + "apply_patch_as_commit", + "commit_files_as_child", + "worktree_at", + "branch_exists", + "commit_exists", + "create_branch", + "create_ref", + "create_worktree", + "delete_branch", + "delete_ref", + "get_commit_message", + "get_current_sha", + "get_tree_sha", + "read_file_at", + "remove_worktree", +] diff --git a/raven/evolver/tree/node.py b/raven/evolver/tree/node.py new file mode 100644 index 0000000..812dce3 --- /dev/null +++ b/raven/evolver/tree/node.py @@ -0,0 +1,705 @@ +"""Harness node schema + JSON round-trip. + +Each evolver iteration produces a :class:`HarnessNode` — one version of +the harness with metadata about how it was made, how it performed, and +what next iterations might do. + +The structure (spec §12.2) has five blocks: + +1. **Identity**: ``node_id`` / ``parent_id`` / Git pointers / created_at. +2. **AppliedPatch**: what was changed relative to ``parent_id`` to create + this node — ``target_file``, ``patch_where``, ``patch_why``, diff, + reasoning, source-evidence pointers back to trajectories. +3. **EvalResult**: what was measured (bandit task subset, per-task + pass/fail, subset pass rate, dense signals, L1 alert count). +4. **JudgeAnalysis**: what the judge inferred from this node's + trajectories — counts of L1/L2/L3 seen + candidate patches the judge + proposes for future child nodes. +5. **Status**: where this node stands in the evolver lifecycle. + +Storage layout (spec §12.3): + +- Git commit handles the physical code state of the node (we don't + duplicate it here). +- One JSON file per node in ``evolver/nodes/.json`` holds the + metadata defined here. +- Trajectory data lives in ``evolver/trajectories/`` keyed by id; this + schema only carries trajectory *pointers*, never inline bodies. + +Why explicit ``to_dict`` / ``from_dict`` instead of stdlib ``dataclasses.asdict``? +Two reasons: + +- We need to deserialise enums (``PatchWhere``, ``PatchWhy``, + ``NodeStatus``) from their string values, which ``asdict`` doesn't help + with on load. +- Tuples like ``turn_range = (start, end)`` round-trip through JSON as + lists — we want to coerce back to tuple on load for type stability. + +Both are surface-level concerns, but the explicit code keeps the failure +modes loud (missing fields raise, unknown enums raise). +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path +from typing import Any, Optional, Union + +from raven.evolver.judge.schema import PatchWhere, PatchWhy + +# --------------------------------------------------------------------------- +# Status enum +# --------------------------------------------------------------------------- + + +class NodeStatus(str, Enum): + """Where a node stands in the evolver lifecycle. + + - ``active``: live in the tree, eligible for bandit-on-nodes selection. + - ``pruned_low_score``: still in archive (Git keeps the commit) but + bandit deprioritises it; rare for it to be re-selected. + - ``pruned_inert``: culled at the zero-GPU preflight (SOP §2 ③) — its + trigger predicate had zero hits over the historical trajectory corpus, + so the mechanism was never applied or evaluated. Distinct from + ``pruned_at_screen``: an inert death indicts the trigger's + reachability, not the mechanism body. + - ``pruned_at_screen``: culled at the K=1 anchor screen — its anchor-mean + pass@1 fell more than ``cull_threshold`` below vanilla, so it never ran + the full-set confirm (SOP §2 ⑤a: screen on the anchor, then return). + - ``pruned_at_confirm``: ran the full-set K=3 confirm but did not beat + vanilla / pass the gates (SOP §2 ⑥). + - ``blocked_l1``: judge flagged an L1 infra bug while evaluating this + node; evolver paused until human review. + - ``promoted_to_baseline``: this node became a new baseline (rare; + happens at major lift inflection points). + - ``archived_methodology_failure``: rewards from this node are + contaminated by a methodology defect; excluded from the scheduler + at load time so the bandit never learns from tainted outcomes. + - ``errored``: apply or evaluation raised for this candidate; the round + recorded the reason and skipped it rather than aborting (a single + candidate's crash must not sink the round). + """ + + active = "active" + pruned_low_score = "pruned_low_score" + pruned_inert = "pruned_inert" + pruned_at_screen = "pruned_at_screen" + pruned_at_confirm = "pruned_at_confirm" + blocked_l1 = "blocked_l1" + promoted_to_baseline = "promoted_to_baseline" + archived_methodology_failure = "archived-methodology-failure" + errored = "errored" + + +# --------------------------------------------------------------------------- +# Patch-related substructures +# --------------------------------------------------------------------------- + + +@dataclass +class SourceEvidence: + """A pointer into a trajectory used as evidence for a patch. + + A single applied patch may cite multiple trajectories (e.g., five + trajectories all hit ``repetition_breaker`` at turns 20-30); each is + one ``SourceEvidence``. + """ + + trajectory_id: str + turn_range: tuple[int, int] + finding: str # one-line summary of what the evidence shows + + def to_dict(self) -> dict[str, Any]: + return { + "trajectory_id": self.trajectory_id, + "turn_range": list(self.turn_range), + "finding": self.finding, + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "SourceEvidence": + tr = d.get("turn_range") + if not isinstance(tr, (list, tuple)) or len(tr) != 2: + raise ValueError(f"SourceEvidence.turn_range must be 2-element list/tuple, got {tr!r}") + return cls( + trajectory_id=_require(d, "trajectory_id", "SourceEvidence"), + turn_range=(int(tr[0]), int(tr[1])), + finding=_require(d, "finding", "SourceEvidence"), + ) + + +@dataclass +class PatchComponent: + """One independently-rollbackable piece of a multi-file patch. + + A patch may bundle several file edits that together implement one + semantic improvement (e.g., add a new hook file + register it in a + config + reference it from a prompt template). Each such file edit + is one ``PatchComponent``. The bundle hangs together via the + ``depends_on`` graph: if ``comp_3`` lists ``comp_1`` as dependency, + ``comp_3`` MUST be applied after ``comp_1`` (and cannot survive in + isolation when ``comp_1`` is dropped during bisect). + + Component-level bisect (spec §18.5.1.x): when an applied patch + causes regression, the evolver tries dropping subsets of components + (respecting ``depends_on``) and keeping the highest-quality subset + that still beats the parent. + + For the simple "1 file, 1 fix" case, an ``AppliedPatch`` carries + exactly one ``PatchComponent`` — the multi-component machinery + becomes a no-op. + """ + + component_id: str # "comp_1" / "comp_2" — unique within one patch + target_file: str # repo-relative path + diff: str # unified diff for THIS file only + rationale: str # what this component does + depends_on: list[str] = field(default_factory=list) # other component_ids + + def to_dict(self) -> dict[str, Any]: + return { + "component_id": self.component_id, + "target_file": self.target_file, + "diff": self.diff, + "rationale": self.rationale, + "depends_on": list(self.depends_on), + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "PatchComponent": + return cls( + component_id=_require(d, "component_id", "PatchComponent"), + target_file=_require(d, "target_file", "PatchComponent"), + diff=_require(d, "diff", "PatchComponent"), + rationale=_require(d, "rationale", "PatchComponent"), + depends_on=list(d.get("depends_on") or []), + ) + + +@dataclass +class AppliedPatch: + """The actual patch that was applied to the parent to create this node. + + Multi-component design (spec §12.2, updated 2026-05-28): + A patch is a non-empty ordered list of ``PatchComponent``. + Most patches have exactly one component (single-file edit). Multi-file + coupled patches use ``components`` with explicit ``depends_on`` so the + evolver can do component-level bisect on regression (§18.5.1.x). + + Convenience properties for the common single-component case: + - ``target_file`` returns the first component's file + - ``diff`` returns the concatenated diff across components + + The canonical code state still lives in the node's Git commit; + components describe *how* the diff is logically structured. + """ + + patch_where: PatchWhere + patch_why: PatchWhy + components: list[PatchComponent] # >= 1 + overall_reasoning: str # judge's reasoning at the whole-patch level + source_evidence: list[SourceEvidence] = field(default_factory=list) + patch_why_extra: Optional[str] = None # only when patch_why=other + + # Bisect bookkeeping (spec §18.5.1.x). Set when a patch was rescued + # from regression by dropping some components. + partial: bool = False + dropped_components: list[str] = field(default_factory=list) + + def __post_init__(self) -> None: + if self.patch_why == PatchWhy.other and not self.patch_why_extra: + raise ValueError("AppliedPatch with patch_why=other requires non-empty patch_why_extra") + if not self.components and self.patch_where != PatchWhere.control: + raise ValueError("AppliedPatch requires at least one component") + # Component ids must be unique within the patch + ids = [c.component_id for c in self.components] + if len(ids) != len(set(ids)): + raise ValueError(f"AppliedPatch components have duplicate component_id: {ids}") + # depends_on references must resolve to existing component_ids + valid_ids = set(ids) + for c in self.components: + for dep in c.depends_on: + if dep not in valid_ids: + raise ValueError( + f"PatchComponent {c.component_id!r} depends_on {dep!r} which is not a sibling component" + ) + + @property + def target_file(self) -> Optional[str]: + """Primary file = first component's file (single-component default). + + Returns None for control-arm nodes that carry no components. + """ + return self.components[0].target_file if self.components else None + + @property + def diff(self) -> str: + """Concatenated diff across all components (Git-applicable as a whole).""" + return "\n".join(c.diff for c in self.components) + + def to_dict(self) -> dict[str, Any]: + return { + "patch_where": self.patch_where.value, + "patch_why": self.patch_why.value, + "patch_why_extra": self.patch_why_extra, + "overall_reasoning": self.overall_reasoning, + "components": [c.to_dict() for c in self.components], + "source_evidence": [e.to_dict() for e in self.source_evidence], + "partial": self.partial, + "dropped_components": list(self.dropped_components), + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "AppliedPatch": + components_raw = _require(d, "components", "AppliedPatch") + patch_where_raw = d.get("patch_where", "") + is_control = patch_where_raw == PatchWhere.control.value + if not isinstance(components_raw, list) or (not components_raw and not is_control): + raise ValueError("AppliedPatch.components must be a non-empty list") + return cls( + patch_where=_coerce_enum( + _require(d, "patch_where", "AppliedPatch"), + PatchWhere, + "patch_where", + ), + patch_why=_coerce_enum( + _require(d, "patch_why", "AppliedPatch"), + PatchWhy, + "patch_why", + ), + components=[PatchComponent.from_dict(c) for c in components_raw], + overall_reasoning=_require(d, "overall_reasoning", "AppliedPatch"), + source_evidence=[SourceEvidence.from_dict(e) for e in (d.get("source_evidence") or [])], + patch_why_extra=d.get("patch_why_extra"), + partial=bool(d.get("partial", False)), + dropped_components=list(d.get("dropped_components") or []), + ) + + +# --------------------------------------------------------------------------- +# Evaluation substructure +# --------------------------------------------------------------------------- + + +@dataclass +class PerTaskResult: + """One task's outcome under this node. + + ``pass_outcome`` rather than ``pass`` to avoid the Python keyword. + ``turns`` and ``elapsed_sec`` are optional because the eval driver + may or may not surface them depending on the task. + """ + + pass_outcome: bool + turns: Optional[int] = None + elapsed_sec: Optional[float] = None + + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {"pass": self.pass_outcome} + if self.turns is not None: + d["turns"] = self.turns + if self.elapsed_sec is not None: + d["elapsed_sec"] = self.elapsed_sec + return d + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "PerTaskResult": + if "pass" not in d: + raise ValueError("PerTaskResult missing required 'pass'") + return cls( + pass_outcome=bool(d["pass"]), + turns=d.get("turns"), + elapsed_sec=d.get("elapsed_sec"), + ) + + +@dataclass +class EvalResult: + """Aggregated evaluation of a single node. + + ``dense_signals`` is a flexible bag — current planned keys + (avg_token_usage / avg_turn_count / redundancy_rate / + test_run_frequency) are documented but not enforced, so future signals + can be added without schema migration. + """ + + bandit_tasks_chosen: list[str] + per_task_results: dict[str, PerTaskResult] + subset_pass_rate: float + dense_signals: dict[str, float] = field(default_factory=dict) + l1_alert_count: int = 0 + + def __post_init__(self) -> None: + if not 0.0 <= self.subset_pass_rate <= 1.0: + raise ValueError(f"subset_pass_rate must be in [0,1], got {self.subset_pass_rate}") + if self.l1_alert_count < 0: + raise ValueError(f"l1_alert_count must be >= 0, got {self.l1_alert_count}") + + def to_dict(self) -> dict[str, Any]: + return { + "bandit_tasks_chosen": list(self.bandit_tasks_chosen), + "per_task_results": {k: v.to_dict() for k, v in self.per_task_results.items()}, + "subset_pass_rate": self.subset_pass_rate, + "dense_signals": dict(self.dense_signals), + "l1_alert_count": self.l1_alert_count, + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "EvalResult": + return cls( + bandit_tasks_chosen=list(_require(d, "bandit_tasks_chosen", "EvalResult")), + per_task_results={ + k: PerTaskResult.from_dict(v) for k, v in _require(d, "per_task_results", "EvalResult").items() + }, + subset_pass_rate=float(_require(d, "subset_pass_rate", "EvalResult")), + dense_signals=dict(d.get("dense_signals") or {}), + l1_alert_count=int(d.get("l1_alert_count", 0)), + ) + + +# --------------------------------------------------------------------------- +# Judge analysis substructure +# --------------------------------------------------------------------------- + + +@dataclass +class ProposedComponent: + """A proposed component of a future patch (judge output, no diff yet). + + Counterpart to :class:`PatchComponent`: the judge describes WHAT to + change (target_file + natural-language summary) but doesn't write + the actual diff. The mutation operator (e.g., GEPA library) later + turns each ``ProposedComponent`` into a concrete ``PatchComponent`` + with a unified diff. + """ + + component_id: str + target_file: str + summary: str # natural language description of the proposed change + depends_on: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "component_id": self.component_id, + "target_file": self.target_file, + "summary": self.summary, + "depends_on": list(self.depends_on), + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "ProposedComponent": + return cls( + component_id=_require(d, "component_id", "ProposedComponent"), + target_file=_require(d, "target_file", "ProposedComponent"), + summary=_require(d, "summary", "ProposedComponent"), + depends_on=list(d.get("depends_on") or []), + ) + + +@dataclass +class CandidatePatch: + """A patch the judge proposes for a *future* child node. + + These are not yet applied — the bandit-on-nodes / bandit-on-WHY will + later pick one of these from this node's pool to spawn an actual child + (with its own ``AppliedPatch``). The bridge from candidate to applied + is the mutation operator producing the unified diff. + + Multi-component design (spec §12.2, updated 2026-05-28): a candidate + patch carries one or more :class:`ProposedComponent`. The judge is + encouraged to keep N=1 for simple single-file fixes, and to use N>1 + only when the change is structurally coupled (e.g., new hook file + + config registration). ``depends_on`` between components is honored + by both the mutation operator and any component-level bisect during + evolution. + """ + + patch_where: PatchWhere + patch_why: PatchWhy + components: list[ProposedComponent] # >= 1 + overall_reasoning: str + source_trajectory_id: str + patch_why_extra: Optional[str] = None # only when patch_why=other + + def __post_init__(self) -> None: + if self.patch_why == PatchWhy.other and not self.patch_why_extra: + raise ValueError("CandidatePatch with patch_why=other requires non-empty patch_why_extra") + if not self.components: + raise ValueError("CandidatePatch requires at least one component") + ids = [c.component_id for c in self.components] + if len(ids) != len(set(ids)): + raise ValueError(f"CandidatePatch components have duplicate component_id: {ids}") + valid_ids = set(ids) + for c in self.components: + for dep in c.depends_on: + if dep not in valid_ids: + raise ValueError( + f"ProposedComponent {c.component_id!r} depends_on {dep!r} which is not a sibling component" + ) + + @property + def target_file(self) -> str: + """Primary file = first component's file (single-component default).""" + return self.components[0].target_file + + def to_dict(self) -> dict[str, Any]: + return { + "patch_where": self.patch_where.value, + "patch_why": self.patch_why.value, + "patch_why_extra": self.patch_why_extra, + "overall_reasoning": self.overall_reasoning, + "components": [c.to_dict() for c in self.components], + "source_trajectory_id": self.source_trajectory_id, + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "CandidatePatch": + components_raw = _require(d, "components", "CandidatePatch") + if not isinstance(components_raw, list) or not components_raw: + raise ValueError("CandidatePatch.components must be a non-empty list") + return cls( + patch_where=_coerce_enum( + _require(d, "patch_where", "CandidatePatch"), + PatchWhere, + "patch_where", + ), + patch_why=_coerce_enum( + _require(d, "patch_why", "CandidatePatch"), + PatchWhy, + "patch_why", + ), + components=[ProposedComponent.from_dict(c) for c in components_raw], + overall_reasoning=_require(d, "overall_reasoning", "CandidatePatch"), + source_trajectory_id=_require(d, "source_trajectory_id", "CandidatePatch"), + patch_why_extra=d.get("patch_why_extra"), + ) + + +@dataclass +class JudgeAnalysis: + """The judge's verdict on this node's trajectories. + + ``issue_types_seen`` counts L1 / L2 / L3 occurrences across the + node's evaluated trajectories — used by the L1 routing layer to + decide whether to pause the evolver (high L1 count) and by + bandit-on-WHY to rebalance pathology coverage. + + ``candidate_patches`` is the menu the evolver chooses from for the + next iteration of child-node creation. + """ + + issue_types_seen: dict[str, int] = field(default_factory=dict) + candidate_patches: list[CandidatePatch] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "issue_types_seen": dict(self.issue_types_seen), + "candidate_patches": [p.to_dict() for p in self.candidate_patches], + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "JudgeAnalysis": + return cls( + issue_types_seen=dict(d.get("issue_types_seen") or {}), + candidate_patches=[CandidatePatch.from_dict(p) for p in (d.get("candidate_patches") or [])], + ) + + +# --------------------------------------------------------------------------- +# Top-level node +# --------------------------------------------------------------------------- + + +SCHEMA_VERSION = 1 # bump when breaking change to JSON layout + + +@dataclass +class HarnessNode: + """One harness version in the evolver tree. + + Three sub-blocks (``patch`` / ``eval`` / ``judge_analysis``) are all + ``Optional`` — they get populated as the node moves through its + lifecycle: + + - Root node: all three are None (no parent, no eval yet). + - Right after creation: ``patch`` set; ``eval`` and ``judge_analysis`` + still None. + - After evaluation: ``eval`` populated. + - After judge runs on trajectories: ``judge_analysis`` populated. + """ + + node_id: str + parent_id: Optional[str] + git_commit_sha: str + git_branch: str + created_at: str # ISO 8601 UTC + created_at_iter: int + # core_version (spec §22.5.4): the immutable-kernel version this node was + # created under. Used for cohort-controlled cross-experiment comparison — + # nodes with different ``core_version`` must not be directly compared on + # eval numbers. Read from ``raven.__version__`` at + # evolver startup and injected into each child node. Defaults to + # ``"unknown"`` so old JSON files without this field deserialise (with a + # warning logged at the call site, not here). + core_version: str = "unknown" + status: NodeStatus = NodeStatus.active + patch: Optional[AppliedPatch] = None + eval: Optional[EvalResult] = None + judge_analysis: Optional[JudgeAnalysis] = None + + def __post_init__(self) -> None: + if self.created_at_iter < 0: + raise ValueError(f"created_at_iter must be >= 0, got {self.created_at_iter}") + if not self.node_id: + raise ValueError("node_id must be non-empty") + if not self.git_commit_sha: + raise ValueError("git_commit_sha must be non-empty") + if not self.core_version: + # Empty string isn't valid — caller must pass either a real + # version like "1.0.0" or the sentinel "unknown". + raise ValueError("core_version must be non-empty (use 'unknown' if not available)") + # Root invariant: no parent → no applied patch + if self.parent_id is None and self.patch is not None: + raise ValueError("root node (parent_id=None) must not carry an AppliedPatch") + + @staticmethod + def utc_now() -> str: + """ISO 8601 UTC timestamp for ``created_at`` fields.""" + return datetime.now(timezone.utc).isoformat() + + # -- serialisation ------------------------------------------------------ + + def to_dict(self) -> dict[str, Any]: + return { + "_schema_version": SCHEMA_VERSION, + "node_id": self.node_id, + "parent_id": self.parent_id, + "git_commit_sha": self.git_commit_sha, + "git_branch": self.git_branch, + "created_at": self.created_at, + "created_at_iter": self.created_at_iter, + "core_version": self.core_version, + "status": self.status.value, + "patch": self.patch.to_dict() if self.patch else None, + "eval": self.eval.to_dict() if self.eval else None, + "judge_analysis": self.judge_analysis.to_dict() if self.judge_analysis else None, + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "HarnessNode": + ver = d.get("_schema_version", 1) + if ver != SCHEMA_VERSION: + raise ValueError( + f"HarnessNode schema version mismatch: file has {ver!r}, " + f"code understands {SCHEMA_VERSION}. Migration not yet implemented." + ) + patch_raw = d.get("patch") + eval_raw = d.get("eval") + judge_raw = d.get("judge_analysis") + # Missing ``core_version`` is tolerated for backwards compat with + # JSON files written before §22.5 — the node falls back to + # ``"unknown"`` and downstream cohort analysis must exclude or + # specially mark these nodes (spec §22.5.4). + return cls( + node_id=_require(d, "node_id", "HarnessNode"), + parent_id=d.get("parent_id"), + git_commit_sha=_require(d, "git_commit_sha", "HarnessNode"), + git_branch=_require(d, "git_branch", "HarnessNode"), + created_at=_require(d, "created_at", "HarnessNode"), + created_at_iter=int(_require(d, "created_at_iter", "HarnessNode")), + core_version=d.get("core_version", "unknown"), + status=_coerce_enum( + d.get("status", NodeStatus.active.value), + NodeStatus, + "status", + ), + patch=AppliedPatch.from_dict(patch_raw) if patch_raw else None, + eval=EvalResult.from_dict(eval_raw) if eval_raw else None, + judge_analysis=JudgeAnalysis.from_dict(judge_raw) if judge_raw else None, + ) + + @staticmethod + def current_core_version() -> str: + """Read the current immutable-kernel version from + :mod:`raven` version (spec §22.5). + + Evolver code should call this when constructing new + :class:`HarnessNode` instances so the node carries the kernel + version it was created under. Falls back to ``"unknown"`` if + the import fails (shouldn't happen in normal operation). + """ + try: + from raven.__core_version__ import __version__ + + return __version__ + except ImportError: + return "unknown" + + def save(self, path: Union[str, Path]) -> None: + """Write this node's metadata to ``path`` as pretty JSON. + + Parent dirs are created on demand so callers can pass + ``evolver/nodes/.json`` without pre-mkdir. + """ + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + # Write through a temp + rename so an interrupted write doesn't + # leave a half-formed JSON file — common pattern for crash-safe + # file replacement on POSIX. + tmp = p.with_suffix(p.suffix + ".tmp") + tmp.write_text( + json.dumps(self.to_dict(), indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + tmp.replace(p) + + @classmethod + def load(cls, path: Union[str, Path]) -> "HarnessNode": + p = Path(path) + raw = json.loads(p.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError(f"node file {p} top-level must be a JSON object, got {type(raw).__name__}") + return cls.from_dict(raw) + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _require(d: dict[str, Any], key: str, what: str) -> Any: + """Pull a required key from ``d`` or raise with a contextual message.""" + if key not in d: + raise ValueError(f"{what} missing required field {key!r}") + return d[key] + + +def _coerce_enum(value: Any, enum_cls: type, field_name: str) -> Any: + """Coerce a string into ``enum_cls`` or raise with valid options.""" + if isinstance(value, enum_cls): + return value + if not isinstance(value, str): + raise ValueError(f"field {field_name!r} must be a string, got {type(value).__name__}") + try: + return enum_cls(value) + except ValueError as exc: + valid = [m.value for m in enum_cls] + raise ValueError(f"field {field_name!r}={value!r} not one of {valid}") from exc + + +__all__ = [ + "AppliedPatch", + "CandidatePatch", + "EvalResult", + "HarnessNode", + "JudgeAnalysis", + "NodeStatus", + "PatchComponent", + "PerTaskResult", + "ProposedComponent", + "SCHEMA_VERSION", + "SourceEvidence", +] diff --git a/raven/evolver/tree/store.py b/raven/evolver/tree/store.py new file mode 100644 index 0000000..9f6378a --- /dev/null +++ b/raven/evolver/tree/store.py @@ -0,0 +1,348 @@ +"""Evolver tree persistence orchestrator (C2.2 + C2.3 + C2.4). + +Decouples HarnessNode metadata (JSON files in ``evolver/nodes/``) +from physical code state (Git commits inside the host repo). The +linkage is :attr:`HarnessNode.git_commit_sha`. + +This module is the **only** place that should: + +- write / read node JSON metadata +- atomically tie a new git commit to a new HarnessNode + +Spec reference: §12.3 storage layout, §12.2 node schema, §22.7 +path_guard integration. + +Storage layout (spec §12.3): + +.. code-block:: + + raven repo (Git) ← code state, white-listed via Git + ├─ commits referenced by HarnessNode.git_commit_sha + └─ optional refs (created by EvolverTreeStore.create_child_node + when caller passes a branch_name) + + / ← typically ``evolver/`` under repo + ├─ nodes/ + │ ├─ root-.json + │ ├─ v1-.json + │ └─ ... + └─ (other files like trajectories/, tree.json — out of scope here) + +The store does **not** own trajectory files or the WHERE×WHY archive — +those are separate concerns landing in later modules. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional, Union + +from raven.evolver.applier import assert_patch_allowed +from raven.evolver.tree import git_ops +from raven.evolver.tree.node import ( + AppliedPatch, + HarnessNode, + NodeStatus, +) + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Read-only topology snapshot +# --------------------------------------------------------------------------- + + +@dataclass +class TreeView: + """A read-only snapshot of the evolver tree topology. + + Built by :meth:`EvolverTreeStore.build_tree`. Subsequent saves do + not update this view — call ``build_tree`` again to refresh. + + Invariants (enforced at construction time by ``build_tree``): + + - Exactly one root (``parent_id is None``) when ``root_id`` is set. + If no nodes have ``parent_id=None``, ``root_id`` is None and + ``orphans`` lists the dangling node IDs. + - ``children_of[parent_id]`` lists every node whose ``parent_id`` + equals that parent. + - Every key in ``children_of`` is a known node id, plus possibly + ``None`` for the "children-of-root-set" entry (kept under the + root's own id, not ``None``). + """ + + nodes: dict[str, HarnessNode] + children_of: dict[str, list[str]] # parent_id → child node_ids + root_id: Optional[str] + orphans: list[str] = field(default_factory=list) + + # ---- traversal helpers --------------------------------------------------- + + def descendants(self, node_id: str) -> list[str]: + """Return all node ids in the subtree rooted at ``node_id`` + (depth-first, ``node_id`` itself not included).""" + out: list[str] = [] + stack = list(self.children_of.get(node_id, [])) + while stack: + nid = stack.pop() + out.append(nid) + stack.extend(self.children_of.get(nid, [])) + return out + + def ancestry(self, node_id: str) -> list[str]: + """Return the chain of ancestor ids from ``node_id`` (exclusive) + up to the root (inclusive), in walking order.""" + out: list[str] = [] + cur = self.nodes.get(node_id) + if cur is None: + return out + while cur.parent_id is not None: + out.append(cur.parent_id) + cur = self.nodes.get(cur.parent_id) + if cur is None: + # parent_id points at an unknown node (dangling) + break + return out + + def __len__(self) -> int: + return len(self.nodes) + + +# --------------------------------------------------------------------------- +# Store +# --------------------------------------------------------------------------- + + +class EvolverTreeStore: + """File-system + Git-backed persistence for the evolver tree. + + Typical lifecycle: + + .. code-block:: python + + store = EvolverTreeStore( + repo_root=Path("/path/to/raven"), + nodes_dir=Path("/path/to/raven/evolver/nodes"), + ) + store.save_node(root) + child = store.create_child_node( + parent_node_id=root.node_id, + patch=applied_patch, + iter_index=1, + commit_message="[auto-evolver] hook_new: repetition_breaker", + ) + view = store.build_tree() + for d in view.descendants(root.node_id): + print(d) + """ + + NODE_FILE_SUFFIX = ".json" + + def __init__( + self, + repo_root: Union[str, Path], + nodes_dir: Union[str, Path], + ) -> None: + self.repo_root = Path(repo_root).resolve() + self.nodes_dir = Path(nodes_dir).resolve() + self.nodes_dir.mkdir(parents=True, exist_ok=True) + + # ---- single-node IO ---------------------------------------------------- + + def _node_path(self, node_id: str) -> Path: + return self.nodes_dir / f"{node_id}{self.NODE_FILE_SUFFIX}" + + def save_node(self, node: HarnessNode) -> Path: + """Write ``node``'s metadata JSON to ``nodes_dir/.json``. + + Atomic on POSIX (temp + rename, inherited from + :meth:`HarnessNode.save`). Idempotent — overwriting an existing + file with the same node id is allowed. + """ + path = self._node_path(node.node_id) + node.save(path) + return path + + def load_node(self, node_id: str) -> HarnessNode: + """Load one node by id. Raises :class:`FileNotFoundError` + if the JSON is missing.""" + path = self._node_path(node_id) + if not path.exists(): + raise FileNotFoundError(f"node {node_id!r} not found in {self.nodes_dir}") + return HarnessNode.load(path) + + def has_node(self, node_id: str) -> bool: + return self._node_path(node_id).exists() + + # ---- multi-node IO ----------------------------------------------------- + + def load_all_nodes(self) -> list[HarnessNode]: + """Load every node JSON in ``nodes_dir``. Sorted by file name + for deterministic test ordering. Files that fail to parse are + skipped with a warning log (they could be in-progress writes + or schema-version mismatches).""" + out: list[HarnessNode] = [] + for jpath in sorted(self.nodes_dir.glob(f"*{self.NODE_FILE_SUFFIX}")): + try: + out.append(HarnessNode.load(jpath)) + except Exception as exc: # pragma: no cover - defensive + logger.warning("Skipping unloadable node file %s: %s", jpath, exc) + return out + + def build_tree(self) -> TreeView: + """Load all nodes and assemble a :class:`TreeView`. + + Topology rules: + + - A node with ``parent_id=None`` is the root. If multiple roots + exist (shouldn't happen but defensible) the *first by node_id* + wins; the others are reported in ``orphans``. + - A node whose ``parent_id`` is not in the loaded set is added + to ``orphans`` (dangling parent pointer). + - ``children_of`` is built only for known parents. + """ + nodes_list = self.load_all_nodes() + nodes = {n.node_id: n for n in nodes_list} + children_of: dict[str, list[str]] = {nid: [] for nid in nodes} + root_candidates: list[str] = [] + orphans: list[str] = [] + + for n in sorted(nodes_list, key=lambda x: x.node_id): + if n.parent_id is None: + root_candidates.append(n.node_id) + continue + if n.parent_id not in nodes: + orphans.append(n.node_id) + continue + children_of[n.parent_id].append(n.node_id) + + root_id: Optional[str] = None + if root_candidates: + root_id = root_candidates[0] + # Any additional roots are reported as orphans-by-policy + if len(root_candidates) > 1: + logger.warning( + "Multiple root nodes detected; keeping %s, treating %s as orphans", + root_id, + root_candidates[1:], + ) + orphans.extend(root_candidates[1:]) + + # Sort each children list for determinism + for k in children_of: + children_of[k].sort() + + return TreeView( + nodes=nodes, + children_of=children_of, + root_id=root_id, + orphans=sorted(orphans), + ) + + # ---- atomic child creation (C2.4, in next step) ------------------------ + # See create_child_node below for the apply-patch+commit+save atomic op. + + def create_child_node( + self, + parent_node_id: str, + patch: AppliedPatch, + iter_index: int, + commit_message: str, + *, + git_branch: Optional[str] = None, + create_branch_ref: bool = False, + author_name: str = "evolver-bot", + author_email: str = "evolver@raven.local", + guard_immutable: bool = True, + ) -> HarnessNode: + """Atomic operation: apply ``patch`` on top of the parent's + git commit, create a child :class:`HarnessNode`, and persist + its JSON. + + On failure at any step, the file system + git ref state is + left as it was before the call (caveat: a dangling commit + without a JSON metadata file is unreachable by node id and + will be garbage-collected by Git eventually). + + :param parent_node_id: id of the parent node (must already be saved) + :param patch: the :class:`AppliedPatch` whose unified diff will + be applied + :param iter_index: evolver iteration number; folded into the + generated child node_id + :param commit_message: passed verbatim to git commit-tree + :param git_branch: branch name to record on the child node. + Inherited from the parent if not given. + :param create_branch_ref: if True, also create a Git branch + ref pointing at the new commit (useful for human-readable + navigation; not required for evolver operation) + :param author_name / author_email: git author / committer + identity + :param guard_immutable: if True (default), runs + :func:`assert_patch_allowed` against the patch's target + files before applying. Set False only in test code. + + :returns: the new child :class:`HarnessNode`, already saved + to disk and ready for evaluation + :raises ImmutablePathError: if the patch hits the immutable + kernel (spec §22) + :raises ValueError: parent missing, malformed patch + :raises GitOpError: git command failed (typically because + the patch didn't apply cleanly to the parent's tree) + """ + # Step 0: load parent (raises FileNotFoundError if missing) + parent = self.load_node(parent_node_id) + + # Step 0.5: gate against immutable kernel modification + if guard_immutable: + target_files = [c.target_file for c in patch.components] + assert_patch_allowed(target_files) + + # Step 1: construct the child commit (does NOT touch working tree) + child_sha = git_ops.apply_patch_as_commit( + repo_root=self.repo_root, + parent_sha=parent.git_commit_sha, + unified_diff=patch.diff, + message=commit_message, + author_name=author_name, + author_email=author_email, + ) + + # Step 2: assemble the HarnessNode metadata + node_id = f"v{iter_index}-{child_sha[:8]}" + # In the rare case the short-SHA collides with an existing + # node id (~1 in 4 billion), retry with a longer suffix. + if self.has_node(node_id): + node_id = f"v{iter_index}-{child_sha[:12]}" + + branch_for_node = git_branch or parent.git_branch + child = HarnessNode( + node_id=node_id, + parent_id=parent_node_id, + git_commit_sha=child_sha, + git_branch=branch_for_node, + created_at=HarnessNode.utc_now(), + created_at_iter=iter_index, + core_version=HarnessNode.current_core_version(), + status=NodeStatus.active, + patch=patch, + ) + + # Step 3: persist JSON (atomic via HarnessNode.save → temp+rename) + self.save_node(child) + + # Step 4 (optional): create a branch ref for human navigation + if create_branch_ref: + ref_name = f"evolver/{node_id}" + if not git_ops.branch_exists(self.repo_root, ref_name): + git_ops.create_branch(self.repo_root, ref_name, child_sha) + + return child + + +__all__ = [ + "EvolverTreeStore", + "TreeView", +] diff --git a/tests/test_appworld_precheck.py b/tests/test_appworld_precheck.py new file mode 100644 index 0000000..105c4bc --- /dev/null +++ b/tests/test_appworld_precheck.py @@ -0,0 +1,102 @@ +"""Unit tests for the Gate0 subject-endpoint probe (benchmarks.appworld.evolve.precheck). + +The probe is what stands between `run`/`check` and burning trials against a +dead or degraded endpoint; each failure mode must map to its own actionable +message (unreachable vs unhealthy vs degraded), because the operator acts on +that text. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +httpx = pytest.importorskip("httpx") + +from benchmarks.appworld.evolve.precheck import ( # noqa: E402 + _endpoint_problem, + _subject_endpoint, +) + + +class TestSubjectEndpoint: + def test_reads_api_base_and_model(self, tmp_path): + cfg = tmp_path / "subject.json" + cfg.write_text( + '{"providers": {"custom": {"api_base": "http://h/v1"}},' + '"agents": {"defaults": {"provider": "custom", "model": "m1"}}}' + ) + assert _subject_endpoint(cfg) == ("http://h/v1", "m1", None) + + def test_missing_model_is_a_problem(self, tmp_path): + cfg = tmp_path / "subject.json" + cfg.write_text( + '{"providers": {"custom": {"api_base": "http://h/v1"}},"agents": {"defaults": {"provider": "custom"}}}' + ) + _, _, problem = _subject_endpoint(cfg) + assert "missing provider api_base/model" in problem + + def test_unreadable_config_is_a_problem(self, tmp_path): + cfg = tmp_path / "subject.json" + cfg.write_text("{not json") + _, _, problem = _subject_endpoint(cfg) + assert "unreadable" in problem + + +def _response(status: int = 200, tokens: int = 300): + return SimpleNamespace( + status_code=status, + text="body", + json=lambda: {"usage": {"completion_tokens": tokens}}, + ) + + +def _probe(monkeypatch, *, post=None, seconds_per_call: float = 1.0, min_tok_per_s: float = 12.0): + clock = {"t": 0.0} + + def monotonic(): + clock["t"] += seconds_per_call + return clock["t"] + + monkeypatch.setattr("time.monotonic", monotonic) + monkeypatch.setattr(httpx, "post", post) + return _endpoint_problem("http://h/v1", "m1", 60.0, min_tok_per_s) + + +class TestEndpointProblem: + def test_healthy_endpoint_is_none(self, monkeypatch): + assert _probe(monkeypatch, post=lambda *a, **k: _response()) is None + + def test_timeout_is_degraded(self, monkeypatch): + def post(*a, **k): + raise httpx.TimeoutException("slow") + + problem = _probe(monkeypatch, post=post) + assert "degraded" in problem and "no 300-token completion" in problem + + def test_connection_error_is_unreachable(self, monkeypatch): + def post(*a, **k): + raise httpx.ConnectError("nodename nor servname") + + problem = _probe(monkeypatch, post=post) + assert "unreachable" in problem and "ConnectError" in problem + + def test_http_error_status_is_unhealthy(self, monkeypatch): + problem = _probe(monkeypatch, post=lambda *a, **k: _response(status=503)) + assert "unhealthy" in problem and "HTTP 503" in problem + + def test_empty_generation_is_unhealthy(self, monkeypatch): + problem = _probe(monkeypatch, post=lambda *a, **k: _response(tokens=0)) + assert "empty generation" in problem + + def test_slow_decode_trips_the_throughput_floor(self, monkeypatch): + # 300 tokens in 30s = 10 tok/s, below the 12 tok/s SOP health bar. + problem = _probe(monkeypatch, post=lambda *a, **k: _response(), seconds_per_call=30.0) + assert "degraded" in problem and "tok/s floor" in problem diff --git a/tests/test_appworld_sandbox.py b/tests/test_appworld_sandbox.py new file mode 100644 index 0000000..ffb1db9 --- /dev/null +++ b/tests/test_appworld_sandbox.py @@ -0,0 +1,130 @@ +"""Unit tests for the design-step sandbox (benchmarks.appworld.evolve.sandbox). + +The sandbox is the boundary between an LLM editing freely and the candidate +that actually gets committed: whitelist refusal, out-of-whitelist reversion, +and exact change/deletion capture. A hole here means a candidate silently +carries (or drops) edits the designer never verified. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from benchmarks.appworld.evolve.sandbox import Sandbox # noqa: E402 + +_ENV = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + "PATH": "/usr/bin:/bin", +} + + +@pytest.fixture() +def subject(tmp_path: Path) -> tuple[Path, str]: + repo = tmp_path / "subject" + (repo / "benchmarks/appworld").mkdir(parents=True) + (repo / "benchmarks/__init__.py").touch() + (repo / "benchmarks/appworld/__init__.py").touch() + (repo / "benchmarks/appworld/agent_cli.py").write_text("PROMPT = 'v1'\n") + (repo / "raven/agent").mkdir(parents=True) + (repo / "raven/agent/loop.py").write_text("x = 1\n") + (repo / "grader.py").write_text("score = 1\n") + for cmd in (["git", "init", "-q"], ["git", "add", "-A"], ["git", "commit", "-qm", "init"]): + subprocess.run(cmd, cwd=repo, check=True, env=_ENV, capture_output=True) + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=repo, check=True, capture_output=True, text=True, env=_ENV + ).stdout.strip() + return repo, sha + + +@pytest.fixture() +def sandbox(subject, tmp_path): + repo, sha = subject + sb = Sandbox(repo, tmp_path / "wt", sha) + yield sb + sb.close() + + +class TestWriteRefusal: + def test_out_of_whitelist_write_is_refused(self, sandbox): + msg = sandbox.write_text("grader.py", "score = 999\n") + assert "refused" in msg and "whitelist" in msg + assert (sandbox.root / "grader.py").read_text() == "score = 1\n" + + def test_path_escape_is_refused(self, sandbox): + assert "escapes" in sandbox.write_text("benchmarks/../../etc/x", "") + + def test_whitelist_write_lands(self, sandbox): + msg = sandbox.write_text("benchmarks/appworld/agent_cli.py", "PROMPT = 'v2'\n") + assert msg.startswith("[wrote") + assert (sandbox.root / "benchmarks/appworld/agent_cli.py").read_text() == "PROMPT = 'v2'\n" + + +class TestBashBoundary: + def test_command_naming_origin_repo_is_refused(self, sandbox): + out = sandbox.bash(f"cat {sandbox.repo_root.resolve()}/grader.py") + assert "refused" in out and "origin repo" in out + + def test_command_runs_inside_worktree(self, sandbox): + out = sandbox.bash("pwd") + assert str(sandbox.root) in out and "[exit 0]" in out + + +class TestScopeRestore: + def test_out_of_whitelist_edits_reverted_whitelist_kept(self, sandbox): + # Simulate the driver escaping via bash: edit the grader (outside the + # whitelist), create an outside file, and make a legit whitelist edit. + (sandbox.root / "grader.py").write_text("score = 999\n") + (sandbox.root / "stray.txt").write_text("junk\n") + sandbox.write_text("raven/agent/loop.py", "x = 2\n") + + reverted = sandbox.scope_restore() + + assert sorted(reverted) == ["grader.py", "stray.txt"] + assert (sandbox.root / "grader.py").read_text() == "score = 1\n" + assert not (sandbox.root / "stray.txt").exists() + assert (sandbox.root / "raven/agent/loop.py").read_text() == "x = 2\n" + + +class TestChangeCapture: + def test_changed_and_deleted_whitelist_files(self, sandbox): + sandbox.write_text("benchmarks/appworld/agent_cli.py", "PROMPT = 'v2'\n") + sandbox.write_text("benchmarks/appworld/hints.md", "new file\n") + (sandbox.root / "raven/agent/loop.py").unlink() + + changed = sandbox.changed_whitelist() + assert changed == { + "benchmarks/appworld/agent_cli.py": b"PROMPT = 'v2'\n", + "benchmarks/appworld/hints.md": b"new file\n", + } + assert sandbox.deleted_whitelist() == ["raven/agent/loop.py"] + assert sandbox.original("raven/agent/loop.py") == b"x = 1\n" + + def test_build_artifacts_are_not_captured(self, sandbox): + pyc = sandbox.root / "benchmarks/appworld/__pycache__" + pyc.mkdir() + (pyc / "agent_cli.cpython-313.pyc").write_bytes(b"\x00") + (sandbox.root / "benchmarks/appworld/x.pyc").write_bytes(b"\x00") + assert sandbox.changed_whitelist() == {} + + +class TestImportCheck: + def test_import_failure_surfaces_before_an_eval_is_burned(self, sandbox): + sandbox.write_text("benchmarks/appworld/broken.py", "import missing_dep_xyz\n") + ok, err = sandbox.import_check("benchmarks.appworld.broken") + assert not ok and "missing_dep_xyz" in err + + def test_import_resolves_from_the_edited_worktree(self, sandbox): + sandbox.write_text("benchmarks/appworld/probe.py", "VALUE = 42\n") + ok, err = sandbox.import_check("benchmarks.appworld.probe") + assert ok, err diff --git a/tests/test_evolver_claude_cli.py b/tests/test_evolver_claude_cli.py new file mode 100644 index 0000000..60f51f1 --- /dev/null +++ b/tests/test_evolver_claude_cli.py @@ -0,0 +1,112 @@ +"""Unit tests for the claude-CLI driver transport (providers.claude_cli). + +Every evolver LLM role rides this seam when provider=claude_cli; a subprocess +is injected so the transcript serialization, JSON result parsing, retry and +rate-limit behavior are pinned without touching a real CLI or sleeping. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from raven.evolver.orchestrator.providers.claude_cli import ( + make_claude_call_fn, + render_messages, +) + + +def _proc(result: str = "ok", *, returncode: int = 0, is_error: bool = False, stderr: str = "") -> SimpleNamespace: + return SimpleNamespace( + returncode=returncode, + stdout=json.dumps({"result": result, "is_error": is_error}), + stderr=stderr, + ) + + +class _FakeRun: + """subprocess.run stand-in replaying scripted results; records invocations.""" + + def __init__(self, script: list): + self.script = list(script) + self.calls: list[dict] = [] + + def __call__(self, argv, **kw): + self.calls.append({"argv": list(argv), **kw}) + item = self.script.pop(0) + if isinstance(item, Exception): + raise item + return item + + +class TestRenderMessages: + def test_single_user_message_passes_through(self): + system, prompt = render_messages( + [ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "hello"}, + ] + ) + assert system == "be terse" and prompt == "hello" + + def test_multi_turn_serializes_with_role_tags(self): + system, prompt = render_messages( + [ + {"role": "user", "content": "do X"}, + {"role": "assistant", "content": "did X"}, + {"role": "user", "content": "now Y"}, + ] + ) + assert system == "" + assert "[USER]\ndo X" in prompt + assert "[ASSISTANT]\ndid X" in prompt + assert prompt.strip().endswith("now Y") + assert "NEXT message only" in prompt # the continue-instruction header + + +class TestCallFn: + def test_success_path_and_argv_shape(self): + run = _FakeRun([_proc("answer")]) + call = make_claude_call_fn("claude-test", run=run, retry_delays=()) + assert call([{"role": "user", "content": "q"}]) == "answer" + argv = run.calls[0]["argv"] + assert argv[:2] == ["claude", "-p"] + assert argv[argv.index("--model") + 1] == "claude-test" + # Tools must be OFF: this is Claude-as-completion, the FSM keeps control. + assert argv[argv.index("--allowedTools") + 1] == "" + # The prompt travels via stdin (argv overflows on rendered trajectories). + assert run.calls[0]["input"] == "q" + + def test_retries_then_succeeds(self, monkeypatch): + sleeps: list[float] = [] + monkeypatch.setattr("time.sleep", sleeps.append) + run = _FakeRun([_proc("", returncode=1, stderr="transient"), _proc("recovered")]) + call = make_claude_call_fn("m", run=run, retry_delays=(3.0,)) + assert call([{"role": "user", "content": "q"}]) == "recovered" + assert sleeps == [3.0] + + def test_rate_limit_stretches_the_delay(self, monkeypatch): + sleeps: list[float] = [] + monkeypatch.setattr("time.sleep", sleeps.append) + run = _FakeRun([_proc("", returncode=1, stderr="429 rate limit"), _proc("ok")]) + call = make_claude_call_fn("m", run=run, retry_delays=(3.0,), rate_limit_delay=120.0) + assert call([{"role": "user", "content": "q"}]) == "ok" + assert sleeps == [120.0] + + def test_exhausted_retries_raise_with_cause(self, monkeypatch): + monkeypatch.setattr("time.sleep", lambda s: None) + run = _FakeRun([_proc("", returncode=1, stderr="dead")] * 2) + call = make_claude_call_fn("m", run=run, retry_delays=(1.0,)) + with pytest.raises(RuntimeError, match="failed after 2 attempts") as exc: + call([{"role": "user", "content": "q"}]) + assert "dead" in str(exc.value.__cause__) + + def test_is_error_and_empty_results_are_failures(self, monkeypatch): + monkeypatch.setattr("time.sleep", lambda s: None) + for bad in (_proc("boom", is_error=True), _proc(" ")): + run = _FakeRun([bad]) + call = make_claude_call_fn("m", run=run, retry_delays=()) + with pytest.raises(RuntimeError): + call([{"role": "user", "content": "q"}]) diff --git a/tests/test_evolver_gates.py b/tests/test_evolver_gates.py new file mode 100644 index 0000000..b2ef9a3 --- /dev/null +++ b/tests/test_evolver_gates.py @@ -0,0 +1,324 @@ +"""Unit tests for the gate arithmetic (raven.evolver.orchestrator.gates). + +These protect the promotion decision itself: paired z statistics, the Fisher +exact test, the three-shield pipeline's narrowing rules, and the two concrete +gate policies. Wrong math here does not crash — it silently promotes or prunes +the wrong candidate — so the expected values below are hand-computed. +""" + +from __future__ import annotations + +import math + +import pytest + +from raven.evolver.orchestrator.gates.fisher import ( + fisher_one_sided, + focused_counts, + train_mean, +) +from raven.evolver.orchestrator.gates.paired import paired_lift +from raven.evolver.orchestrator.gates.pipeline import run_gates +from raven.evolver.orchestrator.gates.policy import Baseline, DecisionContext +from raven.evolver.orchestrator.gates.strategies import ( + FocusedFisherGate, + PairedTwoSigmaGate, + confirm_job_name, +) +from raven.evolver.orchestrator.scoring import TaskEval +from raven.evolver.scheduler.anchor_selection import AnchorSelection +from raven.evolver.tree.node import HarnessNode, NodeStatus + + +def _te(tid: str, passes: int, attempts: int, infra: int = 0) -> TaskEval: + return TaskEval(task_id=tid, passes=passes, attempts=attempts, infra_attempts=infra) + + +def _evals(spec: dict[str, tuple[int, int]]) -> dict[str, TaskEval]: + return {tid: _te(tid, p, a) for tid, (p, a) in spec.items()} + + +class TestPairedLift: + def test_deterministic_win_is_inf_z(self): + ids = ["t1", "t2", "t3", "t4"] + r = paired_lift( + candidate_evals=_evals({t: (3, 3) for t in ids}), + control_evals=_evals({t: (0, 3) for t in ids}), + task_ids=ids, + ) + assert r.mean_lift == 1.0 + assert r.se == 0.0 + assert r.z == math.inf + assert r.promoted and r.credited_2sigma + + def test_hand_computed_z(self): + # diffs = [1, 1, 0, 0]: mean 0.5, stdev sqrt(1/3), se sqrt(1/3)/2, + # z = 0.5 / (sqrt(1/3)/2) = sqrt(3) ~ 1.732 -> promoted, NOT credited. + cand = _evals({"t1": (3, 3), "t2": (3, 3), "t3": (0, 3), "t4": (0, 3)}) + ctrl = _evals({t: (0, 3) for t in ("t1", "t2", "t3", "t4")}) + r = paired_lift(candidate_evals=cand, control_evals=ctrl, task_ids=["t1", "t2", "t3", "t4"]) + assert r.mean_lift == pytest.approx(0.5) + assert r.z == pytest.approx(math.sqrt(3)) + assert r.promoted + assert not r.credited_2sigma + + def test_zero_lift_is_not_promoted(self): + ids = ["t1", "t2"] + same = _evals({t: (1, 3) for t in ids}) + r = paired_lift(candidate_evals=same, control_evals=dict(same), task_ids=ids) + assert r.z == 0.0 + assert not r.promoted and not r.credited_2sigma + + def test_deterministic_regression_is_minus_inf(self): + ids = ["t1", "t2"] + r = paired_lift( + candidate_evals=_evals({t: (0, 3) for t in ids}), + control_evals=_evals({t: (3, 3) for t in ids}), + task_ids=ids, + ) + assert r.z == -math.inf + assert not r.promoted and not r.credited_2sigma + + def test_missing_task_scores_zero_for_that_arm(self): + # Candidate never launched t2: it must score 0.0, not be dropped. + cand = _evals({"t1": (3, 3)}) + ctrl = _evals({"t1": (0, 3), "t2": (3, 3)}) + r = paired_lift(candidate_evals=cand, control_evals=ctrl, task_ids=["t1", "t2"]) + assert r.candidate_mean == pytest.approx(0.5) + assert r.control_mean == pytest.approx(0.5) + assert not r.promoted # tie, not a win + + def test_single_task_has_zero_se(self): + r = paired_lift( + candidate_evals=_evals({"t1": (3, 3)}), + control_evals=_evals({"t1": (0, 3)}), + task_ids=["t1"], + ) + assert r.se == 0.0 and r.z == math.inf + + def test_empty_task_list_refused(self): + with pytest.raises(ValueError, match="non-empty"): + paired_lift(candidate_evals={}, control_evals={}, task_ids=[]) + + +class TestFisher: + def test_hand_computed_extreme(self): + # [[3,0],[0,3]]: P(a=3) = C(3,3)*C(3,0)/C(6,3) = 1/20 = 0.05. + assert fisher_one_sided(3, 0, 0, 3) == pytest.approx(0.05) + + def test_hand_computed_moderate(self): + # [[8,2],[2,8]]: sum_{a=8..10} C(10,a)*C(10,10-a)/C(20,10) + # = (45*45 + 10*10 + 1) / 184756 = 2126/184756. + assert fisher_one_sided(8, 2, 2, 8) == pytest.approx(2126 / 184756) + + def test_candidate_worst_is_one(self): + assert fisher_one_sided(0, 3, 3, 0) == pytest.approx(1.0) + + def test_degenerate_margins_return_one(self): + assert fisher_one_sided(0, 0, 2, 1) == 1.0 # empty candidate row + assert fisher_one_sided(2, 1, 0, 0) == 1.0 # empty control row + assert fisher_one_sided(0, 3, 0, 3) == 1.0 # no passes anywhere + assert fisher_one_sided(3, 0, 3, 0) == 1.0 # all passes everywhere + + def test_focused_counts_keeps_infra_as_fails(self): + evals = {"t1": _te("t1", 1, 3, infra=2), "t2": _te("t2", 3, 3)} + # t3 never launched: skipped here (train_mean owns the denominator). + assert focused_counts(evals, ["t1", "t2", "t3"]) == (4, 2) + + def test_train_mean_fixed_denominator(self): + evals = _evals({"t1": (3, 3)}) + # Missing t2 contributes 0.0 but stays in the denominator. + assert train_mean(evals, ["t1", "t2"]) == pytest.approx(0.5) + assert train_mean(evals, []) == 0.0 + + +class TestRunGates: + def test_infra_reported_but_not_dropped(self): + ids = ["t1", "t2"] + cand = {"t1": _te("t1", 3, 3, infra=1), "t2": _te("t2", 3, 3)} + ctrl = _evals({t: (0, 3) for t in ids}) + g = run_gates(candidate_evals=cand, control_evals=ctrl, task_ids=ids) + assert g.infra_contaminated == ["t1"] + assert g.eligible_tasks == ids # SOP 0: kept in the denominator + assert g.promoted + + def test_gate_b_none_fails_open(self): + ids = ["t1"] + g = run_gates( + candidate_evals=_evals({"t1": (3, 3)}), + control_evals=_evals({"t1": (0, 3)}), + task_ids=ids, + fired_tasks=None, + ) + assert g.unfired_excluded == [] and g.eligible_tasks == ids + + def test_gate_b_empty_set_leaves_nothing_and_refuses(self): + ids = ["t1", "t2"] + g = run_gates( + candidate_evals=_evals({t: (3, 3) for t in ids}), + control_evals=_evals({t: (0, 3) for t in ids}), + task_ids=ids, + fired_tasks=set(), + ) + assert not g.promoted + assert g.paired is None + assert g.unfired_excluded == ids + + def test_gate_b_narrows_paired_to_fired_subset(self): + cand = _evals({"t1": (3, 3), "t2": (0, 3)}) + ctrl = _evals({"t1": (0, 3), "t2": (0, 3)}) + g = run_gates(candidate_evals=cand, control_evals=ctrl, task_ids=["t1", "t2"], fired_tasks={"t1"}) + assert g.eligible_tasks == ["t1"] + assert g.unfired_excluded == ["t2"] + assert g.paired.n_tasks == 1 and g.promoted + + +class _FakeEval: + """EvalFn returning canned evals keyed by job-name suffix; records calls.""" + + def __init__(self, by_suffix: dict[str, dict[str, TaskEval]]): + self.by_suffix = by_suffix + self.calls: list[tuple[list[str], int, str]] = [] + + def __call__(self, node, task_ids, k, job_name, *, split="train"): + self.calls.append((list(task_ids), k, job_name)) + for suffix, evals in self.by_suffix.items(): + if job_name.endswith(suffix): + return {t: ev for t, ev in evals.items() if t in task_ids} + raise AssertionError(f"unexpected eval job {job_name!r}") + + +def _node(nid: str = "cand") -> HarnessNode: + return HarnessNode( + node_id=nid, + parent_id="C0", + git_commit_sha="0" * 40, + git_branch="", + created_at=HarnessNode.utc_now(), + created_at_iter=1, + ) + + +def _ctx(eval_fn, baseline_evals, train_ids, **kw) -> DecisionContext: + return DecisionContext( + node=_node(), + parent_id="C0", + round_index=1, + eval=eval_fn, + baseline=Baseline(baseline_evals, train_mean(baseline_evals, train_ids), "vanilla"), + train_task_ids=train_ids, + **kw, + ) + + +class TestFocusedFisherGate: + def test_promotes_on_full_train_lift(self): + train = ["f1", "t2", "t3"] + base = _evals({"f1": (0, 3), "t2": (3, 3), "t3": (0, 3)}) # mean 1/3 + fake = _FakeEval( + { + "_focused": _evals({"f1": (2, 3)}), + "_confirm": _evals({"f1": (2, 3), "t2": (3, 3), "t3": (0, 3)}), # mean 5/9 + } + ) + out = FocusedFisherGate(k=3).decide(_ctx(fake, base, train, focused_task_ids=["f1"])) + assert out.status == NodeStatus.promoted_to_baseline + assert out.score == pytest.approx(5 / 9) + assert out.stats["full_lift"] == pytest.approx(5 / 9 - 1 / 3) + probe_ids, _, probe_job = fake.calls[0] + assert probe_ids == ["f1"] and probe_job == "cand_focused" + confirm_ids, _, confirm_job = fake.calls[1] + assert confirm_ids == train and confirm_job == confirm_job_name("cand") + + def test_min_confirm_lift_prunes(self): + train = ["f1", "t2", "t3"] + base = _evals({"f1": (0, 3), "t2": (3, 3), "t3": (0, 3)}) + fake = _FakeEval( + { + "_focused": _evals({"f1": (2, 3)}), + "_confirm": _evals({"f1": (2, 3), "t2": (3, 3), "t3": (0, 3)}), + } + ) + out = FocusedFisherGate(k=3, min_confirm_lift=0.5).decide(_ctx(fake, base, train, focused_task_ids=["f1"])) + assert out.status == NodeStatus.pruned_at_confirm # lift 2/9 < 0.5 + + def test_stable_sentinel_regression_prunes_at_screen(self): + train = ["s1", "s2", "t3"] + base = _evals({"s1": (3, 3), "s2": (3, 3), "t3": (0, 3)}) + fake = _FakeEval({"_focused": _evals({"s1": (1, 3), "s2": (3, 3)})}) + # st_c = mean(1/3, 1) = 2/3 < 1.0 - guard(1.5/(2*3)=0.25) = 0.75 -> prune. + out = FocusedFisherGate(k=3).decide(_ctx(fake, base, train, sentinel_task_ids=["s1", "s2"])) + assert out.status == NodeStatus.pruned_at_screen + assert out.stats["sentinel_regression"] is True + assert len(fake.calls) == 1 # no confirm was paid for + + def test_fragile_sentinel_noise_is_tolerated(self): + # A borderline (1/3) sentinel dipping to 0/3 is not significant + # (fisher p = 0.5): wide-pass advances to confirm instead of pruning. + train = ["s1", "t2"] + base = _evals({"s1": (1, 3), "t2": (0, 3)}) + fake = _FakeEval( + { + "_focused": _evals({"s1": (0, 3)}), + "_confirm": _evals({"s1": (1, 3), "t2": (2, 3)}), + } + ) + out = FocusedFisherGate(k=3).decide(_ctx(fake, base, train, sentinel_task_ids=["s1"])) + assert out.status == NodeStatus.promoted_to_baseline + assert out.stats["sent_fragile_p_worse"] == pytest.approx(0.5) + + def test_significantly_worse_probe_prunes_without_confirm(self): + train = ["f1", "f2", "f3"] + base = _evals({t: (3, 3) for t in train}) + fake = _FakeEval({"_focused": _evals({t: (0, 3) for t in train})}) + out = FocusedFisherGate(k=3).decide(_ctx(fake, base, train, focused_task_ids=train)) + assert out.status == NodeStatus.pruned_at_screen + assert out.stats["pruned_significantly_worse"] is True + # fisher_p_worse = P for [[9,0],[0,9]] = 1/C(18,9) = 1/48620. + assert out.stats["fisher_p_worse"] == pytest.approx(1 / 48620) + assert len(fake.calls) == 1 + + +class TestPairedTwoSigmaGate: + def _anchor(self, ids, cull=0.1): + return AnchorSelection(task_ids=ids, sigma_screen=cull, cull_threshold=cull, tasks=[], shortfalls={}) + + def test_requires_anchor(self): + fake = _FakeEval({}) + with pytest.raises(ValueError, match="anchor"): + PairedTwoSigmaGate().decide(_ctx(fake, {}, ["t1"])) + + def test_clear_screen_loss_prunes_before_confirm(self): + train = ["a1", "a2", "t3"] + base = _evals({"a1": (3, 3), "a2": (3, 3), "t3": (0, 3)}) + fake = _FakeEval({"_screen": _evals({"a1": (0, 1), "a2": (0, 1)})}) + out = PairedTwoSigmaGate().decide(_ctx(fake, base, train, anchor=self._anchor(["a1", "a2"]))) + assert out.status == NodeStatus.pruned_at_screen + assert out.screen.bucket == "cull" + assert len(fake.calls) == 1 # confirm never ran + + def test_fired_subset_cannot_promote_a_full_train_regression(self): + # Gate-b narrows the paired stats to {t1} where the candidate wins, + # but the full-train mean regresses (1/3 vs 2/3): the dual condition + # must refuse promotion and report the FULL mean as the score. + train = ["t1", "t2", "t3"] + base = _evals({"t1": (0, 3), "t2": (3, 3), "t3": (3, 3)}) # mean 2/3 + fake = _FakeEval( + { + "_screen": _evals({"t2": (1, 1)}), # within band -> advance + "_confirm": _evals({"t1": (3, 3), "t2": (0, 3), "t3": (0, 3)}), # mean 1/3 + } + ) + out = PairedTwoSigmaGate().decide( + _ctx( + fake, + base, + train, + anchor=self._anchor(["t2"]), + fired_source=lambda node, ids: {"t1"}, + ) + ) + assert out.gate.paired.promoted # the fired subset alone looks like a win + assert out.status == NodeStatus.pruned_at_confirm + assert out.score == pytest.approx(1 / 3) # full-train mean, never the subset mean + assert out.gate.unfired_excluded == ["t2", "t3"] diff --git a/tests/test_evolver_git_ops.py b/tests/test_evolver_git_ops.py new file mode 100644 index 0000000..3dd2102 --- /dev/null +++ b/tests/test_evolver_git_ops.py @@ -0,0 +1,124 @@ +"""Unit tests for the git primitives the evolver builds candidates with. + +These run against the USER'S real repo in production (the subject checkout), +so the invariants under test are the dangerous ones: the main working tree is +never touched, ephemeral worktrees are cleaned up on every exit path, and +evolver commits stay reachable across git gc via plain refs. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from raven.evolver.tree import git_ops + +_ENV = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + "PATH": "/usr/bin:/bin", +} + + +@pytest.fixture(autouse=True) +def _reset_ephemeral_root(): + yield + git_ops.set_ephemeral_root(None) + + +@pytest.fixture() +def repo(tmp_path: Path) -> tuple[Path, str]: + repo = tmp_path / "subject" + (repo / "src").mkdir(parents=True) + (repo / "src/x.py").write_text("x = 1\n") + (repo / "old.txt").write_text("legacy\n") + for cmd in (["git", "init", "-q"], ["git", "add", "-A"], ["git", "commit", "-qm", "init"]): + subprocess.run(cmd, cwd=repo, check=True, env=_ENV, capture_output=True) + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=repo, check=True, capture_output=True, text=True, env=_ENV + ).stdout.strip() + return repo, sha + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run(["git", *args], cwd=repo, check=True, env=_ENV, capture_output=True, text=True).stdout + + +class TestCommitFilesAsChild: + def test_edit_delete_and_create_land_in_the_child(self, repo): + repo_dir, sha = repo + child, changed = git_ops.commit_files_as_child( + repo_dir, + sha, + {"src/x.py": b"x = 2\n", "src/new.py": b"y = 1\n"}, + "evolver: candidate", + deletions=("old.txt",), + ) + assert child != sha + assert _git(repo_dir, "rev-parse", f"{child}^").strip() == sha + assert _git(repo_dir, "show", f"{child}:src/x.py") == "x = 2\n" + assert _git(repo_dir, "show", f"{child}:src/new.py") == "y = 1\n" + tree = _git(repo_dir, "ls-tree", "-r", "--name-only", child) + assert "old.txt" not in tree + assert sorted(changed) == ["old.txt", "src/new.py", "src/x.py"] + + def test_main_working_tree_and_head_are_untouched(self, repo): + repo_dir, sha = repo + git_ops.commit_files_as_child(repo_dir, sha, {"src/x.py": b"x = 99\n"}, "evolver: candidate") + assert (repo_dir / "src/x.py").read_text() == "x = 1\n" + assert _git(repo_dir, "rev-parse", "HEAD").strip() == sha + assert _git(repo_dir, "status", "--porcelain") == "" + assert _git(repo_dir, "worktree", "list").count("\n") == 1 # main only + + def test_unknown_parent_refused(self, repo): + repo_dir, _ = repo + with pytest.raises(ValueError, match="not a known commit"): + git_ops.commit_files_as_child(repo_dir, "f" * 40, {"src/x.py": b""}, "msg") + + +class TestWorktreeAt: + def test_yields_checkout_and_cleans_up(self, repo): + repo_dir, sha = repo + with git_ops.worktree_at(repo_dir, sha) as wt: + assert (wt / "src/x.py").read_text() == "x = 1\n" + kept = wt + assert not kept.exists() + assert "evolver-wt-" not in _git(repo_dir, "worktree", "list") + + def test_cleans_up_on_exception(self, repo): + repo_dir, sha = repo + with pytest.raises(RuntimeError, match="boom"): + with git_ops.worktree_at(repo_dir, sha) as wt: + kept = wt + raise RuntimeError("boom") + assert not kept.exists() + assert "evolver-wt-" not in _git(repo_dir, "worktree", "list") + + def test_ephemeral_root_redirects_tempdirs(self, repo, tmp_path): + repo_dir, sha = repo + root = tmp_path / "run_tmp" + git_ops.set_ephemeral_root(root) + with git_ops.worktree_at(repo_dir, sha) as wt: + assert str(wt).startswith(str(root)) + git_ops.set_ephemeral_root(None) + with git_ops.worktree_at(repo_dir, sha) as wt: + assert not str(wt).startswith(str(root)) + + +class TestRefs: + def test_ref_anchors_evolver_commit_against_gc(self, repo): + repo_dir, sha = repo + child, _ = git_ops.commit_files_as_child(repo_dir, sha, {"src/x.py": b"x = 3\n"}, "evolver: candidate") + git_ops.create_ref(repo_dir, "refs/evolver/n1", child) + assert _git(repo_dir, "rev-parse", "refs/evolver/n1").strip() == child + # Overwrite is allowed (re-promotion re-points the ref). + git_ops.create_ref(repo_dir, "refs/evolver/n1", sha) + assert _git(repo_dir, "rev-parse", "refs/evolver/n1").strip() == sha + git_ops.delete_ref(repo_dir, "refs/evolver/n1") + git_ops.delete_ref(repo_dir, "refs/evolver/n1") # absent: no-op + proc = subprocess.run(["git", "rev-parse", "refs/evolver/n1"], cwd=repo_dir, env=_ENV, capture_output=True) + assert proc.returncode != 0 diff --git a/tests/test_evolver_launch.py b/tests/test_evolver_launch.py new file mode 100644 index 0000000..226e0d9 --- /dev/null +++ b/tests/test_evolver_launch.py @@ -0,0 +1,531 @@ +"""Unit tests for the unified evolution launcher (raven.evolver.launch).""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from raven.evolver.launch.config import ( + SMOKE_BUILTIN, + RunSpecError, + deep_merge, + load_run_spec, +) +from raven.evolver.launch.contract import validate_whitelist +from raven.evolver.launch.registry import load_bench +from raven.evolver.launch.state import RunMeta, atomic_write_json, config_fingerprint + +# The appworld bench plugin lives at the repo root (benchmarks/), outside the +# installed raven package; load_bench imports it via the subject repo root, +# and the direct `import benchmarks...` statements below need it on sys.path. +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + +@pytest.fixture() +def subject_repo(tmp_path: Path) -> tuple[Path, str]: + """A tiny git repo standing in for the evolved subject.""" + repo = tmp_path / "subject" + (repo / "raven/agent").mkdir(parents=True) + (repo / "raven/agent/loop.py").write_text("x = 1\n") + env = {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + for cmd in (["git", "init", "-q"], ["git", "add", "-A"], ["git", "commit", "-qm", "init"]): + subprocess.run(cmd, cwd=repo, check=True, env={**env, "PATH": "/usr/bin:/bin"}, capture_output=True) + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=repo, check=True, capture_output=True, text=True + ).stdout.strip() + return repo, sha + + +def _write_spec(tmp_path: Path, repo: Path, sha: str, **extra) -> Path: + data = { + "bench": "appworld", + "repo_root": str(repo), + "base_sha": sha, + "work_dir": str(tmp_path / "work"), + **extra, + } + path = tmp_path / "spec.yaml" + path.write_text(yaml.safe_dump(data)) + return path + + +class TestRunSpec: + def test_minimal_spec_loads_with_defaults(self, tmp_path, subject_repo): + repo, sha = subject_repo + spec = load_run_spec(_write_spec(tmp_path, repo, sha)) + assert spec.bench == "appworld" + assert spec.funnel.k_confirm == 3 + assert spec.funnel.termination.patience == 10 + assert spec.funnel.sealed_output_dir == spec.work_dir / "sealed" + + def test_omitted_base_sha_resolves_to_head(self, tmp_path, subject_repo): + repo, sha = subject_repo + path = tmp_path / "spec.yaml" + path.write_text( + yaml.safe_dump( + { + "bench": "appworld", + "repo_root": str(repo), + "work_dir": str(tmp_path / "work"), + } + ) + ) + spec = load_run_spec(path) + assert spec.base_sha == sha + assert spec.base_sha_defaulted is True + assert spec.snapshot()["base_sha"] == sha + + def test_explicit_base_sha_kept_verbatim(self, tmp_path, subject_repo): + repo, sha = subject_repo + spec = load_run_spec(_write_spec(tmp_path, repo, sha[:7])) + assert spec.base_sha == sha[:7] + assert spec.base_sha_defaulted is False + + def test_missing_required_keys(self, tmp_path): + path = tmp_path / "bad.yaml" + path.write_text(yaml.safe_dump({"bench": "appworld"})) + with pytest.raises(RunSpecError, match="missing required"): + load_run_spec(path) + + def test_funnel_scalar_rejected(self, tmp_path, subject_repo): + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha, funnel=5) + with pytest.raises(RunSpecError, match="must be a mapping"): + load_run_spec(path) + + def test_funnel_bad_value_type_readable(self, tmp_path, subject_repo): + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha, funnel={"k_confirm": "three"}) + with pytest.raises(RunSpecError, match="funnel"): + load_run_spec(path) + + def test_funnel_nonpositive_rejected(self, tmp_path, subject_repo): + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha, funnel={"termination": {"patience": 0}}) + with pytest.raises(RunSpecError, match=">= 1"): + load_run_spec(path) + + def test_model_role_scalar_rejected(self, tmp_path, subject_repo): + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha, models={"driver": "claude"}) + with pytest.raises(RunSpecError, match="models.driver"): + load_run_spec(path) + + def test_relative_paths_resolve_against_config_dir(self, tmp_path, subject_repo, monkeypatch): + repo, sha = subject_repo + path = tmp_path / "spec.yaml" + path.write_text( + yaml.safe_dump( + { + "bench": "appworld", + "repo_root": str(repo), + "base_sha": sha, + "work_dir": "evo_work", + } + ) + ) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + spec = load_run_spec(path) + assert spec.work_dir == (tmp_path / "evo_work").resolve() + + def test_unknown_funnel_key_rejected(self, tmp_path, subject_repo): + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha, funnel={"k_confrim": 3}) + with pytest.raises(RunSpecError, match="unknown keys"): + load_run_spec(path) + + def test_unknown_model_role_rejected(self, tmp_path, subject_repo): + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha, models={"designer": {}}) + with pytest.raises(RunSpecError, match="unknown roles"): + load_run_spec(path) + + def test_smoke_applies_builtin_shrink_then_user_overlay(self, tmp_path, subject_repo): + repo, sha = subject_repo + path = _write_spec( + tmp_path, + repo, + sha, + funnel={"k_confirm": 3}, + smoke={"funnel": {"termination": {"max_rounds": 2}}}, + ) + spec = load_run_spec(path, smoke=True) + assert spec.funnel.k_confirm == SMOKE_BUILTIN["funnel"]["k_confirm"] + assert spec.funnel.budget.max_why_per_round == 1 + assert spec.funnel.termination.max_rounds == 2 # user overlay wins + assert spec.work_dir.name.endswith("_smoke") + + def test_non_smoke_ignores_smoke_section(self, tmp_path, subject_repo): + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha, smoke={"funnel": {"k_confirm": 1}}) + spec = load_run_spec(path) + assert spec.funnel.k_confirm == 3 + assert "smoke" not in spec.raw + + def test_deep_merge_nested(self): + merged = deep_merge({"a": {"b": 1, "c": 2}}, {"a": {"b": 9}, "d": 3}) + assert merged == {"a": {"b": 9, "c": 2}, "d": 3} + + +class TestRunMeta: + def test_create_load_roundtrip(self, tmp_path): + meta = RunMeta.create(tmp_path, {"bench": "x"}) + again = RunMeta.load(tmp_path) + assert again is not None + assert again.config_hash == meta.config_hash + assert again.unsealed_at is None + + def test_config_drift_detected(self, tmp_path): + meta = RunMeta.create(tmp_path, {"bench": "x", "funnel": {"k": 3}}) + assert meta.check_config({"bench": "x", "funnel": {"k": 3}}) + assert not meta.check_config({"bench": "x", "funnel": {"k": 1}}) + + def test_unseal_stamp_is_persisted(self, tmp_path): + meta = RunMeta.create(tmp_path, {}) + meta.stamp_unsealed(reason="user_finalized") + again = RunMeta.load(tmp_path) + assert again.unsealed_at + assert again.finalize_reason == "user_finalized" + + def test_fingerprint_is_order_insensitive(self): + assert config_fingerprint({"a": 1, "b": 2}) == config_fingerprint({"b": 2, "a": 1}) + + def test_atomic_write_leaves_no_tmp_on_success(self, tmp_path): + target = tmp_path / "x.json" + atomic_write_json(target, {"ok": True}) + assert json.loads(target.read_text()) == {"ok": True} + assert list(tmp_path.glob("*.tmp")) == [] + + +class TestWhitelistValidation: + def test_live_prefix_passes(self, subject_repo): + repo, sha = subject_repo + validate_whitelist(repo, sha, ("raven/agent/",)) + + def test_dead_prefix_refuses(self, subject_repo): + repo, sha = subject_repo + with pytest.raises(ValueError, match="match no files"): + validate_whitelist(repo, sha, ("nonexistent/agent/",)) + + def test_empty_whitelist_refuses(self, subject_repo): + repo, sha = subject_repo + with pytest.raises(ValueError, match="empty"): + validate_whitelist(repo, sha, ()) + + +class TestAppWorldEntry: + def _ctx(self, tmp_path, subject_repo, bench_config, smoke=False): + from raven.evolver.launch.contract import LaunchContext + + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha, bench_config=bench_config) + spec = load_run_spec(path, smoke=smoke) + return LaunchContext( + spec=spec, + models={"driver": None, "design": None, "verdict": None}, + ) + + def _bench_config(self, tmp_path): + cfg = tmp_path / "subject_cfg.json" + cfg.write_text("{}") + (tmp_path / "appworld" / "data" / "tasks").mkdir(parents=True, exist_ok=True) + bin_dir = tmp_path / "appworld" / "appworld-venv" / "bin" + bin_dir.mkdir(parents=True, exist_ok=True) + (bin_dir / "appworld").write_text("#!/bin/sh\n") + return { + "config_path": str(cfg), + "appworld_data_root": str(tmp_path / "appworld"), + "train_task_ids": ["t1", "t2"], + "whitelist": ["raven/agent/"], + } + + def test_build_bundle_shape(self, tmp_path, subject_repo): + build = load_bench("appworld", repo_root=REPO_ROOT) + bundle = build(self._ctx(tmp_path, subject_repo, self._bench_config(tmp_path))) + assert bundle.root_node_id == "C0" + assert bundle.cold_start_total == 2 * 3 # 2 tasks x k_confirm=3 + assert bundle.cold_start_done() == 0 + assert bundle.unseal is None # no test ids -> no sealed test + + def test_cold_start_counts_existing_trials(self, tmp_path, subject_repo): + build = load_bench("appworld", repo_root=REPO_ROOT) + ctx = self._ctx(tmp_path, subject_repo, self._bench_config(tmp_path)) + bundle = build(ctx) + van = ctx.spec.work_dir / "runs" / "vanilla" + van.mkdir(parents=True) + (van / "t1_k0.json").write_text("{}") + (van / "t1_k1.json").write_text("{}") + assert bundle.cold_start_done() == 2 + + def test_unknown_bench_config_key_rejected(self, tmp_path, subject_repo): + build = load_bench("appworld", repo_root=REPO_ROOT) + bc = self._bench_config(tmp_path) + bc["train_tasks"] = ["x"] + with pytest.raises(ValueError, match="unknown keys"): + build(self._ctx(tmp_path, subject_repo, bc)) + + def test_train_test_overlap_rejected(self, tmp_path, subject_repo): + build = load_bench("appworld", repo_root=REPO_ROOT) + bc = self._bench_config(tmp_path) + bc["test_task_ids"] = ["t2", "t3"] + with pytest.raises(ValueError, match="overlap"): + build(self._ctx(tmp_path, subject_repo, bc)) + + def test_placeholder_task_ids_refused(self, tmp_path, subject_repo): + build = load_bench("appworld", repo_root=REPO_ROOT) + bc = self._bench_config(tmp_path) + bc["train_task_ids"] = ["", "t2"] + with pytest.raises(ValueError, match="placeholder"): + build(self._ctx(tmp_path, subject_repo, bc)) + + def test_missing_appworld_install_refuses_at_build(self, tmp_path, subject_repo, monkeypatch): + build = load_bench("appworld", repo_root=REPO_ROOT) + bc = self._bench_config(tmp_path) + bc["appworld_data_root"] = str(tmp_path / "nowhere") + monkeypatch.delenv("APPWORLD_ROOT", raising=False) + with pytest.raises(ValueError, match="no AppWorld install"): + build(self._ctx(tmp_path, subject_repo, bc)) + + def test_dead_whitelist_refuses_at_build(self, tmp_path, subject_repo): + build = load_bench("appworld", repo_root=REPO_ROOT) + bc = self._bench_config(tmp_path) + bc["whitelist"] = ["nonexistent/dir/"] + with pytest.raises(ValueError, match="match no files"): + build(self._ctx(tmp_path, subject_repo, bc)) + + +class TestScorerImmutability: + def test_scorer_surface_is_immutable(self): + from raven.evolver.applier.path_guard import check_patch_paths + + offenders = check_patch_paths( + [ + "benchmarks/appworld/evolve/grade.py", + "benchmarks/appworld/evolve/adapter.py", + "benchmarks/appworld/batch.py", + "raven/evolver/orchestrator/gates/pipeline.py", + ] + ) + assert len(offenders) == 4 + + def test_agent_surface_is_editable(self): + from raven.evolver.applier.path_guard import check_patch_paths + + assert ( + check_patch_paths( + [ + "benchmarks/appworld/agent_cli.py", + "benchmarks/appworld/tool.py", + ] + ) + == [] + ) + + +class TestSecretRedaction: + def _spec_with(self, tmp_path, repo, sha, **driver): + path = _write_spec( + tmp_path, + repo, + sha, + models={"driver": {"provider": "openai_compat", "base_url": "http://h/v1", "model": "m", **driver}}, + ) + return load_run_spec(path) + + def test_api_key_never_reaches_snapshot(self, tmp_path, subject_repo): + repo, sha = subject_repo + snap = self._spec_with(tmp_path, repo, sha, api_key="sk-SECRET").snapshot() + assert "sk-SECRET" not in json.dumps(snap) + assert snap["models"]["driver"]["api_key"] == "" + + def test_fingerprint_stable_across_key_rotation(self, tmp_path, subject_repo): + repo, sha = subject_repo + f1 = config_fingerprint(self._spec_with(tmp_path, repo, sha, api_key="sk-AAA").snapshot()) + f2 = config_fingerprint(self._spec_with(tmp_path, repo, sha, api_key="sk-BBB").snapshot()) + f3 = config_fingerprint(self._spec_with(tmp_path, repo, sha, api_key="sk-AAA", model="other").snapshot()) + assert f1 == f2 + assert f1 != f3 + + +class TestColdStartPrecheck: + def _bundle(self, tmp_path, subject_repo, monkeypatch, calls): + import benchmarks.appworld.evolve.entry as entry_mod + import benchmarks.appworld.evolve.precheck as precheck_mod + + monkeypatch.setattr( + precheck_mod, + "make_appworld_precheck", + lambda cfg, **kw: lambda: calls.__setitem__("precheck", calls["precheck"] + 1), + ) + monkeypatch.setattr( + entry_mod, + "eval_with_infra_rerun", + lambda *a, **k: calls.__setitem__("eval", calls["eval"] + 1), + ) + helper = TestAppWorldEntry() + ctx = helper._ctx(tmp_path, subject_repo, helper._bench_config(tmp_path)) + return load_bench("appworld", repo_root=REPO_ROOT)(ctx), ctx + + def test_precheck_fires_before_fill_and_skips_when_clean(self, tmp_path, subject_repo, monkeypatch): + calls = {"precheck": 0, "eval": 0} + bundle, ctx = self._bundle(tmp_path, subject_repo, monkeypatch, calls) + + bundle.run_cold_start() + assert calls == {"precheck": 1, "eval": 1} + + van = ctx.spec.work_dir / "runs" / "vanilla" + van.mkdir(parents=True, exist_ok=True) + for tid in ("t1", "t2"): + for k in range(3): + (van / f"{tid}_k{k}.json").write_text(json.dumps({"task_id": tid, "success": True})) + bundle.run_cold_start() + assert calls["precheck"] == 1 # complete + infra-clean: no probe + assert calls["eval"] == 2 + + def test_precheck_fires_when_ladder_has_salvage_work(self, tmp_path, subject_repo, monkeypatch): + calls = {"precheck": 0, "eval": 0} + bundle, ctx = self._bundle(tmp_path, subject_repo, monkeypatch, calls) + van = ctx.spec.work_dir / "runs" / "vanilla" + van.mkdir(parents=True, exist_ok=True) + for tid in ("t1", "t2"): + for k in range(3): + rec = {"task_id": tid, "success": False} + if tid == "t2": + rec["infra_error"] = "runner: TimeoutExpired" + (van / f"{tid}_k{k}.json").write_text(json.dumps(rec)) + bundle.run_cold_start() + assert calls["precheck"] == 1 # infra residue -> probe before reruns + + +class TestBatchTrialResume: + def test_existing_result_is_returned_without_rerun(self, tmp_path, monkeypatch): + from benchmarks.appworld import batch + + out_dir = tmp_path + rec = {"task_id": "t1", "success": True} + (out_dir / "t1_k0.json").write_text(json.dumps(rec)) + + def boom(*a, **k): # noqa: ANN002, ANN003 + raise AssertionError("subprocess must not run for a completed trial") + + monkeypatch.setattr(batch.subprocess, "run", boom) + + class Args: + config = "unused" + workspace = str(tmp_path / "ws") + experiment = "e" + model = None + env = "" + task_timeout = 1 + + got = batch._run_one("t1", 0, 9999, Args(), str(out_dir)) + assert got == rec + + def test_corrupt_result_is_rerun(self, tmp_path, monkeypatch): + from benchmarks.appworld import batch + + (tmp_path / "t1_k0.json").write_text("{half written") + calls = [] + + def fake_run(*a, **k): # noqa: ANN002, ANN003 + calls.append(1) + + monkeypatch.setattr(batch.subprocess, "run", fake_run) + + class Args: + config = "unused" + workspace = str(tmp_path / "ws") + experiment = "e" + model = None + env = "" + task_timeout = 1 + + got = batch._run_one("t1", 0, 9999, Args(), str(tmp_path)) + assert calls, "corrupt file must trigger a re-run" + assert got.get("infra_error") # agent_cli never wrote a fresh result + + +class TestCli: + def test_parser_subcommands(self): + from raven.evolver.cli import build_parser + + p = build_parser() + args = p.parse_args(["run", "--config", "x.yaml", "--smoke", "--force"]) + assert args.command == "run" and args.smoke and args.force + args = p.parse_args(["finalize", "--config", "x.yaml", "--yes"]) + assert args.command == "finalize" and args.yes + + def test_status_on_fresh_dir_reports_not_started(self, tmp_path, subject_repo, capsys): + from raven.evolver.launch.runner import cmd_status + + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha) + assert cmd_status(str(path)) == 0 + assert "not started" in capsys.readouterr().out + + def _buildable_spec(self, tmp_path, repo, sha): + """A spec whose bundle+models build cleanly, so guard tests reach + the meta guard (which now runs after bundle validation).""" + cfg = tmp_path / "subject_cfg.json" + cfg.write_text("{}") + (tmp_path / "appworld" / "data" / "tasks").mkdir(parents=True, exist_ok=True) + bin_dir = tmp_path / "appworld" / "appworld-venv" / "bin" + bin_dir.mkdir(parents=True, exist_ok=True) + (bin_dir / "appworld").write_text("#!/bin/sh\n") + return _write_spec( + tmp_path, + repo, + sha, + models={"driver": {"provider": "openai_compat", "base_url": "http://localhost:1/v1", "model": "m"}}, + bench_config={ + "config_path": str(cfg), + "appworld_data_root": str(tmp_path / "appworld"), + "train_task_ids": ["t1"], + "whitelist": ["raven/agent/"], + }, + ) + + def test_run_refuses_after_unseal(self, tmp_path, subject_repo, capsys): + from raven.evolver.launch.runner import cmd_run + + repo, sha = subject_repo + path = self._buildable_spec(tmp_path, repo, sha) + work = tmp_path / "work" + work.mkdir() + meta = RunMeta.create(work, {}) + meta.stamp_unsealed(reason="user_finalized") + with pytest.raises(SystemExit) as exc: + cmd_run(str(path)) + assert exc.value.code == 2 + assert "unsealed" in capsys.readouterr().err + + def test_run_refuses_on_config_drift(self, tmp_path, subject_repo, capsys): + from raven.evolver.launch.runner import cmd_run + + repo, sha = subject_repo + path = self._buildable_spec(tmp_path, repo, sha) + work = tmp_path / "work" + work.mkdir() + RunMeta.create(work, {"bench": "appworld", "different": True}) + with pytest.raises(SystemExit) as exc: + cmd_run(str(path)) + assert exc.value.code == 2 + assert "config drift" in capsys.readouterr().err + + def test_first_launch_config_mistake_leaves_no_meta(self, tmp_path, subject_repo): + from raven.evolver.launch.runner import cmd_run + + repo, sha = subject_repo + path = _write_spec(tmp_path, repo, sha) # no bench_config -> build fails + with pytest.raises(SystemExit): + cmd_run(str(path)) + assert RunMeta.load(tmp_path / "work") is None diff --git a/tests/test_evolver_launch_e2e.py b/tests/test_evolver_launch_e2e.py new file mode 100644 index 0000000..95eb24c --- /dev/null +++ b/tests/test_evolver_launch_e2e.py @@ -0,0 +1,356 @@ +"""End-to-end tests of the launcher state machine with a fake bench. + +Everything is real except the scorer: real CLI entry (cmd_run/status/finalize), +real RunMeta guards, real EvolutionOrchestrator + journal replay. The fake +bench writes trial marker files for cold start and drives one-candidate rounds +that die at preflight, so no subprocess or LLM is needed. Interruption is +injected via KeyboardInterrupt (BaseException — the loop must not swallow it). +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest +import yaml + +from raven.evolver.launch import runner as runner_mod +from raven.evolver.launch.contract import BenchBundle +from raven.evolver.launch.state import RunMeta +from raven.evolver.tree import git_ops + + +@pytest.fixture(autouse=True) +def _reset_ephemeral_root(): + yield + git_ops.set_ephemeral_root(None) + + +@pytest.fixture() +def repo(tmp_path: Path) -> tuple[Path, str]: + repo = tmp_path / "subject" + (repo / "src").mkdir(parents=True) + (repo / "src/x.py").write_text("x = 1\n") + env = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + "PATH": "/usr/bin:/bin", + } + for cmd in (["git", "init", "-q"], ["git", "add", "-A"], ["git", "commit", "-qm", "init"]): + subprocess.run(cmd, cwd=repo, check=True, env=env, capture_output=True) + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=repo, check=True, capture_output=True, text=True + ).stdout.strip() + return repo, sha + + +@pytest.fixture() +def spec_path(tmp_path: Path, repo) -> Path: + repo_dir, sha = repo + path = tmp_path / "spec.yaml" + path.write_text( + yaml.safe_dump( + { + "bench": "fake", + "repo_root": str(repo_dir), + "base_sha": sha, + "work_dir": str(tmp_path / "work"), + "funnel": { + "k_confirm": 1, + "budget": {"max_why_per_round": 1, "candidates_per_why": 1, "recombinations_per_round": 0}, + "termination": {"patience": 5, "max_rounds": 2}, + }, + } + ) + ) + return path + + +@pytest.fixture() +def fake_bench(monkeypatch): + """Install a fake bench and return its control/observation flags.""" + flags = { + "cold_interrupt_after": None, # write N trial files, then KeyboardInterrupt + "kb_on_round": None, # raise inside design of this round + "cold_writes": 0, + "design_calls": 0, + "unseal_calls": 0, + "no_unseal": False, # build the bundle without a sealed test + "unseal_fail_times": 0, # make the next N unseal calls raise + "precheck_error": None, # make the Gate0 precheck raise this message + "precheck_calls": 0, + } + + def build(ctx) -> BenchBundle: + from raven.evolver.analysis.stability_bucket import StabilityBucket, TaskStability + from raven.evolver.orchestrator.loop import EvolutionOrchestrator + from raven.evolver.orchestrator.scoring import EvalBackend, TaskEval + from raven.evolver.scheduler.anchor_selection import simple_anchor + from raven.evolver.tree.node import HarnessNode + + work = Path(ctx.spec.work_dir) + van = work / "runs" / "vanilla" + train = ["t1", "t2", "t3"] + k = ctx.spec.funnel.k_confirm + + def cold_start_done() -> int: + return len(list(van.glob("*.json"))) if van.is_dir() else 0 + + def run_cold_start() -> None: + van.mkdir(parents=True, exist_ok=True) + for tid in train: + for i in range(k): + out = van / f"{tid}_k{i}.json" + if out.exists(): + continue + if ( + flags["cold_interrupt_after"] is not None + and flags["cold_writes"] >= flags["cold_interrupt_after"] + ): + raise KeyboardInterrupt + out.write_text("{}") + flags["cold_writes"] += 1 + + stability = { + t: TaskStability(task_id=t, passes=p, attempts=3, bucket=b) + for t, p, b in [ + ("t1", 3, StabilityBucket.STABLE_PASS), + ("t2", 0, StabilityBucket.STABLE_FAIL), + ("t3", 1, StabilityBucket.BORDERLINE_1_3), + ] + } + backend = EvalBackend( + train_task_ids=train, + test_task_ids=[], + eval=lambda node, ids, k_, job, **kw: {t: TaskEval(task_id=t, passes=0, attempts=k_) for t in ids}, + cold_start=lambda: stability, + anchor=lambda affinity=None: simple_anchor(stability), + ) + + class Cand: + why = "w" + files = {"src/x.py": b"x"} + summary = "fake candidate" + + def design_fn(round_index, failure_map, parent): + flags["design_calls"] += 1 + if flags["kb_on_round"] == round_index: + flags["kb_on_round"] = None + raise KeyboardInterrupt + return [Cand()] + + def build_orchestrator(): + return EvolutionOrchestrator( + ctx.spec.funnel, + backend=backend, + diagnose_fn=lambda ri, parent: {"why_distribution": {"w": 5.0}}, + design_fn=design_fn, + apply_fn=lambda pid, patch, ri: (_ for _ in ()).throw( + AssertionError("preflight must reject before apply") + ), + preflight_fn=lambda cand, parent: False, + ) + + root = HarnessNode( + node_id="C0", + parent_id=None, + git_commit_sha=ctx.spec.base_sha, + git_branch="", + created_at=HarnessNode.utc_now(), + created_at_iter=0, + ) + + def unseal(records, orch) -> dict: + flags["unseal_calls"] += 1 + if flags["unseal_fail_times"] > 0: + flags["unseal_fail_times"] -= 1 + raise RuntimeError("sealed scoring endpoint died") + return {"best_round": len(records), "retention": 0.5} + + def precheck() -> None: + flags["precheck_calls"] += 1 + if flags["precheck_error"]: + raise RuntimeError(flags["precheck_error"]) + + return BenchBundle( + root_node_id="C0", + root_node=root, + journal_path=work / "journal" / "rounds.jsonl", + cold_start_total=len(train) * k, + cold_start_done=cold_start_done, + run_cold_start=run_cold_start, + build_orchestrator=build_orchestrator, + unseal=None if flags["no_unseal"] else unseal, + precheck=precheck, + ) + + monkeypatch.setattr(runner_mod, "load_bench", lambda name, repo_root=None: build) + return flags + + +class TestFullPipeline: + def test_run_completes_all_three_phases(self, spec_path, fake_bench, tmp_path): + assert runner_mod.cmd_run(str(spec_path)) == 0 + work = tmp_path / "work" + assert len(list((work / "runs" / "vanilla").glob("*.json"))) == 3 + journal = (work / "journal" / "rounds.jsonl").read_text().splitlines() + assert len(journal) == 2 # max_rounds=2, all candidates pruned + report = json.loads((work / "retention.json").read_text()) + assert report == {"best_round": 2, "retention": 0.5} + meta = RunMeta.load(work) + assert meta.unsealed_at and "max_rounds" in (meta.finalize_reason or "") + + def test_completed_run_refuses_to_resume(self, spec_path, fake_bench): + assert runner_mod.cmd_run(str(spec_path)) == 0 + with pytest.raises(SystemExit) as exc: + runner_mod.cmd_run(str(spec_path)) + assert exc.value.code == 2 + + def test_status_after_completion(self, spec_path, fake_bench, capsys): + runner_mod.cmd_run(str(spec_path)) + capsys.readouterr() + assert runner_mod.cmd_status(str(spec_path)) == 0 + assert "UNSEALED" in capsys.readouterr().out + + +class TestInterruptResume: + def test_cold_start_interrupt_then_resume(self, spec_path, fake_bench, tmp_path): + fake_bench["cold_interrupt_after"] = 2 + assert runner_mod.cmd_run(str(spec_path)) == 130 + van = tmp_path / "work" / "runs" / "vanilla" + assert len(list(van.glob("*.json"))) == 2 # partial work kept + + fake_bench["cold_interrupt_after"] = None + assert runner_mod.cmd_run(str(spec_path)) == 0 + # Resume filled only the missing trial (2 before + 1 after = 3 writes). + assert fake_bench["cold_writes"] == 3 + + def test_round_interrupt_then_resume_replays_journal(self, spec_path, fake_bench, tmp_path): + fake_bench["kb_on_round"] = 2 # round 1 completes, round 2 dies mid-design + assert runner_mod.cmd_run(str(spec_path)) == 130 + journal = tmp_path / "work" / "journal" / "rounds.jsonl" + assert len(journal.read_text().splitlines()) == 1 + + calls_before = fake_bench["design_calls"] + assert runner_mod.cmd_run(str(spec_path)) == 0 + # Round 1 is replayed from the journal, not re-designed: only the + # re-run of round 2 costs a design call. + assert fake_bench["design_calls"] == calls_before + 1 + assert len(journal.read_text().splitlines()) == 2 + + def test_status_midway_shows_rounds_and_no_test_numbers(self, spec_path, fake_bench, capsys): + fake_bench["kb_on_round"] = 2 + runner_mod.cmd_run(str(spec_path)) + capsys.readouterr() + assert runner_mod.cmd_status(str(spec_path)) == 0 + out = capsys.readouterr().out + assert "1 completed round" in out + assert "sealed" in out # the reminder, not the numbers + assert "retention" not in out + + +class TestFinalize: + def test_unseal_failure_leaves_run_resumable(self, spec_path, fake_bench, tmp_path): + """A dead endpoint during sealed scoring must not stamp the run; + re-running retries the unseal and succeeds.""" + fake_bench["unseal_fail_times"] = 1 + assert runner_mod.cmd_run(str(spec_path)) == 1 + work = tmp_path / "work" + meta = RunMeta.load(work) + assert meta.unsealed_at is None + assert not (work / "retention.json").exists() + + assert runner_mod.cmd_run(str(spec_path)) == 0 + meta = RunMeta.load(work) + assert meta.unsealed_at is not None + assert (work / "retention.json").exists() + assert fake_bench["unseal_calls"] == 2 + + def test_finalize_after_no_sealed_run_is_stable(self, spec_path, fake_bench, tmp_path): + """A run finalized without a sealed test must not have its stamp + rewritten (or --yes bypassed) by later finalize calls.""" + fake_bench["no_unseal"] = True + assert runner_mod.cmd_run(str(spec_path)) == 0 + work = tmp_path / "work" + meta = RunMeta.load(work) + stamped_at, reason = meta.unsealed_at, meta.finalize_reason + assert "no sealed test" in reason + + for _ in range(2): + assert runner_mod.cmd_finalize(str(spec_path), yes=False) == 0 + meta = RunMeta.load(work) + assert (meta.unsealed_at, meta.finalize_reason) == (stamped_at, reason) + assert not (work / "retention.json").exists() + assert fake_bench["unseal_calls"] == 0 + + def test_finalize_midway_unseals_and_locks(self, spec_path, fake_bench, tmp_path): + fake_bench["kb_on_round"] = 2 + runner_mod.cmd_run(str(spec_path)) + + assert runner_mod.cmd_finalize(str(spec_path), yes=False) == 2 # needs --yes + assert fake_bench["unseal_calls"] == 0 + + assert runner_mod.cmd_finalize(str(spec_path), yes=True) == 0 + assert fake_bench["unseal_calls"] == 1 + work = tmp_path / "work" + assert json.loads((work / "retention.json").read_text())["best_round"] == 1 + meta = RunMeta.load(work) + assert meta.finalize_reason == "user_finalized" + + with pytest.raises(SystemExit): + runner_mod.cmd_run(str(spec_path)) + + def test_finalize_before_any_round_refuses(self, spec_path, fake_bench): + fake_bench["cold_interrupt_after"] = 1 + runner_mod.cmd_run(str(spec_path)) # dies in cold start + assert runner_mod.cmd_finalize(str(spec_path), yes=True) == 2 + + +class TestCheck: + def test_check_runs_precheck_and_passes(self, spec_path, fake_bench, capsys): + assert runner_mod.cmd_check(str(spec_path)) == 0 + assert fake_bench["precheck_calls"] == 1 + out = capsys.readouterr().out + assert "bench precheck: OK" in out + assert "check OK" in out + + def test_check_fails_on_dead_environment(self, spec_path, fake_bench, capsys): + """A dead subject endpoint must fail `check`, not the first cold-start + trial hours later.""" + fake_bench["precheck_error"] = "subject endpoint unreachable (http://x:1/v1)" + assert runner_mod.cmd_check(str(spec_path)) == 1 + captured = capsys.readouterr() + assert "subject endpoint unreachable" in captured.err + assert "check OK" not in captured.out + + +class TestEphemeralWorktreeSweep: + def test_run_sweeps_stale_worktrees_from_a_hard_killed_run(self, spec_path, fake_bench, tmp_path, repo): + """SIGKILL leaves ephemeral worktrees behind (context managers never + ran); the next launch of the same run must sweep them and drop their + registration from the subject repo.""" + repo_dir, sha = repo + stale = tmp_path / "work" / "tmp" / "evolver-wt-stale" / "wt" + git_ops.create_worktree(repo_dir, stale, sha) + assert stale.is_dir() + + assert runner_mod.cmd_run(str(spec_path)) == 0 + assert not (tmp_path / "work" / "tmp" / "evolver-wt-stale").exists() + listed = subprocess.run( + ["git", "worktree", "list", "--porcelain"], + cwd=repo_dir, + capture_output=True, + text=True, + check=True, + ).stdout + assert "evolver-wt-stale" not in listed + + def test_ephemeral_worktrees_land_under_work_dir(self, spec_path, fake_bench, tmp_path, repo): + repo_dir, sha = repo + assert runner_mod.cmd_run(str(spec_path)) == 0 + with git_ops.worktree_at(repo_dir, sha) as wt: + assert str(wt).startswith(str(tmp_path / "work" / "tmp")) diff --git a/tests/test_evolver_scoring.py b/tests/test_evolver_scoring.py new file mode 100644 index 0000000..3472c5a --- /dev/null +++ b/tests/test_evolver_scoring.py @@ -0,0 +1,136 @@ +"""Unit tests for the scoring currency and the SOP 0 infra-rerun ladder. + +The ladder decides which measurements are salvaged and which score 0 in a +fixed denominator; a bug here hands every candidate a free lift (or silently +throws away good trials), so the rerun/keep rules are pinned down exactly. +""" + +from __future__ import annotations + +import pytest + +from raven.evolver.orchestrator.scoring import ( + TaskEval, + anchor_mean_pass_rate, + eval_with_infra_rerun, + flip_summary, + with_infra_rerun, +) + + +def _te(tid, passes, attempts, infra=0): + return TaskEval(task_id=tid, passes=passes, attempts=attempts, infra_attempts=infra) + + +class _ScriptedEval: + """EvalFn replaying a scripted list of result maps; records every call.""" + + def __init__(self, results: list[dict[str, TaskEval]]): + self.results = list(results) + self.calls: list[tuple[list[str], str]] = [] + + def __call__(self, node, task_ids, k, job_name, *, split="train"): + self.calls.append((list(task_ids), job_name)) + return self.results.pop(0) + + +class TestTaskEval: + def test_pass_rate(self): + assert _te("t", 2, 3).pass_rate == pytest.approx(2 / 3) + assert _te("t", 0, 0).pass_rate == 0.0 + + def test_anchor_mean_missing_scores_zero(self): + assert anchor_mean_pass_rate({"t1": _te("t1", 3, 3)}, ["t1", "t2"]) == 0.5 + with pytest.raises(ValueError, match="non-empty"): + anchor_mean_pass_rate({}, []) + + +class TestInfraRerunLadder: + def test_clean_eval_triggers_no_rerun(self): + fake = _ScriptedEval([{"t1": _te("t1", 1, 3), "t2": _te("t2", 3, 3)}]) + out = eval_with_infra_rerun(fake, None, ["t1", "t2"], 3, "job") + assert len(fake.calls) == 1 + assert out["t1"].passes == 1 + + def test_infra_task_rerun_and_salvaged(self): + fake = _ScriptedEval( + [ + {"t1": _te("t1", 0, 3, infra=2), "t2": _te("t2", 3, 3)}, + {"t1": _te("t1", 2, 3, infra=0)}, + ] + ) + out = eval_with_infra_rerun(fake, None, ["t1", "t2"], 3, "job") + assert out["t1"].passes == 2 and out["t1"].infra_attempts == 0 + # Only the contaminated task is re-scored, under the ladder job name. + assert fake.calls[1] == (["t1"], "job_infra_rerun1") + + def test_missing_task_is_infra_by_definition(self): + fake = _ScriptedEval( + [ + {"t1": _te("t1", 3, 3)}, # t2 never came back + {"t2": _te("t2", 1, 3)}, + ] + ) + out = eval_with_infra_rerun(fake, None, ["t1", "t2"], 3, "job") + assert fake.calls[1][0] == ["t2"] + assert out["t2"].passes == 1 + + def test_keeps_measurement_with_fewest_infra(self): + # The rerun came back just as contaminated: keep the original (strictly + # fewer infra trials required to replace). + first = _te("t1", 1, 3, infra=1) + fake = _ScriptedEval( + [ + {"t1": first}, + {"t1": _te("t1", 0, 3, infra=1)}, + {"t1": _te("t1", 0, 3, infra=2)}, + ] + ) + out = eval_with_infra_rerun(fake, None, ["t1"], 3, "job") + assert out["t1"] is first + assert len(fake.calls) == 3 # base + 2 reruns, then the ladder ends + + def test_persistent_infra_survives_and_scores_low(self): + results = [{"t1": _te("t1", 0, 3, infra=3)} for _ in range(3)] + fake = _ScriptedEval(results) + out = eval_with_infra_rerun(fake, None, ["t1"], 3, "job", max_reruns=2) + assert len(fake.calls) == 3 + assert out["t1"].infra_attempts == 3 # left to score 0, never dropped + + def test_wrapper_is_identity_at_zero_reruns(self): + inner = _ScriptedEval([]) + assert with_infra_rerun(inner, 0) is inner + + def test_wrapper_applies_ladder(self): + fake = _ScriptedEval( + [ + {"t1": _te("t1", 0, 3, infra=1)}, + {"t1": _te("t1", 2, 3)}, + ] + ) + wrapped = with_infra_rerun(fake, 1) + out = wrapped(None, ["t1"], 3, "job") + assert out["t1"].passes == 2 and len(fake.calls) == 2 + + +class TestFlipSummary: + def test_partial_rescue_and_regression_accounting(self): + cand = {"t1": _te("t1", 2, 3), "t2": _te("t2", 0, 3), "t3": _te("t3", 3, 3)} + ctrl = {"t1": _te("t1", 1, 3), "t2": _te("t2", 1, 3), "t3": _te("t3", 3, 3)} + s = flip_summary(cand, ctrl, ["t1", "t2", "t3"]) + assert s["rescued"] == ["t1"] # 1/3 -> 2/3 counts as rescued + assert s["regressed"] == ["t2"] + assert s["still_failing"] == ["t1", "t2"] # anything below 1.0 + + def test_missing_arm_scores_zero(self): + cand = {"t1": _te("t1", 2, 3)} + s = flip_summary(cand, {}, ["t1", "t2"]) + assert s["rescued"] == ["t1"] # 0.0 -> 2/3 + assert s["n_regressed"] == 0 + + def test_id_lists_capped_but_counts_exact(self): + ids = [f"t{i}" for i in range(15)] + cand = {t: _te(t, 3, 3) for t in ids} + s = flip_summary(cand, {}, ids, max_ids=12) + assert s["n_rescued"] == 15 + assert len(s["rescued"]) == 12