From 0b5c615d5f5b381b0d24fa89a0503ca0c17a905b Mon Sep 17 00:00:00 2001 From: Michael Hohlios Date: Fri, 2 Oct 2026 11:11:41 -0400 Subject: [PATCH] Support Codex gold candidates and preserve incomplete audit evidence Signed-off-by: Michael Hohlios --- .github/workflows/lint.yml | 1 + eval/corpus/subsets/audit-adjudicated.json | 16 ++++ eval/runners/candidate-codex.py | 5 ++ eval/runners/gold.sh | 31 +++++++- eval/runners/test_gold_codex.py | 76 +++++++++++++++++++ .../2026-10-02-audit-adjudicated/README.md | 56 ++++++++++++++ .../crb-grafana-79265/candidate-output.json | 46 +++++++++++ .../audit/crb-grafana-79265/meta.json | 14 ++++ .../crb-keycloak-36880/candidate-output.json | 64 ++++++++++++++++ .../audit/crb-keycloak-36880/meta.json | 14 ++++ .../crb-keycloak-38446/candidate-output.json | 69 +++++++++++++++++ .../audit/crb-keycloak-38446/meta.json | 15 ++++ .../crb-grafana-79265/candidate-output.json | 1 + .../review/crb-grafana-79265/meta.json | 14 ++++ .../crb-keycloak-36880/candidate-output.json | 59 ++++++++++++++ .../review/crb-keycloak-36880/meta.json | 14 ++++ .../crb-keycloak-38446/candidate-output.json | 70 +++++++++++++++++ .../review/crb-keycloak-38446/meta.json | 15 ++++ .../2026-10-02-audit-adjudicated/summary.json | 14 ++++ 19 files changed, 591 insertions(+), 3 deletions(-) create mode 100644 eval/corpus/subsets/audit-adjudicated.json create mode 100644 eval/runners/test_gold_codex.py create mode 100644 eval/runs/2026-10-02-audit-adjudicated/README.md create mode 100644 eval/runs/2026-10-02-audit-adjudicated/audit/crb-grafana-79265/candidate-output.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/audit/crb-grafana-79265/meta.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-36880/candidate-output.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-36880/meta.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-38446/candidate-output.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-38446/meta.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/review/crb-grafana-79265/candidate-output.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/review/crb-grafana-79265/meta.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-36880/candidate-output.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-36880/meta.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-38446/candidate-output.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-38446/meta.json create mode 100644 eval/runs/2026-10-02-audit-adjudicated/summary.json diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 211fa47..17c1153 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,6 +24,7 @@ jobs: run: | python3 eval/runners/test_match.py python3 eval/runners/test_gold_judge.py + python3 eval/runners/test_gold_codex.py - name: Check internal links uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2 with: diff --git a/eval/corpus/subsets/audit-adjudicated.json b/eval/corpus/subsets/audit-adjudicated.json new file mode 100644 index 0000000..409d7c4 --- /dev/null +++ b/eval/corpus/subsets/audit-adjudicated.json @@ -0,0 +1,16 @@ +{ + "name": "audit-adjudicated", + "rationale": "Seven adjudicated cases with eligible defects, two previously matched controls, and one entirely excluded Ruby case. All eligible findings in the original zero-recall cluster are covered. Not a full-corpus or expected-clean sweep.", + "case_ids": [ + "crb-grafana-79265", + "crb-grafana-97529", + "crb-keycloak-36880", + "crb-keycloak-38446", + "crb-sentry-greptile-3", + "crb-sentry-greptile-5", + "crb-cal.com-14943", + "crb-grafana-94942", + "crb-sentry-67876", + "crb-discourse-graphite-9" + ] +} diff --git a/eval/runners/candidate-codex.py b/eval/runners/candidate-codex.py index 6d476d4..e6ad222 100644 --- a/eval/runners/candidate-codex.py +++ b/eval/runners/candidate-codex.py @@ -100,10 +100,15 @@ def main(): "--output-schema", str(here / "codex-output.schema.json"), "--output-last-message", str(out / "candidate-output.json"), prompt, ] + if args.tier == "audit": + command[2:2] = ["--enable", "multi_agent"] before = snapshot(workdir) started = time.monotonic() meta = {"host": "codex", "tier": args.tier, "base_sha": base, "head_sha": head, + "effective_base_sha": git(workdir, "merge-base", base, head).decode().strip(), "requested_model": args.model, "reasoning_effort": args.effort, + "plugin_revision": git(here.parents[1], "rev-parse", "HEAD").decode().strip(), + "plugin_version": (here.parents[1] / "skills/reviso/VERSION").read_text().strip(), "cli_version": subprocess.check_output(["codex", "--version"], text=True).strip(), "status": "failed"} try: diff --git a/eval/runners/gold.sh b/eval/runners/gold.sh index 54f478c..21d7736 100755 --- a/eval/runners/gold.sh +++ b/eval/runners/gold.sh @@ -33,7 +33,14 @@ CORPUS_FILE="${CORPUS_FILE:-$CORPUS_DIR/public.jsonl}" # so this is a no-op there. CORPUS_BASE=$(cd "$(dirname "$CORPUS_FILE")" && pwd) CACHE="$(cd "$HERE/.." && pwd)/.cache/clones" -mkdir -p "$OUT" "$CACHE" +mkdir -p "$CACHE" +case "${CANDIDATE_HOST:-claude}" in + claude) mkdir -p "$OUT" ;; + codex) + : "${CANDIDATE_CODEX_MODEL:?CANDIDATE_CODEX_MODEL is required}" + : "${CANDIDATE_CODEX_EFFORT:?CANDIDATE_CODEX_EFFORT is required}" ;; + *) echo "gold.sh: unknown CANDIDATE_HOST '${CANDIDATE_HOST}' (claude|codex)" >&2; exit 1 ;; +esac ENTRY=$(jq -c --arg id "$CASE" 'select(.id == $id)' "$CORPUS_FILE") [ -n "$ENTRY" ] || { echo "gold.sh: case '$CASE' not in $CORPUS_FILE" >&2; exit 1; } @@ -90,16 +97,34 @@ else if [ ! -d "$CDIR" ]; then git clone -q --filter=blob:none "$CLONE_URL" "$CDIR" fi - git -C "$CDIR" fetch -q origin "$BASE" "$HEAD_SHA" + git -C "$CDIR" fetch -q --no-write-fetch-head origin "$BASE" "$HEAD_SHA" # A worktree (not a shared clone) so the cache's promisor config still # serves lazy blob fetches for the blobless clone during checkout. RWD="$WD/repo" git -C "$CDIR" worktree add -q --detach "$RWD" "$HEAD_SHA" + if [ "${CANDIDATE_HOST:-claude}" = "codex" ]; then + # The read-only candidate cannot lazy-fetch missing blobs. Materialize + # the reviewed diff and its bounded history before entering the sandbox. + DIFF_BASE=$(git -C "$RWD" merge-base "$BASE" "$HEAD_SHA") + git -C "$RWD" diff --no-ext-diff "$DIFF_BASE" "$HEAD_SHA" >/dev/null + git -C "$RWD" rev-list --objects --no-object-names "$DIFF_BASE..$HEAD_SHA" \ + | git -C "$RWD" cat-file --batch >/dev/null + fi fi # Tier named explicitly at the call site — never left to whatever the # environment happened to carry in. -REVISO_TIER="$REVIEW_TIER" sh "$HERE/candidate.sh" "$RWD" "$BASE" "$HEAD_SHA" "$OUT" +if [ "${CANDIDATE_HOST:-claude}" = "codex" ]; then + "${REVISO_PYTHON:-python3}" "$HERE/candidate-codex.py" "$RWD" "$BASE" "$HEAD_SHA" "$OUT" \ + --tier "$REVIEW_TIER" --model "$CANDIDATE_CODEX_MODEL" --effort "$CANDIDATE_CODEX_EFFORT" \ + --timeout "${CANDIDATE_TIMEOUT:-900}" + if [ "$(jq -r '.status' "$OUT/meta.json")" != "returned" ]; then + echo "gold.sh: incomplete Codex coverage — not scored" >&2 + exit 1 + fi +else + REVISO_TIER="$REVIEW_TIER" sh "$HERE/candidate.sh" "$RWD" "$BASE" "$HEAD_SHA" "$OUT" +fi # Judge against the labels (tiering + metrics live in gold-judge.sh so a # recorded run can be re-judged when calibration moves). diff --git a/eval/runners/test_gold_codex.py b/eval/runners/test_gold_codex.py new file mode 100644 index 0000000..1de469e --- /dev/null +++ b/eval/runners/test_gold_codex.py @@ -0,0 +1,76 @@ +"""Exercise the Codex gold adapter and reject incomplete coverage offline.""" + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +GOLD = Path(__file__).with_name("gold.sh") + + +class CodexGoldTests(unittest.TestCase): + def run_gold(self, incomplete=False, tier="review"): + with tempfile.TemporaryDirectory(prefix="reviso-codex-gold-test-") as directory: + root = Path(directory) + (root / "labels.json").write_text(json.dumps({"expected_clean": True, "findings": []})) + (root / "fixture.json").write_text(json.dumps({"files": [ + {"filename": "sample.py", "status": "added", "patch": "+answer = 42"}]})) + (root / "corpus.jsonl").write_text(json.dumps({"id": "fixture", "synthetic": True, + "fixture": "fixture.json", "labels": "labels.json"}) + "\n") + fake = root / "codex" + fake.write_text('''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +if '--version' in sys.argv: + print('codex fixture') + sys.exit(0) +assert sys.argv[sys.argv.index('--sandbox') + 1] == 'read-only' +assert '--ignore-user-config' in sys.argv +assert '--ignore-rules' in sys.argv +assert sys.argv[sys.argv.index('--model') + 1] == 'fixture-model' +tier = os.environ['REVISO_TIER'] +if tier == 'audit': + assert sys.argv[sys.argv.index('--enable') + 1] == 'multi_agent' +lenses = ['bugs', 'conventions', 'history', 'comments', 'slop', 'deterministic'] +if tier == 'audit': + lenses.append('prior-reviews') +coverage = [{'lens': lens, 'outcome': 'returned', 'reason': ''} for lens in lenses] +if os.environ['GOLD_TEST_INCOMPLETE'] == '1': + coverage[0]['outcome'] = 'no result' + coverage[0]['reason'] = 'fixture unavailable' +output = Path(sys.argv[sys.argv.index('--output-last-message') + 1]) +output.write_text(json.dumps({'report': 'No issues found.', 'findings': [], 'coverage': coverage})) +''') + fake.chmod(0o755) + out = root / "out" + env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ["PATH"], + CORPUS_FILE=str(root / "corpus.jsonl"), REVISO_TIER=tier, + CANDIDATE_HOST="codex", CANDIDATE_CODEX_MODEL="fixture-model", + CANDIDATE_CODEX_EFFORT="medium", REVISO_PYTHON=os.sys.executable, + GOLD_TEST_INCOMPLETE="1" if incomplete else "0") + result = subprocess.run(["sh", str(GOLD), "fixture", str(out)], + env=env, capture_output=True, text=True) + meta = json.loads((out / "meta.json").read_text()) + return result, meta, (out / "gold-judge.json").exists() + + def test_codex_gold_keeps_the_repository_unchanged(self): + for tier in ("review", "audit"): + with self.subTest(tier=tier): + result, meta, judged = self.run_gold(tier=tier) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(meta["repository_unchanged"]) + self.assertEqual(meta["host"], "codex") + self.assertTrue(judged) + + def test_incomplete_coverage_is_not_scored(self): + result, meta, judged = self.run_gold(incomplete=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(meta["status"], "incomplete") + self.assertFalse(judged) + + +if __name__ == "__main__": + unittest.main() diff --git a/eval/runs/2026-10-02-audit-adjudicated/README.md b/eval/runs/2026-10-02-audit-adjudicated/README.md new file mode 100644 index 0000000..76eea3b --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/README.md @@ -0,0 +1,56 @@ +# Audit comparison attempt — 2026-10-02 + +**Blocked, zero scored cases.** This is diagnostic evidence for issue #18, +not a recall result. The planned ten cases are recorded in +`eval/corpus/subsets/audit-adjudicated.json` and cover all nine eligible +findings from the adjudicated zero-recall cluster, two previously matched +controls, and one entirely excluded Ruby case. + +The candidate and matcher used Codex `gpt-6.1-sol`, medium effort, +CLI 0.160.0; the plugin was 0.14.0 at the revision in each `meta.json`. +Claude was unavailable. These results cannot establish historical Claude +performance or compare providers. + +The initial Grafana run lacked diff blobs in a partial clone. The adapter +now hydrates the merge-base diff and objects between the pinned revisions +before entering the read-only sandbox. Subsequent Keycloak runs could +inspect current source but some historical blame still required missing +promisor objects. Full history is not promised by this preparation. + +The audit could start several independent finders, then hit the host +agent-thread limit. Comments, anti-slop, prior-reviews, and independent +verification could not all complete. The workflow correctly recorded +missing coverage instead of treating fallback work as a full audit. +The sweeps were stopped after these repeat failures. Completed diagnostic +outputs and metadata are preserved here; interrupted attempts and raw +host event streams remain in the ignored cache. + +Both tiers reported the repository unchanged. No matcher judgments or +gold metrics were produced for incomplete candidates. The comparison +needs a host with sufficient releasable audit agent capacity and prepared +historical objects before issue #18 can be resolved. + +## Reproduction after resolving the blockers + +Install `jsonschema` in the Python environment used by `REVISO_PYTHON`. +Generate a JSONL file in `eval/.cache/` by selecting the subset IDs from +`eval/corpus/public.jsonl` and prefixing each labels/fixture path with +`../corpus/`. Use a fresh output directory for each tier, and these env vars: + +```sh +REVISO_TIER=audit +CANDIDATE_HOST=codex +CANDIDATE_CODEX_MODEL=gpt-6.1-sol +CANDIDATE_CODEX_EFFORT=medium +MATCH_HOST=codex +MATCH_CODEX_MODEL=gpt-6.1-sol +MATCH_CODEX_EFFORT=medium +REVISO_SWEEP_SUBSET=audit-adjudicated +CORPUS_FILE=/absolute/path/to/subset.jsonl +REVISO_PYTHON=/absolute/path/to/python-with-jsonschema +``` + +Export them, then run `sh eval/runners/sweep.sh gold `. +Repeat with `REVISO_TIER=review`. Only complete coverage can be scored. +Do not retry into an existing candidate directory: preserve the failed +attempt and use a fresh output path. diff --git a/eval/runs/2026-10-02-audit-adjudicated/audit/crb-grafana-79265/candidate-output.json b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-grafana-79265/candidate-output.json new file mode 100644 index 0000000..86cbce3 --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-grafana-79265/candidate-output.json @@ -0,0 +1,46 @@ +{ + "report": "## Reviso audit — detached HEAD vs 50f4e78a39914711d0d231a501ba215becf17ebc (5 commits, 11 files)\nBase: 50f4e78a39914711d0d231a501ba215becf17ebc via flag — merge-base 7ddcd0c961\n\nAudit unavailable: missing Git objects prevented reading every changed-file diff. No findings could be verified; this is not a clean audit.\n\nChecked: none.\nNot checked: bugs, conventions, history, comments, slop, prior-reviews (required diff unavailable); deterministic (Git diff failed); best-practices (no --web).\nSkipped: skip-tier files could not be determined because triage was unavailable.\n\n--- explain: pipeline diagnostics (not review findings) ---\nFinders: bugs no result, conventions no result, history no result, comments no result, slop no result, prior-reviews no result.\nDeterministic: no result. The detector emitted [] after Git failed; that output does not establish a clean pass.\nTriage, independent finders, evidence gathering, and scoring did not run because context assembly failed.\nCandidates before the gate: unavailable.\nNo files were written, feedback sent, or web lookups performed.", + "findings": [], + "coverage": [ + { + "lens": "bugs", + "outcome": "no result", + "reason": "Missing Git objects prevented assembling the diff; independent finder did not run." + }, + { + "lens": "conventions", + "outcome": "no result", + "reason": "Missing Git objects prevented assembling the diff; independent finder did not run." + }, + { + "lens": "history", + "outcome": "no result", + "reason": "Commit metadata was available, but required diff objects were missing; independent finder did not run." + }, + { + "lens": "comments", + "outcome": "no result", + "reason": "Missing Git objects prevented assembling the diff; code-comment finder did not run." + }, + { + "lens": "slop", + "outcome": "no result", + "reason": "Missing Git objects prevented assembling the diff; anti-slop finder did not run." + }, + { + "lens": "prior-reviews", + "outcome": "no result", + "reason": "Required diff unavailable; prior-review finder and lookup did not run." + }, + { + "lens": "deterministic", + "outcome": "no result", + "reason": "Detector invocation encountered a Git diff failure from missing objects; its empty array was unusable." + }, + { + "lens": "best-practices", + "outcome": "skipped", + "reason": "no --web" + } + ] +} \ No newline at end of file diff --git a/eval/runs/2026-10-02-audit-adjudicated/audit/crb-grafana-79265/meta.json b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-grafana-79265/meta.json new file mode 100644 index 0000000..2a0e905 --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-grafana-79265/meta.json @@ -0,0 +1,14 @@ +{ + "host": "codex", + "tier": "audit", + "base_sha": "50f4e78a39914711d0d231a501ba215becf17ebc", + "head_sha": "bbd8c507cdf56f6b884373809e0b24abd4a2353d", + "requested_model": "gpt-6.1-sol", + "reasoning_effort": "medium", + "plugin_revision": "2a41361b9aac174a18e47c0498efd53edcd6453e", + "plugin_version": "0.14.0", + "cli_version": "codex-cli 0.160.0", + "status": "incomplete", + "duration_ms": 150392, + "repository_unchanged": true +} diff --git a/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-36880/candidate-output.json b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-36880/candidate-output.json new file mode 100644 index 0000000..756d537 --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-36880/candidate-output.json @@ -0,0 +1,64 @@ +{ + "report": "## Reviso audit — detached HEAD 1950a51102 vs b8c1cd410f4499c77a8b372decfe1718b80ccfd2 (1 commit, 10 files)\nBase: b8c1cd410f4499c77a8b372decfe1718b80ccfd2 via flag — merge-base b7b84d75d1\n\nFound 2 issues:\n\n1. [P1][conf 95] Global view permission bypasses client-specific denials — services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java:78\n Failure: An all-clients VIEW grant causes canView(client) to return true before evaluating an explicit denial for that client. Collection listing also bypasses individual checks when global canView() succeeds.\n Fix: Reserve unconditional view shortcuts for built-in roles; evaluate client-specific permissions before global fallback and preserve that precedence in list filtering.\n (bugs; corroborated by history)\n\n2. [P1][conf 90] Global manage permission fails inherited list and view checks — services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java:87\n Failure: An administrator granted only all-clients MANAGE can create clients and client scopes but receives 403 when listing them without query roles. With query-clients, the client-scope list is empty.\n Fix: Include MANAGE in the global view decision while preserving client-specific permission precedence.\n (bugs)\n\nChecked: bugs, conventions, history, deterministic.\nNot checked: comments, slop, prior-reviews (host agent-thread limit).\nSkipped: nothing.\n\n--- explain: pipeline diagnostics (not review findings) ---\nFinders: conventions 1, bugs 4, history 1, prior-reviews no result, comments no result, slop no result. Deterministic: 0.\nCandidates before the gate (6):\n [conventions] AdminPermissionsSchema.java:68 — score 50, dropped: rubric-score; documentation impact insufficiently established for this experimental feature.\n [bugs] ClientPermissionsV2.java:78 — score 95, reported.\n [bugs] ClientPermissionsV2.java:87 — score 90, reported.\n [bugs] ClientPermissionsV2.java:138 — score 50, dropped: rubric-score; the sole repository consumer is gated to V1.\n [bugs] ClientPermissionsV2.java:140 — score 50, dropped: rubric-score; the sole repository consumer is gated to V1.\n [history] ClientPermissionsV2.java:76 — score 95, reported; merged with the bugs finding at line 78.\n\nAudit coverage is incomplete: remaining finders and separate evidence agents could not be launched after the host rejected another agent with 'agent thread limit reached'. Candidate evidence was checked directly by the orchestrator. History returned a findings array, but missing promisor blobs prevented full blame and historical diff inspection; available reachable history was used. No repository code was executed, files written, web lookups performed, or feedback sent.\n\nWrong about something? Say which finding — I can file feedback (metadata-only by default).", + "findings": [ + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 78, + "severity": "P1", + "dimension": "bugs", + "title": "Global view permission bypasses client-specific denials", + "failure_scenario": "An administrator has an all-clients VIEW grant and an explicit VIEW denial on a sensitive client. canView(client) accepts the global grant before evaluating the client-specific resource, exposing the denied client.", + "suggested_fix": "Short-circuit only for built-in administrative roles, evaluate client-specific permissions before global fallback, and make collection filtering respect those individual decisions.", + "evidence": "ClientPermissionsV2.java:78 accepts global canView() before the specific-resource evaluation at lines 214–224; ClientsResource.java:153 likewise bypasses individual checks when global view succeeds. UserResourceTypeEvaluationTest.java:203–212 and reachable commit bf355f83d3 establish that an individual denial takes precedence over an all-resource grant.", + "confidence": 95 + }, + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 87, + "severity": "P1", + "dimension": "bugs", + "title": "Global manage permission fails inherited list and view checks", + "failure_scenario": "An administrator granted only all-clients MANAGE can create clients and client scopes but receives 403 when listing either collection without query roles. Adding query-clients permits the client-scope request but returns an empty list.", + "suggested_fix": "Include the all-clients MANAGE scope in the global view decision while preserving client-specific permission precedence.", + "evidence": "ClientPermissionsV2.java:73 and :109 accept global MANAGE, but :87 checks only VIEW; inherited ClientPermissions.java:241–259 and :489 derive collection access from canView(). ClientScopesResource.java:91–95 requires list permission and returns an empty stream when canViewClientScopes() is false.", + "confidence": 90 + } + ], + "coverage": [ + { + "lens": "bugs", + "outcome": "returned", + "reason": "Independent finder returned 4 candidates." + }, + { + "lens": "conventions", + "outcome": "returned", + "reason": "Independent finder returned 1 candidate." + }, + { + "lens": "history", + "outcome": "returned", + "reason": "Independent finder returned 1 candidate using reachable local history; missing promisor blobs limited blame and historical diffs." + }, + { + "lens": "comments", + "outcome": "no result", + "reason": "Finder launch failed with host agent-thread limit reached." + }, + { + "lens": "slop", + "outcome": "no result", + "reason": "No agent capacity remained to launch the anti-slop finder." + }, + { + "lens": "prior-reviews", + "outcome": "no result", + "reason": "No agent capacity remained to launch the prior-reviews finder." + }, + { + "lens": "deterministic", + "outcome": "returned", + "reason": "Read-only detector suite completed and returned 0 findings." + } + ] +} \ No newline at end of file diff --git a/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-36880/meta.json b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-36880/meta.json new file mode 100644 index 0000000..b51b1fd --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-36880/meta.json @@ -0,0 +1,14 @@ +{ + "host": "codex", + "tier": "audit", + "base_sha": "b8c1cd410f4499c77a8b372decfe1718b80ccfd2", + "head_sha": "1950a511026d520a7329c7b6b9ee60a4af8f8b55", + "requested_model": "gpt-6.1-sol", + "reasoning_effort": "medium", + "plugin_revision": "2a41361b9aac174a18e47c0498efd53edcd6453e", + "plugin_version": "0.14.0", + "cli_version": "codex-cli 0.160.0", + "status": "incomplete", + "duration_ms": 327407, + "repository_unchanged": true +} diff --git a/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-38446/candidate-output.json b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-38446/candidate-output.json new file mode 100644 index 0000000..697027a --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-38446/candidate-output.json @@ -0,0 +1,69 @@ +{ + "report": "## Reviso review — detached HEAD ce2dcc537b vs 85737f52b59aa2e3403c9b8e48acf71ccc5bfae3 (1 commit, 8 files)\nBase: 85737f52b59aa2e3403c9b8e48acf71ccc5bfae3 via flag — merge-base 85737f52b5\nAudit unavailable: independent agent delegation failed. This is the required single-pass review fallback.\n\nFound 2 issues:\n\n1. [P1][conf 95] Invalidate local recovery codes after external regeneration — server-spi-private/src/main/java/org/keycloak/utils/CredentialHelper.java:126\n Failure: A federated user regenerates previously local recovery codes into external storage. The old local codes remain valid through validation fallback.\n Fix: Remove existing local recovery credentials after successful external creation.\n (bugs)\n\n2. [P2][conf 100] Consume recovery codes when the test provider validates them — testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java:340\n Failure: The provider accepts the same recovery code repeatedly, and every subsequent login still requests the first code.\n Fix: Validate the next unused code and persist its consumption; return the remaining credential state.\n (bugs)\n\nChecked: bugs, conventions, comments, slop, deterministic.\nNot checked: history (missing Git object prevented completing inspection), prior-reviews (unavailable audit agent; feedback not retrieved), best-practices (no --web).\nSkipped: nothing.\n\n--- explain: pipeline diagnostics (not review findings) ---\nLenses: bugs 2, conventions 0, history no result, comments 0, slop 0, prior-reviews no result, deterministic 0, best-practices skipped (no --web).\nIndependent triage, finders, and evidence agents did not run; candidates were self-verified in the review fallback.\nCandidates before the gate (2):\n [bugs] server-spi-private/src/main/java/org/keycloak/utils/CredentialHelper.java:126 score 95 reported\n [bugs] testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java:340 score 100 reported", + "findings": [ + { + "file": "server-spi-private/src/main/java/org/keycloak/utils/CredentialHelper.java", + "line": 126, + "severity": "P1", + "dimension": "bugs", + "title": "Invalidate local recovery codes after external regeneration", + "failure_scenario": "A federated user with existing local recovery codes regenerates them after their storage provider gains recovery-code support. Successful external creation leaves the local credential intact, so an old code rejected by external storage still authenticates through the local validator.", + "suggested_fix": "After successful external creation, remove existing locally stored recovery-code credentials so regeneration invalidates the previous set.", + "evidence": "CredentialHelper.java:125-129 bypasses RecoveryAuthnCodesCredentialProvider.createCredential, whose lines 39-42 remove the previous local credential. UserCredentialManager.java:70-80 retains externally rejected inputs for local validation, and RecoveryAuthnCodesCredentialProvider.java:105-118 accepts matching old local codes.", + "confidence": 95 + }, + { + "file": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java", + "line": 340, + "severity": "P2", + "dimension": "bugs", + "title": "Consume recovery codes when the test provider validates them", + "failure_scenario": "After a successful federated recovery-code login, the provider retains every code and accepts the same code on subsequent logins. Its credential enumeration reconstructs the original full list, so the browser keeps requesting the first code and exhaustion never triggers regeneration.", + "suggested_fix": "Validate only the next unused recovery code, persist its consumption, and expose the remaining credential state from getCredentials.", + "evidence": "BackwardsCompatibilityUserStorage.java:340 returns an anyMatch result without updating stored codes, while lines 237-241 reconstruct all original codes on every read. The local provider at RecoveryAuthnCodesCredentialProvider.java:111-118 validates the next code, removes it, and persists the updated credential.", + "confidence": 100 + } + ], + "coverage": [ + { + "lens": "bugs", + "outcome": "returned", + "reason": "Single-pass review fallback completed; 2 candidates." + }, + { + "lens": "conventions", + "outcome": "returned", + "reason": "Single-pass review fallback completed; 0 candidates; no repository AGENTS.md or CLAUDE.md found." + }, + { + "lens": "history", + "outcome": "no result", + "reason": "Inspection incomplete: blame required a missing promisor object and retrieval failed; independent audit agent unavailable." + }, + { + "lens": "comments", + "outcome": "returned", + "reason": "Code-comment lens completed in single-pass review fallback; 0 candidates." + }, + { + "lens": "slop", + "outcome": "returned", + "reason": "Anti-slop and identifier duplication checks completed in single-pass review fallback; 0 candidates." + }, + { + "lens": "prior-reviews", + "outcome": "no result", + "reason": "Independent audit agent unavailable; prior PR feedback was not retrieved." + }, + { + "lens": "deterministic", + "outcome": "returned", + "reason": "Detector suite returned an empty findings array; 0 findings." + }, + { + "lens": "best-practices", + "outcome": "skipped", + "reason": "no --web" + } + ] +} \ No newline at end of file diff --git a/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-38446/meta.json b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-38446/meta.json new file mode 100644 index 0000000..dd13635 --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/audit/crb-keycloak-38446/meta.json @@ -0,0 +1,15 @@ +{ + "host": "codex", + "tier": "audit", + "base_sha": "85737f52b59aa2e3403c9b8e48acf71ccc5bfae3", + "head_sha": "ce2dcc537b4b1f98a335e88cfe8e0b804483252a", + "effective_base_sha": "85737f52b59aa2e3403c9b8e48acf71ccc5bfae3", + "requested_model": "gpt-6.1-sol", + "reasoning_effort": "medium", + "plugin_revision": "2a41361b9aac174a18e47c0498efd53edcd6453e", + "plugin_version": "0.14.0", + "cli_version": "codex-cli 0.160.0", + "status": "incomplete", + "duration_ms": 167514, + "repository_unchanged": true +} diff --git a/eval/runs/2026-10-02-audit-adjudicated/review/crb-grafana-79265/candidate-output.json b/eval/runs/2026-10-02-audit-adjudicated/review/crb-grafana-79265/candidate-output.json new file mode 100644 index 0000000..513cebc --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/review/crb-grafana-79265/candidate-output.json @@ -0,0 +1 @@ +{"report":"## Reviso review — detached HEAD bbd8c507cd vs 50f4e78a39914711d0d231a501ba215becf17ebc (5 commits, 11 files)\nBase: 50f4e78a39914711d0d231a501ba215becf17ebc via flag — merge-base 7ddcd0c961\n\nNo verified findings; review incomplete. Missing Git blobs prevented reading the changed hunks in all 11 files. This result does not establish that the changes are clean.\n\nChecked: none.\nNot checked: bugs, conventions, history, comments, slop (missing diff blobs); deterministic (Git diff failed inside the detector pipeline; its empty output is invalid).\nSkipped: no files intentionally excluded. Best-practices skipped: no --web.\n\n--- explain: pipeline diagnostics (not review findings) ---\nLenses: bugs unavailable, conventions unavailable, history unavailable, comments unavailable, slop unavailable, deterministic unavailable.\nCandidates before the gate: 0 recorded; unavailable coverage is not a zero-candidate conclusion.\nCommit intent and current source were partially inspected, but changed-line and pre-existing-issue checks could not be completed. No files were written, feedback sent, or web lookups performed.","findings":[],"coverage":[{"lens":"bugs","outcome":"no result","reason":"Missing Git blobs prevented inspecting changed hunks and verifying that potential defects were introduced by this change."},{"lens":"conventions","outcome":"no result","reason":"The final-state diff was unavailable, preventing a complete conventions review."},{"lens":"history","outcome":"no result","reason":"Commit messages were available, but missing diff blobs prevented checking changed behavior against ancestral implementations."},{"lens":"comments","outcome":"no result","reason":"Current comments were partially read, but changed hunks were unavailable for checking compliance with nearby invariants."},{"lens":"slop","outcome":"no result","reason":"Missing diff blobs prevented identifying and reviewing added blocks across the change."},{"lens":"deterministic","outcome":"no result","reason":"The detector was invoked, but its Git diff failed on a missing blob. The pipeline emitted [] despite that failure, so the output cannot establish a clean pass."}]} \ No newline at end of file diff --git a/eval/runs/2026-10-02-audit-adjudicated/review/crb-grafana-79265/meta.json b/eval/runs/2026-10-02-audit-adjudicated/review/crb-grafana-79265/meta.json new file mode 100644 index 0000000..3b5f919 --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/review/crb-grafana-79265/meta.json @@ -0,0 +1,14 @@ +{ + "host": "codex", + "tier": "review", + "base_sha": "50f4e78a39914711d0d231a501ba215becf17ebc", + "head_sha": "bbd8c507cdf56f6b884373809e0b24abd4a2353d", + "requested_model": "gpt-6.1-sol", + "reasoning_effort": "medium", + "plugin_revision": "2a41361b9aac174a18e47c0498efd53edcd6453e", + "plugin_version": "0.14.0", + "cli_version": "codex-cli 0.160.0", + "status": "incomplete", + "duration_ms": 143860, + "repository_unchanged": true +} diff --git a/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-36880/candidate-output.json b/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-36880/candidate-output.json new file mode 100644 index 0000000..0c607c4 --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-36880/candidate-output.json @@ -0,0 +1,59 @@ +{ + "report": "## Reviso review — detached HEAD (1950a51102) vs b8c1cd410f4499c77a8b372decfe1718b80ccfd2 (1 commit, 10 files)\nBase: b8c1cd410f4499c77a8b372decfe1718b80ccfd2 via flag — merge-base b7b84d75d1\n\nFound 2 issues:\n\n1. [P1][conf 95] All-clients view grants bypass client-specific denies — services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java:87\n Failure: Grant VIEW for all clients and deny VIEW for one client. Both direct retrieval and listing still expose that client because global canView() bypasses its permission evaluation.\n Fix: Reserve unconditional visibility for admin roles, evaluate fine-grained visibility per client, and separately authorize entry to the list endpoint.\n (bugs)\n\n2. [P2][conf 95] Manage permission cannot list the client scopes it authorizes — services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java:121\n Failure: Grant only MANAGE for all clients. Individual client scopes can be read and managed, but listing scopes returns 403 without QUERY_CLIENTS, or an empty list with that role.\n Fix: Override canListClientScopes() and canViewClientScopes() to recognize the all-clients MANAGE grant alongside VIEW.\n (bugs)\n\nChecked: bugs, conventions, comments, slop, deterministic.\nNot checked: history (no result: historical patch blobs are missing from the partial clone and unavailable offline).\nSkipped: nothing.\n\n--- explain: pipeline diagnostics (not review findings) ---\nLenses: bugs 2, conventions 0, history no result, comments 0, slop 0, deterministic 0.\nCandidates before the gate (2):\n [bugs] services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java:87 score 95 reported\n [bugs] services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java:121 score 95 reported\nValidation used source inspection and read-only detectors; repository code was not executed. No files were written, feedback sent, or web lookups performed.\n\nWrong about something? Say which finding — I can file feedback (metadata-only by default).", + "findings": [ + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 87, + "severity": "P1", + "dimension": "bugs", + "title": "All-clients view grants bypass client-specific denies", + "failure_scenario": "Grant an administrator VIEW on all Clients, then attach a denying VIEW permission to one client. The administrator can still retrieve and list that client because the global grant bypasses client-specific evaluation.", + "suggested_fix": "Keep unconditional canView() limited to admin roles, evaluate fine-grained VIEW per client in canView(ClientModel), and separately allow list access for administrators with an all-clients VIEW grant so ClientsResource filters each result.", + "evidence": "ClientPermissionsV2.java:87 treats the all-clients VIEW grant as unconditional visibility; ClientPermissionsV2.java:78 and ClientsResource.java:148 consequently skip client-specific evaluation. UserResourceTypeEvaluationTest.java:203 demonstrates the existing contract that an object-specific deny overrides an all-resource VIEW grant.", + "confidence": 95 + }, + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 121, + "severity": "P2", + "dimension": "bugs", + "title": "Manage permission cannot list the client scopes it authorizes", + "failure_scenario": "An administrator with only an all-clients MANAGE grant can create, update, and directly read client scopes, but GET /client-scopes returns 403 without QUERY_CLIENTS. Adding QUERY_CLIENTS allows the request but still yields an empty list.", + "suggested_fix": "Override canListClientScopes() and canViewClientScopes() in ClientPermissionsV2 to accept all-clients MANAGE as well as VIEW, while preserving the existing admin-role checks.", + "evidence": "ClientPermissionsV2.java:109 and :121 recognize MANAGE for managing and reading individual client scopes, but :87 recognizes only VIEW for global visibility. Inherited ClientPermissions.java:259 and :490 use that global visibility result, and ClientScopesResource.java:91 gates listing before :93 returns an empty stream when it is false.", + "confidence": 95 + } + ], + "coverage": [ + { + "lens": "bugs", + "outcome": "returned", + "reason": "Reviewed all 10 changed files and traced authorization guards, evaluator, callers, and tests; 2 candidates, both shipped." + }, + { + "lens": "conventions", + "outcome": "returned", + "reason": "Checked supplied guidance, applicable configuration, established permission classes, and commit intent; 0 candidates." + }, + { + "lens": "history", + "outcome": "no result", + "reason": "Ancestor commit summaries were available, but historical patch blobs were missing from this partial clone; patch inspection failed offline." + }, + { + "lens": "comments", + "outcome": "returned", + "reason": "Checked added Javadocs, fallback comments, and changed test comments against implementations; 0 consequential candidates." + }, + { + "lens": "slop", + "outcome": "returned", + "reason": "Searched distinctive added identifiers and literals for existing implementations and checked duplication against established patterns; 0 candidates." + }, + { + "lens": "deterministic", + "outcome": "returned", + "reason": "Read-only detector suite completed against the merge-base and returned an empty findings array." + } + ] +} \ No newline at end of file diff --git a/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-36880/meta.json b/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-36880/meta.json new file mode 100644 index 0000000..7798d8d --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-36880/meta.json @@ -0,0 +1,14 @@ +{ + "host": "codex", + "tier": "review", + "base_sha": "b8c1cd410f4499c77a8b372decfe1718b80ccfd2", + "head_sha": "1950a511026d520a7329c7b6b9ee60a4af8f8b55", + "requested_model": "gpt-6.1-sol", + "reasoning_effort": "medium", + "plugin_revision": "2a41361b9aac174a18e47c0498efd53edcd6453e", + "plugin_version": "0.14.0", + "cli_version": "codex-cli 0.160.0", + "status": "incomplete", + "duration_ms": 337455, + "repository_unchanged": true +} diff --git a/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-38446/candidate-output.json b/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-38446/candidate-output.json new file mode 100644 index 0000000..44dd766 --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-38446/candidate-output.json @@ -0,0 +1,70 @@ +{ + "report": "## Reviso review — detached HEAD ce2dcc537b vs 85737f52b5 (1 commit, 8 files)\nBase: 85737f52b59aa2e3403c9b8e48acf71ccc5bfae3 via flag — merge-base 85737f52b5\n\nFound 3 issues:\n\n1. [P1][conf 95] Remove old local recovery codes after creating federated replacements — server-spi-private/src/main/java/org/keycloak/utils/CredentialHelper.java:126\n Failure: Regenerating codes into user storage leaves an existing local set valid; credential validation still falls back to that old set.\n Fix: After successful user-storage creation, remove existing locally stored recovery-code credentials.\n (bugs)\n\n2. [P2][conf 100] Consume recovery codes after successful validation — testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java:340\n Failure: The test provider accepts every generated code repeatedly and always presents the first code; exhaustion never triggers regeneration.\n Fix: Validate the next unused code and persist its consumption before returning success.\n (bugs)\n\n3. [P2][conf 95] Preserve recovery credential IDs and support removal — testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java:242\n Failure: Account REST receives a recovery credential with no ID, preventing removal; the provider also lacks recovery-code disabling.\n Fix: Copy the stored credential ID into the returned model and implement recovery-code disabling and its advertised disableable type.\n (bugs)\n\nChecked: bugs, conventions, comments, slop, deterministic.\nNot checked: history (no result: commit logs were available, but blame required an unavailable promisor object).\nSkipped: nothing.\n\n--- explain: pipeline diagnostics (not review findings) ---\nLenses: bugs 3, conventions 0, history no result (0 candidates), comments 0, slop 0, deterministic 0.\nCandidates before the gate (3):\n [bugs] CredentialHelper.java:126 score 95 reported\n [bugs] BackwardsCompatibilityUserStorage.java:340 score 100 reported\n [bugs] BackwardsCompatibilityUserStorage.java:242 score 95 reported\n\nWrong about something? Say which finding — I can file feedback (metadata-only by default).", + "findings": [ + { + "file": "server-spi-private/src/main/java/org/keycloak/utils/CredentialHelper.java", + "line": 126, + "severity": "P1", + "dimension": "bugs", + "title": "Remove old local recovery codes after creating federated replacements", + "failure_scenario": "A federated user with locally stored recovery codes regenerates them after the storage provider gains recovery-code support. The new set is stored externally, but an old local code still authenticates through local validation fallback.", + "suggested_fix": "After updateCredential succeeds, remove the user's existing locally stored recovery-authn-codes credentials so regeneration invalidates the previous set.", + "evidence": "CredentialHelper.java:125-129 bypasses RecoveryAuthnCodesCredentialProvider.createCredential, whose services/src/main/java/org/keycloak/credential/RecoveryAuthnCodesCredentialProvider.java:39-42 removes the previous stored credential. model/storage/src/main/java/org/keycloak/credential/UserCredentialManager.java:66-78 still invokes local validators when external validation fails.", + "confidence": 95 + }, + { + "file": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java", + "line": 340, + "severity": "P2", + "dimension": "bugs", + "title": "Consume recovery codes after successful validation", + "failure_scenario": "After a successful recovery-code login through the test provider, the same code authenticates again indefinitely. Subsequent challenges still request the first code, and exhausting the set never triggers regeneration.", + "suggested_fix": "Validate the next unused recovery code and persist its removal before returning true; retain code numbering and remaining-code state when returning the credential.", + "evidence": "BackwardsCompatibilityUserStorage.java:334-340 checks membership without updating storedRecoveryKeys, while :237-241 reconstructs the full original set on every read. The local implementation in services/src/main/java/org/keycloak/credential/RecoveryAuthnCodesCredentialProvider.java:118-119 explicitly removes and persists the consumed code.", + "confidence": 100 + }, + { + "file": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java", + "line": 242, + "severity": "P2", + "dimension": "bugs", + "title": "Preserve recovery credential IDs and support removal", + "failure_scenario": "A user requests recovery credentials through Account REST and receives a model with a null ID, so the credential cannot be addressed for deletion. Even deletion through the legacy type placeholder leaves the recovery codes configured because the provider only disables OTP.", + "suggested_fix": "Set the returned model's ID from myUser.recoveryCodes, clear recoveryCodes in disableCredentialType for this type, and include it in getDisableableCredentialTypesStream.", + "evidence": "BackwardsCompatibilityUserStorage.java:195 assigns a stored ID, but :237-242 returns a freshly constructed model without copying it; :215-222 only implements OTP disabling. services/src/main/java/org/keycloak/authentication/requiredactions/util/CredentialDeleteHelper.java:64-69 resolves federated credentials by ID and then invokes disableCredentialType.", + "confidence": 95 + } + ], + "coverage": [ + { + "lens": "bugs", + "outcome": "returned", + "reason": "Reviewed all eight changed files and relevant credential callers; 3 candidates, all shipped." + }, + { + "lens": "conventions", + "outcome": "returned", + "reason": "Checked applicable supplied guidance and repository conventions; no repository AGENTS.md or CLAUDE.md found; 0 candidates." + }, + { + "lens": "history", + "outcome": "no result", + "reason": "Read ancestor commit logs, but blame failed because a required promisor object was unavailable; historical verification remained incomplete." + }, + { + "lens": "comments", + "outcome": "returned", + "reason": "Checked code comments and invariant notes around the changed hunks; 0 candidates." + }, + { + "lens": "slop", + "outcome": "returned", + "reason": "Checked added logic against existing utilities and searched distinctive identifiers and literals for duplication; 0 qualifying candidates." + }, + { + "lens": "deterministic", + "outcome": "returned", + "reason": "The detector suite completed successfully and returned an empty findings array." + } + ] +} \ No newline at end of file diff --git a/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-38446/meta.json b/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-38446/meta.json new file mode 100644 index 0000000..b8758bc --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/review/crb-keycloak-38446/meta.json @@ -0,0 +1,15 @@ +{ + "host": "codex", + "tier": "review", + "base_sha": "85737f52b59aa2e3403c9b8e48acf71ccc5bfae3", + "head_sha": "ce2dcc537b4b1f98a335e88cfe8e0b804483252a", + "effective_base_sha": "85737f52b59aa2e3403c9b8e48acf71ccc5bfae3", + "requested_model": "gpt-6.1-sol", + "reasoning_effort": "medium", + "plugin_revision": "2a41361b9aac174a18e47c0498efd53edcd6453e", + "plugin_version": "0.14.0", + "cli_version": "codex-cli 0.160.0", + "status": "incomplete", + "duration_ms": 203221, + "repository_unchanged": true +} diff --git a/eval/runs/2026-10-02-audit-adjudicated/summary.json b/eval/runs/2026-10-02-audit-adjudicated/summary.json new file mode 100644 index 0000000..9e5fe3f --- /dev/null +++ b/eval/runs/2026-10-02-audit-adjudicated/summary.json @@ -0,0 +1,14 @@ +{ + "status": "blocked", + "scored_cases": 0, + "planned_cases": 10, + "host": "codex", + "model": "gpt-6.1-sol", + "effort": "medium", + "cli_version": "0.160.0", + "blockers": [ + "Partial-clone historical blobs unavailable inside read-only sandbox. Diff hydration permits current-source inspection but not every historical query.", + "Audit cannot launch every independent finder and verifier because host agent-thread capacity cannot be released in this environment." + ], + "metrics": null +}