Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ jobs:
run: |
python3 eval/runners/test_match.py
python3 eval/runners/test_gold_judge.py
python3 eval/runners/test_gold_codex.py
- name: Check internal links
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
with:
Expand Down
16 changes: 16 additions & 0 deletions eval/corpus/subsets/audit-adjudicated.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"name": "audit-adjudicated",
"rationale": "Seven adjudicated cases with eligible defects, two previously matched controls, and one entirely excluded Ruby case. All eligible findings in the original zero-recall cluster are covered. Not a full-corpus or expected-clean sweep.",
"case_ids": [
"crb-grafana-79265",
"crb-grafana-97529",
"crb-keycloak-36880",
"crb-keycloak-38446",
"crb-sentry-greptile-3",
"crb-sentry-greptile-5",
"crb-cal.com-14943",
"crb-grafana-94942",
"crb-sentry-67876",
"crb-discourse-graphite-9"
]
}
5 changes: 5 additions & 0 deletions eval/runners/candidate-codex.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,15 @@ def main():
"--output-schema", str(here / "codex-output.schema.json"),
"--output-last-message", str(out / "candidate-output.json"), prompt,
]
if args.tier == "audit":
command[2:2] = ["--enable", "multi_agent"]
before = snapshot(workdir)
started = time.monotonic()
meta = {"host": "codex", "tier": args.tier, "base_sha": base, "head_sha": head,
"effective_base_sha": git(workdir, "merge-base", base, head).decode().strip(),
"requested_model": args.model, "reasoning_effort": args.effort,
"plugin_revision": git(here.parents[1], "rev-parse", "HEAD").decode().strip(),
"plugin_version": (here.parents[1] / "skills/reviso/VERSION").read_text().strip(),
"cli_version": subprocess.check_output(["codex", "--version"], text=True).strip(),
"status": "failed"}
try:
Expand Down
31 changes: 28 additions & 3 deletions eval/runners/gold.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,14 @@ CORPUS_FILE="${CORPUS_FILE:-$CORPUS_DIR/public.jsonl}"
# so this is a no-op there.
CORPUS_BASE=$(cd "$(dirname "$CORPUS_FILE")" && pwd)
CACHE="$(cd "$HERE/.." && pwd)/.cache/clones"
mkdir -p "$OUT" "$CACHE"
mkdir -p "$CACHE"
case "${CANDIDATE_HOST:-claude}" in
claude) mkdir -p "$OUT" ;;
codex)
: "${CANDIDATE_CODEX_MODEL:?CANDIDATE_CODEX_MODEL is required}"
: "${CANDIDATE_CODEX_EFFORT:?CANDIDATE_CODEX_EFFORT is required}" ;;
*) echo "gold.sh: unknown CANDIDATE_HOST '${CANDIDATE_HOST}' (claude|codex)" >&2; exit 1 ;;
esac

ENTRY=$(jq -c --arg id "$CASE" 'select(.id == $id)' "$CORPUS_FILE")
[ -n "$ENTRY" ] || { echo "gold.sh: case '$CASE' not in $CORPUS_FILE" >&2; exit 1; }
Expand Down Expand Up @@ -90,16 +97,34 @@ else
if [ ! -d "$CDIR" ]; then
git clone -q --filter=blob:none "$CLONE_URL" "$CDIR"
fi
git -C "$CDIR" fetch -q origin "$BASE" "$HEAD_SHA"
git -C "$CDIR" fetch -q --no-write-fetch-head origin "$BASE" "$HEAD_SHA"
# A worktree (not a shared clone) so the cache's promisor config still
# serves lazy blob fetches for the blobless clone during checkout.
RWD="$WD/repo"
git -C "$CDIR" worktree add -q --detach "$RWD" "$HEAD_SHA"
if [ "${CANDIDATE_HOST:-claude}" = "codex" ]; then
# The read-only candidate cannot lazy-fetch missing blobs. Materialize
# the reviewed diff and its bounded history before entering the sandbox.
DIFF_BASE=$(git -C "$RWD" merge-base "$BASE" "$HEAD_SHA")
git -C "$RWD" diff --no-ext-diff "$DIFF_BASE" "$HEAD_SHA" >/dev/null
git -C "$RWD" rev-list --objects --no-object-names "$DIFF_BASE..$HEAD_SHA" \
| git -C "$RWD" cat-file --batch >/dev/null
fi
fi

# Tier named explicitly at the call site — never left to whatever the
# environment happened to carry in.
REVISO_TIER="$REVIEW_TIER" sh "$HERE/candidate.sh" "$RWD" "$BASE" "$HEAD_SHA" "$OUT"
if [ "${CANDIDATE_HOST:-claude}" = "codex" ]; then
"${REVISO_PYTHON:-python3}" "$HERE/candidate-codex.py" "$RWD" "$BASE" "$HEAD_SHA" "$OUT" \
--tier "$REVIEW_TIER" --model "$CANDIDATE_CODEX_MODEL" --effort "$CANDIDATE_CODEX_EFFORT" \
--timeout "${CANDIDATE_TIMEOUT:-900}"
if [ "$(jq -r '.status' "$OUT/meta.json")" != "returned" ]; then
echo "gold.sh: incomplete Codex coverage — not scored" >&2
exit 1
fi
else
REVISO_TIER="$REVIEW_TIER" sh "$HERE/candidate.sh" "$RWD" "$BASE" "$HEAD_SHA" "$OUT"
fi

# Judge against the labels (tiering + metrics live in gold-judge.sh so a
# recorded run can be re-judged when calibration moves).
Expand Down
76 changes: 76 additions & 0 deletions eval/runners/test_gold_codex.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
"""Exercise the Codex gold adapter and reject incomplete coverage offline."""

import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest


GOLD = Path(__file__).with_name("gold.sh")


class CodexGoldTests(unittest.TestCase):
def run_gold(self, incomplete=False, tier="review"):
with tempfile.TemporaryDirectory(prefix="reviso-codex-gold-test-") as directory:
root = Path(directory)
(root / "labels.json").write_text(json.dumps({"expected_clean": True, "findings": []}))
(root / "fixture.json").write_text(json.dumps({"files": [
{"filename": "sample.py", "status": "added", "patch": "+answer = 42"}]}))
(root / "corpus.jsonl").write_text(json.dumps({"id": "fixture", "synthetic": True,
"fixture": "fixture.json", "labels": "labels.json"}) + "\n")
fake = root / "codex"
fake.write_text('''#!/usr/bin/env python3
import json, os, sys
from pathlib import Path
if '--version' in sys.argv:
print('codex fixture')
sys.exit(0)
assert sys.argv[sys.argv.index('--sandbox') + 1] == 'read-only'
assert '--ignore-user-config' in sys.argv
assert '--ignore-rules' in sys.argv
assert sys.argv[sys.argv.index('--model') + 1] == 'fixture-model'
tier = os.environ['REVISO_TIER']
if tier == 'audit':
assert sys.argv[sys.argv.index('--enable') + 1] == 'multi_agent'
lenses = ['bugs', 'conventions', 'history', 'comments', 'slop', 'deterministic']
if tier == 'audit':
lenses.append('prior-reviews')
coverage = [{'lens': lens, 'outcome': 'returned', 'reason': ''} for lens in lenses]
if os.environ['GOLD_TEST_INCOMPLETE'] == '1':
coverage[0]['outcome'] = 'no result'
coverage[0]['reason'] = 'fixture unavailable'
output = Path(sys.argv[sys.argv.index('--output-last-message') + 1])
output.write_text(json.dumps({'report': 'No issues found.', 'findings': [], 'coverage': coverage}))
''')
fake.chmod(0o755)
out = root / "out"
env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ["PATH"],
CORPUS_FILE=str(root / "corpus.jsonl"), REVISO_TIER=tier,
CANDIDATE_HOST="codex", CANDIDATE_CODEX_MODEL="fixture-model",
CANDIDATE_CODEX_EFFORT="medium", REVISO_PYTHON=os.sys.executable,
GOLD_TEST_INCOMPLETE="1" if incomplete else "0")
result = subprocess.run(["sh", str(GOLD), "fixture", str(out)],
env=env, capture_output=True, text=True)
meta = json.loads((out / "meta.json").read_text())
return result, meta, (out / "gold-judge.json").exists()

def test_codex_gold_keeps_the_repository_unchanged(self):
for tier in ("review", "audit"):
with self.subTest(tier=tier):
result, meta, judged = self.run_gold(tier=tier)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertTrue(meta["repository_unchanged"])
self.assertEqual(meta["host"], "codex")
self.assertTrue(judged)

def test_incomplete_coverage_is_not_scored(self):
result, meta, judged = self.run_gold(incomplete=True)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(meta["status"], "incomplete")
self.assertFalse(judged)


if __name__ == "__main__":
unittest.main()
56 changes: 56 additions & 0 deletions eval/runs/2026-10-02-audit-adjudicated/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# Audit comparison attempt — 2026-10-02

**Blocked, zero scored cases.** This is diagnostic evidence for issue #18,
not a recall result. The planned ten cases are recorded in
`eval/corpus/subsets/audit-adjudicated.json` and cover all nine eligible
findings from the adjudicated zero-recall cluster, two previously matched
controls, and one entirely excluded Ruby case.

The candidate and matcher used Codex `gpt-6.1-sol`, medium effort,
CLI 0.160.0; the plugin was 0.14.0 at the revision in each `meta.json`.
Claude was unavailable. These results cannot establish historical Claude
performance or compare providers.

The initial Grafana run lacked diff blobs in a partial clone. The adapter
now hydrates the merge-base diff and objects between the pinned revisions
before entering the read-only sandbox. Subsequent Keycloak runs could
inspect current source but some historical blame still required missing
promisor objects. Full history is not promised by this preparation.

The audit could start several independent finders, then hit the host
agent-thread limit. Comments, anti-slop, prior-reviews, and independent
verification could not all complete. The workflow correctly recorded
missing coverage instead of treating fallback work as a full audit.
The sweeps were stopped after these repeat failures. Completed diagnostic
outputs and metadata are preserved here; interrupted attempts and raw
host event streams remain in the ignored cache.

Both tiers reported the repository unchanged. No matcher judgments or
gold metrics were produced for incomplete candidates. The comparison
needs a host with sufficient releasable audit agent capacity and prepared
historical objects before issue #18 can be resolved.

## Reproduction after resolving the blockers

Install `jsonschema` in the Python environment used by `REVISO_PYTHON`.
Generate a JSONL file in `eval/.cache/` by selecting the subset IDs from
`eval/corpus/public.jsonl` and prefixing each labels/fixture path with
`../corpus/`. Use a fresh output directory for each tier, and these env vars:

```sh
REVISO_TIER=audit
CANDIDATE_HOST=codex
CANDIDATE_CODEX_MODEL=gpt-6.1-sol
CANDIDATE_CODEX_EFFORT=medium
MATCH_HOST=codex
MATCH_CODEX_MODEL=gpt-6.1-sol
MATCH_CODEX_EFFORT=medium
REVISO_SWEEP_SUBSET=audit-adjudicated
CORPUS_FILE=/absolute/path/to/subset.jsonl
REVISO_PYTHON=/absolute/path/to/python-with-jsonschema
```

Export them, then run `sh eval/runners/sweep.sh gold <fresh-output-dir>`.
Repeat with `REVISO_TIER=review`. Only complete coverage can be scored.
Do not retry into an existing candidate directory: preserve the failed
attempt and use a fresh output path.
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{
"report": "## Reviso audit — detached HEAD vs 50f4e78a39914711d0d231a501ba215becf17ebc (5 commits, 11 files)\nBase: 50f4e78a39914711d0d231a501ba215becf17ebc via flag — merge-base 7ddcd0c961\n\nAudit unavailable: missing Git objects prevented reading every changed-file diff. No findings could be verified; this is not a clean audit.\n\nChecked: none.\nNot checked: bugs, conventions, history, comments, slop, prior-reviews (required diff unavailable); deterministic (Git diff failed); best-practices (no --web).\nSkipped: skip-tier files could not be determined because triage was unavailable.\n\n--- explain: pipeline diagnostics (not review findings) ---\nFinders: bugs no result, conventions no result, history no result, comments no result, slop no result, prior-reviews no result.\nDeterministic: no result. The detector emitted [] after Git failed; that output does not establish a clean pass.\nTriage, independent finders, evidence gathering, and scoring did not run because context assembly failed.\nCandidates before the gate: unavailable.\nNo files were written, feedback sent, or web lookups performed.",
"findings": [],
"coverage": [
{
"lens": "bugs",
"outcome": "no result",
"reason": "Missing Git objects prevented assembling the diff; independent finder did not run."
},
{
"lens": "conventions",
"outcome": "no result",
"reason": "Missing Git objects prevented assembling the diff; independent finder did not run."
},
{
"lens": "history",
"outcome": "no result",
"reason": "Commit metadata was available, but required diff objects were missing; independent finder did not run."
},
{
"lens": "comments",
"outcome": "no result",
"reason": "Missing Git objects prevented assembling the diff; code-comment finder did not run."
},
{
"lens": "slop",
"outcome": "no result",
"reason": "Missing Git objects prevented assembling the diff; anti-slop finder did not run."
},
{
"lens": "prior-reviews",
"outcome": "no result",
"reason": "Required diff unavailable; prior-review finder and lookup did not run."
},
{
"lens": "deterministic",
"outcome": "no result",
"reason": "Detector invocation encountered a Git diff failure from missing objects; its empty array was unusable."
},
{
"lens": "best-practices",
"outcome": "skipped",
"reason": "no --web"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"host": "codex",
"tier": "audit",
"base_sha": "50f4e78a39914711d0d231a501ba215becf17ebc",
"head_sha": "bbd8c507cdf56f6b884373809e0b24abd4a2353d",
"requested_model": "gpt-6.1-sol",
"reasoning_effort": "medium",
"plugin_revision": "2a41361b9aac174a18e47c0498efd53edcd6453e",
"plugin_version": "0.14.0",
"cli_version": "codex-cli 0.160.0",
"status": "incomplete",
"duration_ms": 150392,
"repository_unchanged": true
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
{
"report": "## Reviso audit — detached HEAD 1950a51102 vs b8c1cd410f4499c77a8b372decfe1718b80ccfd2 (1 commit, 10 files)\nBase: b8c1cd410f4499c77a8b372decfe1718b80ccfd2 via flag — merge-base b7b84d75d1\n\nFound 2 issues:\n\n1. [P1][conf 95] Global view permission bypasses client-specific denials — services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java:78\n Failure: An all-clients VIEW grant causes canView(client) to return true before evaluating an explicit denial for that client. Collection listing also bypasses individual checks when global canView() succeeds.\n Fix: Reserve unconditional view shortcuts for built-in roles; evaluate client-specific permissions before global fallback and preserve that precedence in list filtering.\n (bugs; corroborated by history)\n\n2. [P1][conf 90] Global manage permission fails inherited list and view checks — services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java:87\n Failure: An administrator granted only all-clients MANAGE can create clients and client scopes but receives 403 when listing them without query roles. With query-clients, the client-scope list is empty.\n Fix: Include MANAGE in the global view decision while preserving client-specific permission precedence.\n (bugs)\n\nChecked: bugs, conventions, history, deterministic.\nNot checked: comments, slop, prior-reviews (host agent-thread limit).\nSkipped: nothing.\n\n--- explain: pipeline diagnostics (not review findings) ---\nFinders: conventions 1, bugs 4, history 1, prior-reviews no result, comments no result, slop no result. Deterministic: 0.\nCandidates before the gate (6):\n [conventions] AdminPermissionsSchema.java:68 — score 50, dropped: rubric-score; documentation impact insufficiently established for this experimental feature.\n [bugs] ClientPermissionsV2.java:78 — score 95, reported.\n [bugs] ClientPermissionsV2.java:87 — score 90, reported.\n [bugs] ClientPermissionsV2.java:138 — score 50, dropped: rubric-score; the sole repository consumer is gated to V1.\n [bugs] ClientPermissionsV2.java:140 — score 50, dropped: rubric-score; the sole repository consumer is gated to V1.\n [history] ClientPermissionsV2.java:76 — score 95, reported; merged with the bugs finding at line 78.\n\nAudit coverage is incomplete: remaining finders and separate evidence agents could not be launched after the host rejected another agent with 'agent thread limit reached'. Candidate evidence was checked directly by the orchestrator. History returned a findings array, but missing promisor blobs prevented full blame and historical diff inspection; available reachable history was used. No repository code was executed, files written, web lookups performed, or feedback sent.\n\nWrong about something? Say which finding — I can file feedback (metadata-only by default).",
"findings": [
{
"file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
"line": 78,
"severity": "P1",
"dimension": "bugs",
"title": "Global view permission bypasses client-specific denials",
"failure_scenario": "An administrator has an all-clients VIEW grant and an explicit VIEW denial on a sensitive client. canView(client) accepts the global grant before evaluating the client-specific resource, exposing the denied client.",
"suggested_fix": "Short-circuit only for built-in administrative roles, evaluate client-specific permissions before global fallback, and make collection filtering respect those individual decisions.",
"evidence": "ClientPermissionsV2.java:78 accepts global canView() before the specific-resource evaluation at lines 214–224; ClientsResource.java:153 likewise bypasses individual checks when global view succeeds. UserResourceTypeEvaluationTest.java:203–212 and reachable commit bf355f83d3 establish that an individual denial takes precedence over an all-resource grant.",
"confidence": 95
},
{
"file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java",
"line": 87,
"severity": "P1",
"dimension": "bugs",
"title": "Global manage permission fails inherited list and view checks",
"failure_scenario": "An administrator granted only all-clients MANAGE can create clients and client scopes but receives 403 when listing either collection without query roles. Adding query-clients permits the client-scope request but returns an empty list.",
"suggested_fix": "Include the all-clients MANAGE scope in the global view decision while preserving client-specific permission precedence.",
"evidence": "ClientPermissionsV2.java:73 and :109 accept global MANAGE, but :87 checks only VIEW; inherited ClientPermissions.java:241–259 and :489 derive collection access from canView(). ClientScopesResource.java:91–95 requires list permission and returns an empty stream when canViewClientScopes() is false.",
"confidence": 90
}
],
"coverage": [
{
"lens": "bugs",
"outcome": "returned",
"reason": "Independent finder returned 4 candidates."
},
{
"lens": "conventions",
"outcome": "returned",
"reason": "Independent finder returned 1 candidate."
},
{
"lens": "history",
"outcome": "returned",
"reason": "Independent finder returned 1 candidate using reachable local history; missing promisor blobs limited blame and historical diffs."
},
{
"lens": "comments",
"outcome": "no result",
"reason": "Finder launch failed with host agent-thread limit reached."
},
{
"lens": "slop",
"outcome": "no result",
"reason": "No agent capacity remained to launch the anti-slop finder."
},
{
"lens": "prior-reviews",
"outcome": "no result",
"reason": "No agent capacity remained to launch the prior-reviews finder."
},
{
"lens": "deterministic",
"outcome": "returned",
"reason": "Read-only detector suite completed and returned 0 findings."
}
]
}
Loading
Loading