From 6bbe4e57d638fc7bcd8643139e61e7978eae4330 Mon Sep 17 00:00:00 2001 From: Michael Hohlios Date: Fri, 2 Oct 2026 10:49:07 -0400 Subject: [PATCH 1/2] Adjudicate zero-recall gold labels against pinned source Signed-off-by: Michael Hohlios --- .github/workflows/lint.yml | 4 +- docs/evals.md | 10 + eval/calibration/adjudication-2026-10-02.json | 240 ++++++++++++++++++ eval/corpus/README.md | 17 ++ eval/corpus/labels/crb-cal.com-14943.json | 24 +- .../labels/crb-discourse-graphite-9.json | 24 +- eval/corpus/labels/crb-grafana-103633.json | 26 +- eval/corpus/labels/crb-grafana-107534.json | 13 +- eval/corpus/labels/crb-grafana-76186.json | 25 +- eval/corpus/labels/crb-grafana-79265.json | 62 ++++- eval/corpus/labels/crb-grafana-90939.json | 26 +- eval/corpus/labels/crb-grafana-97529.json | 28 +- eval/corpus/labels/crb-keycloak-36880.json | 36 ++- eval/corpus/labels/crb-keycloak-36882.json | 13 +- eval/corpus/labels/crb-keycloak-38446.json | 25 +- eval/corpus/labels/crb-sentry-80528.json | 26 +- eval/corpus/labels/crb-sentry-95633.json | 39 ++- eval/corpus/labels/crb-sentry-greptile-3.json | 36 ++- eval/corpus/labels/crb-sentry-greptile-5.json | 37 ++- eval/runners/gold-judge.sh | 26 +- eval/runners/test_gold_judge.py | 77 ++++++ .../README.md | 69 +++++ .../crb-cal.com-14943/candidate.json | 18 ++ .../crb-cal.com-14943/gold-judge.json | 77 ++++++ .../crb-cal.com-14943/gold-labels.json | 40 +++ .../crb-cal.com-14943/match-gc.json | 1 + .../crb-cal.com-14943/meta.json | 10 + .../crb-discourse-graphite-9/candidate.json | 26 ++ .../crb-discourse-graphite-9/gold-judge.json | 85 +++++++ .../crb-discourse-graphite-9/gold-labels.json | 40 +++ .../crb-discourse-graphite-9/match-gc.json | 1 + .../crb-discourse-graphite-9/meta.json | 10 + .../crb-grafana-103633/candidate.json | 26 ++ .../crb-grafana-103633/gold-judge.json | 92 +++++++ .../crb-grafana-103633/gold-labels.json | 42 +++ .../crb-grafana-103633/match-gc.json | 6 + .../crb-grafana-103633/meta.json | 10 + .../crb-grafana-107534/candidate.json | 10 + .../crb-grafana-107534/gold-judge.json | 50 ++++ .../crb-grafana-107534/gold-labels.json | 22 ++ .../crb-grafana-107534/match-gc.json | 1 + .../crb-grafana-107534/meta.json | 10 + .../crb-grafana-76186/candidate.json | 10 + .../crb-grafana-76186/gold-judge.json | 70 +++++ .../crb-grafana-76186/gold-labels.json | 41 +++ .../crb-grafana-76186/match-gc.json | 1 + .../crb-grafana-76186/meta.json | 10 + .../crb-grafana-79265/candidate.json | 34 +++ .../crb-grafana-79265/gold-judge.json | 156 ++++++++++++ .../crb-grafana-79265/gold-labels.json | 99 ++++++++ .../crb-grafana-79265/match-gc.json | 1 + .../crb-grafana-79265/meta.json | 10 + .../crb-grafana-90939/candidate.json | 1 + .../crb-grafana-90939/gold-judge.json | 62 +++++ .../crb-grafana-90939/gold-labels.json | 42 +++ .../crb-grafana-90939/match-gc.json | 1 + .../crb-grafana-90939/meta.json | 10 + .../crb-grafana-97529/candidate.json | 18 ++ .../crb-grafana-97529/gold-judge.json | 82 ++++++ .../crb-grafana-97529/gold-labels.json | 42 +++ .../crb-grafana-97529/match-gc.json | 6 + .../crb-grafana-97529/meta.json | 10 + .../crb-keycloak-36880/candidate.json | 34 +++ .../crb-keycloak-36880/gold-judge.json | 113 +++++++++ .../crb-keycloak-36880/gold-labels.json | 59 +++++ .../crb-keycloak-36880/match-gc.json | 1 + .../crb-keycloak-36880/meta.json | 10 + .../crb-keycloak-36882/candidate.json | 18 ++ .../crb-keycloak-36882/gold-judge.json | 58 +++++ .../crb-keycloak-36882/gold-labels.json | 22 ++ .../crb-keycloak-36882/match-gc.json | 1 + .../crb-keycloak-36882/meta.json | 10 + .../crb-keycloak-38446/candidate.json | 18 ++ .../crb-keycloak-38446/gold-judge.json | 79 ++++++ .../crb-keycloak-38446/gold-labels.json | 41 +++ .../crb-keycloak-38446/match-gc.json | 1 + .../crb-keycloak-38446/meta.json | 10 + .../crb-sentry-80528/candidate.json | 1 + .../crb-sentry-80528/gold-judge.json | 62 +++++ .../crb-sentry-80528/gold-labels.json | 42 +++ .../crb-sentry-80528/match-gc.json | 1 + .../crb-sentry-80528/meta.json | 10 + .../crb-sentry-95633/candidate.json | 34 +++ .../crb-sentry-95633/gold-judge.json | 116 +++++++++ .../crb-sentry-95633/gold-labels.json | 62 +++++ .../crb-sentry-95633/match-gc.json | 1 + .../crb-sentry-95633/meta.json | 10 + .../crb-sentry-greptile-3/candidate.json | 42 +++ .../crb-sentry-greptile-3/gold-judge.json | 127 +++++++++ .../crb-sentry-greptile-3/gold-labels.json | 59 +++++ .../crb-sentry-greptile-3/match-gc.json | 6 + .../crb-sentry-greptile-3/meta.json | 10 + .../crb-sentry-greptile-5/candidate.json | 26 ++ .../crb-sentry-greptile-5/gold-judge.json | 107 ++++++++ .../crb-sentry-greptile-5/gold-labels.json | 60 +++++ .../crb-sentry-greptile-5/match-gc.json | 1 + .../crb-sentry-greptile-5/meta.json | 10 + .../summary.json | 176 +++++++++++++ 98 files changed, 3564 insertions(+), 40 deletions(-) create mode 100644 eval/calibration/adjudication-2026-10-02.json create mode 100644 eval/runners/test_gold_judge.py create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/README.md create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/candidate.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/gold-judge.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/gold-labels.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/match-gc.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/meta.json create mode 100644 eval/runs/2026-10-02-zero-recall-adjudication/summary.json diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cb82d67..211fa47 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,7 +21,9 @@ jobs: python3 -m pip install jsonschema python3 eval/runners/test_portability.py - name: Check matcher validation - run: python3 eval/runners/test_match.py + run: | + python3 eval/runners/test_match.py + python3 eval/runners/test_gold_judge.py - name: Check internal links uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2 with: diff --git a/docs/evals.md b/docs/evals.md index b171c29..b67b0c2 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -222,6 +222,16 @@ artifacts document case by case: ## Runs (audit tier) +The first sweep's 15 zero-correctness-recall CRB cases were adjudicated on +2026-10-02: 35 labels became 11 eligible correctness defects, 16 +policy-excluded findings, and 8 wrong or insufficiently specified labels. +The unchanged historical candidate already matched one of the 11; its +map-race label had been incorrectly tiered as efficiency. See the +[rulings and historical re-judge](../eval/runs/2026-10-02-zero-recall-adjudication/README.md). +This selected cluster is not a new full-corpus recall result. The new +[30-pair Codex calibration](../eval/calibration/README.md) does not validate +the archived Claude match decisions. + The deep tier's first recorded measurement. Not a gold row: `reviso-6` carries no labels file, so there is no recall figure to report — what this run establishes is the score distribution and the per-lens yield, which diff --git a/eval/calibration/adjudication-2026-10-02.json b/eval/calibration/adjudication-2026-10-02.json new file mode 100644 index 0000000..b674dbd --- /dev/null +++ b/eval/calibration/adjudication-2026-10-02.json @@ -0,0 +1,240 @@ +{ + "date": "2026-10-02", + "cases": [ + { + "case": "crb-grafana-76186", + "labels": "eval/corpus/labels/crb-grafana-76186.json", + "dispositions": [ + { + "finding_index": 0, + "status": "policy-excluded" + }, + { + "finding_index": 1, + "status": "label-wrong" + } + ] + }, + { + "case": "crb-grafana-79265", + "labels": "eval/corpus/labels/crb-grafana-79265.json", + "dispositions": [ + { + "finding_index": 0, + "status": "real-miss" + }, + { + "finding_index": 1, + "status": "policy-excluded" + }, + { + "finding_index": 2, + "status": "label-wrong" + }, + { + "finding_index": 3, + "status": "policy-excluded" + }, + { + "finding_index": 4, + "status": "label-wrong" + } + ] + }, + { + "case": "crb-grafana-90939", + "labels": "eval/corpus/labels/crb-grafana-90939.json", + "dispositions": [ + { + "finding_index": 0, + "status": "policy-excluded" + }, + { + "finding_index": 1, + "status": "policy-excluded" + } + ] + }, + { + "case": "crb-grafana-97529", + "labels": "eval/corpus/labels/crb-grafana-97529.json", + "dispositions": [ + { + "finding_index": 0, + "status": "policy-excluded" + }, + { + "finding_index": 1, + "status": "real-miss" + } + ] + }, + { + "case": "crb-grafana-103633", + "labels": "eval/corpus/labels/crb-grafana-103633.json", + "dispositions": [ + { + "finding_index": 0, + "status": "policy-excluded" + }, + { + "finding_index": 1, + "status": "policy-excluded" + } + ] + }, + { + "case": "crb-grafana-107534", + "labels": "eval/corpus/labels/crb-grafana-107534.json", + "dispositions": [ + { + "finding_index": 0, + "status": "policy-excluded" + } + ] + }, + { + "case": "crb-keycloak-36880", + "labels": "eval/corpus/labels/crb-keycloak-36880.json", + "dispositions": [ + { + "finding_index": 0, + "status": "real-miss" + }, + { + "finding_index": 1, + "status": "real-miss" + }, + { + "finding_index": 2, + "status": "real-miss" + } + ] + }, + { + "case": "crb-keycloak-36882", + "labels": "eval/corpus/labels/crb-keycloak-36882.json", + "dispositions": [ + { + "finding_index": 0, + "status": "policy-excluded" + } + ] + }, + { + "case": "crb-keycloak-38446", + "labels": "eval/corpus/labels/crb-keycloak-38446.json", + "dispositions": [ + { + "finding_index": 0, + "status": "policy-excluded" + }, + { + "finding_index": 1, + "status": "real-miss" + } + ] + }, + { + "case": "crb-sentry-80528", + "labels": "eval/corpus/labels/crb-sentry-80528.json", + "dispositions": [ + { + "finding_index": 0, + "status": "policy-excluded" + }, + { + "finding_index": 1, + "status": "policy-excluded" + } + ] + }, + { + "case": "crb-sentry-95633", + "labels": "eval/corpus/labels/crb-sentry-95633.json", + "dispositions": [ + { + "finding_index": 0, + "status": "label-wrong" + }, + { + "finding_index": 1, + "status": "policy-excluded" + }, + { + "finding_index": 2, + "status": "policy-excluded" + } + ] + }, + { + "case": "crb-sentry-greptile-3", + "labels": "eval/corpus/labels/crb-sentry-greptile-3.json", + "dispositions": [ + { + "finding_index": 0, + "status": "real-miss" + }, + { + "finding_index": 1, + "status": "label-wrong" + }, + { + "finding_index": 2, + "status": "real-miss" + } + ] + }, + { + "case": "crb-sentry-greptile-5", + "labels": "eval/corpus/labels/crb-sentry-greptile-5.json", + "dispositions": [ + { + "finding_index": 0, + "status": "label-wrong" + }, + { + "finding_index": 1, + "status": "policy-excluded" + }, + { + "finding_index": 2, + "status": "real-miss" + } + ] + }, + { + "case": "crb-cal.com-14943", + "labels": "eval/corpus/labels/crb-cal.com-14943.json", + "dispositions": [ + { + "finding_index": 0, + "status": "real-miss" + }, + { + "finding_index": 1, + "status": "real-miss" + } + ] + }, + { + "case": "crb-discourse-graphite-9", + "labels": "eval/corpus/labels/crb-discourse-graphite-9.json", + "dispositions": [ + { + "finding_index": 0, + "status": "label-wrong" + }, + { + "finding_index": 1, + "status": "label-wrong" + } + ] + } + ], + "counts": { + "policy-excluded": 16, + "label-wrong": 8, + "real-miss": 11 + } +} diff --git a/eval/corpus/README.md b/eval/corpus/README.md index 69917dc..3cdfa06 100644 --- a/eval/corpus/README.md +++ b/eval/corpus/README.md @@ -101,6 +101,23 @@ tiers are not measured against the `slop-*` cases. ## Imported gold cases (CRB) +The 15 zero-correctness-recall cases from the first sweep were adjudicated +on 2026-10-02. Their finding text remains intact; `adjudication` records a +disposition, reason, and pinned source evidence. `policy-excluded` and +`label-wrong` findings are excluded from gold matching and the correctness +denominator, but retained in the judge's `excluded_gold` array. A case whose +gold is entirely excluded is **unscored**, not expected-clean: unmatched +candidate findings remain promotion candidates, not proven false positives. +One imported category was corrected explicitly: Grafana's unlocked cache +iteration can panic, so it is correctness rather than efficiency; the +original category is retained in `imported_category`. + +The [adjudication manifest](../calibration/adjudication-2026-10-02.json) +indexes all 35 rulings. The +[historical re-judge](../runs/2026-10-02-zero-recall-adjudication/summary.json) +uses the original candidate outputs and original match indices; it does +not retroactively rerun the old model or claim current-release recall. + 50 entries came from `withmartian/code-review-benchmark` via `import-crb.sh ` (re-run is idempotent): real PRs in calcom/cal.com (ts), grafana/grafana (go), keycloak/keycloak (java), diff --git a/eval/corpus/labels/crb-cal.com-14943.json b/eval/corpus/labels/crb-cal.com-14943.json index 4289c87..63d8ef7 100644 --- a/eval/corpus/labels/crb-cal.com-14943.json +++ b/eval/corpus/labels/crb-cal.com-14943.json @@ -12,7 +12,17 @@ "severity": "high", "category": "correctness", "crb_category": "bug", - "description": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consider an atomic increment via Prisma (increment: 1) to avoid race conditions (also applies to the similar update in the catch block)." + "description": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consider an atomic increment via Prisma (increment: 1) to avoid race conditions (also applies to the similar update in the catch block).", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "concurrency", + "why": "The new retry updates assign the stale fetched retryCount plus one. Overlapping cron invocations can overwrite one another rather than atomically incrementing the retry count.", + "evidence": [ + "https://github.com/calcom/cal.com/blob/c790227e0cda780e6fea9bb03af27948d9e286b9/packages/features/ee/workflows/api/scheduleSMSReminders.ts#L181", + "https://github.com/calcom/cal.com/compare/917c7b0764f4c6bfdca95db2c5f73dbfdbe11757...c790227e0cda780e6fea9bb03af27948d9e286b9" + ] + } }, { "file": "packages/features/ee/workflows/api/scheduleSMSReminders.ts", @@ -21,7 +31,17 @@ "severity": "high", "category": "robustness", "crb_category": "error-handling", - "description": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) that have retryCount > 1. This occurs because the retryCount condition within the OR clause for deletion lacks a method: WorkflowMethods.SMS filter, causing it to apply to all reminder types instead of only SMS reminders, which is the intended scope of this function." + "description": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) that have retryCount > 1. This occurs because the retryCount condition within the OR clause for deletion lacks a method: WorkflowMethods.SMS filter, causing it to apply to all reminder types instead of only SMS reminders, which is the intended scope of this function.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "filter-scope", + "why": "The new retryCount > 1 OR branch lacks method: SMS, so deleteMany applies it to reminders for every method. The previous method filter scoped the whole delete to SMS.", + "evidence": [ + "https://github.com/calcom/cal.com/blob/c790227e0cda780e6fea9bb03af27948d9e286b9/packages/features/ee/workflows/api/scheduleSMSReminders.ts#L38", + "https://github.com/calcom/cal.com/compare/917c7b0764f4c6bfdca95db2c5f73dbfdbe11757...c790227e0cda780e6fea9bb03af27948d9e286b9" + ] + } } ] } diff --git a/eval/corpus/labels/crb-discourse-graphite-9.json b/eval/corpus/labels/crb-discourse-graphite-9.json index f1c8acf..d798835 100644 --- a/eval/corpus/labels/crb-discourse-graphite-9.json +++ b/eval/corpus/labels/crb-discourse-graphite-9.json @@ -12,7 +12,17 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "Thread-safety issue with lazy @loaded_locales" + "description": "Thread-safety issue with lazy @loaded_locales", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "refuted", + "why": "load_locale already holds LOAD_MUTEX and rechecks membership inside the lock. The label supplies no unsynchronized write or reproducible new duplicate-load race for normally initialized locale state.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/discourse-graphite/blob/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293/lib/freedom_patches/translate_accelerator.rb#L45", + "https://github.com/ai-code-review-evaluation/discourse-graphite/compare/728845d008f06828630a632e5c7d67e11bf15deb...ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293" + ] + } }, { "file": null, @@ -21,7 +31,17 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same locale is passed as a String vs Symbol (also applies to other locations in the PR)." + "description": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same locale is passed as a String vs Symbol (also applies to other locations in the PR).", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "unreachable-input", + "why": "The new caller supplies I18n.locale and symbol-valued fallback locales; it does not introduce a caller passing a string. Existing locale loading already used the same array. No new mixed-string/symbol loading path is established.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/discourse-graphite/blob/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293/config/initializers/i18n.rb#L6", + "https://github.com/ai-code-review-evaluation/discourse-graphite/compare/728845d008f06828630a632e5c7d67e11bf15deb...ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293" + ] + } } ] } diff --git a/eval/corpus/labels/crb-grafana-103633.json b/eval/corpus/labels/crb-grafana-103633.json index 07f6208..4e62b1e 100644 --- a/eval/corpus/labels/crb-grafana-103633.json +++ b/eval/corpus/labels/crb-grafana-103633.json @@ -12,7 +12,18 @@ "severity": "high", "category": "security", "crb_category": "security", - "description": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately, but cached denials from the same permission cache are ignored, leading to a fresh database lookup. This allows stale cached grants to provide access to revoked resources, posing a security risk. " + "description": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately, but cached denials from the same permission cache are ignored, leading to a fresh database lookup. This allows stale cached grants to provide access to revoked resources, posing a security risk. ", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "Cached grants were already trusted in getUserPermissions/getAnonymousPermissions. The new path refreshes a denial from the permission map, while the separate permission-denial cache explicitly short-circuits to false; the label does not establish a new stale-grant regression.", + "evidence": [ + "https://github.com/grafana/grafana/blob/5634ca44f799a82161f0401dadc286452f3bdbf8/pkg/services/authz/rbac/service.go#L316", + "https://github.com/grafana/grafana/compare/5634ca44f799a82161f0401dadc286452f3bdbf8...7562f37880367411a62304bcdcdc178bced23906" + ] + }, + "policy_excluded": true }, { "file": "pkg/services/authz/rbac/service.go", @@ -21,7 +32,18 @@ "severity": "low", "category": "efficiency", "crb_category": "performance", - "description": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so checkPermission will still treat this scope as not allowed." + "description": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so checkPermission will still treat this scope as not allowed.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "test-coverage", + "why": "The false-valued test seed does not exercise the intended cached-grant override. This is a coverage-only gold item, not a production behavior defect; review excludes missing test coverage. The old candidate did surface it.", + "evidence": [ + "https://github.com/grafana/grafana/blob/7562f37880367411a62304bcdcdc178bced23906/pkg/services/authz/rbac/service_test.go#L982", + "https://github.com/grafana/grafana/compare/5634ca44f799a82161f0401dadc286452f3bdbf8...7562f37880367411a62304bcdcdc178bced23906" + ] + }, + "policy_excluded": true } ] } diff --git a/eval/corpus/labels/crb-grafana-107534.json b/eval/corpus/labels/crb-grafana-107534.json index 12b57d9..bf42f09 100644 --- a/eval/corpus/labels/crb-grafana-107534.json +++ b/eval/corpus/labels/crb-grafana-107534.json @@ -12,7 +12,18 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used in the corresponding test setup." + "description": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used in the corresponding test setup.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "test-coverage", + "why": "This label only says the test setup omits the third argument; it gives no behavior defect. Coverage omissions are excluded. The candidate double-interpolation finding is a different root cause.", + "evidence": [ + "https://github.com/grafana/grafana/blob/1cab4143d443c33079566f6ca36931eb8468159f/public/app/plugins/datasource/loki/shardQuerySplitting.test.ts#L53", + "https://github.com/grafana/grafana/compare/d9a8253640464014aa3662c91131278990cfb828...1cab4143d443c33079566f6ca36931eb8468159f" + ] + }, + "policy_excluded": true } ] } diff --git a/eval/corpus/labels/crb-grafana-76186.json b/eval/corpus/labels/crb-grafana-76186.json index 3a07e89..5857366 100644 --- a/eval/corpus/labels/crb-grafana-76186.json +++ b/eval/corpus/labels/crb-grafana-76186.json @@ -12,7 +12,18 @@ "severity": "high", "category": "correctness", "crb_category": "bug", - "description": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request is received. This occurs because they directly access req.PluginContext (via the instrumentContext function) without first checking if req is nil. This is a regression, as previous middleware layers gracefully handled nil requests." + "description": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request is received. This occurs because they directly access req.PluginContext (via the instrumentContext function) without first checking if req is nil. This is a regression, as previous middleware layers gracefully handled nil requests.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The default middleware stack already dereferences nil requests in InstrumentationMiddleware before LoggerMiddleware. The new middleware does not introduce the claimed nil-request regression.", + "evidence": [ + "https://github.com/grafana/grafana/blob/58ba11ecbd60863e2bfc6e32f0399ed4feb3927a/pkg/services/pluginsintegration/clientmiddleware/instrumentation_middleware.go#L161", + "https://github.com/grafana/grafana/compare/58ba11ecbd60863e2bfc6e32f0399ed4feb3927a...303cdc2caf256aa5df660611120c2e3f7392365d" + ] + }, + "policy_excluded": true }, { "file": null, @@ -21,7 +32,17 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract and add traceID from the context to log parameters were removed from the LoggerMiddleware. The newly introduced ContextualLoggerMiddleware does not add this information, resulting in missing traceID in plugin request logs and impacting debugging and request tracing capabilities." + "description": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract and add traceID from the context to log parameters were removed from the LoggerMiddleware. The newly introduced ContextualLoggerMiddleware does not add this information, resulting in missing traceID in plugin request logs and impacting debugging and request tracing capabilities.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "refuted", + "why": "Tracing registers a contextual log provider that supplies traceID, and the changed logger calls FromContext. Removing the explicit argument does not remove the logged field.", + "evidence": [ + "https://github.com/grafana/grafana/blob/303cdc2caf256aa5df660611120c2e3f7392365d/pkg/infra/tracing/tracing.go#L91", + "https://github.com/grafana/grafana/compare/58ba11ecbd60863e2bfc6e32f0399ed4feb3927a...303cdc2caf256aa5df660611120c2e3f7392365d" + ] + } } ] } diff --git a/eval/corpus/labels/crb-grafana-79265.json b/eval/corpus/labels/crb-grafana-79265.json index 3e815ab..3e0c7bd 100644 --- a/eval/corpus/labels/crb-grafana-79265.json +++ b/eval/corpus/labels/crb-grafana-79265.json @@ -12,7 +12,17 @@ "severity": "high", "category": "correctness", "crb_category": "bug", - "description": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond the limit. Consider using a database transaction or lock." + "description": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond the limit. Consider using a database transaction or lock.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "concurrency", + "why": "CountDevices and the insert/upsert run in separate sessions without a transaction or shared lock. Two uncached device requests can both observe capacity and insert beyond the configured limit.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/anonstore/database.go#L112", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + } }, { "file": "pkg/services/anonymous/anonimpl/client.go", @@ -21,7 +31,18 @@ "severity": "medium", "category": "security", "crb_category": "security", - "description": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, device tagging was asynchronous and non-blocking. This change prevents anonymous users from authenticating when the device limit is reached." + "description": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, device tagging was asynchronous and non-blocking. This change prevents anonymous users from authenticating when the device limit is reached.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "intentional", + "why": "Rejecting anonymous authentication at the configured device limit is the purpose of this change; TagDevice now returns the store error and the anonymous client explicitly handles ErrDeviceLimitReached.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/client.go#L44", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + }, + "policy_excluded": true }, { "file": "pkg/services/anonymous/anonimpl/anonstore/database.go", @@ -30,7 +51,17 @@ "severity": "medium", "category": "correctness", "crb_category": "bug", - "description": "This call won’t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec’s signature requires a first parameter of type string (not an interface{} splat)." + "description": "This call won’t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec’s signature requires a first parameter of type string (not an interface{} splat).", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-api", + "why": "DBSession embeds the vendored Xorm Session. Its Exec signature is Exec(sqlOrArgs ...any), not Exec(query string, args ...any); the interface-slice splat is valid.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/util/xorm/session_raw.go#L208", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + } }, { "file": null, @@ -39,7 +70,18 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist." + "description": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "intentional", + "why": "At capacity, an existing recent device may be updated and a missing/expired device must not be inserted. Returning ErrDeviceLimitReached when the update finds no eligible row implements that policy.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/anonstore/database.go#L73", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + }, + "policy_excluded": true }, { "file": null, @@ -48,7 +90,17 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound but device.UpdatedAt as the current time may not match the intended logic. Consider using time.Now().UTC() consistently." + "description": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound but device.UpdatedAt as the current time may not match the intended logic. Consider using time.Now().UTC() consistently.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-premise", + "why": "Both time bounds use device.UpdatedAt.UTC(); the upper bound includes one minute. The label describes a mixed-UTC expression that is not present, and UTC conversion preserves the instant.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/anonstore/database.go#L81", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + } } ] } diff --git a/eval/corpus/labels/crb-grafana-90939.json b/eval/corpus/labels/crb-grafana-90939.json index 7e4bcc9..7ca647f 100644 --- a/eval/corpus/labels/crb-grafana-90939.json +++ b/eval/corpus/labels/crb-grafana-90939.json @@ -12,7 +12,18 @@ "severity": "medium", "category": "efficiency", "crb_category": "performance", - "description": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function first checks if the cache is populated using a read lock (RLock), and if the cache is empty, it acquires a write lock to populate it. However, it fails to re-check whether the cache was populated by another goroutine while waiting to acquire the write lock." + "description": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function first checks if the cache is populated using a read lock (RLock), and if the cache is empty, it acquires a write lock to populate it. However, it fails to re-check whether the cache was populated by another goroutine while waiting to acquire the write lock.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "efficiency", + "why": "The missing second cache check may repeat manifest parsing under cold-start concurrency; it does not invalidate the pointer publication lock. Efficiency is outside the correctness lane.", + "evidence": [ + "https://github.com/grafana/grafana/blob/b1613e320acff00623e6efc59f00ee68c7684a97/pkg/api/webassets/webassets.go#L42", + "https://github.com/grafana/grafana/compare/3ce1a5b0caab215346a51629b0344b90d67e9478...b1613e320acff00623e6efc59f00ee68c7684a97" + ] + }, + "policy_excluded": true }, { "file": "pkg/api/webassets/webassets.go", @@ -21,7 +32,18 @@ "severity": "high", "category": "robustness", "crb_category": "error-handling", - "description": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assigns the fetch result to the cache (line 69: entryPointAssetsCache = result) regardless of whether the fetch succeeded or failed. When an error occurs during asset fetching, result is nil, and this nil value overwrites any previously valid cache entry." + "description": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assigns the fetch result to the cache (line 69: entryPointAssetsCache = result) regardless of whether the fetch succeeded or failed. When an error occurs during asset fetching, result is nil, and this nil value overwrites any previously valid cache entry.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "Assigning result to the global cache on the error path is unchanged. The patch introduces locks around the existing assignment rather than a new cache-wipe behavior.", + "evidence": [ + "https://github.com/grafana/grafana/blob/3ce1a5b0caab215346a51629b0344b90d67e9478/pkg/api/webassets/webassets.go#L59", + "https://github.com/grafana/grafana/compare/3ce1a5b0caab215346a51629b0344b90d67e9478...b1613e320acff00623e6efc59f00ee68c7684a97" + ] + }, + "policy_excluded": true } ] } diff --git a/eval/corpus/labels/crb-grafana-97529.json b/eval/corpus/labels/crb-grafana-97529.json index 5ed779f..c6b36cb 100644 --- a/eval/corpus/labels/crb-grafana-97529.json +++ b/eval/corpus/labels/crb-grafana-97529.json @@ -12,16 +12,38 @@ "severity": "high", "category": "correctness", "crb_category": "bug", - "description": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment. " + "description": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment. ", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "efficiency", + "why": "The label describes redundant expensive builds, without a new failure scenario. getOrCreateIndex performs its cache check outside BuildIndex in both versions, so the old function-wide lock did not recheck the cache or prevent queued duplicate builds. Do not count this performance-only concern as a correctness miss.", + "evidence": [ + "https://github.com/grafana/grafana/blob/26fed312840cd76b766cbd2158e17a7e6c0ec548/pkg/storage/unified/search/bleve.go#L93", + "https://github.com/grafana/grafana/compare/871af07203177ce59a24e742107119409274f48a...26fed312840cd76b766cbd2158e17a7e6c0ec548" + ] + }, + "policy_excluded": true }, { "file": null, "line": 0, "title": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchroniz", "severity": "high", - "category": "efficiency", + "category": "correctness", "crb_category": "performance", - "description": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic." + "description": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "category-correction", + "why": "TotalDocs iterates the map without the cache mutex while BuildIndex writes it; a map read/write panic is correctness, not efficiency. The original candidate already reports this root cause.", + "evidence": [ + "https://github.com/grafana/grafana/blob/26fed312840cd76b766cbd2158e17a7e6c0ec548/pkg/storage/unified/search/bleve.go#L134", + "https://github.com/grafana/grafana/compare/871af07203177ce59a24e742107119409274f48a...26fed312840cd76b766cbd2158e17a7e6c0ec548" + ] + }, + "imported_category": "efficiency" } ] } diff --git a/eval/corpus/labels/crb-keycloak-36880.json b/eval/corpus/labels/crb-keycloak-36880.json index 91894bd..a9a36bc 100644 --- a/eval/corpus/labels/crb-keycloak-36880.json +++ b/eval/corpus/labels/crb-keycloak-36880.json @@ -12,7 +12,17 @@ "severity": "high", "category": "security", "crb_category": "security", - "description": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled." + "description": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "feature-flag", + "why": "The newly added V1 feature guard prevents cleanup listener execution when only V2 is enabled, even though V2 creates per-client authorization resources.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/AdminPermissions.java#L77", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + ] + } }, { "file": null, @@ -21,7 +31,17 @@ "severity": "high", "category": "correctness", "crb_category": "bug", - "description": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions." + "description": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "The per-client lookup supplies the resource-server ID as owner, but getOrCreateResource creates the resource with the server client ID as owner. Client-specific permissions can be missed.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L214", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + ] + } }, { "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", @@ -30,7 +50,17 @@ "severity": "high", "category": "robustness", "crb_category": "error-handling", - "description": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures." + "description": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "getClientsWithPermission enumerates typed resources and returns their names. Individually created client resources have no type; the type-level Clients resource name is not a client ID.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L138", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + ] + } } ] } diff --git a/eval/corpus/labels/crb-keycloak-36882.json b/eval/corpus/labels/crb-keycloak-36882.json index 65db176..9eec77a 100644 --- a/eval/corpus/labels/crb-keycloak-36882.json +++ b/eval/corpus/labels/crb-keycloak-36882.json @@ -12,7 +12,18 @@ "severity": "medium", "category": "correctness", "crb_category": "bug", - "description": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:" + "description": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "intentional", + "why": "The compatibility CLI already uses picocli.exit(result.exitCode()); the new feature-disabled branch follows the same CLI exit convention and returns immediately. No embedding caller or broken contract is identified by the truncated label.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/62bf98939eda58f0d0a1a41a58aa6d4ecdc24dba/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/UpdateCompatibilityCheck.java#L45", + "https://github.com/keycloak/keycloak/compare/8671f860469d6d95e54df3e13f77ebc401249489...62bf98939eda58f0d0a1a41a58aa6d4ecdc24dba" + ] + }, + "policy_excluded": true } ] } diff --git a/eval/corpus/labels/crb-keycloak-38446.json b/eval/corpus/labels/crb-keycloak-38446.json index 0f1db9d..ed32376 100644 --- a/eval/corpus/labels/crb-keycloak-38446.json +++ b/eval/corpus/labels/crb-keycloak-38446.json @@ -12,7 +12,18 @@ "severity": "medium", "category": "security", "crb_category": "security", - "description": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential(user) without checking isPresent() can lead to a NoSuchElementException if the Optional is empty." + "description": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential(user) without checking isPresent() can lead to a NoSuchElementException if the Optional is empty.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The login bean previously called findFirst().get() without a guard. This patch replaces the lookup with one that also checks federated credentials. The raw-list type-safety concern is compiler/linter territory and does not establish the claimed new absent-credential failure.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/services/src/main/java/org/keycloak/forms/login/freemarker/model/RecoveryAuthnCodeInputLoginBean.java#L34", + "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a" + ] + }, + "policy_excluded": true }, { "file": null, @@ -21,7 +32,17 @@ "severity": "low", "category": "security", "crb_category": "security", - "description": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.recoveryCodes.getId()); otherwise getId() will be null and downstream removal by id (e.g., removeStoredCredentialById in the authenticator flow) may not work." + "description": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.recoveryCodes.getId()); otherwise getId() will be null and downstream removal by id (e.g., removeStoredCredentialById in the authenticator flow) may not work.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "The new user-storage getCredentials path creates a recovery model with createFromValues but never transfers the stored ID. createFromValues does not set an ID, while the authenticator later passes getId to removal.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java#L239", + "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a" + ] + } } ] } diff --git a/eval/corpus/labels/crb-sentry-80528.json b/eval/corpus/labels/crb-sentry-80528.json index cda8a3c..915bf4f 100644 --- a/eval/corpus/labels/crb-sentry-80528.json +++ b/eval/corpus/labels/crb-sentry-80528.json @@ -12,7 +12,18 @@ "severity": "high", "category": "correctness", "crb_category": "bug", - "description": "The function modifies the config variable to include display values but then returns the original monitor.config instead of the modified version." + "description": "The function modifies the config variable to include display values but then returns the original monitor.config instead of the modified version.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The copy-and-return-original-config bug exists verbatim in base mark_failed.py and is moved into incident_occurrence.py. It is a real latent bug but not introduced by this PR.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9/src/sentry/monitors/logic/mark_failed.py#L327", + "https://github.com/getsentry/sentry/compare/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9...dcdcadb771128e79259cc9eff9c70c38fc597976" + ] + }, + "policy_excluded": true }, { "file": null, @@ -21,7 +32,18 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates an unnecessary database query." + "description": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates an unnecessary database query.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The MonitorCheckIn query already appears in base mark_failed_threshold and moves unchanged to try_incident_threshold. It also retrieves full model instances where previous_checkins has only ID/date/status; the label supplies no new unnecessary-query regression.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9/src/sentry/monitors/logic/mark_failed.py#L178", + "https://github.com/getsentry/sentry/compare/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9...dcdcadb771128e79259cc9eff9c70c38fc597976" + ] + }, + "policy_excluded": true } ] } diff --git a/eval/corpus/labels/crb-sentry-95633.json b/eval/corpus/labels/crb-sentry-95633.json index 156275c..588f68a 100644 --- a/eval/corpus/labels/crb-sentry-95633.json +++ b/eval/corpus/labels/crb-sentry-95633.json @@ -12,7 +12,18 @@ "severity": "high", "category": "correctness", "crb_category": "bug", - "description": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This could cause AttributeError at runtime. Verify the correct API or implement a custom shutdown mechanism." + "description": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This could cause AttributeError at runtime. Verify the correct API or implement a custom shutdown mechanism.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-version", + "why": "The pinned .python-version is 3.13.1. Queue.shutdown(immediate=False) and queue.ShutDown are documented Python 3.13 APIs.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/9966ec5a13e331659c3ea00981f9b11b0faf821f/.python-version#L1", + "https://github.com/getsentry/sentry/compare/fd358e8a388939959369f08f616e29552cdaf96e...9966ec5a13e331659c3ea00981f9b11b0faf821f", + "https://docs.python.org/3.13/library/queue.html#queue.Queue.shutdown" + ] + } }, { "file": "tests/sentry/uptime/consumers/test_results_consumer.py", @@ -21,7 +32,18 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant to improve maintainability." + "description": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant to improve maintainability.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "test-style", + "why": "Repeated polling bounds in test scaffolding are cleanup, not correctness; no written rule requiring a shared test constant is supplied.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/9966ec5a13e331659c3ea00981f9b11b0faf821f/tests/sentry/uptime/consumers/test_results_consumer.py#L2009", + "https://github.com/getsentry/sentry/compare/fd358e8a388939959369f08f616e29552cdaf96e...9966ec5a13e331659c3ea00981f9b11b0faf821f" + ] + }, + "policy_excluded": true }, { "file": "tests/sentry/uptime/consumers/test_results_consumer.py", @@ -30,7 +52,18 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation." + "description": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "documentation", + "why": "The docstring promises commits for other messages while the test asserts no commit. This is a test-documentation cleanup item, with no runtime defect asserted; review excludes general documentation complaints without a written requirement.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/9966ec5a13e331659c3ea00981f9b11b0faf821f/tests/sentry/uptime/consumers/test_results_consumer.py#L2140", + "https://github.com/getsentry/sentry/compare/fd358e8a388939959369f08f616e29552cdaf96e...9966ec5a13e331659c3ea00981f9b11b0faf821f" + ] + }, + "policy_excluded": true } ] } diff --git a/eval/corpus/labels/crb-sentry-greptile-3.json b/eval/corpus/labels/crb-sentry-greptile-3.json index 5d3c1fe..df7a29e 100644 --- a/eval/corpus/labels/crb-sentry-greptile-3.json +++ b/eval/corpus/labels/crb-sentry-greptile-3.json @@ -12,7 +12,17 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "sample_rate = 0.0 is falsy and skipped" + "description": "sample_rate = 0.0 is falsy and skipped", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "falsy-zero", + "why": "The new factory helper uses if client_sample_rate, so a valid 0.0 is not copied into normalized_data. This changes the fixture data rather than merely omitting a test.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/6ad6fe3899f630ccf922ce7ac681147dc7987435/src/sentry/testutils/factories.py#L345", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/a5d290951def84afdcc4c88d2f1f20023fc36e2a...6ad6fe3899f630ccf922ce7ac681147dc7987435" + ] + } }, { "file": null, @@ -21,7 +31,17 @@ "severity": "low", "category": "efficiency", "crb_category": "performance", - "description": "Using Python’s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys won’t match across workers and invalidate_upsampling_cache may fail to delete them. Use a deterministic serialization of project_ids for the cache key." + "description": "Using Python’s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys won’t match across workers and invalidate_upsampling_cache may fail to delete them. Use a deterministic serialization of project_ids for the cache key.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-premise", + "why": "The hashed value is a sorted tuple of integer project IDs, not strings. Python hash randomization does not randomize integer tuple hashes across processes; both creation and invalidation use the same expression.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/6ad6fe3899f630ccf922ce7ac681147dc7987435/src/sentry/api/helpers/error_upsampling.py#L27", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/a5d290951def84afdcc4c88d2f1f20023fc36e2a...6ad6fe3899f630ccf922ce7ac681147dc7987435" + ] + } }, { "file": "src/sentry/api/endpoints/organization_events_stats.py", @@ -30,7 +50,17 @@ "severity": "medium", "category": "correctness", "crb_category": "bug", - "description": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths where the query ultimately runs against discover (e.g., dashboard split) while the original dataset is metrics, upsampling may be skipped even when all projects are allowlisted." + "description": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths where the query ultimately runs against discover (e.g., dashboard split) while the original dataset is metrics, upsampling may be skipped even when all projects are allowlisted.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "The new eligibility call passes the outer dataset even when _get_event_stats receives a different scoped_dataset for a discover dashboard split. The helper gates transformations on that dataset argument.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/6ad6fe3899f630ccf922ce7ac681147dc7987435/src/sentry/api/endpoints/organization_events_stats.py#L220", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/a5d290951def84afdcc4c88d2f1f20023fc36e2a...6ad6fe3899f630ccf922ce7ac681147dc7987435" + ] + } } ] } diff --git a/eval/corpus/labels/crb-sentry-greptile-5.json b/eval/corpus/labels/crb-sentry-greptile-5.json index fb89414..1d02779 100644 --- a/eval/corpus/labels/crb-sentry-greptile-5.json +++ b/eval/corpus/labels/crb-sentry-greptile-5.json @@ -12,7 +12,17 @@ "severity": "medium", "category": "correctness", "crb_category": "bug", - "description": "Breaking changes in error response format" + "description": "Breaking changes in error response format", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "underspecified", + "why": "The label names no endpoint, old/new format, affected caller, or compatibility requirement. The collector intentionally adds structured validation errors. A format change alone cannot serve as a reproducible correctness gold label.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9/src/sentry/issues/endpoints/browser_reporting_collector.py#L120", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/49a275847631e231672d01b5dcd66558c069d25b...ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9" + ] + } }, { "file": null, @@ -21,7 +31,18 @@ "severity": "medium", "category": "correctness", "crb_category": "bug", - "description": "Detector validator uses wrong key when updating type" + "description": "Detector validator uses wrong key when updating type", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "validated_data.get(\"detector_type\", instance.group_type) is unchanged context in the owner-field patch, not an introduced line or behavior.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9/src/sentry/workflow_engine/endpoints/validators/base/detector.py#L63", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/49a275847631e231672d01b5dcd66558c069d25b...ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9" + ] + }, + "policy_excluded": true }, { "file": null, @@ -30,7 +51,17 @@ "severity": "low", "category": "correctness", "crb_category": "bug", - "description": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment)." + "description": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "NodeStorage.get_multi appends cached entries after uncached entries, so partial cache hits can reorder its values. Zipping those values with input error IDs assigns another event payload to an ID.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9/src/sentry/nodestore/base.py#L184", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/49a275847631e231672d01b5dcd66558c069d25b...ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9" + ] + } } ] } diff --git a/eval/runners/gold-judge.sh b/eval/runners/gold-judge.sh index d48dffe..a84aa72 100755 --- a/eval/runners/gold-judge.sh +++ b/eval/runners/gold-judge.sh @@ -19,6 +19,12 @@ HERE=$(cd "$(dirname "$0")" && pwd) mkdir -p "$OUT" . "$HERE/tiers.sh" +if ! jq -e 'all(.findings[]?; (.adjudication.status // "unreviewed") as $s | + (["unreviewed", "real-miss", "policy-excluded", "label-wrong"] | index($s)) != null)' "$LABELS" >/dev/null; then + echo "gold-judge.sh: unknown adjudication status" >&2 + exit 1 +fi + # Expected-clean short-circuits: any candidate finding is a false positive, # no matcher call needed (gold-eval spec). if [ "$(jq -r '.expected_clean // false' "$LABELS")" = "true" ]; then @@ -34,19 +40,33 @@ jq '.findings' "$LABELS" > "$OUT/gold-labels.json" if [ -f "$OUT/match-gc.json" ]; then echo "gold-judge.sh: reusing recorded match-gc.json (re-judge)" >&2 else - sh "$HERE/match.sh" "$OUT/gold-labels.json" "$CANDIDATE" > "$OUT/match-gc.json" + # Keep original indices so archived matches remain replayable. New model + # calls see only eligible gold, then map their indices back to that array. + jq 'to_entries | map(select(.value.policy_excluded != true and + .value.adjudication.status != "policy-excluded" and + .value.adjudication.status != "label-wrong"))' \ + "$OUT/gold-labels.json" > "$OUT/gold-active-entries.json" + jq 'map(.value)' "$OUT/gold-active-entries.json" > "$OUT/gold-active-labels.json" + sh "$HERE/match.sh" "$OUT/gold-active-labels.json" "$CANDIDATE" \ + | jq --slurpfile g "$OUT/gold-active-entries.json" \ + 'map(.a_idx = $g[0][.a_idx].key)' > "$OUT/match-gc.json" fi jq -n --arg re "$CLEANUP_RE" \ --slurpfile g "$OUT/gold-labels.json" \ --slurpfile c "$CANDIDATE" \ --slurpfile m "$OUT/match-gc.json" ' - ($g[0] | map(. + {tier: (if (.category // "" | test($re)) then "cleanup" else "correctness" end)})) as $g | - ($c[0]) as $c | ($m[0]) as $m | + ($g[0] | map(. + {tier: (if .policy_excluded == true or + .adjudication.status == "policy-excluded" or .adjudication.status == "label-wrong" + then "excluded" elif (.category // "" | test($re)) then "cleanup" else "correctness" end)})) as $g | + ($c[0]) as $c | ($m[0]) as $recorded | + ($recorded | map(select($g[.a_idx].tier != "excluded"))) as $m | ($m | map(.a_idx)) as $gm | ($m | map(.b_idx)) as $cm | ($g | to_entries | map(select(.value.tier == "correctness"))) as $gcorr | { mode: "gold", expected_clean: false, + excluded_gold: [ $g[] | select(.tier == "excluded") ], + excluded_recorded_matches: [ $recorded[] | select($g[.a_idx].tier == "excluded") ], matched: [ $m[] | {gold: $g[.a_idx], candidate: $c[.b_idx]} ], missed_correctness_gold: [ $g | to_entries[] | select(.value.tier == "correctness" and ((.key as $i | $gm | index($i)) == null)) | .value ], diff --git a/eval/runners/test_gold_judge.py b/eval/runners/test_gold_judge.py new file mode 100644 index 0000000..04a713d --- /dev/null +++ b/eval/runners/test_gold_judge.py @@ -0,0 +1,77 @@ +"""Check adjudication denominator changes without renumbering archived matches.""" + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +JUDGE = Path(__file__).with_name("gold-judge.sh") + + +class GoldJudgeTests(unittest.TestCase): + def judge(self, findings, matches): + with tempfile.TemporaryDirectory(prefix="reviso-gold-test-") as directory: + root = Path(directory) + labels, candidate = root / "labels.json", root / "candidate.json" + labels.write_text(json.dumps({"expected_clean": False, "findings": findings})) + candidate.write_text(json.dumps([{"title": "one"}, {"title": "two"}])) + (root / "match-gc.json").write_text(json.dumps(matches)) + result = subprocess.run(["sh", str(JUDGE), str(labels), str(candidate), str(root)], + capture_output=True, text=True) + output = root / "gold-judge.json" + return result, json.loads(output.read_text()) if output.exists() else None + + def test_exclusions_keep_original_match_indices(self): + findings = [{"title": "old", "policy_excluded": True}, + {"title": "miss", "category": "correctness"}, + {"title": "hit", "category": "correctness"}] + result, judged = self.judge(findings, [{"a_idx": 0, "b_idx": 0}, {"a_idx": 2, "b_idx": 1}]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(judged["metrics"]["gold_recall_correctness"], 50) + self.assertEqual(judged["matched"][0]["gold"]["title"], "hit") + self.assertEqual(judged["missed_correctness_gold"][0]["title"], "miss") + self.assertEqual(judged["promotion_candidates"][0]["title"], "one") + self.assertEqual(judged["excluded_recorded_matches"], [{"a_idx": 0, "b_idx": 0}]) + + def test_all_excluded_is_unscored_not_expected_clean(self): + findings = [{"adjudication": {"status": "label-wrong"}}, + {"adjudication": {"status": "policy-excluded"}}] + result, judged = self.judge(findings, []) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIsNone(judged["metrics"]["gold_recall_correctness"]) + self.assertFalse(judged["expected_clean"]) + self.assertNotIn("false_positives", judged) + self.assertEqual(len(judged["excluded_gold"]), 2) + + def test_unknown_disposition_fails_loudly(self): + result, judged = self.judge([{"adjudication": {"status": "typo"}}], []) + self.assertNotEqual(result.returncode, 0) + self.assertIsNone(judged) + self.assertIn("unknown adjudication", result.stderr) + + def test_live_matcher_sees_only_eligible_labels(self): + with tempfile.TemporaryDirectory(prefix="reviso-gold-live-") as directory: + root = Path(directory) + labels, candidate = root / "labels.json", root / "candidate.json" + labels.write_text(json.dumps({"findings": [ + {"title": "excluded-marker", "adjudication": {"status": "label-wrong"}}, + {"title": "eligible", "category": "correctness"}]})) + candidate.write_text(json.dumps([{"title": "candidate"}])) + fake = root / "claude" + fake.write_text('''#!/bin/sh +case "$2" in *excluded-marker*) exit 1 ;; esac +printf '%s\\n' '{"result":"[{\\"a_idx\\":0,\\"b_idx\\":0}]"}' +''') + fake.chmod(0o755) + env = dict(os.environ, MATCH_HOST="claude", PATH=str(root) + os.pathsep + os.environ["PATH"]) + result = subprocess.run(["sh", str(JUDGE), str(labels), str(candidate), str(root)], + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads((root / "match-gc.json").read_text()), [{"a_idx": 1, "b_idx": 0}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/README.md b/eval/runs/2026-10-02-zero-recall-adjudication/README.md new file mode 100644 index 0000000..7efbbeb --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/README.md @@ -0,0 +1,69 @@ +# Zero-recall CRB adjudication — 2026-10-02 + +Source: the 15 cases listed in issue #13, from the 2026-08-07 gold sweep. +All 35 imported findings now carry a dated ruling and pinned source evidence +in `eval/corpus/labels/`. Original descriptions and CRB categories remain +intact. The machine-readable index is +`eval/calibration/adjudication-2026-10-02.json`. + +| Disposition | Findings | +| --- | --- | +| Eligible correctness defects | 11 | +| Policy-excluded | 16 | +| Label-wrong / insufficiently specified | 8 | + +The historical outputs are re-judged here, without rerunning either the +candidate or matcher. The original sweep remains untouched. After the +exclusions and one category correction, the recorded candidate matched +**1 of 11 eligible correctness findings** in this deliberately selected +zero-recall cluster. This is not full-corpus recall, a current-release +result, or a validation of the old Claude matcher. + +The recovered match is Grafana 97529's unlocked cache iteration: the +candidate reported it, but the imported label called a possible map +read/write panic `efficiency`. Its normalized category is now correctness; +`imported_category` preserves the old mapping. + +## Rulings that change the interpretation + +- Grafana 76186: the default old instrumentation middleware already + dereferenced nil requests, and the contextual log provider supplies + `traceID`. Neither label establishes the claimed new regression. +- Grafana 79265: the count-then-insert race remains eligible. Device-limit + rejection is intentional. The alleged compile failure is false: the + vendored Xorm session accepts `Exec(sqlOrArgs ...any)`. +- Grafana 90939 and Sentry 80528: the cache-error assignment and wrong + returned config predate the changes. A latent defect moved unchanged + is excluded by Reviso's changed-behavior policy. +- Sentry 95633: `.python-version` pins 3.13.1, which has `Queue.shutdown`. + Test constants and documentation cleanup do not become correctness + defects merely because the importer mapped them to `bug`. +- Sentry greptile-3: a tuple of integer IDs is not randomized by + `PYTHONHASHSEED`; the label assumes string-hash behavior. The missing + zero sample rate and wrong scoped dataset remain eligible defects. +- Sentry greptile-5: the detector-type key is unchanged context. The + event-ID pairing defect is real: the pinned `NodeStorage.get_multi` + implementation appends cached entries after uncached ones, so partial + cache hits reorder the values before the caller zips them with IDs. +- Discourse graphite-9: locale loading already holds `LOAD_MUTEX` and + rechecks membership. The changed callers use canonical symbol locales; + neither short label identifies a new valid-input failure. + +Two source-based checks supplemented inspection. Executing the pinned +`get_multi` method with A cached and B uncached returned B then A, pairing +the input IDs with the wrong payloads. Integer-tuple hashes were identical +in separate Python processes under hash seeds 1 and 99. No upstream code +is vendored into tracked artifacts. + +## Judge behavior + +Excluded gold remains visible in `excluded_gold`, outside matching and +the denominator. Original indices are retained for replaying recorded +matches. Entirely excluded cases are unscored, not expected-clean: +candidate-only findings still require verification before being called +wins or false positives. The judge rejects unknown adjudication statuses. + +The 11 eligible findings are a starting point for the audit comparison, +not evidence that the current workflow still misses them. Measure both +tiers on the same pinned changes before deciding whether deeper review or +new detection angles are needed. diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/candidate.json new file mode 100644 index 0000000..6573396 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/candidate.json @@ -0,0 +1,18 @@ +[ + { + "file": "packages/features/ee/workflows/api/scheduleSMSReminders.ts", + "line": 190, + "title": "Retry-count bump inside `catch` can throw and abort the whole cron run", + "severity": "P1", + "category": "bugs", + "description": "Row can be deleted during batch processing, causing the retry-count update at line 190 to throw P2025. Since this throw occurs inside a catch block, it escapes the loop and skips all remaining reminders, causing the endpoint to 500." + }, + { + "file": "packages/features/ee/workflows/api/scheduleSMSReminders.ts", + "line": 178, + "title": "SMS-lock is counted as a retryable failure, permanently deleting pending reminders", + "severity": "P1", + "category": "bugs", + "description": "SMS-lock state from twilio.scheduleSMS is treated as a retryable failure and incremented at line 178. Reminders with retryCount > 1 are hard-deleted, so teams locked for ~45 minutes permanently lose all pending reminders in the 7-day window with no logging." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/gold-judge.json new file mode 100644 index 0000000..a0c7726 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/gold-judge.json @@ -0,0 +1,77 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [ + { + "file": null, + "line": 0, + "title": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consid", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consider an atomic increment via Prisma (increment: 1) to avoid race conditions (also applies to the similar update in the catch block).", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "concurrency", + "why": "The new retry updates assign the stale fetched retryCount plus one. Overlapping cron invocations can overwrite one another rather than atomically incrementing the retry count.", + "evidence": [ + "https://github.com/calcom/cal.com/blob/c790227e0cda780e6fea9bb03af27948d9e286b9/packages/features/ee/workflows/api/scheduleSMSReminders.ts#L181", + "https://github.com/calcom/cal.com/compare/917c7b0764f4c6bfdca95db2c5f73dbfdbe11757...c790227e0cda780e6fea9bb03af27948d9e286b9" + ] + }, + "tier": "correctness" + }, + { + "file": "packages/features/ee/workflows/api/scheduleSMSReminders.ts", + "line": 0, + "title": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) tha", + "severity": "high", + "category": "robustness", + "crb_category": "error-handling", + "description": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) that have retryCount > 1. This occurs because the retryCount condition within the OR clause for deletion lacks a method: WorkflowMethods.SMS filter, causing it to apply to all reminder types instead of only SMS reminders, which is the intended scope of this function.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "filter-scope", + "why": "The new retryCount > 1 OR branch lacks method: SMS, so deleteMany applies it to reminders for every method. The previous method filter scoped the whole delete to SMS.", + "evidence": [ + "https://github.com/calcom/cal.com/blob/c790227e0cda780e6fea9bb03af27948d9e286b9/packages/features/ee/workflows/api/scheduleSMSReminders.ts#L38", + "https://github.com/calcom/cal.com/compare/917c7b0764f4c6bfdca95db2c5f73dbfdbe11757...c790227e0cda780e6fea9bb03af27948d9e286b9" + ] + }, + "tier": "correctness" + } + ], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "packages/features/ee/workflows/api/scheduleSMSReminders.ts", + "line": 190, + "title": "Retry-count bump inside `catch` can throw and abort the whole cron run", + "severity": "P1", + "category": "bugs", + "description": "Row can be deleted during batch processing, causing the retry-count update at line 190 to throw P2025. Since this throw occurs inside a catch block, it escapes the loop and skips all remaining reminders, causing the endpoint to 500." + }, + { + "file": "packages/features/ee/workflows/api/scheduleSMSReminders.ts", + "line": 178, + "title": "SMS-lock is counted as a retryable failure, permanently deleting pending reminders", + "severity": "P1", + "category": "bugs", + "description": "SMS-lock state from twilio.scheduleSMS is treated as a retryable failure and incremented at line 178. Reminders with retryCount > 1 are hard-deleted, so teams locked for ~45 minutes permanently lose all pending reminders in the 7-day window with no logging." + } + ], + "metrics": { + "gold_count": 2, + "gold_correctness_count": 2, + "candidate_count": 2, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/gold-labels.json new file mode 100644 index 0000000..89a52c4 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/gold-labels.json @@ -0,0 +1,40 @@ +[ + { + "file": null, + "line": 0, + "title": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consid", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consider an atomic increment via Prisma (increment: 1) to avoid race conditions (also applies to the similar update in the catch block).", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "concurrency", + "why": "The new retry updates assign the stale fetched retryCount plus one. Overlapping cron invocations can overwrite one another rather than atomically incrementing the retry count.", + "evidence": [ + "https://github.com/calcom/cal.com/blob/c790227e0cda780e6fea9bb03af27948d9e286b9/packages/features/ee/workflows/api/scheduleSMSReminders.ts#L181", + "https://github.com/calcom/cal.com/compare/917c7b0764f4c6bfdca95db2c5f73dbfdbe11757...c790227e0cda780e6fea9bb03af27948d9e286b9" + ] + } + }, + { + "file": "packages/features/ee/workflows/api/scheduleSMSReminders.ts", + "line": 0, + "title": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) tha", + "severity": "high", + "category": "robustness", + "crb_category": "error-handling", + "description": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) that have retryCount > 1. This occurs because the retryCount condition within the OR clause for deletion lacks a method: WorkflowMethods.SMS filter, causing it to apply to all reminder types instead of only SMS reminders, which is the intended scope of this function.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "filter-scope", + "why": "The new retryCount > 1 OR branch lacks method: SMS, so deleteMany applies it to reminders for every method. The previous method filter scoped the whole delete to SMS.", + "evidence": [ + "https://github.com/calcom/cal.com/blob/c790227e0cda780e6fea9bb03af27948d9e286b9/packages/features/ee/workflows/api/scheduleSMSReminders.ts#L38", + "https://github.com/calcom/cal.com/compare/917c7b0764f4c6bfdca95db2c5f73dbfdbe11757...c790227e0cda780e6fea9bb03af27948d9e286b9" + ] + } + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/meta.json new file mode 100644 index 0000000..f63ddab --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-cal.com-14943/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.224", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-cal.com-14943", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/candidate.json new file mode 100644 index 0000000..43eb86d --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/candidate.json @@ -0,0 +1,26 @@ +[ + { + "file": "app/controllers/application_controller.rb", + "line": 159, + "title": "Fallbacks are inert outside the web request path", + "severity": "P1", + "category": "bugs", + "description": "I18n.fallbacks.ensure_loaded! is called only from ApplicationController#set_locale. In Sidekiq jobs and other contexts (post_alerter, post_destroyer, db.rake) where I18n.locale is set without ensure_loaded!, fallback locales are never loaded. On non-English default_locale sites, translation keys that exist only in :en resolve to MissingTranslation in emails and notifications." + }, + { + "file": "config/initializers/i18n.rb", + "line": 17, + "title": "Translation LRU cache key doesn't include the site's default_locale", + "severity": "P1", + "category": "bugs", + "description": "Cache key is key + config.locale + backend.object_id, omitting SiteSetting.default_locale which is per-site and mutable. Multisite: two sites with different default_locales can poison the LRU and serve each other wrong translations. Single-site: an admin changing default_locale leaves stale fallback entries cached until process restart." + }, + { + "file": "config/initializers/i18n.rb", + "line": 13, + "title": "Uncached fallback list runs a SiteSetting lookup on every translate", + "severity": "P2", + "category": "slop, comments", + "description": "FallbackLocaleList#[] recomputes [locale, SiteSetting.default_locale.to_sym, :en] on every call instead of memoizing. I18n.Backend.Fallbacks#translate calls it on every backend lookup, and every interpolated I18n.t(key, count: n) bypasses the LRU cache. Stock implementation memoizes to avoid this overhead." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/gold-judge.json new file mode 100644 index 0000000..d6578f2 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/gold-judge.json @@ -0,0 +1,85 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": "lib/freedom_patches/translate_accelerator.rb", + "line": 0, + "title": "Thread-safety issue with lazy @loaded_locales", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Thread-safety issue with lazy @loaded_locales", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "refuted", + "why": "load_locale already holds LOAD_MUTEX and rechecks membership inside the lock. The label supplies no unsynchronized write or reproducible new duplicate-load race for normally initialized locale state.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/discourse-graphite/blob/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293/lib/freedom_patches/translate_accelerator.rb#L45", + "https://github.com/ai-code-review-evaluation/discourse-graphite/compare/728845d008f06828630a632e5c7d67e11bf15deb...ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293" + ] + }, + "tier": "excluded" + }, + { + "file": null, + "line": 0, + "title": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same locale is passed as a String vs Symbol (also applies to other locations in the PR).", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "unreachable-input", + "why": "The new caller supplies I18n.locale and symbol-valued fallback locales; it does not introduce a caller passing a string. Existing locale loading already used the same array. No new mixed-string/symbol loading path is established.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/discourse-graphite/blob/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293/config/initializers/i18n.rb#L6", + "https://github.com/ai-code-review-evaluation/discourse-graphite/compare/728845d008f06828630a632e5c7d67e11bf15deb...ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293" + ] + }, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "app/controllers/application_controller.rb", + "line": 159, + "title": "Fallbacks are inert outside the web request path", + "severity": "P1", + "category": "bugs", + "description": "I18n.fallbacks.ensure_loaded! is called only from ApplicationController#set_locale. In Sidekiq jobs and other contexts (post_alerter, post_destroyer, db.rake) where I18n.locale is set without ensure_loaded!, fallback locales are never loaded. On non-English default_locale sites, translation keys that exist only in :en resolve to MissingTranslation in emails and notifications." + }, + { + "file": "config/initializers/i18n.rb", + "line": 17, + "title": "Translation LRU cache key doesn't include the site's default_locale", + "severity": "P1", + "category": "bugs", + "description": "Cache key is key + config.locale + backend.object_id, omitting SiteSetting.default_locale which is per-site and mutable. Multisite: two sites with different default_locales can poison the LRU and serve each other wrong translations. Single-site: an admin changing default_locale leaves stale fallback entries cached until process restart." + }, + { + "file": "config/initializers/i18n.rb", + "line": 13, + "title": "Uncached fallback list runs a SiteSetting lookup on every translate", + "severity": "P2", + "category": "slop, comments", + "description": "FallbackLocaleList#[] recomputes [locale, SiteSetting.default_locale.to_sym, :en] on every call instead of memoizing. I18n.Backend.Fallbacks#translate calls it on every backend lookup, and every interpolated I18n.t(key, count: n) bypasses the LRU cache. Stock implementation memoizes to avoid this overhead." + } + ], + "metrics": { + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 3, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/gold-labels.json new file mode 100644 index 0000000..5d258e9 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/gold-labels.json @@ -0,0 +1,40 @@ +[ + { + "file": "lib/freedom_patches/translate_accelerator.rb", + "line": 0, + "title": "Thread-safety issue with lazy @loaded_locales", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Thread-safety issue with lazy @loaded_locales", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "refuted", + "why": "load_locale already holds LOAD_MUTEX and rechecks membership inside the lock. The label supplies no unsynchronized write or reproducible new duplicate-load race for normally initialized locale state.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/discourse-graphite/blob/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293/lib/freedom_patches/translate_accelerator.rb#L45", + "https://github.com/ai-code-review-evaluation/discourse-graphite/compare/728845d008f06828630a632e5c7d67e11bf15deb...ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293" + ] + } + }, + { + "file": null, + "line": 0, + "title": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same locale is passed as a String vs Symbol (also applies to other locations in the PR).", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "unreachable-input", + "why": "The new caller supplies I18n.locale and symbol-valued fallback locales; it does not introduce a caller passing a string. Existing locale loading already used the same array. No new mixed-string/symbol loading path is established.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/discourse-graphite/blob/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293/config/initializers/i18n.rb#L6", + "https://github.com/ai-code-review-evaluation/discourse-graphite/compare/728845d008f06828630a632e5c7d67e11bf15deb...ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293" + ] + } + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/meta.json new file mode 100644 index 0000000..cfc27ec --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-discourse-graphite-9/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.224", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-discourse-graphite-9", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/candidate.json new file mode 100644 index 0000000..c9e31c7 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/candidate.json @@ -0,0 +1,26 @@ +[ + { + "file": "pkg/services/authz/rbac/cache.go", + "line": 30, + "title": "Denial cache key omits identity type — anonymous:0 and render:0 collide", + "severity": "P1", + "category": "bugs", + "description": "Denial cache key lacks identity type field. Anonymous users and render service both get identifier \"0\", causing cache collisions. Anonymous denied dashboards:read poisons render service's read check within 30s, breaking image rendering for restricted dashboards; reverse also holds for write." + }, + { + "file": "pkg/services/authz/rbac/service_test.go", + "line": 982, + "title": "Should deny on explicit cache deny entry passes with the denial cache removed", + "severity": "P2", + "category": "bugs", + "description": "Test seeded scope map has false values that grant nothing, not the true values needed to test the cache deny short-circuit. Test passes even with denial cache removed, so Check's denial cache short-circuit at service.go:117 is untested." + }, + { + "file": "pkg/services/authz/rbac/service_test.go", + "line": 337, + "title": "Dead cacheHit scaffolding left in TestService_getUserPermissions", + "severity": "P2", + "category": "slop", + "description": "cacheHit field is false in both remaining test cases; seeding block (lines 367–369) and assertion (lines 388–390) are unreachable. Cached path is no longer tested but future readers will assume it is." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/gold-judge.json new file mode 100644 index 0000000..37b826f --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/gold-judge.json @@ -0,0 +1,92 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": null, + "line": 0, + "title": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately", + "severity": "high", + "category": "security", + "crb_category": "security", + "description": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately, but cached denials from the same permission cache are ignored, leading to a fresh database lookup. This allows stale cached grants to provide access to revoked resources, posing a security risk. ", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "Cached grants were already trusted in getUserPermissions/getAnonymousPermissions. The new path refreshes a denial from the permission map, while the separate permission-denial cache explicitly short-circuits to false; the label does not establish a new stale-grant regression.", + "evidence": [ + "https://github.com/grafana/grafana/blob/5634ca44f799a82161f0401dadc286452f3bdbf8/pkg/services/authz/rbac/service.go#L316", + "https://github.com/grafana/grafana/compare/5634ca44f799a82161f0401dadc286452f3bdbf8...7562f37880367411a62304bcdcdc178bced23906" + ] + }, + "policy_excluded": true, + "tier": "excluded" + }, + { + "file": "pkg/services/authz/rbac/service.go", + "line": 0, + "title": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so check", + "severity": "low", + "category": "efficiency", + "crb_category": "performance", + "description": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so checkPermission will still treat this scope as not allowed.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "test-coverage", + "why": "The false-valued test seed does not exercise the intended cached-grant override. This is a coverage-only gold item, not a production behavior defect; review excludes missing test coverage. The old candidate did surface it.", + "evidence": [ + "https://github.com/grafana/grafana/blob/7562f37880367411a62304bcdcdc178bced23906/pkg/services/authz/rbac/service_test.go#L982", + "https://github.com/grafana/grafana/compare/5634ca44f799a82161f0401dadc286452f3bdbf8...7562f37880367411a62304bcdcdc178bced23906" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [ + { + "a_idx": 1, + "b_idx": 1 + } + ], + "matched": [], + "missed_correctness_gold": [], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "pkg/services/authz/rbac/cache.go", + "line": 30, + "title": "Denial cache key omits identity type — anonymous:0 and render:0 collide", + "severity": "P1", + "category": "bugs", + "description": "Denial cache key lacks identity type field. Anonymous users and render service both get identifier \"0\", causing cache collisions. Anonymous denied dashboards:read poisons render service's read check within 30s, breaking image rendering for restricted dashboards; reverse also holds for write." + }, + { + "file": "pkg/services/authz/rbac/service_test.go", + "line": 982, + "title": "Should deny on explicit cache deny entry passes with the denial cache removed", + "severity": "P2", + "category": "bugs", + "description": "Test seeded scope map has false values that grant nothing, not the true values needed to test the cache deny short-circuit. Test passes even with denial cache removed, so Check's denial cache short-circuit at service.go:117 is untested." + }, + { + "file": "pkg/services/authz/rbac/service_test.go", + "line": 337, + "title": "Dead cacheHit scaffolding left in TestService_getUserPermissions", + "severity": "P2", + "category": "slop", + "description": "cacheHit field is false in both remaining test cases; seeding block (lines 367–369) and assertion (lines 388–390) are unreachable. Cached path is no longer tested but future readers will assume it is." + } + ], + "metrics": { + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 3, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/gold-labels.json new file mode 100644 index 0000000..23fbff3 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/gold-labels.json @@ -0,0 +1,42 @@ +[ + { + "file": null, + "line": 0, + "title": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately", + "severity": "high", + "category": "security", + "crb_category": "security", + "description": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately, but cached denials from the same permission cache are ignored, leading to a fresh database lookup. This allows stale cached grants to provide access to revoked resources, posing a security risk. ", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "Cached grants were already trusted in getUserPermissions/getAnonymousPermissions. The new path refreshes a denial from the permission map, while the separate permission-denial cache explicitly short-circuits to false; the label does not establish a new stale-grant regression.", + "evidence": [ + "https://github.com/grafana/grafana/blob/5634ca44f799a82161f0401dadc286452f3bdbf8/pkg/services/authz/rbac/service.go#L316", + "https://github.com/grafana/grafana/compare/5634ca44f799a82161f0401dadc286452f3bdbf8...7562f37880367411a62304bcdcdc178bced23906" + ] + }, + "policy_excluded": true + }, + { + "file": "pkg/services/authz/rbac/service.go", + "line": 0, + "title": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so check", + "severity": "low", + "category": "efficiency", + "crb_category": "performance", + "description": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so checkPermission will still treat this scope as not allowed.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "test-coverage", + "why": "The false-valued test seed does not exercise the intended cached-grant override. This is a coverage-only gold item, not a production behavior defect; review excludes missing test coverage. The old candidate did surface it.", + "evidence": [ + "https://github.com/grafana/grafana/blob/7562f37880367411a62304bcdcdc178bced23906/pkg/services/authz/rbac/service_test.go#L982", + "https://github.com/grafana/grafana/compare/5634ca44f799a82161f0401dadc286452f3bdbf8...7562f37880367411a62304bcdcdc178bced23906" + ] + }, + "policy_excluded": true + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/match-gc.json new file mode 100644 index 0000000..793b75d --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/match-gc.json @@ -0,0 +1,6 @@ +[ + { + "a_idx": 1, + "b_idx": 1 + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/meta.json new file mode 100644 index 0000000..302cddf --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-103633/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.224", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-grafana-103633", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/candidate.json new file mode 100644 index 0000000..c386780 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/candidate.json @@ -0,0 +1,10 @@ +[ + { + "file": "public/app/plugins/datasource/loki/querySplitting.ts", + "line": 299, + "title": "Ad hoc filters are now applied twice (three times under shard splitting), duplicating label filters", + "severity": "P1", + "category": "bugs", + "description": "applyTemplateVariables is not idempotent when request.filters is non-empty; it is called multiple times through the call chain (runSplitQuery → runSplitGroupedQueries → datasource.runQuery → DataSourceWithBackend.query), causing duplicate label filters in the final Loki query. With shardQuerySplitting enabled, this occurs three times. Fix: interpolate for split decisions without re-interpolating in downstream layers, either by keeping request.targets raw or by passing a skipTemplateVariables marker." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/gold-judge.json new file mode 100644 index 0000000..1b537be --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/gold-judge.json @@ -0,0 +1,50 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": null, + "line": 0, + "title": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used ", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used in the corresponding test setup.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "test-coverage", + "why": "This label only says the test setup omits the third argument; it gives no behavior defect. Coverage omissions are excluded. The candidate double-interpolation finding is a different root cause.", + "evidence": [ + "https://github.com/grafana/grafana/blob/1cab4143d443c33079566f6ca36931eb8468159f/public/app/plugins/datasource/loki/shardQuerySplitting.test.ts#L53", + "https://github.com/grafana/grafana/compare/d9a8253640464014aa3662c91131278990cfb828...1cab4143d443c33079566f6ca36931eb8468159f" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "public/app/plugins/datasource/loki/querySplitting.ts", + "line": 299, + "title": "Ad hoc filters are now applied twice (three times under shard splitting), duplicating label filters", + "severity": "P1", + "category": "bugs", + "description": "applyTemplateVariables is not idempotent when request.filters is non-empty; it is called multiple times through the call chain (runSplitQuery → runSplitGroupedQueries → datasource.runQuery → DataSourceWithBackend.query), causing duplicate label filters in the final Loki query. With shardQuerySplitting enabled, this occurs three times. Fix: interpolate for split decisions without re-interpolating in downstream layers, either by keeping request.targets raw or by passing a skipTemplateVariables marker." + } + ], + "metrics": { + "gold_count": 1, + "gold_correctness_count": 0, + "candidate_count": 1, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/gold-labels.json new file mode 100644 index 0000000..58c82f5 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/gold-labels.json @@ -0,0 +1,22 @@ +[ + { + "file": null, + "line": 0, + "title": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used ", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used in the corresponding test setup.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "test-coverage", + "why": "This label only says the test setup omits the third argument; it gives no behavior defect. Coverage omissions are excluded. The candidate double-interpolation finding is a different root cause.", + "evidence": [ + "https://github.com/grafana/grafana/blob/1cab4143d443c33079566f6ca36931eb8468159f/public/app/plugins/datasource/loki/shardQuerySplitting.test.ts#L53", + "https://github.com/grafana/grafana/compare/d9a8253640464014aa3662c91131278990cfb828...1cab4143d443c33079566f6ca36931eb8468159f" + ] + }, + "policy_excluded": true + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/meta.json new file mode 100644 index 0000000..79b7b52 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-107534/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.224", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-grafana-107534", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/candidate.json new file mode 100644 index 0000000..ac5604b --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/candidate.json @@ -0,0 +1,10 @@ +[ + { + "file": "pkg/services/pluginsintegration/clientmiddleware/contextual_logger_middleware.go", + "line": 12, + "title": "Middleware-ordering invariant was deleted, not carried over", + "severity": "P2", + "category": "configuration", + "description": "Middleware-ordering constraint was removed during refactoring. LoggerMiddleware now depends on context attributes set by NewContextualLoggerMiddleware, but without documented ordering requirement. Reordering the middleware stack would silently lose log fields (pluginId, endpoint, dsName, dsUID, uname) with no compile error or test failure." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/gold-judge.json new file mode 100644 index 0000000..c95e11f --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/gold-judge.json @@ -0,0 +1,70 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": null, + "line": 0, + "title": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request i", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request is received. This occurs because they directly access req.PluginContext (via the instrumentContext function) without first checking if req is nil. This is a regression, as previous middleware layers gracefully handled nil requests.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The default middleware stack already dereferences nil requests in InstrumentationMiddleware before LoggerMiddleware. The new middleware does not introduce the claimed nil-request regression.", + "evidence": [ + "https://github.com/grafana/grafana/blob/58ba11ecbd60863e2bfc6e32f0399ed4feb3927a/pkg/services/pluginsintegration/clientmiddleware/instrumentation_middleware.go#L161", + "https://github.com/grafana/grafana/compare/58ba11ecbd60863e2bfc6e32f0399ed4feb3927a...303cdc2caf256aa5df660611120c2e3f7392365d" + ] + }, + "policy_excluded": true, + "tier": "excluded" + }, + { + "file": null, + "line": 0, + "title": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract a", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract and add traceID from the context to log parameters were removed from the LoggerMiddleware. The newly introduced ContextualLoggerMiddleware does not add this information, resulting in missing traceID in plugin request logs and impacting debugging and request tracing capabilities.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "refuted", + "why": "Tracing registers a contextual log provider that supplies traceID, and the changed logger calls FromContext. Removing the explicit argument does not remove the logged field.", + "evidence": [ + "https://github.com/grafana/grafana/blob/303cdc2caf256aa5df660611120c2e3f7392365d/pkg/infra/tracing/tracing.go#L91", + "https://github.com/grafana/grafana/compare/58ba11ecbd60863e2bfc6e32f0399ed4feb3927a...303cdc2caf256aa5df660611120c2e3f7392365d" + ] + }, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "pkg/services/pluginsintegration/clientmiddleware/contextual_logger_middleware.go", + "line": 12, + "title": "Middleware-ordering invariant was deleted, not carried over", + "severity": "P2", + "category": "configuration", + "description": "Middleware-ordering constraint was removed during refactoring. LoggerMiddleware now depends on context attributes set by NewContextualLoggerMiddleware, but without documented ordering requirement. Reordering the middleware stack would silently lose log fields (pluginId, endpoint, dsName, dsUID, uname) with no compile error or test failure." + } + ], + "metrics": { + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 1, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/gold-labels.json new file mode 100644 index 0000000..fc838ee --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/gold-labels.json @@ -0,0 +1,41 @@ +[ + { + "file": null, + "line": 0, + "title": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request i", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request is received. This occurs because they directly access req.PluginContext (via the instrumentContext function) without first checking if req is nil. This is a regression, as previous middleware layers gracefully handled nil requests.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The default middleware stack already dereferences nil requests in InstrumentationMiddleware before LoggerMiddleware. The new middleware does not introduce the claimed nil-request regression.", + "evidence": [ + "https://github.com/grafana/grafana/blob/58ba11ecbd60863e2bfc6e32f0399ed4feb3927a/pkg/services/pluginsintegration/clientmiddleware/instrumentation_middleware.go#L161", + "https://github.com/grafana/grafana/compare/58ba11ecbd60863e2bfc6e32f0399ed4feb3927a...303cdc2caf256aa5df660611120c2e3f7392365d" + ] + }, + "policy_excluded": true + }, + { + "file": null, + "line": 0, + "title": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract a", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract and add traceID from the context to log parameters were removed from the LoggerMiddleware. The newly introduced ContextualLoggerMiddleware does not add this information, resulting in missing traceID in plugin request logs and impacting debugging and request tracing capabilities.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "refuted", + "why": "Tracing registers a contextual log provider that supplies traceID, and the changed logger calls FromContext. Removing the explicit argument does not remove the logged field.", + "evidence": [ + "https://github.com/grafana/grafana/blob/303cdc2caf256aa5df660611120c2e3f7392365d/pkg/infra/tracing/tracing.go#L91", + "https://github.com/grafana/grafana/compare/58ba11ecbd60863e2bfc6e32f0399ed4feb3927a...303cdc2caf256aa5df660611120c2e3f7392365d" + ] + } + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/meta.json new file mode 100644 index 0000000..6940b24 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-76186/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.224", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-grafana-76186", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/candidate.json new file mode 100644 index 0000000..e14b241 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/candidate.json @@ -0,0 +1,34 @@ +[ + { + "file": "pkg/services/anonymous/anonimpl/impl.go", + "line": 147, + "title": "Device limit is bypassed on every request after the first rejection", + "severity": "P0", + "category": "bugs", + "description": "tagDeviceUI writes cache entry before store call, so ErrDeviceLimitReached results in the device being cached. First request from over-limit device gets 401; subsequent requests hit cache short-circuit and authenticate normally without ever being written to anon_device, never counting toward limit." + }, + { + "file": "pkg/services/anonymous/anonimpl/impl.go", + "line": 43, + "title": "Store is no longer injectable; wire registration is now dead and unbuildable", + "severity": "P1", + "category": "conventions", + "description": "ProvideAnonymousDeviceService now constructs anonstore.ProvideAnonDBStore internally instead of receiving anonstore.AnonStore, but pkg/server/wire.go still registers it. New deviceLimit int64 parameter has no provider in the graph, causing wire to fail when anything asks for anonstore.AnonStore." + }, + { + "file": "pkg/setting/setting.go", + "line": 1654, + "title": "New `device_limit` ini option is undocumented", + "severity": "P2", + "category": "conventions", + "description": "PR checklist requires configuration changes to update conf/defaults.ini, conf/sample.ini, and configuration docs. None were touched. Operators have no way to discover the limit exists or that 0 means unlimited." + }, + { + "file": "packages/grafana-runtime/src/config.ts", + "line": 97, + "title": "`anonymousDeviceLimit` is typed `undefined`, not `number | undefined`", + "severity": "P2", + "category": "slop", + "description": "Un-annotated class field initialized to undefined infers type `undefined` under strict mode. Consumers writing `config.anonymousDeviceLimit > 0` get type errors, though getFrontendSettings always sends an int64." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/gold-judge.json new file mode 100644 index 0000000..f6a40bb --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/gold-judge.json @@ -0,0 +1,156 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": "pkg/services/anonymous/anonimpl/client.go", + "line": 0, + "title": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, de", + "severity": "medium", + "category": "security", + "crb_category": "security", + "description": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, device tagging was asynchronous and non-blocking. This change prevents anonymous users from authenticating when the device limit is reached.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "intentional", + "why": "Rejecting anonymous authentication at the configured device limit is the purpose of this change; TagDevice now returns the store error and the anonymous client explicitly handles ErrDeviceLimitReached.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/client.go#L44", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + }, + "policy_excluded": true, + "tier": "excluded" + }, + { + "file": "pkg/services/anonymous/anonimpl/anonstore/database.go", + "line": 0, + "title": "This call won’t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "This call won’t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec’s signature requires a first parameter of type string (not an interface{} splat).", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-api", + "why": "DBSession embeds the vendored Xorm Session. Its Exec signature is Exec(sqlOrArgs ...any), not Exec(query string, args ...any); the interface-slice splat is valid.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/util/xorm/session_raw.go#L208", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + }, + "tier": "excluded" + }, + { + "file": null, + "line": 0, + "title": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist.", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "intentional", + "why": "At capacity, an existing recent device may be updated and a missing/expired device must not be inserted. Returning ErrDeviceLimitReached when the update finds no eligible row implements that policy.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/anonstore/database.go#L73", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + }, + "policy_excluded": true, + "tier": "excluded" + }, + { + "file": null, + "line": 0, + "title": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound b", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound but device.UpdatedAt as the current time may not match the intended logic. Consider using time.Now().UTC() consistently.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-premise", + "why": "Both time bounds use device.UpdatedAt.UTC(); the upper bound includes one minute. The label describes a mixed-UTC expression that is not present, and UTC conversion preserves the instant.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/anonstore/database.go#L81", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + }, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [ + { + "file": null, + "line": 0, + "title": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond ", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond the limit. Consider using a database transaction or lock.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "concurrency", + "why": "CountDevices and the insert/upsert run in separate sessions without a transaction or shared lock. Two uncached device requests can both observe capacity and insert beyond the configured limit.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/anonstore/database.go#L112", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + }, + "tier": "correctness" + } + ], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "pkg/services/anonymous/anonimpl/impl.go", + "line": 147, + "title": "Device limit is bypassed on every request after the first rejection", + "severity": "P0", + "category": "bugs", + "description": "tagDeviceUI writes cache entry before store call, so ErrDeviceLimitReached results in the device being cached. First request from over-limit device gets 401; subsequent requests hit cache short-circuit and authenticate normally without ever being written to anon_device, never counting toward limit." + }, + { + "file": "pkg/services/anonymous/anonimpl/impl.go", + "line": 43, + "title": "Store is no longer injectable; wire registration is now dead and unbuildable", + "severity": "P1", + "category": "conventions", + "description": "ProvideAnonymousDeviceService now constructs anonstore.ProvideAnonDBStore internally instead of receiving anonstore.AnonStore, but pkg/server/wire.go still registers it. New deviceLimit int64 parameter has no provider in the graph, causing wire to fail when anything asks for anonstore.AnonStore." + }, + { + "file": "pkg/setting/setting.go", + "line": 1654, + "title": "New `device_limit` ini option is undocumented", + "severity": "P2", + "category": "conventions", + "description": "PR checklist requires configuration changes to update conf/defaults.ini, conf/sample.ini, and configuration docs. None were touched. Operators have no way to discover the limit exists or that 0 means unlimited." + }, + { + "file": "packages/grafana-runtime/src/config.ts", + "line": 97, + "title": "`anonymousDeviceLimit` is typed `undefined`, not `number | undefined`", + "severity": "P2", + "category": "slop", + "description": "Un-annotated class field initialized to undefined infers type `undefined` under strict mode. Consumers writing `config.anonymousDeviceLimit > 0` get type errors, though getFrontendSettings always sends an int64." + } + ], + "metrics": { + "gold_count": 5, + "gold_correctness_count": 1, + "candidate_count": 4, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/gold-labels.json new file mode 100644 index 0000000..b87d321 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/gold-labels.json @@ -0,0 +1,99 @@ +[ + { + "file": null, + "line": 0, + "title": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond ", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond the limit. Consider using a database transaction or lock.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "concurrency", + "why": "CountDevices and the insert/upsert run in separate sessions without a transaction or shared lock. Two uncached device requests can both observe capacity and insert beyond the configured limit.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/anonstore/database.go#L112", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + } + }, + { + "file": "pkg/services/anonymous/anonimpl/client.go", + "line": 0, + "title": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, de", + "severity": "medium", + "category": "security", + "crb_category": "security", + "description": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, device tagging was asynchronous and non-blocking. This change prevents anonymous users from authenticating when the device limit is reached.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "intentional", + "why": "Rejecting anonymous authentication at the configured device limit is the purpose of this change; TagDevice now returns the store error and the anonymous client explicitly handles ErrDeviceLimitReached.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/client.go#L44", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + }, + "policy_excluded": true + }, + { + "file": "pkg/services/anonymous/anonimpl/anonstore/database.go", + "line": 0, + "title": "This call won’t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "This call won’t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec’s signature requires a first parameter of type string (not an interface{} splat).", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-api", + "why": "DBSession embeds the vendored Xorm Session. Its Exec signature is Exec(sqlOrArgs ...any), not Exec(query string, args ...any); the interface-slice splat is valid.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/util/xorm/session_raw.go#L208", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + } + }, + { + "file": null, + "line": 0, + "title": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist.", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "intentional", + "why": "At capacity, an existing recent device may be updated and a missing/expired device must not be inserted. Returning ErrDeviceLimitReached when the update finds no eligible row implements that policy.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/anonstore/database.go#L73", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + }, + "policy_excluded": true + }, + { + "file": null, + "line": 0, + "title": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound b", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound but device.UpdatedAt as the current time may not match the intended logic. Consider using time.Now().UTC() consistently.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-premise", + "why": "Both time bounds use device.UpdatedAt.UTC(); the upper bound includes one minute. The label describes a mixed-UTC expression that is not present, and UTC conversion preserves the instant.", + "evidence": [ + "https://github.com/grafana/grafana/blob/bbd8c507cdf56f6b884373809e0b24abd4a2353d/pkg/services/anonymous/anonimpl/anonstore/database.go#L81", + "https://github.com/grafana/grafana/compare/50f4e78a39914711d0d231a501ba215becf17ebc...bbd8c507cdf56f6b884373809e0b24abd4a2353d" + ] + } + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/meta.json new file mode 100644 index 0000000..8ac0f57 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-79265/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.224", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-grafana-79265", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/candidate.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/candidate.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/gold-judge.json new file mode 100644 index 0000000..50f340e --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/gold-judge.json @@ -0,0 +1,62 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": "pkg/api/webassets/webassets.go", + "line": 0, + "title": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function f", + "severity": "medium", + "category": "efficiency", + "crb_category": "performance", + "description": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function first checks if the cache is populated using a read lock (RLock), and if the cache is empty, it acquires a write lock to populate it. However, it fails to re-check whether the cache was populated by another goroutine while waiting to acquire the write lock.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "efficiency", + "why": "The missing second cache check may repeat manifest parsing under cold-start concurrency; it does not invalidate the pointer publication lock. Efficiency is outside the correctness lane.", + "evidence": [ + "https://github.com/grafana/grafana/blob/b1613e320acff00623e6efc59f00ee68c7684a97/pkg/api/webassets/webassets.go#L42", + "https://github.com/grafana/grafana/compare/3ce1a5b0caab215346a51629b0344b90d67e9478...b1613e320acff00623e6efc59f00ee68c7684a97" + ] + }, + "policy_excluded": true, + "tier": "excluded" + }, + { + "file": "pkg/api/webassets/webassets.go", + "line": 0, + "title": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assi", + "severity": "high", + "category": "robustness", + "crb_category": "error-handling", + "description": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assigns the fetch result to the cache (line 69: entryPointAssetsCache = result) regardless of whether the fetch succeeded or failed. When an error occurs during asset fetching, result is nil, and this nil value overwrites any previously valid cache entry.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "Assigning result to the global cache on the error path is unchanged. The patch introduces locks around the existing assignment rather than a new cache-wipe behavior.", + "evidence": [ + "https://github.com/grafana/grafana/blob/3ce1a5b0caab215346a51629b0344b90d67e9478/pkg/api/webassets/webassets.go#L59", + "https://github.com/grafana/grafana/compare/3ce1a5b0caab215346a51629b0344b90d67e9478...b1613e320acff00623e6efc59f00ee68c7684a97" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [], + "metrics": { + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 0, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": null + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/gold-labels.json new file mode 100644 index 0000000..035bb78 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/gold-labels.json @@ -0,0 +1,42 @@ +[ + { + "file": "pkg/api/webassets/webassets.go", + "line": 0, + "title": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function f", + "severity": "medium", + "category": "efficiency", + "crb_category": "performance", + "description": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function first checks if the cache is populated using a read lock (RLock), and if the cache is empty, it acquires a write lock to populate it. However, it fails to re-check whether the cache was populated by another goroutine while waiting to acquire the write lock.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "efficiency", + "why": "The missing second cache check may repeat manifest parsing under cold-start concurrency; it does not invalidate the pointer publication lock. Efficiency is outside the correctness lane.", + "evidence": [ + "https://github.com/grafana/grafana/blob/b1613e320acff00623e6efc59f00ee68c7684a97/pkg/api/webassets/webassets.go#L42", + "https://github.com/grafana/grafana/compare/3ce1a5b0caab215346a51629b0344b90d67e9478...b1613e320acff00623e6efc59f00ee68c7684a97" + ] + }, + "policy_excluded": true + }, + { + "file": "pkg/api/webassets/webassets.go", + "line": 0, + "title": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assi", + "severity": "high", + "category": "robustness", + "crb_category": "error-handling", + "description": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assigns the fetch result to the cache (line 69: entryPointAssetsCache = result) regardless of whether the fetch succeeded or failed. When an error occurs during asset fetching, result is nil, and this nil value overwrites any previously valid cache entry.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "Assigning result to the global cache on the error path is unchanged. The patch introduces locks around the existing assignment rather than a new cache-wipe behavior.", + "evidence": [ + "https://github.com/grafana/grafana/blob/3ce1a5b0caab215346a51629b0344b90d67e9478/pkg/api/webassets/webassets.go#L59", + "https://github.com/grafana/grafana/compare/3ce1a5b0caab215346a51629b0344b90d67e9478...b1613e320acff00623e6efc59f00ee68c7684a97" + ] + }, + "policy_excluded": true + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/meta.json new file mode 100644 index 0000000..a1efe78 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-90939/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.224", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-grafana-90939", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/candidate.json new file mode 100644 index 0000000..2714153 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/candidate.json @@ -0,0 +1,18 @@ +[ + { + "file": "pkg/storage/unified/resource/search.go", + "line": 216, + "title": "Unsynchronized read of the bleve index cache", + "severity": "P1", + "category": "bugs", + "description": "TotalDocs() ranges over b.cache without taking cacheMu.RLock(), while BuildIndex writes under lock. Concurrent map read and write can cause fatal error, killing process. Also reachable from BleveIndexMetrics.Collect on every /metrics scrape. Fix: guard the reader with RLock/RUnlock." + }, + { + "file": "pkg/server/module_server_test.go", + "line": 36, + "title": "Postgres skip papers over a startup-latency regression this PR introduces", + "severity": "P2", + "category": "history; conventions", + "description": "NewResourceServer runs s.Init(ctx) synchronously, executing DB work before module reports running. Test has fixed 500ms sleep before checking /metrics, but on postgres in Drone this exceeds 500ms. Skip was disabled rather than fixed. Fix: replace fixed sleep with poll using require.Eventually." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/gold-judge.json new file mode 100644 index 0000000..86d2f05 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/gold-judge.json @@ -0,0 +1,82 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": "pkg/storage/unified/search/bleve.go", + "line": 0, + "title": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same ke", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment. ", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "efficiency", + "why": "The label describes redundant expensive builds, without a new failure scenario. getOrCreateIndex performs its cache check outside BuildIndex in both versions, so the old function-wide lock did not recheck the cache or prevent queued duplicate builds. Do not count this performance-only concern as a correctness miss.", + "evidence": [ + "https://github.com/grafana/grafana/blob/26fed312840cd76b766cbd2158e17a7e6c0ec548/pkg/storage/unified/search/bleve.go#L93", + "https://github.com/grafana/grafana/compare/871af07203177ce59a24e742107119409274f48a...26fed312840cd76b766cbd2158e17a7e6c0ec548" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [ + { + "gold": { + "file": null, + "line": 0, + "title": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchroniz", + "severity": "high", + "category": "correctness", + "crb_category": "performance", + "description": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "category-correction", + "why": "TotalDocs iterates the map without the cache mutex while BuildIndex writes it; a map read/write panic is correctness, not efficiency. The original candidate already reports this root cause.", + "evidence": [ + "https://github.com/grafana/grafana/blob/26fed312840cd76b766cbd2158e17a7e6c0ec548/pkg/storage/unified/search/bleve.go#L134", + "https://github.com/grafana/grafana/compare/871af07203177ce59a24e742107119409274f48a...26fed312840cd76b766cbd2158e17a7e6c0ec548" + ] + }, + "imported_category": "efficiency", + "tier": "correctness" + }, + "candidate": { + "file": "pkg/storage/unified/resource/search.go", + "line": 216, + "title": "Unsynchronized read of the bleve index cache", + "severity": "P1", + "category": "bugs", + "description": "TotalDocs() ranges over b.cache without taking cacheMu.RLock(), while BuildIndex writes under lock. Concurrent map read and write can cause fatal error, killing process. Also reachable from BleveIndexMetrics.Collect on every /metrics scrape. Fix: guard the reader with RLock/RUnlock." + } + } + ], + "missed_correctness_gold": [], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "pkg/server/module_server_test.go", + "line": 36, + "title": "Postgres skip papers over a startup-latency regression this PR introduces", + "severity": "P2", + "category": "history; conventions", + "description": "NewResourceServer runs s.Init(ctx) synchronously, executing DB work before module reports running. Test has fixed 500ms sleep before checking /metrics, but on postgres in Drone this exceeds 500ms. Skip was disabled rather than fixed. Fix: replace fixed sleep with poll using require.Eventually." + } + ], + "metrics": { + "gold_count": 2, + "gold_correctness_count": 1, + "candidate_count": 2, + "matched_count": 1, + "matched_correctness_count": 1, + "gold_recall_correctness": 100, + "precision_proxy_pct": 50 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/gold-labels.json new file mode 100644 index 0000000..bd53ac8 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/gold-labels.json @@ -0,0 +1,42 @@ +[ + { + "file": "pkg/storage/unified/search/bleve.go", + "line": 0, + "title": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same ke", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment. ", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "efficiency", + "why": "The label describes redundant expensive builds, without a new failure scenario. getOrCreateIndex performs its cache check outside BuildIndex in both versions, so the old function-wide lock did not recheck the cache or prevent queued duplicate builds. Do not count this performance-only concern as a correctness miss.", + "evidence": [ + "https://github.com/grafana/grafana/blob/26fed312840cd76b766cbd2158e17a7e6c0ec548/pkg/storage/unified/search/bleve.go#L93", + "https://github.com/grafana/grafana/compare/871af07203177ce59a24e742107119409274f48a...26fed312840cd76b766cbd2158e17a7e6c0ec548" + ] + }, + "policy_excluded": true + }, + { + "file": null, + "line": 0, + "title": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchroniz", + "severity": "high", + "category": "correctness", + "crb_category": "performance", + "description": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "category-correction", + "why": "TotalDocs iterates the map without the cache mutex while BuildIndex writes it; a map read/write panic is correctness, not efficiency. The original candidate already reports this root cause.", + "evidence": [ + "https://github.com/grafana/grafana/blob/26fed312840cd76b766cbd2158e17a7e6c0ec548/pkg/storage/unified/search/bleve.go#L134", + "https://github.com/grafana/grafana/compare/871af07203177ce59a24e742107119409274f48a...26fed312840cd76b766cbd2158e17a7e6c0ec548" + ] + }, + "imported_category": "efficiency" + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/match-gc.json new file mode 100644 index 0000000..d8da683 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/match-gc.json @@ -0,0 +1,6 @@ +[ + { + "a_idx": 1, + "b_idx": 0 + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/meta.json new file mode 100644 index 0000000..c09753f --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-grafana-97529/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.224", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-grafana-97529", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/candidate.json new file mode 100644 index 0000000..d3d87cd --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/candidate.json @@ -0,0 +1,34 @@ +[ + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 237, + "title": "A permission scoped to a single client grants realm-wide client management", + "severity": "P0", + "category": "bugs", + "description": "hasPermission(String scope) doesn't check that the granting policy is bound to the all-clients resource, allowing a single-client permission to grant realm-wide access through DefaultPolicyEvaluator's resource-type-wide evaluation." + }, + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 262, + "title": "Dead code and unused imports in the new ClientPermissionsV2", + "severity": "P2", + "category": "slop", + "description": "getEvaluationContext() method is private and never called; logger field is unused; six imports (Scope, TOKEN_EXCHANGE, ClientModelIdentity, DefaultEvaluationContext, EvaluationContext) exist only for the dead method." + }, + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java", + "line": 96, + "title": "New javadoc describes two void methods as returning a boolean", + "severity": "P2", + "category": "comments", + "description": "requireView() and requireViewClientScopes() javadoc incorrectly claim they return boolean when they are void methods that throw ForbiddenException, misleading callers about the return type." + }, + { + "file": "server-spi-private/src/main/java/org/keycloak/authorization/AdminPermissionsSchema.java", + "line": 304, + "title": "Clients resources have no display name in the permission UI", + "severity": "P2", + "category": "conventions", + "description": "getResourceName() only handles USERS type; CLIENTS type falls through to resource.getDisplayName() which is never set, resulting in blank display names and raw UUIDs in the admin console instead of client IDs." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-judge.json new file mode 100644 index 0000000..2bf2e79 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-judge.json @@ -0,0 +1,113 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [ + { + "file": null, + "line": 0, + "title": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleanin", + "severity": "high", + "category": "security", + "crb_category": "security", + "description": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "feature-flag", + "why": "The newly added V1 feature guard prevents cleanup listener execution when only V2 is enabled, even though V2 creates per-client authorization resources.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/AdminPermissions.java#L77", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + ] + }, + "tier": "correctness" + }, + { + "file": null, + "line": 0, + "title": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.g", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "The per-client lookup supplies the resource-server ID as owner, but getOrCreateResource creates the resource with the server client ID as owner. Client-specific permissions can be missed.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L214", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + ] + }, + "tier": "correctness" + }, + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 0, + "title": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RES", + "severity": "high", + "category": "robustness", + "crb_category": "error-handling", + "description": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "getClientsWithPermission enumerates typed resources and returns their names. Individually created client resources have no type; the type-level Clients resource name is not a client ID.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L138", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + ] + }, + "tier": "correctness" + } + ], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 237, + "title": "A permission scoped to a single client grants realm-wide client management", + "severity": "P0", + "category": "bugs", + "description": "hasPermission(String scope) doesn't check that the granting policy is bound to the all-clients resource, allowing a single-client permission to grant realm-wide access through DefaultPolicyEvaluator's resource-type-wide evaluation." + }, + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 262, + "title": "Dead code and unused imports in the new ClientPermissionsV2", + "severity": "P2", + "category": "slop", + "description": "getEvaluationContext() method is private and never called; logger field is unused; six imports (Scope, TOKEN_EXCHANGE, ClientModelIdentity, DefaultEvaluationContext, EvaluationContext) exist only for the dead method." + }, + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java", + "line": 96, + "title": "New javadoc describes two void methods as returning a boolean", + "severity": "P2", + "category": "comments", + "description": "requireView() and requireViewClientScopes() javadoc incorrectly claim they return boolean when they are void methods that throw ForbiddenException, misleading callers about the return type." + }, + { + "file": "server-spi-private/src/main/java/org/keycloak/authorization/AdminPermissionsSchema.java", + "line": 304, + "title": "Clients resources have no display name in the permission UI", + "severity": "P2", + "category": "conventions", + "description": "getResourceName() only handles USERS type; CLIENTS type falls through to resource.getDisplayName() which is never set, resulting in blank display names and raw UUIDs in the admin console instead of client IDs." + } + ], + "metrics": { + "gold_count": 3, + "gold_correctness_count": 3, + "candidate_count": 4, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-labels.json new file mode 100644 index 0000000..ee3edaa --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-labels.json @@ -0,0 +1,59 @@ +[ + { + "file": null, + "line": 0, + "title": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleanin", + "severity": "high", + "category": "security", + "crb_category": "security", + "description": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "feature-flag", + "why": "The newly added V1 feature guard prevents cleanup listener execution when only V2 is enabled, even though V2 creates per-client authorization resources.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/AdminPermissions.java#L77", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + ] + } + }, + { + "file": null, + "line": 0, + "title": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.g", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "The per-client lookup supplies the resource-server ID as owner, but getOrCreateResource creates the resource with the server client ID as owner. Client-specific permissions can be missed.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L214", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + ] + } + }, + { + "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 0, + "title": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RES", + "severity": "high", + "category": "robustness", + "crb_category": "error-handling", + "description": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "getClientsWithPermission enumerates typed resources and returns their names. Individually created client resources have no type; the type-level Clients resource name is not a client ID.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L138", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + ] + } + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/meta.json new file mode 100644 index 0000000..840dda0 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.225", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-keycloak-36880", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/candidate.json new file mode 100644 index 0000000..aef3aef --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/candidate.json @@ -0,0 +1,18 @@ +[ + { + "file": "docs/guides/operator/advanced-configuration.adoc", + "line": 448, + "title": "Operator side of the preview feature is never gated — docs claim a failure that no code produces", + "severity": "P1", + "category": "bugs", + "description": "CAUTION documents that rolling-updates must be enabled in the operator, but no operator code reads the flag. UpgradeLogicFactory only checks spec.update.strategy, not the feature flag. The documented remedy (setting features.enabled) cannot work because it only affects server container args, not the operator's Profile. CLI commands are gated but operator path remains ungated." + }, + { + "file": "docs/guides/templates/kc.adoc", + "line": 53, + "title": "Documented --features=rolling-updates on update-compatibility is ignored on distributions built with --features", + "severity": "P2", + "category": "bugs, conventions", + "description": "The --features flag is build-time; persisted build values override CLI values. Pre-built distributions with explicit features ignore the documented --features=rolling-updates flag and require a rebuild instead. The flag also replaces rather than extends existing features, contradicting the guide's own warning to include all configuration options. CI doesn't catch this because tests run against unbuilt distributions." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/gold-judge.json new file mode 100644 index 0000000..5c12a49 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/gold-judge.json @@ -0,0 +1,58 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": null, + "line": 0, + "title": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "intentional", + "why": "The compatibility CLI already uses picocli.exit(result.exitCode()); the new feature-disabled branch follows the same CLI exit convention and returns immediately. No embedding caller or broken contract is identified by the truncated label.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/62bf98939eda58f0d0a1a41a58aa6d4ecdc24dba/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/UpdateCompatibilityCheck.java#L45", + "https://github.com/keycloak/keycloak/compare/8671f860469d6d95e54df3e13f77ebc401249489...62bf98939eda58f0d0a1a41a58aa6d4ecdc24dba" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "docs/guides/operator/advanced-configuration.adoc", + "line": 448, + "title": "Operator side of the preview feature is never gated — docs claim a failure that no code produces", + "severity": "P1", + "category": "bugs", + "description": "CAUTION documents that rolling-updates must be enabled in the operator, but no operator code reads the flag. UpgradeLogicFactory only checks spec.update.strategy, not the feature flag. The documented remedy (setting features.enabled) cannot work because it only affects server container args, not the operator's Profile. CLI commands are gated but operator path remains ungated." + }, + { + "file": "docs/guides/templates/kc.adoc", + "line": 53, + "title": "Documented --features=rolling-updates on update-compatibility is ignored on distributions built with --features", + "severity": "P2", + "category": "bugs, conventions", + "description": "The --features flag is build-time; persisted build values override CLI values. Pre-built distributions with explicit features ignore the documented --features=rolling-updates flag and require a rebuild instead. The flag also replaces rather than extends existing features, contradicting the guide's own warning to include all configuration options. CI doesn't catch this because tests run against unbuilt distributions." + } + ], + "metrics": { + "gold_count": 1, + "gold_correctness_count": 0, + "candidate_count": 2, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/gold-labels.json new file mode 100644 index 0000000..5b09e21 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/gold-labels.json @@ -0,0 +1,22 @@ +[ + { + "file": null, + "line": 0, + "title": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "intentional", + "why": "The compatibility CLI already uses picocli.exit(result.exitCode()); the new feature-disabled branch follows the same CLI exit convention and returns immediately. No embedding caller or broken contract is identified by the truncated label.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/62bf98939eda58f0d0a1a41a58aa6d4ecdc24dba/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/UpdateCompatibilityCheck.java#L45", + "https://github.com/keycloak/keycloak/compare/8671f860469d6d95e54df3e13f77ebc401249489...62bf98939eda58f0d0a1a41a58aa6d4ecdc24dba" + ] + }, + "policy_excluded": true + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/meta.json new file mode 100644 index 0000000..486d421 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36882/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.225", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-keycloak-36882", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/candidate.json new file mode 100644 index 0000000..718e56e --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/candidate.json @@ -0,0 +1,18 @@ +[ + { + "file": "server-spi-private/src/main/java/org/keycloak/utils/CredentialHelper.java", + "line": 115, + "title": "Recovery codes written to user storage lose one-time-use, hashing, and metadata", + "severity": "P0", + "category": "bugs", + "description": "createRecoveryCodesCredential flattens RecoveryAuthnCodesCredentialModel into a bare UserCredentialModel, dropping hashing, per-code ordinal, remaining-count, and metadata. Storage boundary discards the model; provider update logic never runs, so used codes are never consumed and any leaked code becomes a permanent second-factor bypass." + }, + { + "file": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/federation/storage/BackwardsCompatibilityUserStorageTest.java", + "line": 123, + "title": "Test helpers copied verbatim from RecoveryAuthnCodesAuthenticatorTest, typo and dead config included", + "severity": "P2", + "category": "slop", + "description": "configureBrowserFlowWithRecoveryAuthnCodes and enterRecoveryCodes are duplicated from shared test, carrying over a 'delayed-suthenticator-config' typo and unused delayed-authenticator execution (delay=0). Dead code dependency causes unrelated failures if provider changes; copies drift independently." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-judge.json new file mode 100644 index 0000000..84e6934 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-judge.json @@ -0,0 +1,79 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": null, + "line": 0, + "title": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by Recover", + "severity": "medium", + "category": "security", + "crb_category": "security", + "description": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential(user) without checking isPresent() can lead to a NoSuchElementException if the Optional is empty.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The login bean previously called findFirst().get() without a guard. This patch replaces the lookup with one that also checks federated credentials. The raw-list type-safety concern is compiler/linter territory and does not establish the claimed new absent-credential failure.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/services/src/main/java/org/keycloak/forms/login/freemarker/model/RecoveryAuthnCodeInputLoginBean.java#L34", + "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [ + { + "file": null, + "line": 0, + "title": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.r", + "severity": "low", + "category": "security", + "crb_category": "security", + "description": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.recoveryCodes.getId()); otherwise getId() will be null and downstream removal by id (e.g., removeStoredCredentialById in the authenticator flow) may not work.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "The new user-storage getCredentials path creates a recovery model with createFromValues but never transfers the stored ID. createFromValues does not set an ID, while the authenticator later passes getId to removal.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java#L239", + "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a" + ] + }, + "tier": "correctness" + } + ], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "server-spi-private/src/main/java/org/keycloak/utils/CredentialHelper.java", + "line": 115, + "title": "Recovery codes written to user storage lose one-time-use, hashing, and metadata", + "severity": "P0", + "category": "bugs", + "description": "createRecoveryCodesCredential flattens RecoveryAuthnCodesCredentialModel into a bare UserCredentialModel, dropping hashing, per-code ordinal, remaining-count, and metadata. Storage boundary discards the model; provider update logic never runs, so used codes are never consumed and any leaked code becomes a permanent second-factor bypass." + }, + { + "file": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/federation/storage/BackwardsCompatibilityUserStorageTest.java", + "line": 123, + "title": "Test helpers copied verbatim from RecoveryAuthnCodesAuthenticatorTest, typo and dead config included", + "severity": "P2", + "category": "slop", + "description": "configureBrowserFlowWithRecoveryAuthnCodes and enterRecoveryCodes are duplicated from shared test, carrying over a 'delayed-suthenticator-config' typo and unused delayed-authenticator execution (delay=0). Dead code dependency causes unrelated failures if provider changes; copies drift independently." + } + ], + "metrics": { + "gold_count": 2, + "gold_correctness_count": 1, + "candidate_count": 2, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-labels.json new file mode 100644 index 0000000..b679849 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-labels.json @@ -0,0 +1,41 @@ +[ + { + "file": null, + "line": 0, + "title": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by Recover", + "severity": "medium", + "category": "security", + "crb_category": "security", + "description": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential(user) without checking isPresent() can lead to a NoSuchElementException if the Optional is empty.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The login bean previously called findFirst().get() without a guard. This patch replaces the lookup with one that also checks federated credentials. The raw-list type-safety concern is compiler/linter territory and does not establish the claimed new absent-credential failure.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/services/src/main/java/org/keycloak/forms/login/freemarker/model/RecoveryAuthnCodeInputLoginBean.java#L34", + "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a" + ] + }, + "policy_excluded": true + }, + { + "file": null, + "line": 0, + "title": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.r", + "severity": "low", + "category": "security", + "crb_category": "security", + "description": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.recoveryCodes.getId()); otherwise getId() will be null and downstream removal by id (e.g., removeStoredCredentialById in the authenticator flow) may not work.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "The new user-storage getCredentials path creates a recovery model with createFromValues but never transfers the stored ID. createFromValues does not set an ID, while the authenticator later passes getId to removal.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java#L239", + "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a" + ] + } + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/meta.json new file mode 100644 index 0000000..f26214a --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.225", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-keycloak-38446", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/candidate.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/candidate.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/gold-judge.json new file mode 100644 index 0000000..19a9b2c --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/gold-judge.json @@ -0,0 +1,62 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": null, + "line": 0, + "title": "The function modifies the config variable to include display values but then returns the original monitor.config instead", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "The function modifies the config variable to include display values but then returns the original monitor.config instead of the modified version.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The copy-and-return-original-config bug exists verbatim in base mark_failed.py and is moved into incident_occurrence.py. It is a real latent bug but not introduced by this PR.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9/src/sentry/monitors/logic/mark_failed.py#L327", + "https://github.com/getsentry/sentry/compare/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9...dcdcadb771128e79259cc9eff9c70c38fc597976" + ] + }, + "policy_excluded": true, + "tier": "excluded" + }, + { + "file": null, + "line": 0, + "title": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates a", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates an unnecessary database query.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The MonitorCheckIn query already appears in base mark_failed_threshold and moves unchanged to try_incident_threshold. It also retrieves full model instances where previous_checkins has only ID/date/status; the label supplies no new unnecessary-query regression.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9/src/sentry/monitors/logic/mark_failed.py#L178", + "https://github.com/getsentry/sentry/compare/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9...dcdcadb771128e79259cc9eff9c70c38fc597976" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [], + "metrics": { + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 0, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": null + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/gold-labels.json new file mode 100644 index 0000000..5f2c9f1 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/gold-labels.json @@ -0,0 +1,42 @@ +[ + { + "file": null, + "line": 0, + "title": "The function modifies the config variable to include display values but then returns the original monitor.config instead", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "The function modifies the config variable to include display values but then returns the original monitor.config instead of the modified version.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The copy-and-return-original-config bug exists verbatim in base mark_failed.py and is moved into incident_occurrence.py. It is a real latent bug but not introduced by this PR.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9/src/sentry/monitors/logic/mark_failed.py#L327", + "https://github.com/getsentry/sentry/compare/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9...dcdcadb771128e79259cc9eff9c70c38fc597976" + ] + }, + "policy_excluded": true + }, + { + "file": null, + "line": 0, + "title": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates a", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates an unnecessary database query.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "The MonitorCheckIn query already appears in base mark_failed_threshold and moves unchanged to try_incident_threshold. It also retrieves full model instances where previous_checkins has only ID/date/status; the label supplies no new unnecessary-query regression.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9/src/sentry/monitors/logic/mark_failed.py#L178", + "https://github.com/getsentry/sentry/compare/0cfc28e76ddc986d2d89dd9b9f63ee916a18a5f9...dcdcadb771128e79259cc9eff9c70c38fc597976" + ] + }, + "policy_excluded": true + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/meta.json new file mode 100644 index 0000000..4333eed --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-80528/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.225", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-sentry-80528", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/candidate.json new file mode 100644 index 0000000..8817096 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/candidate.json @@ -0,0 +1,34 @@ +[ + { + "file": "src/sentry/remote_subscriptions/consumers/queue_consumer.py", + "line": 338, + "title": "Strategy `close()` destroys the factory-owned queue pool — consumer silently drops every message after a rebalance", + "severity": "P0", + "category": "bugs", + "description": "FixedQueuePool is created once and reused for process lifetime, but SimpleQueueProcessingStrategy.close(), terminate(), and join() all call queue_pool.shutdown(). After first rebalance, pool's worker threads exit and queues shutdown. New strategy passes, then every submit() hits queue.ShutDown, which is caught and silently commits offsets, causing permanent data loss." + }, + { + "file": "src/sentry/remote_subscriptions/consumers/queue_consumer.py", + "line": 253, + "title": "\"Natural backpressure\" is claimed but the queues are unbounded and `submit()` never rejects", + "severity": "P1", + "category": "bugs", + "description": "Class docstring guarantees 'Natural backpressure when queues fill up', but queue.Queue() has no maxsize, submit() never raises MessageRejected, and poll() only emits a gauge. On restart against large backlog or stalled processor, queue grows without bound, pinning full Message objects, until OOM." + }, + { + "file": "src/sentry/remote_subscriptions/consumers/queue_consumer.py", + "line": 231, + "title": "`shutdown()` sets the worker stop flag before `q.shutdown(immediate=False)`, so queued items are abandoned", + "severity": "P2", + "category": "bugs", + "description": "shutdown() sets worker.shutdown = True before calling q.shutdown(immediate=False). Each worker's loop re-checks 'while not self.shutdown' after finishing current item and exits immediately, so with queued items, most are dropped. The immediate=False argument is dead." + }, + { + "file": "src/sentry/remote_subscriptions/consumers/queue_consumer.py", + "line": 31, + "title": "Unused `WorkItem.message` field and unused `wait_until_empty` helper", + "severity": "P2", + "category": "slop", + "description": "WorkItem.message is documented for offset tracking but is written and never read. It keeps decoded KafkaPayload alive for entire time item sits in queue, doubling queue memory and worsening backpressure issue. FixedQueuePool.wait_until_empty is likewise never called." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/gold-judge.json new file mode 100644 index 0000000..e05183b --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/gold-judge.json @@ -0,0 +1,116 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": null, + "line": 0, + "title": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This cou", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This could cause AttributeError at runtime. Verify the correct API or implement a custom shutdown mechanism.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-version", + "why": "The pinned .python-version is 3.13.1. Queue.shutdown(immediate=False) and queue.ShutDown are documented Python 3.13 APIs.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/9966ec5a13e331659c3ea00981f9b11b0faf821f/.python-version#L1", + "https://github.com/getsentry/sentry/compare/fd358e8a388939959369f08f616e29552cdaf96e...9966ec5a13e331659c3ea00981f9b11b0faf821f", + "https://docs.python.org/3.13/library/queue.html#queue.Queue.shutdown" + ] + }, + "tier": "excluded" + }, + { + "file": "tests/sentry/uptime/consumers/test_results_consumer.py", + "line": 0, + "title": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant t", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant to improve maintainability.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "test-style", + "why": "Repeated polling bounds in test scaffolding are cleanup, not correctness; no written rule requiring a shared test constant is supplied.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/9966ec5a13e331659c3ea00981f9b11b0faf821f/tests/sentry/uptime/consumers/test_results_consumer.py#L2009", + "https://github.com/getsentry/sentry/compare/fd358e8a388939959369f08f616e29552cdaf96e...9966ec5a13e331659c3ea00981f9b11b0faf821f" + ] + }, + "policy_excluded": true, + "tier": "excluded" + }, + { + "file": "tests/sentry/uptime/consumers/test_results_consumer.py", + "line": 0, + "title": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "documentation", + "why": "The docstring promises commits for other messages while the test asserts no commit. This is a test-documentation cleanup item, with no runtime defect asserted; review excludes general documentation complaints without a written requirement.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/9966ec5a13e331659c3ea00981f9b11b0faf821f/tests/sentry/uptime/consumers/test_results_consumer.py#L2140", + "https://github.com/getsentry/sentry/compare/fd358e8a388939959369f08f616e29552cdaf96e...9966ec5a13e331659c3ea00981f9b11b0faf821f" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "src/sentry/remote_subscriptions/consumers/queue_consumer.py", + "line": 338, + "title": "Strategy `close()` destroys the factory-owned queue pool — consumer silently drops every message after a rebalance", + "severity": "P0", + "category": "bugs", + "description": "FixedQueuePool is created once and reused for process lifetime, but SimpleQueueProcessingStrategy.close(), terminate(), and join() all call queue_pool.shutdown(). After first rebalance, pool's worker threads exit and queues shutdown. New strategy passes, then every submit() hits queue.ShutDown, which is caught and silently commits offsets, causing permanent data loss." + }, + { + "file": "src/sentry/remote_subscriptions/consumers/queue_consumer.py", + "line": 253, + "title": "\"Natural backpressure\" is claimed but the queues are unbounded and `submit()` never rejects", + "severity": "P1", + "category": "bugs", + "description": "Class docstring guarantees 'Natural backpressure when queues fill up', but queue.Queue() has no maxsize, submit() never raises MessageRejected, and poll() only emits a gauge. On restart against large backlog or stalled processor, queue grows without bound, pinning full Message objects, until OOM." + }, + { + "file": "src/sentry/remote_subscriptions/consumers/queue_consumer.py", + "line": 231, + "title": "`shutdown()` sets the worker stop flag before `q.shutdown(immediate=False)`, so queued items are abandoned", + "severity": "P2", + "category": "bugs", + "description": "shutdown() sets worker.shutdown = True before calling q.shutdown(immediate=False). Each worker's loop re-checks 'while not self.shutdown' after finishing current item and exits immediately, so with queued items, most are dropped. The immediate=False argument is dead." + }, + { + "file": "src/sentry/remote_subscriptions/consumers/queue_consumer.py", + "line": 31, + "title": "Unused `WorkItem.message` field and unused `wait_until_empty` helper", + "severity": "P2", + "category": "slop", + "description": "WorkItem.message is documented for offset tracking but is written and never read. It keeps decoded KafkaPayload alive for entire time item sits in queue, doubling queue memory and worsening backpressure issue. FixedQueuePool.wait_until_empty is likewise never called." + } + ], + "metrics": { + "gold_count": 3, + "gold_correctness_count": 0, + "candidate_count": 4, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/gold-labels.json new file mode 100644 index 0000000..f32bab5 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/gold-labels.json @@ -0,0 +1,62 @@ +[ + { + "file": null, + "line": 0, + "title": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This cou", + "severity": "high", + "category": "correctness", + "crb_category": "bug", + "description": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This could cause AttributeError at runtime. Verify the correct API or implement a custom shutdown mechanism.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-version", + "why": "The pinned .python-version is 3.13.1. Queue.shutdown(immediate=False) and queue.ShutDown are documented Python 3.13 APIs.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/9966ec5a13e331659c3ea00981f9b11b0faf821f/.python-version#L1", + "https://github.com/getsentry/sentry/compare/fd358e8a388939959369f08f616e29552cdaf96e...9966ec5a13e331659c3ea00981f9b11b0faf821f", + "https://docs.python.org/3.13/library/queue.html#queue.Queue.shutdown" + ] + } + }, + { + "file": "tests/sentry/uptime/consumers/test_results_consumer.py", + "line": 0, + "title": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant t", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant to improve maintainability.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "test-style", + "why": "Repeated polling bounds in test scaffolding are cleanup, not correctness; no written rule requiring a shared test constant is supplied.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/9966ec5a13e331659c3ea00981f9b11b0faf821f/tests/sentry/uptime/consumers/test_results_consumer.py#L2009", + "https://github.com/getsentry/sentry/compare/fd358e8a388939959369f08f616e29552cdaf96e...9966ec5a13e331659c3ea00981f9b11b0faf821f" + ] + }, + "policy_excluded": true + }, + { + "file": "tests/sentry/uptime/consumers/test_results_consumer.py", + "line": 0, + "title": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "documentation", + "why": "The docstring promises commits for other messages while the test asserts no commit. This is a test-documentation cleanup item, with no runtime defect asserted; review excludes general documentation complaints without a written requirement.", + "evidence": [ + "https://github.com/getsentry/sentry/blob/9966ec5a13e331659c3ea00981f9b11b0faf821f/tests/sentry/uptime/consumers/test_results_consumer.py#L2140", + "https://github.com/getsentry/sentry/compare/fd358e8a388939959369f08f616e29552cdaf96e...9966ec5a13e331659c3ea00981f9b11b0faf821f" + ] + }, + "policy_excluded": true + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/meta.json new file mode 100644 index 0000000..7dde835 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-95633/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.225", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-sentry-95633", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/candidate.json new file mode 100644 index 0000000..09ac52f --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/candidate.json @@ -0,0 +1,42 @@ +[ + { + "file": "src/sentry/search/events/datasets/discover.py", + "line": 1048, + "title": "`upsampled_count()` drops the `ifNull` default, silently reverting defensive logic", + "severity": "P0", + "category": "bugs", + "description": "commit `6ad6fe38` changes `sum(ifNull(sample_weight, 1))` → `sum(sample_weight)`, silently reverting the defensive default. Events without `sample_weight` count as zero instead of one, causing projects to report count() == 0 for all historical error volume." + }, + { + "file": "sentry-repo", + "line": 1, + "title": "Stray `sentry-repo` gitlink committed with no `.gitmodules`", + "severity": "P1", + "category": "bugs", + "description": "`6ad6fe38` adds mode 160000 entry `sentry-repo` without `.gitmodules`, causing every fresh clone to get an empty directory and `git submodule update --init` to fail." + }, + { + "file": "src/sentry/api/helpers/error_upsampling.py", + "line": 27, + "title": "60s eligibility cache is slower than original and cannot be invalidated", + "severity": "P1", + "category": "bugs", + "description": "Cache swaps local dict read for network cache round-trip on every request, making it slower. Functionally non-revertible: when projects are removed from allowlist, endpoint keeps returning upsampled counts for up to 60s per distinct project-subset." + }, + { + "file": "src/sentry/api/helpers/error_upsampling.py", + "line": 137, + "title": "Negated `!event.type:error` queries misclassified as error-focused", + "severity": "P1", + "category": "bugs", + "description": "`_is_error_focused_query` does substring test, so `!event.type:error` (negation) is incorrectly classified as error-focused, causing non-error rows without `sample_weight` to report count as 0." + }, + { + "file": "src/sentry/api/endpoints/organization_events_stats.py", + "line": 218, + "title": "Endpoint refactor triples line count with no behavior change", + "severity": "P2", + "category": "slop", + "description": "Replaces two lines with an inert alias and three identical copies of the same `if`/transform in each return branch, creating three places to keep in sync." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/gold-judge.json new file mode 100644 index 0000000..2484187 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/gold-judge.json @@ -0,0 +1,127 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": null, + "line": 0, + "title": "Using Python’s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys w", + "severity": "low", + "category": "efficiency", + "crb_category": "performance", + "description": "Using Python’s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys won’t match across workers and invalidate_upsampling_cache may fail to delete them. Use a deterministic serialization of project_ids for the cache key.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-premise", + "why": "The hashed value is a sorted tuple of integer project IDs, not strings. Python hash randomization does not randomize integer tuple hashes across processes; both creation and invalidation use the same expression.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/6ad6fe3899f630ccf922ce7ac681147dc7987435/src/sentry/api/helpers/error_upsampling.py#L27", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/a5d290951def84afdcc4c88d2f1f20023fc36e2a...6ad6fe3899f630ccf922ce7ac681147dc7987435" + ] + }, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [ + { + "a_idx": 1, + "b_idx": 2 + } + ], + "matched": [], + "missed_correctness_gold": [ + { + "file": null, + "line": 0, + "title": "sample_rate = 0.0 is falsy and skipped", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "sample_rate = 0.0 is falsy and skipped", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "falsy-zero", + "why": "The new factory helper uses if client_sample_rate, so a valid 0.0 is not copied into normalized_data. This changes the fixture data rather than merely omitting a test.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/6ad6fe3899f630ccf922ce7ac681147dc7987435/src/sentry/testutils/factories.py#L345", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/a5d290951def84afdcc4c88d2f1f20023fc36e2a...6ad6fe3899f630ccf922ce7ac681147dc7987435" + ] + }, + "tier": "correctness" + }, + { + "file": "src/sentry/api/endpoints/organization_events_stats.py", + "line": 0, + "title": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths where the query ultimately runs against discover (e.g., dashboard split) while the original dataset is metrics, upsampling may be skipped even when all projects are allowlisted.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "The new eligibility call passes the outer dataset even when _get_event_stats receives a different scoped_dataset for a discover dashboard split. The helper gates transformations on that dataset argument.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/6ad6fe3899f630ccf922ce7ac681147dc7987435/src/sentry/api/endpoints/organization_events_stats.py#L220", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/a5d290951def84afdcc4c88d2f1f20023fc36e2a...6ad6fe3899f630ccf922ce7ac681147dc7987435" + ] + }, + "tier": "correctness" + } + ], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "src/sentry/search/events/datasets/discover.py", + "line": 1048, + "title": "`upsampled_count()` drops the `ifNull` default, silently reverting defensive logic", + "severity": "P0", + "category": "bugs", + "description": "commit `6ad6fe38` changes `sum(ifNull(sample_weight, 1))` → `sum(sample_weight)`, silently reverting the defensive default. Events without `sample_weight` count as zero instead of one, causing projects to report count() == 0 for all historical error volume." + }, + { + "file": "sentry-repo", + "line": 1, + "title": "Stray `sentry-repo` gitlink committed with no `.gitmodules`", + "severity": "P1", + "category": "bugs", + "description": "`6ad6fe38` adds mode 160000 entry `sentry-repo` without `.gitmodules`, causing every fresh clone to get an empty directory and `git submodule update --init` to fail." + }, + { + "file": "src/sentry/api/helpers/error_upsampling.py", + "line": 27, + "title": "60s eligibility cache is slower than original and cannot be invalidated", + "severity": "P1", + "category": "bugs", + "description": "Cache swaps local dict read for network cache round-trip on every request, making it slower. Functionally non-revertible: when projects are removed from allowlist, endpoint keeps returning upsampled counts for up to 60s per distinct project-subset." + }, + { + "file": "src/sentry/api/helpers/error_upsampling.py", + "line": 137, + "title": "Negated `!event.type:error` queries misclassified as error-focused", + "severity": "P1", + "category": "bugs", + "description": "`_is_error_focused_query` does substring test, so `!event.type:error` (negation) is incorrectly classified as error-focused, causing non-error rows without `sample_weight` to report count as 0." + }, + { + "file": "src/sentry/api/endpoints/organization_events_stats.py", + "line": 218, + "title": "Endpoint refactor triples line count with no behavior change", + "severity": "P2", + "category": "slop", + "description": "Replaces two lines with an inert alias and three identical copies of the same `if`/transform in each return branch, creating three places to keep in sync." + } + ], + "metrics": { + "gold_count": 3, + "gold_correctness_count": 2, + "candidate_count": 5, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/gold-labels.json new file mode 100644 index 0000000..e682cbb --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/gold-labels.json @@ -0,0 +1,59 @@ +[ + { + "file": null, + "line": 0, + "title": "sample_rate = 0.0 is falsy and skipped", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "sample_rate = 0.0 is falsy and skipped", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "falsy-zero", + "why": "The new factory helper uses if client_sample_rate, so a valid 0.0 is not copied into normalized_data. This changes the fixture data rather than merely omitting a test.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/6ad6fe3899f630ccf922ce7ac681147dc7987435/src/sentry/testutils/factories.py#L345", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/a5d290951def84afdcc4c88d2f1f20023fc36e2a...6ad6fe3899f630ccf922ce7ac681147dc7987435" + ] + } + }, + { + "file": null, + "line": 0, + "title": "Using Python’s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys w", + "severity": "low", + "category": "efficiency", + "crb_category": "performance", + "description": "Using Python’s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys won’t match across workers and invalidate_upsampling_cache may fail to delete them. Use a deterministic serialization of project_ids for the cache key.", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "wrong-premise", + "why": "The hashed value is a sorted tuple of integer project IDs, not strings. Python hash randomization does not randomize integer tuple hashes across processes; both creation and invalidation use the same expression.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/6ad6fe3899f630ccf922ce7ac681147dc7987435/src/sentry/api/helpers/error_upsampling.py#L27", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/a5d290951def84afdcc4c88d2f1f20023fc36e2a...6ad6fe3899f630ccf922ce7ac681147dc7987435" + ] + } + }, + { + "file": "src/sentry/api/endpoints/organization_events_stats.py", + "line": 0, + "title": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths where the query ultimately runs against discover (e.g., dashboard split) while the original dataset is metrics, upsampling may be skipped even when all projects are allowlisted.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "The new eligibility call passes the outer dataset even when _get_event_stats receives a different scoped_dataset for a discover dashboard split. The helper gates transformations on that dataset argument.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/6ad6fe3899f630ccf922ce7ac681147dc7987435/src/sentry/api/endpoints/organization_events_stats.py#L220", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/a5d290951def84afdcc4c88d2f1f20023fc36e2a...6ad6fe3899f630ccf922ce7ac681147dc7987435" + ] + } + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/match-gc.json new file mode 100644 index 0000000..809063c --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/match-gc.json @@ -0,0 +1,6 @@ +[ + { + "a_idx": 1, + "b_idx": 2 + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/meta.json new file mode 100644 index 0000000..ab0926d --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-3/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.225", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-sentry-greptile-3", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/candidate.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/candidate.json new file mode 100644 index 0000000..1572a70 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/candidate.json @@ -0,0 +1,26 @@ +[ + { + "file": "src/sentry/hybridcloud/tasks/deliver_webhooks.py", + "line": 239, + "title": "drain_mailbox_parallel processing deadline (120s) shorter than runtime budget (180s)", + "severity": "P1", + "category": "bugs", + "description": "drain_mailbox_parallel has a 120s processing deadline but budgets 180s for itself (BATCH_SCHEDULE_OFFSET). Taskworker hard-kills at 120s before the graceful break runs at 180s, preventing delivery_deadline log/metric emission and aborting the in-flight batch mid-delivery. drain_mailbox in the same commit got 300s (with headroom), creating inconsistency." + }, + { + "file": "src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py", + "line": 153, + "title": "Replay summary interleaves error timestamps (epoch seconds) with rrweb timestamps (epoch ms)", + "severity": "P1", + "category": "bugs", + "description": "ErrorEvent timestamps are epoch seconds; rrweb timestamps are epoch milliseconds. Since epoch seconds (~1.75e9) < epoch ms (~1.75e12), the comparison guard always sorts all errors before all breadcrumbs, breaking chronological ordering. Seer receives mis-ordered logs and produces summaries with wrong causality." + }, + { + "file": "static/app/views/dashboards/widgetCard/chart.tsx", + "line": 165, + "title": "Dashboard table path renders hardcoded empty data, discarding result", + "severity": "P2", + "category": "bugs", + "description": "use-table-widget-visualization branch passes empty columns and hardcoded empty tableData instead of result.data/result.meta. Empty columns won't trigger fallback, so GridEditable renders zero columns. Any org with the flag on sees blank tables for every table widget. Flag isn't yet registered, but breaks when someone registers it." + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/gold-judge.json new file mode 100644 index 0000000..62cd796 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/gold-judge.json @@ -0,0 +1,107 @@ +{ + "mode": "gold", + "expected_clean": false, + "excluded_gold": [ + { + "file": null, + "line": 0, + "title": "Breaking changes in error response format", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "Breaking changes in error response format", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "underspecified", + "why": "The label names no endpoint, old/new format, affected caller, or compatibility requirement. The collector intentionally adds structured validation errors. A format change alone cannot serve as a reproducible correctness gold label.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9/src/sentry/issues/endpoints/browser_reporting_collector.py#L120", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/49a275847631e231672d01b5dcd66558c069d25b...ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9" + ] + }, + "tier": "excluded" + }, + { + "file": null, + "line": 0, + "title": "Detector validator uses wrong key when updating type", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "Detector validator uses wrong key when updating type", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "validated_data.get(\"detector_type\", instance.group_type) is unchanged context in the owner-field patch, not an introduced line or behavior.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9/src/sentry/workflow_engine/endpoints/validators/base/detector.py#L63", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/49a275847631e231672d01b5dcd66558c069d25b...ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [ + { + "file": null, + "line": 0, + "title": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not gu", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "NodeStorage.get_multi appends cached entries after uncached entries, so partial cache hits can reorder its values. Zipping those values with input error IDs assigns another event payload to an ID.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9/src/sentry/nodestore/base.py#L184", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/49a275847631e231672d01b5dcd66558c069d25b...ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9" + ] + }, + "tier": "correctness" + } + ], + "missed_cleanup_gold_informational": [], + "promotion_candidates": [ + { + "file": "src/sentry/hybridcloud/tasks/deliver_webhooks.py", + "line": 239, + "title": "drain_mailbox_parallel processing deadline (120s) shorter than runtime budget (180s)", + "severity": "P1", + "category": "bugs", + "description": "drain_mailbox_parallel has a 120s processing deadline but budgets 180s for itself (BATCH_SCHEDULE_OFFSET). Taskworker hard-kills at 120s before the graceful break runs at 180s, preventing delivery_deadline log/metric emission and aborting the in-flight batch mid-delivery. drain_mailbox in the same commit got 300s (with headroom), creating inconsistency." + }, + { + "file": "src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py", + "line": 153, + "title": "Replay summary interleaves error timestamps (epoch seconds) with rrweb timestamps (epoch ms)", + "severity": "P1", + "category": "bugs", + "description": "ErrorEvent timestamps are epoch seconds; rrweb timestamps are epoch milliseconds. Since epoch seconds (~1.75e9) < epoch ms (~1.75e12), the comparison guard always sorts all errors before all breadcrumbs, breaking chronological ordering. Seer receives mis-ordered logs and produces summaries with wrong causality." + }, + { + "file": "static/app/views/dashboards/widgetCard/chart.tsx", + "line": 165, + "title": "Dashboard table path renders hardcoded empty data, discarding result", + "severity": "P2", + "category": "bugs", + "description": "use-table-widget-visualization branch passes empty columns and hardcoded empty tableData instead of result.data/result.meta. Empty columns won't trigger fallback, so GridEditable renders zero columns. Any org with the flag on sees blank tables for every table widget. Flag isn't yet registered, but breaks when someone registers it." + } + ], + "metrics": { + "gold_count": 3, + "gold_correctness_count": 1, + "candidate_count": 3, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0 + } +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/gold-labels.json new file mode 100644 index 0000000..0728509 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/gold-labels.json @@ -0,0 +1,60 @@ +[ + { + "file": null, + "line": 0, + "title": "Breaking changes in error response format", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "Breaking changes in error response format", + "adjudication": { + "date": "2026-10-02", + "status": "label-wrong", + "reason": "underspecified", + "why": "The label names no endpoint, old/new format, affected caller, or compatibility requirement. The collector intentionally adds structured validation errors. A format change alone cannot serve as a reproducible correctness gold label.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9/src/sentry/issues/endpoints/browser_reporting_collector.py#L120", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/49a275847631e231672d01b5dcd66558c069d25b...ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9" + ] + } + }, + { + "file": null, + "line": 0, + "title": "Detector validator uses wrong key when updating type", + "severity": "medium", + "category": "correctness", + "crb_category": "bug", + "description": "Detector validator uses wrong key when updating type", + "adjudication": { + "date": "2026-10-02", + "status": "policy-excluded", + "reason": "pre-existing", + "why": "validated_data.get(\"detector_type\", instance.group_type) is unchanged context in the owner-field patch, not an introduced line or behavior.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9/src/sentry/workflow_engine/endpoints/validators/base/detector.py#L63", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/49a275847631e231672d01b5dcd66558c069d25b...ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9" + ] + }, + "policy_excluded": true + }, + { + "file": null, + "line": 0, + "title": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not gu", + "severity": "low", + "category": "correctness", + "crb_category": "bug", + "description": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "cross-file", + "why": "NodeStorage.get_multi appends cached entries after uncached entries, so partial cache hits can reorder its values. Zipping those values with input error IDs assigns another event payload to an ID.", + "evidence": [ + "https://github.com/ai-code-review-evaluation/sentry-greptile/blob/ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9/src/sentry/nodestore/base.py#L184", + "https://github.com/ai-code-review-evaluation/sentry-greptile/compare/49a275847631e231672d01b5dcd66558c069d25b...ea188e2d736fd6ed27c1dba8aae63b7268e2a7a9" + ] + } + } +] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/match-gc.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/match-gc.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/match-gc.json @@ -0,0 +1 @@ +[] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/meta.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/meta.json new file mode 100644 index 0000000..bed14ce --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-sentry-greptile-5/meta.json @@ -0,0 +1,10 @@ +{ + "cli_version": "2.1.225", + "models": [ + "claude-opus-5" + ], + "operation": "rejudge", + "source_artifacts": "eval/runs/2026-08-07-gold-sweep-v0/crb-sentry-greptile-5", + "adjudication_date": "2026-10-02", + "tier": "unrecorded" +} diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/summary.json b/eval/runs/2026-10-02-zero-recall-adjudication/summary.json new file mode 100644 index 0000000..fac01f6 --- /dev/null +++ b/eval/runs/2026-10-02-zero-recall-adjudication/summary.json @@ -0,0 +1,176 @@ +{ + "operation": "rejudge", + "source_run": "eval/runs/2026-08-07-gold-sweep-v0", + "candidate_rerun": false, + "matcher_rerun": false, + "cases": [ + { + "case": "crb-cal.com-14943", + "gold_count": 2, + "gold_correctness_count": 2, + "candidate_count": 2, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0, + "excluded_count": 0 + }, + { + "case": "crb-discourse-graphite-9", + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 3, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0, + "excluded_count": 2 + }, + { + "case": "crb-grafana-103633", + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 3, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0, + "excluded_count": 2 + }, + { + "case": "crb-grafana-107534", + "gold_count": 1, + "gold_correctness_count": 0, + "candidate_count": 1, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0, + "excluded_count": 1 + }, + { + "case": "crb-grafana-76186", + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 1, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0, + "excluded_count": 2 + }, + { + "case": "crb-grafana-79265", + "gold_count": 5, + "gold_correctness_count": 1, + "candidate_count": 4, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0, + "excluded_count": 4 + }, + { + "case": "crb-grafana-90939", + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 0, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": null, + "excluded_count": 2 + }, + { + "case": "crb-grafana-97529", + "gold_count": 2, + "gold_correctness_count": 1, + "candidate_count": 2, + "matched_count": 1, + "matched_correctness_count": 1, + "gold_recall_correctness": 100, + "precision_proxy_pct": 50, + "excluded_count": 1 + }, + { + "case": "crb-keycloak-36880", + "gold_count": 3, + "gold_correctness_count": 3, + "candidate_count": 4, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0, + "excluded_count": 0 + }, + { + "case": "crb-keycloak-36882", + "gold_count": 1, + "gold_correctness_count": 0, + "candidate_count": 2, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0, + "excluded_count": 1 + }, + { + "case": "crb-keycloak-38446", + "gold_count": 2, + "gold_correctness_count": 1, + "candidate_count": 2, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0, + "excluded_count": 1 + }, + { + "case": "crb-sentry-80528", + "gold_count": 2, + "gold_correctness_count": 0, + "candidate_count": 0, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": null, + "excluded_count": 2 + }, + { + "case": "crb-sentry-95633", + "gold_count": 3, + "gold_correctness_count": 0, + "candidate_count": 4, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": null, + "precision_proxy_pct": 0, + "excluded_count": 3 + }, + { + "case": "crb-sentry-greptile-3", + "gold_count": 3, + "gold_correctness_count": 2, + "candidate_count": 5, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0, + "excluded_count": 1 + }, + { + "case": "crb-sentry-greptile-5", + "gold_count": 3, + "gold_correctness_count": 1, + "candidate_count": 3, + "matched_count": 0, + "matched_correctness_count": 0, + "gold_recall_correctness": 0, + "precision_proxy_pct": 0, + "excluded_count": 2 + } + ], + "eligible_correctness_total": 11, + "matched_correctness_total": 1, + "excluded_total": 24 +} From 64b8723684dc1061a465baf1d6286530df6f42be Mon Sep 17 00:00:00 2001 From: Michael Hohlios Date: Fri, 2 Oct 2026 11:09:38 -0400 Subject: [PATCH 2/2] Correct Keycloak adjudications using reachable callers Signed-off-by: Michael Hohlios --- docs/evals.md | 6 +- eval/calibration/adjudication-2026-10-02.json | 10 +-- eval/corpus/labels/crb-keycloak-36880.json | 22 +++--- eval/corpus/labels/crb-keycloak-38446.json | 6 +- .../README.md | 15 ++-- .../crb-keycloak-36880/gold-judge.json | 71 ++++++++++--------- .../crb-keycloak-36880/gold-labels.json | 22 +++--- .../crb-keycloak-38446/gold-judge.json | 6 +- .../crb-keycloak-38446/gold-labels.json | 6 +- .../summary.json | 8 +-- 10 files changed, 98 insertions(+), 74 deletions(-) diff --git a/docs/evals.md b/docs/evals.md index b67b0c2..116be47 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -223,9 +223,9 @@ artifacts document case by case: ## Runs (audit tier) The first sweep's 15 zero-correctness-recall CRB cases were adjudicated on -2026-10-02: 35 labels became 11 eligible correctness defects, 16 -policy-excluded findings, and 8 wrong or insufficiently specified labels. -The unchanged historical candidate already matched one of the 11; its +2026-10-02: 35 labels became 9 eligible correctness defects, 17 +policy-excluded findings, and 9 wrong or insufficiently specified labels. +The unchanged historical candidate already matched one of the 9; its map-race label had been incorrectly tiered as efficiency. See the [rulings and historical re-judge](../eval/runs/2026-10-02-zero-recall-adjudication/README.md). This selected cluster is not a new full-corpus recall result. The new diff --git a/eval/calibration/adjudication-2026-10-02.json b/eval/calibration/adjudication-2026-10-02.json index b674dbd..6f781f0 100644 --- a/eval/calibration/adjudication-2026-10-02.json +++ b/eval/calibration/adjudication-2026-10-02.json @@ -103,11 +103,11 @@ }, { "finding_index": 1, - "status": "real-miss" + "status": "label-wrong" }, { "finding_index": 2, - "status": "real-miss" + "status": "policy-excluded" } ] }, @@ -233,8 +233,8 @@ } ], "counts": { - "policy-excluded": 16, - "label-wrong": 8, - "real-miss": 11 + "policy-excluded": 17, + "label-wrong": 9, + "real-miss": 9 } } diff --git a/eval/corpus/labels/crb-keycloak-36880.json b/eval/corpus/labels/crb-keycloak-36880.json index a9a36bc..0ffb20b 100644 --- a/eval/corpus/labels/crb-keycloak-36880.json +++ b/eval/corpus/labels/crb-keycloak-36880.json @@ -34,12 +34,13 @@ "description": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.", "adjudication": { "date": "2026-10-02", - "status": "real-miss", - "reason": "cross-file", - "why": "The per-client lookup supplies the resource-server ID as owner, but getOrCreateResource creates the resource with the server client ID as owner. Client-specific permissions can be missed.", + "status": "label-wrong", + "reason": "identifier-equivalence", + "why": "ResourceServerAdapter.getClientId returns getId. The owner used when creating resources and the owner used by this lookup are equal, so the asserted mismatch is false.", "evidence": [ "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L214", - "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55", + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/model/jpa/src/main/java/org/keycloak/authorization/jpa/store/ResourceServerAdapter.java#L91" ] } }, @@ -53,14 +54,17 @@ "description": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.", "adjudication": { "date": "2026-10-02", - "status": "real-miss", - "reason": "cross-file", - "why": "getClientsWithPermission enumerates typed resources and returns their names. Individually created client resources have no type; the type-level Clients resource name is not a client ID.", + "status": "policy-excluded", + "reason": "unreachable-caller", + "why": "The cited consumer is reached only with the V1 feature enabled. Profile selects a single version of this feature, and the V2 evaluator is selected only with V2 enabled. The label does not establish a reachable failure in this revision.", "evidence": [ "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L138", - "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55", + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/rest/admin-ui-ext/src/main/java/org/keycloak/admin/ui/rest/AvailableRoleMappingResource.java#L184", + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/common/src/main/java/org/keycloak/common/Profile.java#L272" ] - } + }, + "policy_excluded": true } ] } diff --git a/eval/corpus/labels/crb-keycloak-38446.json b/eval/corpus/labels/crb-keycloak-38446.json index ed32376..2800837 100644 --- a/eval/corpus/labels/crb-keycloak-38446.json +++ b/eval/corpus/labels/crb-keycloak-38446.json @@ -37,10 +37,12 @@ "date": "2026-10-02", "status": "real-miss", "reason": "cross-file", - "why": "The new user-storage getCredentials path creates a recovery model with createFromValues but never transfers the stored ID. createFromValues does not set an ID, while the authenticator later passes getId to removal.", + "why": "The new user-storage getCredentials path creates a recovery model without copying the stored ID. AccountCredentialResource returns these credentials, and CredentialDeleteHelper resolves federated deletion by ID. The missing ID makes the returned credential unaddressable. The label\u2019s authenticator example is not the demonstrated path: this provider reconstructs unconsumed codes on each read.", "evidence": [ "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java#L239", - "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a" + "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a", + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/services/src/main/java/org/keycloak/services/resources/account/AccountCredentialResource.java#L184", + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/services/src/main/java/org/keycloak/authentication/requiredactions/util/CredentialDeleteHelper.java#L64" ] } } diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/README.md b/eval/runs/2026-10-02-zero-recall-adjudication/README.md index 7efbbeb..a0c3cd4 100644 --- a/eval/runs/2026-10-02-zero-recall-adjudication/README.md +++ b/eval/runs/2026-10-02-zero-recall-adjudication/README.md @@ -8,14 +8,14 @@ intact. The machine-readable index is | Disposition | Findings | | --- | --- | -| Eligible correctness defects | 11 | -| Policy-excluded | 16 | -| Label-wrong / insufficiently specified | 8 | +| Eligible correctness defects | 9 | +| Policy-excluded | 17 | +| Label-wrong / insufficiently specified | 9 | The historical outputs are re-judged here, without rerunning either the candidate or matcher. The original sweep remains untouched. After the exclusions and one category correction, the recorded candidate matched -**1 of 11 eligible correctness findings** in this deliberately selected +**1 of 9 eligible correctness findings** in this deliberately selected zero-recall cluster. This is not full-corpus recall, a current-release result, or a validation of the old Claude matcher. @@ -35,6 +35,11 @@ read/write panic `efficiency`. Its normalized category is now correctness; - Grafana 90939 and Sentry 80528: the cache-error assignment and wrong returned config predate the changes. A latent defect moved unchanged is excluded by Reviso's changed-behavior policy. +- Keycloak 36880: resource-server and client IDs are equal in the pinned + adapter. The cited role-mapping consumer runs only with V1 enabled, + while the new implementation runs under mutually exclusive V2. These + labels do not establish reachable regressions. The cleanup flag remains + eligible. - Sentry 95633: `.python-version` pins 3.13.1, which has `Queue.shutdown`. Test constants and documentation cleanup do not become correctness defects merely because the importer mapped them to `bug`. @@ -63,7 +68,7 @@ matches. Entirely excluded cases are unscored, not expected-clean: candidate-only findings still require verification before being called wins or false positives. The judge rejects unknown adjudication statuses. -The 11 eligible findings are a starting point for the audit comparison, +The 9 eligible findings are a starting point for the audit comparison, not evidence that the current workflow still misses them. Measure both tiers on the same pinned changes before deciding whether deeper review or new detection angles are needed. diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-judge.json index 2bf2e79..0237fef 100644 --- a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-judge.json +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-judge.json @@ -1,30 +1,7 @@ { "mode": "gold", "expected_clean": false, - "excluded_gold": [], - "excluded_recorded_matches": [], - "matched": [], - "missed_correctness_gold": [ - { - "file": null, - "line": 0, - "title": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleanin", - "severity": "high", - "category": "security", - "crb_category": "security", - "description": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.", - "adjudication": { - "date": "2026-10-02", - "status": "real-miss", - "reason": "feature-flag", - "why": "The newly added V1 feature guard prevents cleanup listener execution when only V2 is enabled, even though V2 creates per-client authorization resources.", - "evidence": [ - "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/AdminPermissions.java#L77", - "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" - ] - }, - "tier": "correctness" - }, + "excluded_gold": [ { "file": null, "line": 0, @@ -35,15 +12,16 @@ "description": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.", "adjudication": { "date": "2026-10-02", - "status": "real-miss", - "reason": "cross-file", - "why": "The per-client lookup supplies the resource-server ID as owner, but getOrCreateResource creates the resource with the server client ID as owner. Client-specific permissions can be missed.", + "status": "label-wrong", + "reason": "identifier-equivalence", + "why": "ResourceServerAdapter.getClientId returns getId. The owner used when creating resources and the owner used by this lookup are equal, so the asserted mismatch is false.", "evidence": [ "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L214", - "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55", + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/model/jpa/src/main/java/org/keycloak/authorization/jpa/store/ResourceServerAdapter.java#L91" ] }, - "tier": "correctness" + "tier": "excluded" }, { "file": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", @@ -55,11 +33,38 @@ "description": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.", "adjudication": { "date": "2026-10-02", - "status": "real-miss", - "reason": "cross-file", - "why": "getClientsWithPermission enumerates typed resources and returns their names. Individually created client resources have no type; the type-level Clients resource name is not a client ID.", + "status": "policy-excluded", + "reason": "unreachable-caller", + "why": "The cited consumer is reached only with the V1 feature enabled. Profile selects a single version of this feature, and the V2 evaluator is selected only with V2 enabled. The label does not establish a reachable failure in this revision.", "evidence": [ "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L138", + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55", + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/rest/admin-ui-ext/src/main/java/org/keycloak/admin/ui/rest/AvailableRoleMappingResource.java#L184", + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/common/src/main/java/org/keycloak/common/Profile.java#L272" + ] + }, + "policy_excluded": true, + "tier": "excluded" + } + ], + "excluded_recorded_matches": [], + "matched": [], + "missed_correctness_gold": [ + { + "file": null, + "line": 0, + "title": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleanin", + "severity": "high", + "category": "security", + "crb_category": "security", + "description": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.", + "adjudication": { + "date": "2026-10-02", + "status": "real-miss", + "reason": "feature-flag", + "why": "The newly added V1 feature guard prevents cleanup listener execution when only V2 is enabled, even though V2 creates per-client authorization resources.", + "evidence": [ + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/AdminPermissions.java#L77", "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" ] }, @@ -103,7 +108,7 @@ ], "metrics": { "gold_count": 3, - "gold_correctness_count": 3, + "gold_correctness_count": 1, "candidate_count": 4, "matched_count": 0, "matched_correctness_count": 0, diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-labels.json index ee3edaa..6808c5d 100644 --- a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-labels.json +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-36880/gold-labels.json @@ -28,12 +28,13 @@ "description": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.", "adjudication": { "date": "2026-10-02", - "status": "real-miss", - "reason": "cross-file", - "why": "The per-client lookup supplies the resource-server ID as owner, but getOrCreateResource creates the resource with the server client ID as owner. Client-specific permissions can be missed.", + "status": "label-wrong", + "reason": "identifier-equivalence", + "why": "ResourceServerAdapter.getClientId returns getId. The owner used when creating resources and the owner used by this lookup are equal, so the asserted mismatch is false.", "evidence": [ "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L214", - "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55", + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/model/jpa/src/main/java/org/keycloak/authorization/jpa/store/ResourceServerAdapter.java#L91" ] } }, @@ -47,13 +48,16 @@ "description": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.", "adjudication": { "date": "2026-10-02", - "status": "real-miss", - "reason": "cross-file", - "why": "getClientsWithPermission enumerates typed resources and returns their names. Individually created client resources have no type; the type-level Clients resource name is not a client ID.", + "status": "policy-excluded", + "reason": "unreachable-caller", + "why": "The cited consumer is reached only with the V1 feature enabled. Profile selects a single version of this feature, and the V2 evaluator is selected only with V2 enabled. The label does not establish a reachable failure in this revision.", "evidence": [ "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java#L138", - "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55" + "https://github.com/keycloak/keycloak/compare/b8c1cd410f4499c77a8b372decfe1718b80ccfd2...1950a511026d520a7329c7b6b9ee60a4af8f8b55", + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/rest/admin-ui-ext/src/main/java/org/keycloak/admin/ui/rest/AvailableRoleMappingResource.java#L184", + "https://github.com/keycloak/keycloak/blob/1950a511026d520a7329c7b6b9ee60a4af8f8b55/common/src/main/java/org/keycloak/common/Profile.java#L272" ] - } + }, + "policy_excluded": true } ] diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-judge.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-judge.json index 84e6934..3d67026 100644 --- a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-judge.json +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-judge.json @@ -39,10 +39,12 @@ "date": "2026-10-02", "status": "real-miss", "reason": "cross-file", - "why": "The new user-storage getCredentials path creates a recovery model with createFromValues but never transfers the stored ID. createFromValues does not set an ID, while the authenticator later passes getId to removal.", + "why": "The new user-storage getCredentials path creates a recovery model without copying the stored ID. AccountCredentialResource returns these credentials, and CredentialDeleteHelper resolves federated deletion by ID. The missing ID makes the returned credential unaddressable. The label’s authenticator example is not the demonstrated path: this provider reconstructs unconsumed codes on each read.", "evidence": [ "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java#L239", - "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a" + "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a", + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/services/src/main/java/org/keycloak/services/resources/account/AccountCredentialResource.java#L184", + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/services/src/main/java/org/keycloak/authentication/requiredactions/util/CredentialDeleteHelper.java#L64" ] }, "tier": "correctness" diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-labels.json b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-labels.json index b679849..16d012b 100644 --- a/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-labels.json +++ b/eval/runs/2026-10-02-zero-recall-adjudication/crb-keycloak-38446/gold-labels.json @@ -31,10 +31,12 @@ "date": "2026-10-02", "status": "real-miss", "reason": "cross-file", - "why": "The new user-storage getCredentials path creates a recovery model with createFromValues but never transfers the stored ID. createFromValues does not set an ID, while the authenticator later passes getId to removal.", + "why": "The new user-storage getCredentials path creates a recovery model without copying the stored ID. AccountCredentialResource returns these credentials, and CredentialDeleteHelper resolves federated deletion by ID. The missing ID makes the returned credential unaddressable. The label’s authenticator example is not the demonstrated path: this provider reconstructs unconsumed codes on each read.", "evidence": [ "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java#L239", - "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a" + "https://github.com/keycloak/keycloak/compare/85737f52b59aa2e3403c9b8e48acf71ccc5bfae3...ce2dcc537b4b1f98a335e88cfe8e0b804483252a", + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/services/src/main/java/org/keycloak/services/resources/account/AccountCredentialResource.java#L184", + "https://github.com/keycloak/keycloak/blob/ce2dcc537b4b1f98a335e88cfe8e0b804483252a/services/src/main/java/org/keycloak/authentication/requiredactions/util/CredentialDeleteHelper.java#L64" ] } } diff --git a/eval/runs/2026-10-02-zero-recall-adjudication/summary.json b/eval/runs/2026-10-02-zero-recall-adjudication/summary.json index fac01f6..e63cbb0 100644 --- a/eval/runs/2026-10-02-zero-recall-adjudication/summary.json +++ b/eval/runs/2026-10-02-zero-recall-adjudication/summary.json @@ -95,13 +95,13 @@ { "case": "crb-keycloak-36880", "gold_count": 3, - "gold_correctness_count": 3, + "gold_correctness_count": 1, "candidate_count": 4, "matched_count": 0, "matched_correctness_count": 0, "gold_recall_correctness": 0, "precision_proxy_pct": 0, - "excluded_count": 0 + "excluded_count": 2 }, { "case": "crb-keycloak-36882", @@ -170,7 +170,7 @@ "excluded_count": 2 } ], - "eligible_correctness_total": 11, + "eligible_correctness_total": 9, "matched_correctness_total": 1, - "excluded_total": 24 + "excluded_total": 26 }