Skip to content

Commit 4ffa3f1

Browse files
jmjavacursoragent
andcommitted
Put a harder-case job on the MCP apply path so unproven local cannot apply.
Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 1d79acc commit 4ffa3f1

3 files changed

Lines changed: 40 additions & 0 deletions

File tree

‎docs/evaluation-protocol.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,9 @@ The same gate runs after **real stdio MCP** calls when you pass
169169
never call `local_*`. Accept / rewrite / reject then run on the scored
170170
candidate. Security-sensitive delegated jobs first call `local_review`
171171
and attach those notes to the premium packet; that still cannot approve.
172+
`MCP_JOBS` includes `rename_exception_across_files` so the apply gate
173+
sees a harder multi-file case; premium accept of an unproven local
174+
result is blocked (`accept_unproven_local`).
172175

173176
The MCP server also refuses secret filenames, private-key / token
174177
blobs, oversized file sets, and `max_tokens` above 4096 **before**

‎src/local_coding_slm/eval/jobs.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from __future__ import annotations
44

55
from local_coding_slm.eval.cases import CASES_BY_ID
6+
from local_coding_slm.eval.cases_harder import RENAME_EXCEPTION_PARTIAL
67
from local_coding_slm.eval.orchestrate import OrchestrationJob
78
from local_coding_slm.eval.review import accept, reject, rewrite
89
from local_coding_slm.eval.routing import RouteSignals, mechanical_signals
@@ -96,6 +97,17 @@ def clamp(value: int, lo: int, hi: int) -> int:
9697
eval_case=CASES_BY_ID["implement_clamp"],
9798
review=reject(notes="do not apply until bounds are reviewed"),
9899
),
100+
OrchestrationJob(
101+
id="mcp_rename_exception_unproven",
102+
signals=mechanical_signals(),
103+
eval_case=CASES_BY_ID["rename_exception_across_files"],
104+
local_replies=(
105+
RENAME_EXCEPTION_PARTIAL,
106+
RENAME_EXCEPTION_PARTIAL,
107+
RENAME_EXCEPTION_PARTIAL,
108+
),
109+
review=accept(notes="do not apply unproven harder local"),
110+
),
99111
)
100112

101113

‎tests/test_eval_mcp_orchestrate.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@
44

55
import unittest
66

7+
from local_coding_slm.eval.cases_harder import HARDER_CASE_IDS
78
from local_coding_slm.eval.harness import run_campaign, run_orchestrated_campaign
89
from local_coding_slm.eval.jobs import MCP_JOBS
10+
from local_coding_slm.eval.orchestrate import run_job
911
from local_coding_slm.eval.record import summarize
1012

1113

@@ -108,6 +110,29 @@ def test_job_table_covers_keep_and_delegate(self) -> None:
108110
self.assertTrue(any(job.eval_case is None for job in MCP_JOBS))
109111
self.assertTrue(any(job.eval_case is not None for job in MCP_JOBS))
110112

113+
def test_mcp_jobs_intersect_harder_cases(self) -> None:
114+
case_ids = {
115+
job.eval_case.id for job in MCP_JOBS if job.eval_case is not None
116+
}
117+
overlap = case_ids & set(HARDER_CASE_IDS)
118+
self.assertIn("rename_exception_across_files", overlap)
119+
self.assertTrue(overlap)
120+
121+
def test_harder_rename_exception_job_blocks_accept_unproven_local(self) -> None:
122+
job = next(
123+
item
124+
for item in MCP_JOBS
125+
if item.eval_case is not None
126+
and item.eval_case.id == "rename_exception_across_files"
127+
)
128+
result = run_job(job)
129+
self.assertEqual(result.case_id, "rename_exception_across_files")
130+
self.assertTrue(result.delegated)
131+
self.assertFalse(result.local_passed)
132+
self.assertFalse(result.applied)
133+
self.assertEqual(result.outcome, "blocked")
134+
self.assertEqual(result.blocked, "accept_unproven_local")
135+
111136

112137
if __name__ == "__main__":
113138
unittest.main()

0 commit comments

Comments
 (0)