1414if str (SRC ) not in sys .path :
1515 sys .path .insert (0 , str (SRC ))
1616
17- from local_coding_slm .eval .cases import CASES , CASES_BY_ID , FIXTURES # noqa: E402
17+ from local_coding_slm .eval .cases import ( # noqa: E402
18+ CASES ,
19+ CASES_BY_ID ,
20+ FIXTURES ,
21+ SEED_CASE_IDS ,
22+ )
23+ from local_coding_slm .eval .cases_harder import HARDER_CASE_IDS # noqa: E402
1824from local_coding_slm .eval .score import score_candidate # noqa: E402
1925
26+ SUITE_CASE_IDS = {
27+ "all" : None ,
28+ "seed" : set (SEED_CASE_IDS ),
29+ "harder" : set (HARDER_CASE_IDS ),
30+ }
31+
2032
2133def _print_result (label : str , result : object ) -> None :
2234 passed = getattr (result , "passed" )
@@ -30,11 +42,20 @@ def _print_result(label: str, result: object) -> None:
3042 print (f" { layer .status :4} { layer .name } { layer .message } )
3143
3244
33- def _run_fixtures (case_id : str | None ) -> int :
45+ def _wanted (case_id : str , suite : str , only : str | None ) -> bool :
46+ if only and case_id != only :
47+ return False
48+ allowed = SUITE_CASE_IDS [suite ]
49+ return allowed is None or case_id in allowed
50+
51+
52+ def _run_fixtures (case_id : str | None , suite : str ) -> int :
3453 failed = 0
54+ ran = 0
3555 for fixture in FIXTURES :
36- if case_id and fixture .case_id != case_id :
56+ if not _wanted ( fixture .case_id , suite , case_id ) :
3757 continue
58+ ran += 1
3859 case = CASES_BY_ID [fixture .case_id ]
3960 result = score_candidate (fixture .text , case )
4061 ok = result .passed == fixture .expect_pass
@@ -54,13 +75,25 @@ def _run_fixtures(case_id: str | None) -> int:
5475 if not ok :
5576 print (f" expected pass={ fixture .expect_pass } { fixture .expect_first } )
5677 failed += 1
78+ if ran == 0 :
79+ print ("no fixtures matched" )
80+ return 1
5781 return failed
5882
5983
60- async def _run_live (case_id : str | None , model : str ) -> int :
84+ async def _run_live (case_id : str | None , model : str , suite : str ) -> int :
85+ from local_coding_slm .ollama_client import OllamaSettings , is_reachable
6186 from local_coding_slm .server import _load_dotenv
6287
6388 _load_dotenv ()
89+ settings = OllamaSettings .from_env ()
90+ if not is_reachable (settings ):
91+ print (
92+ f"SKIP live: Ollama unreachable at { settings .host_label ()}
93+ "(offline fixtures still pass; this is not a model-quality fail)"
94+ )
95+ return 0
96+
6497 from mcp import ClientSession , StdioServerParameters
6598 from mcp .client .stdio import stdio_client
6699
@@ -73,7 +106,12 @@ async def _run_live(case_id: str | None, model: str) -> int:
73106 cwd = str (ROOT ),
74107 )
75108 failed = 0
76- selected = [case for case in CASES if case_id is None or case .id == case_id ]
109+ selected = [
110+ case for case in CASES if _wanted (case .id , suite , case_id )
111+ ]
112+ if not selected :
113+ print ("no live cases matched" )
114+ return 1
77115 async with stdio_client (params ) as (read , write ):
78116 async with ClientSession (read , write ) as session :
79117 await session .initialize ()
@@ -83,7 +121,7 @@ async def _run_live(case_id: str | None, model: str) -> int:
83121 "files" : list (case .files ),
84122 "language" : case .language ,
85123 "model" : model ,
86- "max_tokens" : 700 ,
124+ "max_tokens" : case . max_tokens ,
87125 }
88126 if case .style :
89127 payload ["style" ] = case .style
@@ -107,10 +145,16 @@ def main() -> None:
107145 parser .add_argument ("--live" , action = "store_true" , help = "Call real Ollama via stdio MCP" )
108146 parser .add_argument ("--model" , choices = ("fast" , "strong" ), default = "fast" )
109147 parser .add_argument ("--case" , dest = "case_id" , default = None )
148+ parser .add_argument (
149+ "--suite" ,
150+ choices = tuple (SUITE_CASE_IDS ),
151+ default = "all" ,
152+ help = "Fixture / live subset. harder = Phase 3 multi-file behavior cases." ,
153+ )
110154 args = parser .parse_args ()
111155 if args .live :
112- raise SystemExit (asyncio .run (_run_live (args .case_id , args .model )))
113- failed = _run_fixtures (args .case_id )
156+ raise SystemExit (asyncio .run (_run_live (args .case_id , args .model , args . suite )))
157+ failed = _run_fixtures (args .case_id , args . suite )
114158 raise SystemExit (1 if failed else 0 )
115159
116160
0 commit comments