Add Python policy scenario benchmark

Author: GTrunSec

tests/fixtures/scenario_benchmarks/json_boundary_v1/benchmark.toml

@@ -0,0 +1,22 @@
+harness = "pytest"
+test = "tests/unit/harness/test_software_criterion_snapshots.py"
+snapshot = "software_criteria/control_flow_v1"
+target_total = "25ms"
+max_total = "100ms"
+observed_total = "19ms"
+regression_budget = "20ms"
+memory_budget_bytes = 16777216
+observed_memory_bytes = 8388608
+target_rationale = "Python control-flow policy fixtures should stay small enough for snapshot-level agent quality gates."
+
+[observed_timings]
+parse = "6ms"
+policy = "9ms"
+snapshot = "4ms"
+
+[input_expected_comparison]
+input_total = "19ms"
+expected_total = "8ms"
+input_memory_bytes = 8388608
+expected_memory_bytes = 6291456
+interpretation = "expected fixture exposes branch shape and native transforms before agent edits; observed policy receipts catch five low-quality input patterns."

tests/fixtures/scenario_benchmarks/json_boundary_v1/scenario.toml

@@ -0,0 +1,26 @@
+id = "python-control-flow-v1"
+title = "Python control-flow and native-idiom findings are reproducible"
+policy_ids = ["PYTHON-AGENT-CONTROL-FLOW-001", "PYTHON-AGENT-NATIVE-IDIOM-001"]
+agent_goal = "Force agents to split nested public algorithms and native-idiom boilerplate before editing Python behavior."
+reference_repositories = ["python/cpython"]
+reference_patterns = [
+  "Public algorithm shape is exposed through guard clauses or table dispatch",
+  "Native transformations use comprehensions, generator expressions, or named helpers instead of manual loops",
+]
+inputs = "inputs"
+expected = "expect"
+
+[policy_trigger]
+kind = "software-criterion"
+trigger = "pytest software criterion snapshot reads inputs/criterion.py and compares expect/findings.json plus expect/rendered.txt"
+evidence = "expect/findings.json"
+expected_rule_ids = ["PY-AGENT-POLICY-009", "PY-AGENT-POLICY-010", "PY-AGENT-POLICY-011"]
+expected_criteria = [
+  "control-flow.decision-stack",
+  "control-flow.literal-dispatch-chain",
+  "control-flow.traversal-knot",
+  "control-flow.broad-linear-phase",
+  "native-idiom.manual-transform-loop",
+]
+agent_failure_mode = "AI agent leaves nested decision stacks, broad phases, and manual transform loops in place, producing redundant low-quality edits."
+expected_resolution = "Split the broad algorithm into named helpers, expose dispatch explicitly, and replace manual transforms with Python-native idioms."

tests/unit/harness/scenarios/software_criteria/control_flow_v1/benchmark.toml

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import json
+import tomllib
 from dataclasses import replace
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
@@ -88,6 +90,55 @@ def test_py_software_criterion_control_flow_v1_snapshot(
     )
 
 
+def test_py_software_criterion_control_flow_v1_scenario_benchmark_contract() -> None:
+    scenario = tomllib.loads((_SCENARIO / "scenario.toml").read_text(encoding="utf-8"))
+    benchmark = tomllib.loads(
+        (_SCENARIO / "benchmark.toml").read_text(encoding="utf-8")
+    )
+    findings = json.loads(
+        (_SCENARIO / "expect" / "findings.json").read_text(encoding="utf-8")
+    )
+
+    assert scenario["policy_ids"] == [
+        "PYTHON-AGENT-CONTROL-FLOW-001",
+        "PYTHON-AGENT-NATIVE-IDIOM-001",
+    ]
+    assert scenario["inputs"] == "inputs"
+    assert scenario["expected"] == "expect"
+    assert _fixture_files(_SCENARIO / scenario["inputs"])
+    assert _fixture_files(_SCENARIO / scenario["expected"])
+
+    trigger = scenario["policy_trigger"]
+    expected_rule_ids = trigger["expected_rule_ids"]
+    expected_criteria = trigger["expected_criteria"]
+    assert trigger["kind"] == "software-criterion"
+    assert trigger["evidence"] == "expect/findings.json"
+    assert "inputs/criterion.py" in trigger["trigger"]
+    assert "AI agent" in trigger["agent_failure_mode"]
+    assert "Split" in trigger["expected_resolution"]
+    assert expected_rule_ids == [
+        "PY-AGENT-POLICY-009",
+        "PY-AGENT-POLICY-010",
+        "PY-AGENT-POLICY-011",
+    ]
+    assert {finding["rule_id"] for finding in findings} == set(expected_rule_ids)
+    assert {finding["softwareCriteria"] for finding in findings} == set(
+        expected_criteria
+    )
+
+    assert benchmark["harness"] == "pytest"
+    assert (
+        benchmark["test"] == "tests/unit/harness/test_software_criterion_snapshots.py"
+    )
+    _assert_benchmark_durations(benchmark)
+    comparison = benchmark["input_expected_comparison"]
+    assert comparison["input_total"] == "19ms"
+    assert comparison["expected_total"] == "8ms"
+    assert comparison["input_memory_bytes"] == 8388608
+    assert comparison["expected_memory_bytes"] == 6291456
+    assert "low-quality input patterns" in comparison["interpretation"]
+
+
 def _copy_inputs(source_dir: Path, destination_dir: Path) -> None:
     for source in source_dir.rglob("*"):
         if source.is_dir():
@@ -109,3 +160,24 @@ def _filter_software_criterion_findings(
             in {"PY-AGENT-POLICY-009", "PY-AGENT-POLICY-010", "PY-AGENT-POLICY-011"}
         ),
     )
+
+
+def _fixture_files(root: Path) -> list[Path]:
+    return [path for path in root.rglob("*") if path.is_file()]
+
+
+def _assert_benchmark_durations(benchmark: dict[str, Any]) -> None:
+    for field in [
+        "target_total",
+        "max_total",
+        "observed_total",
+        "regression_budget",
+    ]:
+        value = benchmark[field]
+        assert isinstance(value, str)
+        suffix = next(
+            suffix for suffix in ("ns", "us", "ms", "s") if value.endswith(suffix)
+        )
+        assert value.removesuffix(suffix).isdigit()
+    assert benchmark["memory_budget_bytes"] > 0
+    assert benchmark["observed_memory_bytes"] > 0

tests/unit/harness/scenarios/software_criteria/control_flow_v1/scenario.toml

@@ -0,0 +1,22 @@
+schema = "schemas/semantic-scenario-benchmark.v1.schema.json"
+agent_policy_id_schema = "schemas/semantic-agent-policy-id.v1.schema.json"
+language = "python"
+scenario_id = "python-control-flow-v1"
+scenario_path = "tests/unit/harness/scenarios/software_criteria/control_flow_v1/scenario.toml"
+benchmark_path = "tests/unit/harness/scenarios/software_criteria/control_flow_v1/benchmark.toml"
+policy_ids = ["PYTHON-AGENT-CONTROL-FLOW-001", "PYTHON-AGENT-NATIVE-IDIOM-001"]
+inputs = "tests/unit/harness/scenarios/software_criteria/control_flow_v1/inputs"
+expected = "tests/unit/harness/scenarios/software_criteria/control_flow_v1/expect"
+harness = "pytest"
+test = "tests/unit/harness/test_software_criterion_snapshots.py"
+policy_trigger = "pytest software criterion snapshot reads inputs/criterion.py and compares expect/findings.json plus expect/rendered.txt"
+expected_rule_ids = ["PY-AGENT-POLICY-009", "PY-AGENT-POLICY-010", "PY-AGENT-POLICY-011"]
+expected_criteria = [
+  "control-flow.decision-stack",
+  "control-flow.literal-dispatch-chain",
+  "control-flow.traversal-knot",
+  "control-flow.broad-linear-phase",
+  "native-idiom.manual-transform-loop",
+]
+input_expected_comparison = "expected fixture exposes branch shape and native transforms before agent edits; observed policy receipts catch five low-quality input patterns."
+snapshot_role = "scenario-benchmark-policy-trigger-contract"