evals: record the synthetic user's tokens, not just its cost

app/desktop/studio_server/eval_api.py

@@ -735,11 +735,26 @@ def scored_trace_usage(trace: TaskRun) -> Usage | None:
       Latency still reads from `usage` — `cumulative_usage` deliberately carries
       none, since per-message latencies don't aggregate meaningfully.
     - An eval-driven conversation stores the synthetic-user driver model's spend
-      in `synthetic_user_usage`, beside the assistant-only `usage`; the honest
-      total sums the two null-tolerantly. Today that field carries cost only, so
-      token counts pass through from the assistant side. Migrated legacy traces
-      have the blend fused inside `usage` with `synthetic_user_usage` None, so
-      the same sum reads both record generations correctly.
+      in `synthetic_user_usage`, beside the assistant-only `usage`. Only its
+      **cost** is blended in here, deliberately, even though the field carries
+      the driver's tokens and latency too:
+
+      * Cost is total-spend semantics — what this trace cost to produce, both
+        models included. That is what a run-config summary should report, and it
+        matches migrated legacy traces, which have the blend fused inside
+        `usage` with `synthetic_user_usage` None.
+      * Tokens are not. The synthetic user is usually a different model on a
+        different provider from the agent under test, so folding its ~3.5k input
+        tokens per conversation into this figure would attribute them to the
+        agent and make `cost / total_tokens` meaningless — the exact conflation
+        `synthetic_user_usage` exists to undo.
+      * Latency is not. This summary reports how responsive the agent is;
+        driver-side wall clock is not the agent's, and summing it would make
+        every driven run config look slower than it is.
+
+      Blending everything would also make this field mean two different
+      quantities depending on record age: legacy records blend cost only, since
+      that is all the old field carried.
 
     None when the record has nothing to report, so it contributes nothing to an
     average instead of counting as a zero.
@@ -760,7 +775,9 @@ def scored_trace_usage(trace: TaskRun) -> Usage | None:
         base = trace.usage
 
     if trace.synthetic_user_usage is not None:
-        base = (base or Usage()) + trace.synthetic_user_usage
+        # Cost only — see the docstring. Adding the whole object would fold the
+        # driver's tokens and latency into the agent's figures.
+        base = (base or Usage()) + Usage(cost=trace.synthetic_user_usage.cost)
     if base is None or all(v is None for v in base.model_dump().values()):
         return None
     return base

app/desktop/studio_server/test_eval_api.py

@@ -2374,6 +2374,66 @@ def test_migrated_legacy_trace_reads_unchanged(self, mock_task, data_source):
         assert trace.synthetic_user_usage is None
         assert scored_trace_usage(trace) == blended
 
+    def test_synthetic_user_blends_cost_only(self, mock_task, data_source):
+        """The driver's cost joins the total; its tokens and latency do not.
+
+        The synthetic user is normally a different model on a different provider
+        from the agent under test. Folding its tokens in would attribute them to
+        the agent (~3.5k input per conversation) and make cost/token meaningless,
+        and folding its latency in would make every driven run config look slower
+        than it is. Cost alone is total-spend semantics, and it is what migrated
+        legacy records already blend — so all three quantities keep one meaning
+        across record generations.
+        """
+        trace = TaskRun(
+            parent=mock_task,
+            input="in",
+            input_source=data_source,
+            output=TaskOutput(output="out", source=data_source),
+            usage=Usage(
+                input_tokens=100,
+                output_tokens=50,
+                total_tokens=150,
+                cost=1.0,
+                total_llm_latency_ms=4000,
+            ),
+            synthetic_user_usage=Usage(
+                input_tokens=3548,
+                output_tokens=61,
+                total_tokens=3609,
+                cost=0.25,
+                total_llm_latency_ms=9000,
+            ),
+        )
+
+        usage = scored_trace_usage(trace)
+
+        assert usage is not None
+        # Cost blends: both models' spend produced this trace.
+        assert usage.cost == pytest.approx(1.25)
+        # Tokens and latency stay the agent's alone.
+        assert usage.input_tokens == 100
+        assert usage.output_tokens == 50
+        assert usage.total_tokens == 150
+        assert usage.total_llm_latency_ms == 4000
+
+    def test_synthetic_user_without_cost_leaves_the_total_alone(
+        self, mock_task, data_source
+    ):
+        """A driver that reported tokens but no cost must not perturb anything —
+        including not turning an all-agent figure into a different object."""
+        agent = Usage(input_tokens=100, total_tokens=150, cost=1.0)
+        trace = TaskRun(
+            parent=mock_task,
+            input="in",
+            input_source=data_source,
+            output=TaskOutput(output="out", source=data_source),
+            usage=agent,
+            synthetic_user_usage=Usage(input_tokens=3548, total_tokens=3609),
+        )
+
+        assert scored_trace_usage(trace) == agent
+
     def test_nothing_to_report_reads_as_none(self, mock_task, data_source):
         trace = TaskRun(
             parent=mock_task,

libs/core/kiln_ai/adapters/eval/eval_runner.py

@@ -1048,10 +1048,12 @@ async def _persist_driven_conversation(
             trace=leaf.trace,
             usage=_conversation_usage(drive_result.chain),
             # The synthetic-user driver's spend rides its own field so `usage`
-            # stays honestly assistant-only; a zero-cost drive records None.
-            synthetic_user_usage=Usage(cost=drive_result.su_total_cost)
-            if drive_result.su_total_cost > 0
-            else None,
+            # stays honestly assistant-only. The whole Usage, not a cost-only
+            # stub: the SU is usually a different model on a different provider
+            # from the agent, so its tokens are the half that makes the figure
+            # reconcilable. `drive_case` already returns None for a drive whose
+            # provider reported nothing.
+            synthetic_user_usage=drive_result.su_usage,
             cumulative_usage=leaf.cumulative_usage
             or MessageUsage.from_trace(leaf.trace),
             eval_source=EvalItemSource(source_type=source_type, source_id=source_id),

libs/core/kiln_ai/adapters/eval/test_eval_runner.py

@@ -3518,12 +3518,16 @@ def multi_turn_eval_input(mock_task):
 def _fresh_leaf(
     task: Task,
     data_source: DataSource,
-    su_total_cost: float = 0.0,
+    su_usage: Usage | None = None,
     cumulative_usage: MessageUsage | None = None,
     trace: list[ChatCompletionMessageParam] | None = MULTI_TURN_TRACE,
 ) -> DriveCaseResult:
     """The in-memory DriveCaseResult drive_case_for_eval would return:
-    an id-less, trace-carrying, never-saved leaf plus the SU-side spend."""
+    an id-less, trace-carrying, never-saved leaf plus the SU-side spend.
+
+    `su_usage` defaults to None — the shape a drive whose provider reported
+    nothing produces, which is what the tests that don't care about SU spend
+    want."""
     leaf = TaskRun(
         input="opening message",
         input_source=data_source,
@@ -3533,7 +3537,7 @@ def _fresh_leaf(
         parent=task,
     )
     leaf.id = None
-    return DriveCaseResult(chain=[leaf], su_total_cost=su_total_cost)
+    return DriveCaseResult(chain=[leaf], su_usage=su_usage)
 
 
 class TestRunV2MultiTurnRedrive:
@@ -4274,7 +4278,7 @@ async def test_unsaved_fresh_run_does_not_crash_record_keys_on_item(
 
 class TestEvalRunUsageRecording:
     @pytest.mark.asyncio
-    async def test_drive_records_agent_usage_plus_su_cost(
+    async def test_drive_records_agent_usage_and_full_su_usage(
         self,
         mock_task,
         mock_run_config,
@@ -4285,7 +4289,9 @@ async def test_drive_records_agent_usage_plus_su_cost(
         drive_result = _fresh_leaf(
             mock_task,
             data_source,
-            su_total_cost=0.25,
+            su_usage=Usage(
+                input_tokens=3548, output_tokens=61, total_tokens=3609, cost=0.25
+            ),
             cumulative_usage=MessageUsage(
                 input_tokens=100, output_tokens=50, total_tokens=150, cost=1.0
             ),
@@ -4327,9 +4333,15 @@ async def test_drive_records_agent_usage_plus_su_cost(
         assert trace.usage.cost == pytest.approx(1.0)
         assert trace.usage.input_tokens == 100
         assert trace.usage.total_tokens == 150
-        # The synthetic-user driver's spend rides its own field.
+        # The synthetic-user driver's spend rides its own field, with its own
+        # tokens — the agent's counts above are a different model on a different
+        # provider, so a cost-only SU record could be reconciled against neither
+        # invoice and split per model not at all.
         assert trace.synthetic_user_usage is not None
         assert trace.synthetic_user_usage.cost == pytest.approx(0.25)
+        assert trace.synthetic_user_usage.input_tokens == 3548
+        assert trace.synthetic_user_usage.output_tokens == 61
+        assert trace.synthetic_user_usage.total_tokens == 3609
         assert trace.cumulative_usage == MessageUsage(
             input_tokens=100, output_tokens=50, total_tokens=150, cost=1.0
         )
@@ -4345,7 +4357,7 @@ async def test_zero_cost_drive_records_no_synthetic_user_usage(
     ):
         """None, not a zero-cost Usage: the rollup's null-tolerant blend would
         read a zero object as a real 0.0 cost and count it in averages."""
-        drive_result = _fresh_leaf(mock_task, data_source, su_total_cost=0.0)
+        drive_result = _fresh_leaf(mock_task, data_source, su_usage=None)
         runner = EvalRunner(
             eval_configs=[mock_v2_redrive_config],
             run_configs=[mock_run_config],

libs/core/kiln_ai/synthetic_user/drive_loop.py

@@ -16,6 +16,7 @@
 from typing import Protocol
 
 from kiln_ai.datamodel.task_run import TaskRun
+from kiln_ai.datamodel.usage import Usage
 from kiln_ai.synthetic_user.driver import SyntheticUserDriver
 from kiln_ai.utils.open_ai_types import ChatCompletionMessageParam
 
@@ -63,14 +64,32 @@ class DriveCaseResult:
     no stop_reason field — every case ends after exactly `turns`
     iterations by design.
 
-    `su_total_cost` sums the SU driver's per-turn LLM cost across the
-    case. SU turns aren't persisted as TaskRuns, so this is the only
-    place that spend surfaces — the runner adds it to the target's
-    `cumulative_usage.cost` to produce an honest total.
+    `su_usage` sums the SU driver model's usage across the case's turns.
+    SU turns aren't persisted as TaskRuns, so this is the only place that
+    spend surfaces at all — and it carries the driver's tokens, not just
+    its cost, because the SU is usually a different model on a different
+    provider from the agent under test. A cost alone can be reconciled
+    against no invoice and split per model not at all.
+
+    None when no turn reported usage, rather than a zeroed Usage: an
+    unmeasured drive must not read as a genuinely free one.
     """
 
     chain: list[TaskRun]
-    su_total_cost: float
+    su_usage: Usage | None
+
+    @property
+    def su_total_cost(self) -> float:
+        """The case's SU cost, or 0.0 when nothing was reported.
+
+        Kept as a derived property so the interactive runner's
+        `CaseCompletedEvent.total_cost` is unchanged in behaviour — it wants a
+        float it can add, and "no usage reported" has always summed as zero
+        there.
+        """
+        if self.su_usage is None or self.su_usage.cost is None:
+            return 0.0
+        return float(self.su_usage.cost)
 
 
 async def drive_case(
@@ -111,7 +130,9 @@ async def drive_case(
     prev_run: TaskRun | None = None
     prev_trace: list[ChatCompletionMessageParam] | None = None
     chain: list[TaskRun] = []
-    su_total_cost: float = 0.0
+    # Stays None until some turn actually reports usage, so a drive whose
+    # provider reported nothing is distinguishable from a free one.
+    su_usage: Usage | None = None
 
     for turn in range(1, turns + 1):
         new_run = await target_invoker(
@@ -128,8 +149,13 @@ async def drive_case(
         su_message: str | None = None
         su_cost = 0.0
         if turn < turns:
-            su_message, su_cost = await su_driver.respond(new_run.trace or [])
-            su_total_cost += su_cost
+            su_message, turn_usage = await su_driver.respond(new_run.trace or [])
+            if turn_usage is not None:
+                # Usage.__add__ is None-graceful per field, so a turn that
+                # reports only cost doesn't erase another turn's token counts.
+                su_usage = turn_usage if su_usage is None else su_usage + turn_usage
+                if turn_usage.cost is not None:
+                    su_cost = float(turn_usage.cost)
 
         if on_turn is not None:
             await on_turn(run=new_run, su_message=su_message, su_cost=su_cost)
@@ -139,4 +165,4 @@ async def drive_case(
         prev_run = new_run
         prev_trace = new_run.trace
 
-    return DriveCaseResult(chain=chain, su_total_cost=su_total_cost)
+    return DriveCaseResult(chain=chain, su_usage=su_usage)

libs/core/kiln_ai/synthetic_user/driver.py

@@ -14,6 +14,7 @@
 from kiln_ai.datamodel.datamodel_enums import StructuredOutputMode
 from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties, ToolsRunConfig
 from kiln_ai.datamodel.task import Task
+from kiln_ai.datamodel.usage import Usage
 from kiln_ai.synthetic_user.models import (
     SyntheticUserDriverConfig,
     SyntheticUserInfo,
@@ -84,11 +85,20 @@ def __init__(
 
     async def respond(
         self, conversation: list[ChatCompletionMessageParam]
-    ) -> tuple[str, float]:
-        """Return the SU's next message and the per-call cost.
+    ) -> tuple[str, Usage | None]:
+        """Return the SU's next message and the driver model's usage for the call.
 
         `conversation` is in the eval frame and must end on an `assistant`
         (target) turn. Drive-loop termination is the caller's concern.
+
+        The whole `Usage` rather than just its cost: the SU's TaskRun is never
+        persisted, so this in-memory value is the only place the driver model's
+        tokens ever exist. A cost alone can neither be split per model against an
+        invoice nor recomputed at a different price, and `cost / total_tokens`
+        over a figure whose tokens are the agent's is meaningless.
+
+        None when the provider reported nothing — distinct from a zeroed Usage,
+        which would read as a genuinely free call rather than an unmeasured one.
         """
         # 1) Filter to visible roles (drop system/tool if present).
         visible = [
@@ -133,12 +143,4 @@ async def respond(
         if not isinstance(raw, str):
             raise RuntimeError("synthetic user returned non-string output")
 
-        # Per-call cost; defaults to 0.0 when the provider doesn't
-        # surface pricing.
-        cost = (
-            float(task_run.usage.cost)
-            if task_run.usage is not None and task_run.usage.cost is not None
-            else 0.0
-        )
-
-        return raw, cost
+        return raw, task_run.usage

libs/core/kiln_ai/synthetic_user/eval_drive.py

@@ -46,7 +46,7 @@ async def drive_case_for_eval(
 
     The result's leaf (`chain[-1]`) has `.trace` holding the full cumulative
     conversation and its id is None (nothing touches disk). The result also
-    carries `su_total_cost` — the synthetic user's LLM spend, which surfaces
+    carries `su_usage` — the synthetic user model's spend, which surfaces
     nowhere else since SU turns aren't persisted. `skills` must be preloaded
     by the caller — the adapter raises on skill tools with no injected dict.
     """