feat(llmobs): propagate agent_version to the agent's subtree

ddtrace/llmobs/_constants.py

@@ -152,13 +152,16 @@ class LLMObsSamplingDecision(str, Enum):
 INSTRUMENTATION_METHOD_AUTO = "auto"
 INSTRUMENTATION_METHOD_ANNOTATED = "annotated"
 
-# Agent tracking tag. Set on agent spans only, at span finish.
+# Agent tracking tag set at span finish.
 AGENT_VERSION_TAG_KEY = "agent_version"
 # Holds the version an annotation supplied, until the span kind is known at finish.
-AGENT_ANNOTATION = "_ml_obs.agent_annotation"
+AGENT_VERSION = "_ml_obs.agent_annotation"
+# Holds the version of the nearest agent ancestor, resolved at activation. Also carried on the
+# in-process context handed to asyncio tasks and threads. It is never propagated across services.
+PARENT_AGENT_VERSION = "_ml_obs.parent_agent_version"
 # Holds the manifest the annotations declared, for the same reason. Each annotation is validated
 # and shallow-merged into it as it runs.
-AGENT_DECLARATION_ANNOTATION = "_ml_obs.agent_declaration_annotation"
+AGENT_MANIFEST = "_ml_obs.agent_declaration_annotation"
 
 DISPATCH_ON_TOOL_CALL_OUTPUT_USED = "on_tool_call_output_used"
 DISPATCH_ON_LLM_TOOL_CHOICE = "on_llm_tool_choice"

ddtrace/llmobs/_llmobs.py

@@ -48,8 +48,8 @@
 from ddtrace.internal.utils.formats import format_trace_id
 from ddtrace.internal.utils.formats import parse_tags_str
 from ddtrace.llmobs import _telemetry as telemetry
-from ddtrace.llmobs._constants import AGENT_ANNOTATION
-from ddtrace.llmobs._constants import AGENT_DECLARATION_ANNOTATION
+from ddtrace.llmobs._constants import AGENT_MANIFEST
+from ddtrace.llmobs._constants import AGENT_VERSION
 from ddtrace.llmobs._constants import AGENT_VERSION_TAG_KEY
 from ddtrace.llmobs._constants import ANNOTATIONS_CONTEXT_ID
 from ddtrace.llmobs._constants import CACHED_LLMOBS_EVENT_CTX_KEY
@@ -81,6 +81,7 @@
 from ddtrace.llmobs._constants import LLMOBS_SAMPLING
 from ddtrace.llmobs._constants import LLMOBS_STRUCT
 from ddtrace.llmobs._constants import ML_APP
+from ddtrace.llmobs._constants import PARENT_AGENT_VERSION
 from ddtrace.llmobs._constants import PROMPT_TRACKING_INSTRUMENTATION_METHOD
 from ddtrace.llmobs._constants import PROPAGATED_LLMOBS_TRACE_ID_KEY
 from ddtrace.llmobs._constants import PROPAGATED_ML_APP_KEY
@@ -721,12 +722,12 @@ def _prepare_llmobs_span_data(self, span: Span, span_kind: Optional[str]) -> boo
             return False
 
         # Agent annotations are applied here, where the span kind is known: annotation_context
-        # reaches every span in its block, but only agent spans carry the tags.
+        # reaches every span in its block, but only agent spans carry the manifest.
         if span_kind == "agent":
-            agent_annotation = span._get_ctx_item(AGENT_ANNOTATION)
-            if agent_annotation:
-                llmobs_data.setdefault(LLMOBS_STRUCT.TAGS, {})[AGENT_VERSION_TAG_KEY] = str(agent_annotation)
-            declared_manifest = span._get_ctx_item(AGENT_DECLARATION_ANNOTATION)
+            agent_version = span._get_ctx_item(AGENT_VERSION) or span._get_ctx_item(PARENT_AGENT_VERSION)
+            if agent_version:
+                llmobs_data.setdefault(LLMOBS_STRUCT.TAGS, {})[AGENT_VERSION_TAG_KEY] = str(agent_version)
+            declared_manifest = span._get_ctx_item(AGENT_MANIFEST)
             if declared_manifest:
                 # Both levels are type-checked because caller metadata is not sanitized until
                 # _normalize_llmobs_meta runs below, so a forged `_dd` is still raw here. A
@@ -758,6 +759,10 @@ def _prepare_llmobs_span_data(self, span: Span, span_kind: Optional[str]) -> boo
                 if not merged.get("name"):
                     merged["name"] = get_llmobs_span_name(span) or span.name
                 _annotate_llmobs_span_data(span, agent_manifest=merged)
+        else:
+            parent_agent_version = span._get_ctx_item(PARENT_AGENT_VERSION)
+            if parent_agent_version:
+                llmobs_data.setdefault(LLMOBS_STRUCT.TAGS, {})[AGENT_VERSION_TAG_KEY] = parent_agent_version
 
         llmobs_meta = llmobs_data.setdefault(LLMOBS_STRUCT.META, _Meta())
         # Before the user processor and _normalize_llmobs_meta, either of which can strip values
@@ -1985,9 +1990,9 @@ def annotation_context(
         :param agent: A dictionary declaring the agent running in this context, accepting
                       ``version``, ``name``, ``instructions``, ``model``, ``model_settings`` and
                       ``tools``; see ``ddtrace.llmobs.Agent``. ``version`` is set as an
-                      ``agent_version`` tag and the rest as the agent's manifest, on every agent
-                      span in the block. All keys are optional; unreportable values are dropped,
-                      not raised.
+                      ``agent_version`` tag on every agent span in the block and its child spans,
+                      and the rest as the agent's manifest on every agent span in the block. All
+                      keys are optional; unreportable values are dropped, not raised.
         """
         # id to track an annotation for registering / de-registering
         annotation_id = rand64bits()
@@ -2507,8 +2512,10 @@ def _current_trace_context(self) -> Optional[Context]:
             # Carry the nearest agent onto the context so spans created in in-process task
             # boundaries (asyncio tasks, thread-pool executors) still attribute to it.
             # Stamped last so the budget check sees the full tagset.
-            parent_agent_name, parent_agent_span_id = _resolve_parent_agent(active)
+            parent_agent_name, parent_agent_span_id, parent_agent_version = _resolve_parent_agent(active)
             _stamp_agent_attribution(context._meta, parent_agent_name, parent_agent_span_id)
+            if parent_agent_version:
+                context._meta[PARENT_AGENT_VERSION] = parent_agent_version
             return context
         return None
 
@@ -2533,7 +2540,7 @@ def _activate_llmobs_span(self, span: Span) -> None:
         llmobs_parent = self._llmobs_context_provider.active()
         # Resolve the nearest agent ancestor once, at activation: O(1) one-level lookup
         # (the parent already resolved its own attribution when it activated).
-        parent_agent_name, parent_agent_span_id = _resolve_parent_agent(llmobs_parent)
+        parent_agent_name, parent_agent_span_id, parent_agent_version = _resolve_parent_agent(llmobs_parent)
         if llmobs_parent:
             parent_id = str(llmobs_parent.span_id)
             if isinstance(llmobs_parent, Span):
@@ -2624,6 +2631,8 @@ def _activate_llmobs_span(self, span: Span) -> None:
                 else sampling_decision
             ),
         )
+        if parent_agent_version:
+            span._set_ctx_item(PARENT_AGENT_VERSION, parent_agent_version)
         # Shared by reference across the trace; absent on spans whose decision came from upstream.
         span._set_ctx_item(LLMOBS_SAMPLING, sampling_state)
         # Tag the local root so the backend OTel trace processor can connect OTel gen_ai spans
@@ -2660,7 +2669,7 @@ def _start_span(
             ml_app=agent_service,
         )
         if agent_version:
-            span._set_ctx_item(AGENT_ANNOTATION, agent_version)
+            span._set_ctx_item(AGENT_VERSION, agent_version)
         if _decorator:
             _annotate_llmobs_span_data(span, tags={"decorator": "1"})
         # First session in the trace becomes the trace-level default (first-writer wins), so later
@@ -2795,8 +2804,8 @@ def agent(
         :param str ml_app: Deprecated. Use ``agent_service`` instead.
         :param str agent_service: The agent service that this span belongs to. If not provided, defaults to the
                            propagated value from a parent span/context, ``DD_LLMOBS_ML_APP``, or ``DD_SERVICE``.
-        :param str version: The version of this agent. Set as an ``agent_version`` tag on this span,
-                            and not on its child spans.
+        :param str version: The version of this agent. Set as an ``agent_version`` tag on this span
+                            and on its child spans within the same process.
 
         :returns: The Span object representing the traced operation.
         """
@@ -3056,10 +3065,14 @@ def annotate(
                         such as `{prompt,completion,total}_tokens`.
         :param agent: A dictionary declaring the agent this span represents, accepting ``version``,
                       ``name``, ``instructions``, ``model``, ``model_settings`` and ``tools``; see
-                      ``ddtrace.llmobs.Agent``. ``version`` is set as an ``agent_version`` tag and
-                      the rest as the agent's manifest, on agent spans only. All keys are optional;
-                      unreportable values are dropped, not raised, and an unset value leaves what
-                      an earlier annotation declared in place.
+                      ``ddtrace.llmobs.Agent``. ``version`` is set as an ``agent_version`` tag on
+                      the agent span and its child spans within the same process. Each span copies
+                      the version from its direct parent when it starts, so a version set after the
+                      agent has started only reaches spans started directly under the agent after
+                      this call, and their descendants. The rest is reported as the agent's
+                      manifest, on agent spans only. All keys are optional; unreportable values are
+                      dropped, not raised, and an unset value leaves what an earlier annotation
+                      declared in place.
         """
         error = None
         try:
@@ -3105,17 +3118,17 @@ def annotate(
                 agent_version = agent.get("version") if isinstance(agent, dict) else None
                 if agent_version:
                     # Stashed rather than tagged: the span kind is not resolved yet.
-                    span._set_ctx_item(AGENT_ANNOTATION, agent_version)
+                    span._set_ctx_item(AGENT_VERSION, agent_version)
                 if isinstance(agent, dict):
                     # Validated here, and unreportable or unset values dropped, so repeated
                     # annotate() calls and nested annotation_context blocks shallow-update field by
                     # field rather than overwrite. A version-only agent declares nothing.
                     declared = build_manual_agent_manifest(agent)
                     if declared:
-                        stashed = span._get_ctx_item(AGENT_DECLARATION_ANNOTATION)
+                        stashed = span._get_ctx_item(AGENT_MANIFEST)
                         if not isinstance(stashed, dict):
                             stashed = {}
-                            span._set_ctx_item(AGENT_DECLARATION_ANNOTATION, stashed)
+                            span._set_ctx_item(AGENT_MANIFEST, stashed)
                         stashed.update(declared)
             validated_cost_tags = cls._validate_cost_tags(span, cost_tags, source=_telemetry_source)
             if validated_cost_tags:
@@ -3588,7 +3601,7 @@ def _inject_llmobs_context(cls, span_context: Context, request_headers: dict[str
         # Propagate the nearest agent so spans in the downstream process attribute correctly.
         # Stamped last so the budget check sees the full tagset; degrades to id-only (or drops)
         # rather than overflowing x-datadog-tags.
-        parent_agent_name, parent_agent_span_id = _resolve_parent_agent(active_span)
+        parent_agent_name, parent_agent_span_id, _ = _resolve_parent_agent(active_span)
         _stamp_agent_attribution(span_context._meta, parent_agent_name, parent_agent_span_id)
 
     @classmethod

ddtrace/llmobs/_utils.py

@@ -23,6 +23,7 @@
 from ddtrace.internal._tagset import encode_tagset_values
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.utils.formats import format_trace_id
+from ddtrace.llmobs._constants import AGENT_VERSION
 from ddtrace.llmobs._constants import CACHE_READ_INPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import CACHE_WRITE_INPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import DEFAULT_PROMPT_NAME
@@ -46,6 +47,7 @@
 from ddtrace.llmobs._constants import ML_APP
 from ddtrace.llmobs._constants import ML_APP_DEFAULT
 from ddtrace.llmobs._constants import OUTPUT_TOKENS_METRIC_KEY
+from ddtrace.llmobs._constants import PARENT_AGENT_VERSION
 from ddtrace.llmobs._constants import PROPAGATED_PARENT_AGENT_ID_KEY
 from ddtrace.llmobs._constants import PROPAGATED_PARENT_AGENT_NAME_KEY
 from ddtrace.llmobs._constants import REASONING_OUTPUT_TOKENS_METRIC_KEY
@@ -463,38 +465,47 @@ def get_llmobs_span_kind(span: Span) -> Optional[str]:
     return kind
 
 
-def _resolve_parent_agent(active) -> tuple[Optional[str], Optional[str]]:
-    """Resolve (parent_agent_name, parent_agent_span_id) from the active LLMObs parent.
+def _resolve_parent_agent(active) -> tuple[Optional[str], Optional[str], Optional[str]]:
+    """Resolve (parent_agent_name, parent_agent_span_id, parent_agent_version) from the active LLMObs parent.
 
     active is the result of _llmobs_context_provider.active():
       - a Span whose kind is "agent": the parent IS the agent, so attribute to it.
       - any other Span: it already resolved its own attribution when it activated, so
         inherit its stored PARENT_AGENT_* values (one level of lookup, no walk).
       - a Context (distributed parent): read the propagated _dd.p.* keys off
-        context._meta. The name may be absent if an upstream hop ran an older SDK.
+        context._meta. The name may be absent if an upstream hop ran an older SDK. The
+        version is only present on in-process contexts handed to asyncio tasks and threads.
       - None: no parent, so there is no agent to attribute to.
 
-    An agent span never attributes itself: resolution always looks at the parent.
+    An agent span never attributes itself: resolution always looks at the parent. An agent without
+    its own version passes on the one it inherited.
     """
     if active is None:
-        return None, None
+        return None, None, None
 
     if isinstance(active, Span):
         # Read the meta_struct once: this runs on every span activation (hot path).
         data = _get_llmobs_data_metastruct(active)
         kind = data.get(LLMOBS_STRUCT.META, {}).get(LLMOBS_STRUCT.SPAN, {}).get(LLMOBS_STRUCT.KIND)
         if kind == "agent":
-            return (data.get(LLMOBS_STRUCT.NAME) or active.name, str(active.span_id))
+            version = active._get_ctx_item(AGENT_VERSION) or active._get_ctx_item(PARENT_AGENT_VERSION)
+            return (
+                data.get(LLMOBS_STRUCT.NAME) or active.name,
+                str(active.span_id),
+                str(version) if version else None,
+            )
         return (
             data.get(LLMOBS_STRUCT.PARENT_AGENT_NAME),
             data.get(LLMOBS_STRUCT.PARENT_AGENT_SPAN_ID),
+            active._get_ctx_item(PARENT_AGENT_VERSION),
         )
 
     # Context parent (distributed). Keys land on context._meta via _dd.p.* propagation.
     ctx = active
     return (
         ctx._meta.get(PROPAGATED_PARENT_AGENT_NAME_KEY),
         ctx._meta.get(PROPAGATED_PARENT_AGENT_ID_KEY),
+        ctx._meta.get(PARENT_AGENT_VERSION),
     )
 
 

ddtrace/llmobs/types.py

@@ -250,8 +250,9 @@ class Agent(TypedDict, total=False):
             extra_headers, since those can carry secrets.
         tools: list[AgentTool] - the tools the agent declares it can call.
 
-    ``version`` becomes an ``agent_version`` tag and the rest the agent's manifest, on agent spans
-    only. Declared through ``annotation_context``, both reach every agent span in the block.
+    ``version`` becomes an ``agent_version`` tag on the agent span and its child spans within the
+    same process. The rest becomes the agent's manifest, on agent spans only. Declared through
+    ``annotation_context``, both reach every agent span in the block.
     Unreportable values are dropped rather than raising, and a key whose value is unset (``None``
     or empty) declares nothing rather than erasing what an earlier annotation declared. Each
     annotation shallow-updates the manifest key by key, including one an integration already

releasenotes/notes/llmobs-agent-version-subtree-f2321e0e547b39e8.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    LLM Observability: The agent version set through ``LLMObs.agent(version=...)``, ``@agent(version=...)``,
+    ``LLMObs.annotate(agent={"version": ...})`` or ``LLMObs.annotation_context(agent={"version": ...})`` is now
+    also reported as an ``agent_version`` tag on child spans of that agent within the same process.

tests/llmobs/test_llmobs_decorators.py

@@ -1130,7 +1130,7 @@ def add(self, a: int, b: int) -> int:
     assert input_value == {"a": 1, "b": 2}
 
 
-def test_agent_decorator_sets_agent_tags_on_agent_span_only(llmobs, test_spans):
+def test_agent_decorator_sets_agent_version_on_subtree(llmobs, test_spans):
     @agent(version="v3")
     def my_agent():
         with llmobs.tool(name="test_tool"):
@@ -1140,7 +1140,7 @@ def my_agent():
     spans = {s.name: s for trace in test_spans.pop_traces() for s in trace if get_llmobs_span_kind(s)}
     assert set(spans) == {"my_agent", "test_tool"}
     assert get_llmobs_tags(spans["my_agent"])["agent_version"] == "v3"
-    assert "agent_version" not in get_llmobs_tags(spans["test_tool"])
+    assert get_llmobs_tags(spans["test_tool"])["agent_version"] == "v3"
 
 
 @pytest.mark.parametrize(