diff --git a/ddtrace/llmobs/_constants.py b/ddtrace/llmobs/_constants.py index 02c20701346..17fb56478f2 100644 --- a/ddtrace/llmobs/_constants.py +++ b/ddtrace/llmobs/_constants.py @@ -152,13 +152,16 @@ class LLMObsSamplingDecision(str, Enum): INSTRUMENTATION_METHOD_AUTO = "auto" INSTRUMENTATION_METHOD_ANNOTATED = "annotated" -# Agent tracking tag. Set on agent spans only, at span finish. +# Agent tracking tag set at span finish. AGENT_VERSION_TAG_KEY = "agent_version" # Holds the version an annotation supplied, until the span kind is known at finish. -AGENT_ANNOTATION = "_ml_obs.agent_annotation" +AGENT_VERSION = "_ml_obs.agent_annotation" +# Holds the version of the nearest agent ancestor, resolved at activation. Also carried on the +# in-process context handed to asyncio tasks and threads. It is never propagated across services. +PARENT_AGENT_VERSION = "_ml_obs.parent_agent_version" # Holds the manifest the annotations declared, for the same reason. Each annotation is validated # and shallow-merged into it as it runs. -AGENT_DECLARATION_ANNOTATION = "_ml_obs.agent_declaration_annotation" +AGENT_MANIFEST = "_ml_obs.agent_declaration_annotation" DISPATCH_ON_TOOL_CALL_OUTPUT_USED = "on_tool_call_output_used" DISPATCH_ON_LLM_TOOL_CHOICE = "on_llm_tool_choice" diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index 3b725baa495..3db29d182c0 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -48,8 +48,8 @@ from ddtrace.internal.utils.formats import format_trace_id from ddtrace.internal.utils.formats import parse_tags_str from ddtrace.llmobs import _telemetry as telemetry -from ddtrace.llmobs._constants import AGENT_ANNOTATION -from ddtrace.llmobs._constants import AGENT_DECLARATION_ANNOTATION +from ddtrace.llmobs._constants import AGENT_MANIFEST +from ddtrace.llmobs._constants import AGENT_VERSION from ddtrace.llmobs._constants import AGENT_VERSION_TAG_KEY from ddtrace.llmobs._constants import ANNOTATIONS_CONTEXT_ID from ddtrace.llmobs._constants import CACHED_LLMOBS_EVENT_CTX_KEY @@ -81,6 +81,7 @@ from ddtrace.llmobs._constants import LLMOBS_SAMPLING from ddtrace.llmobs._constants import LLMOBS_STRUCT from ddtrace.llmobs._constants import ML_APP +from ddtrace.llmobs._constants import PARENT_AGENT_VERSION from ddtrace.llmobs._constants import PROMPT_TRACKING_INSTRUMENTATION_METHOD from ddtrace.llmobs._constants import PROPAGATED_LLMOBS_TRACE_ID_KEY from ddtrace.llmobs._constants import PROPAGATED_ML_APP_KEY @@ -721,12 +722,12 @@ def _prepare_llmobs_span_data(self, span: Span, span_kind: Optional[str]) -> boo return False # Agent annotations are applied here, where the span kind is known: annotation_context - # reaches every span in its block, but only agent spans carry the tags. + # reaches every span in its block, but only agent spans carry the manifest. if span_kind == "agent": - agent_annotation = span._get_ctx_item(AGENT_ANNOTATION) - if agent_annotation: - llmobs_data.setdefault(LLMOBS_STRUCT.TAGS, {})[AGENT_VERSION_TAG_KEY] = str(agent_annotation) - declared_manifest = span._get_ctx_item(AGENT_DECLARATION_ANNOTATION) + agent_version = span._get_ctx_item(AGENT_VERSION) or span._get_ctx_item(PARENT_AGENT_VERSION) + if agent_version: + llmobs_data.setdefault(LLMOBS_STRUCT.TAGS, {})[AGENT_VERSION_TAG_KEY] = str(agent_version) + declared_manifest = span._get_ctx_item(AGENT_MANIFEST) if declared_manifest: # Both levels are type-checked because caller metadata is not sanitized until # _normalize_llmobs_meta runs below, so a forged `_dd` is still raw here. A @@ -758,6 +759,10 @@ def _prepare_llmobs_span_data(self, span: Span, span_kind: Optional[str]) -> boo if not merged.get("name"): merged["name"] = get_llmobs_span_name(span) or span.name _annotate_llmobs_span_data(span, agent_manifest=merged) + else: + parent_agent_version = span._get_ctx_item(PARENT_AGENT_VERSION) + if parent_agent_version: + llmobs_data.setdefault(LLMOBS_STRUCT.TAGS, {})[AGENT_VERSION_TAG_KEY] = parent_agent_version llmobs_meta = llmobs_data.setdefault(LLMOBS_STRUCT.META, _Meta()) # Before the user processor and _normalize_llmobs_meta, either of which can strip values @@ -1985,9 +1990,9 @@ def annotation_context( :param agent: A dictionary declaring the agent running in this context, accepting ``version``, ``name``, ``instructions``, ``model``, ``model_settings`` and ``tools``; see ``ddtrace.llmobs.Agent``. ``version`` is set as an - ``agent_version`` tag and the rest as the agent's manifest, on every agent - span in the block. All keys are optional; unreportable values are dropped, - not raised. + ``agent_version`` tag on every agent span in the block and its child spans, + and the rest as the agent's manifest on every agent span in the block. All + keys are optional; unreportable values are dropped, not raised. """ # id to track an annotation for registering / de-registering annotation_id = rand64bits() @@ -2507,8 +2512,10 @@ def _current_trace_context(self) -> Optional[Context]: # Carry the nearest agent onto the context so spans created in in-process task # boundaries (asyncio tasks, thread-pool executors) still attribute to it. # Stamped last so the budget check sees the full tagset. - parent_agent_name, parent_agent_span_id = _resolve_parent_agent(active) + parent_agent_name, parent_agent_span_id, parent_agent_version = _resolve_parent_agent(active) _stamp_agent_attribution(context._meta, parent_agent_name, parent_agent_span_id) + if parent_agent_version: + context._meta[PARENT_AGENT_VERSION] = parent_agent_version return context return None @@ -2533,7 +2540,7 @@ def _activate_llmobs_span(self, span: Span) -> None: llmobs_parent = self._llmobs_context_provider.active() # Resolve the nearest agent ancestor once, at activation: O(1) one-level lookup # (the parent already resolved its own attribution when it activated). - parent_agent_name, parent_agent_span_id = _resolve_parent_agent(llmobs_parent) + parent_agent_name, parent_agent_span_id, parent_agent_version = _resolve_parent_agent(llmobs_parent) if llmobs_parent: parent_id = str(llmobs_parent.span_id) if isinstance(llmobs_parent, Span): @@ -2624,6 +2631,8 @@ def _activate_llmobs_span(self, span: Span) -> None: else sampling_decision ), ) + if parent_agent_version: + span._set_ctx_item(PARENT_AGENT_VERSION, parent_agent_version) # Shared by reference across the trace; absent on spans whose decision came from upstream. span._set_ctx_item(LLMOBS_SAMPLING, sampling_state) # Tag the local root so the backend OTel trace processor can connect OTel gen_ai spans @@ -2660,7 +2669,7 @@ def _start_span( ml_app=agent_service, ) if agent_version: - span._set_ctx_item(AGENT_ANNOTATION, agent_version) + span._set_ctx_item(AGENT_VERSION, agent_version) if _decorator: _annotate_llmobs_span_data(span, tags={"decorator": "1"}) # First session in the trace becomes the trace-level default (first-writer wins), so later @@ -2795,8 +2804,8 @@ def agent( :param str ml_app: Deprecated. Use ``agent_service`` instead. :param str agent_service: The agent service that this span belongs to. If not provided, defaults to the propagated value from a parent span/context, ``DD_LLMOBS_ML_APP``, or ``DD_SERVICE``. - :param str version: The version of this agent. Set as an ``agent_version`` tag on this span, - and not on its child spans. + :param str version: The version of this agent. Set as an ``agent_version`` tag on this span + and on its child spans within the same process. :returns: The Span object representing the traced operation. """ @@ -3056,10 +3065,14 @@ def annotate( such as `{prompt,completion,total}_tokens`. :param agent: A dictionary declaring the agent this span represents, accepting ``version``, ``name``, ``instructions``, ``model``, ``model_settings`` and ``tools``; see - ``ddtrace.llmobs.Agent``. ``version`` is set as an ``agent_version`` tag and - the rest as the agent's manifest, on agent spans only. All keys are optional; - unreportable values are dropped, not raised, and an unset value leaves what - an earlier annotation declared in place. + ``ddtrace.llmobs.Agent``. ``version`` is set as an ``agent_version`` tag on + the agent span and its child spans within the same process. Each span copies + the version from its direct parent when it starts, so a version set after the + agent has started only reaches spans started directly under the agent after + this call, and their descendants. The rest is reported as the agent's + manifest, on agent spans only. All keys are optional; unreportable values are + dropped, not raised, and an unset value leaves what an earlier annotation + declared in place. """ error = None try: @@ -3105,17 +3118,17 @@ def annotate( agent_version = agent.get("version") if isinstance(agent, dict) else None if agent_version: # Stashed rather than tagged: the span kind is not resolved yet. - span._set_ctx_item(AGENT_ANNOTATION, agent_version) + span._set_ctx_item(AGENT_VERSION, agent_version) if isinstance(agent, dict): # Validated here, and unreportable or unset values dropped, so repeated # annotate() calls and nested annotation_context blocks shallow-update field by # field rather than overwrite. A version-only agent declares nothing. declared = build_manual_agent_manifest(agent) if declared: - stashed = span._get_ctx_item(AGENT_DECLARATION_ANNOTATION) + stashed = span._get_ctx_item(AGENT_MANIFEST) if not isinstance(stashed, dict): stashed = {} - span._set_ctx_item(AGENT_DECLARATION_ANNOTATION, stashed) + span._set_ctx_item(AGENT_MANIFEST, stashed) stashed.update(declared) validated_cost_tags = cls._validate_cost_tags(span, cost_tags, source=_telemetry_source) if validated_cost_tags: @@ -3588,7 +3601,7 @@ def _inject_llmobs_context(cls, span_context: Context, request_headers: dict[str # Propagate the nearest agent so spans in the downstream process attribute correctly. # Stamped last so the budget check sees the full tagset; degrades to id-only (or drops) # rather than overflowing x-datadog-tags. - parent_agent_name, parent_agent_span_id = _resolve_parent_agent(active_span) + parent_agent_name, parent_agent_span_id, _ = _resolve_parent_agent(active_span) _stamp_agent_attribution(span_context._meta, parent_agent_name, parent_agent_span_id) @classmethod diff --git a/ddtrace/llmobs/_utils.py b/ddtrace/llmobs/_utils.py index 1b49d0ce1e8..5cffb62b09c 100644 --- a/ddtrace/llmobs/_utils.py +++ b/ddtrace/llmobs/_utils.py @@ -23,6 +23,7 @@ from ddtrace.internal._tagset import encode_tagset_values from ddtrace.internal.logger import get_logger from ddtrace.internal.utils.formats import format_trace_id +from ddtrace.llmobs._constants import AGENT_VERSION from ddtrace.llmobs._constants import CACHE_READ_INPUT_TOKENS_METRIC_KEY from ddtrace.llmobs._constants import CACHE_WRITE_INPUT_TOKENS_METRIC_KEY from ddtrace.llmobs._constants import DEFAULT_PROMPT_NAME @@ -46,6 +47,7 @@ from ddtrace.llmobs._constants import ML_APP from ddtrace.llmobs._constants import ML_APP_DEFAULT from ddtrace.llmobs._constants import OUTPUT_TOKENS_METRIC_KEY +from ddtrace.llmobs._constants import PARENT_AGENT_VERSION from ddtrace.llmobs._constants import PROPAGATED_PARENT_AGENT_ID_KEY from ddtrace.llmobs._constants import PROPAGATED_PARENT_AGENT_NAME_KEY from ddtrace.llmobs._constants import REASONING_OUTPUT_TOKENS_METRIC_KEY @@ -463,31 +465,39 @@ def get_llmobs_span_kind(span: Span) -> Optional[str]: return kind -def _resolve_parent_agent(active) -> tuple[Optional[str], Optional[str]]: - """Resolve (parent_agent_name, parent_agent_span_id) from the active LLMObs parent. +def _resolve_parent_agent(active) -> tuple[Optional[str], Optional[str], Optional[str]]: + """Resolve (parent_agent_name, parent_agent_span_id, parent_agent_version) from the active LLMObs parent. active is the result of _llmobs_context_provider.active(): - a Span whose kind is "agent": the parent IS the agent, so attribute to it. - any other Span: it already resolved its own attribution when it activated, so inherit its stored PARENT_AGENT_* values (one level of lookup, no walk). - a Context (distributed parent): read the propagated _dd.p.* keys off - context._meta. The name may be absent if an upstream hop ran an older SDK. + context._meta. The name may be absent if an upstream hop ran an older SDK. The + version is only present on in-process contexts handed to asyncio tasks and threads. - None: no parent, so there is no agent to attribute to. - An agent span never attributes itself: resolution always looks at the parent. + An agent span never attributes itself: resolution always looks at the parent. An agent without + its own version passes on the one it inherited. """ if active is None: - return None, None + return None, None, None if isinstance(active, Span): # Read the meta_struct once: this runs on every span activation (hot path). data = _get_llmobs_data_metastruct(active) kind = data.get(LLMOBS_STRUCT.META, {}).get(LLMOBS_STRUCT.SPAN, {}).get(LLMOBS_STRUCT.KIND) if kind == "agent": - return (data.get(LLMOBS_STRUCT.NAME) or active.name, str(active.span_id)) + version = active._get_ctx_item(AGENT_VERSION) or active._get_ctx_item(PARENT_AGENT_VERSION) + return ( + data.get(LLMOBS_STRUCT.NAME) or active.name, + str(active.span_id), + str(version) if version else None, + ) return ( data.get(LLMOBS_STRUCT.PARENT_AGENT_NAME), data.get(LLMOBS_STRUCT.PARENT_AGENT_SPAN_ID), + active._get_ctx_item(PARENT_AGENT_VERSION), ) # Context parent (distributed). Keys land on context._meta via _dd.p.* propagation. @@ -495,6 +505,7 @@ def _resolve_parent_agent(active) -> tuple[Optional[str], Optional[str]]: return ( ctx._meta.get(PROPAGATED_PARENT_AGENT_NAME_KEY), ctx._meta.get(PROPAGATED_PARENT_AGENT_ID_KEY), + ctx._meta.get(PARENT_AGENT_VERSION), ) diff --git a/ddtrace/llmobs/types.py b/ddtrace/llmobs/types.py index d7e61075ac6..36e776cb806 100644 --- a/ddtrace/llmobs/types.py +++ b/ddtrace/llmobs/types.py @@ -250,8 +250,9 @@ class Agent(TypedDict, total=False): extra_headers, since those can carry secrets. tools: list[AgentTool] - the tools the agent declares it can call. - ``version`` becomes an ``agent_version`` tag and the rest the agent's manifest, on agent spans - only. Declared through ``annotation_context``, both reach every agent span in the block. + ``version`` becomes an ``agent_version`` tag on the agent span and its child spans within the + same process. The rest becomes the agent's manifest, on agent spans only. Declared through + ``annotation_context``, both reach every agent span in the block. Unreportable values are dropped rather than raising, and a key whose value is unset (``None`` or empty) declares nothing rather than erasing what an earlier annotation declared. Each annotation shallow-updates the manifest key by key, including one an integration already diff --git a/releasenotes/notes/llmobs-agent-version-subtree-f2321e0e547b39e8.yaml b/releasenotes/notes/llmobs-agent-version-subtree-f2321e0e547b39e8.yaml new file mode 100644 index 00000000000..eee218a2747 --- /dev/null +++ b/releasenotes/notes/llmobs-agent-version-subtree-f2321e0e547b39e8.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + LLM Observability: The agent version set through ``LLMObs.agent(version=...)``, ``@agent(version=...)``, + ``LLMObs.annotate(agent={"version": ...})`` or ``LLMObs.annotation_context(agent={"version": ...})`` is now + also reported as an ``agent_version`` tag on child spans of that agent within the same process. diff --git a/tests/llmobs/test_llmobs_decorators.py b/tests/llmobs/test_llmobs_decorators.py index 371b9e43f97..3c470be2762 100644 --- a/tests/llmobs/test_llmobs_decorators.py +++ b/tests/llmobs/test_llmobs_decorators.py @@ -1130,7 +1130,7 @@ def add(self, a: int, b: int) -> int: assert input_value == {"a": 1, "b": 2} -def test_agent_decorator_sets_agent_tags_on_agent_span_only(llmobs, test_spans): +def test_agent_decorator_sets_agent_version_on_subtree(llmobs, test_spans): @agent(version="v3") def my_agent(): with llmobs.tool(name="test_tool"): @@ -1140,7 +1140,7 @@ def my_agent(): spans = {s.name: s for trace in test_spans.pop_traces() for s in trace if get_llmobs_span_kind(s)} assert set(spans) == {"my_agent", "test_tool"} assert get_llmobs_tags(spans["my_agent"])["agent_version"] == "v3" - assert "agent_version" not in get_llmobs_tags(spans["test_tool"]) + assert get_llmobs_tags(spans["test_tool"])["agent_version"] == "v3" @pytest.mark.parametrize( diff --git a/tests/llmobs/test_llmobs_service.py b/tests/llmobs/test_llmobs_service.py index b229a22d753..faecffca4d9 100644 --- a/tests/llmobs/test_llmobs_service.py +++ b/tests/llmobs/test_llmobs_service.py @@ -1993,18 +1993,39 @@ def test_agent_span_sets_agent_version_tag(llmobs): assert get_llmobs_tags(span)["agent_version"] == "v3" -def test_agent_span_version_not_set_on_children(llmobs): - """The version identifies the agent, so it stays on the agent span.""" +def test_agent_span_version_set_on_descendants(llmobs): with llmobs.agent(name="test_agent", version="v3"): with llmobs.workflow(name="test_workflow") as workflow_span: with llmobs.llm(name="test_llm", model_name="test") as llm_span: pass - with llmobs.tool(name="test_tool") as tool_span: - pass + with llmobs.tool(name="test_tool") as tool_span: + pass for span in (workflow_span, llm_span, tool_span): + assert get_llmobs_tags(span)["agent_version"] == "v3" + + +def test_agent_version_not_set_outside_the_agent(llmobs): + with llmobs.workflow(name="root") as root_span: + with llmobs.agent(name="test_agent", version="v3"): + pass + with llmobs.tool(name="sibling") as sibling_span: + pass + for span in (root_span, sibling_span): assert "agent_version" not in get_llmobs_tags(span) +def test_sibling_agents_version_their_own_subtrees(llmobs): + with llmobs.workflow(name="root"): + with llmobs.agent(name="flight_agent", version="v1"): + with llmobs.tool(name="flight_tool") as flight_tool: + pass + with llmobs.agent(name="hotel_agent", version="v2"): + with llmobs.tool(name="hotel_tool") as hotel_tool: + pass + assert get_llmobs_tags(flight_tool)["agent_version"] == "v1" + assert get_llmobs_tags(hotel_tool)["agent_version"] == "v2" + + def test_agent_span_without_version_sets_no_tag(llmobs): with llmobs.agent(name="test_agent") as span: pass @@ -2019,25 +2040,57 @@ def test_nested_agent_spans_each_carry_their_own_version(llmobs): assert get_llmobs_tags(inner_span)["agent_version"] == "v3" -def test_nested_agent_span_does_not_inherit_ancestor_version(llmobs): - """An unversioned sub-agent stays unversioned rather than claiming its parent's version.""" +def test_nested_agent_without_version_inherits_ancestor_version(llmobs): with llmobs.agent(name="outer_agent", version="v1"): with llmobs.agent(name="inner_agent") as inner_span: + with llmobs.llm(name="inner_llm", model_name="test") as inner_llm: + pass + assert get_llmobs_tags(inner_span)["agent_version"] == "v1" + assert get_llmobs_tags(inner_llm)["agent_version"] == "v1" + + +def test_nested_agent_version_scoped_to_its_subtree(llmobs): + with llmobs.agent(name="outer_agent", version="v1"): + with llmobs.agent(name="inner_agent", version="v2"): + with llmobs.llm(name="inner_llm", model_name="test") as inner_llm: + pass + with llmobs.llm(name="outer_llm", model_name="test") as outer_llm: pass - assert "agent_version" not in get_llmobs_tags(inner_span) + assert get_llmobs_tags(inner_llm)["agent_version"] == "v2" + assert get_llmobs_tags(outer_llm)["agent_version"] == "v1" + + +def test_inherited_agent_version_wins_over_explicit_tag(llmobs): + with llmobs.agent(name="test_agent", version="v3"): + with llmobs.tool(name="test_tool") as tool_span: + llmobs.annotate(span=tool_span, tags={"agent_version": "from_tags"}) + assert get_llmobs_tags(tool_span)["agent_version"] == "v3" -def test_annotation_context_sets_agent_tags_on_agent_span_only(llmobs): +def test_annotation_context_agent_version_reaches_agent_subtrees_only(llmobs): with llmobs.annotation_context(agent={"version": "v3"}): - with llmobs.agent(name="test_agent") as agent_span: - with llmobs.llm(name="test_llm", model_name="test") as llm_span: - pass + with llmobs.workflow(name="test_workflow") as workflow_span: + with llmobs.agent(name="test_agent") as agent_span: + with llmobs.llm(name="test_llm", model_name="test") as llm_span: + pass assert get_llmobs_tags(agent_span)["agent_version"] == "v3" - assert "agent_version" not in get_llmobs_tags(llm_span) + assert get_llmobs_tags(llm_span)["agent_version"] == "v3" + assert "agent_version" not in get_llmobs_tags(workflow_span) + + +def test_agent_version_option_wins_over_annotation_context(llmobs): + with llmobs.annotation_context(agent={"version": "from_context"}): + with llmobs.agent(name="test_agent", version="from_option") as agent_span: + with llmobs.tool(name="test_tool") as tool_span: + pass + for span in (agent_span, tool_span): + assert get_llmobs_tags(span)["agent_version"] == "from_option" def test_user_supplied_agent_version_tag_is_left_alone(llmobs): - """`tags` is an arbitrary user namespace, so an app already using this key keeps it.""" + """`tags` is an arbitrary user namespace, so an app already using this key keeps it when no + versioned agent is above the span. + """ with llmobs.annotation_context(tags={"agent_version": "mine-v1"}): with llmobs.workflow(name="test_workflow") as workflow_span: pass diff --git a/tests/llmobs/test_propagation.py b/tests/llmobs/test_propagation.py index e6171064f36..9b0f61acb6d 100644 --- a/tests/llmobs/test_propagation.py +++ b/tests/llmobs/test_propagation.py @@ -1032,6 +1032,46 @@ async def child(): } +def test_agent_version_propagates_across_asyncio_task(llmobs, llmobs_events, patched_asyncio): + import asyncio + + async def main(): + with llmobs.agent(name="my_agent", version="v3"): + + async def child(): + with llmobs.tool(name="async_tool"): + pass + + await asyncio.create_task(child()) + + asyncio.run(main()) + matches = [e for e in llmobs_events if e["name"] == "async_tool"] + assert len(matches) == 1 + assert "agent_version:v3" in matches[0]["tags"] + + +def test_agent_version_propagates_across_thread_pool(llmobs, llmobs_events, patched_futures): + import concurrent.futures + + def child(): + with llmobs.tool(name="thread_tool"): + pass + + with llmobs.agent(name="my_agent", version="v3"): + with concurrent.futures.ThreadPoolExecutor() as executor: + executor.submit(child).result() + matches = [e for e in llmobs_events if e["name"] == "thread_tool"] + assert len(matches) == 1 + assert "agent_version:v3" in matches[0]["tags"] + + +def test_agent_version_not_propagated_across_services(llmobs): + with llmobs.agent(name="my_agent", version="v3"): + with llmobs.tool(name="my_tool") as tool_span: + headers = llmobs.inject_distributed_headers({}, span=tool_span) + assert "v3" not in json.dumps(headers) + + def test_inject_no_stale_agent_attribution_after_agent_finishes(llmobs): """A sibling with no agent ancestor must not inherit the agent that injected before it.