Fix DD_APM_TRACING_ENABLED to work with LLMObs

dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java

@@ -30,6 +30,11 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) {
       return;
     }
 
+    if (!config.isTraceEnabled()) {
+      LOGGER.debug("LLM Observability is disabled: tracing is disabled");
+      return;
+    }
+
     sco.createRemaining(config);
 
     String mlApp = config.getLlmObsMlApp();

dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java

@@ -5,7 +5,9 @@
 import datadog.trace.api.DDSpanTypes;
 import datadog.trace.api.DDTraceApiInfo;
 import datadog.trace.api.DDTraceId;
+import datadog.trace.api.ProductTraceSource;
 import datadog.trace.api.WellKnownTags;
+import datadog.trace.api.internal.TraceSegment;
 import datadog.trace.api.llmobs.LLMObs;
 import datadog.trace.api.llmobs.LLMObsContext;
 import datadog.trace.api.llmobs.LLMObsSpan;
@@ -110,6 +112,12 @@ public DDLLMObsSpan(
     }
     span.setTag(LLMOBS_TAG_PREFIX + PARENT_ID_TAG_INTERNAL, parentSpanID);
     scope = LLMObsContext.attach(span.context());
+
+    // Mark this span as originating from LLM Observability product
+    TraceSegment segment = AgentTracer.get().getTraceSegment();
+    if (segment != null) {
+      segment.setTagTop(Tags.PROPAGATED_TRACE_SOURCE, ProductTraceSource.LLMOBS);
+    }
   }
 
   @Override

dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/LLMObsSystemTest.groovy

@@ -0,0 +1,53 @@
+package datadog.trace.llmobs
+
+import datadog.communication.ddagent.SharedCommunicationObjects
+import datadog.trace.test.util.DDSpecification
+import okhttp3.HttpUrl
+
+class LLMObsSystemTest extends DDSpecification {
+
+  void 'start disabled when llmobs is disabled'() {
+    setup:
+    injectSysConfig('llmobs.enabled', 'false')
+    rebuildConfig()
+    final inst = Mock(java.lang.instrument.Instrumentation)
+    final sco = Mock(SharedCommunicationObjects)
+
+    when:
+    LLMObsSystem.start(inst, sco)
+
+    then:
+    0 * sco._
+  }
+
+  void 'start disabled when trace is disabled'() {
+    setup:
+    injectSysConfig('llmobs.enabled', 'true')
+    injectSysConfig('trace.enabled', 'false')
+    rebuildConfig()
+    final inst = Mock(java.lang.instrument.Instrumentation)
+    final sco = Mock(SharedCommunicationObjects)
+
+    when:
+    LLMObsSystem.start(inst, sco)
+
+    then:
+    0 * sco._
+  }
+
+  void 'start enabled when apm tracing disabled but llmobs enabled'() {
+    setup:
+    injectSysConfig('llmobs.enabled', 'true')
+    injectSysConfig('apm.tracing.enabled', 'false')
+    rebuildConfig()
+    final inst = Mock(java.lang.instrument.Instrumentation)
+    final sco = Mock(SharedCommunicationObjects)
+    sco.agentUrl = HttpUrl.parse('http://localhost:8126')
+
+    when:
+    LLMObsSystem.start(inst, sco)
+
+    then:
+    1 * sco.createRemaining(_)
+  }
+}

dd-smoke-tests/apm-tracing-disabled/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java

@@ -1,5 +1,7 @@
 package datadog.smoketest.apmtracingdisabled;
 
+import datadog.trace.api.llmobs.LLMObs;
+import datadog.trace.api.llmobs.LLMObsSpan;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import io.opentracing.Span;
 import io.opentracing.util.GlobalTracer;
@@ -73,6 +75,17 @@ public void write(
     }
   }
 
+  @GetMapping("/llmobs/test")
+  public String llmobsTest() {
+    // Create LLMObs span using public API
+    LLMObsSpan llmSpan =
+        LLMObs.startLLMSpan("llmobs-test-operation", "gpt-4", "openai", null, null);
+    llmSpan.annotateIO("test input", "test output");
+    llmSpan.finish();
+
+    return "LLMObs test completed";
+  }
+
   private String forceKeepSpan() {
     final Span span = GlobalTracer.get().activeSpan();
     if (span != null) {

dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmDisabledSmokeTest.groovy

@@ -0,0 +1,92 @@
+package datadog.smoketest.apmtracingdisabled
+
+import datadog.trace.api.sampling.PrioritySampling
+import okhttp3.Request
+
+class LlmObsApmDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest {
+
+  static final String LLMOBS_SERVICE_NAME = "llmobs-apm-disabled-test"
+
+  static final String[] LLMOBS_APM_DISABLED_PROPERTIES = [
+    "-Ddd.apm.tracing.enabled=false",
+    "-Ddd.llmobs.enabled=true",
+    "-Ddd.llmobs.ml-app=test-app",
+    "-Ddd.service.name=${LLMOBS_SERVICE_NAME}",
+  ]
+
+  @Override
+  ProcessBuilder createProcessBuilder() {
+    return createProcess(LLMOBS_APM_DISABLED_PROPERTIES)
+  }
+
+  void 'When APM disabled and LLMObs enabled, LLMObs spans should be kept and APM spans should be dropped'() {
+    setup:
+    final llmobsUrl = "http://localhost:${httpPort}/rest-api/llmobs/test"
+    final llmobsRequest = new Request.Builder().url(llmobsUrl).get().build()
+
+    final apmUrl = "http://localhost:${httpPort}/rest-api/greetings"
+    final apmRequest = new Request.Builder().url(apmUrl).get().build()
+
+    when: "Create LLMObs span"
+    final llmobsResponse = client.newCall(llmobsRequest).execute()
+
+    then: "LLMObs request should succeed"
+    llmobsResponse.successful
+
+    when: "Create regular APM span"
+    final apmResponse = client.newCall(apmRequest).execute()
+
+    then: "APM request should succeed"
+    apmResponse.successful
+
+    and: "Wait for traces"
+    waitForTraceCount(2)
+
+    and: "LLMObs trace should be kept (SAMPLER_KEEP)"
+    def llmobsTrace = traces.find { trace ->
+      trace.spans.find { span ->
+        span.meta["http.url"] == llmobsUrl
+      }
+    }
+    assert llmobsTrace != null
+    // The LLMObs child span should have LLMObs tags
+    def llmobsChildSpan = llmobsTrace.spans.find { span ->
+      span.meta["_ml_obs_tag.model_name"] == "gpt-4"
+    }
+    assert llmobsChildSpan != null : "LLMObs child span with model_name=gpt-4 should exist"
+
+    and: "Regular APM trace should be dropped (SAMPLER_DROP)"
+    def apmTrace = traces.find { trace ->
+      trace.spans.find { span ->
+        span.meta["http.url"] == apmUrl
+      }
+    }
+    assert apmTrace != null
+    checkRootSpanPrioritySampling(apmTrace, PrioritySampling.SAMPLER_DROP)
+
+    and: "No NPE or errors in logs"
+    !isLogPresent { it.contains("NullPointerException") }
+    !isLogPresent { it.contains("ERROR") }
+  }
+
+  void 'LLMObs spans should have PROPAGATED_TRACE_SOURCE tag set'() {
+    setup:
+    final llmobsUrl = "http://localhost:${httpPort}/rest-api/llmobs/test"
+    final llmobsRequest = new Request.Builder().url(llmobsUrl).get().build()
+
+    when:
+    final response = client.newCall(llmobsRequest).execute()
+
+    then:
+    response.successful
+    waitForTraceCount(1)
+
+    and: "LLMObs span should be created successfully"
+    def trace = traces[0]
+    assert trace != null
+    def llmobsSpan = trace.spans.find { span ->
+      span.meta["_ml_obs_tag.model_name"] == "gpt-4"
+    }
+    assert llmobsSpan != null : "LLMObs span with model_name should exist"
+  }
+}

dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy

@@ -0,0 +1,34 @@
+package datadog.smoketest.apmtracingdisabled
+
+import okhttp3.Request
+
+class LlmObsTraceDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest {
+
+  static final String[] LLMOBS_TRACE_DISABLED_PROPERTIES = [
+    "-Ddd.trace.enabled=false",
+    "-Ddd.llmobs.enabled=true",
+    "-Ddd.llmobs.ml-app=test-app",
+    "-Ddd.service.name=llmobs-trace-disabled-test",
+  ]
+
+  @Override
+  ProcessBuilder createProcessBuilder() {
+    return createProcess(LLMOBS_TRACE_DISABLED_PROPERTIES)
+  }
+
+  void 'DD_TRACE_ENABLED=false with DD_LLMOBS_ENABLED=true should disable LLMObs gracefully'() {
+    setup:
+    final llmobsUrl = "http://localhost:${httpPort}/rest-api/llmobs/test"
+    final llmobsRequest = new Request.Builder().url(llmobsUrl).get().build()
+
+    when: "Call LLMObs endpoint"
+    final response = client.newCall(llmobsRequest).execute()
+
+    then: "Request should succeed"
+    response.successful
+    response.code() == 200
+
+    and: "LLMObs disabled message in logs"
+    isLogPresent { it.contains("LLM Observability is disabled: tracing is disabled") }
+  }
+}

dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java

@@ -12,6 +12,7 @@
 import datadog.trace.api.sampling.SamplingRule;
 import datadog.trace.core.CoreSpan;
 import java.time.Clock;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -36,9 +37,17 @@ final class Builder {
     public static Sampler forConfig(final Config config, final TraceConfig traceConfig) {
       Sampler sampler;
       if (config != null) {
-        if (!config.isApmTracingEnabled() && isAsmEnabled(config)) {
-          log.debug("APM is disabled. Only 1 trace per minute will be sent.");
-          return new AsmStandaloneSampler(Clock.systemUTC());
+        if (!config.isApmTracingEnabled()) {
+          List<StandaloneProduct> active = new ArrayList<>();
+          if (config.isLlmObsEnabled()) active.add(StandaloneProduct.LLMOBS);
+          if (isAsmEnabled(config)) active.add(StandaloneProduct.ASM);
+          if (active.isEmpty()) {
+            log.debug("APM is disabled. All APM traces will be dropped.");
+            return new ForcePrioritySampler(
+                PrioritySampling.SAMPLER_DROP, SamplingMechanism.DEFAULT);
+          }
+          log.debug("APM is disabled, standalone products active: {}.", active);
+          return new StandaloneSampler(active, Clock.systemUTC());
         }
         final Map<String, String> serviceRules = config.getTraceSamplingServiceRules();
         final Map<String, String> operationRules = config.getTraceSamplingOperationRules();

dd-trace-core/src/main/java/datadog/trace/common/sampling/StandaloneProduct.java

@@ -0,0 +1,52 @@
+package datadog.trace.common.sampling;
+
+import datadog.trace.api.ProductTraceSource;
+import datadog.trace.api.sampling.SamplingMechanism;
+
+/**
+ * Represents a standalone product that can function when APM tracing is disabled. Each product
+ * defines which traces to keep, which sampling mechanism to report, and whether it requires a
+ * 1-per-minute billing trace for service catalog / billing purposes.
+ *
+ * <p>To add a new standalone product:
+ *
+ * <ol>
+ *   <li>Add an enum entry here.
+ *   <li>Add one line in {@link Sampler.Builder#forConfig}.
+ *   <li>Update {@code ProductTraceSource.STANDALONE_PRODUCTS_MASK} to include the new product's
+ *       {@link ProductTraceSource} bit — otherwise {@code
+ *       TraceCollector.setSamplingPriorityIfNecessary} will not recognize the product's traces.
+ * </ol>
+ */
+public enum StandaloneProduct {
+
+  /**
+   * LLM Observability: keeps all LLMOBS-marked traces. No billing trace is needed because LLMObs
+   * requires capturing every LLM interaction.
+   */
+  LLMOBS(ProductTraceSource.LLMOBS, SamplingMechanism.DEFAULT, false),
+
+  /**
+   * Application Security Management: keeps all ASM-marked traces and allows 1 APM trace per minute
+   * so the service catalog and billing can detect the service as live.
+   */
+  ASM(ProductTraceSource.ASM, SamplingMechanism.APPSEC, true);
+
+  /** The {@link ProductTraceSource} bit used to identify traces belonging to this product. */
+  public final int traceSourceBit;
+
+  /** The sampling mechanism to report when a trace is kept for this product. */
+  public final byte samplingMechanism;
+
+  /**
+   * Whether this product requires a billing trace (1 APM trace per minute) even in the absence of
+   * product-marked spans.
+   */
+  public final boolean needsBillingTrace;
+
+  StandaloneProduct(int traceSourceBit, byte samplingMechanism, boolean needsBillingTrace) {
+    this.traceSourceBit = traceSourceBit;
+    this.samplingMechanism = samplingMechanism;
+    this.needsBillingTrace = needsBillingTrace;
+  }
+}