test: guard against leaked enabled=True between llmobs tests

tests/llmobs/conftest.py

@@ -280,27 +280,31 @@ def llmobs(
     with override_global_config(global_config):
         # Pin agentless_enabled=False: the default flips to agentless when the Agent is
         # unreachable, swapping the ``tracer`` fixture's DummyWriter and breaking ``test_spans``.
-        llmobs_service.enable(_tracer=tracer, agentless_enabled=False, **llmobs_enable_opts)
-        llmobs_service._instance._llmobs_span_writer = llmobs_span_writer
-        llmobs_service._instance._llmobs_span_writer.start()
-        # The cassette proxy stands in for intake, so keep this client in direct mode. Without an
-        # app key it would otherwise pick the agent proxy and prefix every path with /evp_proxy/v2,
-        # which no recording matches.
-        dne_client = llmobs_service._instance._dne_client
-        dne_client._agentless = True
-        dne_client._endpoint = dne_client.ENDPOINT
-        dne_client._intake = llmobs_api_proxy_url
-        tracer._span_aggregator.llmobs_processor = LLMObsProcessor(
-            llmobs_span_writer,
-            tracer,
-            keep_meta_struct=True,
-            sampling_resolver=llmobs_service._instance._sampling_resolver,
-        )
         try:
+            # Setup lives inside the try so a failure here cannot leak an enabled instance holding
+            # this test's mocked writers into later tests on the same worker.
+            llmobs_service.enable(_tracer=tracer, agentless_enabled=False, **llmobs_enable_opts)
+            llmobs_service._instance._llmobs_span_writer = llmobs_span_writer
+            llmobs_service._instance._llmobs_span_writer.start()
+            # The cassette proxy stands in for intake, so keep this client in direct mode. Without an
+            # app key it would otherwise pick the agent proxy and prefix every path with /evp_proxy/v2,
+            # which no recording matches.
+            dne_client = llmobs_service._instance._dne_client
+            dne_client._agentless = True
+            dne_client._endpoint = dne_client.ENDPOINT
+            dne_client._intake = llmobs_api_proxy_url
+            tracer._span_aggregator.llmobs_processor = LLMObsProcessor(
+                llmobs_span_writer,
+                tracer,
+                keep_meta_struct=True,
+                sampling_resolver=llmobs_service._instance._sampling_resolver,
+            )
             yield llmobs_service
         finally:
-            tracer.shutdown()
-            llmobs_service.disable()
+            try:
+                tracer.shutdown()
+            finally:
+                llmobs_service.disable()
 
 
 @pytest.fixture

tests/llmobs/test_llmobs_service.py

@@ -2584,25 +2584,33 @@ def test_service_enable_starts_evaluator_runner_when_evaluators_exist(tracer):
     pytest.importorskip("ragas")
     with override_global_config(dict(_dd_api_key="<not-a-real-api-key>", _llmobs_ml_app="<ml-app-name>")):
         with override_env(dict(DD_LLMOBS_EVALUATORS="ragas_faithfulness")):
-            llmobs_service.enable(_tracer=tracer)
-            llmobs_instance = llmobs_service._instance
-            assert llmobs_instance is not None
-            assert llmobs_service.enabled
-            assert llmobs_service._instance._llmobs_eval_metric_writer.status.value == "running"
-            assert llmobs_service._instance._evaluator_runner.status.value == "running"
+            # Guard against leaked enabled=True from a prior failed test
             llmobs_service.disable()
+            llmobs_service.enable(_tracer=tracer)
+            try:
+                llmobs_instance = llmobs_service._instance
+                assert llmobs_instance is not None
+                assert llmobs_service.enabled
+                assert llmobs_service._instance._llmobs_eval_metric_writer.status.value == "running"
+                assert llmobs_service._instance._evaluator_runner.status.value == "running"
+            finally:
+                llmobs_service.disable()
 
 
 def test_service_enable_does_not_start_evaluator_runner(tracer):
     with override_global_config(dict(_dd_api_key="<not-a-real-api-key>", _llmobs_ml_app="<ml-app-name>")):
-        llmobs_service.enable(_tracer=tracer)
-        llmobs_instance = llmobs_service._instance
-        assert llmobs_instance is not None
-        assert llmobs_service.enabled
-        assert llmobs_service._instance._llmobs_eval_metric_writer.status.value == "running"
-        assert llmobs_service._instance._llmobs_span_writer.status.value == "running"
-        assert llmobs_service._instance._evaluator_runner.status.value == "stopped"
+        # Guard against leaked enabled=True from a prior failed test
         llmobs_service.disable()
+        llmobs_service.enable(_tracer=tracer)
+        try:
+            llmobs_instance = llmobs_service._instance
+            assert llmobs_instance is not None
+            assert llmobs_service.enabled
+            assert llmobs_service._instance._llmobs_eval_metric_writer.status.value == "running"
+            assert llmobs_service._instance._llmobs_span_writer.status.value == "running"
+            assert llmobs_service._instance._evaluator_runner.status.value == "stopped"
+        finally:
+            llmobs_service.disable()
 
 
 def test_export_span_when_llmobs_is_disabled_returns_none(llmobs):