Deadline-aware cancellation (prototype)

src/Adapter/MSTest.TestAdapter/InternalAPI/InternalAPI.Unshipped.txt

@@ -1,4 +1,9 @@
 #nullable enable
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestGracefulStopTestExecutionCapability.TryStopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<bool>!
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestGracefulStopTestExecutionCapability.NotifyTestExecutionCompleted() -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestGracefulStopTestExecutionCapability.NotifyTestExecutionPending() -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestGracefulStopTestExecutionCapability.NotifyTestExecutionStarting() -> void
+static Microsoft.VisualStudio.TestTools.UnitTesting.MSTestGracefulStopTestExecutionCapability.Create() -> Microsoft.VisualStudio.TestTools.UnitTesting.MSTestGracefulStopTestExecutionCapability!
 abstract Microsoft.Testing.Extensions.RunSettingsCommandLineOptionsProviderBase.EnvironmentVariablesNotSupportedOnBrowserError.get -> string!
 abstract Microsoft.Testing.Extensions.RunSettingsConfigurationProviderBase.DisplayName.get -> string!
 abstract Microsoft.Testing.Extensions.RunSettingsConfigurationProviderBase.ReadRunSettings(Microsoft.Testing.Platform.CommandLine.CommandLineParseResult! commandLineParseResult) -> string?

src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs

@@ -8,18 +8,126 @@
 namespace Microsoft.VisualStudio.TestTools.UnitTesting;
 
 [SuppressMessage("ApiDesign", "RS0030:Do not use banned APIs", Justification = "We can use MTP from this folder")]
-internal sealed class MSTestGracefulStopTestExecutionCapability : IGracefulStopTestExecutionCapability
+internal sealed class MSTestGracefulStopTestExecutionCapability : IGracefulStopTestExecutionResultCapability
 {
+#pragma warning disable IDE0330 // Use 'System.Threading.Lock' - not available on all target frameworks of this project.
+    private static readonly object Sync = new();
+#pragma warning restore IDE0330
+    private static int s_activeExecutionCount;
+    private static int s_pendingStopOwnerCount;
+    private ExecutionState _executionState;
+    private bool _isStopRequested;
+
     private MSTestGracefulStopTestExecutionCapability()
     {
     }
 
     public static MSTestGracefulStopTestExecutionCapability Instance { get; } = new();
 
+    internal static MSTestGracefulStopTestExecutionCapability Create() => new();
+
     public Task StopTestExecutionAsync(CancellationToken cancellationToken)
     {
-        PlatformServiceProvider.Instance.IsGracefulStopRequested = true;
+        _ = TryRequestGracefulStop();
+
         return Task.CompletedTask;
     }
+
+    public Task<bool> TryStopTestExecutionAsync(CancellationToken cancellationToken)
+        => Task.FromResult(TryRequestGracefulStop());
+
+    internal void NotifyTestExecutionPending()
+    {
+        lock (Sync)
+        {
+            UnregisterPendingStopOwner(_executionState == ExecutionState.Pending && _isStopRequested);
+            _isStopRequested = false;
+            _executionState = ExecutionState.Pending;
+        }
+    }
+
+    internal void NotifyTestExecutionStarting()
+    {
+        lock (Sync)
+        {
+            // Preserve a stop accepted while the run was pending. Otherwise this is a new run, so clear
+            // the process-wide engine flag left by a previous request, but only when no overlapping run
+            // still owns that flag. Discovery never reaches this path.
+            if (_isStopRequested)
+            {
+                UnregisterPendingStopOwner(_executionState == ExecutionState.Pending && _isStopRequested);
+                PlatformServiceProvider.Instance.IsGracefulStopRequested = true;
+            }
+            else if (s_activeExecutionCount == 0 && s_pendingStopOwnerCount == 0)
+            {
+                PlatformServiceProvider.Instance.IsGracefulStopRequested = false;
+            }
+
+            RegisterActiveExecution();
+            _executionState = ExecutionState.Active;
+        }
+    }
+
+    internal void NotifyTestExecutionCompleted()
+    {
+        lock (Sync)
+        {
+            if (_executionState == ExecutionState.Active)
+            {
+                UnregisterActiveExecution();
+            }
+            else
+            {
+                UnregisterPendingStopOwner(_executionState == ExecutionState.Pending && _isStopRequested);
+            }
+
+            _executionState = ExecutionState.Completed;
+        }
+    }
+
+    private bool TryRequestGracefulStop()
+    {
+        lock (Sync)
+        {
+            if (_executionState == ExecutionState.Completed || _isStopRequested)
+            {
+                return false;
+            }
+
+            RegisterPendingStopOwner(_executionState == ExecutionState.Pending && !_isStopRequested);
+            _isStopRequested = true;
+            PlatformServiceProvider.Instance.IsGracefulStopRequested = true;
+            return true;
+        }
+    }
+
+    private static void RegisterActiveExecution()
+        => s_activeExecutionCount++;
+
+    private static void UnregisterActiveExecution()
+        => s_activeExecutionCount--;
+
+    private static void RegisterPendingStopOwner(bool shouldRegister)
+    {
+        if (shouldRegister)
+        {
+            s_pendingStopOwnerCount++;
+        }
+    }
+
+    private static void UnregisterPendingStopOwner(bool shouldUnregister)
+    {
+        if (shouldUnregister)
+        {
+            s_pendingStopOwnerCount--;
+        }
+    }
+
+    private enum ExecutionState
+    {
+        Pending,
+        Active,
+        Completed,
+    }
 }
 #endif

src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestTestFramework.cs

@@ -40,6 +40,7 @@ internal sealed class MSTestTestFramework : ITestFramework, IDataProducer, IDisp
     private readonly ITrxReportCapability? _trxReportCapability;
     private readonly PlatformServicesConfigurationAdapter _configuration;
     private readonly ILoggerFactory _loggerFactory;
+    private readonly MSTestGracefulStopTestExecutionCapability _gracefulStopCapability;
     private readonly CountdownEvent _incomingRequestCounter = new(1);
     private bool? _isTrxEnabled;
     private bool _isDisposed;
@@ -54,7 +55,9 @@ public MSTestTestFramework(MSTestExtension extension, Func<IEnumerable<Assembly>
         _trxReportCapability = capabilities.GetCapability<ITrxReportCapability>();
         _configuration = new(serviceProvider.GetConfiguration());
         _loggerFactory = serviceProvider.GetRequiredService<ILoggerFactory>();
+        _gracefulStopCapability = (MSTestGracefulStopTestExecutionCapability)capabilities.GetCapability<IGracefulStopTestExecutionCapability>()!;
         PlatformServiceProvider.Instance.AdapterTraceLogger = new MTPTraceLogger(_loggerFactory.CreateLogger("mstest-trace"));
+        _gracefulStopCapability.NotifyTestExecutionPending();
 
         // Let the engine emit fixture/test-method spans that nest under the platform's test-case spans. This is a
         // no-op unless the OpenTelemetry extension is registered.
@@ -176,18 +179,26 @@ private async Task RunTestsAsync(RunTestExecutionRequest request, IMessageBus me
         // through the VSTest MSTestExecutor class. Results are published natively via MtpTestResultRecorder and the
         // MTP-specific filter provider evaluates the filter from the neutral UnitTestElement model so this path
         // never materializes a vstest TestCase (see #9769).
-        await new MSTestEngine(cancellationToken, CreateTelemetrySender())
-            .RunFromSourcesAsync(
-                assemblyPaths,
-                runSettings.SettingsXml,
-                runContext.TestRunDirectory,
-                handle.ToAdapterMessageLogger(),
-                settings => new MtpTestResultRecorder(messageBus, this, sessionUid, IsTrxEnabled, settings),
-                new MtpTestElementFilterProvider(runContext),
-                _configuration,
-                new TestSourceHandler(),
-                isMTP: true)
-            .ConfigureAwait(false);
+        _gracefulStopCapability.NotifyTestExecutionStarting();
+        try
+        {
+            await new MSTestEngine(cancellationToken, CreateTelemetrySender())
+                .RunFromSourcesAsync(
+                    assemblyPaths,
+                    runSettings.SettingsXml,
+                    runContext.TestRunDirectory,
+                    handle.ToAdapterMessageLogger(),
+                    settings => new MtpTestResultRecorder(messageBus, this, sessionUid, IsTrxEnabled, settings),
+                    new MtpTestElementFilterProvider(runContext),
+                    _configuration,
+                    new TestSourceHandler(),
+                    isMTP: true)
+                .ConfigureAwait(false);
+        }
+        finally
+        {
+            _gracefulStopCapability.NotifyTestExecutionCompleted();
+        }
     }
 
     private MSTestRunSettings CreateRunSettings(MSTestFrameworkHandle handle)
@@ -232,6 +243,7 @@ public void Dispose()
     {
         if (!_isDisposed)
         {
+            _gracefulStopCapability.NotifyTestExecutionCompleted();
             _incomingRequestCounter.Dispose();
             _isDisposed = true;
         }

src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs

@@ -72,7 +72,7 @@ public static void AddMSTest(this ITestApplicationBuilder testApplicationBuilder
             serviceProvider => new TestFrameworkCapabilities(
                 new MSTestCapabilities(),
                 new MSTestBannerCapability(serviceProvider.GetRequiredService<IPlatformInformation>()),
-                MSTestGracefulStopTestExecutionCapability.Instance),
+                MSTestGracefulStopTestExecutionCapability.Create()),
             (capabilities, serviceProvider) => new MSTestTestFramework(extension, getTestAssemblies, serviceProvider, capabilities));
     }
 }

src/Adapter/MSTestAdapter.PlatformServices/PlatformServiceProvider.cs

@@ -14,6 +14,8 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
 /// </summary>
 internal sealed class PlatformServiceProvider : IPlatformServiceProvider
 {
+    private int _isGracefulStopRequested;
+
     /// <summary>
     /// Initializes a new instance of the <see cref="PlatformServiceProvider"/> class - a singleton.
     /// </summary>
@@ -109,7 +111,11 @@ public IReflectionOperations ReflectionOperations
     /// </summary>
     public TestRunCancellationToken? TestRunCancellationToken { get; set; }
 
-    public bool IsGracefulStopRequested { get; set; }
+    public bool IsGracefulStopRequested
+    {
+        get => Volatile.Read(ref _isGracefulStopRequested) != 0;
+        set => Volatile.Write(ref _isGracefulStopRequested, value ? 1 : 0);
+    }
 
     /// <summary>
     /// Gets or sets the instance for the platform service.

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/InternalAPI/InternalAPI.Unshipped.txt

@@ -14,6 +14,10 @@ static Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsSummaryReporter
 *REMOVED*Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsSummaryReporter.AzureDevOpsSummaryReporter(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Configurations.IConfiguration! configuration, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory) -> void
 *REMOVED*abstract Microsoft.Testing.Extensions.SlowTestReporterBase.EmitSlowTestAsync(string! testName, System.TimeSpan elapsed, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task!
 Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode
+Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode
+const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE = "TESTINGPLATFORM_DEADLINE" -> string!
+const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN = "TESTINGPLATFORM_DEADLINE_STOP_MARGIN" -> string!
+const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN" -> string!
 static Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsTestResultsClient.CreateHttpClientHandler() -> System.Net.Http.HttpClientHandler!
 static Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsTestResultsClient.ShouldOptInToAutomaticDecompression(System.Net.Http.HttpClientHandler! handler) -> bool
 Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsTestResultsPublisher.AzureDevOpsTestResultsPublisher(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Configurations.IConfiguration! configuration, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.Services.ITestApplicationProcessExitCode! testApplicationProcessExitCode, Microsoft.Testing.Extensions.AzureDevOpsReport.IAzureDevOpsTestResultsClient! client, Microsoft.Testing.Platform.Helpers.ITask! task, Microsoft.Testing.Platform.Helpers.IClock! clock, Microsoft.Testing.Platform.Logging.ILogger! logger, Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsTestResultsPublisherOptions! options) -> void

src/Platform/Microsoft.Testing.Extensions.CtrfReport/InternalAPI/InternalAPI.Unshipped.txt

@@ -27,6 +27,10 @@ static Microsoft.Testing.Extensions.CtrfReport.CtrfReportMerger.MergeToFileAsync
 static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipeDirectoryNotWritableErrorMessage.get -> string!
 static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipePathTooLongErrorMessage.get -> string!
 Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode
+Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode
+const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE = "TESTINGPLATFORM_DEADLINE" -> string!
+const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN = "TESTINGPLATFORM_DEADLINE_STOP_MARGIN" -> string!
+const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN" -> string!
 static Microsoft.Testing.Platform.Services.ArtifactNamingHelper.ResolveAndSanitize(string! template, string! processName, string! processId, System.DateTimeOffset timestamp, System.Func<string!, string!>! sanitizeLeafFileName) -> string!
 Microsoft.Testing.Extensions.MergeOutputFileHelper
 static Microsoft.Testing.Extensions.MergeOutputFileHelper.BuildCaseFoldedProbePath(string! directory, string! probeFileName) -> string!

src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs

@@ -60,6 +60,7 @@ public static string GetReason(int exitCode)
             (int)ExitCode.IncompatibleProtocolVersion => GitHubActionsResources.ExitCodeReasonIncompatibleProtocolVersion,
             (int)ExitCode.TestExecutionStoppedForMaxFailedTests => GitHubActionsResources.ExitCodeReasonTestExecutionStoppedForMaxFailedTests,
             (int)ExitCode.CoverageThresholdFailed => GitHubActionsResources.ExitCodeReasonCoverageThresholdFailed,
+            (int)ExitCode.TestExecutionStoppedAtDeadline => GitHubActionsResources.ExitCodeReasonTestExecutionStoppedAtDeadline,
             _ => GitHubActionsResources.ExitCodeReasonUnknown,
         };
 }

src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt

@@ -99,6 +99,10 @@ Microsoft.Testing.Extensions.TestFailureDetails.TestFailureDetails(string? messa
 Microsoft.Testing.Extensions.TestRecord.Failure.get -> Microsoft.Testing.Extensions.TestFailureDetails?
 Microsoft.Testing.Extensions.TestRecord.TestRecord(string! displayName, string! fullyQualifiedName, Microsoft.Testing.Extensions.TerminalKind kind, System.TimeSpan duration, Microsoft.Testing.Extensions.TestFailureDetails? failure) -> void
 Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode
+Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode
+const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE = "TESTINGPLATFORM_DEADLINE" -> string!
+const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN = "TESTINGPLATFORM_DEADLINE_STOP_MARGIN" -> string!
+const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN" -> string!
 Microsoft.Testing.Platform.Helpers.RuntimeFeatureHelper
 override Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsCommandLineProvider.ValidateCommandLineOptionsAsync(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions) -> System.Threading.Tasks.Task<Microsoft.Testing.Platform.Extensions.ValidationResult>!
 static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsAnnotationReporter.GetErrorAnnotation(string! testName, string? explanation, System.Exception? exception, string? repoRoot, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.Logging.ILogger! logger, bool skipAssertionFrames, Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSourceLocation? declaredLocation = null) -> string!