diff --git a/Orleans.slnx b/Orleans.slnx index e4047b5f194..877afac409d 100644 --- a/Orleans.slnx +++ b/Orleans.slnx @@ -42,6 +42,7 @@ + @@ -152,6 +153,7 @@ + diff --git a/docs/site/src/content/docs/grains/journaling/configuration.md b/docs/site/src/content/docs/grains/journaling/configuration.md index 70c53059e5c..f40ea04f877 100644 --- a/docs/site/src/content/docs/grains/journaling/configuration.md +++ b/docs/site/src/content/docs/grains/journaling/configuration.md @@ -150,7 +150,7 @@ Serializer naming policies affect application payload values. Journal command na ## Migrate a journal format -Providers expose the persisted format key as and . Recovery selects the stored reader independently of the configured write format. When they differ, the next write creates a full snapshot using the configured format and updates the metadata. supplies the JSON Lines format key. +Providers expose the persisted format key as and . Recovery selects the stored reader independently of the configured write format. When stored format metadata is absent, recovery uses the configured format. New empty journals use the configured write format. When the stored and configured keys differ, the next write creates a full snapshot using the configured format and updates the metadata. supplies the JSON Lines format key. Use this deployment sequence: diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index e513f837fc9..6583f5ea8aa 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -25,6 +25,8 @@ During , the manager: 1. Resets and replays each registered durable state. 1. Completes activation setup after replay finishes. +Recovery uses the configured format when stored format metadata is absent. New empty journals use the configured write format. + and requests observe recovered durable state after setup succeeds, whether the grain derives directly from , from an application-owned base, or from . A storage read, format, codec, or malformed-data failure fails activation and preserves the stored journal for diagnosis and recovery. Provider registration makes Journaling services available. Per-grain journal I/O begins only for activations which resolve the manager, directly or through durable-state dependencies. Grains which use other persistence models keep their existing activation behavior. @@ -68,6 +70,46 @@ Orleans executes that synchronous block on a single activation thread. Another g the operation awaits, so keep shared state safe to commit at each await. Any caller's write can include staged mutations from other calls. +## Journal operation hooks + +Activation-scoped features coordinate prerequisites and completion through +. The owner exposes a stable, lazily +allocated list of registrations. Inspect and +deduplicate feature registrations on the owner's logical execution context while persistence +is quiescent. Registration survives recovery and deletion; the standard manager rejects hook-list +mutation while persistence is queued or running. +Every journal owner implementation provides this list and runs its registered callbacks at the +operation boundaries below. Delegating owners forward the list to the inner owner. + +Each actual append, snapshot, or deletion runs ordinary before callbacks in list order, outside +the manager lock. At most one supplies the +final prerequisite. Its before callback runs last, and the work loop awaits it directly before +synchronous capture or storage deletion. Prerequisites cover changes staged during asynchronous +preparation, including the final hook's own I/O wait. Preserve operation-local bookkeeping for +the captured batch separately from changes staged later. + +Storage acknowledgement and registered-state acknowledgement or reset precede after callbacks. +All after callbacks run in list order, including for successful zero-byte writes. Coalesced callers +share callbacks for the actual operation. Features implement the before and after callbacks on +their own hook, retaining feature identity and operation-local bookkeeping there. + +A failed prerequisite reports with pending +state retained for an explicit persistence retry after the prerequisite is restored. A failed after +callback reports with persistence completed. +Remaining after callbacks run, multiple failures are aggregated, and the manager stays usable. +The feature's durable recovery protocol resumes interrupted post-persistence work. Storage and +state-processing failures retain the manager's fencing and fresh-recovery behavior. + +Hook callbacks receive the owner's shutdown token. Cancelling a caller's wait leaves the owned +operation running through its actual outcome. Disposal drains owned hooks and storage before releasing +journal resources, including when cancellation callbacks or cleanup fail. Concurrent disposal callers +share this completion. Hook implementations complete without calling initialization, persistence, +or disposal on their own owner. Awaiting an operation serialized behind the current callback creates +a circular dependency. Shutdown closes work admission and cancels queued operations while the current +operation drains to its actual storage and hook outcome. For deletion, the feature owner stops +admission and drains feature operations before +queuing the whole-journal reset. + ## Consistency and competing writers Orleans grain placement normally supplies a single active writer for a grain identity. Journal storage providers also use optimistic concurrency to protect the journal when a stale or competing writer reaches storage. diff --git a/docs/site/src/content/docs/grains/timers.md b/docs/site/src/content/docs/grains/timers.md index b5e40e3814b..45ec72ca2d2 100644 --- a/docs/site/src/content/docs/grains/timers.md +++ b/docs/site/src/content/docs/grains/timers.md @@ -32,9 +32,13 @@ Register timers with . controls delayed tick timing and resolution; Orleans queues the provider's tick notifications on the activation. ### Interleaving @@ -50,9 +54,11 @@ With set to `true`, e ## Change or stop a timer -Call to replace the due time and period. The new due time schedules the next callback, and the new period applies after that callback completes. A change made inside a running callback takes effect after the callback completes. +Call to replace the due time and period. When a callback is already queued or running, it keeps its turn and the change takes effect after it completes. Repeated changes use the latest due time and period for the following schedule. + +A physical tick already dispatched by the provider can arrive after a change to another delayed schedule. Orleans admits that tick according to the activation's scheduling rules. After its callback completes, the configured period determines the next tick, or a change made during the callback supplies the next due time. -Dispose to cancel its callback token and stop future callbacks. Orleans also cancels the token and disposes the timer when the activation begins deactivating. +Dispose to invalidate queued ticks, cancel the token of an admitted callback, and stop further scheduling. Queued messages drain through activation scheduling. Orleans also cancels the token and disposes the timer when the activation begins deactivating. ## Handle callback failures diff --git a/docs/site/src/content/docs/snippets/compiled/Grains/JournalingSnippets.cs b/docs/site/src/content/docs/snippets/compiled/Grains/JournalingSnippets.cs index 7396ef52697..30369581c97 100644 --- a/docs/site/src/content/docs/snippets/compiled/Grains/JournalingSnippets.cs +++ b/docs/site/src/content/docs/snippets/compiled/Grains/JournalingSnippets.cs @@ -9,6 +9,20 @@ namespace Documentation.Grains.Journaling; +// +internal static class JournalHookRegistration +{ + internal static void Register(IJournaledStateManager owner, IJournaledStateHook featureHook) + { + var hooks = owner.Hooks; + if (!hooks.Contains(featureHook)) + { + hooks.Add(featureHook); + } + } +} +// + // public interface IShoppingCartGrain : IGrainWithStringKey { diff --git a/src/Orleans.Core/Messaging/MessageFactory.cs b/src/Orleans.Core/Messaging/MessageFactory.cs index d56e027b7b9..25f2fdfc282 100644 --- a/src/Orleans.Core/Messaging/MessageFactory.cs +++ b/src/Orleans.Core/Messaging/MessageFactory.cs @@ -27,7 +27,10 @@ public MessageFactory(DeepCopier deepCopier, ILogger logger, Mes _seed = unchecked((ulong)Random.Shared.NextInt64()); } - public Message CreateMessage(object? body, InvokeMethodOptions options) + public Message CreateMessage(object? body, InvokeMethodOptions options) => + CreateMessage(body, options, RequestContextExtensions.Export(_deepCopier)); + + public Message CreateMessage(object? body, InvokeMethodOptions options, Dictionary? requestContextData) { var message = new Message { @@ -37,7 +40,7 @@ public Message CreateMessage(object? body, InvokeMethodOptions options) IsUnordered = (options & InvokeMethodOptions.Unordered) != 0, IsAlwaysInterleave = (options & InvokeMethodOptions.AlwaysInterleave) != 0, BodyObject = body, - RequestContextData = RequestContextExtensions.Export(_deepCopier), + RequestContextData = requestContextData, }; return message; diff --git a/src/Orleans.Journaling/CompatibilitySuppressions.xml b/src/Orleans.Journaling/CompatibilitySuppressions.xml index eb687e7cf91..ed87a41fc9b 100644 --- a/src/Orleans.Journaling/CompatibilitySuppressions.xml +++ b/src/Orleans.Journaling/CompatibilitySuppressions.xml @@ -267,6 +267,13 @@ lib/net10.0/Orleans.Journaling.dll true + + CP0006 + P:Orleans.Journaling.IJournaledStateManager.Hooks + lib/net10.0/Orleans.Journaling.dll + lib/net10.0/Orleans.Journaling.dll + true + CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey @@ -302,6 +309,13 @@ lib/net8.0/Orleans.Journaling.dll true + + CP0006 + P:Orleans.Journaling.IJournaledStateManager.Hooks + lib/net8.0/Orleans.Journaling.dll + lib/net8.0/Orleans.Journaling.dll + true + CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey diff --git a/src/Orleans.Journaling/IJournaledStateCaptureHook.cs b/src/Orleans.Journaling/IJournaledStateCaptureHook.cs new file mode 100644 index 00000000000..cd6d0b42693 --- /dev/null +++ b/src/Orleans.Journaling/IJournaledStateCaptureHook.cs @@ -0,0 +1,15 @@ +namespace Orleans.Journaling; + +/// +/// Establishes the final prerequisite immediately before journal capture or deletion. +/// +/// +/// An owner admits at most one capture hook in . +/// Its before callback runs after all ordinary before callbacks. The work loop awaits it directly, +/// then captures state or starts deletion without a further asynchronous phase. +/// Its prerequisites cover changes staged while its own I/O awaited. After callbacks retain +/// normal list order. Features inspect and deduplicate this registration using the same hook list. +/// +public interface IJournaledStateCaptureHook : IJournaledStateHook +{ +} diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs new file mode 100644 index 00000000000..3639b6c658a --- /dev/null +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -0,0 +1,58 @@ +namespace Orleans.Journaling; + +/// +/// Identifies a journal persistence operation. +/// +public enum JournaledStateOperation +{ + /// Appends pending journal entries. + Write, + + /// Replaces storage with a snapshot. + Snapshot, + + /// Deletes storage and resets registered states. + Delete +} + +/// +/// Participates in the prerequisites and completion of actual journal operations. +/// +/// +/// Ordinary before hooks run in list order on the owner's logical execution context, outside its lock. +/// An optional runs last immediately before capture or deletion. +/// Prerequisites must cover changes staged during asynchronous preparation. After hooks run after +/// storage acknowledgement and state acknowledgement or reset, including successful writes which +/// produce no storage bytes. Hooks retain operation-local data across these boundaries and keep later +/// pending changes separate. Hook implementations must complete without calling initialization, +/// persistence, or disposal on their own journal owner. Those operations are serialized behind the +/// current operation, so awaiting them from a hook would create a circular dependency. +/// +public interface IJournaledStateHook +{ + /// + /// Establishes prerequisites before capture or storage deletion. + /// + /// The operation about to execute. + /// The token for the owned operation's lifetime. + /// A completion representing the prerequisite work. + /// + /// Failure reports and retains pending changes + /// for an explicit retry. All staged changes remain safe to commit. Full deletion requires the + /// owner to stop admission and drain feature operations before queuing deletion. + /// + ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; + + /// + /// Completes post-persistence work after the operation succeeds. + /// + /// The completed operation. + /// The token for the owned operation's lifetime. + /// A completion representing post-persistence work. + /// + /// Every after hook is invoked even when an earlier after hook fails. Failures are surfaced as + /// and leave the successfully persisted owner usable. + /// Durable feature state supplies recovery for interrupted post-persistence work. + /// + ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; +} diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index b6363a1b4cc..ffd59075f22 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -8,10 +8,23 @@ namespace Orleans.Journaling; /// /// The owner registers state machines and initializes the journal before using recovered state. /// State machine instances and their dependencies retain the lifetime assigned by their caller. -/// Disposing this manager stops journal processing and releases its journal resources. +/// Disposal drains owned storage and hook operations before releasing resources. /// public interface IJournaledStateManager : IAsyncDisposable { + /// + /// Gets the mutable, lazily allocated list of journal operation hooks. + /// + /// + /// Inspect, add, remove, and deduplicate hooks on the owner's logical execution context while + /// persistence is quiescent. Mutation while persistence is queued or running is rejected. Ordinary before hooks + /// and all after hooks execute in list order. The optional single + /// supplies the final prerequisite. + /// Registration is independent of state-machine registration and persists through recovery and deletion. + /// Every implementation provides this list and invokes its hooks at the documented operation boundaries. + /// + IList Hooks { get; } + /// ValueTask IAsyncDisposable.DisposeAsync() => default; @@ -51,8 +64,11 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// Stage mutations only after establishing that they are safe to commit. Pending changes are shared /// by all callers using this manager. Storage acknowledgement establishes durability. - /// A failed journal operation fences the manager; recovery requires a new manager and state machine instances. + /// A storage or state-processing failure fences the manager; recovery requires a new manager and state machine instances. /// Cancellation stops the caller's wait; an already queued write continues to its storage outcome. + /// Before-hook failure reports and retains pending + /// changes for an explicit retry. After-hook failure reports + /// after successful persistence. /// /// The token used to cancel the caller's wait. /// A task representing the write acknowledgement. @@ -64,7 +80,10 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// The caller keeps other operations quiescent through completion: deletion resets every registered state machine. /// Cancellation ends the caller's wait; an already queued deletion continues to its storage and reset outcome. - /// A failed deletion permanently fences the manager and requests deactivation of its owning grain. + /// A storage or state-reset failure permanently fences the manager and requests deactivation of its owning grain. + /// Before-hook failure reports with state retained. + /// After-hook failure reports after storage deletion + /// and state reset succeed. /// /// The cancellation token. /// A which represents the operation. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index cbf6be752bc..6f8d8479d36 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -1,6 +1,8 @@ using System.Buffers; +using System.Collections.ObjectModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; using Microsoft.Extensions.Logging; using Orleans.Diagnostics; using Orleans.Serialization.Buffers; @@ -33,6 +35,20 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private Exception? _failure; private bool _migrationSnapshotRequired; private int _disposed; + private Task? _disposeTask; + private HookCollection? _hooks; + private bool _hookOperationRunning; + + public IList Hooks + { + get + { + lock (_lock) + { + return _hooks ??= new(this); + } + } + } public JournaledStateManager(JournaledStateManagerShared shared, IJournalStorageProvider storageProvider, IGrainContext grainContext) : this(shared, CreateStorage(storageProvider, CreateJournalId(grainContext)), grainContext) @@ -245,6 +261,11 @@ private async Task WorkLoop() WorkItem workItem; lock (_lock) { + if (_shutdownCancellation.IsCancellationRequested) + { + return; + } + if (!_workQueue.TryDequeue(out var dequeuedWorkItem)) { // Wait for the queue to be signaled again. @@ -252,6 +273,7 @@ private async Task WorkLoop() } workItem = dequeuedWorkItem; + _hookOperationRunning = workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem; } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -273,6 +295,9 @@ private async Task WorkLoop() storageActivity.SetTag(ActivityTagKeys.JournalStorageOperation, queueOperation); } + JournaledStateOperation? hookOperation = null; + var beforeHookRunning = false; + var afterHookRunning = false; try { if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem @@ -282,6 +307,29 @@ private async Task WorkLoop() "The journaled state operation was queued before deletion reset its state."); } + hookOperation = workItem switch + { + AppendJournalWorkItem or WriteSnapshotWorkItem => + workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCompactionRequested + ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write, + DeleteStateWorkItem => JournaledStateOperation.Delete, + _ => null + }; + if (hookOperation is { } operation && _hooks is { Count: > 0 }) + { + beforeHookRunning = true; + var captureHook = await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + + if (captureHook is not null) + { + // Await the final prerequisite in this frame so capture follows its completion directly. + await captureHook.BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + } + + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + beforeHookRunning = false; + } + // Note that the implementation of each command is inlined to avoid allocating unnecessary async states. // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) @@ -291,9 +339,7 @@ private async Task WorkLoop() { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. - var isSnapshot = workItem is WriteSnapshotWorkItem - || _migrationSnapshotRequired - || _storage.IsCompactionRequested; + var isSnapshot = hookOperation == JournaledStateOperation.Snapshot; var operationLabel = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; @@ -540,13 +586,47 @@ private async Task WorkLoop() } } + if (hookOperation is { } completedOperation && _hooks is { Count: > 0 }) + { + afterHookRunning = true; + await InvokeAfterHooksAsync(completedOperation, _shutdownCancellation.Token).ConfigureAwait(true); + afterHookRunning = false; + } + if (recordQueueDuration && queueOperation is not null) { _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: true); } storageActivity?.SetStatus(ActivityStatusCode.Ok); - workItem.SetResult(); + lock (_lock) + { + _hookOperationRunning = false; + workItem.SetResult(); + } + } + catch (Exception exception) when (beforeHookRunning || afterHookRunning) + { + if (beforeHookRunning && exception is OperationCanceledException && _shutdownCancellation.IsCancellationRequested) + { + workItem.TrySetCanceled(_shutdownCancellation.Token); + continue; + } + + LogOperationHookFailed(_shared.Logger, exception, hookOperation!.Value, afterHookRunning); + storageActivity?.SetStatus(ActivityStatusCode.Error, "Journal operation hook failed."); + if (recordQueueDuration && queueOperation is not null) + { + _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: afterHookRunning); + } + + lock (_lock) + { + _hookOperationRunning = false; + workItem.SetException(afterHookRunning + ? new JournaledStatePostCommitException(hookOperation.Value, exception) + : new JournaledStatePreCommitException(hookOperation.Value, exception)); + } } catch (Exception exception) { @@ -580,6 +660,11 @@ private async Task WorkLoop() } finally { + lock (_lock) + { + _hookOperationRunning = false; + } + storageActivity?.Dispose(); } } @@ -596,6 +681,127 @@ private async Task WorkLoop() } } + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + IJournaledStateCaptureHook? captureHook = null; + for (var i = 0; i < _hooks!.Count; i++) + { + var hook = _hooks[i]; + if (hook is IJournaledStateCaptureHook capture) + { + captureHook = capture; + continue; + } + + cancellationToken.ThrowIfCancellationRequested(); + await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + + cancellationToken.ThrowIfCancellationRequested(); + return captureHook; + } + + private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + List? failures = null; + for (var i = 0; i < _hooks!.Count; i++) + { + try + { + await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + catch (Exception exception) + { + (failures ??= []).Add(exception); + } + } + + if (failures is { Count: 1 }) + { + ExceptionDispatchInfo.Capture(failures[0]).Throw(); + } + + if (failures is not null) + { + throw new AggregateException(failures); + } + } + + private sealed class HookCollection(JournaledStateManager owner) : Collection + { + protected override void InsertItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + EnsureMutationAllowed(); + ValidateHook(item, replacingIndex: -1); + base.InsertItem(index, item); + } + } + + protected override void SetItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + EnsureMutationAllowed(); + ValidateHook(item, index); + base.SetItem(index, item); + } + } + + protected override void RemoveItem(int index) + { + lock (owner._lock) + { + EnsureMutationAllowed(); + base.RemoveItem(index); + } + } + + protected override void ClearItems() + { + lock (owner._lock) + { + EnsureMutationAllowed(); + base.ClearItems(); + } + } + + private void EnsureMutationAllowed() + { + ObjectDisposedException.ThrowIf(owner._disposed != 0, owner); + owner._shutdownCancellation.Token.ThrowIfCancellationRequested(); + owner.ThrowIfFenced(); + if (owner._hookOperationRunning) + { + throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); + } + + foreach (var workItem in owner._workQueue) + { + if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem) + { + throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); + } + } + } + + private void ValidateHook(IJournaledStateHook item, int replacingIndex) + { + ArgumentNullException.ThrowIfNull(item); + if (item is IJournaledStateCaptureHook) + { + for (var i = 0; i < Count; i++) + { + if (i != replacingIndex && this[i] is IJournaledStateCaptureHook) + { + throw new InvalidOperationException("A journal owner supports one final capture prerequisite hook. Inspect and deduplicate the hook list before registration."); + } + } + } + } + } + private void Fence(Exception exception) { lock (_lock) @@ -1062,9 +1268,22 @@ void ILifecycleParticipant.Participate(IGrainLifecycle observer private async Task StopAsync(CancellationToken cancellationToken) { + AggregateException? cancellationFailure = null; lock (_lock) { - _shutdownCancellation.Cancel(); + try + { + _shutdownCancellation.Cancel(); + } + catch (AggregateException exception) + { + cancellationFailure = exception; + } + } + + if (cancellationFailure is not null) + { + LogShutdownCancellationFailed(_shared.Logger, cancellationFailure); } _workSignal.Signal(); @@ -1079,6 +1298,11 @@ private async Task StopAsync(CancellationToken cancellationToken) { CancelQueuedWorkItems(_shutdownCancellation.Token); } + + if (cancellationFailure is not null) + { + ExceptionDispatchInfo.Capture(cancellationFailure).Throw(); + } } private void CancelQueuedWorkItems(CancellationToken cancellationToken) @@ -1097,13 +1321,17 @@ void IDisposable.Dispose() DisposeAsync().AsTask().GetAwaiter().GetResult(); } - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - if (Interlocked.Exchange(ref _disposed, 1) != 0) + lock (_lock) { - return; + return new(_disposeTask ??= DisposeCoreAsync()); } + } + private async Task DisposeCoreAsync() + { + _disposed = 1; try { await StopAsync(CancellationToken.None).ConfigureAwait(false); @@ -1352,6 +1580,16 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal {Operation} hook failed. Persistence completed: {Committed}.")] + private static partial void LogOperationHookFailed(ILogger logger, Exception exception, JournaledStateOperation operation, bool committed); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal shutdown cancellation callback failed; owned operations are drained before resources are released.")] + private static partial void LogShutdownCancellationFailed(ILogger logger, Exception exception); + [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/Orleans.Journaling/JournaledStatePostCommitException.cs b/src/Orleans.Journaling/JournaledStatePostCommitException.cs new file mode 100644 index 00000000000..8274c8726d6 --- /dev/null +++ b/src/Orleans.Journaling/JournaledStatePostCommitException.cs @@ -0,0 +1,22 @@ +namespace Orleans.Journaling; + +/// +/// Reports a hook failure after the journal operation and its state acknowledgement or reset succeeded. +/// +/// +/// The journal owner remains usable. The caller handles the failed post-persistence work using +/// the feature's durable recovery protocol, preserving the completed business operation. +/// +[GenerateSerializer] +public sealed class JournaledStatePostCommitException : Exception +{ + /// + /// Initializes a post-persistence hook failure. + /// + /// The successfully completed journal operation. + /// The failure or aggregate of failures from after hooks. + public JournaledStatePostCommitException(JournaledStateOperation operation, Exception innerException) + : base($"Journal operation '{operation}' completed, but a post-persistence hook failed.", innerException) + { + } +} diff --git a/src/Orleans.Journaling/JournaledStatePreCommitException.cs b/src/Orleans.Journaling/JournaledStatePreCommitException.cs new file mode 100644 index 00000000000..fe8d2eb79d4 --- /dev/null +++ b/src/Orleans.Journaling/JournaledStatePreCommitException.cs @@ -0,0 +1,22 @@ +namespace Orleans.Journaling; + +/// +/// Reports a prerequisite hook failure which prevented the journal storage operation. +/// +/// +/// Pending changes remain staged and safe to commit. The caller can restore the prerequisite +/// and explicitly retry persistence, or retire its owner and recover from durable state. +/// +[GenerateSerializer] +public sealed class JournaledStatePreCommitException : Exception +{ + /// + /// Initializes a prerequisite hook failure. + /// + /// The journal operation prevented by the failed prerequisite. + /// The original prerequisite failure. + public JournaledStatePreCommitException(JournaledStateOperation operation, Exception innerException) + : base($"Journal operation '{operation}' was prevented by a prerequisite hook failure.", innerException) + { + } +} diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 2f5046a30be..ba7257516a2 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -92,7 +92,7 @@ Recovery reads the selected provider's physical namespace using the existing journal identity. Changing the selection for a grain type with existing journals requires a deliberate data migration or cutover strategy, including rollback. -JSON Lines is the default `JournaledStateManagerOptions.JournalFormatKey`. Storage providers expose the stored journal format key through `IJournalMetadata.FormatKey` and `JournalMetadata.FormatKey`. During recovery, Orleans uses that stored key to select the matching journal format and durable operation codecs. If a non-empty journal has no stored format metadata, Orleans treats it as legacy OrleansBinary data for compatibility. +JSON Lines is the default `JournaledStateManagerOptions.JournalFormatKey`. Storage providers expose the stored journal format key through `IJournalMetadata.FormatKey` and `JournalMetadata.FormatKey`. During recovery, Orleans uses that stored key to select the matching journal format and durable operation codecs. When stored format metadata is absent, recovery uses the configured format. New empty journals use the configured write format. If you already have data written with the OrleansBinary format, you can keep using it while you plan a migration: @@ -264,6 +264,44 @@ the operation's token before use, or deliberately supply them through a registra lifecycle ownership. Creation through the explicit-journal factory keeps failure handling independent of the ambient grain context, including when the caller subsequently enrolls the manager in a lifecycle. +## Journal operation hooks + +`IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` +registrations. Features inspect and deduplicate their registrations on the owner's logical +execution context while persistence is quiescent. Registration persists through recovery and +whole-journal deletion. The standard manager rejects mutation of the list while persistence is +queued or running and admits at most one `IJournaledStateCaptureHook`. +Every journal owner implementation provides the hook list and invokes registered callbacks at +the operation boundaries described below. Delegating owners forward the list to their inner owner. + +For each actual write, snapshot, or deletion, ordinary before hooks run in list order outside +the manager lock. The capture hook runs last: the work loop awaits it directly, then synchronously +captures the registered states or starts deletion. Prerequisites cover all changes staged during +asynchronous preparation, including changes arriving while the capture hook awaits its own I/O. +Keep operation-local bookkeeping for the captured batch separate from later pending changes. + +After hooks run in list order after storage acknowledgement and state acknowledgement or reset. +They also run for a successful zero-byte write. Coalesced callers share the hooks for their actual +operation. Features implement the before and after callbacks on their own hook, retaining their +identity and operation-local bookkeeping there. + +| Outcome | Owner and caller behavior | +| --- | --- | +| Before hook fails | `JournaledStatePreCommitException` retains pending state for an explicit retry after restoring the prerequisite. | +| Storage or state processing fails | The original failure fences the manager; create a fresh owner and recover the durable outcome. | +| After hook fails | `JournaledStatePostCommitException` reports successful persistence. Every remaining after hook runs, failures are aggregated, and the manager stays usable. The feature's durable recovery protocol resumes interrupted post-persistence work. | + +Hooks receive the owner's shutdown token. Caller cancellation ends the caller's wait while +the owned prerequisite, capture, storage, and completion phases continue. Disposal cancels the +owner token and drains owned work before releasing journal resources, including when cancellation +callbacks or after-hook cleanup fail. Concurrent disposal callers share that drain and its outcome. +Shutdown closes work admission and cancels queued operations while an already running operation +drains to its actual storage and hook outcome. +Hook implementations complete without calling initialization, persistence, or disposal on their +own owner: awaiting an operation serialized behind the current callback creates a circular dependency. +Before whole-journal deletion, the feature owner stops admission and drains its own operations; +deletion completion follows storage deletion and registered-state reset. + ## State identity and retirement Preserve state names across activations and deployments. A stream absent from the setup declarations @@ -312,7 +350,7 @@ Each record contains the state id as element 0 and the durable operation payload Inside the operation payload array, element 0 is the command name, followed by command-specific operands such as keys, values, item arrays, or versions. Storage write batches append one or more complete JSON Lines records without adding a separate extent envelope or final container-close step. -Existing data is read using its stored format metadata, or as legacy OrleansBinary data when metadata is absent, and migrated to the configured write format by the next snapshot write. +Existing data is read using its stored format key, or the configured format when metadata is absent, and migrated to the configured write format by the next snapshot write. ## Catalog enumeration diff --git a/src/Orleans.Messaging/Configuration/InboxOptions.cs b/src/Orleans.Messaging/Configuration/InboxOptions.cs new file mode 100644 index 00000000000..15e43c902fd --- /dev/null +++ b/src/Orleans.Messaging/Configuration/InboxOptions.cs @@ -0,0 +1,238 @@ +using System; + +namespace Orleans.Messaging.Configuration; + +/// +/// Configuration options for the inbox messaging system. +/// +/// +/// +/// These options control the behavior of the inbox, including capacity limits, +/// deduplication tracking, retry behavior, and pump batch sizes. +/// +/// +/// Transport is at-least-once. Deduplication provides effectively-once handler effects only +/// while the processed-message record is retained. Configuration values affect memory usage, +/// throughput, and recovery characteristics. +/// +/// +public class InboxOptions +{ + internal const int MaximumBackoffExponent = 6; + internal const int MaximumBackoffMultiplier = 1 << MaximumBackoffExponent; + + // The runtime multiplies this base delay by at most MaximumBackoffMultiplier. + // Keep the expanded delay within the timer implementation's uint-millisecond limit. + private static readonly TimeSpan MaxSupportedRetryDelay = + TimeSpan.FromTicks( + TimeSpan.FromMilliseconds(uint.MaxValue - 1).Ticks + / MaximumBackoffMultiplier); + + /// + /// Gets or sets the maximum number of pending messages in the inbox. + /// When this limit is reached, new message deliveries will return DeliveryResult.Backpressured(). + /// + /// + /// + /// A lower value (e.g., 100) provides stronger backpressure but may reduce throughput. + /// A higher value (e.g., 10,000) allows more buffering but increases memory usage and recovery time. + /// + /// + /// The inbox capacity is checked before accepting new messages. Messages are persisted to durable + /// storage, so capacity limits affect both in-memory state and storage I/O during recovery. + /// + /// + /// + /// The maximum inbox capacity. Must be greater than zero. Defaults to 1000. + /// + public int MaxCapacity { get; set; } = 1000; + + /// + /// Gets or sets the time window for tracking processed messages to prevent duplicates. + /// Messages that were processed within this window will be rejected with DeliveryResult.Duplicate(). + /// + /// + /// + /// A longer window (e.g., 30 days) provides stronger deduplication guarantees but increases + /// memory usage and storage I/O. A shorter window (e.g., 1 hour) reduces overhead but may + /// allow duplicate processing if retries are delayed. + /// + /// + /// Processed message tracking uses the exact receiver-local MessageId with timestamps. + /// Expired entries are removed atomically when a replay is accepted and are also eligible for + /// compaction during inbox pump maintenance. + /// + /// + /// Consider your retry policies when setting this value. For example, if senders retry for + /// up to 24 hours, set the window to at least 48 hours to ensure deduplication coverage. + /// + /// + /// + /// The deduplication window. Must be greater than zero. Defaults to 7 days. + /// + public TimeSpan DeduplicationWindow { get; set; } = TimeSpan.FromDays(7); + + /// + /// Gets or sets the base delay between retry attempts when delivery encounters backpressure. + /// + /// + /// + /// When the target inbox is at capacity and returns DeliveryResult.Backpressured(), + /// the outbox delivery pump applies exponential backoff from this duration before retrying. + /// + /// + /// A shorter delay (e.g., 100ms) enables faster recovery when the target processes messages quickly, + /// but may increase CPU usage during sustained backpressure. A longer delay (e.g., 5 seconds) + /// reduces retry overhead but increases latency for message delivery. + /// + /// + /// For high-throughput scenarios where quick recovery from backpressure is important, + /// consider values between 100-500ms. For less time-sensitive workloads, 1-5 seconds is appropriate. + /// + /// + /// + /// The base backpressure retry delay. Must be greater than zero. Defaults to 1 second. + /// + public TimeSpan BackpressureRetryDelay { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Gets or sets the maximum number of attempts before an inbox message is dead-lettered. + /// + public int MaxProcessingAttempts { get; set; } = 5; + + /// + /// Gets or sets the maximum number of attempts before an outbox message is dead-lettered. + /// + public int MaxDeliveryAttempts { get; set; } = 100; + + /// + /// Gets or sets the maximum age of an outbox message. + /// + public TimeSpan MaxOutboxRetryAge { get; set; } = TimeSpan.FromDays(1); + + /// + /// Gets or sets how long inbox and outbox dead letters are retained. + /// + public TimeSpan DeadLetterRetentionPeriod { get; set; } = TimeSpan.FromDays(30); + + /// + /// Gets or sets the maximum number of dead letters retained per inbox and per outbox. + /// + public int MaxRetainedDeadLetters { get; set; } = 1000; + + /// + /// Gets or sets the maximum number of inbox messages processed by one durable job attempt. + /// + public int InboxBatchSize { get; set; } = 32; + + /// + /// Gets or sets the maximum number of outbox messages processed by one durable job attempt. + /// + public int OutboxBatchSize { get; set; } = 32; + + /// + /// Gets or sets how long an empty outbox retains its acknowledged physical recovery job. + /// + /// + /// The default is 100 milliseconds. Zero selects immediate retirement. Ready local work + /// wakes its pump immediately, while delivery, acceptance, and acknowledgement keep their + /// normal durability boundaries. Once idle, the job is rescheduled at the idle deadline; + /// provider polling and activation latency also contribute to recovery timing. + /// A durable wakeup is scheduled before outbound work can be acknowledged. + /// + /// A non-negative interval within the supported timer range. + public TimeSpan OutboxIdleRetirementGracePeriod { get; set; } = TimeSpan.FromMilliseconds(100); + + /// + /// Validates the configuration values and throws if any are invalid. + /// + /// + /// Thrown if is less than or equal to zero, + /// or if is less than or equal to , + /// or if a retry, retention, or batch option is outside its supported range. + /// + /// + /// This method is typically called by the dependency injection container during service registration + /// to ensure configuration values are valid before the system starts. + /// + public void Validate() + { + if (OutboxIdleRetirementGracePeriod < TimeSpan.Zero + || OutboxIdleRetirementGracePeriod > TimeSpan.FromMilliseconds(uint.MaxValue - 1)) + { + throw new ArgumentOutOfRangeException(nameof(OutboxIdleRetirementGracePeriod), OutboxIdleRetirementGracePeriod, + "OutboxIdleRetirementGracePeriod must be non-negative and within the supported timer range."); + } + + if (MaxCapacity <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxCapacity), MaxCapacity, "MaxCapacity must be greater than zero."); + } + + if (DeduplicationWindow <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(DeduplicationWindow), DeduplicationWindow, "DeduplicationWindow must be greater than TimeSpan.Zero."); + } + + if (BackpressureRetryDelay <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(BackpressureRetryDelay), BackpressureRetryDelay, "BackpressureRetryDelay must be greater than TimeSpan.Zero."); + } + if (BackpressureRetryDelay > MaxSupportedRetryDelay) + { + throw new ArgumentOutOfRangeException( + nameof(BackpressureRetryDelay), + BackpressureRetryDelay, + $"BackpressureRetryDelay must be less than or equal to {MaxSupportedRetryDelay}."); + } + + if (MaxProcessingAttempts <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxProcessingAttempts), MaxProcessingAttempts, "MaxProcessingAttempts must be greater than zero."); + } + + if (MaxDeliveryAttempts <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxDeliveryAttempts), MaxDeliveryAttempts, "MaxDeliveryAttempts must be greater than zero."); + } + + if (MaxOutboxRetryAge <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(MaxOutboxRetryAge), MaxOutboxRetryAge, "MaxOutboxRetryAge must be greater than TimeSpan.Zero."); + } + + if (MaxOutboxRetryAge >= DeduplicationWindow) + { + throw new ArgumentOutOfRangeException( + nameof(MaxOutboxRetryAge), + MaxOutboxRetryAge, + "MaxOutboxRetryAge must be less than DeduplicationWindow."); + } + + if (DeadLetterRetentionPeriod <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(DeadLetterRetentionPeriod), + DeadLetterRetentionPeriod, + "DeadLetterRetentionPeriod must be greater than TimeSpan.Zero."); + } + + if (MaxRetainedDeadLetters <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(MaxRetainedDeadLetters), + MaxRetainedDeadLetters, + "MaxRetainedDeadLetters must be greater than zero."); + } + + if (InboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(InboxBatchSize), InboxBatchSize, "InboxBatchSize must be greater than zero."); + } + + if (OutboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(OutboxBatchSize), OutboxBatchSize, "OutboxBatchSize must be greater than zero."); + } + } +} diff --git a/src/Orleans.Messaging/DeadLetterRetention.cs b/src/Orleans.Messaging/DeadLetterRetention.cs new file mode 100644 index 00000000000..e2388a608e9 --- /dev/null +++ b/src/Orleans.Messaging/DeadLetterRetention.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Orleans.Messaging; + +internal static class DeadLetterRetention +{ + public static bool Compact( + IDictionary entries, + DateTimeOffset now, + TimeSpan retentionPeriod, + int maxRetainedEntries, + Func getTimestamp, + int reservedCapacity = 0) + where TKey : notnull + { + var removed = false; + foreach (var entry in entries + .Where(entry => MessagingTime.IsExpired(now, getTimestamp(entry.Value), retentionPeriod)) + .ToList()) + { + entries.Remove(entry.Key); + removed = true; + } + + var removeCount = entries.Count + reservedCapacity - maxRetainedEntries; + if (removeCount <= 0) + { + return removed; + } + + foreach (var entry in entries.OrderBy(entry => getTimestamp(entry.Value)).Take(removeCount).ToList()) + { + entries.Remove(entry.Key); + removed = true; + } + + return removed; + } +} diff --git a/src/Orleans.Messaging/DeliveryResult.cs b/src/Orleans.Messaging/DeliveryResult.cs new file mode 100644 index 00000000000..6124b87548c --- /dev/null +++ b/src/Orleans.Messaging/DeliveryResult.cs @@ -0,0 +1,55 @@ +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// Result of attempting to deliver a message to an inbox. +/// +[GenerateSerializer, Alias("Orleans.Messaging.DeliveryResult")] +public readonly struct DeliveryResult +{ + /// + /// The status of the delivery attempt. + /// + [Id(0)] + public DeliveryStatus Status { get; init; } + + /// + /// Optional diagnostic message (e.g., reason for rejection). + /// + [Id(1)] + public string? Message { get; init; } + + /// + /// Creates a result indicating the message was accepted and persisted to inbox. + /// + public static DeliveryResult Accepted() => new() { Status = DeliveryStatus.Accepted }; + + /// + /// Creates a result indicating the message was a duplicate. + /// + public static DeliveryResult Duplicate() => new() { Status = DeliveryStatus.Duplicate }; + + /// + /// Creates a result indicating the inbox is at capacity. + /// + public static DeliveryResult Backpressured() => new() { Status = DeliveryStatus.Backpressured }; + + /// + /// Creates a result indicating the receiving inbox has no registered handler. + /// + public static DeliveryResult HandlerNotFound() => new() + { + Status = DeliveryStatus.HandlerNotFound, + Message = "No inbox handler is registered." + }; + + /// + /// Creates a result indicating the message was dead-lettered. + /// + public static DeliveryResult DeadLettered(string reason) => new() + { + Status = DeliveryStatus.DeadLettered, + Message = reason + }; +} diff --git a/src/Orleans.Messaging/DeliveryStatus.cs b/src/Orleans.Messaging/DeliveryStatus.cs new file mode 100644 index 00000000000..d4d24333e2d --- /dev/null +++ b/src/Orleans.Messaging/DeliveryStatus.cs @@ -0,0 +1,32 @@ +namespace Orleans.Messaging; + +/// +/// Status codes for delivery attempts. +/// +public enum DeliveryStatus +{ + /// + /// Message was accepted and persisted to inbox. + /// + Accepted = 0, + + /// + /// Message was a duplicate (already processed or in inbox). + /// + Duplicate = 1, + + /// + /// Inbox is at capacity; sender should retry later. + /// + Backpressured = 2, + + /// + /// The receiving inbox has no registered handler. + /// + HandlerNotFound = 3, + + /// + /// The message was moved to the receiver's dead-letter store. + /// + DeadLettered = 4 +} diff --git a/src/Orleans.Messaging/Envelope.cs b/src/Orleans.Messaging/Envelope.cs new file mode 100644 index 00000000000..5627271c703 --- /dev/null +++ b/src/Orleans.Messaging/Envelope.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// Contains a command identity and independently encoded headers in one managed buffer. +/// +/// Build an envelope before publication using . Payload is required +/// and can be empty. Subject, sender, and application headers are optional. Raw views share the final +/// buffer and remain available across asynchronous preparation. Keep the published bytes unchanged; +/// ordinary Orleans deep copying and deserialization produce independent buffers. +/// +[GenerateSerializer, Alias("Orleans.Messaging.Envelope")] +public readonly struct Envelope +{ + [Id(1)] + private readonly byte[]? _headers; + + internal Envelope(HierarchicalKey messageId, byte[] headers) + { + MessageId = messageId; + _headers = headers; + } + + /// Gets the exact application-defined identity within the receiving inbox. + [Id(0)] + public HierarchicalKey MessageId { get; } + + /// Gets the required payload as a slice of the encoded header buffer. + public ReadOnlyMemory Payload => PackedEnvelopeHeaders.GetPayload(Data); + + /// Gets the complete packed header directory and values. + /// The bytes follow the current envelope format and remain unchanged after publication. + public ReadOnlyMemory EncodedHeaders => Data; + + /// Enumerates ordinal header names, decoding custom names on demand. + public IEnumerable Keys => PackedEnvelopeHeaders.GetKeys(Data); + + private byte[] Data => _headers ?? throw new FormatException("The envelope header buffer is missing."); + + /// Creates an envelope by validating and copying an encoded header buffer. + /// The required command identity. + /// The complete packed directory and value bytes. + /// An envelope with its own exact-length managed buffer. + /// The identity exceeds admission limits or is unset. + /// The framing, keys, or required payload are invalid. + public static Envelope FromEncodedHeaders(HierarchicalKey messageId, ReadOnlySpan encodedHeaders) + { + EnvelopeValidation.ValidateMessageId(messageId); + PackedEnvelopeHeaders.Validate(encodedHeaders); + return new(messageId, encodedHeaders.ToArray()); + } + + /// Looks up raw value bytes without decoding other headers. + /// The exact ordinal header name. + /// A slice of the common buffer when present. + /// Whether the header is present, including a present empty value. + public bool TryGetBytes(string key, out ReadOnlyMemory value) => PackedEnvelopeHeaders.TryGetBytes(Data, key, out value); + + /// Decodes one independently serialized header using an externally bound serializer. + /// The exact header name. Subjects use raw UTF-8 retrieval. + /// The Orleans serializer for this header's value format. + /// The decoded value, which can be null for a present header. + /// Whether the header is present. Decoding failures propagate to the caller. + public bool TryGetValue(string key, Serializer serializer, out T? value) + { + ArgumentNullException.ThrowIfNull(serializer); + if (key == MessageHeaders.Subject) + { + throw new ArgumentException("Subjects use raw UTF-8 bytes.", nameof(key)); + } + if (TryGetBytes(key, out var bytes)) + { + value = serializer.Deserialize(bytes.Span); + return true; + } + value = default; + return false; + } + + /// Decodes the optional subject for diagnostics or application inspection. + /// The exact subject when present. + /// Whether a subject is present. + /// Dispatch can compare the raw subject slice against preencoded UTF-8 bytes instead. + public bool TryGetSubject(out string? subject) + { + if (TryGetBytes(MessageHeaders.Subject, out var bytes)) + { + subject = PackedEnvelopeHeaders.Utf8.GetString(bytes.Span); + return true; + } + subject = null; + return false; + } +} diff --git a/src/Orleans.Messaging/EnvelopeBuilder.cs b/src/Orleans.Messaging/EnvelopeBuilder.cs new file mode 100644 index 00000000000..ca09bcb9663 --- /dev/null +++ b/src/Orleans.Messaging/EnvelopeBuilder.cs @@ -0,0 +1,155 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; + +namespace Orleans.Messaging; + +/// Encodes headers into temporary pooled storage and publishes one exact-length managed buffer. +/// +/// This builder is single-caller and disposable. Values have independent serialization sessions. +/// A failed value writer leaves its entry unpublished; later successful additions and builds retain +/// only committed entries. Every build creates independent bytes and preserves earlier publications. +/// +public sealed class EnvelopeBuilder : IDisposable, IBufferWriter +{ + private readonly HierarchicalKey _messageId; + private readonly Dictionary _entries = new(StringComparer.Ordinal); + private PooledBuffer _values = new(); + private bool _disposed; + + /// Creates a builder for a required application command identity. + /// The identity, limited to 1,024 UTF-8 bytes and 32 segments. + public EnvelopeBuilder(HierarchicalKey messageId) + { + EnvelopeValidation.ValidateMessageId(messageId); + _messageId = messageId; + } + + /// Copies a raw header value into the common temporary buffer. + /// The unique ordinal name. Subject bytes must be nonempty canonical UTF-8. + /// The bytes to copy, including an empty payload. + /// The key, subject, or duplicate insertion is invalid. + public void AddBytes(string key, ReadOnlySpan value) + { + if (!TryAddBytes(key, value)) + { + throw new ArgumentException("The header already exists.", nameof(key)); + } + } + + /// Adds raw bytes when the header is absent, preserving the first value on a duplicate. + /// The unique ordinal header name. + /// The bytes to copy. + /// Whether the value was added. + public bool TryAddBytes(string key, ReadOnlySpan value) + { + ThrowIfDisposed(); + PackedEnvelopeHeaders.ValidateKey(key); + if (_entries.ContainsKey(key)) return false; + if (key == MessageHeaders.Subject) PackedEnvelopeHeaders.ValidateSubject(value); + var start = _values.Length; + _values.Write(value); + _entries.Add(key, (start, value.Length)); + return true; + } + + /// Serializes one value directly into the common buffer using an independent session. + /// The header value type. + /// The unique header name. Subjects use . + /// The value, including a typed null. + /// The externally bound serializer. + /// The key is a duplicate or uses typed subject encoding. + public void AddValue(string key, T? value, Serializer serializer) + { + if (!TryAddValue(key, value, serializer)) + { + throw new ArgumentException("The header already exists.", nameof(key)); + } + } + + /// Serializes a value only when its header is absent. + /// The unique ordinal header name. + /// The value to encode. + /// The externally bound serializer. + /// Whether the value was added. Writer failures propagate without publishing an entry. + public bool TryAddValue(string key, T? value, Serializer serializer) + { + ThrowIfDisposed(); + ArgumentNullException.ThrowIfNull(serializer); + PackedEnvelopeHeaders.ValidateKey(key); + if (key == MessageHeaders.Subject) throw new ArgumentException("Subjects use raw UTF-8 bytes.", nameof(key)); + if (_entries.ContainsKey(key)) return false; + var start = _values.Length; + serializer.Serialize(value!, this); + _entries.Add(key, (start, _values.Length - start)); + return true; + } + + /// Publishes committed entries in one buffer, with the required payload first. + /// A complete envelope owning independent managed bytes. + /// The required payload was not added. + public Envelope Build() + { + ThrowIfDisposed(); + if (!_entries.TryGetValue(MessageHeaders.Payload, out var payload)) + { + throw new InvalidOperationException("The payload header is required, including for an empty payload."); + } + var directoryLength = PackedEnvelopeHeaders.VarIntLength(payload.Length); + var valueLength = payload.Length; + foreach (var entry in _entries) + { + if (entry.Key == MessageHeaders.Payload) continue; + var token = PackedEnvelopeHeaders.GetKeyToken(entry.Key); + directoryLength = checked(directoryLength + PackedEnvelopeHeaders.VarIntLength(token) + + (token >= 4 ? token - 3 : 0) + PackedEnvelopeHeaders.VarIntLength(entry.Value.Length)); + valueLength = checked(valueLength + entry.Value.Length); + } + var prefixLength = 1 + PackedEnvelopeHeaders.VarIntLength(_entries.Count) + PackedEnvelopeHeaders.VarIntLength(directoryLength); + var result = new byte[checked(prefixLength + directoryLength + valueLength)]; + var position = 0; + result[position++] = PackedEnvelopeHeaders.Version; + PackedEnvelopeHeaders.WriteVarInt(result, ref position, _entries.Count); + PackedEnvelopeHeaders.WriteVarInt(result, ref position, directoryLength); + PackedEnvelopeHeaders.WriteVarInt(result, ref position, payload.Length); + var valuePosition = prefixLength + directoryLength; + CopyValue(payload, result, ref valuePosition); + foreach (var entry in _entries) + { + if (entry.Key == MessageHeaders.Payload) continue; + var token = PackedEnvelopeHeaders.GetKeyToken(entry.Key); + PackedEnvelopeHeaders.WriteVarInt(result, ref position, token); + if (token >= 4) + { + position += PackedEnvelopeHeaders.Utf8.GetBytes(entry.Key, result.AsSpan(position, token - 3)); + } + PackedEnvelopeHeaders.WriteVarInt(result, ref position, entry.Value.Length); + CopyValue(entry.Value, result, ref valuePosition); + } + System.Diagnostics.Debug.Assert(position == prefixLength + directoryLength && valuePosition == result.Length); + return new(_messageId, result); + } + + private void CopyValue((int Offset, int Length) entry, byte[] result, ref int position) + { + _values.AsReadOnlySequence().Slice(entry.Offset, entry.Length).CopyTo(result.AsSpan(position, entry.Length)); + position += entry.Length; + } + + /// Releases temporary storage. Previously built envelopes retain their managed bytes. + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _values.Dispose(); + _entries.Clear(); + } + + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); + + void IBufferWriter.Advance(int count) { ThrowIfDisposed(); _values.Advance(count); } + Memory IBufferWriter.GetMemory(int sizeHint) { ThrowIfDisposed(); return _values.GetMemory(sizeHint); } + Span IBufferWriter.GetSpan(int sizeHint) { ThrowIfDisposed(); return _values.GetSpan(sizeHint); } +} diff --git a/src/Orleans.Messaging/EnvelopeEquivalence.cs b/src/Orleans.Messaging/EnvelopeEquivalence.cs new file mode 100644 index 00000000000..dc40394cbee --- /dev/null +++ b/src/Orleans.Messaging/EnvelopeEquivalence.cs @@ -0,0 +1,23 @@ +using System; + +namespace Orleans.Messaging; + +internal static class EnvelopeEquivalence +{ + public static bool AreEquivalent(OutboxMessage left, OutboxMessage right) => + left.ReceiverId == right.ReceiverId && AreSameCommand(left.Envelope, right.Envelope); + + public static bool AreSameCommand(InboxMessage left, InboxMessage right) => AreSameCommand(left.Envelope, right.Envelope); + + public static bool AreSameCommand(Envelope left, Envelope right) + { + if (left.MessageId != right.MessageId) return false; + var leftPayload = left.Payload; + var rightPayload = right.Payload; + if (leftPayload.Length != rightPayload.Length) return false; + var hasLeft = left.TryGetBytes(MessageHeaders.Subject, out var leftSubject); + var hasRight = right.TryGetBytes(MessageHeaders.Subject, out var rightSubject); + return hasLeft == hasRight && (!hasLeft || leftSubject.Span.SequenceEqual(rightSubject.Span)) + && leftPayload.Span.SequenceEqual(rightPayload.Span); + } +} diff --git a/src/Orleans.Messaging/EnvelopeValidation.cs b/src/Orleans.Messaging/EnvelopeValidation.cs new file mode 100644 index 00000000000..a20b87b329e --- /dev/null +++ b/src/Orleans.Messaging/EnvelopeValidation.cs @@ -0,0 +1,43 @@ +using System; +using System.Text; + +namespace Orleans.Messaging; + +internal static class EnvelopeValidation +{ + internal const int MaxMessageIdBytes = 1024; + internal const int MaxMessageIdSegments = 32; + internal const int MaxSubjectBytes = 256; + private static readonly UTF8Encoding Utf8 = new(false, true); + + // External admission and recovery validate generated values before publishing shared mutations. + public static void Validate(Envelope envelope) + { + ValidateMessageId(envelope.MessageId); + PackedEnvelopeHeaders.Validate(envelope.EncodedHeaders.Span); + } + + public static void Validate(InboxMessage message) => Validate(message.Envelope); + + public static void Validate(OutboxMessage message) + { + if (message.ReceiverId.IsDefault) throw new ArgumentException("The outgoing receiver must not be default.", nameof(message)); + Validate(message.Envelope); + } + + internal static void ValidateMessageId(HierarchicalKey messageId) + { + if (messageId.IsDefault) throw new ArgumentException("The message ID must not be unset.", nameof(messageId)); + if (messageId.SegmentCount > MaxMessageIdSegments) + throw new ArgumentException($"The message ID exceeds {MaxMessageIdSegments} segments.", nameof(messageId)); + if (Utf8.GetByteCount(messageId.ToString()) > MaxMessageIdBytes) + throw new ArgumentException($"The message ID exceeds {MaxMessageIdBytes} UTF-8 bytes.", nameof(messageId)); + } + + public static void ValidateSubject(string subject) + { + ArgumentException.ThrowIfNullOrEmpty(subject); + if (Utf8.GetByteCount(subject) > MaxSubjectBytes) + throw new ArgumentException($"The subject exceeds {MaxSubjectBytes} UTF-8 bytes.", nameof(subject)); + } +} diff --git a/src/Orleans.Messaging/HierarchicalKey.cs b/src/Orleans.Messaging/HierarchicalKey.cs new file mode 100644 index 00000000000..0debc8647be --- /dev/null +++ b/src/Orleans.Messaging/HierarchicalKey.cs @@ -0,0 +1,289 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text; + +namespace Orleans.Messaging; + +/// +/// An immutable, ordinal application identity formed from nonempty hierarchical segments. +/// +/// +/// Create constructs literal segments and escapes slash and backslash characters exactly once. +/// Parse reads the canonical escaped path. Assignment shares immutable backing data; equality and +/// hashing use the full canonical identity. Applications preserve the identity across retries. +/// +[Immutable, Alias("Orleans.Messaging.HierarchicalKey")] +public readonly struct HierarchicalKey : ISpanFormattable, IEquatable, IParsable, ISpanParsable +{ + /// The escape character used within canonical segments. + public const char EscapeCharacter = '\\'; + + /// The separator between canonical segments. + public const char SegmentSeparator = '/'; + + private readonly KeyData? _data; + + private HierarchicalKey(string canonical, int segmentCount) => _data = new(canonical, segmentCount); + + /// Gets whether this value is unset. + public bool IsDefault => _data is null; + + /// Gets the canonical path length in UTF-16 characters. + public int Length => _data?.Canonical.Length ?? 0; + + /// Gets the number of segments, or zero for an unset key. + public int SegmentCount => _data?.SegmentCount ?? 0; + + /// Creates one literal segment, escaping slash and backslash characters. + /// The nonempty literal segment. + /// The segment identity. + /// is null. + /// is empty. + public static HierarchicalKey Create(string value) + { + ArgumentException.ThrowIfNullOrEmpty(value); + return new(Escape(value), 1); + } + + /// Creates a hierarchy from literal segments in root-first order. + /// The nonempty literal segments. + /// A flat canonical identity which owns its immutable backing string. + /// A segment is null. + /// The input or a segment is empty. + public static HierarchicalKey Create(params ReadOnlySpan values) + { + if (values.IsEmpty) + { + throw new ArgumentException("Values must not be empty.", nameof(values)); + } + var builder = new StringBuilder(); + foreach (var value in values) + { + ArgumentException.ThrowIfNullOrEmpty(value); + if (builder.Length > 0) builder.Append(SegmentSeparator); + AppendEscaped(builder, value); + } + return new(builder.ToString(), values.Length); + } + + /// Appends one literal child segment. + /// The nonempty literal child segment. + /// The child identity. + /// This key is unset. + /// is null. + /// is empty. + public HierarchicalKey CreateChildKey(string value) + { + EnsureSet(); + ArgumentException.ThrowIfNullOrEmpty(value); + return new(string.Concat(_data!.Canonical, "/", Escape(value)), checked(SegmentCount + 1)); + } + + /// Composes this hierarchy with an already constructed suffix hierarchy. + /// The constructed suffix. + /// A flat concatenated identity preserving both paths' segment boundaries. + /// This key is unset. + /// is unset. + public HierarchicalKey Append(HierarchicalKey suffix) + { + EnsureSet(); + if (suffix.IsDefault) throw new ArgumentException("The suffix must not be unset.", nameof(suffix)); + return new(string.Concat(_data!.Canonical, "/", suffix._data!.Canonical), checked(SegmentCount + suffix.SegmentCount)); + } + + /// Gets the immediate parent, or null for a root or unset key. + /// The parent identity when this key has more than one segment. + public HierarchicalKey? GetParent() + { + if (SegmentCount < 2) return null; + var lastSeparator = 0; + var path = _data!.Canonical.AsSpan(); + for (var i = 0; i < path.Length; i++) + { + if (path[i] == EscapeCharacter) i++; + else if (path[i] == SegmentSeparator) lastSeparator = i; + } + return new HierarchicalKey(_data.Canonical[..lastSeparator], SegmentCount - 1); + } + + /// Tests whether this key is the other's immediate child. + /// The potential parent. + /// Whether there is exactly one additional segment. + public bool IsChildOf(HierarchicalKey other) => other.IsParentOf(this); + + /// Tests whether this key is the other's immediate parent. + /// The potential child. + /// Whether the other key extends this key by exactly one segment. + public bool IsParentOf(HierarchicalKey other) => !IsDefault && other.SegmentCount == SegmentCount + 1 && IsAncestorOf(other); + + /// Tests whether this key is equal to or an ancestor of the other key. + /// The identity to inspect. + /// Whether the full prefix consists of equal ordinal segments. Unset keys return false. + public bool IsAncestorOf(HierarchicalKey other) => !IsDefault && !other.IsDefault + && (Equals(other) || (other.Length > Length && other._data!.Canonical[Length] == SegmentSeparator + && other._data.Canonical.StartsWith(_data!.Canonical, StringComparison.Ordinal))); + + /// + public static HierarchicalKey Parse(string s, IFormatProvider? provider = null) + { + ArgumentNullException.ThrowIfNull(s); + return TryParse(s, provider, out var result) ? result : throw new FormatException("The value is not a valid canonical hierarchical key."); + } + + /// + public static HierarchicalKey Parse(ReadOnlySpan s, IFormatProvider? provider = null) => + TryParse(s, provider, out var result) ? result : throw new FormatException("The value is not a valid canonical hierarchical key."); + + /// + public static bool TryParse([NotNullWhen(true)] string? s, IFormatProvider? provider, out HierarchicalKey result) + { + if (s is not null && TryCountSegments(s, out var count)) + { + result = new(s, count); + return true; + } + result = default; + return false; + } + + /// + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, out HierarchicalKey result) + { + if (TryCountSegments(s, out var count)) + { + result = new(new string(s), count); + return true; + } + result = default; + return false; + } + + /// + public bool Equals(HierarchicalKey other) => ReferenceEquals(_data, other._data) + || string.Equals(_data?.Canonical, other._data?.Canonical, StringComparison.Ordinal); + + /// + public override bool Equals(object? obj) => obj is HierarchicalKey other && Equals(other); + + /// + public override int GetHashCode() => _data?.Hash ?? 0; + + /// Compares complete ordinal key values. + /// The first identity. + /// The second identity. + /// Whether the identities are equal. + public static bool operator ==(HierarchicalKey left, HierarchicalKey right) => left.Equals(right); + + /// Compares complete ordinal key values for inequality. + /// The first identity. + /// The second identity. + /// Whether the identities differ. + public static bool operator !=(HierarchicalKey left, HierarchicalKey right) => !left.Equals(right); + + /// + public override string ToString() => _data?.Canonical ?? string.Empty; + + /// + public string ToString(string? format, IFormatProvider? formatProvider) => ToString(); + + /// + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + if (ToString().AsSpan().TryCopyTo(destination)) + { + charsWritten = Length; + return true; + } + charsWritten = 0; + return false; + } + + /// Enumerates escaped canonical segment spans in root-first order. + /// An allocation-free segment enumerator. + public SegmentEnumerator GetEnumerator() => new(ToString().AsSpan()); + + private void EnsureSet() + { + if (IsDefault) throw new InvalidOperationException("The key must not be unset."); + } + + private static string Escape(string value) + { + if (value.AsSpan().IndexOfAny(EscapeCharacter, SegmentSeparator) < 0) return value; + var builder = new StringBuilder(value.Length); + AppendEscaped(builder, value); + return builder.ToString(); + } + + private static void AppendEscaped(StringBuilder builder, string value) + { + foreach (var character in value) + { + if (character is EscapeCharacter or SegmentSeparator) builder.Append(EscapeCharacter); + builder.Append(character); + } + } + + private static bool TryCountSegments(ReadOnlySpan path, out int count) + { + count = 0; + var segmentLength = 0; + for (var i = 0; i < path.Length; i++) + { + if (path[i] == EscapeCharacter) + { + if (++i == path.Length || path[i] is not (EscapeCharacter or SegmentSeparator)) return false; + } + else if (path[i] == SegmentSeparator) + { + if (segmentLength == 0) return false; + count++; + segmentLength = 0; + continue; + } + segmentLength++; + } + if (segmentLength == 0) return false; + count++; + return true; + } + + private sealed class KeyData(string canonical, int segmentCount) + { + public string Canonical { get; } = canonical; + public int Hash { get; } = canonical.GetHashCode(StringComparison.Ordinal); + public int SegmentCount { get; } = segmentCount; + } + + /// Enumerates borrowed spans of the immutable canonical key. + public ref struct SegmentEnumerator + { + private readonly ReadOnlySpan _path; + private int _next; + + internal SegmentEnumerator(ReadOnlySpan path) => _path = path; + + /// Gets the current escaped canonical segment. + public ReadOnlySpan Current { get; private set; } + + /// Advances to the next segment. + /// Whether a segment is available. + public bool MoveNext() + { + if (_next >= _path.Length) + { + Current = default; + return false; + } + var start = _next; + for (; _next < _path.Length; _next++) + { + if (_path[_next] == EscapeCharacter) _next++; + else if (_path[_next] == SegmentSeparator) break; + } + Current = _path[start.._next]; + _next++; + return true; + } + } +} diff --git a/src/Orleans.Messaging/HierarchicalKeyCodec.cs b/src/Orleans.Messaging/HierarchicalKeyCodec.cs new file mode 100644 index 00000000000..98a43ecea81 --- /dev/null +++ b/src/Orleans.Messaging/HierarchicalKeyCodec.cs @@ -0,0 +1,48 @@ +using System; +using System.Buffers; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Cloning; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging; + +[RegisterSerializer] +internal sealed class HierarchicalKeyCodec(IFieldCodec strings) : IFieldCodec +{ + public void WriteField(ref Writer writer, uint fieldIdDelta, + Type? expectedType, HierarchicalKey value) where TBufferWriter : IBufferWriter + { + ReferenceCodec.MarkValueField(writer.Session); + writer.WriteFieldHeader(fieldIdDelta, expectedType, typeof(HierarchicalKey), WireType.TagDelimited); + strings.WriteField(ref writer, 0, typeof(string), value.IsDefault ? null! : value.ToString()); + writer.WriteEndObject(); + } + + public HierarchicalKey ReadValue(ref Reader reader, Field field) + { + field.EnsureWireTypeTagDelimited(); + ReferenceCodec.MarkValueField(reader.Session); + HierarchicalKey result = default; + uint fieldId = 0; + while (true) + { + var header = reader.ReadFieldHeader(); + if (header.IsEndBaseOrEndObject) return result; + fieldId += header.FieldIdDelta; + if (fieldId == 0) + { + var canonical = strings.ReadValue(ref reader, header); + result = canonical is null ? default : HierarchicalKey.Parse(canonical); + } + else reader.ConsumeUnknownField(header); + } + } +} + +[RegisterCopier] +internal sealed class HierarchicalKeyCopier : IDeepCopier +{ + public HierarchicalKey DeepCopy(HierarchicalKey input, CopyContext context) => input; +} diff --git a/src/Orleans.Messaging/IInbox.cs b/src/Orleans.Messaging/IInbox.cs new file mode 100644 index 00000000000..64dd0ad1127 --- /dev/null +++ b/src/Orleans.Messaging/IInbox.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.Messaging; + +/// +/// Registers a grain's opaque-message handler and exposes pending inbox state. +/// +public interface IInbox +{ + /// + /// Gets the number of unprocessed messages. + /// + int Count { get; } + + /// + /// Gets the capacity at which delivery returns . + /// + int Capacity { get; } + + /// + /// Gets pending messages in unspecified order. + /// + /// Messages share their finalized envelope buffers, which remain unchanged after publication. + IEnumerable Messages { get; } + + /// + /// Looks up a pending command by its application identity within this inbox. + /// + /// The exact command identity, independent of its immediate sender and subject. + /// The matching stored message when found. + /// Whether the message is pending. + bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out InboxMessage message); + + /// + /// Registers the handler for this inbox. + /// + /// The handler responsible for application decoding and dispatch. + /// is null. + /// A handler has already been registered. + void RegisterHandler(IInboxHandler handler); +} diff --git a/src/Orleans.Messaging/IInboxExtension.cs b/src/Orleans.Messaging/IInboxExtension.cs new file mode 100644 index 00000000000..4893ab90e81 --- /dev/null +++ b/src/Orleans.Messaging/IInboxExtension.cs @@ -0,0 +1,36 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Orleans; +using Orleans.Runtime; +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// Non-generic grain extension for inbox message delivery. +/// +[Alias("IInboxExtension")] +public interface IInboxExtension : IGrainExtension +{ + /// + /// Delivers a message to this grain's inbox. + /// + /// The received command with a required identity and payload; its receiver is this grain. + /// Cancels the caller's wait for delivery. + /// + /// Direct admission shares the finalized envelope buffer under the immutable-publication contract. + /// RPC copying and deserialization isolate envelope buffers using ordinary Orleans serialization. + /// Once delivery owns inbox admission, it retains its gate and ownership reservation until + /// its operation completes. Caller cancellation leaves that operation running to its durable outcome. + /// The grain owner keeps delivery quiescent during journal deletion and resumes delivery + /// after the deletion task completes successfully. + /// + /// Result indicating delivery/processing status. + /// + /// has an unset message ID, exceeds identity admission limits, or has invalid header framing. + /// Optional standard metadata is validated where its contract is consumed. + /// + [Alias("DeliverAsync")] + ValueTask DeliverAsync(InboxMessage message, CancellationToken cancellationToken = default); +} diff --git a/src/Orleans.Messaging/IInboxHandler.cs b/src/Orleans.Messaging/IInboxHandler.cs new file mode 100644 index 00000000000..f06bf473ccb --- /dev/null +++ b/src/Orleans.Messaging/IInboxHandler.cs @@ -0,0 +1,41 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Messaging; + +/// +/// Processes opaque messages accepted by a grain's inbox. +/// +/// +/// Register one handler per inbox. Applications perform payload decoding and dispatch in their handler +/// and stage outbound messages through an injected . +/// +public interface IInboxHandler +{ + /// + /// Handles a message and stages its logical completion. + /// + /// The received envelope and attempt-scoped completion operation. + /// The token to check during preparation and before shared mutation. + /// The handler's method outcome, awaited by the inbox runtime. + /// + /// + /// Decode the payload, perform asynchronous I/O, validate local results, build outgoing envelopes, + /// and check cancellation before the first shared business or journaled mutation. + /// + /// + /// From the first shared mutation through completion of this method, execute synchronously. + /// Apply complete, safe-to-commit business changes, stage outgoing messages through + /// , call , and return. + /// This final block includes the method's return after Complete and relies on the trusted + /// handler contract. Every successful outcome calls Complete, including outcomes with no business effects. + /// + /// + /// Complete stages inbox completion and transport deduplication alongside the business changes + /// and outgoing intents. The runtime owns the subsequent journal write, persistence acknowledgement, + /// and retirement. Failures during local preparation follow the inbox retry and dead-letter policy. + /// An error after Complete preserves the completed logical outcome and is reported by the runtime. + /// + /// + ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken); +} diff --git a/src/Orleans.Messaging/IInboxHandlerContext.cs b/src/Orleans.Messaging/IInboxHandlerContext.cs new file mode 100644 index 00000000000..752047b06e5 --- /dev/null +++ b/src/Orleans.Messaging/IInboxHandlerContext.cs @@ -0,0 +1,31 @@ +namespace Orleans.Messaging; + +/// +/// Exposes the received message and its attempt-scoped logical completion. +/// +public interface IInboxHandlerContext +{ + /// + /// Gets the received message and its finalized application bytes. + /// + /// + /// The context keeps the message available through actual handler completion, including + /// after Complete removes the pending message. Keep its envelope buffer unchanged while handling the command. + /// + InboxMessage Message { get; } + + /// + /// Synchronously stages inbox completion and transport deduplication for the active attempt. + /// + /// + /// Apply safe-to-commit business mutations and stage outgoing messages before calling Complete + /// in the same synchronous final block. Return from the handler without further awaits. + /// The runtime owns the subsequent journal write and acknowledgement. + /// Repeated completion within the same active attempt coalesces. Completion retains its + /// logical outcome when cancellation arrives after the final block starts. + /// + /// + /// The context is retired or belongs to another activation or handler attempt. + /// + void Complete(); +} diff --git a/src/Orleans.Messaging/IMessagingDiagnostics.cs b/src/Orleans.Messaging/IMessagingDiagnostics.cs new file mode 100644 index 00000000000..fb73bd880ea --- /dev/null +++ b/src/Orleans.Messaging/IMessagingDiagnostics.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Provides operational access to a grain's messaging state. +/// +public interface IMessagingDiagnostics +{ + /// + /// Gets messages which failed during inbox processing. + /// + IReadOnlyList> InboxDeadLetters { get; } + + /// + /// Gets messages which could not be delivered from the outbox. + /// + IReadOnlyList> OutboxDeadLetters { get; } + + /// + /// Stages removal of an inbox dead letter. + /// + /// The receiver-local command identity. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveInboxDeadLetter(HierarchicalKey messageId); + + /// + /// Stages removal of an outbox dead letter. + /// + /// The message identifier. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveOutboxDeadLetter(HierarchicalKey messageId); +} + +/// +/// Describes a dead-lettered durable message. +/// +/// The incoming command or outgoing routed intent type. +public sealed class DeadLetter +{ + /// + /// Gets the message. + /// + /// + /// Treat the message payload as immutable. Ordinary RPC serialization copies the payload + /// when returning diagnostic results. + /// + public required TMessage Message { get; init; } + + /// + /// Gets when the message was dead-lettered. + /// + public DateTimeOffset DeadLetteredAt { get; init; } + + /// + /// Gets the terminal failure reason. + /// + public required string Reason { get; init; } + + /// + /// Gets the number of attempts made. + /// + public int AttemptCount { get; init; } +} + +internal sealed class MessagingDiagnostics( + [Microsoft.Extensions.DependencyInjection.FromKeyedServices(MessagingStateNames.InboxDeadLetters)] + IDurableDictionary inbox, + [Microsoft.Extensions.DependencyInjection.FromKeyedServices(MessagingStateNames.OutboxDeadLetters)] + IDurableDictionary outbox) : IMessagingDiagnostics +{ + public IReadOnlyList> InboxDeadLetters => + inbox.Values.Select(static entry => new DeadLetter + { + Message = entry.Message, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount + }).ToList(); + + public IReadOnlyList> OutboxDeadLetters => + outbox.Values.Select(static entry => new DeadLetter + { + Message = entry.Message, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount + }).ToList(); + + public bool RemoveInboxDeadLetter(HierarchicalKey messageId) => + inbox.Remove(messageId); + + public bool RemoveOutboxDeadLetter(HierarchicalKey messageId) => outbox.Remove(messageId); +} diff --git a/src/Orleans.Messaging/IMessagingGrain.cs b/src/Orleans.Messaging/IMessagingGrain.cs new file mode 100644 index 00000000000..718d6952c0f --- /dev/null +++ b/src/Orleans.Messaging/IMessagingGrain.cs @@ -0,0 +1,14 @@ +namespace Orleans.Messaging; + +/// +/// Identifies grain implementations which initialize inbox and outbox services during activation setup. +/// +/// +/// Implement this local capability on a grain class, an application base class, or an application grain interface. +/// With messaging services registered, selected activations validate their execution model and bind +/// messaging state before journal recovery. Grains deriving from +/// receive the same setup automatically. +/// +public interface IMessagingGrain +{ +} diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs new file mode 100644 index 00000000000..69e135efc09 --- /dev/null +++ b/src/Orleans.Messaging/IOutbox.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Stages opaque durable messages alongside a grain's journaled business state. +/// +/// +/// The journal capture hook establishes a durable self-wakeup before capturing pending state. +/// Dispatch begins after persistence acknowledgement. Messages remain pending until the destination +/// acknowledges durable acceptance, recognizes a duplicate, or reports a terminal delivery outcome. +/// Applications namespace command identities and define ordering in their protocols. +/// +public interface IOutbox +{ + /// + /// Gets the grain identity which owns this outbox and sends its messages. + /// + GrainId SenderId { get; } + + /// + /// Gets the number of pending outbound messages. + /// + int Count { get; } + + /// + /// Gets pending messages in unspecified order. + /// + /// Messages share their finalized envelope buffers, which remain unchanged after publication. + IEnumerable Messages { get; } + + /// + /// Synchronously stages an outgoing command for the grain's next journal write. + /// + /// The fully built outgoing message and required receiver. + /// + /// Direct sends share the finalized envelope buffer. Keep its bytes unchanged after publication. + /// A supplied sender must identify this outbox's grain. Equivalent repeated identities retain + /// the original message and headers; payload, optional subject, and destination must match. + /// Custom headers and sender provenance do not change pending intent equality. Conflicts fail explicitly. + /// Inbox handlers stage outgoing + /// messages in their synchronous final block before calling . + /// Ordinary callers persist staged messages using their journaled state manager. + /// An explicit write retry retains pending business changes and messages after a scheduling failure. + /// + /// The envelope has an unset identity, invalid subject or destination, or exceeds identity/subject limits. + /// + /// The sender differs from the owning grain, the identity conflicts, or the outbox is unavailable. + /// + void Send(OutboxMessage message); + + /// + /// Looks up a pending outbound message. + /// + /// The message identifier. + /// The matching stored message when found. + /// Whether the message is pending. + bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage message); +} diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs new file mode 100644 index 00000000000..edbfa198acd --- /dev/null +++ b/src/Orleans.Messaging/Inbox.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Provides access to journaled pending messages and their capacity limit. +/// Registers the single handler for this inbox. +/// +internal sealed class Inbox : IInbox +{ + private readonly IDurableDictionary _inbox; + private IInboxHandler? _handler; + private readonly int _capacity; + + /// + /// Creates an inbox over journaled message storage. + /// + /// Durable dictionary for storing unprocessed messages. + /// Maximum inbox capacity (default: 1000). + public Inbox( + IDurableDictionary inbox, + int capacity = 1000) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity); + + _inbox = inbox; + _capacity = capacity; + } + + internal Inbox( + IDurableDictionary inbox, + IEnumerable handlers, + int capacity) + : this(inbox, capacity) + { + foreach (var handler in handlers) + { + RegisterHandler(handler); + } + } + + /// + /// Number of unprocessed messages. + /// + public int Count => _inbox.Count; + + /// + /// Gets the maximum inbox capacity. + /// + public int Capacity => _capacity; + + /// + /// Gets all pending messages (no ordering guarantee). + /// + public IEnumerable Messages => _inbox.Values; + + /// + /// Tries to get a specific message by its key. + /// + /// The receiver-local command identity. + /// The envelope if found. + /// True if the message exists in the inbox; otherwise, false. + public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out InboxMessage envelope) => + _inbox.TryGetValue(messageId, out envelope); + + /// + /// Registers the single handler for this inbox. + /// + public void RegisterHandler(IInboxHandler handler) + { + ArgumentNullException.ThrowIfNull(handler); + if (_handler is not null) + { + throw new InvalidOperationException("A handler is already registered for this inbox."); + } + + _handler = handler; + } + + internal bool TryGetHandler([MaybeNullWhen(false)] out IInboxHandler handler) + { + handler = _handler; + return handler is not null; + } +} diff --git a/src/Orleans.Messaging/InboxExtension.cs b/src/Orleans.Messaging/InboxExtension.cs new file mode 100644 index 00000000000..1a04ddfc353 --- /dev/null +++ b/src/Orleans.Messaging/InboxExtension.cs @@ -0,0 +1,1409 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.TypeSystem; +using Orleans.Timers; + +namespace Orleans.Messaging; + +/// +/// Implementation of inbox extension for grain message delivery. +/// Handles message persistence, deduplication, and processing. +/// +internal sealed partial class InboxExtension : + IInboxExtension, + IDurableJobFeatureHandler, + ILifecycleObserver, + IDisposable +{ + internal const string JobName = "orleans.messaging.inbox-drain"; + + // Nested handler requests and independent interleaved calls have different logical execution contexts. + private static readonly AsyncLocal _handlerExecution = new(); + + public bool CanHandle(string jobName) => string.Equals(jobName, JobName, StringComparison.Ordinal); + + private readonly IGrainContext _grainContext; + private readonly string _grainType; + private readonly ITimerRegistry _timerRegistry; + private readonly IJournaledStateManager _stateManager; + private readonly ILogger _logger; + private readonly MessagingInstruments _instruments; + private readonly Inbox _inbox; + private readonly IDictionary _inboxDict; + private readonly IDictionary _processed; + private readonly IDictionary _messageStates; + private readonly IDictionary _deadLetters; + private readonly IDurableValue _jobId; + private readonly IDurableValue _job; + private readonly IDurableValue _completedJobId; + private readonly IDurableValue _jobSequence; + private readonly ILocalDurableJobManager _jobManager; + private readonly TimeProvider _timeProvider; + private readonly TimeProvider _jobTimeProvider; + private readonly HashSet _provisionalAcceptances = []; + private readonly MessagingPumpResults _pumpResults; + private readonly MessagingPumpCoordinator _pumpCoordinator = new(); + private readonly int _maxCapacity; + private readonly TimeSpan _deduplicationWindow; + private readonly TimeSpan _processedCompactionInterval; + private readonly int _maxProcessingAttempts; + private readonly int _batchSize; + private readonly TimeSpan _retryDelay; + private readonly TimeSpan _deadLetterRetentionPeriod; + private readonly int _maxRetainedDeadLetters; + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly CancellationTokenSource _shutdownCts = new(); + private readonly CancellationToken _shutdownToken; + private Task _activeDelivery = Task.CompletedTask; + private PumpTimerState? _pumpTimer; + private LocalDrainTimerState? _localDrainTimer; + private bool _localDrainRequested; + private int _disposed; + private int _metricsActive; + private int _reportedDepth; + private string? _durableOwnershipId; + private DurableJob? _durableJob; + private string _ownershipEpoch = Guid.NewGuid().ToString("N"); + private long _stateGeneration; + private bool _recoveryCompleted; + private string? _ownershipStateError; + private long _reservedSequence; + private ExceptionDispatchInfo? _failure; + private readonly HashSet _pendingOwnershipIds = new(StringComparer.Ordinal); + private readonly List _pendingWrites = []; + private string? _durableCompletedJobId; + private DateTimeOffset? _nextProcessedExpiry; + private DateTimeOffset? _lastProcessedCompaction; + + /// + /// Creates a new inbox extension instance. + /// + /// The grain context for this extension. + /// State manager for atomic persistence. + /// Logger for diagnostics. + /// Journaling metrics. + /// The grain's inbox (shared with grain DI). + /// Durable dictionary for inbox messages. + /// Durable dictionary for processed message tracking. + /// Messaging options. + public InboxExtension( + IGrainContext grainContext, + ITimerRegistry timerRegistry, + IJournaledStateManager stateManager, + ILogger logger, + MessagingInstruments instruments, + Inbox inbox, + IDictionary inboxDict, + IDictionary processed, + IDictionary messageStates, + IDictionary deadLetters, + IDurableValue jobId, + IDurableValue job, + IDurableValue completedJobId, + IDurableValue jobSequence, + ILocalDurableJobManager jobManager, + IDurableJobHandlerRegistry jobHandlers, + MessagingPumpResults pumpResults, + TimeProvider timeProvider, + TimeProvider jobTimeProvider, + InboxOptions options) + { + ArgumentNullException.ThrowIfNull(grainContext); + ArgumentNullException.ThrowIfNull(timerRegistry); + ArgumentNullException.ThrowIfNull(stateManager); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(instruments); + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(inboxDict); + ArgumentNullException.ThrowIfNull(processed); + ArgumentNullException.ThrowIfNull(messageStates); + ArgumentNullException.ThrowIfNull(deadLetters); + ArgumentNullException.ThrowIfNull(jobId); + ArgumentNullException.ThrowIfNull(job); + ArgumentNullException.ThrowIfNull(completedJobId); + ArgumentNullException.ThrowIfNull(jobSequence); + ArgumentNullException.ThrowIfNull(jobManager); + ArgumentNullException.ThrowIfNull(jobHandlers); + ArgumentNullException.ThrowIfNull(pumpResults); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentNullException.ThrowIfNull(jobTimeProvider); + ArgumentNullException.ThrowIfNull(options); + _shutdownToken = _shutdownCts.Token; + _grainContext = grainContext; + _grainType = grainContext.GrainId.Type.ToString(); + _timerRegistry = timerRegistry; + _stateManager = stateManager; + _logger = logger; + _instruments = instruments; + _inbox = inbox; + _inboxDict = inboxDict; + _processed = processed; + _messageStates = messageStates; + _deadLetters = deadLetters; + _jobId = jobId; + _job = job; + _completedJobId = completedJobId; + _jobSequence = jobSequence; + _jobManager = jobManager; + _pumpResults = pumpResults; + _timeProvider = timeProvider; + _jobTimeProvider = jobTimeProvider; + _maxCapacity = options.MaxCapacity; + _deduplicationWindow = options.DeduplicationWindow; + _processedCompactionInterval = TimeSpan.FromTicks(Math.Max(1, _deduplicationWindow.Ticks / 4)); + _maxProcessingAttempts = options.MaxProcessingAttempts; + _batchSize = options.InboxBatchSize; + _retryDelay = options.BackpressureRetryDelay; + _deadLetterRetentionPeriod = options.DeadLetterRetentionPeriod; + _maxRetainedDeadLetters = options.MaxRetainedDeadLetters; + jobHandlers.Register(this); + grainContext.ObservableLifecycle.Subscribe( + RuntimeTypeNameFormatter.Format(GetType()), + GrainLifecycleStage.Activate, + this); + } + + private static void ThrowIfHandlerOperationRejected(HandlerExecution execution) => execution.RejectionFailure?.Throw(); + + public int Count => _inboxDict.Count; + public int Capacity => _maxCapacity; + + public async ValueTask DeliverAsync(InboxMessage envelope, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + ValidateReady(); + EnvelopeValidation.Validate(envelope); + + EnsureMetricsActive(); + // Direct publication shares immutable payload bytes; the admitted task keeps the + // envelope reachable independently of cancellation of its caller's wait. + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + var delivery = DeliverUnderGateAsync(envelope); + _activeDelivery = delivery; + delivery.Ignore(); + return await delivery.WaitAsync(cancellationToken).ConfigureAwait(true); + } + + private async Task DeliverUnderGateAsync(InboxMessage envelope) + { + try + { + ValidateReady(); + var key = envelope.Envelope.MessageId; + if (_processed.TryGetValue(key, out var processedAt) + && !MessagingTime.IsExpired(_timeProvider.GetUtcNow(), processedAt, _deduplicationWindow)) + { + _instruments.OnInboxMessageReceived(_grainType, "duplicate"); + return DeliveryResult.Duplicate(); + } + + if (_inboxDict.TryGetValue(key, out var pending)) + { + if (!EnvelopeEquivalence.AreSameCommand(pending, envelope)) + { + throw new InvalidOperationException( + $"The inbox already contains a different command with message ID '{key}'."); + } + await EnsureJobScheduledUnderGateAsync(CancellationToken.None).ConfigureAwait(true); + ScheduleLocalDrain(); + _instruments.OnInboxMessageReceived(_grainType, "duplicate"); + return DeliveryResult.Duplicate(); + } + + if (_inboxDict.Count >= _maxCapacity) + { + _instruments.OnInboxMessageReceived(_grainType, "backpressured"); + return DeliveryResult.Backpressured(); + } + + if (!_inbox.TryGetHandler(out _)) + { + _instruments.OnInboxMessageReceived(_grainType, "handler_not_found"); + LogHandlerNotFound(_logger, envelope.Envelope.MessageId, _grainContext.GrainId); + return DeliveryResult.HandlerNotFound(); + } + + var generation = _stateGeneration; + OwnershipProposal? proposal = null; + try + { + if (GetCommittedInboxCount() == 0 || !HasCommittedOwnership()) + { + proposal = await PrepareOwnershipAsync(CancellationToken.None).ConfigureAwait(true); + } + + ValidateReady(); + var operation = new AcceptanceWrite(generation, envelope, proposal); + await SubmitAsync(operation).ConfigureAwait(true); + ValidateReady(); + ScheduleLocalDrain(); + _instruments.OnInboxMessageReceived(_grainType, "accepted"); + LogMessageAccepted(_logger, envelope.Envelope.MessageId, _grainContext.GrainId); + return DeliveryResult.Accepted(); + } + finally + { + if (proposal is not null) + { + _pendingOwnershipIds.Remove(proposal.Id); + } + } + } + catch (Exception exception) + { + LogDeliveryOperationFailed(_logger, exception, envelope.Envelope.MessageId, _grainContext.GrainId); + throw; + } + finally + { + _gate.Release(); + } + } + + private async Task PrepareOwnershipAsync(CancellationToken cancellationToken) + { + ValidateReady(); + var generation = _stateGeneration; + var previousId = _jobId.Value; + var previousJob = _job.Value; + var sequence = checked(++_reservedSequence); + var id = MessagingJobOwnership.CreateId(_ownershipEpoch, sequence); + _pendingOwnershipIds.Add(id); + try + { + using var cancellation = MessagingCancellation.Combine(cancellationToken, _shutdownCts.Token, out var combinedToken); + var job = await _jobManager.ScheduleJobAsync(new ScheduleJobRequest + { + Target = _grainContext.GrainId, + JobName = JobName, + DueTime = _jobTimeProvider.GetUtcNow(), + Metadata = MessagingJobOwnership.CreateMetadata(id) + }, combinedToken).ConfigureAwait(true); + ValidateReady(); + return new(id, sequence, job, generation, previousId, previousJob); + } + catch + { + _pendingOwnershipIds.Remove(id); + _failure?.Throw(); + throw; + } + } + + private async ValueTask EnsureJobScheduledUnderGateAsync(CancellationToken cancellationToken) + { + ValidateReady(); + if (GetCommittedInboxCount() == 0 || HasCommittedOwnership()) + { + return; + } + + var proposal = await PrepareOwnershipAsync(cancellationToken).ConfigureAwait(true); + try + { + await SubmitAsync(new OwnershipWrite(_stateGeneration, proposal)).ConfigureAwait(true); + } + finally + { + _pendingOwnershipIds.Remove(proposal.Id); + } + } + + private async ValueTask SubmitAsync(InboxWrite operation) + { + ValidateReady(); + _pendingWrites.Add(operation); + try + { + if (operation is HandlerWrite handler) + { + await InvokeHandlerAsync(handler).ConfigureAwait(true); + if (handler.Skipped) + { + return; + } + } + + if (operation is not HandlerWrite { Completed: true } && !StageWrite(operation)) + { + return; + } + try + { + await _stateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + } + catch (JournaledStatePostCommitException) + { + AcknowledgeWrite(operation); + throw; + } + catch (Exception exception) + { + LatchFailure(exception); + throw; + } + AcknowledgeWrite(operation); + } + catch (OperationCanceledException) when (_failure is null + && operation is HandlerWrite { Completed: false } handler + && handler.Cancellation.IsCancellationRequested + && handler.Execution?.RejectionFailure is null) + { + throw; + } + catch (Exception exception) when (exception is not JournaledStatePostCommitException + && (exception is not OperationCanceledException || _failure is not null || !_shutdownToken.IsCancellationRequested + || operation is HandlerWrite { Completed: true } + || operation is HandlerWrite { Execution.RejectionFailure: not null })) + { + LatchFailure(exception); + try + { + _grainContext.Deactivate(new DeactivationReason(DeactivationReasonCode.ApplicationError, _failure!.SourceException, "Inbox operation failed.")); + } + catch (Exception deactivationException) + { + LogDeactivationRequestFailure(_logger, deactivationException); + } + _failure!.Throw(); + throw; + } + finally + { + _pendingWrites.Remove(operation); + operation.SignalFinished(); + } + + if (operation is HandlerWrite { PostCompletionFailure: { } failure }) + { + failure.Throw(); + } + } + + private void ValidateReady() + { + _failure?.Throw(); + ThrowIfOwnershipStateInvalid(); + _shutdownToken.ThrowIfCancellationRequested(); + if (!_recoveryCompleted) + { + throw new InvalidOperationException("Inbox initialization must complete before processing."); + } + } + + private void ThrowIfOwnershipStateInvalid() + { + var error = _ownershipStateError ?? MessagingJobOwnership.GetPairError(_jobId.Value, _job.Value); + if (error is not null) + { + throw new InvalidOperationException(error); + } + } + + private void ValidateGeneration(long generation) + { + ValidateReady(); + if (generation != _stateGeneration) + { + throw new InvalidOperationException("The admitted inbox operation belongs to an obsolete activation or deletion generation."); + } + } + + private bool HasCommittedOwnership() => + MessagingJobOwnership.HasOwner(_jobId.Value, _job.Value) + && string.Equals(_durableOwnershipId, _jobId.Value, StringComparison.Ordinal) + && MessagingJobOwnership.IsSamePhysicalJob(_durableJob, _job.Value); + + private bool IsCurrentOwner(PumpOwner owner) => owner.Generation == _stateGeneration + && string.Equals(owner.Id, _jobId.Value, StringComparison.Ordinal) + && MessagingJobOwnership.IsSamePhysicalJob(owner.Job, _job.Value) + && HasCommittedOwnership(); + + private void ValidateOwner(PumpOwner owner) + { + ValidateGeneration(owner.Generation); + if (!IsCurrentOwner(owner)) + { + throw new InvalidOperationException("The admitted inbox operation no longer owns the acknowledged physical job."); + } + } + + // Every provisional key is an inbox key; acknowledgement removes only its provisional marker. + private int GetCommittedInboxCount() => _inboxDict.Count - _provisionalAcceptances.Count; + + private async ValueTask InvokeHandlerAsync(HandlerWrite operation) + { + ValidateOwner(operation.Owner); + if (!_inboxDict.ContainsKey(operation.Key)) + { + throw new InvalidOperationException("The admitted inbox message is no longer pending."); + } + + if (operation.Cancellation.IsCancellationRequested) + { + operation.Skipped = true; + return; + } + + // Both pump entry points already combine activation shutdown with their attempt scope. + // Borrow that scope through handler completion and the owned journal write. + var combinedToken = operation.Cancellation; + var previous = _handlerExecution.Value; + var execution = operation.Execution = new HandlerExecution(this, operation); + _handlerExecution.Value = execution; + try + { + var found = _inbox.TryGetHandler(out var handler); + if (!found) + { + operation.Error = new InvalidOperationException("No inbox handler is registered."); + operation.DeadLetter = true; + return; + } + + execution.Active = true; + await handler!.HandleAsync(new InboxHandlerContext( + operation.Envelope, execution.Complete), + combinedToken).ConfigureAwait(true); + ThrowIfHandlerOperationRejected(execution); + if (!operation.Completed) + { + execution.RejectOperation(new InvalidOperationException( + "Inbox handlers must call Complete before returning successfully.")); + } + } + catch (Exception exception) when (operation.Completed) + { + operation.PostCompletionFailure = execution.RejectionFailure ?? ExceptionDispatchInfo.Capture(exception); + LogHandlerException(_logger, operation.PostCompletionFailure.SourceException, + operation.Envelope.Envelope.MessageId, _grainContext.GrainId); + } + catch (Exception) when (execution.RejectionFailure is not null) + { + execution.RejectionFailure.Throw(); + throw; + } + catch (Exception exception) when (!combinedToken.IsCancellationRequested && _failure is null) + { + // Before Complete, handler failures retain the message for retry or dead-lettering. + operation.Error = exception; + LogHandlerException(_logger, exception, operation.Envelope.Envelope.MessageId, _grainContext.GrainId); + } + finally + { + execution.Active = false; + _handlerExecution.Value = previous; + } + + if (operation.Completed) + { + return; + } + ValidateOwner(operation.Owner); + + if (operation.Error is not null) + { + var attempts = _messageStates.TryGetValue(operation.Key, out var state) ? state.AttemptCount : 0; + var count = checked(attempts + 1); + operation.DeadLetter = count >= _maxProcessingAttempts; + operation.Retry = new InboxMessageState + { + AttemptCount = count, + LastError = operation.Error.ToString(), + NextAttemptAt = MessagingTime.AddClamped(_timeProvider.GetUtcNow(), + TimeSpan.FromTicks(_retryDelay.Ticks * (1L << Math.Min(count - 1, InboxOptions.MaximumBackoffExponent)))) + }; + } + } + + private bool StageWrite(InboxWrite operation) + { + ValidateGeneration(operation.Generation); + var processedBefore = operation is CompactWrite ? _processed.Count : 0; + var deadLettersBefore = operation is CompactWrite ? _deadLetters.Count : 0; + switch (operation) + { + case AcceptanceWrite acceptance: + if (_inboxDict.ContainsKey(acceptance.Key) || _inboxDict.Count >= _maxCapacity) + { + throw new InvalidOperationException("The admitted inbox acceptance no longer matches available capacity or message identity."); + } + + if (acceptance.Owner is { } acceptanceOwner) + { + ApplyOwnership(acceptanceOwner); + } + else if (!HasCommittedOwnership()) + { + throw new InvalidOperationException("The admitted inbox acceptance requires acknowledged job ownership."); + } + + _processed.Remove(acceptance.Key); + _inboxDict.Add(acceptance.Key, acceptance.Envelope); + _messageStates.Add(acceptance.Key, new InboxMessageState()); + _provisionalAcceptances.Add(acceptance.Key); + UpdateInboxDepth(1); + break; + case OwnershipWrite ownership: + if (GetCommittedInboxCount() == 0) + { + throw new InvalidOperationException("The admitted ownership repair has no pending inbox work."); + } + ApplyOwnership(ownership.Owner); + break; + case HandlerWrite handler: + ValidateOwner(handler.Owner); + handler.Cancellation.ThrowIfCancellationRequested(); + if (handler.Skipped) + { + break; + } + + if (!_inboxDict.ContainsKey(handler.Key)) + { + throw new InvalidOperationException("The invoked inbox handler lost its pending message before capture."); + } + + if (handler.Error is not null && !handler.DeadLetter) + { + _messageStates[handler.Key] = handler.Retry!; + } + else + { + var now = _timeProvider.GetUtcNow(); + if (handler.DeadLetter) + { + DeadLetterRetention.Compact(_deadLetters, now, _deadLetterRetentionPeriod, + _maxRetainedDeadLetters, static entry => entry.DeadLetteredAt, + reservedCapacity: _deadLetters.ContainsKey(handler.Key) ? 0 : 1); + _deadLetters[handler.Key] = new InboxDeadLetter + { + Message = handler.Envelope, + DeadLetteredAt = now, + Reason = handler.Error!.Message, + AttemptCount = handler.Retry?.AttemptCount ?? 0 + }; + } + StageHandlerCompletion(handler); + } + break; + case ClearOwnerWrite clear: + ValidateOwner(clear.Owner); + if (_inboxDict.Count != 0) + { + throw new InvalidOperationException("The admitted inbox owner cannot be cleared while work is pending."); + } + _completedJobId.Value = clear.Owner.Id; + _jobId.Value = null; + _job.Value = null; + break; + case CompactWrite: + DeadLetterRetention.Compact(_deadLetters, _timeProvider.GetUtcNow(), _deadLetterRetentionPeriod, + _maxRetainedDeadLetters, static entry => entry.DeadLetteredAt); + break; + } + + var maintenanceTime = _timeProvider.GetUtcNow(); + if (IsProcessedMaintenanceDue(maintenanceTime)) + { + CompactProcessedMessages(maintenanceTime); + } + + return operation is not CompactWrite || _processed.Count != processedBefore || _deadLetters.Count != deadLettersBefore; + } + + private void StageHandlerCompletion(HandlerWrite operation) + { + var now = _timeProvider.GetUtcNow(); + RemoveMessage(operation.Key); + _messageStates.Remove(operation.Key); + _processed[operation.Key] = now; + TrackProcessedExpiry(now); + operation.Completed = true; + } + + private void ApplyOwnership(OwnershipProposal proposal) + { + ValidateGeneration(proposal.Generation); + if (!string.Equals(_jobId.Value, proposal.PreviousId, StringComparison.Ordinal) + || !(proposal.PreviousJob is null && _job.Value is null + || MessagingJobOwnership.IsSamePhysicalJob(proposal.PreviousJob, _job.Value))) + { + throw new InvalidOperationException("The admitted inbox ownership proposal no longer matches the preceding owner."); + } + _jobId.Value = proposal.Id; + _job.Value = proposal.Job; + _jobSequence.Value = proposal.Sequence; + } + + private void AcknowledgeWrite(InboxWrite operation) + { + // Ownership-changing operations retain the inbox gate through their own write acknowledgement. + switch (operation) + { + case AcceptanceWrite acceptance: + if (acceptance.Owner is { } owner) + { + _durableOwnershipId = owner.Id; + _durableJob = owner.Job; + } + _provisionalAcceptances.Remove(acceptance.Key); + break; + case OwnershipWrite ownership: + _durableOwnershipId = ownership.Owner.Id; + _durableJob = ownership.Owner.Job; + break; + case ClearOwnerWrite clear: + _durableCompletedJobId = clear.Owner.Id; + _durableOwnershipId = null; + _durableJob = null; + break; + } + } + + private void LatchFailure(Exception exception) + { + _failure ??= ExceptionDispatchInfo.Capture(exception); + _pumpCoordinator.Reset(); + _pumpResults.Clear(JobName); + CancelProcessing(); + } + + private void CancelProcessing() + { + _localDrainRequested = false; + try + { + _shutdownCts.Cancel(); + } + catch (AggregateException cancellationException) + { + LogCancellationCallbackFailure(_logger, cancellationException); + } + finally + { + // Activation cancellation owns callback failures; disposing timers first would + // transfer those errors to the runtime timer logger instead of preserving them here. + _pumpTimer?.Dispose(); + _localDrainTimer?.Dispose(); + } + } + + [LoggerMessage(Level = LogLevel.Error, Message = "An inbox cancellation callback failed while stopping processing.")] + private static partial void LogCancellationCallbackFailure(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "Requesting deactivation after an inbox persistence failure failed.")] + private static partial void LogDeactivationRequestFailure(ILogger logger, Exception exception); + + private void InitializeRecoveredState() + { + _stateGeneration++; + _reservedSequence = _jobSequence.Value; + _durableOwnershipId = _jobId.Value; + _durableJob = _job.Value; + _durableCompletedJobId = _completedJobId.Value; + _ownershipStateError = MessagingJobOwnership.GetPairError(_jobId.Value, _job.Value); + _lastProcessedCompaction = null; + RebuildProcessedExpiry(); + _recoveryCompleted = true; + ReconcileInboxDepth(); + } + + internal async Task ResumeProcessingAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + ValidateReady(); + EnsureMetricsActive(); + await EnsureJobScheduledUnderGateAsync(cancellationToken).ConfigureAwait(true); + ScheduleLocalDrain(); + } + finally + { + _gate.Release(); + } + } + + public async Task OnStart(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_grainContext.GrainInstance is not DurableGrain and not IMessagingGrain) + { + throw new InvalidOperationException("Inbox activation requires IMessagingGrain or DurableGrain."); + } + foreach (var (key, message) in _inboxDict) + { + EnvelopeValidation.Validate(message); + if (key != message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered inbox key does not match its command identity."); + } + } + foreach (var entry in _deadLetters.Values) + { + EnvelopeValidation.Validate(entry.Message); + } + InitializeRecoveredState(); + await ResumeProcessingAsync(cancellationToken).ConfigureAwait(true); + if (_deadLetters.Values.Any(entry => MessagingTime.IsExpired(_timeProvider.GetUtcNow(), entry.DeadLetteredAt, _deadLetterRetentionPeriod)) + || _deadLetters.Count > _maxRetainedDeadLetters + || HasExpiredProcessedMessages(_timeProvider.GetUtcNow())) + { + await SubmitAsync(new CompactWrite(_stateGeneration)).ConfigureAwait(true); + } + } + + public async Task OnStop(CancellationToken cancellationToken) + { + var operations = _pendingWrites.Select(static operation => operation.WaitForFinished()).Append(_activeDelivery).ToArray(); + StopProcessing(); + // Owned work outlives caller cancellation and drains through handler completion and actual persistence. + await Task.WhenAll(operations).ConfigureAwait( + ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + } + + internal void StopProcessing() + { + try + { + CancelProcessing(); + } + finally + { + _pumpCoordinator.Reset(); + _pumpResults.Clear(JobName); + if (Interlocked.Exchange(ref _metricsActive, 0) != 0) + { + _instruments.OnInboxDepthChanged(-Interlocked.Exchange(ref _reportedDepth, 0)); + } + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + try + { + StopProcessing(); + } + finally + { + // Scope teardown has drained actual operations and releases durable values. + // Retired admission metadata must not continue reporting provisional work. + _provisionalAcceptances.Clear(); + _shutdownCts.Dispose(); + } + } + } + + private sealed class HandlerExecution(InboxExtension owner, HandlerWrite operation) + { + public InboxExtension Owner { get; } = owner; + public bool Active { get; set; } + public ExceptionDispatchInfo? RejectionFailure { get; private set; } + + public void Complete() + { + ValidateAttempt(); + if (operation.Completed) + { + return; + } + try + { + // Business state and directly injected outbox sends have already been staged. + // Stage inbox removal and dedupe synchronously, before the owned write can await. + Owner.StageHandlerCompletion(operation); + } + catch (Exception exception) + { + RejectOperation(exception); + throw; + } + } + + private void ValidateAttempt() + { + if (!Active || !ReferenceEquals(_handlerExecution.Value, this)) + { + RejectOperation(new InvalidOperationException( + "The inbox handler context belongs to an inactive or different attempt.")); + } + ThrowIfHandlerOperationRejected(this); + } + + [DoesNotReturn] + public void RejectOperation(Exception exception) + { + RejectionFailure ??= ExceptionDispatchInfo.Capture(exception); + if (_handlerExecution.Value is { } current && ReferenceEquals(current.Owner, Owner)) + { + current.RejectionFailure ??= RejectionFailure; + } + RejectionFailure.Throw(); + } + } + + private readonly record struct PumpOwner(string Id, DurableJob Job, long Generation); + private sealed record OwnershipProposal(string Id, long Sequence, DurableJob Job, long Generation, string? PreviousId, DurableJob? PreviousJob); + + private abstract class InboxWrite(long generation) + { + public long Generation { get; } = generation; + private TaskCompletionSource? _finished; + private bool _retired; + + public TaskCompletionSource Finished + { + get + { + var finished = _finished ??= new(TaskCreationOptions.RunContinuationsAsynchronously); + if (_retired) finished.TrySetResult(); + return finished; + } + } + public Task WaitForFinished() => _retired ? Task.CompletedTask : Finished.Task; + + public void SignalFinished() + { + _retired = true; + _finished?.TrySetResult(); + } + } + + private sealed class AcceptanceWrite(long generation, InboxMessage envelope, OwnershipProposal? owner) : InboxWrite(generation) + { + public InboxMessage Envelope { get; } = envelope; + public HierarchicalKey Key => Envelope.Envelope.MessageId; + public OwnershipProposal? Owner { get; } = owner; + } + + private sealed class OwnershipWrite(long generation, OwnershipProposal owner) : InboxWrite(generation) + { + public OwnershipProposal Owner { get; } = owner; + } + + private sealed class HandlerWrite(PumpOwner owner, InboxMessage envelope, CancellationToken cancellation) : InboxWrite(owner.Generation) + { + public PumpOwner Owner { get; } = owner; + public InboxMessage Envelope { get; } = envelope; + public HierarchicalKey Key => Envelope.Envelope.MessageId; + public CancellationToken Cancellation { get; } = cancellation; + public HandlerExecution? Execution { get; set; } + public bool Completed { get; set; } + public ExceptionDispatchInfo? PostCompletionFailure { get; set; } + public bool Skipped { get; set; } + public bool DeadLetter { get; set; } + public Exception? Error { get; set; } + public InboxMessageState? Retry { get; set; } + } + + private sealed class ClearOwnerWrite(PumpOwner owner) : InboxWrite(owner.Generation) + { + public PumpOwner Owner { get; } = owner; + } + + private sealed class CompactWrite(long generation) : InboxWrite(generation); + + public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) + { + _failure?.Throw(); + _shutdownToken.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); + if (!_recoveryCompleted) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + ThrowIfOwnershipStateInvalid(); + if (!MessagingJobOwnership.TryGetOwnershipId(context.Job, out var ownershipId)) + { + return DurableJobRunResult.Completed; + } + + if (_pendingOwnershipIds.Count != 0 || _pendingWrites.Any(static operation => operation is ClearOwnerWrite)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + var key = new MessagingPumpExecutionKey( + JobName, + context.Job.Id, + context.RunId, + Volatile.Read(ref _stateGeneration)); + if (!string.Equals(_jobId.Value, ownershipId, StringComparison.Ordinal)) + { + var disposition = MessagingJobOwnership.ResolveMismatch( + _recoveryCompleted, + HasCommittedOwnership(), + MessagingJobOwnership.IsCompleted(_durableCompletedJobId, ownershipId), + _inboxDict.Count > 0); + if (disposition == OwnershipMismatchDisposition.ReclaimOrphan) + { + LogOrphanedJobReclaimed(_logger, ownershipId, _grainContext.GrainId); + _instruments.OnOrphanedJobReclaimed(_grainContext.GrainId.Type.ToString(), JobName); + return CompleteObsoleteExecution(key); + } + + if (disposition == OwnershipMismatchDisposition.CompleteStale) + { + return CompleteObsoleteExecution(key); + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!HasCommittedOwnership()) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!MessagingJobOwnership.IsSamePhysicalJob(_job.Value, context.Job)) + { + return DurableJobRunResult.Completed; + } + + if (_pumpResults.TryTake(key, out var result, out var exception)) + { + if (exception is not null) + { + throw exception; + } + + return result!; + } + + if (!_pumpCoordinator.TryAcquire(ownershipId, cancellationToken, out var lease)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!_pumpResults.TryStart(key, cancellationToken, out var execution)) + { + _pumpCoordinator.Release(lease); + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + try + { + (_pumpTimer ??= new(this)).Queue(new(execution, lease, new(ownershipId, context.Job, key.StateGeneration), cancellationToken)); + } + catch (Exception registrationException) + { + _pumpCoordinator.Release(lease); + _pumpResults.Fail(execution, registrationException); + throw; + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + private DurableJobRunResult CompleteObsoleteExecution(MessagingPumpExecutionKey key) + { + // Committed ownership establishes retirement; this run's retained result will no longer be polled. + _pumpResults.TryTake(key, out _, out _); + return DurableJobRunResult.Completed; + } + + private async Task RunPumpTimerAsync( + MessagingPumpExecution execution, + MessagingPumpLease lease, + PumpOwner pumpOwner, + CancellationToken jobCancellation, + CancellationToken timerCancellation) + { + if (!_pumpCoordinator.IsCurrent(lease) || !IsCurrentOwner(pumpOwner)) + { + _pumpResults.Discard(execution); + return; + } + + if (!_pumpResults.TryBegin(execution)) + { + return; + } + + DurableJobRunResult? result = null; + Exception? failure = null; + try + { + using var linkedCancellation = MessagingCancellation.Combine( + jobCancellation, timerCancellation, _shutdownCts.Token, out var combinedToken); + result = await ExecuteJobCoreAsync( + pumpOwner, + clearOwnershipWhenEmpty: true, + combinedToken); + } + catch (Exception exception) + { + failure = exception; + } + finally + { + if (failure is null) + { + _pumpResults.Complete(execution, result!); + } + else + { + _pumpResults.Fail(execution, failure); + } + } + } + + private async ValueTask ExecuteJobCoreAsync(PumpOwner owner, bool clearOwnershipWhenEmpty, CancellationToken cancellationToken) + { + ValidateReady(); + if (!IsCurrentOwner(owner)) + { + return DurableJobRunResult.Completed; + } + + var now = _timeProvider.GetUtcNow(); + var pending = new List(Math.Min(_inboxDict.Count, _batchSize)); + foreach (var pair in _inboxDict) + { + if (!_provisionalAcceptances.Contains(pair.Key) + && (!_messageStates.TryGetValue(pair.Key, out var state) || state.NextAttemptAt is null || state.NextAttemptAt <= now)) + { + pending.Add(pair.Value); + if (pending.Count == _batchSize) + { + break; + } + } + } + foreach (var envelope in pending) + { + cancellationToken.ThrowIfCancellationRequested(); + ValidateOwner(owner); + var start = Stopwatch.GetTimestamp(); + var operation = new HandlerWrite(owner, envelope, cancellationToken); + await SubmitAsync(operation).ConfigureAwait(true); + if (operation.Skipped) + { + cancellationToken.ThrowIfCancellationRequested(); + } + var status = operation.Error is null ? "success" : operation.DeadLetter ? "dead_lettered" : "retry"; + _instruments.OnInboxMessageProcessed(_grainType, status); + _instruments.OnInboxProcessingDuration(Stopwatch.GetElapsedTime(start), _grainType); + } + + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + ValidateReady(); + if (!IsCurrentOwner(owner)) + { + return DurableJobRunResult.Completed; + } + if (_inboxDict.Count == 0) + { + if (clearOwnershipWhenEmpty) + { + await SubmitAsync(new ClearOwnerWrite(owner)).ConfigureAwait(true); + } + return DurableJobRunResult.Completed; + } + } + finally + { + _gate.Release(); + } + + if (HasExpiredProcessedMessages(_timeProvider.GetUtcNow())) + { + await SubmitAsync(new CompactWrite(_stateGeneration)).ConfigureAwait(true); + } + + var nextAttempt = GetNextAttemptAt(); + if (GetNextProcessedMaintenance() is { } maintenance && maintenance < nextAttempt) + { + nextAttempt = maintenance; + } + var delay = nextAttempt - _timeProvider.GetUtcNow(); + return DurableJobRunResult.RescheduleAt(MessagingTime.AddClamped( + _jobTimeProvider.GetUtcNow(), delay > TimeSpan.Zero ? delay : TimeSpan.Zero)); + } + + private DateTimeOffset GetNextAttemptAt() + { + var now = _timeProvider.GetUtcNow(); + var next = DateTimeOffset.MaxValue; + foreach (var key in _inboxDict.Keys) + { + if (!_messageStates.TryGetValue(key, out var state) || state.NextAttemptAt is not { } at || at <= now) + { + return now; + } + if (at < next) + { + next = at; + } + } + return next; + } + + private DateTimeOffset? GetNextProcessedMaintenance() + { + if (_nextProcessedExpiry is not { } next) + { + return null; + } + if (_lastProcessedCompaction is { } last) + { + if (_processedCompactionInterval.Ticks > DateTimeOffset.MaxValue.UtcTicks - last.UtcTicks) + { + return null; + } + var earliestScan = MessagingTime.AddClamped(last, _processedCompactionInterval); + if (earliestScan > next) + { + next = earliestScan; + } + } + return next; + } + + private bool IsProcessedMaintenanceDue(DateTimeOffset now) => + GetNextProcessedMaintenance() is { } next && now >= next; + + private bool HasExpiredProcessedMessages(DateTimeOffset now) + { + if (!IsProcessedMaintenanceDue(now)) + { + return false; + } + if (_processed.Any(pair => MessagingTime.IsExpired(now, pair.Value, _deduplicationWindow))) + { + return true; + } + + _lastProcessedCompaction = now; + RebuildProcessedExpiry(); + return false; + } + + private void TrackProcessedExpiry(DateTimeOffset processedAt) + { + if (_deduplicationWindow.Ticks > DateTimeOffset.MaxValue.UtcTicks - processedAt.UtcTicks) + { + return; + } + var expiry = MessagingTime.AddClamped(processedAt, _deduplicationWindow); + if (_nextProcessedExpiry is null || expiry < _nextProcessedExpiry) + { + _nextProcessedExpiry = expiry; + } + } + + private void RebuildProcessedExpiry() + { + _nextProcessedExpiry = null; + foreach (var entry in _processed) + { + TrackProcessedExpiry(entry.Value); + } + } + + private void CompactProcessedMessages(DateTimeOffset now) + { + // Amortize dictionary scans over retention time; ordinary completion and owner-clear writes only check the deadline. + List? expired = null; + _nextProcessedExpiry = null; + foreach (var entry in _processed) + { + if (MessagingTime.IsExpired(now, entry.Value, _deduplicationWindow)) + { + (expired ??= []).Add(entry.Key); + } + else + { + TrackProcessedExpiry(entry.Value); + } + } + if (expired is not null) + { + foreach (var key in expired) + { + _processed.Remove(key); + } + } + _lastProcessedCompaction = now; + } + + private void EnsureMetricsActive() + { + if (Interlocked.Exchange(ref _metricsActive, 1) == 0) + { + Volatile.Write(ref _reportedDepth, _inboxDict.Count); + _instruments.OnInboxDepthChanged(_inboxDict.Count); + } + } + + private void UpdateInboxDepth(int delta) + { + if (Volatile.Read(ref _metricsActive) != 0) + { + Interlocked.Add(ref _reportedDepth, delta); + _instruments.OnInboxDepthChanged(delta); + } + } + + private void ReconcileInboxDepth() + { + if (Volatile.Read(ref _metricsActive) == 0) + { + return; + } + + var count = _inboxDict.Count; + var delta = count - Interlocked.Exchange(ref _reportedDepth, count); + if (delta != 0) + { + _instruments.OnInboxDepthChanged(delta); + } + } + + private bool RemoveMessage(HierarchicalKey key) + { + if (!_inboxDict.Remove(key)) + { + return false; + } + + UpdateInboxDepth(-1); + + return true; + } + + // Structured logging using LoggerMessage source generator + + [LoggerMessage(Level = LogLevel.Error, EventName = "DeliveryOperationFailed", + Message = "Inbox delivery of message {MessageId} to {GrainId} failed")] + private static partial void LogDeliveryOperationFailed(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId grainId); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "No inbox handler registered for message {MessageId} to {ReceiverId}")] + private static partial void LogHandlerNotFound(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Accepted message {MessageId} to {ReceiverId}")] + private static partial void LogMessageAccepted(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Handler threw exception for message {MessageId} to {ReceiverId}")] + private static partial void LogHandlerException(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Reclaimed orphaned inbox job ownership {OwnershipId} for grain {GrainId}")] + private static partial void LogOrphanedJobReclaimed(ILogger logger, string ownershipId, GrainId grainId); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Error scheduling inbox recovery for grain {GrainId}")] + private static partial void LogRecoverySchedulingError(ILogger logger, Exception exception, GrainId grainId); + + private readonly record struct PumpTurn(MessagingPumpExecution Execution, + MessagingPumpLease Lease, PumpOwner Owner, CancellationToken JobCancellation); + private readonly record struct LocalDrainTurn(MessagingPumpLease Lease, PumpOwner Owner); + + private sealed class PumpTimerState(InboxExtension owner) : MessagingTurn + { + protected override IGrainTimer RegisterTimer(long registrationGeneration) => owner._timerRegistry.RegisterGrainTimer( + owner._grainContext, (state, token) => state.RunAsync(registrationGeneration, token), this, + new GrainTimerCreationOptions(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan) { Interleave = false, KeepAlive = true }); + + protected override async Task ExecuteAsync(PumpTurn turn, CancellationToken timerCancellation) + { + try + { + await owner.RunPumpTimerAsync(turn.Execution, turn.Lease, turn.Owner, turn.JobCancellation, timerCancellation); + } + finally + { + owner._pumpCoordinator.Release(turn.Lease); + if (owner._localDrainRequested) owner.ScheduleLocalDrain(); + } + } + + protected override void Discard(PumpTurn turn) + { + owner._pumpResults.Discard(turn.Execution); + owner._pumpCoordinator.Release(turn.Lease); + } + } + + private void ScheduleLocalDrain() + { + if (_shutdownToken.IsCancellationRequested || _failure is not null + || _jobId.Value is not { Length: > 0 } jobId + || GetCommittedInboxCount() == 0 + || !HasCommittedOwnership()) + { + return; + } + + _localDrainRequested = true; + if (_pumpCoordinator.IsActive || !_pumpCoordinator.TryAcquire(jobId, _shutdownToken, out var lease)) + { + return; + } + + _localDrainRequested = false; + try + { + (_localDrainTimer ??= new(this)).Queue(new(lease, new PumpOwner(jobId, _job.Value!, _stateGeneration))); + } + catch + { + _pumpCoordinator.Release(lease); + throw; + } + } + + private sealed class LocalDrainTimerState(InboxExtension owner) : MessagingTurn + { + protected override IGrainTimer RegisterTimer(long registrationGeneration) => owner._timerRegistry.RegisterGrainTimer( + owner._grainContext, (state, token) => state.RunAsync(registrationGeneration, token), this, + new GrainTimerCreationOptions(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan) { Interleave = false, KeepAlive = true }); + + protected override async Task ExecuteAsync(LocalDrainTurn turn, CancellationToken cancellationToken) + { + try + { + if (owner._pumpCoordinator.IsCurrent(turn.Lease)) + { + using var cancellation = MessagingCancellation.Combine( + cancellationToken, owner._shutdownToken, out var combinedToken); + _ = await owner.ExecuteJobCoreAsync(turn.Owner, clearOwnershipWhenEmpty: false, combinedToken); + } + } + finally + { + owner._pumpCoordinator.Release(turn.Lease); + if (owner._localDrainRequested) owner.ScheduleLocalDrain(); + } + } + + protected override void Discard(LocalDrainTurn turn) => owner._pumpCoordinator.Release(turn.Lease); + } + +} diff --git a/src/Orleans.Messaging/InboxHandlerContext.cs b/src/Orleans.Messaging/InboxHandlerContext.cs new file mode 100644 index 00000000000..a9278c2d532 --- /dev/null +++ b/src/Orleans.Messaging/InboxHandlerContext.cs @@ -0,0 +1,12 @@ +using System; + +namespace Orleans.Messaging; + +internal sealed class InboxHandlerContext(InboxMessage message, Action complete) : IInboxHandlerContext +{ + private readonly Action _complete = complete ?? throw new ArgumentNullException(nameof(complete)); + + public InboxMessage Message { get; } = message; + + public void Complete() => _complete(); +} diff --git a/src/Orleans.Messaging/InboxMessage.cs b/src/Orleans.Messaging/InboxMessage.cs new file mode 100644 index 00000000000..0a9e5aeb3ee --- /dev/null +++ b/src/Orleans.Messaging/InboxMessage.cs @@ -0,0 +1,10 @@ +namespace Orleans.Messaging; + +/// A received command whose destination is the grain owning the inbox. +[GenerateSerializer, Alias("Orleans.Messaging.InboxMessage")] +public readonly struct InboxMessage +{ + /// Gets the command identity and common header buffer. + [Id(0)] + public required Envelope Envelope { get; init; } +} diff --git a/src/Orleans.Messaging/MessageHeaders.cs b/src/Orleans.Messaging/MessageHeaders.cs new file mode 100644 index 00000000000..c883edf216f --- /dev/null +++ b/src/Orleans.Messaging/MessageHeaders.cs @@ -0,0 +1,14 @@ +namespace Orleans.Messaging; + +/// Names the standard envelope headers. +public static class MessageHeaders +{ + /// The required application payload, including an empty payload. + public const string Payload = "payload"; + + /// The optional, nonempty protocol subject encoded as canonical UTF-8 bytes. + public const string Subject = "subject"; + + /// The optional immediate sender encoded using an externally supplied Orleans serializer. + public const string Sender = "sender"; +} diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs new file mode 100644 index 00000000000..6cb3e4ed292 --- /dev/null +++ b/src/Orleans.Messaging/MessageState.cs @@ -0,0 +1,65 @@ +using System; + +namespace Orleans.Messaging; + +[GenerateSerializer, Alias("Orleans.Messaging.InboxMessageState")] +internal sealed class InboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessageState")] +internal sealed class OutboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + + [Id(3)] + public DateTimeOffset? EnqueuedAt { get; set; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxDeadLetter")] +internal sealed class InboxDeadLetter +{ + [Id(0)] + public required InboxMessage Message { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxDeadLetter")] +internal sealed class OutboxDeadLetter +{ + [Id(0)] + public required OutboxMessage Message { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } +} diff --git a/src/Orleans.Messaging/MessagingActivationValidator.cs b/src/Orleans.Messaging/MessagingActivationValidator.cs new file mode 100644 index 00000000000..cfdfaf65056 --- /dev/null +++ b/src/Orleans.Messaging/MessagingActivationValidator.cs @@ -0,0 +1,49 @@ +using System; +using System.Linq; +using Orleans; +using Orleans.Concurrency; +using Orleans.Metadata; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal static class MessagingActivationValidator +{ + // The public attribute exposes the runtime's internal, sealed strategy type. + private static readonly Type _statelessWorkerPlacementType = new StatelessWorkerAttribute().PlacementStrategy.GetType(); + + public static void Validate(IGrainContext grainContext, GrainProperties properties, PlacementStrategy placementStrategy) + { + var grain = grainContext.GrainInstance + ?? throw new InvalidOperationException("Messaging activation requires an initialized grain instance."); + var grainType = grain.GetType(); + if (placementStrategy.GetType() == _statelessWorkerPlacementType) + { + throw new InvalidOperationException( + $"Messaging requires one activation per grain identity, but grain type '{grainType}' is a stateless worker."); + } + + if (properties.Properties.TryGetValue(WellKnownGrainTypeProperties.Reentrant, out var reentrant) && bool.Parse(reentrant) + || properties.Properties.ContainsKey(WellKnownGrainTypeProperties.MayInterleavePredicate)) + { + throw new InvalidOperationException( + $"Messaging requires non-reentrant grain execution, but grain type '{grainType}' enables interleaving."); + } + + var grainInterfaces = grainType + .GetInterfaces() + .Where(static type => typeof(IGrain).IsAssignableFrom(type)) + .ToArray(); + var interleavableMethod = grainInterfaces + .SelectMany(static type => type.GetInterfaces().Append(type)) + .Distinct() + .SelectMany(static type => type.GetMethods()) + .FirstOrDefault(static method => method.IsDefined(typeof(AlwaysInterleaveAttribute), inherit: true)); + if (interleavableMethod is not null) + { + throw new InvalidOperationException( + $"Messaging grain type '{grainType}' implements interleavable method " + + $"'{interleavableMethod.DeclaringType}.{interleavableMethod.Name}'."); + } + } +} diff --git a/src/Orleans.Messaging/MessagingCancellation.cs b/src/Orleans.Messaging/MessagingCancellation.cs new file mode 100644 index 00000000000..c868da367f7 --- /dev/null +++ b/src/Orleans.Messaging/MessagingCancellation.cs @@ -0,0 +1,44 @@ +using System.Threading; + +namespace Orleans.Messaging; + +internal static class MessagingCancellation +{ + // A borrowed token needs no resource; distinct scopes still require a link owned by the caller. + public static CancellationTokenSource? Combine(CancellationToken first, CancellationToken second, out CancellationToken token) + { + if (!first.CanBeCanceled || first == second) + { + token = second; + return null; + } + if (!second.CanBeCanceled) + { + token = first; + return null; + } + var source = CancellationTokenSource.CreateLinkedTokenSource(first, second); + token = source.Token; + return source; + } + + public static CancellationTokenSource? Combine( + CancellationToken first, CancellationToken second, CancellationToken third, out CancellationToken token) + { + if (!first.CanBeCanceled || first == second || first == third) + { + return Combine(second, third, out token); + } + if (!second.CanBeCanceled || second == third) + { + return Combine(first, third, out token); + } + if (!third.CanBeCanceled) + { + return Combine(first, second, out token); + } + var source = CancellationTokenSource.CreateLinkedTokenSource(first, second, third); + token = source.Token; + return source; + } +} diff --git a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs new file mode 100644 index 00000000000..bd0187a2a4f --- /dev/null +++ b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs @@ -0,0 +1,32 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal sealed class MessagingGrainTypeConfigurator(GrainClassMap grainClasses) : IConfigureGrainTypeComponents +{ + public void Configure(GrainType grainType, GrainProperties properties, GrainTypeSharedContext shared) + { + if (!grainClasses.TryGetGrainClass(grainType, out var grainClass)) + { + throw new InvalidOperationException($"No grain implementation is registered for '{grainType}'."); + } + + if (typeof(DurableGrain).IsAssignableFrom(grainClass) || typeof(IMessagingGrain).IsAssignableFrom(grainClass)) + { + var placementStrategy = shared.PlacementStrategy; + shared.AddActivationSetup(context => + { + MessagingActivationValidator.Validate(context, properties, placementStrategy); + var services = context.ActivationServices; + _ = services.GetRequiredService(); + _ = services.GetRequiredService(); + _ = services.GetRequiredService(); + _ = services.GetRequiredService(); + }); + } + } +} diff --git a/src/Orleans.Messaging/MessagingInstruments.cs b/src/Orleans.Messaging/MessagingInstruments.cs new file mode 100644 index 00000000000..01af41137fb --- /dev/null +++ b/src/Orleans.Messaging/MessagingInstruments.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Threading; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal sealed class MessagingInstruments(OrleansInstruments instruments) +{ + private const string MillisecondsUnit = "ms"; + private const string GrainTypeTagName = "grain_type"; + private const string StatusTagName = "status"; + + private readonly Counter _inboxMessagesReceived = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-received"); + private readonly Counter _inboxMessagesProcessed = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-processed"); + private readonly Counter _outboxMessagesSent = instruments.Meter.CreateCounter("orleans-messaging-outbox-messages-sent"); + private readonly Counter _outboxMessagesDelivered = instruments.Meter.CreateCounter("orleans-messaging-outbox-messages-delivered"); + private readonly Counter _orphanedJobsReclaimed = instruments.Meter.CreateCounter("orleans-messaging-orphaned-jobs-reclaimed"); + private readonly Histogram _inboxProcessingDuration = instruments.Meter.CreateHistogram("orleans-messaging-inbox-processing-duration", MillisecondsUnit); + private readonly Histogram _outboxDeliveryDuration = instruments.Meter.CreateHistogram("orleans-messaging-outbox-delivery-duration", MillisecondsUnit); + private readonly DepthTracker _inboxDepth = new(instruments.Meter, "orleans-messaging-inbox-depth"); + private readonly DepthTracker _outboxDepth = new(instruments.Meter, "orleans-messaging-outbox-depth"); + + internal static MessagingInstruments CreateForDirectConstruction() => new(new OrleansInstruments(new DirectMeterFactory())); + + internal void OnInboxDepthChanged(int delta) => _inboxDepth.Adjust(delta); + + internal void OnOutboxDepthChanged(int delta) => _outboxDepth.Adjust(delta); + + internal void OnInboxMessageReceived(string grainType, string status) => + Add(_inboxMessagesReceived, grainType, status); + + internal void OnInboxMessageProcessed(string grainType, string status) => + Add(_inboxMessagesProcessed, grainType, status); + + internal void OnInboxProcessingDuration(TimeSpan duration, string grainType) => + Record(_inboxProcessingDuration, duration, grainType); + + internal void OnOutboxMessageSent(string grainType) + { + if (_outboxMessagesSent.Enabled) + { + _outboxMessagesSent.Add(1, CreateTags(grainType)); + } + } + + internal void OnOutboxMessageDelivered(string grainType, string status) => + Add(_outboxMessagesDelivered, grainType, status); + + internal void OnOutboxDeliveryDuration(TimeSpan duration, string grainType) => + Record(_outboxDeliveryDuration, duration, grainType); + + internal void OnOrphanedJobReclaimed(string grainType, string jobName) + { + if (_orphanedJobsReclaimed.Enabled) + { + _orphanedJobsReclaimed.Add( + 1, + [ + new(GrainTypeTagName, grainType), + new("job_name", jobName) + ]); + } + } + + private static void Add(Counter counter, string grainType, string status) + { + if (counter.Enabled) + { + counter.Add( + 1, + [ + new(GrainTypeTagName, grainType), + new(StatusTagName, status) + ]); + } + } + + private static void Record(Histogram histogram, TimeSpan duration, string grainType) + { + if (histogram.Enabled) + { + histogram.Record(Math.Max(0, duration.TotalMilliseconds), CreateTags(grainType)); + } + } + + private static KeyValuePair[] CreateTags(string grainType) => + [ + new(GrainTypeTagName, grainType) + ]; + + private sealed class DirectMeterFactory : IMeterFactory + { + public Meter Create(MeterOptions options) => new(options); + + public void Dispose() + { + } + } + + private sealed class DepthTracker + { + private readonly ObservableGauge _gauge; + private long _value; + + public DepthTracker(Meter meter, string name) + { + _gauge = meter.CreateObservableGauge(name, () => Volatile.Read(ref _value)); + } + + public void Adjust(int delta) => Interlocked.Add(ref _value, delta); + } +} diff --git a/src/Orleans.Messaging/MessagingJobOwnership.cs b/src/Orleans.Messaging/MessagingJobOwnership.cs new file mode 100644 index 00000000000..a73ae264e66 --- /dev/null +++ b/src/Orleans.Messaging/MessagingJobOwnership.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal static class MessagingJobOwnership +{ + private const string MetadataKey = "orleans.messaging.ownership-id"; + + public static IReadOnlyDictionary CreateMetadata(string ownershipId) => + new Dictionary(1, StringComparer.Ordinal) + { + [MetadataKey] = ownershipId + }; + + public static bool TryGetOwnershipId(DurableJob job, out string ownershipId) + { + if (job.Metadata is not null + && job.Metadata.TryGetValue(MetadataKey, out var value) + && !string.IsNullOrWhiteSpace(value)) + { + ownershipId = value; + return true; + } + + ownershipId = string.Empty; + return false; + } + + + public static bool HasOwner(string? ownershipId, DurableJob? job) => + !string.IsNullOrWhiteSpace(ownershipId) && job is not null; + + public static string? GetPairError(string? ownershipId, DurableJob? job) + { + var hasOwnershipId = !string.IsNullOrWhiteSpace(ownershipId); + if (hasOwnershipId != (job is not null)) + { + return "The messaging ownership generation and job handle must either both be present or both be absent."; + } + + if (hasOwnershipId + && (!TryGetOwnershipId(job!, out var jobOwnershipId) + || !string.Equals(jobOwnershipId, ownershipId, StringComparison.Ordinal))) + { + return "The messaging job handle metadata does not match its ownership generation."; + } + + return null; + } + + public static bool IsSamePhysicalJob(DurableJob? expected, DurableJob? actual) => + expected is not null + && actual is not null + && string.Equals(expected.Id, actual.Id, StringComparison.Ordinal) + && string.Equals(expected.ShardId, actual.ShardId, StringComparison.Ordinal); + + public static string CreateId(string epoch, long sequence) + { + ArgumentException.ThrowIfNullOrWhiteSpace(epoch); + return $"{epoch}:{sequence.ToString(CultureInfo.InvariantCulture)}"; + } + + public static bool IsCompleted(string? completedOwnershipId, string ownershipId) + { + if (string.Equals(completedOwnershipId, ownershipId, StringComparison.Ordinal)) + { + return true; + } + + return TryParse(completedOwnershipId, out var completedEpoch, out var completed) + && TryParse(ownershipId, out var currentEpoch, out var current) + && string.Equals(completedEpoch, currentEpoch, StringComparison.Ordinal) + && current <= completed; + } + + private static bool TryParse(string? value, out string epoch, out long sequence) + { + sequence = 0; + var separator = value?.LastIndexOf(':') ?? -1; + if (separator <= 0 + || !long.TryParse( + value.AsSpan(separator + 1), + NumberStyles.None, + CultureInfo.InvariantCulture, + out sequence)) + { + epoch = string.Empty; + return false; + } + + epoch = value![..separator]; + return true; + } + + public static OwnershipMismatchDisposition ResolveMismatch( + bool recoveryCompleted, + bool hasCurrentOwner, + bool ownershipCompleted, + bool hasWork) + { + if (!recoveryCompleted) + { + return OwnershipMismatchDisposition.WaitForRecovery; + } + + if (hasCurrentOwner || ownershipCompleted) + { + return OwnershipMismatchDisposition.CompleteStale; + } + + return hasWork + ? OwnershipMismatchDisposition.WaitForReplacement + : OwnershipMismatchDisposition.ReclaimOrphan; + } +} + +internal enum OwnershipMismatchDisposition +{ + WaitForRecovery, + WaitForReplacement, + CompleteStale, + ReclaimOrphan +} diff --git a/src/Orleans.Messaging/MessagingPumpCoordinator.cs b/src/Orleans.Messaging/MessagingPumpCoordinator.cs new file mode 100644 index 00000000000..c4d57c76f83 --- /dev/null +++ b/src/Orleans.Messaging/MessagingPumpCoordinator.cs @@ -0,0 +1,82 @@ +using System; +using System.Threading; + +namespace Orleans.Messaging; + +/// +/// Coalesces physical durable-job callbacks which represent the same logical pump ownership. +/// +internal sealed class MessagingPumpCoordinator +{ + private readonly object _lock = new(); + private string? _activeOwnershipId; + private CancellationToken _activeCancellationToken; + private long _activeGeneration; + + public bool IsActive + { + get + { + lock (_lock) + { + return _activeOwnershipId is not null; + } + } + } + + public bool TryAcquire( + string ownershipId, + CancellationToken cancellationToken, + out MessagingPumpLease lease) + { + ArgumentException.ThrowIfNullOrWhiteSpace(ownershipId); + + lock (_lock) + { + if (string.Equals(_activeOwnershipId, ownershipId, StringComparison.Ordinal) + && !_activeCancellationToken.IsCancellationRequested) + { + lease = default; + return false; + } + + lease = new MessagingPumpLease(ownershipId, ++_activeGeneration); + _activeOwnershipId = ownershipId; + _activeCancellationToken = cancellationToken; + return true; + } + } + + public bool IsCurrent(MessagingPumpLease lease) + { + lock (_lock) + { + return _activeGeneration == lease.Generation + && string.Equals(_activeOwnershipId, lease.OwnershipId, StringComparison.Ordinal); + } + } + + public void Release(MessagingPumpLease lease) + { + lock (_lock) + { + if (_activeGeneration == lease.Generation + && string.Equals(_activeOwnershipId, lease.OwnershipId, StringComparison.Ordinal)) + { + _activeOwnershipId = null; + _activeCancellationToken = default; + } + } + } + + public void Reset() + { + lock (_lock) + { + _activeOwnershipId = null; + _activeCancellationToken = default; + } + } +} + +internal readonly record struct MessagingPumpLease(string OwnershipId, long Generation); diff --git a/src/Orleans.Messaging/MessagingPumpResults.cs b/src/Orleans.Messaging/MessagingPumpResults.cs new file mode 100644 index 00000000000..16a3bdbe3d0 --- /dev/null +++ b/src/Orleans.Messaging/MessagingPumpResults.cs @@ -0,0 +1,428 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal readonly record struct MessagingPumpExecutionKey(string JobName, string JobId, string RunId, long StateGeneration); + +internal readonly record struct MessagingPumpExecution(MessagingPumpExecutionKey Key, long Generation); + +internal sealed class MessagingPumpResults +{ + private const int DefaultMaxRetainedEntries = 65_536; + private static readonly TimeSpan DefaultRetentionPeriod = TimeSpan.FromMinutes(10); + + private readonly object _lock = new(); + private readonly Dictionary _entries = []; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _completedRetentionPeriod; + private readonly TimeSpan _abandonedRetentionPeriod; + private readonly TimeSpan _cleanupInterval; + private readonly int _maxRetainedEntries; + private DateTimeOffset _nextCleanup; + private long _generation; + + internal MessagingPumpResults() + : this(TimeProvider.System, DefaultRetentionPeriod, DefaultRetentionPeriod, DefaultMaxRetainedEntries) + { + } + + internal MessagingPumpResults( + TimeProvider timeProvider, + TimeSpan completedRetentionPeriod, + TimeSpan abandonedRetentionPeriod, + int maxRetainedEntries) + { + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(completedRetentionPeriod, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(abandonedRetentionPeriod, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxRetainedEntries); + + _timeProvider = timeProvider; + _completedRetentionPeriod = completedRetentionPeriod; + _abandonedRetentionPeriod = abandonedRetentionPeriod; + _maxRetainedEntries = maxRetainedEntries; + _cleanupInterval = TimeSpan.FromTicks(Math.Max( + TimeSpan.FromSeconds(1).Ticks, + Math.Min(TimeSpan.FromMinutes(1).Ticks, Math.Min(completedRetentionPeriod.Ticks, abandonedRetentionPeriod.Ticks) / 4))); + _nextCleanup = MessagingTime.AddClamped(timeProvider.GetUtcNow(), _cleanupInterval); + } + + public bool TryStart( + MessagingPumpExecutionKey key, + CancellationToken cancellationToken, + out MessagingPumpExecution execution) + { + List? removed; + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + removed = Prune(now, force: _entries.Count >= _maxRetainedEntries); + if (_entries.ContainsKey(key)) + { + execution = default; + } + else + { + if (_entries.Count >= _maxRetainedEntries) + { + MessagingPumpExecutionKey? candidateKey = null; + Entry? candidateEntry = null; + DateTimeOffset candidateTime = DateTimeOffset.MaxValue; + + foreach (var (entryKey, entryValue) in _entries) + { + if (entryValue.State != EntryState.Running) + { + var timestamp = entryValue.State == EntryState.Completed ? entryValue.CompletedAt : entryValue.CreatedAt; + if (candidateEntry is null || timestamp < candidateTime) + { + candidateKey = entryKey; + candidateEntry = entryValue; + candidateTime = timestamp; + } + } + } + + if (candidateKey is not null && _entries.Remove(candidateKey.Value)) + { + (removed ??= []).Add(candidateEntry!); + } + } + + if (_entries.Count >= _maxRetainedEntries) + { + execution = default; + } + else + { + execution = new(key, ++_generation); + _entries.Add(key, new Entry(execution.Generation, now)); + } + } + } + + DisposeRegistrations(removed); + if (execution == default) + { + return false; + } + + if (!cancellationToken.CanBeCanceled) + { + return true; + } + + var registration = cancellationToken.UnsafeRegister( + static state => + { + var cancellation = (CancellationState)state!; + cancellation.Owner.CancelWaiting(cancellation.Execution, cancellation.Token); + }, + new CancellationState(this, execution, cancellationToken)); + + var disposeRegistration = false; + lock (_lock) + { + if (_entries.TryGetValue(key, out var current) + && current.Generation == execution.Generation + && current.State == EntryState.Waiting) + { + current.CancellationRegistration = registration; + } + else + { + disposeRegistration = true; + } + } + + if (disposeRegistration) + { + registration.Dispose(); + } + + return true; + } + + public bool TryBegin(MessagingPumpExecution execution) + { + CancellationTokenRegistration registration = default; + lock (_lock) + { + if (!_entries.TryGetValue(execution.Key, out var entry) + || entry.Generation != execution.Generation + || entry.State != EntryState.Waiting) + { + return false; + } + + entry.State = EntryState.Running; + registration = entry.CancellationRegistration; + entry.CancellationRegistration = default; + } + + registration.Dispose(); + return true; + } + + public void Complete(MessagingPumpExecution execution, DurableJobRunResult result) + { + ArgumentNullException.ThrowIfNull(result); + Finish(execution, result, exception: null); + } + + public void Fail(MessagingPumpExecution execution, Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + Finish(execution, result: null, exception); + } + + public void Discard(MessagingPumpExecution execution) + { + Entry? removed = null; + lock (_lock) + { + if (_entries.TryGetValue(execution.Key, out var entry) + && entry.Generation == execution.Generation + && entry.State != EntryState.Running) + { + _entries.Remove(execution.Key); + removed = entry; + } + } + + if (removed is not null) + { + DisposeRegistration(removed); + } + } + + public void Clear(string jobName) + { + List? removed = null; + lock (_lock) + { + foreach (var pair in _entries.ToArray()) + { + if (string.Equals(pair.Key.JobName, jobName, StringComparison.Ordinal)) + { + _entries.Remove(pair.Key); + (removed ??= []).Add(pair.Value); + } + } + } + + DisposeRegistrations(removed); + } + + public bool TryTake( + MessagingPumpExecutionKey key, + out DurableJobRunResult? result, + out Exception? exception) + { + Entry? removedEntry = null; + List? pruned; + lock (_lock) + { + pruned = Prune(_timeProvider.GetUtcNow(), force: false); + if (!_entries.TryGetValue(key, out var entry) || entry.State != EntryState.Completed) + { + result = null; + exception = null; + } + else + { + _entries.Remove(key); + removedEntry = entry; + result = entry.Result; + exception = entry.Exception; + } + } + + DisposeRegistrations(pruned); + if (removedEntry is null) + { + return false; + } + + removedEntry.CancellationRegistration.Dispose(); + return true; + } + + private void Finish( + MessagingPumpExecution execution, + DurableJobRunResult? result, + Exception? exception) + { + CancellationTokenRegistration registration = default; + List? removed; + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + if (_entries.TryGetValue(execution.Key, out var entry) + && entry.Generation == execution.Generation + && entry.State != EntryState.Completed) + { + entry.Result = result; + entry.Exception = exception; + entry.State = EntryState.Completed; + entry.CompletedAt = now; + registration = entry.CancellationRegistration; + entry.CancellationRegistration = default; + } + + removed = Prune(now, force: _entries.Count > _maxRetainedEntries); + } + + registration.Dispose(); + DisposeRegistrations(removed); + } + + private void CancelWaiting(MessagingPumpExecution execution, CancellationToken cancellationToken) + { + CancellationTokenRegistration registration = default; + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + if (_entries.TryGetValue(execution.Key, out var entry) + && entry.Generation == execution.Generation + && entry.State == EntryState.Waiting) + { + entry.Exception = new OperationCanceledException(cancellationToken); + entry.State = EntryState.Completed; + entry.CompletedAt = now; + registration = entry.CancellationRegistration; + entry.CancellationRegistration = default; + } + } + + registration.Dispose(); + } + + private List? Prune(DateTimeOffset now, bool force) + { + if (!force && now < _nextCleanup) + { + return null; + } + + _nextCleanup = MessagingTime.AddClamped(now, _cleanupInterval); + List? removed = null; + foreach (var pair in _entries.ToArray()) + { + var entry = pair.Value; + var expired = entry.State switch + { + EntryState.Completed => now - entry.CompletedAt >= _completedRetentionPeriod, + EntryState.Waiting => now - entry.CreatedAt >= _abandonedRetentionPeriod, + _ => false + }; + if (expired && _entries.Remove(pair.Key)) + { + (removed ??= []).Add(entry); + } + } + + if (_entries.Count <= _maxRetainedEntries) + { + return removed; + } + + foreach (var pair in _entries + .Where(static pair => pair.Value.State != EntryState.Running) + .OrderBy(static pair => pair.Value.State == EntryState.Completed ? pair.Value.CompletedAt : pair.Value.CreatedAt) + .ToArray()) + { + if (_entries.Count <= _maxRetainedEntries) + { + break; + } + + if (_entries.Remove(pair.Key)) + { + (removed ??= []).Add(pair.Value); + } + } + + return removed; + } + + private static void DisposeRegistrations(List? entries) + { + if (entries is null) + { + return; + } + + foreach (var entry in entries) + { + DisposeRegistration(entry); + } + } + + private static void DisposeRegistration(Entry entry) + { + var registration = entry.CancellationRegistration; + entry.CancellationRegistration = default; + registration.Dispose(); + } + + private sealed class Entry(long generation, DateTimeOffset createdAt) + { + public long Generation { get; } = generation; + public DateTimeOffset CreatedAt { get; } = createdAt; + public DateTimeOffset CompletedAt { get; set; } + public EntryState State { get; set; } + public DurableJobRunResult? Result { get; set; } + public Exception? Exception { get; set; } + public CancellationTokenRegistration CancellationRegistration { get; set; } + } + + private sealed record CancellationState( + MessagingPumpResults Owner, + MessagingPumpExecution Execution, + CancellationToken Token); + + private enum EntryState + { + Waiting, + Running, + Completed + } +} + +internal sealed class OneShotTimerHandle +{ + private readonly object _lock = new(); + private IGrainTimer? _timer; + private bool _completed; + + public void Attach(IGrainTimer timer) + { + lock (_lock) + { + if (_completed) + { + timer.Dispose(); + } + else + { + _timer = timer; + } + } + } + + public void Complete() + { + lock (_lock) + { + _completed = true; + _timer?.Dispose(); + _timer = null; + } + } +} diff --git a/src/Orleans.Messaging/MessagingStateNames.cs b/src/Orleans.Messaging/MessagingStateNames.cs new file mode 100644 index 00000000000..b4a99609f08 --- /dev/null +++ b/src/Orleans.Messaging/MessagingStateNames.cs @@ -0,0 +1,22 @@ +namespace Orleans.Messaging; + +internal static class MessagingStateNames +{ + private const string Prefix = "__orleans.messaging."; + + public const string Inbox = Prefix + "inbox"; + public const string InboxProcessed = Prefix + "inbox-processed"; + public const string InboxMessageState = Prefix + "inbox-message-state"; + public const string InboxDeadLetters = Prefix + "inbox-dead-letters"; + public const string InboxJobId = Prefix + "inbox-job-id"; + public const string InboxJobHandle = Prefix + "inbox-job-handle"; + public const string InboxCompletedJobId = Prefix + "inbox-completed-job-id"; + public const string InboxJobSequence = Prefix + "inbox-job-sequence"; + public const string Outbox = Prefix + "outbox"; + public const string OutboxMessageState = Prefix + "outbox-message-state"; + public const string OutboxDeadLetters = Prefix + "outbox-dead-letters"; + public const string OutboxJobId = Prefix + "outbox-job-id"; + public const string OutboxJobHandle = Prefix + "outbox-job-handle"; + public const string OutboxCompletedJobId = Prefix + "outbox-completed-job-id"; + public const string OutboxJobSequence = Prefix + "outbox-job-sequence"; +} diff --git a/src/Orleans.Messaging/MessagingTime.cs b/src/Orleans.Messaging/MessagingTime.cs new file mode 100644 index 00000000000..c185c232926 --- /dev/null +++ b/src/Orleans.Messaging/MessagingTime.cs @@ -0,0 +1,21 @@ +using System; + +namespace Orleans.Messaging; + +internal static class MessagingTime +{ + public static bool IsExpired(DateTimeOffset now, DateTimeOffset timestamp, TimeSpan retention) + { + var nowTicks = now.UtcTicks; + var timestampTicks = timestamp.UtcTicks; + return nowTicks >= timestampTicks + && nowTicks - timestampTicks >= retention.Ticks; + } + + public static DateTimeOffset AddClamped(DateTimeOffset timestamp, TimeSpan duration) + { + var utcTicks = timestamp.UtcDateTime.Ticks; + var remainingTicks = DateTimeOffset.MaxValue.Ticks - utcTicks; + return new DateTimeOffset(utcTicks + Math.Min(duration.Ticks, remainingTicks), TimeSpan.Zero); + } +} diff --git a/src/Orleans.Messaging/MessagingTurn.cs b/src/Orleans.Messaging/MessagingTurn.cs new file mode 100644 index 00000000000..5030e326dea --- /dev/null +++ b/src/Orleans.Messaging/MessagingTurn.cs @@ -0,0 +1,75 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +// Accessed on the owning activation. Runtime admission stays non-interleaving and neutral. +// Only the timer/state is reused: every logical arm carries an immutable owner-bound payload. +internal abstract class MessagingTurn : IDisposable where T : struct +{ + private IGrainTimer? _timer; + private T? _pending; + private bool _disposed; + private long _registrationGeneration; + private CancellationToken _timerCancellation; + + protected abstract IGrainTimer RegisterTimer(long registrationGeneration); + protected abstract Task ExecuteAsync(T payload, CancellationToken cancellationToken); + protected abstract void Discard(T payload); + + public void Queue(T payload) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_pending is { } superseded) + { + _pending = null; + Discard(superseded); + } + _pending = payload; + try + { + // Register disarmed so no callback can observe a partially installed timer handle. + if (_timerCancellation.IsCancellationRequested) + { + // A disposed physical handle is never reanimated. The active callback keeps its + // original payload/token; only future arms receive a new registration generation. + _timer?.Dispose(); + _timer = null; + _timerCancellation = default; + } + _timer ??= RegisterTimer(++_registrationGeneration); + _timer.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); + } + catch + { + _pending = null; // Queue's caller still owns this failed admission and its cleanup. + throw; + } + } + + public Task RunAsync(long registrationGeneration, CancellationToken cancellationToken) + { + if (_disposed || registrationGeneration != _registrationGeneration || _pending is not { } payload) + { + return Task.CompletedTask; + } + _pending = null; + _timerCancellation = cancellationToken; + // The active payload is a value snapshot, never the mutable slot used by a subsequent arm. + return ExecuteAsync(payload, cancellationToken); + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + if (_pending is { } queued) + { + _pending = null; + Discard(queued); + } + _timer?.Dispose(); + } +} diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj new file mode 100644 index 00000000000..2069ef3a10b --- /dev/null +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -0,0 +1,29 @@ + + + Microsoft.Orleans.Messaging + Microsoft Orleans Messaging + Opaque messaging contracts and journaled inbox processing for Microsoft Orleans. + $(PackageTags) Messaging Inbox Outbox + false + + true + true + $(DefaultTargetFrameworks) + disable + enable + $(NoWarn);ORLEANSEXP005 + $(VersionSuffix).alpha.1 + alpha.1 + + + + + + + + + + + + + diff --git a/src/Orleans.Messaging/OrleansContracts.txt b/src/Orleans.Messaging/OrleansContracts.txt new file mode 100644 index 00000000000..d082ad687ac --- /dev/null +++ b/src/Orleans.Messaging/OrleansContracts.txt @@ -0,0 +1,17 @@ +# This file is generated by the Orleans contract analyzer. +# To regenerate this project from the repository root: +# dotnet format PATH_TO_PROJECT.csproj analyzers --severity info --diagnostics ORLEANS0016 ORLEANS0017 ORLEANS0018 ORLEANS0019 ORLEANS0020 ORLEANS0022 ORLEANS0023 ORLEANS0024 +# Run the command once per contract project; do not pass a .sln or .slnx path. +# Verify with: dotnet build PATH_TO_PROJECT.csproj +# The regeneration command edits this manifest only; it does not change source attributes. +# OrleansContracts format: 2 +# Method lines use: wire-identity: CLR-signature. +# The identity is the identifier Orleans uses at runtime, whether generated or declared in source. +# Review every diff: identity or signature changes can break wire compatibility during rolling upgrades. +# Details: https://aka.ms/orleans/OrleansContracts.txt + +*RETIRED* interface [GrainInterfaceType("Orleans.DurableMessaging.IDurableInboxExtension")] Orleans.DurableMessaging.IDurableInboxExtension [Version(0)] + DeliverAsync: DeliverAsync(Orleans.DurableMessaging.InboxMessage, System.Threading.CancellationToken) -> ValueTask + +interface [GrainInterfaceType("Orleans.Messaging.IInboxExtension")] Orleans.Messaging.IInboxExtension [Version(0)] + DeliverAsync: DeliverAsync(Orleans.Messaging.InboxMessage, System.Threading.CancellationToken) -> ValueTask diff --git a/src/Orleans.Messaging/OutboxMessage.cs b/src/Orleans.Messaging/OutboxMessage.cs new file mode 100644 index 00000000000..32210bb8fde --- /dev/null +++ b/src/Orleans.Messaging/OutboxMessage.cs @@ -0,0 +1,20 @@ +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// An outgoing command with destination routing separate from its common envelope. +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessage")] +public readonly struct OutboxMessage +{ + /// Gets the command identity and common header buffer. + [Id(0)] + public required Envelope Envelope { get; init; } + + /// Gets the required destination grain identity. + [Id(1)] + public required GrainId ReceiverId { get; init; } + + /// Creates the incoming view by sharing the common envelope and its buffer. + /// A received message whose destination is implied by the receiving grain. + public InboxMessage ToInboxMessage() => new() { Envelope = Envelope }; +} diff --git a/src/Orleans.Messaging/PackedEnvelopeHeaders.cs b/src/Orleans.Messaging/PackedEnvelopeHeaders.cs new file mode 100644 index 00000000000..6f4121dd03f --- /dev/null +++ b/src/Orleans.Messaging/PackedEnvelopeHeaders.cs @@ -0,0 +1,211 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Text; + +namespace Orleans.Messaging; + +internal static class PackedEnvelopeHeaders +{ + // Version, count and directory length precede payload length, keyed lengths, then contiguous values. + internal const byte Version = 1; + internal static readonly UTF8Encoding Utf8 = new(false, true); + + internal static void ValidateKey(string key) + { + ArgumentException.ThrowIfNullOrEmpty(key); + _ = Utf8.GetByteCount(key); + } + + internal static int GetKeyToken(string key) => key switch + { + MessageHeaders.Payload => 0, + MessageHeaders.Subject => 1, + MessageHeaders.Sender => 2, + _ => checked(3 + Utf8.GetByteCount(key)) + }; + + internal static void ValidateSubject(ReadOnlySpan subject) + { + if (subject.IsEmpty || subject.Length > EnvelopeValidation.MaxSubjectBytes) + { + throw new ArgumentException($"Subjects require 1 to {EnvelopeValidation.MaxSubjectBytes} UTF-8 bytes.", nameof(subject)); + } + _ = Utf8.GetCharCount(subject); + } + + internal static int VarIntLength(int value) + { + var length = 1; + while ((value >>= 7) > 0) length++; + return length; + } + + internal static void WriteVarInt(Span bytes, ref int position, int value) + { + do + { + var next = (byte)(value & 127); + value >>= 7; + bytes[position++] = (byte)(next | (value > 0 ? 128 : 0)); + } while (value > 0); + } + + private static int ReadVarInt(ReadOnlySpan bytes, ref int position) + { + uint value = 0; + for (var shift = 0; shift <= 28; shift += 7) + { + if (position >= bytes.Length) throw Invalid("Truncated integer."); + var next = bytes[position++]; + if (shift == 28 && next > 7) throw Invalid("Integer exceeds the buffer address space."); + value |= (uint)(next & 127) << shift; + if (next < 128) + { + if (shift > 0 && next == 0) throw Invalid("Noncanonical integer."); + return (int)value; + } + } + throw Invalid("Invalid integer."); + } + + private static (int Count, int DirectoryEnd) ReadPrefix(ReadOnlySpan bytes, ref int position) + { + if (bytes.IsEmpty || bytes[position++] != Version) throw Invalid("Unknown or missing envelope format."); + var count = ReadVarInt(bytes, ref position); + var directoryLength = ReadVarInt(bytes, ref position); + if (count == 0 || directoryLength == 0 || directoryLength > bytes.Length - position + || count - 1 > (directoryLength - 1) / 2) + { + throw Invalid("Invalid header count or directory length."); + } + return (count, position + directoryLength); + } + + internal static void Validate(ReadOnlySpan bytes) + { + var position = 0; + var (count, directoryEnd) = ReadPrefix(bytes, ref position); + var directory = bytes[..directoryEnd]; + var valuePosition = directoryEnd; + var payloadLength = ReadVarInt(directory, ref position); + AdvanceValue(bytes, ref valuePosition, payloadLength); + HashSet? customKeys = null; + var standardKeys = 0; + for (var i = 1; i < count; i++) + { + var token = ReadVarInt(directory, ref position); + if (token is 1 or 2) + { + var mask = 1 << token; + if ((standardKeys & mask) != 0) throw Invalid("Duplicate standard header."); + standardKeys |= mask; + } + else + { + var nameLength = token - 3; + if (nameLength <= 0 || nameLength > directoryEnd - position) throw Invalid("Invalid header key length."); + string key; + try { key = Utf8.GetString(directory.Slice(position, nameLength)); } + catch (DecoderFallbackException error) { throw Invalid("Invalid UTF-8 key.", error); } + position += nameLength; + if (key is MessageHeaders.Payload or MessageHeaders.Subject or MessageHeaders.Sender) + throw Invalid("Standard keys require their reserved encoding."); + if (!(customKeys ??= new(StringComparer.Ordinal)).Add(key)) throw Invalid("Duplicate custom header."); + } + var length = ReadVarInt(directory, ref position); + var start = valuePosition; + AdvanceValue(bytes, ref valuePosition, length); + if (token == 1) + { + try { ValidateSubject(bytes.Slice(start, length)); } + catch (ArgumentException error) { throw Invalid("Invalid subject header.", error); } + } + } + if (position != directoryEnd || valuePosition != bytes.Length) throw Invalid("Trailing or missing directory/value bytes."); + } + + private static void AdvanceValue(ReadOnlySpan bytes, ref int position, int length) + { + if (length > bytes.Length - position) throw Invalid("Header value exceeds the buffer."); + position += length; + } + + internal static ReadOnlyMemory GetPayload(byte[] bytes) + { + var position = 0; + var (_, directoryEnd) = ReadPrefix(bytes, ref position); + var length = ReadVarInt(bytes.AsSpan(0, directoryEnd), ref position); + if (length > bytes.Length - directoryEnd) throw Invalid("Payload exceeds the buffer."); + return bytes.AsMemory(directoryEnd, length); + } + + internal static bool TryGetBytes(byte[] bytes, string key, out ReadOnlyMemory value) + { + ValidateKey(key); + if (key == MessageHeaders.Payload) + { + value = GetPayload(bytes); + return true; + } + var wanted = GetKeyToken(key); + byte[]? rented = null; + var keyLength = wanted >= 4 ? wanted - 3 : 0; + Span encodedKey = keyLength <= 256 ? stackalloc byte[keyLength] : (rented = ArrayPool.Shared.Rent(keyLength)).AsSpan(0, keyLength); + try + { + if (keyLength > 0) Utf8.GetBytes(key, encodedKey); + var position = 0; + var (count, directoryEnd) = ReadPrefix(bytes, ref position); + var directory = bytes.AsSpan(0, directoryEnd); + var valuePosition = directoryEnd + ReadVarInt(directory, ref position); + for (var i = 1; i < count; i++) + { + var token = ReadVarInt(directory, ref position); + var match = token == wanted; + if (token >= 4) + { + var nameLength = token - 3; + match &= directory.Slice(position, nameLength).SequenceEqual(encodedKey); + position += nameLength; + } + var length = ReadVarInt(directory, ref position); + if (match) + { + value = bytes.AsMemory(valuePosition, length); + return true; + } + valuePosition += length; + } + } + finally + { + if (rented is not null) ArrayPool.Shared.Return(rented); + } + value = default; + return false; + } + + internal static IEnumerable GetKeys(byte[] bytes) + { + var position = 0; + var (count, directoryEnd) = ReadPrefix(bytes, ref position); + _ = ReadVarInt(bytes.AsSpan(0, directoryEnd), ref position); + yield return MessageHeaders.Payload; + for (var i = 1; i < count; i++) + { + var token = ReadVarInt(bytes.AsSpan(0, directoryEnd), ref position); + var key = token switch + { + 1 => MessageHeaders.Subject, + 2 => MessageHeaders.Sender, + _ => Utf8.GetString(bytes, position, token - 3) + }; + if (token >= 4) position += token - 3; + _ = ReadVarInt(bytes.AsSpan(0, directoryEnd), ref position); + yield return key; + } + } + + private static FormatException Invalid(string message, Exception? inner = null) => new(message, inner); +} diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md new file mode 100644 index 00000000000..cd50fee59ee --- /dev/null +++ b/src/Orleans.Messaging/README.md @@ -0,0 +1,59 @@ +# Microsoft Orleans Messaging + +This intermediate project supplies command identities, extensible envelopes, distinct incoming and outgoing +messages, handler contracts, and journaled inbox processing. It remains non-packable while the outgoing +runtime and hosting layers assemble +`Microsoft.Orleans.Messaging`. + +`EnvelopeBuilder` encodes raw bytes and independently Orleans-serialized values into one final, +exact-length managed array. `Envelope` retains its `HierarchicalKey MessageId` and that array. +Its required `payload` header can be empty. `subject`, `sender`, and custom ordinal headers are optional. +Use `AddBytes` for a subject's nonempty UTF-8 bytes and `AddValue` with an externally bound serializer for +sender or custom values. `TryAddBytes` and `TryAddValue` preserve the first value of an existing header. +Dispose the builder after building; published envelopes keep their managed bytes. + +Payload and raw header retrieval expose `ReadOnlyMemory` slices of the common array. Typed retrieval +uses a caller-supplied `Serializer` and decodes only the requested header, including a present typed null. +Enumerating keys decodes custom names on demand. Unknown values remain opaque across copying and serialization. +A failed value serialization propagates its error and leaves no published entry; later builds include only +successfully added values. Every build owns independent bytes. Direct staging and local delivery share the +finalized array, whose contents remain stable after publication. Ordinary Orleans copying and deserialization +produce independent arrays. + +`OutboxMessage` adds a required destination `ReceiverId`. `ToInboxMessage()` shares its envelope with an +`InboxMessage`, whose receiver is the grain owning the inbox. Pending incoming equality compares exact message +ID, payload, and optional subject; outgoing equality also compares destination. Equivalent repeated submissions +retain the original message and all its headers, including sender provenance and custom metadata. Completed +records deduplicate by receiver-local message ID for their configured lifetime. + +Admission validates the exact identity's 1,024 UTF-8 byte and 32 segment limits, required payload, packed counts, +canonical integers, strict UTF-8 keys, unique names, and directory/value bounds before shared mutation. Present +subjects have 1 to 256 UTF-8 bytes. Raw handlers accept identity and payload alone; consuming typed protocols +supply their own subject and sender requirements. + +`HierarchicalKey` is a readonly ordinal value with an immutable canonical path and cached hash. `Create` and +`CreateChildKey` accept literal segments; `Parse` reads escaped canonical paths; `Append` composes hierarchies. +The default key is unset. Serialization stores the canonical path and reconstructs navigation and hash state. + +`IInboxHandler.HandleAsync` receives an `InboxMessage` through its context and explicitly calls `Complete()`. +Decode local results, do asynchronous preparation, build outgoing messages, and check cancellation before the +first shared mutation. From that mutation through method return, execute synchronously: apply safe-to-commit +changes, send through `IOutbox`, complete, and return. The runtime owns persistence and acknowledgement. + +The receiver acknowledges admission after its wakeup is scheduled and the journal persists the incoming +message with the logical generation and exact returned physical job handle. Pending-message conflicts fail +before scheduling or mutation. Caller cancellation ends the wait while an admitted operation retains its gate +through the actual persistence outcome. Handler preparation failures follow bounded retry and dead-letter +policy; errors after Complete preserve the logical outcome through acknowledgement. + +Recovery validates pending and dead-letter header buffers before marking the inbox ready or scheduling work. +Diagnostics expose `DeadLetter` and `DeadLetter`, preserving header +content while keeping outgoing destinations separate. Stop closes admission and drains actual operations +before deletion or scope disposal. Storage failures retain their first cause; fresh activations restore only +the actual journal outcome. + +The packed format directly replaces the earlier unreleased five-field envelope. Use fresh journal state when +adopting this representation. The format is a version byte, compact header count and directory length, +payload length, keyed value lengths, and contiguous value bytes. Standard keys have reserved identifiers; +custom names are UTF-8. Values are independent and header offsets are derived from lengths. Generated envelopes +received from external calls or recovery are validated by the owning admission/recovery boundary. diff --git a/src/Orleans.Runtime/Timers/GrainTimer.cs b/src/Orleans.Runtime/Timers/GrainTimer.cs index 525ce76e30a..dc2a1248d2f 100644 --- a/src/Orleans.Runtime/Timers/GrainTimer.cs +++ b/src/Orleans.Runtime/Timers/GrainTimer.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Reflection; using Microsoft.Extensions.Logging; @@ -21,11 +22,18 @@ internal abstract partial class GrainTimer : IGrainTimer private readonly bool _interleave; private readonly bool _keepAlive; private readonly TimerTickInvoker _invoker; - private bool _changed; - private bool _firing; + private TimerState _state; + // The current arm while idle; the next delay once a tick is queued. Change replaces it. private TimeSpan _dueTime; private TimeSpan _period; + private enum TimerState : byte + { + Idle, + Busy, + Disposed + } + public GrainTimer(TimerRegistry shared, IGrainContext grainContext, bool interleave, bool keepAlive) { ArgumentNullException.ThrowIfNull(shared); @@ -38,17 +46,29 @@ public GrainTimer(TimerRegistry shared, IGrainContext grainContext, bool interle _dueTime = Timeout.InfiniteTimeSpan; _period = Timeout.InfiniteTimeSpan; _invoker = new(this); - - // Avoid capturing async locals. using (new ExecutionContextSuppressor()) { - _timer = shared.TimeProvider.CreateTimer(TimerCallback, this, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + _timer = _shared.TimeProvider.CreateTimer(TimerCallback, this, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); } - } protected IGrainContext GrainContext => _grainContext; + // Called with _cts locked, after Change or callback completion. + private bool ScheduleNextTick() + { + Debug.Assert(_state == TimerState.Idle); + _timer.Change(_dueTime == TimeSpan.Zero ? Timeout.InfiniteTimeSpan : _dueTime, Timeout.InfiniteTimeSpan); + if (_dueTime == TimeSpan.Zero) + { + _state = TimerState.Busy; + _dueTime = _period; + return true; + } + + return false; + } + private ILogger Logger => _shared.TimerLogger; [DoesNotReturn] @@ -65,13 +85,30 @@ private static void ThrowInvalidSchedulingContext() protected void ScheduleTickOnActivation() { - try + lock (_cts) { - // Indicate that the timer is firing so that the effect of the next change call is deferred until after the tick completes. - _firing = true; + // Provider callbacks may race Change. A reserved tick stays busy until completion. + if (_state != TimerState.Idle || _dueTime == Timeout.InfiniteTimeSpan) + { + return; + } - // Note: this does not execute on the activation's execution context. - var msg = _shared.MessageFactory.CreateMessage(body: _invoker, options: InvokeMethodOptions.OneWay); + _state = TimerState.Busy; + _dueTime = _period; + } + + QueueTickOnActivation(); + } + + // The admission is reserved under _cts, but delivered outside it: activation shutdown disposes + // timers under its own lock. Taking that lock while holding _cts would invert the lock order. + // Changes during delivery affect the following tick; disposal prevents callback admission. + private void QueueTickOnActivation() + { + try + { + // Timer requests start a new call chain, including ticks queued immediately during registration. + var msg = _shared.MessageFactory.CreateMessage(body: _invoker, options: InvokeMethodOptions.OneWay, requestContextData: null); msg.SetInfiniteTimeToLive(); msg.SendingGrain = _grainContext.GrainId; msg.TargetGrain = _grainContext.GrainId; @@ -88,14 +125,23 @@ protected void ScheduleTickOnActivation() } catch (Exception exception) { + lock (_cts) + { + if (_state != TimerState.Disposed) + { + _state = TimerState.Idle; + // Release failed admission without immediately retrying the failed delivery. + _timer.Change(_dueTime == TimeSpan.Zero ? Timeout.InfiniteTimeSpan : _dueTime, Timeout.InfiniteTimeSpan); + } + } + try { LogErrorScheduleTickOnActivation(Logger, exception, this); } catch { - // Ignore. - // Allowing an exception to escape here would crash the process. + // Allowing an exception to escape a physical timer callback would crash the process. } } } @@ -104,11 +150,20 @@ protected void ScheduleTickOnActivation() private ValueTask InvokeGrainTimerCallbackAsync() { + lock (_cts) + { + if (_state == TimerState.Disposed) + { + return new(Response.Completed); + } + + Debug.Assert(_state == TimerState.Busy); + } + try { LogTraceBeforeCallback(Logger, this); - _changed = false; GrainTimerEvents.EmitTickStart(GrainContext, this); var task = InvokeCallbackAsync(_cts.Token); @@ -138,32 +193,22 @@ private ValueTask InvokeGrainTimerCallbackAsync() private void OnTickCompleted() { - // Schedule the next tick. - try + bool queueTick; + lock (_cts) { - if (_cts.IsCancellationRequested) + if (_state == TimerState.Disposed) { - // The instance has been disposed. No further ticks should be fired. return; } - if (!_changed) - { - // If the timer was not modified during the tick, schedule the next tick based on the period. - _timer.Change(_period, Timeout.InfiniteTimeSpan); - } - else - { - // If the timer was modified during the tick, schedule the next tick based on the new due time. - _timer.Change(_dueTime, Timeout.InfiniteTimeSpan); - } - } - catch (ObjectDisposedException) - { + Debug.Assert(_state == TimerState.Busy); + _state = TimerState.Idle; + queueTick = ScheduleNextTick(); } - finally + + if (queueTick) { - _firing = false; + QueueTickOnActivation(); } } @@ -201,24 +246,28 @@ public void Change(TimeSpan dueTime, TimeSpan period) { ValidateArguments(dueTime, period); - _changed = true; - _dueTime = dueTime; - _period = period; - - // If the timer is currently firing, the change will be deferred until after the tick completes. - // Otherwise, perform the change now. - if (!_firing) + var queueTick = false; + lock (_cts) { - try + if (_state == TimerState.Disposed) { - // This method resets the timer, so the next tick will be scheduled at the new due time and subsequent - // ticks will be scheduled after the specified period. - _timer.Change(dueTime, Timeout.InfiniteTimeSpan); + return; } - catch (ObjectDisposedException) + + _dueTime = dueTime; + _period = period; + + // A queued or running callback keeps its turn; changes schedule work after it completes. + if (_state == TimerState.Idle) { + queueTick = ScheduleNextTick(); } } + + if (queueTick) + { + QueueTickOnActivation(); + } } private static void ValidateArguments(TimeSpan dueTime, TimeSpan period) @@ -238,8 +287,21 @@ private static void ValidateArguments(TimeSpan dueTime, TimeSpan period) public void Dispose() { + lock (_cts) + { + if (_state == TimerState.Disposed) + { + return; + } + + // Publish disposal before cancellation, whose registrations can reenter Change/Dispose. + _state = TimerState.Disposed; + _timer.Dispose(); + } + try { + // Do not run cancellation registrations under the timer's state lock. _cts.Cancel(); } catch (Exception exception) @@ -247,8 +309,6 @@ public void Dispose() LogErrorCancellingCallback(Logger, exception); } - _timer.Dispose(); - GrainTimerEvents.EmitDisposed(GrainContext, this); var timerRegistry = _grainContext.GetComponent(); diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 396a1692cad..76ff42769dd 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -219,8 +219,20 @@ public partial interface IDurableValue T? Value { get; set; } } + public partial interface IJournaledStateCaptureHook : IJournaledStateHook + { + } + + public partial interface IJournaledStateHook + { + System.Threading.Tasks.ValueTask AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.ValueTask BeforeOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); + } + public partial interface IJournaledStateManager : System.IAsyncDisposable { + System.Collections.Generic.IList Hooks { get; } + long PendingWriteByteCount { get; } System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken = default); @@ -424,6 +436,25 @@ public sealed partial class JournaledStateManagerOptions public System.TimeSpan RetirementGracePeriod { get { throw null; } set { } } } + public enum JournaledStateOperation + { + Write = 0, + Snapshot = 1, + Delete = 2 + } + + [GenerateSerializer] + public sealed partial class JournaledStatePostCommitException : System.Exception + { + public JournaledStatePostCommitException(JournaledStateOperation operation, System.Exception innerException) { } + } + + [GenerateSerializer] + public sealed partial class JournaledStatePreCommitException : System.Exception + { + public JournaledStatePreCommitException(JournaledStateOperation operation, System.Exception innerException) { } + } + public readonly ref partial struct JournalEntry { private readonly object _dummy; @@ -800,10 +831,62 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. where TBufferWriter : System.Buffers.IBufferWriter { } } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_JournaledStatePostCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePostCommitException instance) { } + + public global::Orleans.Journaling.JournaledStatePostCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePostCommitException instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePostCommitException value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_JournaledStatePreCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePreCommitException instance) { } + + public global::Orleans.Journaling.JournaledStatePreCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePreCommitException instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePreCommitException value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] public sealed partial class Copier_DurableTaskCompletionSourceState : global::Orleans.Serialization.Cloning.ShallowCopier> { } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_JournaledStatePostCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier + { + public Copier_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_JournaledStatePreCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier + { + public Copier_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } + } } \ No newline at end of file diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs new file mode 100644 index 00000000000..5e5e5794dea --- /dev/null +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -0,0 +1,451 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ +namespace Orleans.Messaging +{ + public sealed partial class DeadLetter + { + public int AttemptCount { get { throw null; } init { } } + + public System.DateTimeOffset DeadLetteredAt { get { throw null; } init { } } + + public required TMessage Message { get { throw null; } init { } } + + public required string Reason { get { throw null; } init { } } + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.DeliveryResult")] + public readonly partial struct DeliveryResult + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + [Id(1)] + public string? Message { get { throw null; } init { } } + + [Id(0)] + public DeliveryStatus Status { get { throw null; } init { } } + + public static DeliveryResult Accepted() { throw null; } + + public static DeliveryResult Backpressured() { throw null; } + + public static DeliveryResult DeadLettered(string reason) { throw null; } + + public static DeliveryResult Duplicate() { throw null; } + + public static DeliveryResult HandlerNotFound() { throw null; } + } + + public enum DeliveryStatus + { + Accepted = 0, + Duplicate = 1, + Backpressured = 2, + HandlerNotFound = 3, + DeadLettered = 4 + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.Envelope")] + public readonly partial struct Envelope + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public System.ReadOnlyMemory EncodedHeaders { get { throw null; } } + + public System.Collections.Generic.IEnumerable Keys { get { throw null; } } + + [Id(0)] + public HierarchicalKey MessageId { get { throw null; } } + + public System.ReadOnlyMemory Payload { get { throw null; } } + + public static Envelope FromEncodedHeaders(HierarchicalKey messageId, System.ReadOnlySpan encodedHeaders) { throw null; } + + public readonly bool TryGetBytes(string key, out System.ReadOnlyMemory value) { throw null; } + + public readonly bool TryGetSubject(out string? subject) { throw null; } + + public readonly bool TryGetValue(string key, Serialization.Serializer serializer, out T? value) { throw null; } + } + + public sealed partial class EnvelopeBuilder : System.IDisposable, System.Buffers.IBufferWriter + { + public EnvelopeBuilder(HierarchicalKey messageId) { } + + public void AddBytes(string key, System.ReadOnlySpan value) { } + + public void AddValue(string key, T? value, Serialization.Serializer serializer) { } + + public Envelope Build() { throw null; } + + public void Dispose() { } + + void System.Buffers.IBufferWriter.Advance(int count) { } + + System.Memory System.Buffers.IBufferWriter.GetMemory(int sizeHint) { throw null; } + + System.Span System.Buffers.IBufferWriter.GetSpan(int sizeHint) { throw null; } + + public bool TryAddBytes(string key, System.ReadOnlySpan value) { throw null; } + + public bool TryAddValue(string key, T? value, Serialization.Serializer serializer) { throw null; } + } + + [Immutable] + [Alias("Orleans.Messaging.HierarchicalKey")] + public readonly partial struct HierarchicalKey : System.ISpanFormattable, System.IFormattable, System.IEquatable, System.IParsable, System.ISpanParsable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public const char EscapeCharacter = '\\'; + public const char SegmentSeparator = '/'; + public bool IsDefault { get { throw null; } } + + public int Length { get { throw null; } } + + public int SegmentCount { get { throw null; } } + + public readonly HierarchicalKey Append(HierarchicalKey suffix) { throw null; } + + public static HierarchicalKey Create(scoped params System.ReadOnlySpan values) { throw null; } + + public static HierarchicalKey Create(string value) { throw null; } + + public readonly HierarchicalKey CreateChildKey(string value) { throw null; } + + public readonly bool Equals(HierarchicalKey other) { throw null; } + + public override readonly bool Equals(object? obj) { throw null; } + + public readonly SegmentEnumerator GetEnumerator() { throw null; } + + public override readonly int GetHashCode() { throw null; } + + public readonly HierarchicalKey? GetParent() { throw null; } + + public readonly bool IsAncestorOf(HierarchicalKey other) { throw null; } + + public readonly bool IsChildOf(HierarchicalKey other) { throw null; } + + public readonly bool IsParentOf(HierarchicalKey other) { throw null; } + + public static bool operator ==(HierarchicalKey left, HierarchicalKey right) { throw null; } + + public static bool operator !=(HierarchicalKey left, HierarchicalKey right) { throw null; } + + static HierarchicalKey System.ISpanParsable.Parse(System.ReadOnlySpan s, System.IFormatProvider? provider) { throw null; } + + static HierarchicalKey System.IParsable.Parse(string s, System.IFormatProvider? provider) { throw null; } + + public override readonly string ToString() { throw null; } + + public readonly string ToString(string? format, System.IFormatProvider? formatProvider) { throw null; } + + public readonly bool TryFormat(System.Span destination, out int charsWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } + + static bool System.ISpanParsable.TryParse(System.ReadOnlySpan s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + static bool System.IParsable.TryParse(string? s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + public ref partial struct SegmentEnumerator + { + private object _dummy; + private int _dummyPrimitive; + public System.ReadOnlySpan Current { get { throw null; } } + + public bool MoveNext() { throw null; } + } + } + + public partial interface IInbox + { + int Capacity { get; } + + int Count { get; } + + System.Collections.Generic.IEnumerable Messages { get; } + + void RegisterHandler(IInboxHandler handler); + bool TryGetMessage(HierarchicalKey messageId, out InboxMessage message); + } + + [Alias("IInboxExtension")] + public partial interface IInboxExtension : Runtime.IGrainExtension, Runtime.IAddressable + { + [Alias("DeliverAsync")] + System.Threading.Tasks.ValueTask DeliverAsync(InboxMessage message, System.Threading.CancellationToken cancellationToken = default); + } + + public partial interface IInboxHandler + { + System.Threading.Tasks.ValueTask HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + } + + public partial interface IInboxHandlerContext + { + InboxMessage Message { get; } + + void Complete(); + } + + public partial interface IMessagingDiagnostics + { + System.Collections.Generic.IReadOnlyList> InboxDeadLetters { get; } + + System.Collections.Generic.IReadOnlyList> OutboxDeadLetters { get; } + + bool RemoveInboxDeadLetter(HierarchicalKey messageId); + bool RemoveOutboxDeadLetter(HierarchicalKey messageId); + } + + public partial interface IMessagingGrain + { + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.InboxMessage")] + public readonly partial struct InboxMessage + { + [Id(0)] + public required Envelope Envelope { get { throw null; } init { } } + } + + public partial interface IOutbox + { + int Count { get; } + + System.Collections.Generic.IEnumerable Messages { get; } + + Runtime.GrainId SenderId { get; } + + void Send(OutboxMessage message); + bool TryGetMessage(HierarchicalKey messageId, out OutboxMessage message); + } + + public static partial class MessageHeaders + { + public const string Payload = "payload"; + public const string Sender = "sender"; + public const string Subject = "subject"; + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.OutboxMessage")] + public readonly partial struct OutboxMessage + { + [Id(0)] + public required Envelope Envelope { get { throw null; } init { } } + + [Id(1)] + public required Runtime.GrainId ReceiverId { get { throw null; } init { } } + + public readonly InboxMessage ToInboxMessage() { throw null; } + } +} + +namespace Orleans.Messaging.Configuration +{ + public partial class InboxOptions + { + public System.TimeSpan BackpressureRetryDelay { get { throw null; } set { } } + + public System.TimeSpan DeadLetterRetentionPeriod { get { throw null; } set { } } + + public System.TimeSpan DeduplicationWindow { get { throw null; } set { } } + + public int InboxBatchSize { get { throw null; } set { } } + + public int MaxCapacity { get { throw null; } set { } } + + public int MaxDeliveryAttempts { get { throw null; } set { } } + + public System.TimeSpan MaxOutboxRetryAge { get { throw null; } set { } } + + public int MaxProcessingAttempts { get { throw null; } set { } } + + public int MaxRetainedDeadLetters { get { throw null; } set { } } + + public int OutboxBatchSize { get { throw null; } set { } } + + public System.TimeSpan OutboxIdleRetirementGracePeriod { get { throw null; } set { } } + + public void Validate() { } + } +} + +namespace OrleansCodeGen.Orleans.Messaging +{ + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_DeliveryResult : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_DeliveryResult(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.DeliveryResult instance) { } + + public global::Orleans.Messaging.DeliveryResult ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.DeliveryResult instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.DeliveryResult value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_Envelope : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_Envelope(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.Envelope instance) { } + + public global::Orleans.Messaging.Envelope ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.Envelope instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.Envelope value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_InboxMessage : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_InboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.InboxMessage instance) { } + + public global::Orleans.Messaging.InboxMessage ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.InboxMessage instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.InboxMessage value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 instance) { } + + public Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_OutboxMessage : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_OutboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.OutboxMessage instance) { } + + public global::Orleans.Messaging.OutboxMessage ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.OutboxMessage instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.OutboxMessage value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_Envelope : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public global::Orleans.Messaging.Envelope DeepCopy(global::Orleans.Messaging.Envelope result, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_InboxMessage : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_InboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.InboxMessage DeepCopy(global::Orleans.Messaging.InboxMessage original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 DeepCopy(Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_OutboxMessage : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_OutboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.OutboxMessage DeepCopy(global::Orleans.Messaging.OutboxMessage original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + [global::Orleans.CompoundTypeAlias(new[] { "inv", typeof(global::Orleans.Runtime.GrainReference), "Ext", typeof(global::Orleans.Messaging.IInboxExtension), typeof(global::Orleans.Messaging.IInboxExtension), "DeliverAsync" })] + public sealed partial class Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Runtime.Request + { + public global::Orleans.Messaging.InboxMessage arg0; + public System.Threading.CancellationToken arg1; + public override bool IsCancellable { get { throw null; } } + + public override void Dispose() { } + + public override string GetActivityName() { throw null; } + + public override object GetArgument(int index) { throw null; } + + public override int GetArgumentCount() { throw null; } + + public override System.Threading.CancellationToken GetCancellationToken() { throw null; } + + public override string GetInterfaceName() { throw null; } + + public override System.Type GetInterfaceType() { throw null; } + + public override System.Reflection.MethodInfo GetMethod() { throw null; } + + public override string GetMethodName() { throw null; } + + public override object GetTarget() { throw null; } + + protected override System.Threading.Tasks.ValueTask InvokeInner() { throw null; } + + public override void SetArgument(int index, object value) { } + + public override void SetTarget(global::Orleans.Serialization.Invocation.ITargetHolder holder) { } + + public override bool TryCancel() { throw null; } + } +} \ No newline at end of file diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index c94bd6045e2..f5931b7e85b 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -126,6 +126,8 @@ private void ReplayEncodedJournalData() private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter buffer, JournalStreamId streamId) : IJournaledStateManager { + public IList Hooks { get; } = []; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(buffer.CreateJournalStreamWriter(streamId)); diff --git a/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs new file mode 100644 index 00000000000..c8caa18f4ac --- /dev/null +++ b/test/Orleans.Core.Tests/Runtime/GrainTimerTests.cs @@ -0,0 +1,605 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization; +using Orleans.Serialization.Invocation; +using Orleans.Timers; +using Xunit; + +namespace NonSilo.Tests.Runtime; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestCategory("BVT")] +public sealed class GrainTimerTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ImmediateTick_UsesOrdinaryActivationAdmission(bool interleave, bool keepAlive) + { + using var fixture = new TimerFixture(); + using var reentrancy = RequestContext.AllowCallChainReentrancy(); + var state = new object(); + var parent = new UncopyableContextValue(); + var calls = 0; + RequestContext.Set("parent", parent); + try + { + using var timer = fixture.Register((value, token) => + { + Assert.Same(state, value); + Assert.False(token.IsCancellationRequested); + Assert.Equal(Guid.Empty, RequestContext.ReentrancyId); + Assert.Null(RequestContext.Get("parent")); + calls++; + return Task.CompletedTask; + }, state, TimeSpan.Zero, Timeout.InfiniteTimeSpan, interleave, keepAlive); + + var message = Assert.Single(fixture.Messages); + Assert.Equal(0, calls); + Assert.Equal(1, fixture.Time.TimerCreations); + Assert.Equal(Message.Directions.OneWay, message.Direction); + Assert.Equal(interleave, message.IsAlwaysInterleave); + Assert.Equal(keepAlive, message.IsKeepAlive); + Assert.True(message.IsLocalOnly); + Assert.Null(message.TimeToLive); + Assert.Null(message.RequestContextData); + Assert.Same(parent, RequestContext.Get("parent")); + Assert.Equal(fixture.Grain.GrainId, message.TargetGrain); + Assert.Equal(message.SendingGrain, message.TargetGrain); + Assert.Equal(message.SendingSilo, message.TargetSilo); + await fixture.InvokeAsync(message); + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Equal(1, calls); + Assert.Single(fixture.Messages); + } + finally + { + RequestContext.Remove("parent"); + } + } + + [Fact] + public void MessageFactory_DefaultContextIsCopiedWhileExplicitContextSkipsExport() + { + using var fixture = new TimerFixture(); + var value = new List { 1, 2 }; + RequestContext.Set("parent", value); + try + { + var message = fixture.MessageFactory.CreateMessage(null, Orleans.CodeGeneration.InvokeMethodOptions.OneWay); + var copied = Assert.IsType>(message.RequestContextData!["parent"]); + Assert.Equal(value, copied); + Assert.NotSame(value, copied); + value.Add(3); + Assert.Equal(new[] { 1, 2 }, copied); + + RequestContext.Set("parent", new UncopyableContextValue()); + Assert.Throws(() => + fixture.MessageFactory.CreateMessage(null, Orleans.CodeGeneration.InvokeMethodOptions.OneWay)); + var independent = fixture.MessageFactory.CreateMessage(null, Orleans.CodeGeneration.InvokeMethodOptions.OneWay, requestContextData: null); + Assert.Null(independent.RequestContextData); + Assert.IsType(RequestContext.Get("parent")); + } + finally + { + RequestContext.Remove("parent"); + } + } + + [Theory] + [InlineData(-1)] + [InlineData(0)] + [InlineData(2000)] + public async Task ChangeWhileQueued_PreservesTickAndUsesLatestFollowingSchedule(int milliseconds) + { + using var fixture = new TimerFixture(); + var calls = 0; + using var timer = fixture.Register((_, _) => + { + calls++; + return Task.CompletedTask; + }, 0, TimeSpan.Zero, TimeSpan.FromSeconds(1)); + timer.Change(TimeSpan.Zero, TimeSpan.Zero); + timer.Change(TimeSpan.FromMilliseconds(milliseconds), Timeout.InfiniteTimeSpan); + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Single(fixture.Messages); + await fixture.InvokeAsync(fixture.Messages[0]); + Assert.Equal(1, calls); + await VerifyFollowingTickAsync(fixture, milliseconds); + Assert.Equal(milliseconds < 0 ? 1 : 2, calls); + } + + [Theory] + [InlineData(-1)] + [InlineData(0)] + [InlineData(2000)] + public async Task ChangeDuringCallback_DefersFollowingTickUntilCompletion(int milliseconds) + { + using var fixture = new TimerFixture(); + var release = NewCompletion(); + var started = NewCompletion(); + var calls = 0; + using var timer = fixture.Register(async (_, _) => + { + if (++calls == 1) + { + started.SetResult(); + await release.Task; + } + }, 0, TimeSpan.Zero, TimeSpan.Zero, interleave: true); + var invocation = fixture.InvokeAsync(Assert.Single(fixture.Messages)); + try + { + await started.Task; + timer.Change(TimeSpan.Zero, TimeSpan.Zero); + timer.Change(TimeSpan.FromMilliseconds(milliseconds), Timeout.InfiniteTimeSpan); + fixture.Time.Advance(TimeSpan.FromDays(1)); + fixture.Time.FireDispatchedCallback(); + Assert.Equal(1, calls); + Assert.Single(fixture.Messages); + } + finally + { + release.SetResult(); + await invocation; + } + await VerifyFollowingTickAsync(fixture, milliseconds); + Assert.Equal(milliseconds < 0 ? 1 : 2, calls); + } + + [Fact] + public async Task ZeroPeriod_QueuesSuccessiveTurnsAndDisposalDrainsLastTurn() + { + using var fixture = new TimerFixture(); + var calls = 0; + using var timer = fixture.Register((_, _) => + { + Assert.Equal(++calls, fixture.Messages.Count); + return Task.CompletedTask; + }, 0, TimeSpan.Zero, TimeSpan.Zero); + for (var i = 0; i < 8; i++) + { + await fixture.InvokeAsync(fixture.Messages[i]); + Assert.Equal(i + 2, fixture.Messages.Count); + } + timer.Dispose(); + await fixture.InvokeAsync(fixture.Messages[8]); + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Equal(8, calls); + Assert.Equal(9, fixture.Messages.Count); + Assert.Equal(1, fixture.Time.TimerCreations); + } + + [Theory] + [InlineData(0.5)] + [InlineData(100.5)] + public async Task DelayedTick_UsesProviderTimingAndResumesPeriod(double milliseconds) + { + using var fixture = new TimerFixture(); + var calls = 0; + var delay = TimeSpan.FromMilliseconds(milliseconds); + using var timer = fixture.Register((_, _) => + { + calls++; + return Task.CompletedTask; + }, 0, delay, delay); + for (var i = 0; i < 2; i++) + { + fixture.Time.Advance(delay - TimeSpan.FromTicks(1)); + Assert.Equal(i, fixture.Messages.Count); + fixture.Time.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(i + 1, fixture.Messages.Count); + await fixture.InvokeAsync(fixture.Messages[i]); + Assert.Equal(i + 1, calls); + } + Assert.Equal(1, fixture.Time.TimerCreations); + } + + [Fact] + public async Task DispatchedTick_RacingDelayedChange_IsAdmittedOnce() + { + using var fixture = new TimerFixture(); + var calls = 0; + using var timer = fixture.Register((_, _) => + { + calls++; + return Task.CompletedTask; + }, 0, TimeSpan.FromSeconds(1), Timeout.InfiniteTimeSpan); + timer.Change(TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)); + fixture.Time.FireDispatchedCallback(); + fixture.Time.FireDispatchedCallback(); + Assert.Equal(0, calls); + await fixture.InvokeAsync(Assert.Single(fixture.Messages)); + fixture.Time.Advance(TimeSpan.FromSeconds(5) - TimeSpan.FromTicks(1)); + Assert.Single(fixture.Messages); + fixture.Time.Advance(TimeSpan.FromTicks(1)); + await fixture.InvokeAsync(fixture.Messages[1]); + Assert.Equal(2, calls); + } + + [Fact] + public async Task PausedTimer_DispatchedTickLeavesSchedulePaused() + { + using var fixture = new TimerFixture(); + using var timer = fixture.Register((_, _) => throw new InvalidOperationException("Unexpected tick"), + 0, TimeSpan.FromSeconds(1), TimeSpan.Zero); + timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + fixture.Time.FireDispatchedCallback(); + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Empty(fixture.Messages); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task DisposedTimer_QueuedTickDrains(bool delayed, bool interleave) + { + using var fixture = new TimerFixture(); + var calls = 0; + using var timer = fixture.Register((_, _) => + { + calls++; + return Task.CompletedTask; + }, 0, delayed ? TimeSpan.FromSeconds(1) : TimeSpan.Zero, TimeSpan.Zero, interleave); + if (delayed) + { + fixture.Time.Advance(TimeSpan.FromSeconds(1)); + } + timer.Dispose(); + timer.Change(TimeSpan.Zero, TimeSpan.Zero); + fixture.Time.FireDispatchedCallback(); + await fixture.InvokeAsync(Assert.Single(fixture.Messages)); + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Equal(0, calls); + Assert.Single(fixture.Messages); + Assert.Collection(fixture.Events, + evt => Assert.IsType(evt), + evt => Assert.IsType(evt)); + } + + [Fact] + public async Task DisposeDuringDelivery_StateLockIsReleasedBeforeActivationAdmission() + { + using var fixture = new TimerFixture(); + var entered = NewCompletion(); + var release = NewCompletion(); + using var timer = fixture.Register((_, _) => throw new InvalidOperationException("Unexpected tick"), + 0, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + fixture.BeforeReceive = () => + { + entered.SetResult(); + release.Task.GetAwaiter().GetResult(); + }; + var admission = Task.Run(() => timer.Change(TimeSpan.Zero, TimeSpan.Zero), TestContext.Current.CancellationToken); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await Task.Run(timer.Dispose, TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } + finally + { + release.SetResult(); + await admission; + } + await fixture.InvokeAsync(Assert.Single(fixture.Messages)); + Assert.Empty(fixture.Events.OfType()); + } + + [Fact] + public async Task DisposeWithBlockedCancellation_QueuedTickDrainsBeforeCancellationReturns() + { + using var fixture = new TimerFixture(); + var entered = NewCompletion(); + var release = NewCompletion(); + var calls = 0; + CancellationToken token = default; + using var timer = fixture.Register((_, value) => + { + calls++; + token = value; + return Task.CompletedTask; + }, 0, TimeSpan.Zero, Timeout.InfiniteTimeSpan, interleave: true); + await fixture.InvokeAsync(Assert.Single(fixture.Messages)); + using var registration = token.Register(() => + { + entered.SetResult(); + release.Task.GetAwaiter().GetResult(); + }); + timer.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); + var disposal = Task.Run(timer.Dispose, TestContext.Current.CancellationToken); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await Task.Run(() => fixture.InvokeAsync(fixture.Messages[1]), TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.True(token.IsCancellationRequested); + Assert.False(disposal.IsCompleted); + Assert.Equal(1, calls); + } + finally + { + release.SetResult(); + await disposal; + } + Assert.Single(fixture.Events.OfType()); + Assert.Single(fixture.Events.OfType()); + } + + [Fact] + public async Task DisposeDuringCallback_ReentrantFailingCancellationCompletesDisposalOnce() + { + using var fixture = new TimerFixture(); + var registry = Substitute.For(); + fixture.Grain.GetComponent(typeof(IGrainTimerRegistry)).Returns(registry); + var started = NewCompletion(); + var canceled = NewCompletion(); + var exception = new InvalidOperationException("Cancellation registration failed"); + IGrainTimer? timer = null; + timer = fixture.Register(async (_, token) => + { + using var registration = token.Register(() => + { + timer!.Change(TimeSpan.Zero, TimeSpan.Zero); + timer.Dispose(); + canceled.SetResult(); + throw exception; + }); + started.SetResult(); + await canceled.Task; + Assert.True(token.IsCancellationRequested); + }, 0, TimeSpan.Zero, TimeSpan.Zero); + using (timer) + { + var invocation = fixture.InvokeAsync(Assert.Single(fixture.Messages)); + try + { + await started.Task; + } + finally + { + timer.Dispose(); + await invocation; + } + timer.Dispose(); + var cancellationError = Assert.IsType(fixture.SingleLoggedError()); + Assert.Same(exception, Assert.Single(cancellationError.InnerExceptions)); + registry.Received(1).OnTimerDisposed(timer); + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Single(fixture.Messages); + Assert.Single(fixture.Events.OfType()); + Assert.Single(fixture.Events.OfType()); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CallbackFailure_PairsDiagnosticsAndSchedulesNextPeriod(bool asynchronous) + { + using var fixture = new TimerFixture(); + var release = NewCompletion(); + var started = NewCompletion(); + var exception = new InvalidOperationException("Callback failed"); + var calls = 0; + using var timer = fixture.Register((_, _) => + { + if (++calls > 1) + { + return Task.CompletedTask; + } + if (asynchronous) + { + return FailAsync(); + } + throw exception; + }, 0, TimeSpan.Zero, TimeSpan.FromSeconds(2)); + var invocation = fixture.InvokeAsync(Assert.Single(fixture.Messages), exception); + try + { + if (asynchronous) + { + await started.Task; + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Single(fixture.Messages); + } + } + finally + { + release.SetResult(); + await invocation; + } + fixture.Time.Advance(TimeSpan.FromSeconds(2)); + await fixture.InvokeAsync(fixture.Messages[1]); + Assert.Equal(2, calls); + Assert.Collection(fixture.Events, + evt => Assert.IsType(evt), + evt => Assert.IsType(evt), + evt => Assert.Same(exception, Assert.IsType(evt).Exception), + evt => Assert.IsType(evt), + evt => Assert.Null(Assert.IsType(evt).Exception)); + + async Task FailAsync() + { + started.SetResult(); + await release.Task; + throw exception; + } + } + + [Theory] + [InlineData(-1)] + [InlineData(0)] + [InlineData(2000)] + public async Task AdmissionFailure_LogsErrorWithoutImplicitImmediateRetry(int replacementMilliseconds) + { + using var fixture = new TimerFixture(); + var exception = new InvalidOperationException("Admission failed"); + var calls = 0; + using var timer = fixture.Register((_, _) => + { + calls++; + return Task.CompletedTask; + }, 0, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + fixture.BeforeReceive = () => + { + if (replacementMilliseconds >= 0) + { + timer.Change(TimeSpan.FromMilliseconds(replacementMilliseconds), Timeout.InfiniteTimeSpan); + } + throw exception; + }; + timer.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); + Assert.Empty(fixture.Messages); + Assert.Same(exception, fixture.SingleLoggedError()); + fixture.BeforeReceive = null; + if (replacementMilliseconds > 0) + { + fixture.Time.Advance(TimeSpan.FromSeconds(2)); + } + else + { + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Empty(fixture.Messages); + timer.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); + } + await fixture.InvokeAsync(Assert.Single(fixture.Messages)); + Assert.Equal(1, calls); + } + + [Fact] + public async Task LegacyTimer_UsesInterleavingTurnsAndStopsAfterDisposal() + { + using var fixture = new TimerFixture(); + var calls = 0; + using var timer = fixture.Registry.RegisterTimer(fixture.Grain, _ => + { + calls++; + return Task.CompletedTask; + }, null, TimeSpan.Zero, Timeout.InfiniteTimeSpan); + var message = Assert.Single(fixture.Messages); + Assert.True(message.IsAlwaysInterleave); + Assert.False(message.IsKeepAlive); + await fixture.InvokeAsync(message); + timer.Dispose(); + fixture.Time.FireDispatchedCallback(); + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Equal(1, calls); + Assert.Single(fixture.Messages); + } + + private static TaskCompletionSource NewCompletion() => new(TaskCreationOptions.RunContinuationsAsynchronously); + + private sealed class UncopyableContextValue { } + + private static async Task VerifyFollowingTickAsync(TimerFixture fixture, int milliseconds) + { + if (milliseconds > 0) + { + Assert.Single(fixture.Messages); + fixture.Time.Advance(TimeSpan.FromMilliseconds(milliseconds) - TimeSpan.FromTicks(1)); + Assert.Single(fixture.Messages); + fixture.Time.Advance(TimeSpan.FromTicks(1)); + } + if (milliseconds >= 0) + { + Assert.Equal(2, fixture.Messages.Count); + await fixture.InvokeAsync(fixture.Messages[1]); + } + fixture.Time.Advance(TimeSpan.FromDays(1)); + Assert.Equal(milliseconds < 0 ? 1 : 2, fixture.Messages.Count); + } + + private sealed class TimerFixture : IObserver, IDisposable + { + private readonly ServiceProvider _services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + private readonly IDisposable _subscription; + private readonly ILogger _logger = Substitute.For(); + public TrackingTimeProvider Time { get; } = new(); + public IGrainContext Grain { get; } = Substitute.For(); + public List Messages { get; } = []; + public List Events { get; } = []; + public Action? BeforeReceive { get; set; } + public MessageFactory MessageFactory { get; } + public TimerRegistry Registry { get; } + + public TimerFixture() + { + _logger.IsEnabled(LogLevel.Error).Returns(true); + var loggerFactory = Substitute.For(); + loggerFactory.CreateLogger(Arg.Any()).Returns(_logger); + Grain.GrainId.Returns(GrainId.Create("timer-test", "one")); + Grain.When(context => context.ReceiveMessage(Arg.Any())).Do(call => + { + BeforeReceive?.Invoke(); + Messages.Add((Message)call[0]); + }); + var details = Substitute.For(); + details.SiloAddress.Returns(SiloAddress.New(System.Net.IPAddress.Loopback, 11111, 1)); + MessageFactory = new(_services.GetRequiredService(), NullLogger.Instance, null!); + Registry = new(loggerFactory, Time, MessageFactory, details); + _subscription = GrainTimerEvents.AllEvents.Subscribe(this); + } + + public IGrainTimer Register(Func callback, T state, TimeSpan dueTime, + TimeSpan period, bool interleave = false, bool keepAlive = false) => + Registry.RegisterGrainTimer(Grain, callback, state, new(dueTime, period) { Interleave = interleave, KeepAlive = keepAlive }); + + public async Task InvokeAsync(Message message, Exception? expectedException = null) + { + RequestContextExtensions.Import(message.RequestContextData); + var invokable = Assert.IsAssignableFrom(message.BodyObject); + invokable.SetTarget(Grain); + using var response = await invokable.Invoke(); + Assert.Same(expectedException, response.Exception); + } + + public Exception SingleLoggedError() + { + var log = Assert.Single(_logger.ReceivedCalls(), call => + call.GetMethodInfo().Name == nameof(ILogger.Log) && Equals(call.GetArguments()[0], LogLevel.Error)); + return Assert.IsAssignableFrom(log.GetArguments()[3]); + } + + public void OnNext(GrainTimerEvents.TimerEvent value) + { + if (ReferenceEquals(value.GrainContext, Grain)) + { + Events.Add(value); + } + } + + public void OnCompleted() { } + public void OnError(Exception error) => throw error; + public void Dispose() + { + _subscription.Dispose(); + _services.Dispose(); + } + } + + private sealed class TrackingTimeProvider : FakeTimeProvider + { + private TimerCallback? _callback; + private object? _state; + public int TimerCreations { get; private set; } + public void FireDispatchedCallback() => _callback!(_state); + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + TimerCreations++; + _callback = callback; + _state = state; + return base.CreateTimer(callback, state, dueTime, period); + } + } +} diff --git a/test/Orleans.DefaultCluster.Tests/TimerOrleansTest.cs b/test/Orleans.DefaultCluster.Tests/TimerOrleansTest.cs index d11e5b7e5d2..ed3b4daba3d 100644 --- a/test/Orleans.DefaultCluster.Tests/TimerOrleansTest.cs +++ b/test/Orleans.DefaultCluster.Tests/TimerOrleansTest.cs @@ -106,8 +106,10 @@ private async Task DriveExternalTickUntilTimerTicks(INonReentrantTimerCallG { using var callbackObserver = TimerCallbackDiagnosticObserver.Create(); var grainId = grain.GetGrainId(); + var waitForDelay = callbackObserver.WaitForDelayScheduledCountAsync( + grainId, callbackObserver.GetDelayScheduledCount(grainId) + 1, TimerDiagnosticTimeout); var externalTick = grain.ExternalTick("external"); - await callbackObserver.WaitForDelayScheduledCountAsync(grainId, callbackObserver.GetDelayScheduledCount(grainId) + 1, TimerDiagnosticTimeout); + await waitForDelay; await fixture.AdvanceTimeAsync(TimerCallbackDelay); await externalTick; await AdvanceTimerToTickCountAsync(grain, timerObserver, dueTime, TimerCallbackDelay, expectedTimerTicks); diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index 7a9862c5cab..75432b23712 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -727,6 +727,8 @@ public IJournaledStateManager CreateStandalone(JournalId journalId) private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager { + public IList Hooks => inner.Hooks; + public int DisposeCalls { get; private set; } public long PendingWriteByteCount => inner.PendingWriteByteCount; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 26cf16aac46..4e46d24cde7 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -193,6 +193,8 @@ public void Collections_UseDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { + public IList Hooks { get; } = []; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index e1cc3708672..bb3648d69fa 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -92,6 +92,8 @@ public void Add_UsesDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { + public IList Hooks { get; } = []; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index e9822dd9557..97b65742677 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -746,6 +746,7 @@ public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { + public IList Hooks => inner.Hooks; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); public void RegisterStateMachine(string name, IStateMachine state) => inner.RegisterStateMachine(name, state); public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs new file mode 100644 index 00000000000..6b25f0cf9fe --- /dev/null +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -0,0 +1,864 @@ +using System.Buffers; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Xunit; + +namespace Orleans.Journaling.Tests; + +public partial class StateManagerTests +{ + [Fact] + public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() + { + await using var manager = CreateTestSystem().Manager; + var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; + Assert.Null(field.GetValue(manager)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Null(field.GetValue(manager)); + + var hook = new DelegateTestHook(); + var hooks = manager.Hooks; + hooks.Add(hook); + Assert.True(hooks.Contains(hook)); + Assert.Same(hooks, manager.Hooks); + Assert.Same(hook, Assert.Single(hooks)); + Assert.True(hooks.Remove(hook)); + Assert.Empty(manager.Hooks); + Assert.Throws(() => hooks.Add(null!)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot, BlockNextAppend = !snapshot, BlockNextReplace = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; + CancellationToken ownedToken = default; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (operation, token) => + { + Assert.Equal(expected, operation); + Assert.True(token.CanBeCanceled); + ownedToken = token; + events.Add("before-sync"); + state["hook"] = 2; + }, + BeforeOperationAsync = async (operation, token) => + { + Assert.Equal(expected, operation); + Assert.Equal(ownedToken, token); + await Task.Yield(); + events.Add("before-async"); + }, + AfterOperation = (operation, token) => + { + Assert.Equal(expected, operation); + Assert.Equal(ownedToken, token); + Assert.Single(snapshot ? storage.Replaces : storage.Appends); + events.Add("after-sync"); + }, + AfterOperationAsync = async (_, token) => + { + Assert.Equal(ownedToken, token); + await Task.Yield(); + events.Add("after-async"); + } + }); + state["business"] = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await (snapshot ? storage.ReplaceEntered : storage.BlockedAppendStarted).Task + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "before-sync", "before-async", "capture" }, events); + Assert.False(write.IsCompleted); + (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "before-sync", "before-async", "capture", "ack", "after-sync", "after-async" }, events); + } + finally + { + (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); + } + + await using var replayManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["hook"]); + } + + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledStateOperation operation) + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var capture = new AlwaysWritingState(); + manager.RegisterStateMachine("capture", capture); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var pendingBytes = manager.PendingWriteByteCount; + var failure = new IOException("Prerequisite failed."); + var after = 0; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (_, _) => throw failure, + AfterOperation = (_, _) => after++ + }); + var caught = await Assert.ThrowsAsync(() => InvokeAsync()); + Assert.Same(failure, caught.InnerException); + Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); + Assert.Equal(1, state["business"]); + Assert.Equal(pendingBytes, manager.PendingWriteByteCount); + Assert.Equal(0, capture.AppendEntriesCount); + Assert.Equal(0, capture.WriteCompletedCount); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, after); + manager.Hooks.Clear(); + await InvokeAsync(); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); + Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); + + Task InvokeAsync() => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) + : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask(); + } + + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHooks(JournaledStateOperation operation) + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var first = new IOException("First cleanup failed."); + var second = new IOException("Second cleanup failed."); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw first }); + manager.Hooks.Add(new DelegateTestHook + { + AfterOperationAsync = async (_, _) => + { + await Task.Yield(); + events.Add("second"); + throw second; + } + }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) + : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); + Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); + Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); + Assert.Equal(operation == JournaledStateOperation.Delete + ? new[] { "second", "last" } + : new[] { "capture", "ack", "second", "last" }, events); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + manager.Hooks.Clear(); + + await using var replayManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, recovered.Count); + if (operation != JournaledStateOperation.Delete) + { + Assert.Equal(1, recovered["business"]); + } + + state["later"] = 2; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, state["later"]); + } + + [Fact] + public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() + { + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage { NextAppendException = failure }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var after = 0; + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => after++ }); + state["business"] = 1; + Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); + Assert.Equal(0, after); + var fenced = await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(failure, fenced.InnerException); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinaryHook(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + manager.Hooks.Add(new CaptureTestHook( + (_, _) => + { + Assert.Equal(3, state["late"]); + state["owner"] = 2; + events.Add("capture-prerequisite"); + return default; + }, + (_, _) => events.Add("capture-after"))); + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperationAsync = async (_, token) => + { + events.Add("ordinary-before"); + entered.TrySetResult(); + await release.Task.WaitAsync(token); + events.Add("ordinary-complete"); + }, + AfterOperation = (_, _) => events.Add("ordinary-after") + }); + state["business"] = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "ordinary-before" }, events); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + state["late"] = 3; + release.TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "ordinary-before", "ordinary-complete", "capture-prerequisite", "capture", "ack", "capture-after", "ordinary-after" }, events); + } + finally + { + release.TrySetResult(); + } + + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["owner"]); + Assert.Equal(3, recovered["late"]); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(bool captureHook) + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken ownedToken = default; + Func before = async (_, token) => + { + ownedToken = token; + entered.TrySetResult(); + await release.Task.WaitAsync(token); + state["scheduled"] = 2; + }; + Action after = (_, token) => + { + Assert.Equal(ownedToken, token); + Assert.False(token.IsCancellationRequested); + Assert.Single(storage.Appends); + completed.TrySetResult(); + }; + manager.Hooks.Add(captureHook + ? new CaptureTestHook(before, after) + : new DelegateTestHook { BeforeOperationAsync = before, AfterOperation = after }); + using var caller = new CancellationTokenSource(); + state["business"] = 1; + var write = manager.WriteStateAsync(caller.Token).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.NotEqual(caller.Token, ownedToken); + state["late"] = 3; + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => write); + Assert.False(ownedToken.IsCancellationRequested); + Assert.Empty(storage.Appends); + Assert.False(completed.Task.IsCompleted); + release.TrySetResult(); + await completed.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(2, state["scheduled"]); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["scheduled"]); + Assert.Equal(3, recovered["late"]); + } + finally + { + release.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_RegistrationEnforcesSingleCapturePrerequisite() + { + await using var manager = CreateTestSystem().Manager; + var first = new CaptureTestHook((_, _) => default); + var second = new CaptureTestHook((_, _) => default); + var ordinary = new DelegateTestHook(); + manager.Hooks.Add(first); + manager.Hooks.Add(ordinary); + Assert.Throws(() => manager.Hooks.Add(second)); + Assert.Throws(() => manager.Hooks[1] = second); + Assert.Equal(new IJournaledStateHook[] { first, ordinary }, manager.Hooks); + manager.Hooks[0] = second; + Assert.Same(second, manager.Hooks[0]); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var after = 0; + manager.Hooks.Add(new CaptureTestHook( + async (operation, token) => + { + Assert.Equal(JournaledStateOperation.Delete, operation); + Assert.Equal(1, state.Value); + entered.TrySetResult(); + await release.Task.WaitAsync(token); + }, + (operation, _) => + { + Assert.Equal(JournaledStateOperation.Delete, operation); + Assert.Equal(0, state.Value); + Assert.Equal(1, storage.DeleteCount); + after++; + })); + var deletion = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, after); + Assert.False(deletion.IsCompleted); + release.TrySetResult(); + await deletion.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, after); + Assert.Single(manager.Hooks); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_MutationIsRejectedWhilePersistenceIsQueued(JournaledStateOperation operation) + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + var before = 0; + var after = 0; + var hook = new CaptureTestHook( + (actual, _) => + { + Assert.Equal(operation, actual); + before++; + return default; + }, + (actual, _) => + { + Assert.Equal(operation, actual); + after++; + }); + manager.Hooks.Add(hook); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + using var caller = new CancellationTokenSource(); + var canceledWaiter = InvokeAsync(caller.Token); + var remainingWaiter = InvokeAsync(TestContext.Current.CancellationToken); + caller.Cancel(); + + Assert.Equal(0, state.CaptureCount); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, before); + Assert.False(remainingWaiter.IsCompleted); + Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); + Assert.Throws(() => manager.Hooks.Insert(0, new DelegateTestHook())); + Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); + Assert.Throws(() => manager.Hooks.Remove(hook)); + Assert.Throws(() => manager.Hooks.Clear()); + Assert.Same(hook, Assert.Single(manager.Hooks)); + + await Assert.ThrowsAnyAsync(() => canceledWaiter); + await remainingWaiter; + Assert.Equal(1, before); + Assert.Equal(1, after); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.CaptureCount); + Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); + Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + manager.Hooks.Clear(); + Assert.Empty(manager.Hooks); + + Task InvokeAsync(CancellationToken token) => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(token) + : manager.WriteStateAsync(token)).AsTask(); + }); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_MutationIsRejectedThroughoutOwnedOperation(bool after) + { + var storage = new CapturingStorage { BlockNextAppend = !after }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hook = new DelegateTestHook + { + BeforeOperation = (_, _) => Assert.Throws(() => manager.Hooks.Clear()), + AfterOperationAsync = after ? async (_, _) => + { + entered.TrySetResult(); + await release.Task; + } + : null + }; + manager.Hooks.Add(hook); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await (after ? entered : storage.BlockedAppendStarted).Task + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); + Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); + Assert.Throws(() => manager.Hooks.Remove(hook)); + Assert.Throws(() => manager.Hooks.Clear()); + Assert.Same(hook, Assert.Single(manager.Hooks)); + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + manager.Hooks.Clear(); + Assert.Empty(manager.Hooks); + } + finally + { + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var drained = false; + var failure = new IOException("Cancellation callback failed."); + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperationAsync = async (_, token) => + { + using var registration = token.Register(() => throw failure); + entered.TrySetResult(); + await release.Task; + drained = true; + } + }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + var concurrentShutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, concurrentShutdown); + Assert.False(shutdown.IsCompleted); + Assert.False(drained); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => + shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Same(failure, Assert.Single(caught.InnerExceptions)); + await Assert.ThrowsAnyAsync(() => write); + await Assert.ThrowsAnyAsync(() => queued); + Assert.True(drained); + Assert.Empty(storage.Appends); + Assert.Throws(() => manager.Hooks.Clear()); + } + finally + { + release.TrySetResult(); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDrains(bool after) + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = NewSignal(); + var release = NewSignal(); + Func callback = async (_, _) => + { + entered.TrySetResult(); + await release.Task; + }; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperationAsync = after ? null : callback, + AfterOperationAsync = after ? callback : null + }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var initializing = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var deleting = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.False(shutdown.IsCompleted); + Assert.False(initializing.IsCompleted); + Assert.False(deleting.IsCompleted); + release.TrySetResult(); + await WaitFor(shutdown); + if (after) + { + await WaitFor(write); + } + else + { + await Assert.ThrowsAnyAsync(() => WaitFor(write)); + } + + await Assert.ThrowsAnyAsync(() => WaitFor(initializing)); + await Assert.ThrowsAnyAsync(() => WaitFor(deleting)); + Assert.True(initializing.IsCanceled); + Assert.True(deleting.IsCanceled); + Assert.Equal(after ? 1 : 0, state.CaptureCount); + Assert.Equal(after ? 1 : 0, state.WriteCompletedCount); + Assert.Equal(after ? 1 : 0, storage.Appends.Count); + Assert.Equal(0, storage.DeleteCount); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesItsOutcome(bool snapshot, bool storageFails) + { + var entered = NewSignal(); + var release = NewSignal(); + var cancellationFailure = new IOException("Cancellation callback failed."); + var storageFailure = new IOException("Actual storage failure."); + var completed = false; + var storage = Substitute.For(); + storage.IsCompactionRequested.Returns(snapshot); + storage.ReadAsync(Arg.Any(), Arg.Any()).Returns(call => + { + call.Arg().Complete(metadata: null); + return ValueTask.CompletedTask; + }); + storage.AppendAsync(Arg.Any>(), Arg.Any()) + .Returns(call => WriteAsync(call.Arg>(), call.Arg())); + storage.ReplaceAsync(Arg.Any>(), Arg.Any()) + .Returns(call => WriteAsync(call.Arg>(), call.Arg())); + var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + CancellationTokenRegistration registration = default; + var after = 0; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), + AfterOperation = (_, _) => after++ + }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, manager.DisposeAsync().AsTask()); + Assert.False(shutdown.IsCompleted); + Assert.False(completed); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Equal(0, after); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => WaitFor(shutdown)); + Assert.Same(cancellationFailure, Assert.Single(caught.InnerExceptions)); + Assert.True(completed); + if (storageFails) + { + Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(write))); + Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(queued))); + } + else + { + await WaitFor(write); + await Assert.ThrowsAnyAsync(() => WaitFor(queued)); + } + + Assert.Equal(storageFails ? 0 : 1, state.WriteCompletedCount); + Assert.Equal(storageFails ? 0 : 1, after); + } + finally + { + release.TrySetResult(); + registration.Dispose(); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + } + + async ValueTask WriteAsync(ReadOnlySequence bytes, CancellationToken token) + { + var original = bytes.ToArray(); + entered.TrySetResult(); + await release.Task; + Assert.True(token.IsCancellationRequested); + Assert.Equal(original, bytes.ToArray()); + completed = true; + if (storageFails) + { + throw storageFailure; + } + } + } + + [Fact] + public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var first = new IOException("First cleanup failed."); + var second = new IOException("Second cleanup failed."); + List events = []; + manager.Hooks.Add(new DelegateTestHook + { + AfterOperationAsync = async (_, token) => + { + Assert.Single(storage.Appends); + entered.TrySetResult(); + await release.Task; + Assert.True(token.IsCancellationRequested); + events.Add("first"); + throw first; + } + }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw second }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var shutdown = manager.DisposeAsync().AsTask(); + var concurrentShutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, concurrentShutdown); + Assert.False(shutdown.IsCompleted); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => + write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "first", "last" }, events); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Fact] + public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var before = 0; + var after = 0; + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + state["first"] = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await storage.BlockedAppendStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(0, after); + state["second"] = 2; + var second = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + state["third"] = 3; + var third = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(2, before); + Assert.Equal(2, after); + Assert.Equal(2, storage.Appends.Count); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + var before = 0; + var after = 0; + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + Assert.Single(storage.Appends); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Hooks_FailureOutcomesRoundTripWithTypeAndOriginalCause(bool after) + { + Exception original = after + ? new JournaledStatePostCommitException(JournaledStateOperation.Delete, new IOException("Cleanup failed.")) + : new JournaledStatePreCommitException(JournaledStateOperation.Snapshot, new IOException("Scheduling failed.")); + var serializer = ServiceProvider.GetRequiredService(); + var copy = serializer.Deserialize(serializer.SerializeToArray(original)); + if (after) + { + Assert.IsType(copy); + } + else + { + Assert.IsType(copy); + } + + Assert.Equal(original.InnerException!.Message, Assert.IsType(copy.InnerException).Message); + Assert.Equal(original.Message, copy.Message); + } + + private sealed class CaptureTestHook( + Func before, + Action? after = null) : IJournaledStateCaptureHook + { + public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => + before(operation, cancellationToken); + + public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + after?.Invoke(operation, cancellationToken); + return default; + } + } + + private sealed class DelegateTestHook : IJournaledStateHook + { + public Action? BeforeOperation { get; init; } + public Func? BeforeOperationAsync { get; init; } + public Action? AfterOperation { get; init; } + public Func? AfterOperationAsync { get; init; } + + ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + BeforeOperation?.Invoke(operation, cancellationToken); + return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + + ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + AfterOperation?.Invoke(operation, cancellationToken); + return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + } + + private sealed class HookAcknowledgementState(List events) : IStateMachine + { + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) { } + public void Reset(JournalStreamWriter writer) { } + public void WritePendingEntries(JournalStreamWriter writer) => events.Add("capture"); + public void WriteSnapshot(JournalStreamWriter writer) => events.Add("capture"); + public void OnWriteCompleted() => events.Add("ack"); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs b/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs new file mode 100644 index 00000000000..a86d99f456b --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs @@ -0,0 +1,148 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using NSubstitute; +using Orleans.Concurrency; +using Orleans.Messaging.Tests.Support; +using Orleans.Messaging.Tests.Functional; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Placement; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ActivationValidationTests +{ + private static readonly Action ValidateActivation = ReceiverTestServices + .GetImplementationType("MessagingActivationValidator") + .GetMethod("Validate")! + .CreateDelegate>(); + + [Fact] + public void ExternalConsumerAssembly_HasNoFriendAccessToMessaging() + { + var sourceAssembly = typeof(IInbox).Assembly; + var consumerName = typeof(ActivationValidationTests).Assembly.GetName().Name; + var friendDeclarations = sourceAssembly + .GetCustomAttributesData() + .Where(attribute => attribute.AttributeType.FullName == "System.Runtime.CompilerServices.InternalsVisibleToAttribute") + .Select(attribute => attribute.ConstructorArguments[0].Value?.ToString()) + .ToArray(); + + Assert.DoesNotContain(friendDeclarations, declaration => + declaration?.StartsWith(consumerName!, StringComparison.Ordinal) == true); + } + + [Theory] + [InlineData("reentrant", "non-reentrant")] + [InlineData("stateless", "one activation")] + [InlineData("interleavable-method", "interleavable method")] + [InlineData("may-interleave", "non-reentrant")] + public void UnsupportedExecutionModel_FailsWithSpecificDiagnostic(string model, string expected) + { + object grain = model switch + { + "reentrant" => new ReentrantGrain(), + "stateless" => new StatelessGrain(), + "interleavable-method" => new InterleavableGrain(), + "may-interleave" => new MayInterleaveGrain(), + _ => throw new ArgumentOutOfRangeException(nameof(model)) + }; + var context = Substitute.For(); + context.GrainInstance.Returns(grain); + + var values = new Dictionary(); + new AttributeGrainPropertiesProvider(Substitute.For()) + .Populate(grain.GetType(), GrainType.Create("activation-validation"), values); + var properties = new GrainProperties(values.ToImmutableDictionary(StringComparer.Ordinal)); + var placement = model == "stateless" + ? new StatelessWorkerAttribute().PlacementStrategy + : new RandomPlacementAttribute().PlacementStrategy; + var exception = Assert.Throws(() => ValidateActivation(context, properties, placement)); + + Assert.Contains(expected, exception.Message, StringComparison.Ordinal); + Assert.Contains(grain.GetType().ToString(), exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void MissingGrainInstance_FailsBeforeExecutionModelValidation() + { + var context = Substitute.For(); + + var properties = new GrainProperties(ImmutableDictionary.Empty.WithComparers(StringComparer.Ordinal)); + var exception = Assert.Throws(() => + ValidateActivation(context, properties, new RandomPlacementAttribute().PlacementStrategy)); + + Assert.Contains("initialized grain instance", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task StandardStateConstruction_MissingSelectedCodecFailsBeforeEnrollment() + { + const string selectedFormat = "orleans-binary"; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(selectedFormat); + builder.Services.RemoveAllKeyed(typeof(IDurableValueCommandCodec<>), selectedFormat); + var id = new JournalId("missing-codec/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var exception = Assert.Throws(() => + scope.ServiceProvider.GetRequiredKeyedService>("state")); + Assert.Contains(selectedFormat, exception.Message, StringComparison.Ordinal); + Assert.False(owner.TryGetStateMachine("state", out _)); + } + + [Fact] + public async Task CodecResolution_PreservesUnrelatedFailure() + { + var cause = new InvalidOperationException("codec resolution failure"); + const string selectedFormat = "orleans-binary"; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(selectedFormat); + builder.Services.AddKeyedSingleton>(selectedFormat, (_, _) => throw cause); + var id = new JournalId("codec-resolution/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + + var exception = Assert.Throws(() => + scope.ServiceProvider.GetRequiredKeyedService>("state")); + + Assert.Same(cause, exception); + Assert.False(owner.TryGetStateMachine("state", out _)); + } + + [Reentrant] + private sealed class ReentrantGrain; + + [StatelessWorker] + private sealed class StatelessGrain; + + [MayInterleave(nameof(Interleave))] + private sealed class MayInterleaveGrain + { + public static bool Interleave(IInvokable request) => true; + } + + public interface IInterleavableBase + { + [AlwaysInterleave] + Task PingAsync(); + } + + public interface IInterleavableGrain : IGrain, IInterleavableBase; + + private sealed class InterleavableGrain : IInterleavableGrain + { + public Task PingAsync() => Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs new file mode 100644 index 00000000000..4ec4d1b583c --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -0,0 +1,306 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Functional; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Timers; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DeliveryAndOptionsContractTests +{ + [Fact] + public void RetentionTimeArithmetic_HandlesMaximumDurationsWithoutOverflow() + { + var timeType = typeof(IInbox).Assembly.GetType( + "Orleans.Messaging.MessagingTime", + throwOnError: true)!; + var isExpired = timeType.GetMethod( + "IsExpired", + BindingFlags.Static | BindingFlags.Public)!; + var addClamped = timeType.GetMethod( + "AddClamped", + BindingFlags.Static | BindingFlags.Public)!; + var timestamp = DateTimeOffset.MaxValue - TimeSpan.FromTicks(1); + + Assert.False((bool)isExpired.Invoke( + null, + [DateTimeOffset.MaxValue, timestamp, TimeSpan.MaxValue])!); + var fullDateTimeRange = TimeSpan.FromTicks( + DateTimeOffset.MaxValue.UtcTicks - DateTimeOffset.MinValue.UtcTicks); + Assert.True((bool)isExpired.Invoke( + null, + [DateTimeOffset.MaxValue, DateTimeOffset.MinValue, fullDateTimeRange])!); + Assert.False((bool)isExpired.Invoke( + null, + [DateTimeOffset.MaxValue, DateTimeOffset.MinValue, TimeSpan.MaxValue])!); + Assert.False((bool)isExpired.Invoke( + null, + [DateTimeOffset.MinValue, DateTimeOffset.MaxValue, TimeSpan.FromTicks(1)])!); + Assert.Equal( + DateTimeOffset.MaxValue, + (DateTimeOffset)addClamped.Invoke(null, [timestamp, TimeSpan.MaxValue])!); + } + + [Fact] + public void DeadLetterCompaction_SaturatesMaximumRetentionWithoutOverflow() + { + var retentionType = typeof(IInbox).Assembly.GetType( + "Orleans.Messaging.DeadLetterRetention", + throwOnError: true)!; + var compact = retentionType.GetMethod( + "Compact", + BindingFlags.Static | BindingFlags.Public)! + .MakeGenericMethod(typeof(string), typeof(DateTimeOffset)); + var entries = new Dictionary + { + ["oldest"] = DateTimeOffset.MinValue, + ["newest"] = DateTimeOffset.MaxValue + }; + + var removed = (bool)compact.Invoke( + null, + [ + entries, + DateTimeOffset.MaxValue, + TimeSpan.MaxValue, + int.MaxValue, + (Func)(static timestamp => timestamp), + 0 + ])!; + + Assert.False(removed); + Assert.Equal(2, entries.Count); + + var fullDateTimeRange = TimeSpan.FromTicks( + DateTimeOffset.MaxValue.UtcTicks - DateTimeOffset.MinValue.UtcTicks); + removed = (bool)compact.Invoke( + null, + [ + entries, + DateTimeOffset.MaxValue, + fullDateTimeRange, + int.MaxValue, + (Func)(static timestamp => timestamp), + 0 + ])!; + + Assert.True(removed); + Assert.DoesNotContain("oldest", entries); + Assert.Contains("newest", entries); + } + + [Fact] + public void DeliveryResult_EachFactory_PreservesStatusAndPayload() + { + var handlerMissing = DeliveryResult.HandlerNotFound(); + var deadLettered = DeliveryResult.DeadLettered("poison body"); + + Assert.Equal(DeliveryStatus.Accepted, DeliveryResult.Accepted().Status); + Assert.Equal(DeliveryStatus.Duplicate, DeliveryResult.Duplicate().Status); + Assert.Equal(DeliveryStatus.Backpressured, DeliveryResult.Backpressured().Status); + Assert.Equal(DeliveryStatus.HandlerNotFound, handlerMissing.Status); + Assert.Equal("No inbox handler is registered.", handlerMissing.Message); + Assert.Equal(DeliveryStatus.DeadLettered, deadLettered.Status); + Assert.Equal("poison body", deadLettered.Message); + } + + [Fact] + public void DeliveryStatus_AllValues_HaveContiguousDistinctValues() + { + Assert.Equal( + [ + DeliveryStatus.Accepted, + DeliveryStatus.Duplicate, + DeliveryStatus.Backpressured, + DeliveryStatus.HandlerNotFound, + DeliveryStatus.DeadLettered + ], + Enum.GetValues()); + Assert.Equal([0, 1, 2, 3, 4], Enum.GetValues().Select(static value => (int)value)); + } + + [Fact] + public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() + { + var options = new InboxOptions(); + + options.Validate(); + + Assert.Equal(1000, options.MaxCapacity); + Assert.Equal(TimeSpan.FromDays(7), options.DeduplicationWindow); + Assert.Equal(TimeSpan.FromDays(1), options.MaxOutboxRetryAge); + Assert.Equal(TimeSpan.FromDays(30), options.DeadLetterRetentionPeriod); + Assert.Equal(5, options.MaxProcessingAttempts); + Assert.Equal(100, options.MaxDeliveryAttempts); + Assert.Equal(1000, options.MaxRetainedDeadLetters); + Assert.Equal(32, options.InboxBatchSize); + Assert.Equal(32, options.OutboxBatchSize); + Assert.Equal(TimeSpan.FromMilliseconds(100), options.OutboxIdleRetirementGracePeriod); + } + + [Fact] + public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContract() + { + var invalidCases = new (string Parameter, Action Mutate)[] + { + (nameof(InboxOptions.MaxCapacity), options => options.MaxCapacity = 0), + (nameof(InboxOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.Zero), + (nameof(InboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.Zero), + (nameof(InboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.MaxValue), + (nameof(InboxOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), + (nameof(InboxOptions.MaxDeliveryAttempts), options => options.MaxDeliveryAttempts = 0), + (nameof(InboxOptions.MaxOutboxRetryAge), options => options.MaxOutboxRetryAge = TimeSpan.Zero), + (nameof(InboxOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), + (nameof(InboxOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), + (nameof(InboxOptions.InboxBatchSize), options => options.InboxBatchSize = 0), + (nameof(InboxOptions.OutboxBatchSize), options => options.OutboxBatchSize = 0), + (nameof(InboxOptions.OutboxIdleRetirementGracePeriod), options => options.OutboxIdleRetirementGracePeriod = TimeSpan.FromTicks(-1)), + (nameof(InboxOptions.OutboxIdleRetirementGracePeriod), options => options.OutboxIdleRetirementGracePeriod = TimeSpan.MaxValue), + }; + + foreach (var (parameter, mutate) in invalidCases) + { + var options = new InboxOptions(); + mutate(options); + var exception = Assert.Throws(options.Validate); + Assert.Equal(parameter, exception.ParamName); + } + } + + [Theory] + [InlineData(0)] + [InlineData(100)] + [InlineData(4294967294)] + public void Validate_OutboxIdleGrace_SupportsImmediateAndMaximumTimerBoundary(long milliseconds) + { + new InboxOptions { OutboxIdleRetirementGracePeriod = TimeSpan.FromMilliseconds(milliseconds) }.Validate(); + } + + [Fact] + public void Validate_MaxOutboxRetryAgeNotLessThanDeduplicationWindow_FailsAtBoundaryAndAbove() + { + foreach (var retryAge in new[] { TimeSpan.FromHours(2), TimeSpan.FromHours(3) }) + { + var options = new InboxOptions + { + DeduplicationWindow = TimeSpan.FromHours(2), + MaxOutboxRetryAge = retryAge, + }; + + var exception = Assert.Throws(options.Validate); + Assert.Equal(nameof(InboxOptions.MaxOutboxRetryAge), exception.ParamName); + Assert.Contains("less than DeduplicationWindow", exception.Message, StringComparison.Ordinal); + } + } + + [Fact] + public void InboxDispose_CancelsWorkOnceAndSupportsRepeatedDisposal() + { + var assembly = typeof(IInbox).Assembly; + var extensionType = assembly.GetType("Orleans.Messaging.InboxExtension", throwOnError: true)!; + var coordinatorType = assembly.GetType("Orleans.Messaging.MessagingPumpCoordinator", throwOnError: true)!; + var extension = (IDisposable)RuntimeHelpers.GetUninitializedObject(extensionType); + extensionType.GetField("_provisionalAcceptances", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(extension, new HashSet()); + var coordinator = Activator.CreateInstance(coordinatorType)!; + var results = Activator.CreateInstance(assembly.GetType("Orleans.Messaging.MessagingPumpResults", throwOnError: true)!, nonPublic: true)!; + extensionType.GetField("_pumpResults", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(extension, results); + using var shutdown = new CancellationTokenSource(); + var token = shutdown.Token; + var cancellationCount = 0; + using var registration = token.Register(() => cancellationCount++); + extensionType.GetField("_shutdownCts", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(extension, shutdown); + extensionType.GetField("_pumpCoordinator", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(extension, coordinator); + object?[] acquireArguments = ["owner", token, null]; + Assert.True((bool)coordinatorType.GetMethod("TryAcquire")!.Invoke(coordinator, acquireArguments)!); + + extension.Dispose(); + + Assert.True(token.IsCancellationRequested); + Assert.Equal(1, cancellationCount); + Assert.False((bool)coordinatorType.GetMethod("IsCurrent")!.Invoke(coordinator, [acquireArguments[2]])!); + Assert.Throws(() => shutdown.Token); + + extension.Dispose(); + + Assert.Equal(1, cancellationCount); + } + + [Fact] + public async Task InboxLifecycleStart_ObservesPreCanceledLifecycleToken() + { + var extensionType = typeof(IInbox).Assembly.GetType( + "Orleans.Messaging.InboxExtension", + throwOnError: true)!; + var extension = (ILifecycleObserver)RuntimeHelpers.GetUninitializedObject(extensionType); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + () => extension.OnStart(cancellation.Token)); + } + + [Fact] + public async Task InboxLifecycleStart_CancellationInterruptsBlockedResume() + { + var builder = InboxStateManagerBoundaryTests.CreateBuilder("orleans-binary"); + var id = new JournalId("lifecycle-start/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + builder.Services.AddScoped(sp => + { + var context = Substitute.For(); + context.GrainId.Returns(GrainId.Create("lifecycle-start", "blocked")); + context.GrainInstance.Returns(Substitute.For()); + context.ActivationServices.Returns(sp); + context.ObservableLifecycle.Returns(Substitute.For()); + return context; + }); + var jobs = Substitute.For(); + var timers = Substitute.For(); + builder.Services.AddSingleton(jobs); + builder.Services.AddSingleton(timers); + builder.Services.AddSingleton(Substitute.For()); + var instrumentsType = ReceiverTestServices.GetImplementationType("MessagingInstruments"); + builder.Services.AddSingleton(instrumentsType, + instrumentsType.GetMethod("CreateForDirectConstruction", BindingFlags.NonPublic | BindingFlags.Static)!.Invoke(null, null)!); + ReceiverTestServices.Add(builder.Services, static _ => { }); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var extensionType = ReceiverTestServices.GetImplementationType("InboxExtension"); + var extension = (ILifecycleObserver)scope.ServiceProvider.GetRequiredService(extensionType); + var owner = scope.ServiceProvider.GetRequiredService(); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + var gate = (SemaphoreSlim)extensionType.GetField("_gate", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + await gate.WaitAsync(TestContext.Current.CancellationToken); + try + { + using var cancellation = new CancellationTokenSource(); + var start = extension.OnStart(cancellation.Token); + Assert.False(start.IsCompleted); + cancellation.Cancel(); + + var canceled = await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(cancellation.Token, canceled.CancellationToken); + Assert.Equal(0, gate.CurrentCount); + Assert.Empty(jobs.ReceivedCalls()); + Assert.Empty(timers.ReceivedCalls()); + Assert.Empty(scope.ServiceProvider.GetRequiredKeyedService>( + "__orleans.messaging.inbox")); + } + finally + { + gate.Release(); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs new file mode 100644 index 00000000000..6abda98b3f0 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -0,0 +1,474 @@ +using System.Buffers; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class EnvelopeContractTests +{ + private static HierarchicalKey Id => HierarchicalKey.Create("orders", "123"); + private static GrainId Receiver => GrainId.Create("receiver", "123"); + + [Fact] + public void RequiredPayloadAndOptionalMetadata() + { + using var builder = new EnvelopeBuilder(Id); + Assert.Throws(() => builder.Build()); + builder.AddBytes(MessageHeaders.Payload, []); + var envelope = builder.Build(); + Validate(envelope); + Assert.Equal(Id, envelope.MessageId); + Assert.Empty(envelope.Payload.ToArray()); + Assert.Equal(new byte[] { 1, 1, 1, 0 }, envelope.EncodedHeaders.ToArray()); + Assert.Equal(new[] { MessageHeaders.Payload }, envelope.Keys); + Assert.True(envelope.TryGetBytes(MessageHeaders.Payload, out var empty)); + Assert.Empty(empty.ToArray()); + Assert.False(envelope.TryGetBytes("missing", out var missing)); + Assert.Equal(default, missing); + Assert.False(envelope.TryGetSubject(out var subject)); + Assert.Null(subject); + Validate(new InboxMessage { Envelope = envelope }); + Assert.Throws(() => Validate(new OutboxMessage { Envelope = envelope, ReceiverId = default })); + Assert.Throws(() => Validate(default(InboxMessage))); + } + + [Fact] + public void RawTypedAndNullHeadersUseIndependentSessions() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var strings = services.GetRequiredService>(); + var integers = services.GetRequiredService>(); + var grains = services.GetRequiredService>(); + using var builder = new EnvelopeBuilder(Id); + var value = new string('v', 25); + builder.AddBytes("raw", [7, 0, 255]); + builder.AddValue(MessageHeaders.Payload, 42, integers); + builder.AddValue("first", value, strings); + builder.AddValue("second", value, strings); + builder.AddValue("null", null, strings); + builder.AddBytes("empty", []); + builder.AddBytes(MessageHeaders.Subject, "orders.v1"u8); + var sender = GrainId.Create("sender", "123"); + builder.AddValue(MessageHeaders.Sender, sender, grains); + var envelope = builder.Build(); + Validate(envelope); + Assert.True(envelope.TryGetValue(MessageHeaders.Payload, integers, out var payload)); + Assert.Equal(42, payload); + Assert.True(envelope.TryGetValue(MessageHeaders.Sender, grains, out var actualSender)); + Assert.Equal(sender, actualSender); + Assert.True(envelope.TryGetValue("second", strings, out var second)); + Assert.Equal(value, second); + Assert.True(envelope.TryGetValue("first", strings, out var first)); + Assert.Equal(value, first); + Assert.True(envelope.TryGetValue("null", strings, out var nullValue)); + Assert.Null(nullValue); + Assert.True(envelope.TryGetBytes("raw", out var raw)); + Assert.Equal(new byte[] { 7, 0, 255 }, raw.ToArray()); + Assert.True(envelope.TryGetBytes("empty", out var empty)); + Assert.Empty(empty.ToArray()); + Assert.True(envelope.TryGetSubject(out var subject)); + Assert.Equal("orders.v1", subject); + Assert.True(envelope.TryGetBytes("second", out var secondBytes)); + Assert.Equal(strings.SerializeToArray(value), secondBytes.ToArray()); + Assert.False(envelope.TryGetValue("missing", strings, out var absent)); + Assert.Null(absent); + Assert.Throws(() => envelope.TryGetValue(MessageHeaders.Subject, strings, out _)); + Assert.Throws(() => builder.AddValue(MessageHeaders.Subject, "typed", strings)); + Assert.Throws(() => builder.AddValue("bad", "value", null!)); + Assert.Throws(() => envelope.TryGetValue("first", (Serializer)null!, out _)); + } + + [Fact] + public void DuplicateHeadersRetainFirstValueAndOrdinalNames() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var strings = services.GetRequiredService>(); + using var builder = new EnvelopeBuilder(Id); + builder.AddBytes(MessageHeaders.Payload, [1]); + builder.AddValue("custom", "first", strings); + Assert.False(builder.TryAddValue("custom", "later", strings)); + Assert.Throws(() => builder.AddBytes("custom", [3])); + Assert.False(builder.TryAddBytes(MessageHeaders.Payload, [2])); + builder.AddValue("CUSTOM", "different", strings); + builder.AddBytes("\u00e9", [4]); + builder.AddBytes("e\u0301", [5]); + var envelope = builder.Build(); + Validate(envelope); + Assert.Equal(new[] { "payload", "custom", "CUSTOM", "\u00e9", "e\u0301" }, envelope.Keys); + Assert.True(envelope.TryGetValue("custom", strings, out var first)); + Assert.Equal("first", first); + Assert.True(envelope.TryGetValue("CUSTOM", strings, out var upper)); + Assert.Equal("different", upper); + Assert.Equal(new byte[] { 1 }, envelope.Payload.ToArray()); + Assert.Throws(() => builder.AddBytes("", [])); + Assert.Throws(() => builder.AddBytes(null!, [])); + Assert.Throws(() => builder.AddBytes("\ud800", [])); + } + + [Fact] + public void FailedValueSerializationPublishesOnlyCommittedEntries() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var pool = services.GetRequiredService(); + var failure = new InvalidOperationException("failed after writing bytes"); + var broken = new Serializer(new FailingCodec(failure), pool); + using var builder = new EnvelopeBuilder(Id); + builder.AddBytes("before", [6]); + Assert.Same(failure, Assert.Throws(() => builder.AddValue("failed", 42, broken))); + builder.AddBytes(MessageHeaders.Payload, [1, 2]); + builder.AddBytes("after", [7]); + var envelope = builder.Build(); + Validate(envelope); + Assert.Equal(new[] { "payload", "before", "after" }, envelope.Keys); + Assert.False(envelope.TryGetBytes("failed", out _)); + Assert.Equal(new byte[] { 1, 2 }, envelope.Payload.ToArray()); + Assert.True(envelope.TryGetBytes("before", out var before)); + Assert.Equal(new byte[] { 6 }, before.ToArray()); + Assert.True(envelope.TryGetBytes("after", out var after)); + Assert.Equal(new byte[] { 7 }, after.ToArray()); + Assert.Equal(12, envelope.EncodedHeaders.Length - Encoding.UTF8.GetByteCount("beforeafter")); + } + + [Fact] + public void BuilderPublicationAndSliceOwnership() + { + byte[] source = [0, 255, 128]; + var builder = new EnvelopeBuilder(Id); + builder.AddBytes("custom", [6, 7]); + builder.AddBytes(MessageHeaders.Payload, source); + var first = builder.Build(); + var second = builder.Build(); + System.Array.Fill(source, (byte)42); + builder.AddBytes("later", [8]); + var third = builder.Build(); + builder.Dispose(); + builder.Dispose(); + Assert.Equal(new byte[] { 0, 255, 128 }, first.Payload.ToArray()); + Assert.NotSame(Array(first.EncodedHeaders), Array(second.EncodedHeaders)); + Assert.False(first.TryGetBytes("later", out _)); + Assert.True(third.TryGetBytes("later", out var later)); + Assert.Equal(new byte[] { 8 }, later.ToArray()); + Assert.True(first.TryGetBytes("custom", out var custom)); + Assert.Same(Array(first.EncodedHeaders), Array(custom)); + Assert.Same(Array(first.EncodedHeaders), Array(first.Payload)); + Assert.Equal(first.EncodedHeaders.Length, Array(first.EncodedHeaders).Length); + Assert.Throws(() => builder.Build()); + Assert.Throws(() => builder.AddBytes("disposed", [])); + Assert.Throws(() => ((IBufferWriter)builder).GetMemory(1)); + } + + [Theory] + [InlineData(0)] [InlineData(1)] [InlineData(127)] [InlineData(128)] + [InlineData(16383)] [InlineData(16384)] [InlineData(100000)] + public void FramesRoundTripAcrossLengthBoundaries(int length) + { + var payload = Enumerable.Range(0, length).Select(i => (byte)i).ToArray(); + using var builder = new EnvelopeBuilder(Id); + builder.AddBytes("unknown", [255, 0, 128]); + builder.AddBytes(MessageHeaders.Payload, payload); + var original = builder.Build(); + var encoded = original.EncodedHeaders.ToArray(); + var decoded = Envelope.FromEncodedHeaders(Id, encoded); + System.Array.Fill(encoded, (byte)0); + Assert.Equal(payload, decoded.Payload.ToArray()); + Assert.Equal(original.EncodedHeaders.ToArray(), decoded.EncodedHeaders.ToArray()); + Assert.True(decoded.TryGetBytes("unknown", out var unknown)); + Assert.Equal(new byte[] { 255, 0, 128 }, unknown.ToArray()); + Validate(decoded); + } + + [Theory] + [InlineData(1)] [InlineData(127)] [InlineData(128)] [InlineData(300)] + public void CountsAndCustomKeyLengthsRoundTrip(int count) + { + using var builder = new EnvelopeBuilder(Id); + builder.AddBytes(MessageHeaders.Payload, []); + for (var i = 0; i < count; i++) builder.AddBytes($"key-{i}", [(byte)i]); + var longKey = new string('x', count); + builder.AddBytes(longKey, [3]); + var envelope = builder.Build(); + Validate(envelope); + var decoded = Envelope.FromEncodedHeaders(Id, envelope.EncodedHeaders.Span); + Assert.Equal(count + 2, decoded.Keys.Count()); + Assert.True(decoded.TryGetBytes($"key-{count - 1}", out var last)); + Assert.Equal(new byte[] { (byte)(count - 1) }, last.ToArray()); + Assert.True(decoded.TryGetBytes(longKey, out var longest)); + Assert.Equal(new byte[] { 3 }, longest.ToArray()); + } + + public static TheoryData MalformedFrames => new() + { + new byte[] { }, new byte[] { 0, 1, 1, 0 }, new byte[] { 1, 0, 1, 0 }, new byte[] { 1, 1, 0 }, new byte[] { 1, 1, 1 }, new byte[] { 1, 1, 1, 1 }, + new byte[] { 1, 1, 1, 0, 7 }, new byte[] { 1, 129, 0, 1, 0 }, new byte[] { 1, 255, 255, 255, 255, 8, 1, 0 }, + new byte[] { 1, 100, 1, 0 }, new byte[] { 1, 1, 5, 255, 255, 255, 255, 7 }, + new byte[] { 1, 2, 3, 0, 0, 0 }, new byte[] { 1, 2, 3, 0, 3, 0 }, + new byte[] { 1, 2, 4, 0, 4, 255, 0 }, new byte[] { 1, 2, 4, 0, 6, 120, 0 }, + new byte[] { 1, 3, 7, 0, 4, 120, 0, 4, 120, 0 }, + new byte[] { 1, 3, 5, 0, 1, 1, 1, 1, 97, 98 }, new byte[] { 1, 3, 5, 0, 2, 0, 2, 0 }, + new byte[] { 1, 2, 10, 0, 10, 112, 97, 121, 108, 111, 97, 100, 0 }, + new byte[] { 1, 2, 3, 0, 1, 0 }, new byte[] { 1, 2, 3, 0, 1, 1, 255 } + }; + + [Theory] + [MemberData(nameof(MalformedFrames))] + public void FramingRejectsMalformedBuffers(byte[] bytes) + { + Assert.Throws(() => Envelope.FromEncodedHeaders(Id, bytes)); + } + + [Fact] + public void ExternalAndRecoveryBoundaryRejectsMalformedGeneratedValuesBeforeMutation() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + object boxed = Build([1]); + typeof(Envelope).GetField("_headers", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(boxed, new byte[] { 1, 0, 1, 0 }); + var malformed = (Envelope)boxed; + var codec = services.GetRequiredService>(); + var recovered = codec.Deserialize(codec.SerializeToArray(malformed)); + Assert.Throws(() => Validate(new InboxMessage { Envelope = recovered })); + Assert.Throws(() => Validate(new OutboxMessage { Envelope = recovered, ReceiverId = Receiver })); + Assert.Throws(() => Validate(default(Envelope))); + } + + [Fact] + public void OrdinaryCopyAndIndependentWireRoundTrip() + { + using var sending = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var receiving = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var builder = new EnvelopeBuilder(Id); + builder.AddBytes(MessageHeaders.Payload, [0, 255, 128]); + builder.AddBytes("unknown", [4, 5]); + var envelope = builder.Build(); + var serializer = sending.GetRequiredService>(); + var wire = serializer.SerializeToArray(envelope); + var decoded = receiving.GetRequiredService>().Deserialize(wire); + var copied = sending.GetRequiredService().Copy(envelope); + foreach (var value in new[] { decoded, copied }) + { + Validate(value); + Assert.Equal(Id, value.MessageId); + Assert.Equal(envelope.EncodedHeaders.ToArray(), value.EncodedHeaders.ToArray()); + Assert.NotSame(Array(envelope.EncodedHeaders), Array(value.EncodedHeaders)); + Assert.True(value.TryGetBytes("unknown", out var unknown)); + Assert.Same(Array(value.EncodedHeaders), Array(unknown)); + Array(value.Payload)[Segment(value.Payload).Offset] = 42; + Assert.Equal(new byte[] { 0, 255, 128 }, envelope.Payload.ToArray()); + } + Assert.Equal(wire, serializer.SerializeToArray(envelope)); + } + + [Theory] + [InlineData("InboxMessage")] [InlineData("OutboxMessage")] + [InlineData("InboxDeadLetter")] [InlineData("OutboxDeadLetter")] + public void MessageAndDeadLetterCopyPreservesUnknownHeadersAndIsolation(string kind) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var envelope = Build([0, 255], "subject.v1"); + object value = kind.StartsWith("Inbox", StringComparison.Ordinal) ? new InboxMessage { Envelope = envelope } + : new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + if (kind.EndsWith("DeadLetter", StringComparison.Ordinal)) + { + var type = typeof(Envelope).Assembly.GetType("Orleans.Messaging." + kind, true)!; + var record = Activator.CreateInstance(type)!; + type.GetProperty("Message")!.SetValue(record, value); + type.GetProperty("DeadLetteredAt")!.SetValue(record, DateTimeOffset.UnixEpoch); + type.GetProperty("Reason")!.SetValue(record, "failure"); + type.GetProperty("AttemptCount")!.SetValue(record, 3); + value = record; + } + var serializer = services.GetRequiredService(); + foreach (var result in new[] { serializer.Deserialize(serializer.SerializeToArray(value)), services.GetRequiredService().Copy(value) }) + { + Assert.NotNull(result); + var message = kind.EndsWith("DeadLetter", StringComparison.Ordinal) ? result.GetType().GetProperty("Message")!.GetValue(result)! : result; + var actual = message is InboxMessage incoming ? incoming.Envelope : Assert.IsType(message).Envelope; + Assert.Equal(envelope.EncodedHeaders.ToArray(), actual.EncodedHeaders.ToArray()); + Assert.NotSame(Array(envelope.EncodedHeaders), Array(actual.EncodedHeaders)); + if (message is OutboxMessage outgoing) Assert.Equal(Receiver, outgoing.ReceiverId); + if (kind.EndsWith("DeadLetter", StringComparison.Ordinal)) Assert.Equal(3, result.GetType().GetProperty("AttemptCount")!.GetValue(result)); + } + } + + [Fact] + public void DistinctMessageSchemasAndLocalConversion() + { + var envelope = Build([4, 5]); + var outgoing = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + var incoming = outgoing.ToInboxMessage(); + Assert.Equal(envelope.EncodedHeaders, incoming.Envelope.EncodedHeaders); + Assert.Same(Array(envelope.EncodedHeaders), Array(incoming.Envelope.EncodedHeaders)); + Assert.Equal(Id, incoming.Envelope.MessageId); + Assert.Equal(16, Unsafe.SizeOf()); + Assert.Equal(Unsafe.SizeOf(), Unsafe.SizeOf()); + Assert.Null(typeof(InboxMessage).GetProperty("ReceiverId")); + Assert.Null(typeof(Envelope).GetProperty("ReceiverId")); + Assert.Null(typeof(Envelope).GetCustomAttribute()); + Assert.Single(typeof(Envelope).GetCustomAttributes()); + AssertIds(typeof(Envelope), ("MessageId", 0u), ("_headers", 1u)); + AssertIds(typeof(InboxMessage), ("Envelope", 0u)); + AssertIds(typeof(OutboxMessage), ("Envelope", 0u), ("ReceiverId", 1u)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var other = outgoing with { ReceiverId = GrainId.Create("receiver", "other") }; + Assert.Equal(serializer.SerializeToArray(incoming), serializer.SerializeToArray(other.ToInboxMessage())); + Assert.True(services.GetRequiredService>().SerializeToArray(outgoing).Length > serializer.SerializeToArray(incoming).Length); + Assert.Equal(typeof(InboxMessage), typeof(IInboxExtension).GetMethod("DeliverAsync")!.GetParameters()[0].ParameterType); + Assert.Equal(typeof(OutboxMessage), typeof(IOutbox).GetMethod("Send")!.GetParameters()[0].ParameterType); + Assert.Equal(typeof(InboxMessage), typeof(IInboxHandlerContext).GetProperty("Message")!.PropertyType); + Assert.Equal(typeof(IEnumerable), typeof(IInbox).GetProperty("Messages")!.PropertyType); + Assert.Equal(typeof(IEnumerable), typeof(IOutbox).GetProperty("Messages")!.PropertyType); + Assert.Equal("IInboxExtension", typeof(IInboxExtension).GetCustomAttribute()!.Alias); + Assert.Equal("DeliverAsync", typeof(IInboxExtension).GetMethod("DeliverAsync")!.GetCustomAttribute()!.Alias); + Assert.DoesNotContain(typeof(IDisposable), typeof(Envelope).GetInterfaces()); + } + + [Theory] + [InlineData("same", true, true)] [InlineData("custom", true, true)] [InlineData("sender", true, true)] + [InlineData("receiver", false, true)] [InlineData("id", false, false)] [InlineData("child", false, false)] + [InlineData("subject", false, false)] [InlineData("subject-case", false, false)] [InlineData("no-subject", false, false)] + [InlineData("payload", false, false)] [InlineData("payload-length", false, false)] + public void PendingCommandEqualityIgnoresCustomHeadersAndSender(string change, bool outbox, bool inbox) + { + var original = new OutboxMessage { Envelope = Build([0, 255, 128], "subject.v1", "first"), ReceiverId = Receiver }; + var id = change == "id" ? HierarchicalKey.Create("other") : change == "child" ? Id.CreateChildKey("child") : Id; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var builder = new EnvelopeBuilder(id); + builder.AddBytes(MessageHeaders.Payload, change == "payload" ? [0, 255, 129] : change == "payload-length" ? [0, 255] : [0, 255, 128]); + if (change != "no-subject") builder.AddBytes(MessageHeaders.Subject, Encoding.UTF8.GetBytes(change == "subject" ? "other" : change == "subject-case" ? "Subject.v1" : "subject.v1")); + builder.AddBytes("custom", Encoding.UTF8.GetBytes(change == "custom" ? "second" : "first")); + if (change == "sender") builder.AddValue(MessageHeaders.Sender, GrainId.Create("sender", "different"), services.GetRequiredService>()); + var repeated = new OutboxMessage { Envelope = builder.Build(), ReceiverId = change == "receiver" ? GrainId.Create("receiver", "other") : Receiver }; + Assert.Equal(outbox, Invoke("EnvelopeEquivalence", "AreEquivalent", original, repeated)); + Assert.Equal(inbox, Invoke("EnvelopeEquivalence", "AreSameCommand", original.ToInboxMessage(), repeated.ToInboxMessage())); + Assert.Equal(inbox, Invoke("EnvelopeEquivalence", "AreSameCommand", original.Envelope, repeated.Envelope)); + Assert.True(original.Envelope.TryGetBytes("custom", out var retained)); + Assert.Equal("first", Encoding.UTF8.GetString(retained.Span)); + Assert.Equal(new byte[] { 0, 255, 128 }, original.Envelope.Payload.ToArray()); + } + + [Fact] + public void PendingRawCommandsKeepAbsentSubjectAndOriginalHeaders() + { + var first = new OutboxMessage { Envelope = Build([], custom: "first"), ReceiverId = Receiver }; + var repeated = new OutboxMessage { Envelope = Build([], custom: "second"), ReceiverId = Receiver }; + Assert.True(Invoke("EnvelopeEquivalence", "AreEquivalent", first, repeated)); + Assert.True(Invoke("EnvelopeEquivalence", "AreSameCommand", first.ToInboxMessage(), repeated.ToInboxMessage())); + Assert.False(first.Envelope.TryGetSubject(out _)); + Assert.True(first.Envelope.TryGetBytes("custom", out var original)); + Assert.Equal("first", Encoding.UTF8.GetString(original.Span)); + Assert.NotEqual(first.Envelope.EncodedHeaders.ToArray(), repeated.Envelope.EncodedHeaders.ToArray()); + } + + [Theory] + [InlineData("id", 1024, true)] [InlineData("id", 1025, false)] + [InlineData("depth", 32, true)] [InlineData("depth", 33, false)] + [InlineData("unicode-id", 512, true)] [InlineData("unicode-id", 513, false)] + [InlineData("escaped-id", 512, true)] [InlineData("escaped-id", 513, false)] + [InlineData("subject", 256, true)] [InlineData("subject", 257, false)] + [InlineData("unicode-subject", 128, true)] [InlineData("unicode-subject", 129, false)] + public void IdentityAndSubjectAdmissionBoundaries(string field, int length, bool valid) + { + var id = field switch + { + "id" => HierarchicalKey.Create(new string('x', length)), + "depth" => HierarchicalKey.Create(Enumerable.Repeat("x", length).ToArray()), + "unicode-id" => HierarchicalKey.Create(new string('\u00e9', length)), + "escaped-id" => HierarchicalKey.Create(new string('/', length)), + _ => Id + }; + Envelope Create() + { + using var builder = new EnvelopeBuilder(id); + builder.AddBytes(MessageHeaders.Payload, []); + if (field.Contains("subject", StringComparison.Ordinal)) builder.AddBytes(MessageHeaders.Subject, + Encoding.UTF8.GetBytes(new string(field == "subject" ? 's' : '\u00e9', length))); + return builder.Build(); + } + if (valid) + { + var envelope = Create(); + Validate(envelope); + Assert.Equal(id, envelope.MessageId); + Assert.Empty(envelope.Payload.ToArray()); + } + else Assert.Throws(() => Create()); + } + + [Fact] + public void InvalidIdentitySubjectAndTypedDecodingAreExplicit() + { + Assert.Throws(() => new EnvelopeBuilder(default)); + Assert.Throws(() => new EnvelopeBuilder(HierarchicalKey.Create("\ud800"))); + using var builder = new EnvelopeBuilder(Id); + Assert.Throws(() => builder.AddBytes(MessageHeaders.Subject, [])); + Assert.Throws(() => builder.AddBytes(MessageHeaders.Subject, [255])); + builder.AddBytes(MessageHeaders.Subject, " Subject.V1 "u8); + builder.AddBytes(MessageHeaders.Payload, [255]); + var envelope = builder.Build(); + Assert.True(envelope.TryGetSubject(out var subject)); + Assert.Equal(" Subject.V1 ", subject); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + Assert.ThrowsAny(() => envelope.TryGetValue(MessageHeaders.Payload, services.GetRequiredService>(), out _)); + var key = HierarchicalKey.Create(Enumerable.Repeat(new string('x', 1024), 33).ToArray()); + Assert.Equal(key, HierarchicalKey.Parse(key.ToString())); + Assert.Throws(() => new EnvelopeBuilder(key)); + } + + private static Envelope Build(byte[] payload, string? subject = null, string? custom = null) + { + using var builder = new EnvelopeBuilder(Id); + builder.AddBytes(MessageHeaders.Payload, payload); + if (subject is not null) builder.AddBytes(MessageHeaders.Subject, Encoding.UTF8.GetBytes(subject)); + if (custom is not null) builder.AddBytes("custom", Encoding.UTF8.GetBytes(custom)); + return builder.Build(); + } + + private static ArraySegment Segment(ReadOnlyMemory memory) + { + Assert.True(MemoryMarshal.TryGetArray(memory, out var segment)); + return segment; + } + private static byte[] Array(ReadOnlyMemory memory) => Segment(memory).Array!; + private static void Validate(T message) => Invoke("EnvelopeValidation", "Validate", message!); + private static T Invoke(string type, string method, params object[] args) + { + var helper = typeof(Envelope).Assembly.GetType("Orleans.Messaging." + type, true)!; + try { return (T)helper.GetMethod(method, args.Select(a => a.GetType()).ToArray())!.Invoke(null, args)!; } + catch (TargetInvocationException exception) when (exception.InnerException is { } cause) + { + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(cause).Throw(); + throw; + } + } + private static void AssertIds(Type type, params (string Name, uint Id)[] expected) + { + var actual = type.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .SelectMany(member => member.GetCustomAttributes().Select(id => (member.Name, id.Id))) + .OrderBy(p => p.Id).ToArray(); + Assert.Equal(expected.OrderBy(p => p.Id), actual); + } + private sealed class FailingCodec(Exception failure) : IFieldCodec + { + public void WriteField(ref Writer writer, uint fieldIdDelta, Type? expectedType, int value) + where TBufferWriter : IBufferWriter + { + writer.WriteByte(99); + writer.Commit(); + throw failure; + } + public int ReadValue(ref Reader reader, Field field) => throw new NotSupportedException(); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs new file mode 100644 index 00000000000..ad86770da5e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs @@ -0,0 +1,284 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class HandlerRoutingContractTests +{ + [Fact] + public void RegisterHandler_FirstRegistration_PreservesHandlerIdentity() + { + var inbox = CreateInbox(out var storage); + Assert.Null(RegisteredHandler(inbox)); + var handler = Substitute.For(); + + inbox.RegisterHandler(handler); + + Assert.Same(handler, RegisteredHandler(inbox)); + Assert.Equal(1000, inbox.Capacity); + storage.DidNotReceiveWithAnyArgs().TryGetValue(default, out _); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterHandler_SecondRegistration_RejectsWithoutReplacingOriginal(bool sameInstance) + { + var inbox = CreateInbox(out _); + var first = Substitute.For(); + inbox.RegisterHandler(first); + var replacement = sameInstance ? first : Substitute.For(); + + var exception = Assert.Throws(() => inbox.RegisterHandler(replacement)); + + Assert.Equal("A handler is already registered for this inbox.", exception.Message); + Assert.Same(first, RegisteredHandler(inbox)); + Assert.Equal(1000, inbox.Capacity); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterHandler_NullRegistration_RejectsWithoutChangingHandler(bool alreadyRegistered) + { + var inbox = CreateInbox(out _); + var handler = Substitute.For(); + if (alreadyRegistered) inbox.RegisterHandler(handler); + + var exception = Assert.Throws(() => inbox.RegisterHandler(null!)); + + Assert.Equal("handler", exception.ParamName); + if (alreadyRegistered) + { + Assert.Same(handler, RegisteredHandler(inbox)); + } + else + { + Assert.Null(RegisteredHandler(inbox)); + inbox.RegisterHandler(handler); + Assert.Same(handler, RegisteredHandler(inbox)); + } + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public void HandlerContext_Complete_ForwardsToOwnerSynchronously(int calls) + { + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => count++); + + for (var i = 0; i < calls; i++) + { + context.Complete(); + Assert.Equal(i + 1, count); + } + + Assert.Equal(calls, count); + AssertEnvelope(envelope, context.Message.Envelope); + } + + [Fact] + public void HandlerContext_Complete_PropagatesOriginalCallbackFailureWithoutChangingEnvelope() + { + var sentinel = new InvalidOperationException("owner failure"); + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => + { + count++; + throw sentinel; + }); + + var exception = Assert.Throws(context.Complete); + + Assert.Same(sentinel, exception); + Assert.Equal(1, count); + AssertEnvelope(envelope, context.Message.Envelope); + } + + [Fact] + public void HandlerContext_Envelope_PreservesIdentityAndPayload() + { + byte[] source = [0x00, 0xff, 0x80]; + var envelope = Envelope(source.ToArray()); + var count = 0; + var context = CreateContext(envelope, () => count++); + Array.Fill(source, (byte)0x42); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + + var decoded = services.GetRequiredService>() + .Deserialize(services.GetRequiredService>().SerializeToArray(context.Message.Envelope)); + context.Complete(); + + Assert.Equal(1, count); + AssertEnvelope(envelope, context.Message.Envelope); + Assert.Equal(envelope.MessageId, decoded.MessageId); + Assert.Equal(envelope.EncodedHeaders.ToArray(), decoded.EncodedHeaders.ToArray()); + Assert.Equal(new byte[] { 0x00, 0xff, 0x80 }, decoded.Payload.ToArray()); + Assert.Equal(new byte[] { 0x00, 0xff, 0x80 }, context.Message.Envelope.Payload.ToArray()); + } + + [Fact] + public async Task HandleAsync_SynchronousHandling_PreservesContextTokenAndExplicitCompletion() + { + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => count++); + using var cancellation = new CancellationTokenSource(); + var handler = Substitute.For(); + handler.HandleAsync(context, cancellation.Token).Returns(_ => + { + context.Complete(); + return ValueTask.CompletedTask; + }); + var inbox = CreateInbox(out _); + inbox.RegisterHandler(handler); + + var result = RegisteredHandler(inbox)!.HandleAsync(context, cancellation.Token); + Assert.True(result.IsCompletedSuccessfully); + await result; + + Assert.Equal(1, count); + AssertEnvelope(envelope, context.Message.Envelope); + await handler.Received(1).HandleAsync(context, cancellation.Token); + } + + [Fact] + public async Task HandleAsync_DeferredWork_PreservesContextAndCompletesOnlyAfterRelease() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => count++); + using var cancellation = new CancellationTokenSource(); + var handler = Substitute.For(); + async ValueTask Handle() + { + entered.SetResult(); + await release.Task; + context.Complete(); + } + handler.HandleAsync(context, cancellation.Token).Returns(_ => Handle()); + var inbox = CreateInbox(out _); + inbox.RegisterHandler(handler); + + var pending = RegisteredHandler(inbox)!.HandleAsync(context, cancellation.Token).AsTask(); + await entered.Task; + Assert.False(pending.IsCompleted); + Assert.Equal(0, count); + release.SetResult(); + await pending; + + Assert.Equal(1, count); + AssertEnvelope(envelope, context.Message.Envelope); + await handler.Received(1).HandleAsync(context, cancellation.Token); + } + + [Fact] + public async Task HandleAsync_ReturnWithoutComplete_DoesNotInvokeCompletion() + { + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => count++); + var handler = Substitute.For(); + handler.HandleAsync(context, CancellationToken.None).Returns(ValueTask.CompletedTask); + var inbox = CreateInbox(out _); + inbox.RegisterHandler(handler); + + await RegisteredHandler(inbox)!.HandleAsync(context, CancellationToken.None); + + Assert.Equal(0, count); + AssertEnvelope(envelope, context.Message.Envelope); + await handler.Received(1).HandleAsync(context, CancellationToken.None); + } + + [Fact] + public void HandlerNotFound_RoundTripsStatusAndDiagnostic() + { + var result = DeliveryResult.HandlerNotFound(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + + var decoded = serializer.Deserialize(serializer.SerializeToArray(result)); + + Assert.Equal(DeliveryStatus.HandlerNotFound, result.Status); + Assert.Equal(DeliveryStatus.HandlerNotFound, decoded.Status); + Assert.Equal("No inbox handler is registered.", result.Message); + Assert.Equal("No inbox handler is registered.", decoded.Message); + } + + [Fact] + public void ExternalConsumerAssembly_HasNoFriendAccessToMessaging() + { + var consumer = typeof(HandlerRoutingContractTests).Assembly.GetName().Name!; + Assert.DoesNotContain(typeof(IInboxHandler).Assembly.GetCustomAttributes(), + attribute => attribute.AssemblyName.Split(',')[0] == consumer); + } + + private static IInbox CreateInbox(out IDurableDictionary storage) + { + var type = typeof(IInbox).Assembly.GetType("Orleans.Messaging.Inbox", throwOnError: true)!; + storage = Substitute.For>(); + return Assert.IsAssignableFrom(Activator.CreateInstance(type, storage, 1000)); + } + + private static IInboxHandler? RegisteredHandler(IInbox inbox) + { + var method = inbox.GetType().GetMethod("TryGetHandler", BindingFlags.NonPublic | BindingFlags.Instance)!; + object?[] arguments = [null]; + var found = Assert.IsType(method.Invoke(inbox, arguments)); + if (!found) + { + Assert.Null(arguments[0]); + return null; + } + return Assert.IsAssignableFrom(arguments[0]); + } + + [Fact] + public void HandlerContext_Constructor_RequiresCompletionCallback() + { + var envelope = Envelope(); + var type = typeof(IInboxHandlerContext).Assembly.GetType("Orleans.Messaging.InboxHandlerContext", throwOnError: true)!; + var exception = Assert.Throws(() => Activator.CreateInstance(type, [new InboxMessage { Envelope = envelope }, null])); + Assert.Equal("complete", Assert.IsType(exception.InnerException).ParamName); + var parameters = Assert.Single(type.GetConstructors()).GetParameters(); + Assert.Equal(2, parameters.Length); + Assert.Equal(typeof(Action), parameters[1].ParameterType); + Assert.False(parameters[1].IsOptional); + } + + private static IInboxHandlerContext CreateContext(Envelope envelope, Action complete) + { + var type = typeof(IInboxHandlerContext).Assembly.GetType("Orleans.Messaging.InboxHandlerContext", throwOnError: true)!; + return Assert.IsAssignableFrom(Activator.CreateInstance(type, new InboxMessage { Envelope = envelope }, complete)); + } + + private static Envelope Envelope(byte[]? payload = null) + { + using var builder = new EnvelopeBuilder(HierarchicalKey.Create("44444444-4444-4444-4444-444444444444")); + builder.AddBytes(MessageHeaders.Payload, payload ?? [0x00, 0xff, 0x80]); + return builder.Build(); + } + + private static void AssertEnvelope(Envelope expected, Envelope actual) + { + Assert.Equal(expected.MessageId, actual.MessageId); + Assert.Equal(expected.EncodedHeaders, actual.EncodedHeaders); + Assert.Equal(expected.Payload, actual.Payload); + Assert.Equal(new byte[] { 0x00, 0xff, 0x80 }, actual.Payload.ToArray()); + Assert.False(actual.TryGetSubject(out _)); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs new file mode 100644 index 00000000000..8f1f82cc1c2 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs @@ -0,0 +1,349 @@ +using System.Buffers; +using System.Reflection; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class HierarchicalKeyTests +{ + [Fact] + public void SerializationContract_IdentifiesCurrentType() + { + var type = typeof(HierarchicalKey); + Assert.Equal("Orleans.Messaging.HierarchicalKey", Assert.Single(type.GetCustomAttributes()).Alias); + Assert.True(type.IsValueType); + Assert.Single(type.GetCustomAttributes()); + Assert.Empty(type.GetCustomAttributes()); + var field = Assert.Single(type.GetFields(BindingFlags.Instance | BindingFlags.NonPublic)); + Assert.True(field.IsInitOnly); + Assert.True(field.FieldType.IsSealed); + Assert.Equal(typeof(string), field.FieldType.GetProperty("Canonical")!.PropertyType); + Assert.Equal(typeof(int), field.FieldType.GetProperty("Hash")!.PropertyType); + Assert.DoesNotContain(field.FieldType.GetFields(BindingFlags.Instance | BindingFlags.NonPublic), + value => value.FieldType == typeof(HierarchicalKey) || value.FieldType == field.FieldType); + } + + [Fact] + public void Default_IsDistinguishedUnsetValue() + { + var key = default(HierarchicalKey); + Assert.True(key.IsDefault); + Assert.Equal(0, key.Length); + Assert.Equal(0, key.SegmentCount); + Assert.Equal(0, key.GetHashCode()); + Assert.Equal(string.Empty, key.ToString()); + Assert.True(key == default); + Assert.False(key != default); + Assert.False(key.Equals(null)); + Assert.Null(key.GetParent()); + Assert.False(key.GetEnumerator().MoveNext()); + Assert.False(key.IsAncestorOf(default)); + Assert.False(key.IsParentOf(default)); + Assert.False(key.IsChildOf(default)); + Assert.False(HierarchicalKey.Create("a").IsAncestorOf(key)); + Assert.Throws(() => key.CreateChildKey("a")); + Assert.Throws(() => key.Append(HierarchicalKey.Create("a"))); + Assert.Throws(() => HierarchicalKey.Create("a").Append(default)); + Span empty = []; + Assert.True(key.TryFormat(empty, out var written, default, null)); + Assert.Equal(0, written); + } + + [Theory] + [InlineData("foo", "foo")] + [InlineData("foo/bar", @"foo\/bar")] + [InlineData(@"foo\bar", @"foo\\bar")] + [InlineData(@"foo\/bar/baz", @"foo\\\/bar\/baz")] + [InlineData("/", @"\/")] + [InlineData(@"\", @"\\")] + [InlineData("/a//b/", @"\/a\/\/b\/")] + [InlineData(" ", " ")] + public void Create_LiteralSegment_EscapesExactlyOnce(string literal, string canonical) + { + var key = HierarchicalKey.Create(literal); + Assert.Equal(canonical, key.ToString()); + Assert.Equal(canonical.Length, key.Length); + Assert.Equal(1, key.SegmentCount); + Assert.Null(key.GetParent()); + Assert.Equal(key, HierarchicalKey.Parse(canonical)); + Assert.Equal([canonical], Segments(key)); + Assert.Equal(key, HierarchicalKey.Create(new[] { literal })); + } + + [Fact] + public void Create_WithValues_CreatesRootFirstHierarchy() + { + var key = HierarchicalKey.Create("orders", "42", "payment"); + var root = HierarchicalKey.Create("orders"); + var parent = root.CreateChildKey("42"); + Assert.Equal("orders/42/payment", key.ToString()); + Assert.Equal(["orders", "42", "payment"], Segments(key)); + Assert.Equal(3, key.SegmentCount); + Assert.Equal(17, key.Length); + Assert.Equal(parent.CreateChildKey("payment"), key); + Assert.Equal(HierarchicalKey.Parse("orders/42/payment"), key); + Assert.Equal(parent, key.GetParent()); + Assert.Equal(root, key.GetParent()!.Value.GetParent()); + Assert.Null(key.GetParent()!.Value.GetParent()!.Value.GetParent()); + Assert.True(parent.IsParentOf(key)); + Assert.True(key.IsChildOf(parent)); + Assert.True(root.IsAncestorOf(key)); + Assert.False(root.IsParentOf(key)); + } + + [Fact] + public void Create_WithValues_ArrayAndSpanSlicesUseOnlyProvidedFragments() + { + string[] input = ["ignored", "orders", "42", "payment", "ignored"]; + var expected = HierarchicalKey.Parse("orders/42/payment"); + Assert.Equal(expected, HierarchicalKey.Create(input.AsSpan(1, 3))); + ReadOnlySpan slice = input.AsSpan(1, 3); + Assert.Equal(expected, HierarchicalKey.Create(slice)); + Assert.Equal(expected, HierarchicalKey.Create(["orders", "42", "payment"])); + } + + [Fact] + public void Create_WithValues_ArrayMutationDoesNotChangeKey() + { + string[] input = ["orders", "42", "payment"]; + var key = HierarchicalKey.Create(input); + var hash = key.GetHashCode(); + input[0] = "other"; + input[1] = null!; + Assert.Equal("orders/42/payment", key.ToString()); + Assert.Equal(hash, key.GetHashCode()); + Assert.Equal(HierarchicalKey.Parse("orders/42"), key.GetParent()); + } + + [Fact] + public void Create_WithValues_EmptyInputThrowsArgumentException() + { + Assert.Equal("values", Assert.Throws(() => HierarchicalKey.Create()).ParamName); + Assert.Throws(() => HierarchicalKey.Create(ReadOnlySpan.Empty)); + Assert.Throws(() => HierarchicalKey.Create(Array.Empty())); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void Create_WithValues_NullFragmentThrowsArgumentNullException(int position) + { + string[] input = ["a", "b", "c"]; + input[position] = null!; + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(input)).ParamName); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void Create_WithValues_EmptyLiteralSegmentRejects(int position) + { + string[] input = ["a", "b", "c"]; + input[position] = ""; + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(input)).ParamName); + } + + [Fact] + public void Create_LiteralSegment_RejectsOnlyNullAndEmpty() + { + Assert.Throws(() => HierarchicalKey.Create((string)null!)); + Assert.Throws(() => HierarchicalKey.Create("")); + var parent = HierarchicalKey.Create("root"); + Assert.Throws(() => parent.CreateChildKey(null!)); + Assert.Throws(() => parent.CreateChildKey("")); + } + + [Theory] + [InlineData("")] + [InlineData("/a")] + [InlineData("a/")] + [InlineData("a//b")] + [InlineData(@"a\")] + [InlineData(@"a\q")] + [InlineData(@"a\/b/")] + public void Parse_InvalidCanonicalPath_RejectsStringAndSpan(string path) + { + Assert.Throws(() => HierarchicalKey.Parse(path)); + Assert.Throws(() => HierarchicalKey.Parse(path.AsSpan())); + Assert.False(HierarchicalKey.TryParse(path, null, out var first)); + Assert.True(first.IsDefault); + Assert.False(HierarchicalKey.TryParse(path.AsSpan(), null, out var second)); + Assert.True(second.IsDefault); + } + + [Fact] + public void Parse_WithNullString_ThrowsArgumentNullException() + { + Assert.Throws(() => HierarchicalKey.Parse((string)null!)); + Assert.False(HierarchicalKey.TryParse(null, null, out var key)); + Assert.True(key.IsDefault); + } + + [Theory] + [InlineData("a/b/c", 3)] + [InlineData(@"a\/b/c", 2)] + [InlineData(@"a\\/b/c", 3)] + [InlineData(@"a\\\/b/c", 2)] + public void Parse_CanonicalEscapes_PreserveSegmentBoundaries(string path, int count) + { + var key = HierarchicalKey.Parse(path); + Assert.Equal(path, key.ToString()); + Assert.Equal(count, key.SegmentCount); + Assert.Equal(count, Segments(key).Count); + Assert.True(HierarchicalKey.TryParse(path.AsSpan(), null, out var copy)); + Assert.True(key == copy); + Assert.Equal(key.GetHashCode(), copy.GetHashCode()); + var characters = path.ToCharArray(); + var fromCaller = HierarchicalKey.Parse(characters.AsSpan()); + characters[0] = 'z'; + Assert.Equal(key, fromCaller); + } + + [Fact] + public void Append_ComposesConstructedPaths_ChildAppendsOneLiteral() + { + var parent = HierarchicalKey.Parse(@"root/fo\/o"); + var suffix = HierarchicalKey.Parse(@"ba\\r/baz"); + var composed = parent.Append(suffix); + Assert.Equal(@"root/fo\/o/ba\\r/baz", composed.ToString()); + Assert.Equal(4, composed.SegmentCount); + Assert.Equal(["root", @"fo\/o", @"ba\\r", "baz"], Segments(composed)); + Assert.Equal(parent.Append(HierarchicalKey.Create(@"ba\r")), composed.GetParent()); + Assert.True(parent.IsAncestorOf(composed)); + Assert.False(parent.IsParentOf(composed)); + var literal = parent.CreateChildKey("ba/r/baz"); + Assert.Equal(@"root/fo\/o/ba\/r\/baz", literal.ToString()); + Assert.Equal(3, literal.SegmentCount); + Assert.Equal(parent, literal.GetParent()); + Assert.True(parent.IsParentOf(literal)); + Assert.NotEqual(composed, literal); + } + + [Theory] + [InlineData("foo", "foo/bar", true, true)] + [InlineData("foo", "foo/bar/baz", false, true)] + [InlineData("foo/bar", "foo/bar", false, true)] + [InlineData("foo", "foobar/child", false, false)] + [InlineData("foo", @"foo\/bar/child", false, false)] + [InlineData("foo/bar", "foo/baz", false, false)] + [InlineData("Foo", "foo/bar", false, false)] + public void Navigation_UsesExactOrdinalSegments(string root, string path, bool immediate, bool ancestor) + { + var parent = HierarchicalKey.Parse(root); + var child = HierarchicalKey.Parse(path); + Assert.Equal(immediate, parent.IsParentOf(child)); + Assert.Equal(immediate, child.IsChildOf(parent)); + Assert.Equal(ancestor, parent.IsAncestorOf(child)); + if (immediate) Assert.Equal(parent, child.GetParent()); + } + + [Fact] + public void Equality_HashOperatorsAndDictionary_UseCanonicalOrdinalIdentity() + { + var direct = HierarchicalKey.Parse(@"tenant/acme/orders\/42"); + var composed = HierarchicalKey.Create("tenant", "acme", "orders/42"); + Assert.True(direct == composed); + Assert.False(direct != composed); + Assert.True(direct.Equals((object)composed)); + Assert.Equal(direct.GetHashCode(), composed.GetHashCode()); + Assert.Equal(direct.ToString().GetHashCode(StringComparison.Ordinal), direct.GetHashCode()); + var other = HierarchicalKey.Create("Tenant", "acme", "orders/42"); + Assert.True(other != direct); + Assert.False(other == direct); + Assert.False(direct.Equals("tenant/acme/orders\\/42")); + var dictionary = new Dictionary { [direct] = 42, [other] = 81, [default] = 5 }; + Assert.Equal(42, dictionary[composed]); + Assert.Equal(81, dictionary[other]); + Assert.Equal(5, dictionary[default]); + Assert.Equal(3, dictionary.Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SerializationAndCopy_RebuildCanonicalCacheAndPreserveValue(bool unset) + { + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var key = unset ? default : HierarchicalKey.Create("tenant", @"a\/b", "42"); + var serializer = first.GetRequiredService>(); + var bytes = serializer.SerializeToArray(key); + var decoded = second.GetRequiredService>().Deserialize(bytes); + var copy = first.GetRequiredService().Copy(key); + Assert.Equal(key, decoded); + Assert.Equal(key.IsDefault, decoded.IsDefault); + Assert.Equal(key.GetHashCode(), decoded.GetHashCode()); + Assert.Equal(key.SegmentCount, decoded.SegmentCount); + Assert.Equal(key, copy); + var data = Assert.Single(typeof(HierarchicalKey).GetFields(BindingFlags.Instance | BindingFlags.NonPublic)); + Assert.Same(data.GetValue(key), data.GetValue(copy)); + if (!unset) Assert.NotSame(data.GetValue(key), data.GetValue(decoded)); + Assert.Equal(bytes, serializer.SerializeToArray(decoded)); + var map = first.GetRequiredService>>(); + var dictionary = Assert.IsType>(map.Deserialize(map.SerializeToArray(new() { [key] = 42 }))); + Assert.Equal(42, dictionary[decoded]); + } + + [Theory] + [InlineData("")] + [InlineData("a//b")] + [InlineData(@"a\q")] + public void Codec_InvalidCanonicalWire_RejectsInsteadOfConstructingKey(string canonical) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var session = services.GetRequiredService().GetSession(); + var output = new ArrayBufferWriter(); + var writer = Writer.Create(output, session); + ReferenceCodec.MarkValueField(session); + writer.WriteFieldHeader(0, typeof(HierarchicalKey), typeof(HierarchicalKey), WireType.TagDelimited); + services.GetRequiredService>().WriteField(ref writer, 0, typeof(string), canonical); + writer.WriteEndObject(); + writer.Commit(); + Assert.Throws(() => services.GetRequiredService>().Deserialize(output.WrittenSpan)); + } + + [Fact] + public void LongHierarchy_ConstructionNavigationAndFormatting_AreIterative() + { + var key = HierarchicalKey.Create(Enumerable.Repeat("x", 10000).ToArray()); + Assert.Equal(10000, key.SegmentCount); + Assert.Equal(19999, key.Length); + Assert.Equal(10000, Segments(key).Count); + Assert.Equal(9999, key.GetParent()!.Value.SegmentCount); + Assert.Equal(key, HierarchicalKey.Parse(key.ToString())); + Assert.True(HierarchicalKey.Create("x").IsAncestorOf(key)); + var span = new char[key.Length]; + Assert.True(key.TryFormat(span, out var written, default, null)); + Assert.Equal(key.Length, written); + Assert.Equal(key.ToString(), new string(span)); + Array.Fill(span, 'z'); + Assert.False(key.TryFormat(span.AsSpan(0, key.Length - 1), out written, default, null)); + Assert.Equal(0, written); + Assert.All(span, character => Assert.Equal('z', character)); + } + + [Fact] + public void EscapeCharacter_IsBackslash() => Assert.Equal('\\', HierarchicalKey.EscapeCharacter); + + [Fact] + public void SegmentSeparator_IsForwardSlash() => Assert.Equal('/', HierarchicalKey.SegmentSeparator); + + private static List Segments(HierarchicalKey key) + { + var values = new List(); + foreach (var segment in key) values.Add(segment.ToString()); + return values; + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingPumpResultsTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingPumpResultsTests.cs new file mode 100644 index 00000000000..ec9b8270d5a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingPumpResultsTests.cs @@ -0,0 +1,356 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.Time.Testing; +using Orleans.DurableJobs; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingPumpResultsTests +{ + [Fact] + public void ConcurrentStarts_SuppressDuplicateExecution() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + var starts = new bool[64]; + + Parallel.For(0, starts.Length, index => starts[index] = results.TryStart(key, out _)); + + Assert.Equal(1, starts.Count(static started => started)); + } + + [Fact] + public void CanceledWaitingExecution_BecomesTakeableAndDoesNotRun() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + using var cancellation = new CancellationTokenSource(); + + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + cancellation.Cancel(); + + Assert.False(results.HasCancellationRegistration(key)); + Assert.False(results.TryBegin(execution)); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Null(result); + Assert.IsType(exception); + } + + + [Fact] + public void AlreadyCanceledToken_CompletesBeforeRegistrationInstallWithoutRetainingRegistration() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + + Assert.False(results.HasCancellationRegistration(key)); + Assert.False(results.TryBegin(execution)); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Null(result); + Assert.IsType(exception); + } + + [Fact] + public void RunningExecution_DoesNotRetainAttemptCancellationRegistration() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + using var cancellation = new CancellationTokenSource(); + + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + Assert.True(results.HasCancellationRegistration(key)); + Assert.True(results.TryBegin(execution)); + + Assert.False(results.HasCancellationRegistration(key)); + } + + + [Fact] + public void CompletedWaitingExecution_DoesNotRetainAttemptCancellationRegistration() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + using var cancellation = new CancellationTokenSource(); + + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + Assert.True(results.HasCancellationRegistration(key)); + results.Complete(execution); + + Assert.False(results.HasCancellationRegistration(key)); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Fact] + public void CompletedResultWithoutSecondPoll_Expires() + { + var clock = new FakeTimeProvider(); + var results = new PumpResults(clock, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 16); + var key = results.CreateKey("job", "id", "run"); + Assert.True(results.TryStart(key, out var execution)); + Assert.True(results.TryBegin(execution)); + results.Complete(execution); + + clock.Advance(TimeSpan.FromMinutes(2)); + _ = results.TryStart(results.CreateKey("job", "other", "run"), out _); + + Assert.False(results.TryTake(key, out _, out _)); + } + + [Fact] + public void RetainedEntries_AreBounded() + { + var results = new PumpResults(new FakeTimeProvider(), TimeSpan.FromHours(1), TimeSpan.FromHours(1), 4); + + for (var index = 0; index < 100; index++) + { + Assert.True(results.TryStart(results.CreateKey("job", index.ToString(), "run"), out _)); + } + + Assert.InRange(results.Count, 0, 4); + } + + [Fact] + public void RunningExecution_IsNotExpiredOrDuplicated() + { + var clock = new FakeTimeProvider(); + var results = new PumpResults(clock, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 4); + var key = results.CreateKey("job", "id", "run"); + Assert.True(results.TryStart(key, out var execution)); + Assert.True(results.TryBegin(execution)); + + clock.Advance(TimeSpan.FromHours(1)); + + Assert.False(results.TryStart(key, out _)); + results.Complete(execution); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Fact] + public void CapacityExhaustedByRunningExecution_RejectsNewStartWithoutLosingRunningExecution() + { + var results = new PumpResults(new FakeTimeProvider(), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 1); + var runningKey = results.CreateKey("job", "running", "run"); + var rejectedKey = results.CreateKey("job", "rejected", "run"); + Assert.True(results.TryStart(runningKey, out var execution)); + Assert.True(results.TryBegin(execution)); + + Assert.False(results.TryStart(rejectedKey, out _)); + results.Complete(execution); + + Assert.True(results.TryTake(runningKey, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Fact] + public void DifferentRunId_DoesNotObserveOlderResult() + { + var results = new PumpResults(); + var firstKey = results.CreateKey("job", "id", "run-1"); + var secondKey = results.CreateKey("job", "id", "run-2"); + Assert.True(results.TryStart(firstKey, out var firstExecution)); + Assert.True(results.TryBegin(firstExecution)); + results.Complete(firstExecution); + + Assert.True(results.TryStart(secondKey, out var secondExecution)); + Assert.False(results.TryTake(secondKey, out _, out _)); + Assert.True(results.TryTake(firstKey, out var firstResult, out _)); + Assert.Same(DurableJobRunResult.Completed, firstResult); + Assert.True(results.TryBegin(secondExecution)); + } + + [Fact] + public void DifferentStateGeneration_DoesNotObservePreRecoveryResult() + { + var results = new PumpResults(); + var beforeRecovery = results.CreateKey("job", "id", "run", stateGeneration: 1); + var afterRecovery = results.CreateKey("job", "id", "run", stateGeneration: 2); + Assert.True(results.TryStart(beforeRecovery, out var execution)); + Assert.True(results.TryBegin(execution)); + results.Complete(execution); + + Assert.False(results.TryTake(afterRecovery, out _, out _)); + Assert.True(results.TryStart(afterRecovery, out _)); + Assert.True(results.TryTake(beforeRecovery, out var result, out _)); + Assert.Same(DurableJobRunResult.Completed, result); + } + + [Fact] + public void DiscardWaitingExecution_ReleasesRegistrationAndCapacity() + { + var results = new PumpResults(new FakeTimeProvider(), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 1); + var key = results.CreateKey("inbox", "id", "run"); + using var cancellation = new CancellationTokenSource(); + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + var entry = results.GetEntry(key); + Assert.True(results.HasCancellationRegistration(key)); + + results.Discard(execution); + + Assert.Equal(0, results.Count); + Assert.Equal(default, PumpResults.GetRegistration(entry)); + Assert.False(results.TryBegin(execution)); + cancellation.Cancel(); + Assert.False(results.TryTake(key, out _, out _)); + Assert.True(results.TryStart(results.CreateKey("inbox", "other", "run"), out var next)); + Assert.True(results.TryBegin(next)); + } + + [Fact] + public void DiscardOldExecution_PreservesNewGenerationWithSameKey() + { + var results = new PumpResults(); + var key = results.CreateKey("inbox", "id", "run"); + Assert.True(results.TryStart(key, out var previous)); + results.Discard(previous); + using var cancellation = new CancellationTokenSource(); + Assert.True(results.TryStartWithCancellation(key, out var current, cancellation.Token)); + + results.Discard(previous); + + Assert.Equal(1, results.Count); + Assert.True(results.HasCancellationRegistration(key)); + Assert.True(results.TryBegin(current)); + } + + [Fact] + public void DiscardRunningExecution_PreservesInFlightResult() + { + var results = new PumpResults(); + var key = results.CreateKey("inbox", "id", "run"); + Assert.True(results.TryStart(key, out var execution)); + Assert.True(results.TryBegin(execution)); + + results.Discard(execution); + + Assert.Equal(1, results.Count); + Assert.False(results.TryStart(key, out _)); + results.Complete(execution); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ClearJob_ReleasesOnlyThatFeaturesEntriesAndFencesLateCompletion(bool running) + { + var results = new PumpResults(); + var inbox = results.CreateKey("inbox", "id", "run"); + var outbox = results.CreateKey("outbox", "id", "run"); + using var cancellation = new CancellationTokenSource(); + Assert.True(results.TryStartWithCancellation(inbox, out var oldInbox, cancellation.Token)); + var removed = results.GetEntry(inbox); + if (running) + { + Assert.True(results.TryBegin(oldInbox)); + } + Assert.True(results.TryStartWithCancellation(outbox, out var liveOutbox, cancellation.Token)); + + results.Clear("inbox"); + + Assert.Equal(1, results.Count); + Assert.Equal(default, PumpResults.GetRegistration(removed)); + Assert.True(results.HasCancellationRegistration(outbox)); + Assert.False(results.TryBegin(oldInbox)); + Assert.True(results.TryStart(inbox, out var newInbox)); + results.Complete(oldInbox); + Assert.False(results.TryTake(inbox, out _, out _)); + Assert.True(results.TryBegin(newInbox)); + Assert.True(results.TryBegin(liveOutbox)); + } + + private sealed class PumpResults + { + private static readonly Assembly Assembly = typeof(IOutbox).Assembly; + private static readonly Type ResultsType = Assembly.GetType("Orleans.Messaging.MessagingPumpResults", throwOnError: true)!; + private static readonly Type KeyType = Assembly.GetType("Orleans.Messaging.MessagingPumpExecutionKey", throwOnError: true)!; + private readonly object _instance; + + public PumpResults() + { + _instance = Activator.CreateInstance(ResultsType, nonPublic: true)!; + } + + public PumpResults(TimeProvider timeProvider, TimeSpan completedRetention, TimeSpan abandonedRetention, int maxEntries) + { + _instance = Activator.CreateInstance( + ResultsType, + BindingFlags.Instance | BindingFlags.NonPublic, + binder: null, + [timeProvider, completedRetention, abandonedRetention, maxEntries], + culture: null)!; + } + + public int Count => ((IDictionary)ResultsType + .GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_instance)!).Count; + + public object GetEntry(object key) => ((IDictionary)ResultsType + .GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_instance)!)[key]!; + + public static CancellationTokenRegistration GetRegistration(object entry) => + (CancellationTokenRegistration)entry.GetType().GetProperty("CancellationRegistration")!.GetValue(entry)!; + + public void Discard(object execution) => ResultsType.GetMethod("Discard")!.Invoke(_instance, [execution]); + public void Clear(string jobName) => ResultsType.GetMethod("Clear")!.Invoke(_instance, [jobName]); + + public bool HasCancellationRegistration(object key) + { + var entries = (IDictionary)ResultsType + .GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_instance)!; + var entry = entries[key]!; + var registration = (CancellationTokenRegistration)entry.GetType() + .GetProperty("CancellationRegistration")! + .GetValue(entry)!; + return registration != default; + } + + public object CreateKey(string jobName, string jobId, string runId, long stateGeneration = 0) => + Activator.CreateInstance(KeyType, [jobName, jobId, runId, stateGeneration])!; + + public bool TryStart(object key, out object execution) => + TryStartWithCancellation(key, out execution, TestContext.Current.CancellationToken); + + public bool TryStartWithCancellation( + object key, + out object execution, + CancellationToken cancellationToken) + { + object?[] arguments = [key, cancellationToken, null]; + var result = (bool)ResultsType.GetMethod("TryStart")!.Invoke(_instance, arguments)!; + execution = arguments[2]!; + return result; + } + + public bool TryBegin(object execution) => + (bool)ResultsType.GetMethod("TryBegin")!.Invoke(_instance, [execution])!; + + public void Complete(object execution) => + ResultsType.GetMethod("Complete")!.Invoke(_instance, [execution, DurableJobRunResult.Completed]); + + public bool TryTake(object key, out DurableJobRunResult? result, out Exception? exception) + { + object?[] arguments = [key, null, null]; + var taken = (bool)ResultsType.GetMethod("TryTake")!.Invoke(_instance, arguments)!; + result = arguments[1] as DurableJobRunResult; + exception = arguments[2] as Exception; + return taken; + } + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs b/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs new file mode 100644 index 00000000000..1b4de3c3296 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs @@ -0,0 +1,105 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Messaging.Tests.Functional; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class StandardJournaledStateTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Dictionary_WriteAcknowledgementPreservesLaterMutations(bool snapshot) + { + var id = new JournalId("standard-dictionary/" + Guid.NewGuid().ToString("N")); + var storage = new ControlledJournalStorageProvider(); + await using var provider = CreateProvider(storage, id); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var items = scope.ServiceProvider.GetRequiredKeyedService>("state"); + Assert.False(owner is IDurableStateManager); + Assert.True(owner.TryGetStateMachine("state", out var registered)); + Assert.Same(items, registered); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + items.Add("first", 1); + if (snapshot) storage.RequestSnapshot(id); + using var blocked = storage.BlockWrite(id); + var first = owner.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await blocked.WaitUntilEnteredAsync(); + items["later"] = 2; + Assert.False(first.IsCompleted); + blocked.Release(); + await first; + await using (var replay = provider.CreateAsyncScope()) + { + var recovered = replay.ServiceProvider.GetRequiredService(); + var persisted = replay.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(new KeyValuePair("first", 1), Assert.Single(persisted)); + } + Assert.Equal(2, items["later"]); + await owner.WriteStateAsync(TestContext.Current.CancellationToken); + await using var finalScope = provider.CreateAsyncScope(); + var finalOwner = finalScope.ServiceProvider.GetRequiredService(); + var final = finalScope.ServiceProvider.GetRequiredKeyedService>("state"); + await finalOwner.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(new[] { new KeyValuePair("first", 1), new KeyValuePair("later", 2) }, final.OrderBy(pair => pair.Key)); + Assert.Equal(2, storage.GetSuccessfulWriteCount(id)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Value_WriteAcknowledgementPreservesNewerPendingValue(bool snapshot) + { + var id = new JournalId("standard-value/" + Guid.NewGuid().ToString("N")); + var storage = new ControlledJournalStorageProvider(); + await using var provider = CreateProvider(storage, id); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var value = scope.ServiceProvider.GetRequiredKeyedService>("state"); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + value.Value = 1; + if (snapshot) storage.RequestSnapshot(id); + using var blocked = storage.BlockWrite(id); + var first = owner.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await blocked.WaitUntilEnteredAsync(); + value.Value = 2; + blocked.Release(); + await first; + await using (var replay = provider.CreateAsyncScope()) + { + var recovered = replay.ServiceProvider.GetRequiredService(); + var persisted = replay.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, persisted.Value); + } + Assert.Equal(2, value.Value); + await owner.WriteStateAsync(TestContext.Current.CancellationToken); + await using var finalScope = provider.CreateAsyncScope(); + var finalOwner = finalScope.ServiceProvider.GetRequiredService(); + var final = finalScope.ServiceProvider.GetRequiredKeyedService>("state"); + await finalOwner.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, final.Value); + Assert.Equal(2, storage.GetSuccessfulWriteCount(id)); + } + + private static ServiceProvider CreateProvider(ControlledJournalStorageProvider storage, JournalId id) + { + var builder = InboxStateManagerBoundaryTests.CreateBuilder("orleans-binary"); + builder.Services.AddSingleton(sp => + { + storage.Configure(sp.GetRequiredService>()); + return storage; + }); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + return builder.Services.BuildServiceProvider(validateScopes: true); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs new file mode 100644 index 00000000000..e9151a1a184 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs @@ -0,0 +1,150 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ByteMessagingPayloadTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task DirectAdmission_CallerCancellationLeavesActualAcceptanceAndHandlerPayloadAvailable() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + var envelope = CreateEnvelope(receiver, NewMessage(300, "gc-admission")); + var publishedPayload = envelope.Envelope.Payload; + var expected = envelope.Envelope.Payload.ToArray(); + using var handler = new PayloadProbe(expected); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + using var scheduling = Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName); + using var cancellation = new CancellationTokenSource(); + Task waiting = null!; + await OnTurnAsync(context, () => waiting = extension.DeliverAsync(envelope, cancellation.Token).AsTask()); + await scheduling.WaitUntilEnteredAsync(); + var actual = (Task)extension.GetType().GetField("_activeDelivery", + BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + envelope = default; + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => waiting); + Assert.False(actual.IsCompleted); + scheduling.Continue(); + await actual.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(publishedPayload, handler.Payload); + Assert.Equal(expected, handler.Payload.ToArray()); + handler.Release.TrySetResult(); + await handler.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.InboxCount == 0 && snapshot.ProcessedMessageCount == 1); + Assert.Empty(completed.InboxDeadLetters); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, handler.Payload.ToArray()); + } + + [Fact] + public async Task RpcAcceptanceAndDuplicate_CopyPayloadAndPreserveCallerBytesAcrossReplay() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(302, "rpc-bytes")); + var expected = envelope.Envelope.Payload.ToArray(); + using var handler = new PayloadProbe(expected); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.True(MemoryMarshal.TryGetArray(envelope.Envelope.Payload, out var caller)); + Assert.True(MemoryMarshal.TryGetArray(handler.Payload, out var received)); + Assert.NotSame(caller.Array, received.Array); + caller.Array![caller.Offset] ^= 0xff; + Assert.Equal(expected, handler.Payload.ToArray()); + caller.Array[caller.Offset] ^= 0xff; + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + handler.Release.TrySetResult(); + await handler.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.InboxCount == 0 && snapshot.ProcessedMessageCount == 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task RpcRejection_PreservesCallerBytesWithoutStagingState() + { + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + var original = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await original.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(301, "rejected-bytes")); + var expected = envelope.Envelope.Payload.ToArray(); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(receiver, envelope)).Status); + var rejected = await receiver.GetSnapshotAsync(); + Assert.Equal(0, rejected.InboxCount); + Assert.Equal(0, rejected.ProcessedMessageCount); + Assert.Empty(rejected.Effects); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completed.SetResult(); } + catch (Exception exception) { completed.SetException(exception); } + }); + return completed.Task; + } + + private sealed class PayloadProbe(byte[] expected) : IInboxHandler, IDisposable + { + public ReadOnlyMemory Payload { get; private set; } + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + try + { + Payload = context.Message.Envelope.Payload; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + Assert.Equal(expected, Payload.ToArray()); + context.Complete(); + Assert.Equal(Payload, context.Message.Envelope.Payload); + Assert.Equal(expected, context.Message.Envelope.Payload.ToArray()); + Completed.TrySetResult(); + } + catch (Exception exception) + { + Completed.TrySetException(exception); + throw; + } + } + + public void Dispose() => Release.TrySetResult(); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs new file mode 100644 index 00000000000..c7ebf76f74d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs @@ -0,0 +1,106 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DedupeExpiryClusterCollection : ICollectionFixture +{ + public const string Name = "Messaging dedupe expiry cluster"; +} + +[Collection(DedupeExpiryClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DedupeExpiryBehaviorTests(DedupeExpiryClusterFixture fixture) +{ + [Fact] + public async Task IdleReplay_AtDeduplicationBoundary_IsAcceptedWithoutCompactionTrigger() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var original = new TestMessage(TestApplicationProtocol.NewMessageId(), 15, "expires"); + var first = CreateEnvelope(receiver, original); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, first)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + + fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + var state = await fixture.WaitForEffectCountAsync(receiver, 2); + + Assert.Equal(2, state.Effects.Single(effect => effect.LogicalId == original.LogicalId).Count); + } + + [Fact] + public async Task ExpiryReplacementFailure_ReplaysDedupeRecordBeforeExpiryMaintenance() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), 16, "failed-expiry-replacement")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + var oldContext = fixture.GetGrainContext(receiver); + fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await Assert.ThrowsAnyAsync(() => DeliverAsync(receiver, envelope)); + + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var failed = await receiver.GetSnapshotAsync(); + Assert.Equal(0, failed.InboxCount); + Assert.Equal(1, Assert.Single(failed.Effects).Count); + var replayed = Assert.IsType(fixture.GetGrainContext(receiver).GrainInstance).ReplayedSnapshot; + Assert.Equal(1, Assert.IsType(replayed).ProcessedMessageCount); + Assert.Equal(0, failed.ProcessedMessageCount); + + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(failed.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(0, recovered.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var retried = await fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(2, Assert.Single(retried.Effects).Count); + } + + private Task WaitForIdleInboxAsync(IMessagingTestGrain receiver) => + fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && string.IsNullOrEmpty(snapshot.InboxJobId)); + + private static async Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + await receiver.AsReference().DeliverAsync(envelope); + + private InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + TestMessage message, + HierarchicalKey? messageId = null) + { + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("expiry-test-sender", "stable"); + var built = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/expiry", message); + if (messageId is { } id) + { + built = TestApplicationProtocol.With(built, sessions, messageId: id); + } + + return built; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/DeliveryCancellationTests.cs b/test/Orleans.Messaging.Tests/Functional/DeliveryCancellationTests.cs new file mode 100644 index 00000000000..021791b4d19 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/DeliveryCancellationTests.cs @@ -0,0 +1,227 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DeliveryCancellationTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false, "schedule")] + [InlineData(true, "schedule")] + [InlineData(false, "acknowledgement")] + [InlineData(true, "acknowledgement")] + [InlineData(false, "storage")] + [InlineData(true, "storage")] + public async Task CallerCancellation_RetainsOwnedDeliveryUntilCommit(bool proxy, string phase) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var extension = GetExtension(context); + using var blocked = Block(receiver, phase); + var envelope = CreateEnvelope(receiver, NewMessage(150, phase)); + using var cancellation = new CancellationTokenSource(); + var delivery = StartDelivery(proxy, receiver, context, extension, envelope, cancellation.Token); + await blocked.Entered(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => delivery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + + Assert.Equal(0, GetGate(extension).CurrentCount); + Assert.Single(GetPendingOwners(extension)); + var duplicate = StartDelivery(false, receiver, context, extension, envelope, TestContext.Current.CancellationToken); + await OnTurnAsync(context, static () => { }); + Assert.False(duplicate.IsCompleted); + Assert.Equal(1, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + blocked.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(3, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.Equal(1, GetGate(extension).CurrentCount); + Assert.Empty(GetPendingOwners(extension)); + Assert.False(((MessagingTestGrain)context.GrainInstance!).DeactivationFailure.Task.IsCompleted); + } + + [Theory] + [InlineData(false, "acknowledgement")] + [InlineData(true, "acknowledgement")] + [InlineData(false, "storage")] + [InlineData(true, "storage")] + public async Task CanceledCaller_LateFailureIsLoggedAndReleasesOwnership(bool proxy, string phase) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = GetExtension(context); + var envelope = CreateEnvelope(receiver, NewMessage(151, phase)); + using var logs = new DeliveryLogProbe(envelope.Envelope.MessageId); + Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService().AddProvider(logs); + using var blocked = Block(receiver, phase); + using var cancellation = new CancellationTokenSource(); + var delivery = StartDelivery(proxy, receiver, context, extension, envelope, cancellation.Token); + await blocked.Entered(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Equal(0, GetGate(extension).CurrentCount); + Assert.Single(GetPendingOwners(extension)); + + Assert.IsType(blocked.Fail)(); + var failure = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var logged = await logs.Failure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + Assert.IsType(failure); + Assert.Same(failure, logged); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(1, GetGate(extension).CurrentCount); + Assert.Empty(GetPendingOwners(extension)); + Assert.Empty(grain.GetSnapshotForTest().Effects); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + if (phase == "acknowledgement") + { + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + else + { + Assert.Equal(0, recovered.InboxCount); + Assert.Null(recovered.InboxJobId); + Assert.Empty(recovered.Effects); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeactivationAfterCallerCancellation_DrainsOwnedDelivery(bool proxy) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = GetExtension(context); + var shutdown = (CancellationTokenSource)extension.GetType().GetField("_shutdownCts", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + var stopping = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = shutdown.Token.Register(() => stopping.TrySetResult()); + using var blocked = Block(receiver, "storage"); + var envelope = CreateEnvelope(receiver, NewMessage(152, "deactivation")); + using var cancellation = new CancellationTokenSource(); + var delivery = StartDelivery(proxy, receiver, context, extension, envelope, cancellation.Token); + await blocked.Entered(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + + _ = await receiver.GetSnapshotAsync().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Drain canceled delivery waiter."), TestContext.Current.CancellationToken); + await stopping.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.False(context.Deactivated.IsCompleted); + Assert.Equal(0, GetGate(extension).CurrentCount); + Assert.Single(GetPendingOwners(extension)); + Assert.True(shutdown.Token.CanBeCanceled); + blocked.Release(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(1, GetGate(extension).CurrentCount); + Assert.Empty(GetPendingOwners(extension)); + Assert.Throws(() => shutdown.Token); + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + private BlockedPhase Block(IMessagingTestGrain receiver, string phase) + { + switch (phase) + { + case "schedule": + var schedule = Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName); + return new(schedule.WaitUntilEnteredAsync, schedule.Continue, null); + case "acknowledgement": + var acknowledgement = Fixture.Storage.BlockAcknowledgement(JournalId.FromGrainId(receiver.GetGrainId())); + return new(acknowledgement.WaitUntilEnteredAsync, acknowledgement.Release, acknowledgement.Fail); + case "storage": + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + return new(storage.WaitUntilEnteredAsync, storage.Release, storage.Fail); + default: + throw new ArgumentOutOfRangeException(nameof(phase)); + } + } + + private static Task StartDelivery(bool proxy, IMessagingTestGrain receiver, IGrainContext context, + IInboxExtension extension, InboxMessage envelope, CancellationToken cancellationToken) + { + if (proxy) + { + return DeliverWithCancellationAsync(receiver, envelope, cancellationToken); + } + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(envelope, cancellationToken).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private static IInboxExtension GetExtension(IGrainContext context) => + (IInboxExtension)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + private static SemaphoreSlim GetGate(IInboxExtension extension) => + (SemaphoreSlim)extension.GetType().GetField("_gate", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + private static HashSet GetPendingOwners(IInboxExtension extension) => + (HashSet)extension.GetType().GetField("_pendingOwnershipIds", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completed.SetResult(); } + catch (Exception exception) { completed.SetException(exception); } + }); + return completed.Task; + } + + private sealed record BlockedPhase(Func Entered, Action Release, Action? Fail) : IDisposable + { + public void Dispose() => Release(); + } + + private sealed class DeliveryLogProbe(HierarchicalKey messageId) : ILoggerProvider + { + public TaskCompletionSource Failure { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public ILogger CreateLogger(string categoryName) => new Logger(this, categoryName, messageId.ToString()); + public void Dispose() { } + private sealed class Logger(DeliveryLogProbe owner, string category, string messageId) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel level) => level >= LogLevel.Error; + public void Log(LogLevel level, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (exception is not null && category == "Orleans.Messaging.InboxExtension" + && eventId.Name == "DeliveryOperationFailed" && formatter(state, exception).Contains(messageId, StringComparison.Ordinal)) + { + owner.Failure.TrySetResult(exception); + } + } + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs new file mode 100644 index 00000000000..174ac17952a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs @@ -0,0 +1,91 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Messaging.Tests.Support; +using Orleans.Metadata; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class EarlyActivationValidationTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(typeof(ReentrantActivationValidationTestGrain), "non-reentrant")] + [InlineData(typeof(StatelessActivationValidationTestGrain), "one activation")] + [InlineData(typeof(MayInterleaveActivationValidationTestGrain), "non-reentrant")] + [InlineData(typeof(AlwaysInterleaveActivationValidationTestGrain), "interleavable method")] + public async Task UnsupportedModel_FailsBeforeJournalInitializationReplayAndWork(Type grainType, string diagnostic) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + await AssertRejectedBeforeRecoveryAsync(grain, grainType, diagnostic); + } + + [Theory] + [InlineData(typeof(MetadataReentrantActivationValidationGrain), WellKnownGrainTypeProperties.Reentrant, "TrUe")] + [InlineData(typeof(MetadataMayInterleaveActivationValidationGrain), WellKnownGrainTypeProperties.MayInterleavePredicate, "Interleave")] + public async Task ResolvedInterleavingMetadata_FailsBeforeJournalRecovery(Type grainType, string key, string value) + { + Assert.False(grainType.IsDefined(typeof(ReentrantAttribute), inherit: true)); + Assert.False(grainType.IsDefined(typeof(MayInterleaveAttribute), inherit: true)); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + var properties = Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService() + .GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(value, properties.Properties[key]); + await AssertRejectedBeforeRecoveryAsync(grain, grainType, "non-reentrant"); + } + + private async Task AssertRejectedBeforeRecoveryAsync(IActivationValidationTestGrain grain, Type grainType, string diagnostic) + { + var failure = await Assert.ThrowsAnyAsync(() => grain.IncrementAsync()); + Assert.Contains(diagnostic, failure.ToString(), StringComparison.Ordinal); + Assert.Contains(grainType.Name, failure.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Fixture.ActivationProbe.Get(grain.GetGrainId())); + Assert.False(observation.InstanceAvailableDuringConstruction); + Assert.Equal(0, observation.Activated); + Assert.Equal(0, observation.Calls); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + Assert.Equal(0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, grain.GetGrainId())); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount("orleans.messaging.outbox-drain", grain.GetGrainId())); + } + + [Fact] + public async Task SupportedModel_InitializesAndReplaysFreshActivationNormally() + { + await AssertSupportedReplayAsync(typeof(SupportedActivationValidationTestGrain)); + } + + [Fact] + public async Task ResolvedNonReentrantMetadata_InitializesAndReplaysNormally() + { + await AssertSupportedReplayAsync(typeof(MetadataNonReentrantActivationValidationGrain)); + } + + private async Task AssertSupportedReplayAsync(Type grainType) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + Assert.Equal(1, await grain.IncrementAsync()); + var first = Assert.Single(Fixture.ActivationProbe.Get(grain.GetGrainId())); + Assert.False(first.InstanceAvailableDuringConstruction); + Assert.NotNull(first.Context.GrainInstance); + Assert.Equal(1, first.Activated); + Assert.Equal(1, first.Calls); + Assert.Equal(1, Fixture.Storage.GetReadCount(JournalId.FromGrainId(grain.GetGrainId()))); + await grain.DeactivateAsync(); + await first.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(2, await grain.IncrementAsync()); + var observations = Fixture.ActivationProbe.Get(grain.GetGrainId()); + Assert.Equal(2, observations.Length); + Assert.NotSame(first.Context, observations[1].Context); + Assert.Equal(1, observations[1].Activated); + Assert.Equal(1, observations[1].Calls); + Assert.Equal(2, Fixture.Storage.GetReadCount(JournalId.FromGrainId(grain.GetGrainId()))); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs new file mode 100644 index 00000000000..cc271252d8e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs @@ -0,0 +1,234 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxAcceptanceBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task Deliver_AcceptedOnlyAfterInboxAndStableJobOwnershipAreDurable() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var barrier = Fixture.Storage.BlockWrite(journalId); + var envelope = CreateEnvelope(receiver, NewMessage(1, "durability")); + + var delivery = DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + + Assert.False(delivery.IsCompleted); + var staged = Fixture.GetSnapshot(receiver); + Assert.Equal(1, staged.InboxCount); + Assert.Empty(staged.Effects); + var scheduledJob = Assert.Single( + Fixture.JobManagerProbe.GetScheduledJobs("orleans.messaging.inbox-drain", receiver.GetGrainId())); + Assert.Equal(staged.InboxJobId, scheduledJob.Metadata!["orleans.messaging.ownership-id"]); + Assert.Same(scheduledJob, staged.InboxJob); + Assert.Equal(scheduledJob.Id, staged.InboxJob?.Id); + Assert.Equal(scheduledJob.ShardId, staged.InboxJob?.ShardId); + + barrier.Release(); + var result = await delivery; + Assert.Equal(DeliveryStatus.Accepted, result.Status); + EndpointSnapshot completed; + try + { + completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Accepted message did not drain. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}, job={snapshot.InboxJobId}.", + exception); + } + Assert.Single(completed.Effects); + Assert.Equal(0, completed.InboxCount); + completed = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxJobId is null && snapshot.InboxJob is null); + Assert.Null(completed.InboxJob); + Assert.Equal(3, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + } + + [Fact] + public async Task Deliver_CancellationStopsWaitingForInboxGate() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var barrier = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var firstEnvelope = CreateEnvelope(receiver, NewMessage(72, "holds-gate")); + var secondEnvelope = CreateEnvelope(receiver, NewMessage(73, "canceled")); + var firstDelivery = DeliverAsync(receiver, firstEnvelope); + await barrier.WaitUntilEnteredAsync(); + using var cancellation = new CancellationTokenSource(); + + var canceledDelivery = DeliverWithCancellationAsync( + receiver, + secondEnvelope, + cancellation.Token); + Assert.False(canceledDelivery.IsCompleted); + cancellation.Cancel(); + + try + { + await Assert.ThrowsAnyAsync(() => canceledDelivery); + } + finally + { + barrier.Release(); + } + + Assert.Equal(DeliveryStatus.Accepted, (await firstDelivery).Status); + } + + [Fact] + public async Task ConcurrentWriteDuringLocalAcceptancePreparation_PreservesSafeStagingOnly() + { + var receiver = NewGrain(); + var effect = new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 74, "prior-safe-state"); + await receiver.StageEffectAsync(effect); + using var schedule = Fixture.JobManagerProbe.BlockNext("orleans.messaging.inbox-drain"); + var envelope = CreateEnvelope(receiver, NewMessage(75, "schedule-barrier")); + var delivery = DeliverAsync(receiver, envelope); + await schedule.WaitUntilEnteredAsync(); + var staged = Fixture.GetSnapshot(receiver); + Assert.Equal(0, staged.InboxCount); + Assert.Null(staged.InboxJobId); + Assert.Null(staged.InboxJob); + await Fixture.WriteStateAsync(receiver); + Assert.Equal(effect, Assert.Single(Fixture.GetSnapshot(receiver).Effects)); + Assert.False(delivery.IsCompleted); + schedule.Continue(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(new[] { "prior-safe-state", "schedule-barrier" }, completed.Effects.Select(static e => e.Value)); + } + + [Fact] + public async Task ConcurrentDuplicateDeliveries_ConvergeToOneEffectWithinRetention() + { + var receiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/blocked-duplicate"); + var envelope = CreateEnvelope(receiver, NewMessage(11, "duplicate"), "messages/blocked-duplicate"); + + var first = await DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + var second = DeliverAsync(receiver, envelope); + + Assert.Equal(DeliveryStatus.Accepted, first.Status); + Assert.False(second.IsCompleted); + Assert.Equal(1, Fixture.GetSnapshot(receiver).InboxCount); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await second).Status); + var state = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(state.Effects).Count); + Assert.Equal(1, state.MaxConcurrentHandlers); + } + + [Fact] + public async Task DuplicateAfterReactivationWithinRetention_RemainsEffectivelyOnce() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(13, "reactivation")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var before = await Fixture.WaitForEffectCountAsync(receiver, 1); + await receiver.RequestDeactivationAsync(); + var after = await receiver.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, after.ActivationId); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task SameMessageIdFromDistinctSenders_DeduplicatesOneReceiverCommand() + { + var receiver = NewGrain(); + var original = CreateEnvelope(receiver, NewMessage(14, "sender-scoped")); + var other = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("other-sender", "same-message-id")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, other)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, original)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, other)).Status); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task ReorderedDistinctAndDuplicateMessages_ConvergeByApplicationSequence() + { + var receiver = NewGrain(); + var messages = new[] + { + NewMessage(3, "third"), + NewMessage(1, "first"), + NewMessage(2, "second"), + }; + + foreach (var message in messages) + { + var envelope = CreateEnvelope(receiver, message); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + var state = await Fixture.WaitForEffectCountAsync(receiver, 3); + Assert.Equal([1, 2, 3], state.Effects.Select(static effect => effect.Sequence)); + Assert.All(state.Effects, static effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task ConcurrentDelivery_WaitsWhileHandlersRemainSequential() + { + var receiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/sequential"); + var first = CreateEnvelope(receiver, NewMessage(21, "first"), "messages/sequential"); + var second = CreateEnvelope(receiver, NewMessage(22, "second"), "messages/sequential"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await WaitForBarrierAsync(receiver, barrier); + var secondDelivery = DeliverAsync(receiver, second); + Assert.False(secondDelivery.IsCompleted); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await secondDelivery).Status); + EndpointSnapshot state; + try + { + state = await Fixture.WaitForEffectCountAsync(receiver, 2); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Second message did not complete. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + Assert.Equal(1, state.MaxConcurrentHandlers); + Assert.Equal([21, 22], state.Effects.Select(static effect => effect.Sequence)); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs new file mode 100644 index 00000000000..1f9aa7e260e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs @@ -0,0 +1,372 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxAdmissionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task ExplicitPreparation_AllowsPriorStateWriteThenCapturesAppliedCohortAtomically() + { + using var attempt = await PrepareAttemptAsync("atomic-admission"); + var state = attempt.Grain.GetSnapshotForTest(); + Assert.Equal(1, state.InboxCount); + Assert.Equal(0, state.ProcessedMessageCount); + Assert.Empty(state.Effects); + Assert.Empty(attempt.Outbox); + await OnTurnAsync(attempt.Context, () => + attempt.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "previous-state"); + await attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var priorCapture = attempt.Grain.Captures[^1]; + Assert.Empty(priorCapture.Effects); + Assert.Equal(1, priorCapture.InboxCount); + Assert.Equal(0, priorCapture.ProcessedMessageCount); + Assert.Equal(0, priorCapture.OutboxCount); + Assert.False(attempt.Grain.ApplyAttempted.Task.IsCompleted); + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + using var storage = Fixture.Storage.BlockWrite(attempt.JournalId); + attempt.Preparation.Release(); + await storage.WaitUntilEnteredAsync(); + var queued = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var coalesced = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(queued.IsCompleted); + Assert.False(coalesced.IsCompleted); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + + var captured = attempt.Grain.GetSnapshotForTest(); + Assert.Equal(0, captured.InboxCount); + Assert.Equal(1, captured.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(captured.Effects).Count); + Assert.Single(attempt.Grain.OutputCaptures[^1]); + Assert.False(queued.IsCompleted); + storage.Release(); + await Task.WhenAll(queued, coalesced); + await attempt.Receiver.RequestDeactivationAsync(); + var recovered = await attempt.Receiver.GetSnapshotAsync(); + Assert.NotEqual(state.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task HandlerStorageFailure_DeactivatesAndFailsQueuedWritesAfterPriorAcknowledgement() + { + using var attempt = await PrepareAttemptAsync("fault-ordering"); + var extension = attempt.Context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + await OnTurnAsync(attempt.Context, () => + { + attempt.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "prior-captured"; + }); + using var storage = Fixture.Storage.BlockWrite(attempt.JournalId); + var preceding = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(attempt.JournalId); + var queued = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + attempt.Preparation.Release(); + await attempt.Grain.ApplyAttempted.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await OnTurnAsync(attempt.Context, () => + { + Assert.Equal(1, attempt.Outbox.SendCalls); + Assert.Single(attempt.Grain.GetSnapshotForTest().Effects); + }); + Assert.False(queued.IsCompleted); + storage.Release(); + await preceding; + + var failure = await Assert.ThrowsAsync(() => queued); + Assert.Same(failure, await attempt.Grain.DeactivationFailure.Task); + await attempt.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Same(failure, await Assert.ThrowsAsync(async () => + await ((IInboxExtension)extension).DeliverAsync(attempt.Envelope, TestContext.Current.CancellationToken))); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + var failed = attempt.Grain.GetSnapshotForTest(); + Assert.Single(failed.Effects); + Assert.Equal(0, failed.InboxCount); + Assert.Equal(1, failed.ProcessedMessageCount); + _ = await attempt.Receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(attempt.Receiver, 1); + Assert.NotEqual(failed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task PhysicalCallbackMismatch_DuringPreparationPreservesCommittedOwner() + { + using var attempt = await PrepareAttemptAsync("owner-invalidation"); + var value = attempt.Context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-job-handle"); + var previous = value.Value!; + var mismatched = new DurableJob + { + Id = previous.Id, + ShardId = "different-physical-shard", + Metadata = previous.Metadata, + Name = previous.Name, + TargetGrainId = previous.TargetGrainId, + DueTime = previous.DueTime + }; + var feature = (IDurableJobFeatureHandler)attempt.Context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + DurableJobRunResult result = null!; + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + await OnTurnTaskAsync(attempt.Context, async () => + result = await feature.ExecuteJobAsync(new CallbackContext(mismatched), TestContext.Current.CancellationToken)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Same(previous, value.Value); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + var pending = attempt.Grain.GetSnapshotForTest(); + Assert.Empty(pending.Effects); + Assert.Equal(1, pending.InboxCount); + Assert.Equal(0, pending.ProcessedMessageCount); + Assert.False(attempt.Grain.ApplyAttempted.Task.IsCompleted); + Assert.Empty(attempt.Outbox.Messages); + attempt.Preparation.Release(); + var recovered = await Fixture.WaitForEffectCountAsync(attempt.Receiver, 1); + Assert.Equal(pending.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Empty(recovered.InboxDeadLetters); + } + + [Fact] + public async Task LateOutgoingIntent_AfterCapture_WaitsForNextCapture() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var grain = Assert.IsType(context.GrainInstance); + var durable = context.ActivationServices.GetRequiredKeyedService>("test-handler-output"); + var sessions = Fixture.Client.ServiceProvider.GetRequiredService(); + var first = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 1); + var late = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 2); + await receiver.StageOutputAsync(first); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var write = Fixture.WriteStateAsync(receiver).AsTask(); + await storage.WaitUntilEnteredAsync(); + await OnTurnTaskAsync(context, async () => + { + outbox.Send(late); + }); + Assert.Equal(new[] { first.Envelope.MessageId }, grain.OutputCaptures[^1]); + Assert.Equal(2, outbox.Count); + storage.Release(); + await write; + Assert.Equal(new[] { first.Envelope.MessageId }, grain.OutputCaptures[^1]); + await receiver.RetryWriteStateAsync(); + Assert.Equal(new[] { first.Envelope.MessageId, late.Envelope.MessageId }.OrderBy(static key => key.ToString(), StringComparer.Ordinal), + grain.OutputCaptures[^1].OrderBy(static key => key.ToString(), StringComparer.Ordinal)); + Assert.Equal(2, durable.Count); + await receiver.RequestDeactivationAsync(); + Assert.Equal(2, (await receiver.GetSnapshotAsync()).OutboxCount); + } + + [Fact] + public async Task AcceptanceAfterCapture_IsAcknowledgedByLaterOperation() + { + var receiver = NewGrain(); + var seed = CreateEnvelope(receiver, NewMessage(110, "seed"), "messages/cutoff"); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-capture-cutoff"); + var turn = receiver.HoldPumpTurnAsync("hold-capture-cutoff", deactivate: false); + await hold.WaitUntilEnteredAsync(); + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + Task initial = null!; + await OnTurnAsync(context, () => initial = extension.DeliverAsync(seed, TestContext.Current.CancellationToken).AsTask()); + Assert.Equal(DeliveryStatus.Accepted, (await initial).Status); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + grain.ClearCaptures(); + await OnTurnAsync(context, () => context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "capture-cutoff"); + var preceding = Fixture.WriteStateAsync(receiver).AsTask(); + await storage.WaitUntilEnteredAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/cutoff"); + var incoming = CreateEnvelope(receiver, NewMessage(111, "late-acceptance"), "messages/cutoff"); + Task delivery = null!; + await OnTurnAsync(context, () => delivery = extension.DeliverAsync(incoming).AsTask()); + Assert.False(delivery.IsCompleted); + Assert.Equal(2, grain.GetSnapshotForTest().InboxCount); + storage.Release(); + await preceding; + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + Assert.Equal(1, grain.Captures[0].InboxCount); + Assert.Equal(2, grain.Captures[1].InboxCount); + Assert.Equal(2, grain.GetSnapshotForTest().InboxCount); + hold.Release(); + await turn; + await handler.WaitUntilEnteredAsync(); + handler.Release(); + Assert.Equal(2, (await Fixture.WaitForEffectCountAsync(receiver, 2)).ProcessedMessageCount); + } + + [Fact] + public async Task CallerWaitCancellation_DoesNotWithdrawCapturedAcceptance() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var storage = Fixture.Storage.BlockWrite(journal); + var envelope = CreateEnvelope(receiver, NewMessage(112, "cancel-wait")); + using var cancellation = new CancellationTokenSource(); + var delivery = DeliverWithCancellationAsync(receiver, envelope, cancellation.Token); + await storage.WaitUntilEnteredAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery); + storage.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task OwnedTimerCancellation_DuringHandlerPreparation_PreservesActivationAndRetries() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(113, "attempt-cancel"), "messages/attempt-cancel"); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/attempt-cancel"); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var timer = Assert.Single(events.Events.Select(item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, context) + && item.Timer.GetType().GenericTypeArguments is [var state] + && state.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension")).Timer; + await OnTurnAsync(context, timer.Dispose); + await events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + Assert.False(context.Deactivated.IsCompleted); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.Equal(1, grain.GetSnapshotForTest().InboxCount); + handler.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task MessagingPrimaryStates_AreTheRegisteredScopedDataInstances() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(receiver).ActivationServices; + var manager = services.GetRequiredService(); + Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox", out var inbox)); + Assert.Same(services.GetRequiredKeyedService>("__orleans.messaging.inbox"), inbox); + Assert.True(manager.TryGetStateMachine("test-handler-output", out var output)); + Assert.Same(((JournaledTestOutbox)services.GetRequiredService()).StoredMessages, output); + Assert.Same(services.GetRequiredKeyedService>("test-handler-output"), output); + } + + [Fact] + public async Task AcceptanceWriteFailure_DeactivatesAndFreshScopeRetries() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Fixture.Storage.FailWrite(journal); + var envelope = CreateEnvelope(receiver, NewMessage(114, "acceptance-write-failure")); + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Contains("Injected journal write failure", failure.Message, StringComparison.Ordinal); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var rejected = grain.GetSnapshotForTest(); + Assert.Equal(before.ActivationId, rejected.ActivationId); + Assert.Equal(1, rejected.InboxCount); + Assert.NotNull(rejected.InboxJobId); + Assert.NotNull(rejected.InboxJob); + Assert.Empty(rejected.Effects); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var deactivation = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.IsType(deactivation); + Assert.Equal(failure.Message, deactivation.Message); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(rejected.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Null(recovered.InboxJobId); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + var jobs = Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId()); + Assert.Equal(2, jobs.Count); + Assert.NotEqual(jobs[0].Metadata!["orleans.messaging.ownership-id"], jobs[1].Metadata!["orleans.messaging.ownership-id"]); + } + + private async Task PrepareAttemptAsync(string routeSuffix) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var route = "messages/" + routeSuffix; + var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(109, routeSuffix) with { ForwardTo = GrainId.Create("output", routeSuffix) }, route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var preparation = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route + "/application-preparation"); + handler.Release(); + await preparation.WaitUntilEnteredAsync(); + return new(receiver, context, outbox, handler, preparation, envelope); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completion.SetResult(); } + catch (Exception exception) { completion.SetException(exception); } + }); + return completion.Task; + } + + private static Task OnTurnTaskAsync(IGrainContext context, Func action) + { + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(action()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private sealed record Attempt(IMessagingTestGrain Receiver, IGrainContext Context, JournaledTestOutbox Outbox, + HandlerProbe.Barrier Handler, HandlerProbe.Barrier Preparation, InboxMessage Envelope) : IDisposable + { + public MessagingTestGrain Grain { get; } = Assert.IsType(Context.GrainInstance); + public IJournaledStateManager Manager => Context.ActivationServices.GetRequiredService(); + public JournalId JournalId => JournalId.FromGrainId(Receiver.GetGrainId()); + public void Dispose() { Preparation.Dispose(); Handler.Dispose(); } + } + + private sealed class CallbackContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCallbackOwnershipTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCallbackOwnershipTests.cs new file mode 100644 index 00000000000..fb5882609dc --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCallbackOwnershipTests.cs @@ -0,0 +1,295 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCallbackOwnershipTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task MissingOwnerWithWork_RepairsScheduleAndCommitBeforeOrphanRetires() + { + var receiver = NewGrain(); + const string route = "messages/repair-owner"; + var envelope = CreateEnvelope(receiver, NewMessage(90, "repaired"), route); + await receiver.SeedInboxStateAsync(envelope, null, null); + var oldContext = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var orphan = CreateJob(receiver, "orphan", "old-shard", "old:1"); + using var schedule = Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName); + using var write = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + + var activation = receiver.GetSnapshotAsync(); + await schedule.WaitUntilEnteredAsync(); + var extension = GetExtension(receiver); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, orphan)).Status); + Assert.Empty(Fixture.GetSnapshot(receiver).Effects); + Assert.Empty(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + + schedule.Continue(); + await write.WaitUntilEnteredAsync(); + var scheduled = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, orphan)).Status); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, scheduled)).Status); + Assert.Empty(Fixture.GetSnapshot(receiver).Effects); + + write.Release(); + await activation; + await handler.WaitUntilEnteredAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await ExecuteAsync(extension, orphan)).Status); + var owned = Fixture.GetSnapshot(receiver); + Assert.Same(scheduled, owned.InboxJob); + Assert.Equal(scheduled.Metadata!["orleans.messaging.ownership-id"], owned.InboxJobId); + Assert.Equal(1, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + handler.Release(); + + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + } + + [Theory] + [InlineData("generation-only")] + [InlineData("handle-only")] + [InlineData("mismatched-metadata")] + public async Task RecoveredInvalidPair_FailsStartupAndStopsCallbacksWithoutRepair(string fault) + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(91, fault)); + var handle = CreateJob(receiver, "physical", "shard", fault == "mismatched-metadata" ? "other:1" : "owner:1"); + await receiver.SeedInboxStateAsync( + envelope, + fault == "handle-only" ? null : "owner:1", + fault == "generation-only" ? null : handle); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journalId); + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + using var read = Fixture.Storage.BlockRead(journalId); + var activation = receiver.GetSnapshotAsync(); + await read.WaitUntilEnteredAsync(); + var extension = GetExtension(receiver); + var current = Fixture.GetGrainContext(receiver); + read.Release(); + var lifecycle = await Assert.ThrowsAnyAsync(() => activation); + await current.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Assert.ThrowsAnyAsync(async () => await ExecuteAsync(extension, handle)); + var delivery = await Assert.ThrowsAsync(() => + OnTurnAsync(extension.Context, () => + ((IInboxExtension)extension.Handler).DeliverAsync(envelope, TestContext.Current.CancellationToken))); + Assert.Contains(delivery.Message, lifecycle.ToString(), StringComparison.Ordinal); + Assert.Contains(fault == "mismatched-metadata" ? "metadata does not match" : "both be present or both be absent", delivery.Message, StringComparison.Ordinal); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + var replayFailure = await Assert.ThrowsAnyAsync(() => receiver.GetSnapshotAsync()); + Assert.Contains(delivery.Message, replayFailure.ToString(), StringComparison.Ordinal); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + } + + [Theory] + [InlineData("different-id")] + [InlineData("different-shard")] + [InlineData("missing-metadata")] + public async Task NonAuthoritativeCallback_RetiresWithoutChangingCommittedOwner(string fault) + { + var receiver = NewGrain(); + const string route = "messages/physical-owner"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(92, fault), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var owned = Fixture.GetSnapshot(receiver); + var handle = Assert.IsType(owned.InboxJob); + var callback = CreateJob( + receiver, + fault == "different-id" ? "other-id" : handle.Id, + fault == "different-shard" ? "other-shard" : handle.ShardId, + fault == "missing-metadata" ? null : owned.InboxJobId); + + var result = await ExecuteAsync(GetExtension(receiver), callback); + + Assert.Equal(DurableJobRunStatus.Completed, result.Status); + Assert.Same(handle, Fixture.GetSnapshot(receiver).InboxJob); + Assert.Equal(owned.InboxJobId, Fixture.GetSnapshot(receiver).InboxJobId); + Assert.Empty(Fixture.GetSnapshot(receiver).Effects); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.MaxConcurrentHandlers); + } + + [Fact] + public async Task CallbacksDuringFreshInitialization_WaitUntilCommittedOwnerIsRestored() + { + var receiver = NewGrain(); + const string route = "messages/callback-initialization"; + var envelope = CreateEnvelope(receiver, NewMessage(93, "initialization"), route); + var handle = CreateJob(receiver, "committed-physical", "committed-shard", "owner:1"); + await receiver.SeedInboxStateAsync(envelope, "owner:1", handle); + var oldContext = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + using var read = Fixture.Storage.BlockRead(JournalId.FromGrainId(receiver.GetGrainId())); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var activation = receiver.GetSnapshotAsync(); + await read.WaitUntilEnteredAsync(); + var extension = GetExtension(receiver); + var orphan = CreateJob(receiver, "orphan", "orphan-shard", "stale:1"); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, handle)).Status); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, orphan)).Status); + read.Release(); + await activation; + await handler.WaitUntilEnteredAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await ExecuteAsync(extension, orphan)).Status); + var recovered = Fixture.GetSnapshot(receiver); + Assert.Equal(handle.Id, recovered.InboxJob?.Id); + Assert.Equal(handle.ShardId, recovered.InboxJob?.ShardId); + handler.Release(); + await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(DurableJobRunStatus.Completed, (await ExecuteAsync(extension, orphan)).Status); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + } + + [Fact] + public async Task DuplicateScheduledJobs_RetireExtraHandleAfterAcceptanceCommits() + { + var receiver = NewGrain(); + const string route = "messages/duplicate-jobs"; + _ = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(94, "duplicate-jobs"), route); + Fixture.JobManagerProbe.DuplicateNext(ReceiverTestServices.InboxJobName); + using var write = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var delivery = DeliverAsync(receiver, envelope); + await write.WaitUntilEnteredAsync(); + var jobs = Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId()); + Assert.Equal(2, jobs.Count); + var extension = GetExtension(receiver); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, jobs[0])).Status); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, jobs[1])).Status); + Assert.False(delivery.IsCompleted); + write.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await handler.WaitUntilEnteredAsync(); + + Assert.Equal(DurableJobRunStatus.Completed, (await ExecuteAsync(extension, jobs[1])).Status); + Assert.Same(jobs[0], Fixture.GetSnapshot(receiver).InboxJob); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.MaxConcurrentHandlers); + } + + [Fact] + public async Task PreviousOwnerCallback_DuringReplacementPreparation_QuiescesUntilAcceptanceCommits() + { + var receiver = NewGrain(); + var oldJob = CreateJob(receiver, "previous-physical", "previous-shard", "previous:1"); + await receiver.SetInboxOwnershipAsync("previous:1", oldJob); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId())); + using var schedule = Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/replacement-quiescence"); + var envelope = CreateEnvelope(receiver, NewMessage(116, "replacement-quiescence"), "messages/replacement-quiescence"); + using var timers = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var delivery = DeliverAsync(receiver, envelope); + await schedule.WaitUntilEnteredAsync(); + + var callback = await InvokeDurableCallbackAsync(receiver, oldJob); + + Assert.Equal(DurableJobRunStatus.InProgress, callback.Status); + Assert.DoesNotContain(timers.Events, item => item.Payload is GrainTimerEvents.Created created + && ReferenceEquals(created.GrainContext, context)); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.False(delivery.IsCompleted); + Assert.Equal(oldJob.Id, Fixture.GetSnapshot(receiver).InboxJob?.Id); + schedule.Continue(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await handler.WaitUntilEnteredAsync(); + var accepted = Fixture.GetSnapshot(receiver); + var replacement = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + Assert.Same(replacement, accepted.InboxJob); + Assert.Equal(replacement.Metadata!["orleans.messaging.ownership-id"], accepted.InboxJobId); + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeDurableCallbackAsync(receiver, oldJob, dequeueCount: 2)).Status); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + } + + private static async Task InvokeDurableCallbackAsync(IMessagingTestGrain receiver, DurableJob job, int dequeueCount = 1) + { + var assembly = typeof(DurableJob).Assembly; + var receiverType = assembly.GetType("Orleans.DurableJobs.IDurableJobReceiverExtension", throwOnError: true)!; + var reference = receiver.AsReference(receiverType); + var run = Activator.CreateInstance(assembly.GetType("Orleans.DurableJobs.JobRunContext", throwOnError: true)!, + job, Guid.NewGuid().ToString("N"), dequeueCount)!; + return await (ValueTask)receiverType.GetMethod("HandleDurableJobAsync")! + .Invoke(reference, [run, TestContext.Current.CancellationToken])!; + } + + private (IGrainContext Context, IDurableJobFeatureHandler Handler) GetExtension(IMessagingTestGrain receiver) + { + var context = Fixture.GetGrainContext(receiver); + return (context, (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension"))); + } + + private static Task ExecuteAsync( + (IGrainContext Context, IDurableJobFeatureHandler Handler) extension, DurableJob job) + { + var cancellationToken = TestContext.Current.CancellationToken; + return OnTurnAsync(extension.Context, () => + extension.Handler.ExecuteJobAsync(new CallbackContext(job), cancellationToken)); + } + + private static Task OnTurnAsync(IGrainContext context, Func> action) + { + var task = new Task>(async () => + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + var result = await action(); + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + return result; + }); + context.Scheduler.QueueTask(task); + return task.Unwrap(); + } + + private static DurableJob CreateJob(IMessagingTestGrain receiver, string id, string shardId, string? ownershipId) => + new() + { + Id = id, + ShardId = shardId, + Name = ReceiverTestServices.InboxJobName, + TargetGrainId = receiver.GetGrainId(), + DueTime = DateTimeOffset.UtcNow, + Metadata = ownershipId is null ? null : new Dictionary + { + ["orleans.messaging.ownership-id"] = ownershipId + } + }; + + private sealed class CallbackContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCancellationCleanupTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCancellationCleanupTests.cs new file mode 100644 index 00000000000..8d2a670eb50 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCancellationCleanupTests.cs @@ -0,0 +1,306 @@ +using System.Collections; +using System.Collections.Concurrent; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCancellationCleanupTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task HandlerCancellation_CallbackFailurePreservesCleanupAndOriginalCause(bool throws, bool terminalFailure) + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var shutdown = CancellationCleanupProbe.Field(extension, "_shutdownCts"); + var token = shutdown.Token; + using var logs = new CancellationCleanupProbe.Logs(); + Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService().AddProvider(logs); + using var handler = new CancelingHandler(throws); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + var envelope = CreateEnvelope(receiver, NewMessage(310, "cancellation"), "cancel-cleanup"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId())); + IOException? firstFailure = null; + using var seeded = await OnTurnAsync(context, () => CancellationCleanupProbe.SeedResults(extension)); + if (terminalFailure) + { + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var failingInput = CreateEnvelope(receiver, NewMessage(311, "capture-failure")); + var delivery = await OnTurnAsync(context, () => + ((IInboxExtension)extension).DeliverAsync(failingInput, TestContext.Current.CancellationToken).AsTask()); + firstFailure = await Assert.ThrowsAsync(() => delivery); + } + Task stopping = null!; + await OnTurnAsync(context, () => + { + if (!terminalFailure) Assert.True(CancellationCleanupProbe.CoordinatorIsActive(extension)); + stopping = ((ILifecycleObserver)extension).OnStop(CancellationToken.None); + Assert.False(stopping.IsCompleted); + CancellationCleanupProbe.AssertClean(extension, seeded); + Assert.True(shutdown.Token.IsCancellationRequested); + Assert.Equal(1, handler.CallbackCalls); + logs.AssertCallback(throws, handler.CallbackFailure); + if (terminalFailure) Assert.Same(firstFailure, CancellationCleanupProbe.Field(extension, "_failure").SourceException); + else Assert.Null(CancellationCleanupProbe.Field(extension, "_failure")); + }); + Assert.True(token.IsCancellationRequested); + Assert.False(handler.Finished); + handler.Release.TrySetResult(); + await stopping.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Exception failure; + if (terminalFailure) + { + failure = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Same(firstFailure, failure); + } + else + { + failure = Assert.IsType(handler.Cancellation); + Assert.Null(CancellationCleanupProbe.Field(extension, "_failure")); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + await receiver.RequestDeactivationAsync(); + } + Assert.DoesNotContain(handler.CallbackFailure, failure is AggregateException aggregate ? aggregate.Flatten().InnerExceptions : [failure]); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.True(handler.Finished); + Assert.Throws(() => shutdown.Token); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.Equal(1, CancellationCleanupProbe.Field(extension, "_gate").CurrentCount); + Assert.Empty(CancellationCleanupProbe.Field(extension, "_pendingWrites")); + ((IDisposable)extension).Dispose(); + Assert.Equal(1, handler.CallbackCalls); + logs.AssertCallback(throws, handler.CallbackFailure); + await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + private sealed class CancelingHandler(bool throws) : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Exception CallbackFailure { get; } = new InvalidOperationException("Application cancellation callback failed."); + public int CallbackCalls { get; private set; } + public bool Finished { get; private set; } + public OperationCanceledException? Cancellation { get; private set; } + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + using var registration = cancellationToken.Register(() => + { + CallbackCalls++; + if (throws) throw CallbackFailure; + }); + Entered.TrySetResult(); + try + { + await Release.Task; + cancellationToken.ThrowIfCancellationRequested(); + throw new InvalidOperationException("A canceled handler must never complete."); + } + catch (OperationCanceledException exception) + { + Cancellation = exception; + throw; + } + finally { Finished = true; } + } + public void Dispose() => Release.TrySetResult(); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => OnTurnAsync(context, () => { action(); return true; }); + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { done.SetResult(action()); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCancellationCleanupContractTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Dispose_CallbackFailureStillReleasesOwnedResourcesOnce(bool throws) + { + using var probe = new CancellationCleanupProbe(throws); + ((IDisposable)probe.Extension).Dispose(); + probe.AssertClean(); + Assert.Throws(() => probe.Shutdown.Token); + ((IDisposable)probe.Extension).Dispose(); + probe.AssertClean(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OnStop_CallbackFailureStillDrainsOwnedDelivery(bool throws) + { + using var probe = new CancellationCleanupProbe(throws); + var owned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationCleanupProbe.SetField(probe.Extension, "_activeDelivery", owned.Task); + var stopping = ((ILifecycleObserver)probe.Extension).OnStop(CancellationToken.None); + Assert.False(stopping.IsCompleted); + probe.AssertClean(); + Assert.True(probe.Shutdown.Token.CanBeCanceled); + owned.SetException(new IOException("Already observed delivery failure.")); + await stopping; + ((IDisposable)probe.Extension).Dispose(); + Assert.Throws(() => probe.Shutdown.Token); + probe.AssertClean(); + } +} + +internal sealed class CancellationCleanupProbe : IDisposable +{ + internal static readonly Type ExtensionType = ReceiverTestServices.GetImplementationType("InboxExtension"); + private readonly CancellationTokenSource _handlerCancellation; + private readonly CancellationTokenRegistration _registration; + private readonly ILoggerFactory _loggerFactory; + private readonly bool _throws; + private readonly Seeded _seeded; + private readonly Logs _logs = new(); + private readonly Exception _callbackFailure = new InvalidOperationException("Linked handler cancellation failed."); + private int _calls; + public object Extension { get; } = RuntimeHelpers.GetUninitializedObject(ExtensionType); + public CancellationTokenSource Shutdown { get; } = new(); + public CancellationCleanupProbe(bool throws) + { + _throws = throws; + _loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(_logs)); + SetField(Extension, "_logger", Activator.CreateInstance(typeof(Logger<>).MakeGenericType(ExtensionType), _loggerFactory)!); + SetField(Extension, "_shutdownCts", Shutdown); + SetField(Extension, "_shutdownToken", Shutdown.Token); + SetField(Extension, "_pumpCoordinator", Activator.CreateInstance(ReceiverTestServices.GetImplementationType("MessagingPumpCoordinator"))!); + SetField(Extension, "_pumpResults", Activator.CreateInstance(ReceiverTestServices.GetImplementationType("MessagingPumpResults"), nonPublic: true)!); + var instruments = ReceiverTestServices.GetImplementationType("MessagingInstruments") + .GetMethod("CreateForDirectConstruction", BindingFlags.NonPublic | BindingFlags.Static)!.Invoke(null, null)!; + SetField(Extension, "_instruments", instruments); + instruments.GetType().GetMethod("OnInboxDepthChanged", BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(instruments, [1]); + SetField(Extension, "_metricsActive", 1); + SetField(Extension, "_reportedDepth", 1); + SetField(Extension, "_activeDelivery", Task.CompletedTask); + SetField(Extension, "_provisionalAcceptances", Activator.CreateInstance( + ExtensionType.GetField("_provisionalAcceptances", BindingFlags.NonPublic | BindingFlags.Instance)!.FieldType)!); + SetField(Extension, "_pendingWrites", Activator.CreateInstance( + ExtensionType.GetField("_pendingWrites", BindingFlags.NonPublic | BindingFlags.Instance)!.FieldType)!); + var coordinator = Field(Extension, "_pumpCoordinator"); + object?[] arguments = ["owner", Shutdown.Token, null]; + Assert.True((bool)coordinator.GetType().GetMethod("TryAcquire")!.Invoke(coordinator, arguments)!); + _seeded = SeedResults(Extension); + _handlerCancellation = CancellationTokenSource.CreateLinkedTokenSource(Shutdown.Token); + _registration = _handlerCancellation.Token.Register(() => { _calls++; if (_throws) throw _callbackFailure; }); + } + public void AssertClean() + { + Assert.True(Shutdown.IsCancellationRequested); + Assert.True(_handlerCancellation.IsCancellationRequested); + Assert.Equal(1, _calls); + AssertClean(Extension, _seeded); + _logs.AssertCallback(_throws, _callbackFailure); + } + public static T Field(object instance, string name) => (T)ExtensionType.GetField(name, BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(instance)!; + public static void SetField(object instance, string name, object value) => ExtensionType.GetField(name, BindingFlags.NonPublic | BindingFlags.Instance)!.SetValue(instance, value); + public static bool CoordinatorIsActive(object extension) + { + var coordinator = Field(extension, "_pumpCoordinator"); + return (bool)coordinator.GetType().GetProperty("IsActive")!.GetValue(coordinator)!; + } + public static Seeded SeedResults(object extension) + { + var results = Field(extension, "_pumpResults"); + var keyType = ReceiverTestServices.GetImplementationType("MessagingPumpExecutionKey"); + var entries = (IDictionary)results.GetType().GetField("_entries", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(results)!; + object? captured = null; + var cancellation = new CancellationTokenSource(); + foreach (var name in new[] { ReceiverTestServices.InboxJobName, "test/other-pump" }) + { + var key = Activator.CreateInstance(keyType, name, "cleanup", "run", 1L)!; + object?[] arguments = [key, cancellation.Token, null]; + Assert.True((bool)results.GetType().GetMethod("TryStart")!.Invoke(results, arguments)!); + if (name == ReceiverTestServices.InboxJobName) captured = entries[key]; + } + var registration = (CancellationTokenRegistration)captured!.GetType().GetProperty("CancellationRegistration")!.GetValue(captured)!; + Assert.NotEqual(default, registration); + return new Seeded(cancellation, captured); + } + public static void AssertClean(object extension, Seeded seeded) + { + Assert.False(CoordinatorIsActive(extension)); + Assert.Equal(0, Field(extension, "_metricsActive")); + Assert.Equal(0, Field(extension, "_reportedDepth")); + var instruments = Field(extension, "_instruments"); + var depth = instruments.GetType().GetField("_inboxDepth", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instruments)!; + Assert.Equal(0L, depth.GetType().GetField("_value", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(depth)); + var results = Field(extension, "_pumpResults"); + var entries = (IDictionary)results.GetType().GetField("_entries", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(results)!; + var key = Assert.Single(entries.Keys.Cast()); + Assert.Equal("test/other-pump", key.GetType().GetProperty("JobName")!.GetValue(key)); + Assert.Equal(default(CancellationTokenRegistration), seeded.Entry.GetType().GetProperty("CancellationRegistration")!.GetValue(seeded.Entry)); + } + public void Dispose() + { + _seeded.Dispose(); + _registration.Dispose(); + _handlerCancellation.Dispose(); + Shutdown.Dispose(); + _loggerFactory.Dispose(); + } + internal sealed record Seeded(CancellationTokenSource Cancellation, object Entry) : IDisposable + { + public void Dispose() => Cancellation.Dispose(); + } + + internal sealed class Logs : ILoggerProvider + { + private readonly ConcurrentQueue _exceptions = new(); + public ILogger CreateLogger(string categoryName) => new Sink(this, categoryName); + public void Dispose() { } + public void AssertCallback(bool throws, Exception failure) + { + if (throws) Assert.Same(failure, Assert.Single(Assert.IsType(Assert.Single(_exceptions)).Flatten().InnerExceptions)); + else Assert.Empty(_exceptions); + } + private sealed class Sink(Logs owner, string category) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Error; + public void Log(LogLevel level, EventId id, TState state, Exception? exception, Func formatter) + { + if (category == "Orleans.Messaging.InboxExtension" && id.Name == "LogCancellationCallbackFailure" && exception is not null) + owner._exceptions.Enqueue(exception); + } + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs new file mode 100644 index 00000000000..abe8cb45fd4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs @@ -0,0 +1,113 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class InboxCapacityCollection : ICollectionFixture +{ + public const string Name = "Messaging inbox capacity"; +} + +[Collection(InboxCapacityCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCapacityBehaviorTests(InboxCapacityClusterFixture fixture) +{ + [Fact] + public async Task InboxAtCapacity_BackpressuresWithoutPersistenceAndRecoversWhenCapacityFrees() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("capacity-test-sender", Guid.NewGuid().ToString("N")); + var poison = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/capacity", new TestMessage(TestApplicationProtocol.NewMessageId(), 31, "poison", ThrowDuringPreparation: true)); + var rejected = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/capacity", new TestMessage(TestApplicationProtocol.NewMessageId(), 32, "accepted-after-capacity")); + const string processedInstrument = "orleans-messaging-inbox-messages-processed"; + var firstRetryAcknowledged = fixture.Metrics.WaitForCountAsync(processedInstrument, 1, "retry"); + using var timers = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, poison)).Status); + var full = await fixture.WaitForInboxCountAsync(receiver, 1); + Assert.Empty(full.Effects); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, rejected)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + + await firstRetryAcknowledged; + Assert.Equal(1, fixture.Metrics.GetCount(processedInstrument, "retry")); + var originalContext = fixture.GetGrainContext(receiver); + var timer = Assert.Single(timers.Events.Select(static item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, originalContext) + && item.Timer.GetType().GenericTypeArguments is [var state] + && state.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension") + && state.Name == "LocalDrainTimerState").Timer; + var stopped = await timers.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Null(Assert.IsType(stopped.Payload).Exception); + DateTimeOffset nextAttempt = default; + await OnTurnAsync(originalContext, () => + { + var stateType = ReceiverTestServices.GetImplementationType("InboxMessageState"); + var statesType = typeof(IDurableDictionary<,>).MakeGenericType(typeof(HierarchicalKey), stateType); + var states = originalContext.ActivationServices.GetRequiredKeyedService( + statesType, "__orleans.messaging.inbox-message-state"); + var dictionaryType = typeof(IDictionary<,>).MakeGenericType(typeof(HierarchicalKey), stateType); + var state = dictionaryType.GetProperty("Item")!.GetValue(states, [poison.Envelope.MessageId])!; + Assert.Equal(1, stateType.GetProperty("AttemptCount")!.GetValue(state)); + nextAttempt = Assert.IsType(stateType.GetProperty("NextAttemptAt")!.GetValue(state)); + Assert.Equal(fixture.Clock.GetUtcNow() + TimeSpan.FromHours(1), nextAttempt); + }); + fixture.Clock.Advance(TimeSpan.FromHours(2)); + Assert.True(nextAttempt <= fixture.Clock.GetUtcNow()); + var previous = fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + Assert.NotSame(previous, fixture.GetGrainContext(receiver)); + var current = fixture.GetGrainContext(receiver); + var currentGrain = Assert.IsType(current.GrainInstance); + try + { + await fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId == currentGrain.GetSnapshotForTest().ActivationId + && snapshot.InboxDeadLetters.Count == 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Recovered retry did not complete. Activation={snapshot.ActivationId}, inbox={snapshot.InboxCount}, " + + $"processed={snapshot.ProcessedMessageCount}, deadLetters={snapshot.InboxDeadLetters.Count}, " + + $"nextAttempt={nextAttempt:O}, now={fixture.Clock.GetUtcNow():O}, job={snapshot.InboxJobId}, " + + $"failureCompleted={currentGrain.DeactivationFailure.Task.IsCompleted}.", exception); + } + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, rejected)).Status); + var recovered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("accepted-after-capacity", Assert.Single(recovered.Effects).Value); + Assert.Equal(2, Assert.Single(recovered.InboxDeadLetters).AttemptCount); + } + + private static async Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + await receiver.AsReference().DeliverAsync(envelope); + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var finished = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); finished.SetResult(); } + catch (Exception exception) { finished.SetException(exception); } + }); + return finished.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCommittedCountTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCommittedCountTests.cs new file mode 100644 index 00000000000..e9264a683d1 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCommittedCountTests.cs @@ -0,0 +1,382 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCommittedCountTests() : MessagingBehaviorTestBase(new CountFixture()) +{ + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(1)] + [InlineData(32)] + [InlineData(128)] + public async Task QueuedDeliveryBurst_CountChecksPerformConstantCollectionWork(int messages) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var probe = new CountProbe(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-count-burst"); + var turn = receiver.HoldPumpTurnAsync("hold-count-burst", deactivate: false); + await hold.WaitUntilEnteredAsync(); + try + { + for (var i = 0; i < messages; i++) + { + var envelope = CreateEnvelope(receiver, NewMessage(i, "queued-count")); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery(context, probe.Extension, envelope, Cancellation)).Status); + } + await OnTurnAsync(context, () => + { + Assert.Equal(new Counts(messages, 0, messages), probe.Read()); + Assert.Equal(0, probe.Dictionary.KeyCollections); + Assert.Equal(0, probe.Dictionary.KeyVisits); + Assert.Equal(0, probe.Dictionary.EntryEnumerations); + var reads = probe.Dictionary.CountReads; + Assert.Equal(messages, probe.DurableCount()); + Assert.Equal(reads + 1, probe.Dictionary.CountReads); + }); + Assert.Equal(1, ScheduleCount(receiver)); + } + finally + { + hold.Release(); + } + await turn; + await Fixture.WaitForEffectCountAsync(receiver, messages); + _ = await receiver.GetSnapshotAsync(); + await AssertCountsAsync(context, probe, new(0, 0, 0)); + } + + [Theory] + [InlineData(0)] + [InlineData(2)] + public async Task AcceptanceCount_TracksPreparationCommitCancellationAndAcknowledgement(int existing) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var probe = new CountProbe(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-count-phases"); + var turn = receiver.HoldPumpTurnAsync("hold-count-phases", deactivate: false); + await hold.WaitUntilEnteredAsync(); + try + { + for (var i = 0; i < existing; i++) + { + var queued = CreateEnvelope(receiver, NewMessage(i, "existing")); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery(context, probe.Extension, queued, Cancellation)).Status); + } + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var storage = Fixture.Storage.BlockWrite(journal); + try + { + using var scheduling = existing == 0 ? Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName) : null; + var envelope = CreateEnvelope(receiver, NewMessage(140, "pending-count")); + using var cancellation = new CancellationTokenSource(); + var delivery = StartDelivery(context, probe.Extension, envelope, cancellation.Token); + if (scheduling is not null) + { + await scheduling.WaitUntilEnteredAsync(); + await AssertCountsAsync(context, probe, new(0, 0, 0)); + scheduling.Continue(); + } + await storage.WaitUntilEnteredAsync(); + await AssertCountsAsync(context, probe, new(existing + 1, 1, existing)); + var duplicate = StartDelivery(context, probe.Extension, envelope, Cancellation); + Assert.False(delivery.IsCompleted); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery); + await AssertCountsAsync(context, probe, new(existing + 1, 1, existing)); + Assert.False(duplicate.IsCompleted); + storage.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + await AssertCountsAsync(context, probe, new(existing + 1, 0, existing + 1)); + Assert.Equal(1, ScheduleCount(receiver)); + } + finally + { + storage.Release(); + } + } + finally + { + hold.Release(); + } + await turn; + await Fixture.WaitForEffectCountAsync(receiver, existing + 1); + _ = await receiver.GetSnapshotAsync(); + await AssertCountsAsync(context, probe, new(0, 0, 0)); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + Assert.Equal(DurableJobRunStatus.Completed, (await RunPumpAsync(context, probe.Extension, job)).Status); + var completed = await receiver.GetSnapshotAsync(); + Assert.Null(completed.InboxJobId); + Assert.Equal(existing + 1, completed.ProcessedMessageCount); + await AssertCountsAsync(context, probe, new(0, 0, 0)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task FailedAcceptanceCount_PreservesOldStateAndUsesFreshReplay(bool committed) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var probe = new CountProbe(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-count-fault"); + using var handlers = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/count-replay"); + var turn = receiver.HoldPumpTurnAsync("hold-count-fault", deactivate: false); + await hold.WaitUntilEnteredAsync(); + try + { + var first = CreateEnvelope(receiver, NewMessage(150, "acknowledged"), "messages/count-replay"); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery(context, probe.Extension, first, Cancellation)).Status); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var storage = Fixture.Storage.BlockWrite(journal); + try + { + if (committed) Fixture.Storage.FailAfterWrite(journal); + var second = CreateEnvelope(receiver, NewMessage(151, "ambiguous"), "messages/count-replay"); + var delivery = StartDelivery(context, probe.Extension, second, Cancellation); + await storage.WaitUntilEnteredAsync(); + await AssertCountsAsync(context, probe, new(2, 1, 1)); + if (committed) storage.Release(); + else storage.Fail(); + var failure = await Assert.ThrowsAsync(() => delivery); + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation)); + await AssertCountsAsync(context, probe, new(2, 1, 1)); + } + finally + { + storage.Release(); + } + } + finally + { + hold.Release(); + } + await turn; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(new Counts(2, 0, 2), probe.Read()); + // The recovered drain turn can hold admission ahead of this snapshot request. + var recoverySnapshot = receiver.GetSnapshotAsync(); + await handlers.WaitUntilEnteredAsync(); + var freshContext = Fixture.GetGrainContext(receiver); + Assert.NotSame(context, freshContext); + var fresh = new CountProbe(freshContext); + var restored = committed ? 2 : 1; + await AssertCountsAsync(freshContext, fresh, new(restored, 0, restored)); + Assert.Equal(new Counts(2, 0, 2), probe.Read()); + handlers.Release(); + _ = await recoverySnapshot; + await Fixture.WaitForEffectCountAsync(receiver, restored); + _ = await receiver.GetSnapshotAsync(); + await AssertCountsAsync(freshContext, fresh, new(0, 0, 0)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task FailedAcceptanceCount_DistinguishesSchedulingFailureFromStagedWriteFailure(bool scheduling) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var probe = new CountProbe(context); + if (scheduling) Fixture.JobManagerProbe.FailNext(ReceiverTestServices.InboxJobName); + else Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var envelope = CreateEnvelope(receiver, NewMessage(152, "rejected")); + if (scheduling) + { + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Contains("Injected durable job scheduling failure", failure.Message, StringComparison.Ordinal); + } + else + { + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Contains("Injected journal write failure", failure.Message, StringComparison.Ordinal); + } + if (!scheduling) await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(scheduling ? new Counts(0, 0, 0) : new Counts(1, 0, 1), probe.Read()); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + if (!scheduling) + { + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + } + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + var current = Fixture.GetGrainContext(receiver); + await AssertCountsAsync(current, new CountProbe(current), new(0, 0, 0)); + if (!scheduling) Assert.Equal(new Counts(1, 0, 1), probe.Read()); + } + + [Fact] + public async Task StoppedOwnerDeletionCount_ResetsAndLaterAcceptanceUsesFreshGeneration() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(153, "delete-pending")); + var job = new DurableJob + { + Id = "count-delete-job", + ShardId = "count-delete-shard", + Name = ReceiverTestServices.InboxJobName, + TargetGrainId = receiver.GetGrainId(), + DueTime = Fixture.Clock.GetUtcNow(), + Metadata = new Dictionary { ["orleans.messaging.ownership-id"] = "count-delete:1" } + }; + await receiver.SeedInboxStateAsync(envelope, "count-delete:1", job); + var context = Fixture.GetGrainContext(receiver); + var probe = new CountProbe(context); + await AssertCountsAsync(context, probe, new(1, 0, 1)); + await receiver.DeleteStateAndDeactivateAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(new Counts(0, 0, 0), probe.Read()); + _ = await receiver.GetSnapshotAsync(); + var replacement = Fixture.GetGrainContext(receiver); + Assert.NotSame(context, replacement); + var replacementProbe = new CountProbe(replacement); + await AssertCountsAsync(replacement, replacementProbe, new(0, 0, 0)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + await AssertCountsAsync(replacement, replacementProbe, new(0, 0, 0)); + await receiver.RequestDeactivationAsync(); + await replacement.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + _ = await receiver.GetSnapshotAsync(); + var freshContext = Fixture.GetGrainContext(receiver); + Assert.NotSame(replacement, freshContext); + await AssertCountsAsync(freshContext, new CountProbe(freshContext), new(0, 0, 0)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + private int ScheduleCount(IMessagingTestGrain receiver) => + Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId()); + private static Task AssertCountsAsync(IGrainContext context, CountProbe probe, Counts expected) => + OnTurnAsync(context, () => Assert.Equal(expected, probe.Read())); + private static Task StartDelivery(IGrainContext context, IInboxExtension extension, InboxMessage envelope, CancellationToken cancellation) + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(envelope, cancellation).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } + private static async Task RunPumpAsync(IGrainContext context, IInboxExtension extension, DurableJob job) + { + var feature = (IDurableJobFeatureHandler)extension; + var run = new PumpContext(job); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + DurableJobRunResult result = null!; + await OnTurnAsync(context, () => result = feature.ExecuteJobAsync(run, Cancellation).GetAwaiter().GetResult()); + if (result.IsInProgress) + { + await events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.GrainContext, context), + TimeSpan.FromSeconds(30), Cancellation); + await OnTurnAsync(context, () => result = feature.ExecuteJobAsync(run, Cancellation).GetAwaiter().GetResult()); + } + return result; + } + private sealed record Counts(int Stored, int Provisional, int Durable, bool ProvisionalSubset = true); + private sealed class CountProbe + { + private readonly HashSet _provisional; + public CountProbe(IGrainContext context) + { + var type = ReceiverTestServices.GetImplementationType("InboxExtension"); + Extension = (IInboxExtension)context.ActivationServices.GetRequiredService(type); + var field = type.GetField("_inboxDict", BindingFlags.Instance | BindingFlags.NonPublic)!; + Dictionary = new CountingDictionary((IDictionary)field.GetValue(Extension)!); + field.SetValue(Extension, Dictionary); + _provisional = (HashSet)type.GetField("_provisionalAcceptances", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(Extension)!; + DurableCount = type.GetMethod("GetCommittedInboxCount", BindingFlags.Instance | BindingFlags.NonPublic)!.CreateDelegate>(Extension); + } + public IInboxExtension Extension { get; } + public CountingDictionary Dictionary { get; } + public Func DurableCount { get; } + public Counts Read() => new(Dictionary.Inner.Count, _provisional.Count, DurableCount(), _provisional.All(Dictionary.Inner.ContainsKey)); + } + private sealed class CountingDictionary(IDictionary inner) : IDictionary + { + public IDictionary Inner => inner; + public int CountReads { get; private set; } + public int KeyCollections { get; private set; } + public int KeyVisits { get; private set; } + public int EntryEnumerations { get; private set; } + public int Count { get { CountReads++; return inner.Count; } } + public ICollection Keys { get { KeyCollections++; return new CountingKeys(this); } } + public ICollection Values => inner.Values; + public InboxMessage this[HierarchicalKey key] { get => inner[key]; set => inner[key] = value; } + public bool IsReadOnly => inner.IsReadOnly; + public void Add(HierarchicalKey key, InboxMessage value) => inner.Add(key, value); + public void Add(KeyValuePair item) => inner.Add(item); + public void Clear() => inner.Clear(); + public bool Contains(KeyValuePair item) => inner.Contains(item); + public bool ContainsKey(HierarchicalKey key) => inner.ContainsKey(key); + public void CopyTo(KeyValuePair[] array, int index) => inner.CopyTo(array, index); + public bool Remove(HierarchicalKey key) => inner.Remove(key); + public bool Remove(KeyValuePair item) => inner.Remove(item); + public bool TryGetValue(HierarchicalKey key, out InboxMessage value) => inner.TryGetValue(key, out value); + public IEnumerator> GetEnumerator() { EntryEnumerations++; return inner.GetEnumerator(); } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + private sealed class CountingKeys(CountingDictionary owner) : ICollection + { + public int Count => owner.Inner.Keys.Count; + public bool IsReadOnly => true; + public bool Contains(HierarchicalKey item) => owner.Inner.Keys.Contains(item); + public void CopyTo(HierarchicalKey[] array, int index) => owner.Inner.Keys.CopyTo(array, index); + public IEnumerator GetEnumerator() + { + foreach (var key in owner.Inner.Keys) { owner.KeyVisits++; yield return key; } + } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + public void Add(HierarchicalKey item) => throw new NotSupportedException(); + public void Clear() => throw new NotSupportedException(); + public bool Remove(HierarchicalKey item) => throw new NotSupportedException(); + } + } + private sealed class PumpContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + private sealed class CountFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxCapacity = 256; + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs new file mode 100644 index 00000000000..dbbdbb57238 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs @@ -0,0 +1,196 @@ +using System.Buffers; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Orleans.Timers; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxEagerManagerTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + public async Task SynchronousManager_StagesBeforeWriteAndCompletesWithoutStateFaultNotifications(bool reverseStateOrder, bool failWrite) + { + var source = NewGrain(); + _ = await source.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(source).ActivationServices; + var format = services.GetRequiredService>().Value.JournalFormatKey; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(format); + builder.Services.AddScoped(sp => + new EagerManager(sp.GetRequiredKeyedService(format), reverseStateOrder)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var stateServices = scope.ServiceProvider; + var manager = Assert.IsType(stateServices.GetRequiredService()); + var messages = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox"); + var processed = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed"); + var attempts = stateServices.GetRequiredKeyedService(typeof(IDurableDictionary<,>) + .MakeGenericType(typeof(HierarchicalKey), ReceiverTestServices.GetImplementationType("InboxMessageState")), "__orleans.messaging.inbox-message-state"); + var deadLetters = stateServices.GetRequiredKeyedService(typeof(IDurableDictionary<,>) + .MakeGenericType(typeof(HierarchicalKey), ReceiverTestServices.GetImplementationType("InboxDeadLetter")), "__orleans.messaging.inbox-dead-letters"); + var ownerId = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-job-id"); + var ownerJob = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-job-handle"); + var completed = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-completed-job-id"); + var sequence = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-job-sequence"); + var handler = Substitute.For(); + var inbox = Create("Inbox", messages, new[] { handler }, 10); + var context = Substitute.For(); + var grainId = GrainId.Create("eager-inbox", "standalone"); + context.GrainId.Returns(grainId); + context.ActivationServices.Returns(stateServices); + context.GrainInstance.Returns(Substitute.For()); + context.ObservableLifecycle.Returns(Substitute.For()); + var jobs = Substitute.For(); + DurableJob scheduled = null!; + jobs.ScheduleJobAsync(Arg.Any(), Arg.Any()).Returns(call => + { + var request = call.Arg(); + scheduled = new DurableJob + { + Id = "scheduler-returned-id", + ShardId = "scheduler-returned-shard", + Name = request.JobName, + TargetGrainId = request.Target, + DueTime = request.DueTime, + Metadata = request.Metadata + }; + return Task.FromResult(scheduled); + }); + var failure = new IOException("Owner failed while capturing the staged write."); + if (failWrite) + { + context.When(value => value.Deactivate(Arg.Any(), Arg.Any())) + .Do(_ => throw new IOException("Secondary deactivation failure.")); + } + var timers = Substitute.For(); + var type = ReceiverTestServices.GetImplementationType("InboxExtension"); + var extension = (IInboxExtension)Create("InboxExtension", context, + timers, manager, + services.GetRequiredService(typeof(ILogger<>).MakeGenericType(type)), + services.GetRequiredService(ReceiverTestServices.GetImplementationType("MessagingInstruments")), + inbox, messages, processed, attempts, deadLetters, ownerId, ownerJob, completed, sequence, + jobs, Substitute.For(), + Create("MessagingPumpResults"), TimeProvider.System, TimeProvider.System, + new InboxOptions { MaxCapacity = 10 }); + using var lifetime = (IDisposable)extension; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await ((ILifecycleObserver)extension).OnStart(TestContext.Current.CancellationToken); + var envelope = TestApplicationProtocol.CreateInbox(stateServices.GetRequiredService(), GrainId.Create("sender", "eager"), "route", 42); + manager.BeforeCapture = () => + { + Assert.Equal(envelope, Assert.Single(messages).Value); + Assert.Same(scheduled, ownerJob.Value); + Assert.Equal(scheduled.Metadata!["orleans.messaging.ownership-id"], ownerId.Value); + Assert.Equal(1, sequence.Value); + if (failWrite) throw failure; + }; + var delivery = extension.DeliverAsync(envelope, TestContext.Current.CancellationToken); + if (failWrite) + { + Assert.True(delivery.IsCompleted); + Assert.Same(failure, await Assert.ThrowsAsync(() => delivery.AsTask())); + var captured = (System.Runtime.ExceptionServices.ExceptionDispatchInfo)type + .GetField("_failure", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + Assert.Same(failure, captured.SourceException); + Assert.Equal(1, manager.Requests); + Assert.Equal(0, manager.Writes); + Assert.True(manager.IsFenced); + Assert.Empty(manager.Batches); + Assert.Single(messages); + Assert.Empty(timers.ReceivedCalls()); + } + else + { + Assert.True(delivery.IsCompletedSuccessfully); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + Assert.Equal(1, manager.Writes); + Assert.NotEmpty(Assert.Single(manager.Batches)); + Assert.Empty((IEnumerable)type.GetField("_provisionalAcceptances", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!); + } + await jobs.Received(1).ScheduleJobAsync(Arg.Any(), Arg.Any()); + } + + private static object Create(string name, params object[] args) => Activator.CreateInstance( + ReceiverTestServices.GetImplementationType(name), BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DoNotWrapExceptions, + binder: null, args, culture: null)!; + + private sealed class EagerManager(IJournalFormat format, bool reverseStateOrder) : IJournaledStateManager + { + private readonly Dictionary _states = []; + private readonly JournalBufferWriter _writer = format.CreateWriter(); + private readonly Dictionary _ids = []; + private IEnumerable States => reverseStateOrder ? _states.Values.Reverse() : _states.Values; + private System.Runtime.ExceptionServices.ExceptionDispatchInfo? _failure; + public IList Hooks { get; } = []; + public Action? BeforeCapture { get; set; } + public int Writes { get; private set; } + public int Requests { get; private set; } + public bool IsFenced => _failure is not null; + public List Batches { get; } = []; + public void RegisterStateMachine(string name, IStateMachine state) + { + _states.Add(name, state); + _ids.Add(state, checked((uint)(_ids.Count + 8))); + } + public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) => _states.TryGetValue(name, out state); + public ValueTask InitializeAsync(CancellationToken token) + { + _failure?.Throw(); + token.ThrowIfCancellationRequested(); + foreach (var state in States) state.Reset(_writer.CreateJournalStreamWriter(new(_ids[state]))); + foreach (var state in States) state.OnRecoveryCompleted(); + return default; + } + public ValueTask WriteStateAsync(CancellationToken token) + { + Requests++; + _failure?.Throw(); + token.ThrowIfCancellationRequested(); + try + { + BeforeCapture?.Invoke(); + foreach (var state in States) state.WritePendingEntries(_writer.CreateJournalStreamWriter(new(_ids[state]))); + using var buffer = _writer.GetBuffer(); + if (buffer.Length > 0) + { + Batches.Add(buffer.AsReadOnlySequence().ToArray()); + _writer.Reset(); + foreach (var state in States) state.OnWriteCompleted(); + } + Writes++; + return ValueTask.CompletedTask; + } + catch (Exception exception) + { + _failure ??= System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(exception); + throw; + } + } + public ValueTask DeleteStateAsync(CancellationToken token) + { + _failure?.Throw(); + token.ThrowIfCancellationRequested(); + _writer.Reset(); + foreach (var state in States) state.Reset(_writer.CreateJournalStreamWriter(new(_ids[state]))); + Batches.Clear(); + return default; + } + public ValueTask DisposeAsync() { _writer.Dispose(); return default; } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs new file mode 100644 index 00000000000..9e885113e92 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs @@ -0,0 +1,75 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxEnvelopeValidationTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DefaultMessage_RejectsBeforeAcceptanceOrDuplicate(bool existingKey) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + // External deserialization can materialize invalid default fields before admission. + var malformed = new InboxMessage { Envelope = default }; + var context = Fixture.GetGrainContext(receiver); + if (existingKey) + { + var processed = context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed"); + await OnTurnAsync(context, () => processed.Add(malformed.Envelope.MessageId, Fixture.Clock.GetUtcNow())); + await receiver.RetryWriteStateAsync(); + } + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + + var exception = await Assert.ThrowsAsync(() => DeliverAsync(receiver, malformed)); + + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + await OnTurnAsync(context, () => + { + var direct = extension.DeliverAsync(malformed, TestContext.Current.CancellationToken); + Assert.True(direct.IsCompleted); + Assert.Throws(() => direct.GetAwaiter().GetResult()); + }); + Assert.Contains("message", exception.Message, StringComparison.OrdinalIgnoreCase); + var after = await receiver.GetSnapshotAsync(); + Assert.Equal(0, after.InboxCount); + Assert.Null(after.InboxJobId); + Assert.Empty(after.Effects); + Assert.Equal(existingKey ? 1 : 0, after.ProcessedMessageCount); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + } + + [Fact] + public async Task DirectEnvelopeWithExplicitValidIdentity_PreservesDeliveryAndDedupe() + { + var receiver = NewGrain(); + var template = CreateEnvelope(receiver, NewMessage(187, "explicit-identity")); + var envelope = TestApplicationProtocol.With(template, Sessions, + messageId: HierarchicalKey.Parse("tenant/test/command/explicit", null)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs new file mode 100644 index 00000000000..39119f2a21f --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -0,0 +1,698 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHandlerCompletionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task LateAcknowledgedAcceptance_ImmediatelyRearmsSameLocalTimerWithoutClockAdvance() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + Assert.Null(RequestContext.Get("reusable-turn-parent")); + self.Mutate(); + self.Context.Complete(); + return default; + }; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + RequestContext.Set("reusable-turn-parent", "first-call-chain"); + InboxMessage first; + try { first = await DeliverAsync(rig); } + finally { RequestContext.Remove("reusable-turn-parent"); } + var timer = GetTimer(events, rig); + var second = CreateEnvelope(rig.Receiver, NewMessage(502, "late-local"), "async/handler"); + Task accepted = null!; + // The pump is deliberately non-interleaving. Inject the owned late admission on its + // scheduler, as capture/hook tests do, rather than waiting on a blocked ordinary RPC. + await OnTurnAsync(rig.Context, () => + { + RequestContext.Set("reusable-turn-parent", "second-call-chain"); + try + { + var extension = (IInboxExtension)rig.Context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + accepted = extension.DeliverAsync(second).AsTask(); + } + finally { RequestContext.Remove("reusable-turn-parent"); } + }); + Assert.Equal(DeliveryStatus.Accepted, (await accepted).Status); + Assert.Equal(2, rig.Inbox.Count); + var drained = events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer) + && rig.Inbox.Count == 0 && rig.Effects.Count == 2, + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + handler.Release.TrySetResult(); + await drained; + Assert.Equal(2, rig.Effects.Count); + Assert.Equal(2, rig.Processed.Count); + Assert.Single(events.Events.Select(item => item.Payload).OfType(), + created => ReferenceEquals(created.GrainContext, rig.Context) && IsLocalDrainTimer(created.Timer)); + Assert.Equal(2, events.Events.Select(item => item.Payload).OfType() + .Count(stopped => ReferenceEquals(stopped.Timer, timer))); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + [Fact] + public async Task Complete_StagesBusinessOutputAndDedupeBeforeHandlerReturn() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, token) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + AssertCompletedState(rig, self.Context.Message); + Assert.Equal(1, Assert.Single(rig.Effects).Value.Count); + Assert.Single(rig.Outbox); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockAcknowledgement(rig.Journal); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + Assert.False(finished.IsCompleted); + storage.Release(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 1); + await DeactivateAsync(rig); + var recovered = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + + [Fact] + public async Task Complete_PrecedingWriterCapturesCompletionWithBusinessAcrossReturnContinuation() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var writes = Writes(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var preceding = OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "prior"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + await storage.WaitUntilEnteredAsync(); + var priorCapture = rig.Grain.Captures[^1]; + Assert.Empty(priorCapture.Effects); + Assert.Equal(1, priorCapture.InboxCount); + Assert.Equal(0, priorCapture.ProcessedMessageCount); + Assert.Equal(0, priorCapture.OutboxCount); + var queued = OnTurnAsync(rig.Context, async () => await rig.Manager.WriteStateAsync(CancellationToken.None)); + var staged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var returnContinuation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + handler.Body = async (self, _) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + AssertCompletedState(rig, input); + staged.TrySetResult(); + await returnContinuation.Task; + }; + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + try + { + await WaitAsync(staged.Task); + Assert.False(finished.IsCompleted); + storage.Release(); + await WaitAsync(Task.WhenAll(preceding, queued)); + Assert.False(finished.IsCompleted); + Assert.Equal(writes + 2, Writes(rig)); + var capturedEffects = Assert.Single(rig.Grain.Captures, snapshot => snapshot.Effects.Count != 0); + Assert.Equal(0, capturedEffects.InboxCount); + Assert.Equal(1, capturedEffects.ProcessedMessageCount); + Assert.Equal(1, capturedEffects.OutboxCount); + Assert.Equal(new MessageEffect(input.Envelope.MessageId, 1, 501, "async-handler"), Assert.Single(capturedEffects.Effects)); + } + finally + { + returnContinuation.TrySetResult(); + } + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 1); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandlerFailure_BeforeVsAfterCompleteUsesActualLogicalOutcome(bool completed) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var error = new IOException("Handler outcome."); + handler.Body = (self, _) => + { + if (completed) + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + } + throw error; + }; + var input = await DeliverAsync(rig); + var timer = GetTimer(events, rig); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + var stop = (GrainTimerEvents.TickStop)(await TimerStoppedAsync(events, timer)).Payload!; + if (completed) + { + Assert.Same(error, stop.Exception); + AssertSuccess(rig, input, outputCount: 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + else + { + Assert.Null(stop.Exception); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters).MessageId); + } + await AssertHealthyAsync(rig); + await DeactivateAsync(rig); + var replay = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(completed ? 1 : 0, replay.Effects.Count); + Assert.Equal(1, replay.ProcessedMessageCount); + Assert.Equal(completed ? 0 : 1, replay.InboxDeadLetters.Count); + } + + [Fact] + public async Task SuccessfulReturnWithoutComplete_IsExplicitTerminalMisuse() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (_, _) => ValueTask.CompletedTask; + var input = await DeliverAsync(rig); + var writes = Writes(rig); + handler.Release.TrySetResult(); + var error = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); + Assert.Contains("must call Complete", error.Message, StringComparison.Ordinal); + Assert.Equal(writes, Writes(rig)); + await rig.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Inbox).Key); + Assert.Equal(input.Envelope.Payload, Assert.Single(rig.Inbox).Value.Envelope.Payload); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Effects); + await AssertFailureReplayAsync(rig, input); + } + + [Fact] + public async Task RepeatedComplete_SameActiveAttemptStagesExactlyOnce() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + self.Mutate(); + self.Context.Complete(); + var timestamp = Assert.Single(rig.Processed).Value; + self.Context.Complete(); + Assert.Equal(timestamp, Assert.Single(rig.Processed).Value); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData("complete")] + public async Task RetainedContext_AfterRetirementRejectsAndKeepsOwnerHealthy(string operation) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + var error = await Assert.ThrowsAsync(() => + OnTurnAsync(rig.Context, () => InvokeContextOperation(handler, operation))); + Assert.Contains("inactive or different attempt", error.Message, StringComparison.Ordinal); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData("complete")] + public async Task RetainedContext_DuringOtherAttemptRetainsFirstMisuse(string operation) + { + var rig = await CreateAsync(); + using var first = rig.Handler; + var one = await DeliverAsync(rig); + var finished = await FinishedAsync(rig); + first.Release.TrySetResult(); + await WaitAsync(finished); + using var second = new TestHandler(rig.Effects, rig.Context); + InvalidOperationException? rejection = null; + second.Body = (_, _) => + { + try { InvokeContextOperation(first, operation); } + catch (InvalidOperationException error) { rejection = error; } + throw new IOException("Replacement failure."); + }; + await OnTurnAsync(rig.Context, () => + rig.Grain.HandlerOverride = second); + var two = CreateEnvelope(rig.Receiver, NewMessage(502, "next"), "async/next"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, two)).Status); + await WaitAsync(second.Entered.Task); + second.Release.TrySetResult(); + var failure = await WaitAsync(rig.Grain.DeactivationFailure.Task); + Assert.Same(Assert.IsType(rejection), failure); + Assert.Equal(1, Assert.Single(rig.Effects).Value.Count); + await WaitAsync(rig.Context.Deactivated); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DurableJobCancellation_BeforeCompleteRetainsExactOwnerAndRetries(bool cancellationCheck) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var localEvents = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var input = await DeliverAsync(rig); + var localTimer = GetTimer(localEvents, rig); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var resume = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retry = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var attempts = 0; + OperationCanceledException? expected = null; + handler.Body = async (self, token) => + { + if (++attempts > 1) + { + await retry.Task.WaitAsync(token); + self.Mutate(); + self.Context.Complete(); + return; + } + entered.TrySetResult(); + try + { + if (cancellationCheck) + { + await resume.Task; + token.ThrowIfCancellationRequested(); + } + else await resume.Task.WaitAsync(token); + } + catch (OperationCanceledException error) { expected = error; throw; } + }; + var snapshot = rig.Grain.GetSnapshotForTest(); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, rig.Receiver.GetGrainId())); + var run = new JobContext(job); + var feature = (IDurableJobFeatureHandler)rig.Context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + using var jobEvents = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + using var cancellation = new CancellationTokenSource(); + var start = new StartAtTimerStop(rig, localTimer, feature, run, cancellation.Token, jobEvents); + using var subscription = GrainTimerEvents.AllEvents.Subscribe(start); + await OnTurnAsync(rig.Context, localTimer.Dispose); + var timer = await WaitAsync(start.Started.Task); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + handler.Release.TrySetResult(); + try + { + await WaitAsync(entered.Task); + await OnTurnAsync(rig.Context, cancellation.Cancel); + } + finally + { + resume.TrySetResult(); + } + await TimerStoppedAsync(jobEvents, timer); + var failure = await Assert.ThrowsAnyAsync(() => + OnTurnAsync(rig.Context, async () => await feature.ExecuteJobAsync(run, TestContext.Current.CancellationToken))); + Assert.Same(expected, failure); + Assert.Single(rig.Inbox); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Effects); + Assert.Equal(snapshot.InboxJobId, rig.Grain.GetSnapshotForTest().InboxJobId); + Assert.Same(snapshot.InboxJob, rig.Grain.GetSnapshotForTest().InboxJob); + await AssertHealthyAsync(rig); + var duplicate = DeliverAsync(rig.Receiver, input); + retry.TrySetResult(); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + AssertSuccess(rig, input, outputCount: 0); + Assert.Same(rig.Context, Fixture.GetGrainContext(rig.Receiver)); + } + + [Fact] + public async Task CanceledJobCallback_LeavesActiveHandlerAndCommittedInboxUndisturbed() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var snapshot = rig.Grain.GetSnapshotForTest(); + var writes = Writes(rig); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, rig.Receiver.GetGrainId())); + var feature = (IDurableJobFeatureHandler)rig.Context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var canceled = new CancellationToken(canceled: true); + var error = await Assert.ThrowsAnyAsync(() => OnTurnAsync(rig.Context, async () => + await feature.ExecuteJobAsync(new JobContext(job), canceled))); + Assert.Equal(canceled, error.CancellationToken); + Assert.Equal(writes, Writes(rig)); + Assert.Single(rig.Inbox); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Processed); + Assert.Equal(snapshot.InboxJobId, rig.Grain.GetSnapshotForTest().InboxJobId); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 0); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CancellationInFinalBlock_CompleteAndOwnedWritePreserveOutcome(bool failWrite) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + IGrainTimer timer = null!; + handler.Body = (self, token) => + { + self.Mutate(); + timer.Dispose(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + timer = GetTimer(events, rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + AssertCompletedState(rig, input); + Assert.False(finished.IsCompleted); + var failure = new OperationCanceledException("Actual storage canceled.", new CancellationToken(canceled: true)); + if (failWrite) storage.Fail(failure); + else storage.Release(); + await WaitAsync(finished); + if (failWrite) + { + Assert.Same(failure, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + } + else + { + AssertSuccess(rig, input, outputCount: 1); + await AssertHealthyAsync(rig); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletionWrite_HookOutcomePreservesActualCommit(bool postCommit) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var input = await DeliverAsync(rig); + var error = new IOException("Journal hook failed."); + await OnTurnAsync(rig.Context, () => rig.Manager.Hooks.Add(new FailingHook(postCommit, error))); + var timer = GetTimer(events, rig); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + if (postCommit) + { + Assert.IsType( + ((GrainTimerEvents.TickStop)(await TimerStoppedAsync(events, timer)).Payload!).Exception); + AssertSuccess(rig, input, outputCount: 0); + await OnTurnAsync(rig.Context, rig.Manager.Hooks.Clear); + await AssertHealthyAsync(rig); + } + else + { + var failure = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); + Assert.Same(error, failure.InnerException); + await AssertFailureReplayAsync(rig, input); + } + } + + [Fact] + public async Task PostCompleteHandlerFailure_WithStorageFailureKeepsStorageCauseAuthoritative() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var handlerError = new InvalidOperationException("After Complete."); + handler.Body = (self, _) => { self.Mutate(); self.Context.Complete(); throw handlerError; }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + var storageError = new IOException("Storage failed."); + storage.Fail(storageError); + await WaitAsync(finished); + Assert.Same(storageError, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + } + + private async Task CreateAsync() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var services = context.ActivationServices; + var effects = services.GetRequiredKeyedService>("test-effects"); + var handler = new TestHandler(effects, context); + var rig = new Rig(receiver, context, Assert.IsType(context.GrainInstance), + services.GetRequiredService(), (JournaledTestOutbox)services.GetRequiredService(), + effects, handler, + services.GetRequiredKeyedService>("__orleans.messaging.inbox"), + services.GetRequiredKeyedService>("__orleans.messaging.inbox-processed")); + await OnTurnAsync(context, () => rig.Grain.HandlerOverride = handler); + return rig; + } + + private async Task DeliverAsync(Rig rig) + { + var input = CreateEnvelope(rig.Receiver, NewMessage(501, "async-handler"), "async/handler"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, input)).Status); + await WaitAsync(rig.Handler.Entered.Task); + return input; + } + + private static void InvokeContextOperation(TestHandler handler, string operation) + { + switch (operation) + { + case "complete": handler.Context.Complete(); break; + default: throw new ArgumentException(nameof(operation)); + } + } + + private static void AssertCompletedState(Rig rig, InboxMessage input) + { + Assert.Empty(rig.Inbox); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Processed).Key); + } + + private static void AssertSuccess(Rig rig, InboxMessage input, int outputCount) + { + AssertCompletedState(rig, input); + Assert.Equal(new MessageEffect(input.Envelope.MessageId, 1, 501, "async-handler"), Assert.Single(rig.Effects).Value); + Assert.Equal(outputCount, rig.Outbox.Count); + Assert.Empty(rig.Grain.GetSnapshotForTest().InboxDeadLetters); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + private int Writes(Rig rig) => Fixture.Storage.GetSuccessfulWriteCount(rig.Journal); + private static async Task AssertHealthyAsync(Rig rig) + { + await OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "healthy"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + private static async Task DeactivateAsync(Rig rig) + { + await rig.Receiver.RequestDeactivationAsync(); + await WaitAsync(rig.Context.Deactivated); + } + + private async Task AssertFailureReplayAsync(Rig rig, InboxMessage input) + { + await WaitAsync(rig.Context.Deactivated); + await rig.Receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForDeadLetterCountAsync(rig.Receiver, 1); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(input.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + private static async Task FinishedAsync(Rig rig) + { + Task result = null!; + await OnTurnAsync(rig.Context, () => + { + var extension = rig.Context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var pending = CancellationCleanupProbe.Field(extension, "_pendingWrites"); + var operation = Assert.Single(pending.Cast(), item => item.GetType().Name == "HandlerWrite"); + result = ((TaskCompletionSource)operation.GetType().GetProperty("Finished")!.GetValue(operation)!).Task; + }); + return result; + } + + private static IGrainTimer GetTimer(DiagnosticEventCollector events, Rig rig) => + Assert.Single(events.Events.Select(item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, rig.Context) && IsLocalDrainTimer(item.Timer)).Timer; + private static IGrainTimer GetPumpTimer(DiagnosticEventCollector events, Rig rig) => + Assert.Single(events.Events.Select(item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, rig.Context) && IsInboxTimer(item.Timer) + && !IsLocalDrainTimer(item.Timer)).Timer; + private static bool IsLocalDrainTimer(IGrainTimer timer) => IsInboxTimer(timer) + && timer.GetType().GenericTypeArguments[0].Name == "LocalDrainTimerState"; + private static bool IsInboxTimer(IGrainTimer timer) => timer.GetType().GenericTypeArguments is [var type] + && type.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension"); + private static Task TimerStoppedAsync(DiagnosticEventCollector events, IGrainTimer timer) => + events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = CompleteAsync(); }); + return done.Task; + async Task CompleteAsync() + { + try + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + await action(); + done.SetResult(); + } + catch (Exception error) { done.SetException(error); } + } + } + + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, MessagingTestGrain Grain, + IJournaledStateManager Manager, JournaledTestOutbox Outbox, IDurableDictionary Effects, + TestHandler Handler, IDurableDictionary Inbox, + IDurableDictionary Processed) + { + public JournalId Journal => JournalId.FromGrainId(Receiver.GetGrainId()); + } + + private sealed class TestHandler(IDurableDictionary effects, IGrainContext grainContext) : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public IInboxHandlerContext Context { get; private set; } = null!; + public OutboxMessage Output { get; private set; } + public Func Body { get; set; } = + static (self, _) => { self.Mutate(); self.Context.Complete(); return default; }; + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + Context = context; + + Output = TestApplicationProtocol.Create(grainContext.ActivationServices.GetRequiredService(), grainContext.GrainId, grainContext.GrainId, "output", 41); + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + await Body(this, cancellationToken); + } + public void Mutate() + { + effects.TryGetValue(Context.Message.Envelope.MessageId, out var previous); + effects[Context.Message.Envelope.MessageId] = + new MessageEffect(Context.Message.Envelope.MessageId, (previous?.Count ?? 0) + 1, 501, "async-handler"); + } + public void Dispose() + { + Release.TrySetResult(); + + } + } + private sealed class JobContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + + private sealed class FailingHook(bool postCommit, Exception error) : IJournaledStateHook + { + public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (!postCommit) throw error; + return default; + } + + public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (postCommit) throw error; + return default; + } + } + + private sealed class StartAtTimerStop(Rig rig, IGrainTimer preceding, IDurableJobFeatureHandler feature, + JobContext run, CancellationToken cancellationToken, DiagnosticEventCollector events) + : IObserver + { + private bool _started; + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void OnNext(GrainTimerEvents.TimerEvent value) + { + if (_started || value is not GrainTimerEvents.TickStop stopped + || !ReferenceEquals(stopped.Timer, preceding)) + { + return; + } + _started = true; + try + { + Assert.Same(rig.Context, ReceiverTestServices.CurrentGrainContext); + var result = feature.ExecuteJobAsync(run, cancellationToken); + Assert.True(result.IsCompletedSuccessfully); + Assert.True(result.GetAwaiter().GetResult().IsInProgress); + Started.SetResult(GetPumpTimer(events, rig)); + } + catch (Exception error) + { + Started.SetException(error); + } + } + public void OnCompleted() { } + public void OnError(Exception error) => Started.TrySetException(error); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs new file mode 100644 index 00000000000..ec2d4ed9bd7 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs @@ -0,0 +1,150 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHandlerTransactionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task HandlerSuccess_CommitsEffectCompletionDedupeAndStagedOutputAtomically() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var logicalId = TestApplicationProtocol.NewMessageId(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(logicalId, 7, "atomic", sink.GetGrainId())); + + var result = await DeliverAsync(receiver, envelope); + var receiverState = await Fixture.WaitForEffectCountAsync(receiver, 1); + var output = Fixture.GetStagedOutput(receiver); + + Assert.Equal(DeliveryStatus.Accepted, result.Status); + var effect = Assert.Single(receiverState.Effects); + Assert.Equal(new MessageEffect(logicalId, 1, 7, "atomic"), effect); + Assert.Equal(0, receiverState.InboxCount); + Assert.Equal(1, receiverState.OutboxCount); + var outgoing = Assert.Single(output); + Assert.Equal(sink.GetGrainId(), outgoing.ReceiverId); + Assert.Equal("messages/forwarded", TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), outgoing).Route); + var body = TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), outgoing).Body; + Assert.Equal(new TestMessage(logicalId, 7, "atomic"), body); + Assert.Equal(1, receiverState.ProcessedMessageCount); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(receiverState.ActivationId, recovered.ActivationId); + Assert.Equal(effect, Assert.Single(recovered.Effects)); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(outgoing.Envelope.MessageId, Assert.Single(Fixture.GetStagedOutput(receiver)).Envelope.MessageId); + + var duplicate = await DeliverAsync(receiver, envelope); + Assert.Equal(DeliveryStatus.Duplicate, duplicate.Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + Assert.Single(Fixture.GetStagedOutput(receiver)); + } + + [Fact] + public async Task HandlerPreparationFailure_DeadLettersWithoutStagingEffectsOrOutput() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), 9, "preparation-failure", sink.GetGrainId(), ThrowDuringPreparation: true)); + + var accepted = await DeliverAsync(receiver, envelope); + var state = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Equal(DeliveryStatus.Accepted, accepted.Status); + Assert.Empty(state.Effects); + Assert.Equal(0, state.InboxCount); + Assert.Equal(0, state.OutboxCount); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal(1, deadLetter.AttemptCount); + Assert.Contains("Injected handler preparation failure", deadLetter.Reason, StringComparison.Ordinal); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(state.ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + [Fact] + public async Task HandlerPreparation_OrdinaryWriteLeavesEffectsLocalUntilApplyAndCompletion() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(10, "premature-commit") with { CommitDuringHandling = true }); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal("premature-commit", Assert.Single(completed.Effects).Value); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(oldGrain.DeactivationFailure.Task.IsCompleted); + Assert.Same(oldContext, Fixture.GetGrainContext(receiver)); + } + + [Fact] + public async Task HandlerCompletionWriteFailure_FencesOldManagerAndFreshActivationRetries() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/completion-failure"); + var envelope = CreateEnvelope(receiver, NewMessage(78, "completion-failure"), "messages/completion-failure"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Assert.ThrowsAnyAsync(() => oldManager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId != oldGrain.GetSnapshotForTest().ActivationId + && snapshot.Effects.Sum(static effect => effect.Count) == 1); + Assert.NotEqual(oldGrain.GetSnapshotForTest().ActivationId, completed.ActivationId); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task PreparationFailureAccountingWriteFailure_RecoversInFreshActivation() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/failure-accounting"); + var envelope = CreateEnvelope(receiver, NewMessage(79, "failure-accounting") with { ThrowDuringPreparation = true }, "messages/failure-accounting"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, completed.ActivationId); + Assert.Equal(1, Assert.Single(completed.InboxDeadLetters).AttemptCount); + Assert.Empty(completed.Effects); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs new file mode 100644 index 00000000000..7056ae69a10 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -0,0 +1,280 @@ +using System.Text; +using System.Runtime.InteropServices; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHeaderBehaviorTests : MessagingBehaviorTestBase +{ + public InboxHeaderBehaviorTests() : base(new HeaderFixture()) { } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RawMessage_AbsentOptionalMetadataCommitsAndDeduplicates(bool empty) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + byte[] payload = empty ? [] : [0, 255, 128, 42]; + var message = Raw(HierarchicalKey.Create("raw", Guid.NewGuid().ToString("N")), payload); + Assert.False(message.Envelope.TryGetSubject(out _)); + Assert.False(message.Envelope.TryGetBytes(MessageHeaders.Sender, out _)); + using var handler = new RawHandler(context); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + using var storage = Fixture.Storage.BlockAcknowledgement(JournalId.FromGrainId(context.GrainId)); + var delivery = DeliverAsync(receiver, message); + await storage.WaitUntilEnteredAsync(); + Assert.False(delivery.IsCompleted); + Assert.False(handler.Entered.Task.IsCompleted); + storage.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await WaitAsync(handler.Entered.Task); + Assert.Equal(payload, handler.Message.Envelope.Payload.ToArray()); + handler.Release.TrySetResult(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(message.Envelope.MessageId, Assert.Single(completed.Effects).LogicalId); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + await receiver.RequestDeactivationAsync(); + await WaitAsync(context.Deactivated); + var replayed = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, replayed.ActivationId); + Assert.Equal(1, Assert.Single(replayed.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + } + + [Fact] + public async Task PendingDuplicate_ChangedCustomHeadersAndSenderPreservesOriginal() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var key = HierarchicalKey.Create("custom", Guid.NewGuid().ToString("N")); + var original = Raw(key, [1, 2, 3], "original", GrainId.Create("producer", "first")); + var changed = Raw(key, [1, 2, 3], "changed", GrainId.Create("forwarder", "second")); + using var handler = new RawHandler(context); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var pending = context.ActivationServices.GetRequiredService(); + Envelope first = default; + await OnTurnAsync(context, () => first = Assert.Single(pending.Messages).Envelope); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); + var schedules = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, context.GrainId); + await OnTurnAsync(context, async () => + { + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(changed)).Status); + var retained = Assert.Single(pending.Messages).Envelope; + Assert.Equal(first.EncodedHeaders, retained.EncodedHeaders); + Assert.True(retained.TryGetBytes("custom.trace", out var trace)); + Assert.Equal("original", Encoding.UTF8.GetString(trace.Span)); + Assert.Equal(GrainId.Create("producer", "first"), TestApplicationProtocol.Sender(Sessions, retained)); + }); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); + Assert.Equal(schedules, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, context.GrainId)); + handler.Release.TrySetResult(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + [Fact] + public async Task UnknownHeaders_RecoverAndRemainAvailableAfterRetryAndDeadLetter() + { + var receiver = NewGrain(); + var key = HierarchicalKey.Create("headers", Guid.NewGuid().ToString("N")); + var body = NewMessage(810, "poison") with { LogicalId = key, ThrowDuringPreparation = true }; + var application = CreateEnvelope(receiver, body); + using var builder = new EnvelopeBuilder(key); + foreach (var header in application.Envelope.Keys) + { + Assert.True(application.Envelope.TryGetBytes(header, out var bytes)); + builder.AddBytes(header, bytes.Span); + } + byte[] opaque = [0xff, 0, 0x80, 0x7f]; + builder.AddBytes("future.opaque", opaque); + builder.AddValue("future.number", 8675309, new Serializer(Sessions)); + var message = new InboxMessage { Envelope = builder.Build() }; + var retry = Fixture.Metrics.WaitForCountAsync("orleans-messaging-inbox-messages-processed", 1, "retry"); + await receiver.SeedInboxStateAsync(message, null, null); + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(previous.Deactivated); + _ = await receiver.GetSnapshotAsync(); + var current = Fixture.GetGrainContext(receiver); + Assert.NotSame(previous, current); + await retry; + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.InboxDeadLetters); + await OnTurnAsync(current, () => + { + var stored = Assert.Single(current.ActivationServices.GetRequiredService().Messages).Envelope; + Assert.True(stored.TryGetBytes("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes.ToArray()); + Assert.True(stored.TryGetValue("future.number", new Serializer(Sessions), out var number)); + Assert.Equal(8675309, number); + }); + Fixture.Clock.Advance(TimeSpan.FromHours(1)); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + _ = await receiver.GetSnapshotAsync(); + current = Fixture.GetGrainContext(receiver); + var dead = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Equal(key, Assert.Single(dead.InboxDeadLetters).MessageId); + Assert.Equal(2, Assert.Single(dead.InboxDeadLetters).AttemptCount); + Assert.Empty(dead.Effects); + var diagnostics = current.ActivationServices.GetRequiredService(); + await OnTurnAsync(current, () => + { + var retained = Assert.Single(diagnostics.InboxDeadLetters).Message.Envelope; + Assert.True(retained.TryGetBytes("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes.ToArray()); + Assert.True(retained.TryGetValue("future.number", new Serializer(Sessions), out var number)); + Assert.Equal(8675309, number); + Assert.Equal(application.Envelope.Payload.ToArray(), retained.Payload.ToArray()); + }); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + _ = await receiver.GetSnapshotAsync(); + var recovered = Fixture.GetGrainContext(receiver); + await OnTurnAsync(recovered, () => + { + var retained = Assert.Single(recovered.ActivationServices.GetRequiredService() + .InboxDeadLetters).Message.Envelope; + Assert.True(retained.TryGetBytes("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes.ToArray()); + Assert.True(retained.TryGetValue("future.number", new Serializer(Sessions), out var number)); + Assert.Equal(8675309, number); + }); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, Raw(key, []))).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MalformedMessage_RejectsBeforeSchedulingOrMutation(bool completed) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var key = HierarchicalKey.Create("malformed", Guid.NewGuid().ToString("N")); + var message = Raw(key, [1, 2]); + if (completed) + { + await OnTurnAsync(context, async () => + { + context.ActivationServices.GetRequiredKeyedService>( + "__orleans.messaging.inbox-processed").Add(key, Fixture.Clock.GetUtcNow()); + await context.ActivationServices.GetRequiredService().WriteStateAsync(CancellationToken.None); + }); + } + Assert.True(MemoryMarshal.TryGetArray(message.Envelope.EncodedHeaders, out var encoded)); + encoded.Array![encoded.Offset] = 255; // Represents malformed external/generated input. + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); + await OnTurnAsync(context, async () => + { + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + await Assert.ThrowsAsync(() => extension.DeliverAsync(message, TestContext.Current.CancellationToken).AsTask()); + }); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, context.GrainId)); + var snapshot = await receiver.GetSnapshotAsync(); + Assert.Equal(0, snapshot.InboxCount); + Assert.Equal(completed ? 1 : 0, snapshot.ProcessedMessageCount); + Assert.Empty(snapshot.Effects); + } + + [Fact] + public async Task MalformedRecoveredHeaders_FailBeforeSchedulingOrBusinessMutation() + { + var receiver = NewGrain(); + var key = HierarchicalKey.Create("recovery", Guid.NewGuid().ToString("N")); + var message = Raw(key, []); + Assert.True(MemoryMarshal.TryGetArray(message.Envelope.EncodedHeaders, out var encoded)); + encoded.Array![encoded.Offset] = 255; + await receiver.SeedInboxStateAsync(message, null, null); + var previous = Fixture.GetGrainContext(receiver); + var journal = JournalId.FromGrainId(previous.GrainId); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + await receiver.RequestDeactivationAsync(); + await WaitAsync(previous.Deactivated); + var failure = await Assert.ThrowsAnyAsync(() => receiver.GetSnapshotAsync()); + Assert.Contains("envelope format", failure.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, previous.GrainId)); + } + + private InboxMessage Raw(HierarchicalKey key, byte[] payload, string? trace = null, GrainId? sender = null) + { + using var builder = new EnvelopeBuilder(key); + builder.AddBytes(MessageHeaders.Payload, payload); + if (trace is not null) builder.AddBytes("custom.trace", Encoding.UTF8.GetBytes(trace)); + if (sender is { } value) builder.AddValue(MessageHeaders.Sender, value, new Serializer(Sessions)); + return new() { Envelope = builder.Build() }; + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var result = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = Run(); }); + return result.Task; + async Task Run() + { + try { await action(); result.SetResult(); } + catch (Exception exception) { result.SetException(exception); } + } + } + + private static Task WaitAsync(Task task) => + task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + private sealed class RawHandler(IGrainContext context) : IInboxHandler, IDisposable + { + public InboxMessage Message { get; private set; } + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async ValueTask HandleAsync(IInboxHandlerContext handler, CancellationToken cancellationToken) + { + Message = handler.Message; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + var key = handler.Message.Envelope.MessageId; + var effects = context.ActivationServices.GetRequiredKeyedService>("test-effects"); + effects[key] = new(key, 1, 800, "raw"); + handler.Complete(); + } + + public void Dispose() => Release.TrySetResult(); + } + + private sealed class HeaderFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxProcessingAttempts = 2; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs new file mode 100644 index 00000000000..6ae704bf821 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs @@ -0,0 +1,473 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxIdentityTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task PendingCommand_NewSenderCoalescesAndPreservesOriginalEnvelope() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1042", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = TestApplicationProtocol.With(original, Sessions, sender: GrainId.Create("forwarder", "second")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId); + ReadOnlyMemory storedPayload = default; + await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Envelope.Payload); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, repeated)).Status); + await OnTurnAsync(rig.Context, () => + { + var inbox = rig.Context.ActivationServices.GetRequiredService(); + Assert.True(inbox.TryGetMessage(key, out var stored)); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, stored.Envelope)); + Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(stored.Envelope)); + Assert.Equal(storedPayload, stored.Envelope.Payload); + Assert.Equal(original.Envelope.Payload.ToArray(), stored.Envelope.Payload.ToArray()); + Assert.Single(rig.Pending); + Assert.Empty(rig.Processed); + }); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId)); + handler.Release.TrySetResult(); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, handler.Calls); + Assert.Equal(key, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).LogicalId); + Assert.Equal(1, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).Count); + } + + [Theory] + [InlineData("subject")] + [InlineData("subject-case")] + [InlineData("body")] + public async Task PendingCommand_ConflictingContentRejectsBeforeSchedulingOrMutation(string difference) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1043", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var conflictingSubject = difference switch + { + "subject" => "inventory.release.v1", + "subject-case" => "Inventory.reserve.v1", + _ => Assert.IsType(TestApplicationProtocol.Subject(original.Envelope)) + }; + var replacement = Create(rig, key, conflictingSubject, "different-parameters", + GrainId.Create("forwarder", "other")); + var conflict = difference == "body" ? replacement : TestApplicationProtocol.With(original, Sessions, + sender: TestApplicationProtocol.Sender(Sessions, replacement.Envelope), subject: conflictingSubject); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var writes = Writes(rig); + var job = rig.Grain.GetSnapshotForTest().InboxJob; + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId); + ReadOnlyMemory storedPayload = default; + await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Envelope.Payload); + var failure = await Assert.ThrowsAsync(() => DeliverOnTurnAsync(rig, conflict)); + Assert.Contains(key.ToString(), failure.Message, StringComparison.Ordinal); + Assert.Contains("different command", failure.Message, StringComparison.Ordinal); + await OnTurnAsync(rig.Context, () => + { + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, Assert.Single(rig.Pending).Value.Envelope)); + Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(Assert.Single(rig.Pending).Value.Envelope)); + Assert.Equal(storedPayload, Assert.Single(rig.Pending).Value.Envelope.Payload); + Assert.Equal(original.Envelope.Payload.ToArray(), Assert.Single(rig.Pending).Value.Envelope.Payload.ToArray()); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().Effects); + Assert.Same(job, rig.Grain.GetSnapshotForTest().InboxJob); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + }); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId)); + await OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "healthy"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, original)).Status); + handler.Release.TrySetResult(); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, handler.Calls); + Assert.Equal(1, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletedCommand_ChangedSenderSubjectAndBodyAcknowledgesWithoutFingerprint(bool restart) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1044", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = Create(rig, key, "completely.changed.subject.v2", "different-body", + GrainId.Create("forwarder", "new-owner")); + handler.Release.TrySetResult(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + await Fixture.SnapshotProbe.WaitAsync(rig.Receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + var prior = rig.Context; + if (restart) + { + await rig.Receiver.RequestDeactivationAsync(); + await WaitAsync(prior.Deactivated); + await rig.Receiver.GetSnapshotAsync(); + Assert.NotSame(prior, Fixture.GetGrainContext(rig.Receiver)); + } + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, repeated)).Status); + var completed = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId)); + } + + [Fact] + public async Task ExactParentChildAndSiblingCommandsHaveIndependentCompletions() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var parent = HierarchicalKey.Create("tenant", "orders", "1045"); + var keys = new[] { parent, parent.CreateChildKey("reserve"), parent.CreateChildKey("release") }; + for (var index = 0; index < keys.Length; index++) + { + var command = Create(rig, keys[index], "inventory.command.v1", $"command-{index}"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, command)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, index + 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, command)).Status); + } + var completed = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(3, completed.Effects.Count); + Assert.Equal(3, completed.ProcessedMessageCount); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + Assert.Equal(keys.OrderBy(static key => key.ToString(), StringComparer.Ordinal), + completed.Effects.Select(static effect => effect.LogicalId).OrderBy(static key => key.ToString(), StringComparer.Ordinal)); + } + + [Fact] + public async Task SameCommandKey_InDifferentReceiversCompletesIndependently() + { + var first = await CreateAsync(); + var second = await CreateAsync(); + using var firstHandler = first.Handler; + using var secondHandler = second.Handler; + firstHandler.Release.TrySetResult(); + secondHandler.Release.TrySetResult(); + var key = HierarchicalKey.Create("tenant", "orders", "1049", "reserve"); + var one = Create(first, key, "inventory.reserve.v1", "first-receiver"); + var two = Create(second, key, "inventory.reserve.v1", "second-receiver"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(first.Receiver, one)).Status); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(second.Receiver, two)).Status); + var firstOutcome = await Fixture.WaitForEffectCountAsync(first.Receiver, 1); + var secondOutcome = await Fixture.WaitForEffectCountAsync(second.Receiver, 1); + Assert.Equal(key, Assert.Single(firstOutcome.Effects).LogicalId); + Assert.Equal(key, Assert.Single(secondOutcome.Effects).LogicalId); + Assert.Equal(1, firstOutcome.ProcessedMessageCount); + Assert.Equal(1, secondOutcome.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(first.Receiver, one)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(second.Receiver, two)).Status); + Assert.Equal(1, firstHandler.Calls); + Assert.Equal(1, secondHandler.Calls); + } + + [Fact] + public async Task PendingIdentityAfterReplayStillRejectsConflictAndDeduplicatesNewSender() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var timers = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var key = HierarchicalKey.Create("tenant", "orders", "1046", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var timer = Assert.Single(timers.Events.Select(static item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, rig.Context) + && item.Timer.GetType().GenericTypeArguments is [var type] + && type.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension") + && type.Name == "LocalDrainTimerState").Timer; + await OnTurnAsync(rig.Context, () => + { + timer.Dispose(); + rig.Context.Deactivate(new DeactivationReason( + DeactivationReasonCode.ApplicationRequested, "Verify pending command identity replay."), + TestContext.Current.CancellationToken); + }); + await WaitAsync(rig.Context.Deactivated); + using var recoveredHandler = new IdentityHandler(); + using var read = Fixture.Storage.BlockRead(JournalId.FromGrainId(rig.Context.GrainId)); + var activation = rig.Receiver.GetSnapshotAsync(); + await read.WaitUntilEnteredAsync(); + var recoveredContext = Fixture.GetGrainContext(rig.Receiver); + var recoveredGrain = Assert.IsType(recoveredContext.GrainInstance); + await OnTurnAsync(recoveredContext, () => recoveredGrain.HandlerOverride = recoveredHandler); + read.Release(); + await WaitAsync(recoveredHandler.Entered.Task); + var recovered = CreateRig(rig.Receiver, recoveredContext, recoveredHandler); + Assert.NotSame(rig.Context, recovered.Context); + var repeated = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("forwarder", "after-replay")); + var conflict = Create(recovered, key, "inventory.reserve.v1", "changed", + GrainId.Create("forwarder", "after-replay")); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(recovered, repeated)).Status); + await Assert.ThrowsAsync(() => DeliverOnTurnAsync(recovered, conflict)); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, Assert.Single(recovered.Pending).Value.Envelope)); + Assert.Empty(recovered.Processed); + Assert.False(recovered.Grain.DeactivationFailure.Task.IsCompleted); + recoveredHandler.Release.TrySetResult(); + await activation; + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + } + + [Fact] + public async Task DeadLetterInspectionAndRemovalUseReceiverLocalKeyAcrossReplay() + { + var receiver = NewGrain(); + var key = HierarchicalKey.Create("tenant", "orders", "1048", "invalid-command"); + var body = new TestMessage(key, 611, "poison", ThrowDuringPreparation: true); + var original = CreateEnvelope(receiver, body); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + var deadLetter = Assert.Single((await Fixture.WaitForDeadLetterCountAsync(receiver, 1)).InboxDeadLetters); + Assert.Equal(key, deadLetter.MessageId); + var old = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(old.Deactivated); + var recovered = await receiver.GetSnapshotAsync(); + Assert.Equal(key, Assert.Single(recovered.InboxDeadLetters).MessageId); + var duplicate = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("forwarder", "deadletter"), subject: "another.subject.v1"); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, duplicate)).Status); + Assert.True(await receiver.RemoveInboxDeadLetterAsync(key)); + Assert.False(await receiver.RemoveInboxDeadLetterAsync(key)); + var current = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, duplicate)).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MissingOptionalMetadata_RawHandlerCompletesWithoutSubject(bool removeSender) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "invalid-subject", "command"); + var envelope = Create(rig, key, "valid.subject.v1", "missing-subject"); + var raw = TestApplicationProtocol.With(envelope, Sessions, removeSubject: true, removeSender: removeSender); + var bytes = envelope.Envelope.Payload.ToArray(); + handler.Release.TrySetResult(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverOnTurnAsync(rig, raw)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(raw.Envelope.TryGetSubject(out _)); + if (removeSender) Assert.False(raw.Envelope.TryGetBytes(MessageHeaders.Sender, out _)); + Assert.Equal(bytes, raw.Envelope.Payload.ToArray()); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, raw)).Status); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + [Theory] + [InlineData("key-bytes")] + [InlineData("key-utf8")] + [InlineData("key-depth")] + [InlineData("subject-bytes")] + [InlineData("subject-utf8")] + public async Task OversizedMetadata_RejectsBeforeSchedulingOrPersistence(string field) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = field switch + { + "key-bytes" => HierarchicalKey.Create(new string('x', 1025)), + "key-utf8" => HierarchicalKey.Create(new string('\u00e9', 513)), + "key-depth" => HierarchicalKey.Create(Enumerable.Repeat("segment", 33).ToArray()), + _ => HierarchicalKey.Create("tenant", "bounded", "command") + }; + var subject = field switch + { + "subject-bytes" => new string('x', 257), + "subject-utf8" => new string('\u00e9', 129), + _ => "inventory.reserve.v1" + }; + var writes = Writes(rig); + Assert.ThrowsAny(() => Create(rig, key, subject, "oversized")); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId)); + Assert.Empty(rig.Pending); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().Effects); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + Assert.False(handler.Entered.Task.IsCompleted); + } + + [Theory] + [InlineData("key-bytes")] + [InlineData("key-utf8")] + [InlineData("key-depth")] + [InlineData("subject-bytes")] + [InlineData("subject-utf8")] + [InlineData("subject-space")] + public async Task MetadataAtExactDefaultLimit_IsAccepted(string field) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var key = field switch + { + "key-bytes" => HierarchicalKey.Create(new string('x', 1024)), + "key-utf8" => HierarchicalKey.Create(new string('\u00e9', 512)), + "key-depth" => HierarchicalKey.Create(Enumerable.Repeat("segment", 32).ToArray()), + _ => HierarchicalKey.Create("tenant", "bounded", "command") + }; + var subject = field switch + { + "subject-bytes" => new string('x', 256), + "subject-utf8" => new string('\u00e9', 128), + "subject-space" => " ", + _ => "inventory.reserve.v1" + }; + var envelope = Create(rig, key, subject, "maximum"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, envelope)).Status); + } + + [Fact] + public async Task ExpiredIdentityCanAcceptChangedSenderSubjectAndBodyAtExactBoundary() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var key = HierarchicalKey.Create("tenant", "orders", "1047", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = Create(rig, key, "inventory.reserve.v2", "changed", + GrainId.Create("forwarder", "new")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + await Fixture.SnapshotProbe.WaitAsync(rig.Receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, repeated)).Status); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, repeated)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 2); + Assert.Equal(2, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(2, handler.Calls); + } + + private async Task CreateAsync() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var handler = new IdentityHandler(); + var rig = CreateRig(receiver, context, handler); + await OnTurnAsync(context, () => rig.Grain.HandlerOverride = handler); + return rig; + } + + private static Rig CreateRig(IMessagingTestGrain receiver, IGrainContext context, IdentityHandler handler) => + new(receiver, context, Assert.IsType(context.GrainInstance), handler, + context.ActivationServices.GetRequiredService(), + context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox"), + context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed")); + + private InboxMessage Create(Rig rig, HierarchicalKey key, string subject, string value, GrainId sender = default) => + TestApplicationProtocol.CreateInbox(Sessions, + sender.IsDefault ? GrainId.Create("producer", "first") : sender, + subject, new TestMessage(key, 610, value), key); + + private int Writes(Rig rig) => Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(rig.Context.GrainId)); + + private static Task DeliverOnTurnAsync(Rig rig, InboxMessage envelope) + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + rig.Context.Scheduler.QueueAction(() => + { + try + { + Assert.Same(rig.Context, ReceiverTestServices.CurrentGrainContext); + var extension = (IInboxExtension)rig.Context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + started.SetResult(extension.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); + } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = CompleteAsync(); }); + return done.Task; + async Task CompleteAsync() + { + try + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + await action(); + done.SetResult(); + } + catch (Exception exception) { done.SetException(exception); } + } + } + + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, + MessagingTestGrain Grain, IdentityHandler Handler, IJournaledStateManager Manager, + IDurableDictionary Pending, + IDurableDictionary Processed); + + private sealed class IdentityHandler : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int Calls { get; private set; } + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + var services = ReceiverTestServices.CurrentGrainContext!.ActivationServices; + var effects = services.GetRequiredKeyedService>("test-effects"); + var application = TestApplicationProtocol.Read(services.GetRequiredService(), context.Message); + var body = Assert.IsType(application.Body); + Calls++; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + effects.TryGetValue(context.Message.Envelope.MessageId, out var prior); + effects[context.Message.Envelope.MessageId] = new MessageEffect(context.Message.Envelope.MessageId, + (prior?.Count ?? 0) + 1, body.Sequence, body.Value); + context.Complete(); + } + public void Dispose() => Release.TrySetResult(); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs new file mode 100644 index 00000000000..b5e286df763 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs @@ -0,0 +1,118 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxLifecycleBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task Deactivation_WaitsForActiveInboxTimerBeforeDisposingShutdownSource() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var shutdown = GetShutdownSource(context); + var token = shutdown.Token; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName, GrainLifecycleEvents.ListenerName); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/teardown-active"); + var envelope = CreateEnvelope(receiver, NewMessage(95, "active-timer"), "messages/teardown-active"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var started = Assert.Single(events.Events.Select(static item => item.Payload) + .OfType(), item => ReferenceEquals(item.GrainContext, context)); + var tickStopped = events.WaitForEventAsync( + nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, started.Timer), + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + var stoppedAtCancellation = false; + var cancellations = 0; + using var registration = token.Register(() => + { + stoppedAtCancellation = events.Events.Any(item => + item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, started.Timer)); + Interlocked.Increment(ref cancellations); + }); + + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Verify active inbox timer teardown."), TestContext.Current.CancellationToken); + + Assert.False(context.Deactivated.IsCompleted); + Assert.False(tickStopped.IsCompleted); + Assert.False(token.IsCancellationRequested); + Assert.Equal(token, shutdown.Token); + handler.Release(); + var stopped = Assert.IsType((await tickStopped).Payload); + await WaitForDeactivationAsync(context); + + Assert.Null(stopped.Exception); + Assert.True(stoppedAtCancellation); + Assert.Equal(1, cancellations); + Assert.True(token.IsCancellationRequested); + Assert.Throws(() => shutdown.Token); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task Deactivation_DisposesQueuedInboxTimerBeforeScopeTeardownAndRecoveryDrainsOnce() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var shutdown = GetShutdownSource(context); + var token = shutdown.Token; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName, GrainLifecycleEvents.ListenerName); + var envelope = CreateEnvelope(receiver, NewMessage(96, "queued-timer")); + + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync(envelope)).Status); + await WaitForDeactivationAsync(context); + + var captured = events.Events.Select(static item => item.Payload).ToList(); + var created = Assert.Single(captured.OfType(), + item => ReferenceEquals(item.GrainContext, context)); + Assert.DoesNotContain(captured.OfType(), item => ReferenceEquals(item.GrainContext, context)); + var disposed = Assert.Single(captured.OfType(), + item => ReferenceEquals(item.Timer, created.Timer)); + var deactivated = Assert.Single(captured.OfType(), + item => ReferenceEquals(item.GrainContext, context)); + Assert.True(captured.IndexOf(disposed) < captured.IndexOf(deactivated)); + Assert.True(token.IsCancellationRequested); + Assert.Throws(() => shutdown.Token); + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + } + + private static CancellationTokenSource GetShutdownSource(IGrainContext context) + { + var extensionType = ReceiverTestServices.GetImplementationType("InboxExtension"); + var extension = context.ActivationServices.GetRequiredService(extensionType); + return (CancellationTokenSource)extensionType.GetField("_shutdownCts", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(extension)!; + } + + private static async Task WaitForDeactivationAsync(IGrainContext context) + { + try + { + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + catch (TimeoutException exception) + { + throw new TimeoutException($"Inbox activation {context.GrainId} did not complete timer/lifecycle teardown.", exception); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxMissingHandlerTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxMissingHandlerTests.cs new file mode 100644 index 00000000000..4fa27922827 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxMissingHandlerTests.cs @@ -0,0 +1,492 @@ +using System.Collections; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Orleans.Timers; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxMissingHandlerTests() : MessagingBehaviorTestBase(new RetryFixture()) +{ + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandlerInvocationFailure_BeforeCompletePreservesOriginalCauseAndRetryState(bool deferred) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var failure = new IOException("Original application invocation failure."); + var armed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var handler = new InvocationFaultHandler(failure, deferred, () => armed.TrySetResult(Fixture.Storage.BlockWrite(journal))); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = WaitForPumpAsync(events, receiver.GetGrainId()); + var envelope = CreateEnvelope(receiver, NewMessage(192, "invocation-retry")); + var now = Fixture.Clock.GetUtcNow(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + var owner = grain.GetSnapshotForTest().InboxJob; + var ownershipId = grain.GetSnapshotForTest().InboxJobId; + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (deferred) + { + Assert.False(armed.Task.IsCompleted); + var initialEntry = Assert.Single(GetAttemptStates(context).Cast()); + var initialState = initialEntry.GetType().GetProperty("Value")!.GetValue(initialEntry)!; + Assert.Equal(0, initialState.GetType().GetProperty("AttemptCount")!.GetValue(initialState)); + Assert.Null(initialState.GetType().GetProperty("LastError")!.GetValue(initialState)); + Assert.Null(initialState.GetType().GetProperty("NextAttemptAt")!.GetValue(initialState)); + Assert.Equal(1, grain.GetSnapshotForTest().InboxCount); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.Equal(0, grain.GetSnapshotForTest().ProcessedMessageCount); + Assert.Equal(0, grain.GetSnapshotForTest().OutboxCount); + handler.Release.TrySetResult(); + } + using var accounting = await armed.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + await accounting.WaitUntilEnteredAsync(); + var pending = CancellationCleanupProbe.Field(extension, "_pendingWrites"); + var operation = Assert.Single(pending.Cast(), item => item.GetType().Name == "HandlerWrite"); + Assert.Same(failure, operation.GetType().GetProperty("Error")!.GetValue(operation)); + Assert.Equal(false, operation.GetType().GetProperty("Completed")!.GetValue(operation)); + AssertPendingRetry(context, grain.GetSnapshotForTest(), 1, now + TimeSpan.FromMinutes(1)); + Assert.Same(owner, grain.GetSnapshotForTest().InboxJob); + Assert.Equal(ownershipId, grain.GetSnapshotForTest().InboxJobId); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + await OnTurnAsync(context, () => grain.HandlerOverride = null); + accounting.Release(); + await stopped; + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + AssertPendingRetry(context, await receiver.GetSnapshotAsync(), 1, now + TimeSpan.FromMinutes(1)); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + using var retry = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/record"); + Fixture.Clock.Advance(TimeSpan.FromMinutes(1)); + // The grain-facing retry clock is fake while shard-executor scheduling uses + // real background time. Request the authoritative callback explicitly so this + // test does not depend on whether the original background dequeue happened + // before or after the fake-time advance. RunPumpAsync still runs the real + // non-interleaving inbox timer and awaits that exact requested run's outcome. + var retryPump = RunPumpAsync(receiver, Assert.IsType(owner)); + await retry.WaitUntilEnteredAsync(); + AssertPendingRetry(context, grain.GetSnapshotForTest(), 1, now + TimeSpan.FromMinutes(1)); + retry.Release(); + Assert.Equal(DurableJobRunStatus.Completed, (await retryPump).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(1, ScheduleCount(receiver)); + Assert.Same(context, Fixture.GetGrainContext(receiver)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + private sealed class InvocationFaultHandler(Exception failure, bool deferred, Action beforeThrow) : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Entered.TrySetResult(); + if (deferred) return ThrowAfterBarrierAsync(cancellationToken); + beforeThrow(); + throw failure; + } + private async ValueTask ThrowAfterBarrierAsync(CancellationToken cancellationToken) + { + await Release.Task.WaitAsync(cancellationToken); + beforeThrow(); + throw failure; + } + public void Dispose() => Release.TrySetResult(); + } + + [Fact] + public async Task AcceptedHandlerMissingOnFreshActivation_DeadLettersImmediatelyAndRetainsDedupe() + { + var receiver = NewGrain(); + const string route = "activation-only/handler"; + _ = await receiver.GetSnapshotAsync(); + await receiver.ConfigureHandlerAsync(false); + var originalContext = Fixture.GetGrainContext(receiver); + var originalGrain = Assert.IsType(originalContext.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(190, "missing-after-replay"), route); + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync(envelope)).Status); + await originalContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + var accepted = Assert.IsType(originalGrain.AcceptedSnapshot); + Assert.Equal(1, accepted.InboxCount); + Assert.Empty(accepted.Effects); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + var now = Fixture.Clock.GetUtcNow(); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = WaitForPumpAsync(events, receiver.GetGrainId()); + + _ = await receiver.GetSnapshotAsync(); + await stopped; + var completed = await receiver.GetSnapshotAsync(); + + Assert.NotEqual(accepted.ActivationId, completed.ActivationId); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.Effects); + Assert.Equal(0, completed.OutboxCount); + Assert.Equal(0, completed.MaxConcurrentHandlers); + var deadLetter = Assert.Single(completed.InboxDeadLetters); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal("No inbox handler is registered.", deadLetter.Reason); + Assert.Equal(0, deadLetter.AttemptCount); + Assert.Equal(now, deadLetter.DeadLetteredAt); + var context = Fixture.GetGrainContext(receiver); + var retained = Assert.Single(context.ActivationServices.GetRequiredService().InboxDeadLetters); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, envelope.Envelope), TestApplicationProtocol.Sender(Sessions, retained.Message.Envelope)); + Assert.Empty(GetAttemptStates(context)); + var processed = context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed"); + Assert.Equal(now, processed[envelope.Envelope.MessageId]); + Assert.Equal(job.Id, completed.InboxJob!.Id); + Assert.Equal(job.ShardId, completed.InboxJob.ShardId); + Assert.Equal(accepted.InboxJobId, completed.InboxJobId); + Assert.Equal(1, ScheduleCount(receiver)); + Assert.False(Assert.IsType(context.GrainInstance).DeactivationFailure.Task.IsCompleted); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + + Assert.Equal(DurableJobRunStatus.Completed, (await RunPumpAsync(receiver, job)).Status); + var retired = await receiver.GetSnapshotAsync(); + Assert.Null(retired.InboxJobId); + Assert.Null(retired.InboxJob); + Assert.Equal(1, ScheduleCount(receiver)); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(deadLetter, Assert.Single(recovered.InboxDeadLetters)); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Empty(recovered.Effects); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(deadLetter, Assert.Single((await receiver.GetSnapshotAsync()).InboxDeadLetters)); + Assert.Equal(1, ScheduleCount(receiver)); + } + + [Fact] + public Task OrdinaryHandlerFailures_RetryUntilExactConfiguredAttemptLimit() => + RetryUntilExactConfiguredAttemptLimitAsync(startBackgroundBeforeDuplicate: false); + + [Fact] + public Task RetryDuplicateDelivery_QueuedBehindActiveHandler_DoesNotDeadlock() => + RetryUntilExactConfiguredAttemptLimitAsync(startBackgroundBeforeDuplicate: true); + + private async Task RetryUntilExactConfiguredAttemptLimitAsync(bool startBackgroundBeforeDuplicate) + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(191, "retry-limit") with { ThrowDuringPreparation = true }); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = WaitForPumpAsync(events, receiver.GetGrainId()); + var now = Fixture.Clock.GetUtcNow(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await stopped; + var first = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + AssertPendingRetry(context, first, expectedAttempts: 1, now + TimeSpan.FromMinutes(1)); + var job = Assert.IsType(first.InboxJob); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await RunPumpAsync(receiver, job)).Status); + AssertPendingRetry(context, await receiver.GetSnapshotAsync(), expectedAttempts: 1, now + TimeSpan.FromMinutes(1)); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + + Task? background = null; + using var tick = startBackgroundBeforeDuplicate + ? Fixture.Clock.CreateTimer(_ => background = RunPumpAsync(receiver, job), null, TimeSpan.FromMinutes(1), Timeout.InfiniteTimeSpan) + : null; + using (var runningLocalDrain = ArmHandlerBeforeAdvance(receiver.GetGrainId(), TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), envelope).Route, TimeSpan.FromMinutes(1))) + { + if (startBackgroundBeforeDuplicate) + { + await runningLocalDrain.WaitUntilEnteredAsync(); + } + var duplicate = DeliverAsync(receiver, envelope); + await runningLocalDrain.WaitUntilEnteredAsync(); + var retry = startBackgroundBeforeDuplicate + ? Assert.IsAssignableFrom>(background) + : RunPumpAsync(receiver, job); + await OnTurnAsync(context, static () => { }); + await AssertUnrelatedTimerDoesNotCompleteAsync(context, events, retry); + if (startBackgroundBeforeDuplicate) + { + Assert.False(duplicate.IsCompleted); + } + runningLocalDrain.Release(); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await retry).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + } + AssertPendingRetry(context, await receiver.GetSnapshotAsync(), expectedAttempts: 2, now + TimeSpan.FromMinutes(3)); + using var runningRequestedPump = ArmHandlerBeforeAdvance(receiver.GetGrainId(), TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), envelope).Route, TimeSpan.FromMinutes(2)); + var terminal = RunPumpAsync(receiver, job); + await runningRequestedPump.WaitUntilEnteredAsync(); + await AssertUnrelatedTimerDoesNotCompleteAsync(context, events, terminal); + runningRequestedPump.Release(); + Assert.Equal(DurableJobRunStatus.Completed, (await terminal).Status); + var completed = await receiver.GetSnapshotAsync(); + var deadLetter = Assert.Single(completed.InboxDeadLetters); + Assert.Equal(3, deadLetter.AttemptCount); + Assert.Contains("Injected handler preparation failure", deadLetter.Reason, StringComparison.Ordinal); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Empty(completed.Effects); + Assert.Equal(0, completed.OutboxCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(GetAttemptStates(context)); + Assert.Null(completed.InboxJobId); + Assert.Equal(1, ScheduleCount(receiver)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.False(Assert.IsType(context.GrainInstance).DeactivationFailure.Task.IsCompleted); + } + + [Fact] + public async Task RetryBarrier_IsArmedWhenClockMakesBackgroundPumpEligible() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(192, "clock-boundary") with { ThrowDuringPreparation = true }); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = WaitForPumpAsync(events, receiver.GetGrainId()); + var now = Fixture.Clock.GetUtcNow(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await stopped; + var first = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var job = Assert.IsType(first.InboxJob); + AssertPendingRetry(context, first, expectedAttempts: 1, now + TimeSpan.FromMinutes(1)); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId())); + HandlerProbe.Barrier? observedBarrier = null; + Task? background = null; + using var tick = Fixture.Clock.CreateTimer(_ => + { + Fixture.HandlerProbe.TryGet(receiver.GetGrainId(), TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), envelope).Route, out observedBarrier); + background = RunPumpAsync(receiver, job); + }, null, TimeSpan.FromMinutes(1), Timeout.InfiniteTimeSpan); + + using var handler = ArmHandlerBeforeAdvance(receiver.GetGrainId(), TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), envelope).Route, TimeSpan.FromMinutes(1)); + Assert.Same(handler, observedBarrier); + Assert.NotNull(background); + await handler.WaitUntilEnteredAsync(); + await OnTurnAsync(context, () => AssertPendingRetry(context, grain.GetSnapshotForTest(), expectedAttempts: 1, now + TimeSpan.FromMinutes(1))); + Assert.False(background.IsCompleted); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + await AssertUnrelatedTimerDoesNotCompleteAsync(context, events, background); + handler.Release(); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await background).Status); + AssertPendingRetry(context, await receiver.GetSnapshotAsync(), expectedAttempts: 2, now + TimeSpan.FromMinutes(3)); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.Equal(1, ScheduleCount(receiver)); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + private HandlerProbe.Barrier ArmHandlerBeforeAdvance(GrainId grainId, string route, TimeSpan advance) + { + var handler = Fixture.HandlerProbe.Arm(grainId, route); + try + { + Fixture.Clock.Advance(advance); + return handler; + } + catch + { + handler.Dispose(); + throw; + } + } + + private static async Task AssertUnrelatedTimerDoesNotCompleteAsync(IGrainContext context, DiagnosticEventCollector events, Task pump) + { + IGrainTimer unrelated = null!; + await OnTurnAsync(context, () => unrelated = context.ActivationServices.GetRequiredService() + .RegisterGrainTimer(context, static (_, _) => Task.CompletedTask, 0, + new GrainTimerCreationOptions(TimeSpan.Zero, Timeout.InfiniteTimeSpan) { Interleave = true })); + using (unrelated) + { + await events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, unrelated), + TimeSpan.FromSeconds(30), Cancellation); + Assert.False(pump.IsCompleted); + } + } + + private static void AssertPendingRetry(IGrainContext context, EndpointSnapshot snapshot, int expectedAttempts, DateTimeOffset nextAttempt) + { + Assert.Equal(1, snapshot.InboxCount); + Assert.Equal(0, snapshot.ProcessedMessageCount); + Assert.Empty(snapshot.InboxDeadLetters); + Assert.Empty(snapshot.Effects); + Assert.Equal(0, snapshot.OutboxCount); + var entry = Assert.Single(GetAttemptStates(context).Cast()); + var state = entry.GetType().GetProperty("Value")!.GetValue(entry)!; + Assert.Equal(expectedAttempts, state.GetType().GetProperty("AttemptCount")!.GetValue(state)); + Assert.Equal(nextAttempt, state.GetType().GetProperty("NextAttemptAt")!.GetValue(state)); + } + private static IEnumerable GetAttemptStates(IGrainContext context) + { + var manager = context.ActivationServices.GetRequiredService(); + Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox-message-state", out var state)); + return Assert.IsAssignableFrom(state); + } + private int ScheduleCount(IMessagingTestGrain receiver) => + Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId()); + private static async Task WaitForPumpAsync(DiagnosticEventCollector events, GrainId grainId) + { + var created = await events.WaitForEventAsync(nameof(GrainTimerEvents.Created), + item => item.Payload is GrainTimerEvents.Created timer && timer.GrainContext.GrainId == grainId && IsInboxTimer(timer.Timer), + TimeSpan.FromSeconds(30), Cancellation); + var timer = Assert.IsType(created.Payload).Timer; + return await events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), Cancellation); + } + private static bool IsInboxTimer(IGrainTimer timer) => timer.GetType().GenericTypeArguments is [var state] + && state.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension"); + private async Task RunPumpAsync(IMessagingTestGrain receiver, DurableJob job) + { + var context = Fixture.GetGrainContext(receiver); + var feature = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var completion = new PumpCompletion(context, feature, new PumpContext(job)); + using var subscription = GrainTimerEvents.AllEvents.Subscribe(completion); + await OnTurnAsync(context, completion.Start); + return await completion.Result.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + } + private sealed class PumpCompletion(IGrainContext context, IDurableJobFeatureHandler feature, IJobRunContext run) + : IObserver + { + private IGrainTimer? _timer; + private bool _capturing; + private bool _started; + public TaskCompletionSource Result { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Start() + { + _started = true; + StartRequestedPump(allowCoalescing: true); + } + private void StartRequestedPump(bool allowCoalescing) + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + DurableJobRunResult result; + _capturing = true; + try + { + result = feature.ExecuteJobAsync(run, Cancellation).GetAwaiter().GetResult(); + } + finally + { + _capturing = false; + } + if (result.IsInProgress && _timer is null) + { + // Bind only this exact requested payload, never an unrelated coalesced turn. + var state = feature.GetType().GetField("_pumpTimer", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!.GetValue(feature); + if (state is not null) + { + var pending = state.GetType().BaseType!.GetField("_pending", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!.GetValue(state); + if (pending is not null) + { + var execution = pending.GetType().GetProperty("Execution")!.GetValue(pending)!; + var key = execution.GetType().GetProperty("Key")!.GetValue(execution)!; + if (Equals(run.RunId, key.GetType().GetProperty("RunId")!.GetValue(key)) + && Equals(run.Job.Id, key.GetType().GetProperty("JobId")!.GetValue(key))) + { + _timer = Assert.IsAssignableFrom(state.GetType().BaseType! + .GetField("_timer", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!.GetValue(state)); + } + } + } + } + if (!result.IsInProgress) + { + Result.TrySetResult(result); + } + else if (!allowCoalescing) + { + Assert.NotNull(_timer); + } + } + public void OnNext(GrainTimerEvents.TimerEvent item) + { + if (!ReferenceEquals(item.GrainContext, context) || Result.Task.IsCompleted) return; + try + { + if (item is GrainTimerEvents.Created && _capturing) + { + Assert.Null(_timer); + _timer = item.Timer; + } + else if (item is GrainTimerEvents.TickStop stop && _started) + { + if (ReferenceEquals(stop.Timer, _timer)) + { + Assert.Null(stop.Exception); + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + var result = feature.ExecuteJobAsync(run, Cancellation).GetAwaiter().GetResult(); + Assert.False(result.IsInProgress); + Result.TrySetResult(result); + } + else if (_timer is null && IsInboxTimer(stop.Timer)) + { + // The coalesced pump released its lease before this event; start the requested run on this turn. + Assert.Null(stop.Exception); + StartRequestedPump(allowCoalescing: false); + } + } + } + catch (Exception exception) + { + Result.TrySetException(exception); + } + } + public void OnError(Exception error) => Result.TrySetException(error); + public void OnCompleted() { } + } + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } + private sealed class PumpContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + private sealed class RetryFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxProcessingAttempts = 3; + options.BackpressureRetryDelay = TimeSpan.FromMinutes(1); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxPumpResultLifetimeTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxPumpResultLifetimeTests.cs new file mode 100644 index 00000000000..80608adae59 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxPumpResultLifetimeTests.cs @@ -0,0 +1,241 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxPumpResultLifetimeTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task ReplacedQueuedPump_DiscardsUnconsumableExecutionAndRegistration() + { + var receiver = NewGrain(); + var oldJob = CreateJob(receiver, "replaced:1"); + await receiver.SetInboxOwnershipAsync("replaced:1", oldJob); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var results = GetEntries(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-replacement"); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/replacement"); + var envelope = CreateEnvelope(receiver, NewMessage(170, "replacement"), "messages/replacement"); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var turn = receiver.HoldPumpTurnAsync("hold-replacement", envelope, deactivate: false); + await hold.WaitUntilEnteredAsync(); + var run = CreateRun(oldJob); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeAsync(receiver, run)).Status); + var entry = Assert.Single(results.Values.Cast()); + Assert.NotEqual(default, GetRegistration(entry)); + var timer = Assert.Single(events.Events.Select(static e => e.Payload).OfType(), e => ReferenceEquals(e.GrainContext, context)).Timer; + var stopped = events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + e => e.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + hold.Release(); + await turn; + await stopped; + await handler.WaitUntilEnteredAsync(); + + Assert.DoesNotContain(results.Keys.Cast(), key => GetJobId(key) == oldJob.Id); + Assert.Equal(default, GetRegistration(entry)); + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeAsync(receiver, CreateRun(oldJob, 2))).Status); + Assert.False(Assert.IsType(context.GrainInstance).DeactivationFailure.Task.IsCompleted); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + } + + [Fact] + public async Task OwnerClearCompletion_PollConsumesResultBeforeOwnershipMismatchReturns() + { + var receiver = NewGrain(); + var job = CreateJob(receiver, "clear:1"); + await receiver.SetInboxOwnershipAsync("clear:1", job); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var entries = GetEntries(context); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + e => e.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.GrainContext, context), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var run = CreateRun(job); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeAsync(receiver, run)).Status); + await stopped; + Assert.Null((await receiver.GetSnapshotAsync()).InboxJobId); + Assert.Single(entries); + var feature = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + DurableJobRunResult result = null!; + await OnTurnAsync(context, () => result = feature.ExecuteJobAsync(run, TestContext.Current.CancellationToken).GetAwaiter().GetResult()); + Assert.Equal(DurableJobRunStatus.Completed, result.Status); + Assert.Empty(entries); + } + + [Fact] + public async Task ShutdownWithQueuedPump_ReleasesRegistrationAndFreshActivationRetries() + { + var receiver = NewGrain(); + var job = CreateJob(receiver, "shutdown:1"); + await receiver.SetInboxOwnershipAsync("shutdown:1", job); + await RefreshSeededOwnerAsync(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(171, "fresh-after-stop")); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var entries = GetEntries(context); + var feature = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-stop"); + var turn = receiver.HoldPumpTurnAsync("hold-stop", deactivate: true); + await hold.WaitUntilEnteredAsync(); + var run = CreateRun(job); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeAsync(receiver, run)).Status); + var entry = Assert.Single(entries.Values.Cast()); + Assert.NotEqual(default, GetRegistration(entry)); + var messages = context.ActivationServices.GetRequiredKeyedService>( + "__orleans.messaging.inbox"); + var manager = context.ActivationServices.GetRequiredService(); + Task persisted = null!; + await OnTurnAsync(context, () => + { + messages.Add(envelope.Envelope.MessageId, envelope); + persisted = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + }); + await persisted; + hold.Release(); + await turn; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Empty(entries); + Assert.Equal(default, GetRegistration(entry)); + await Assert.ThrowsAnyAsync(async () => await feature.ExecuteJobAsync(run, TestContext.Current.CancellationToken)); + Assert.Empty(grain.GetSnapshotForTest().Effects); + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + } + + [Fact] + public async Task TerminalFaultWithQueuedPump_ReleasesEntryWithoutCompletingDurableJob() + { + var receiver = NewGrain(); + var job = CreateJob(receiver, "fault:1"); + await receiver.SetInboxOwnershipAsync("fault:1", job); + await RefreshSeededOwnerAsync(receiver); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 172, "uncommitted")); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var entries = GetEntries(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-fault"); + var turn = receiver.HoldPumpTurnAsync("hold-fault", deactivate: false); + await hold.WaitUntilEnteredAsync(); + var run = CreateRun(job); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeAsync(receiver, run)).Status); + var entry = Assert.Single(entries.Values.Cast()); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var failedWrite = new DurableJob + { + Id = Guid.NewGuid().ToString("N"), + ShardId = "pump-result-lifetime", + Name = "test/write-journal", + TargetGrainId = receiver.GetGrainId(), + DueTime = DateTimeOffset.UtcNow + }; + + var result = await InvokeAsync(receiver, CreateRun(failedWrite)); + + Assert.Equal(DurableJobRunStatus.Failed, result.Status); + Assert.IsType(await grain.DeactivationFailure.Task); + hold.Release(); + await turn; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Empty(entries); + Assert.Equal(default, GetRegistration(entry)); + Assert.Equal(job.Id, grain.GetSnapshotForTest().InboxJob!.Id); + Assert.Single(grain.GetSnapshotForTest().Effects); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + + [Fact] + public async Task CanceledQueuedAttempt_DiscardsOnlySupersededRunAndAllowsRetry() + { + var receiver = NewGrain(); + var job = CreateJob(receiver, "attempt:1"); + await receiver.SetInboxOwnershipAsync("attempt:1", job); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var entries = GetEntries(context); + var feature = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-attempt"); + var turn = receiver.HoldPumpTurnAsync("hold-attempt", deactivate: false); + await hold.WaitUntilEnteredAsync(); + using var cancellation = new CancellationTokenSource(); + var canceledRun = CreateRun(job); + var retryRun = CreateRun(job, 2); + await OnTurnAsync(context, () => Assert.Equal(DurableJobRunStatus.InProgress, + feature.ExecuteJobAsync(canceledRun, cancellation.Token).GetAwaiter().GetResult().Status)); + cancellation.Cancel(); + await OnTurnAsync(context, () => + { + Assert.Throws(() => feature.ExecuteJobAsync(canceledRun, cancellation.Token).GetAwaiter().GetResult()); + Assert.Equal(DurableJobRunStatus.InProgress, + feature.ExecuteJobAsync(retryRun, TestContext.Current.CancellationToken).GetAwaiter().GetResult().Status); + }); + var queuedRetry = Assert.Single(entries.Keys.Cast()); + Assert.Equal(retryRun.RunId, queuedRetry.GetType().GetProperty("RunId")!.GetValue(queuedRetry)); + hold.Release(); + await turn; + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + _ = await receiver.GetSnapshotAsync(); + var retained = Assert.Single(entries.Keys.Cast()); + Assert.Equal(retryRun.RunId, retained.GetType().GetProperty("RunId")!.GetValue(retained)); + await OnTurnAsync(context, () => Assert.Equal(DurableJobRunStatus.Completed, + feature.ExecuteJobAsync(retryRun, TestContext.Current.CancellationToken).GetAwaiter().GetResult().Status)); + Assert.Empty(entries); + } + + private static IDictionary GetEntries(IGrainContext context) + { + var type = ReceiverTestServices.GetImplementationType("MessagingPumpResults"); + return (IDictionary)type.GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(context.ActivationServices.GetRequiredService(type))!; + } + private static string GetJobId(object key) => (string)key.GetType().GetProperty("JobId")!.GetValue(key)!; + private static CancellationTokenRegistration GetRegistration(object entry) => + (CancellationTokenRegistration)entry.GetType().GetProperty("CancellationRegistration")!.GetValue(entry)!; + private static DurableJob CreateJob(IMessagingTestGrain receiver, string owner) => new() + { + Id = Guid.NewGuid().ToString("N"), + ShardId = "pump-result-lifetime", + Name = ReceiverTestServices.InboxJobName, + TargetGrainId = receiver.GetGrainId(), + DueTime = DateTimeOffset.UtcNow, + Metadata = new Dictionary { ["orleans.messaging.ownership-id"] = owner } + }; + private static IJobRunContext CreateRun(DurableJob job, int dequeueCount = 1) => + (IJobRunContext)Activator.CreateInstance(typeof(DurableJob).Assembly.GetType("Orleans.DurableJobs.JobRunContext", throwOnError: true)!, + job, Guid.NewGuid().ToString("N"), dequeueCount)!; + private static async Task InvokeAsync(IMessagingTestGrain receiver, IJobRunContext run) + { + var type = typeof(DurableJob).Assembly.GetType("Orleans.DurableJobs.IDurableJobReceiverExtension", throwOnError: true)!; + return await (ValueTask)type.GetMethod("HandleDurableJobAsync")! + .Invoke(receiver.AsReference(type), [run, TestContext.Current.CancellationToken])!; + } + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs new file mode 100644 index 00000000000..b197dde574a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs @@ -0,0 +1,332 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxQuiescenceTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task AlwaysInterleaveCallback_ObservesAdmittedClearOnSameActivationScheduler() + { + var receiver = NewGrain(); + var owner = CreateJob(receiver, ReceiverTestServices.InboxJobName, "owner:1"); + await receiver.SetInboxOwnershipAsync("owner:1", owner); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + using var barrier = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeJobAsync(receiver, owner)).Status); + await barrier.WaitUntilEnteredAsync(); + var scheduler = barrier.EntryScheduler; + Assert.NotSame(TaskScheduler.Default, scheduler); + Assert.Same(context, barrier.EntryContext); + var operation = Assert.Single(GetPending(context).Cast()); + Assert.Equal("ClearOwnerWrite", operation.GetType().Name); + var finished = (TaskCompletionSource)operation.GetType().GetProperty("Finished")!.GetValue(operation)!; + + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeJobAsync(receiver, CreateJob(receiver, "test/probe-scheduler"))).Status); + Assert.Same(scheduler, grain.JobScheduler); + Assert.Same(context, grain.JobGrainContext); + for (var attempt = 2; attempt <= 4; attempt++) + { + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeJobAsync(receiver, owner, attempt)).Status); + Assert.Same(operation, Assert.Single(GetPending(context).Cast())); + } + Assert.Single(events.Events, item => item.Payload is GrainTimerEvents.Created created && ReferenceEquals(created.GrainContext, context)); + barrier.Release(); + await finished.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeJobAsync(receiver, CreateJob(receiver, "test/probe-scheduler"))).Status); + Assert.Same(scheduler, grain.JobScheduler); + Assert.Same(context, grain.JobGrainContext); + Assert.Empty(GetPending(context)); + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeJobAsync(receiver, owner, 5)).Status); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OwnerDelete_AfterAcknowledgedWork_StopsComponentsAndUsesFreshActivation(bool pump) + { + var receiver = NewGrain(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/delete-ack"); + var first = CreateEnvelope(receiver, NewMessage(160, "first"), "messages/delete-ack"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var outbox = GetOutbox(context); + var second = CreateEnvelope(receiver, NewMessage(161, "second"), "messages/delete-ack"); + Task? secondDelivery = null; + + if (pump) + { + var owner = Assert.IsType(grain.GetSnapshotForTest().InboxJob); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeJobAsync(receiver, owner)).Status); + } + else + { + secondDelivery = DeliverAsync(receiver, second); + } + + handler.Release(); + if (secondDelivery is not null) Assert.Equal(DeliveryStatus.Accepted, (await secondDelivery).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, pump ? 1 : 2); + Assert.Equal(pump ? 1 : 2, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + using var storage = Fixture.Storage.BlockDelete(JournalId.FromGrainId(receiver.GetGrainId())); + var deletion = receiver.DeleteStateAndDeactivateAsync(); + await storage.WaitUntilEnteredAsync(); + AssertStopped(extension, outbox); + Assert.False(deletion.IsCompleted); + storage.Release(); + await deletion; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var deleted = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, deleted.ActivationId); + Assert.Empty(deleted.Effects); + Assert.Equal(0, deleted.ProcessedMessageCount); + Assert.Null(deleted.InboxJobId); + } + + + [Fact] + public async Task OwnerDelete_DrainsCanceledDeliveryAndGateWaiterBeforeReset() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + using var blocked = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var first = CreateEnvelope(receiver, NewMessage(163, "owned-after-cancel")); + var second = CreateEnvelope(receiver, NewMessage(164, "gate-waiter")); + using var cancellation = new CancellationTokenSource(); + var delivery = DeliverWithCancellationAsync(receiver, first, cancellation.Token); + await blocked.WaitUntilEnteredAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery); + _ = await receiver.GetSnapshotAsync(); + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var token = TestContext.Current.CancellationToken; + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(second, token).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + var waiting = await started.Task; + Assert.False(waiting.IsCompleted); + var outbox = GetOutbox(context); + using var deletionStorage = Fixture.Storage.BlockDelete(JournalId.FromGrainId(receiver.GetGrainId())); + var deleteEntered = deletionStorage.WaitUntilEnteredAsync(); + var deletion = receiver.DeleteStateAndDeactivateAsync(); + await outbox.Stopping.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.False(deletion.IsCompleted); + Assert.False(deleteEntered.IsCompleted); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_gate").CurrentCount); + Assert.Single(CancellationCleanupProbe.Field>(extension, "_pendingOwnershipIds")); + blocked.Release(); + await Assert.ThrowsAnyAsync(() => waiting); + await deleteEntered; + AssertStopped(extension, outbox); + Assert.Equal(1, CancellationCleanupProbe.Field(extension, "_gate").CurrentCount); + Assert.Empty(CancellationCleanupProbe.Field>(extension, "_pendingOwnershipIds")); + Assert.Equal(1, grain.GetSnapshotForTest().InboxCount); + deletionStorage.Release(); + await deletion; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var fresh = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, fresh.ActivationId); + Assert.Equal(0, fresh.ProcessedMessageCount); + Assert.Equal(0, fresh.InboxCount); + Assert.Empty(fresh.Effects); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeliveryWhileOwnerClearIsAdmitted_WaitsAndAcceptsWithoutFencing(bool interleaved) + { + var receiver = NewGrain(); + var owner = CreateJob(receiver, ReceiverTestServices.InboxJobName, "clear-overlap:1"); + await receiver.SetInboxOwnershipAsync("clear-overlap:1", owner); + await RefreshSeededOwnerAsync(receiver); + var incoming = CreateEnvelope(receiver, NewMessage(181, "after-clear")); + await receiver.SetControlEnvelopeAsync(incoming); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + using var preparation = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeJobAsync(receiver, owner)).Status); + await preparation.WaitUntilEnteredAsync(); + Assert.Equal("ClearOwnerWrite", Assert.Single(GetPending(context).Cast()).GetType().Name); + Task delivery; + if (interleaved) + { + delivery = InvokeJobAsync(receiver, CreateJob(receiver, "test/deliver-envelope")); + await grain.ControlDeliveryEntered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + else + { + delivery = DeliverAsync(receiver, incoming); + } + Assert.False(delivery.IsCompleted); + preparation.Release(); + await delivery; + if (interleaved) + { + Assert.Equal(DurableJobRunStatus.Completed, (await (Task)delivery).Status); + } + else + { + Assert.Equal(DeliveryStatus.Accepted, (await (Task)delivery).Status); + } + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + Assert.Equal(1, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OwnerDeleteFailure_DeactivatesAndFreshOwnerObservesActualStorageOutcome(bool committed) + { + var receiver = NewGrain(); + var seed = new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 188, "preserved-before-delete"); + await receiver.StageEffectAsync(seed); + await receiver.RetryWriteStateAsync(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var outbox = GetOutbox(context); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journalId); + using var storage = Fixture.Storage.BlockDelete(journalId); + if (committed) Fixture.Storage.FailAfterDelete(journalId); + var deletion = receiver.DeleteStateAndDeactivateAsync(); + await storage.WaitUntilEnteredAsync(); + AssertStopped(extension, outbox); + if (committed) storage.Release(); + else storage.Fail(); + var failure = await Assert.ThrowsAsync(() => deletion); + var reason = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.IsType(reason); + Assert.Equal(failure.Message, reason.Message); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + Assert.Equal(seed, Assert.Single(grain.GetSnapshotForTest().Effects)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + if (committed) + { + Assert.Empty(recovered.Effects); + } + else + { + Assert.Equal(seed, Assert.Single(recovered.Effects)); + } + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(0, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task OwnerDelete_DiscardsStagedOutputAndLaterDeliveryUsesFreshOwner() + { + var receiver = NewGrain(); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 190, "delete-this")); + await receiver.RetryWriteStateAsync(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = GetOutbox(context); + var staged = TestApplicationProtocol.Create(Sessions, receiver.GetGrainId(), receiver.GetGrainId(), + "output/staged", NewMessage(193, "staged-output")); + { + await receiver.StageOutputAsync(staged); + Assert.Equal(staged.Envelope.MessageId, Assert.Single(outbox.Messages).Envelope.MessageId); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + await receiver.DeleteStateAndDeactivateAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(outbox.Messages); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.True(outbox.Stopping.IsCompleted); + Assert.Throws(() => outbox.Send(staged)); + } + Assert.Empty(outbox.Messages); + var fresh = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, fresh.ActivationId); + Assert.Empty(fresh.Effects); + Assert.Equal(0, fresh.OutboxCount); + var envelope = CreateEnvelope(receiver, NewMessage(191, "new-state")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var after = await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(fresh.ActivationId, after.ActivationId); + Assert.Equal("new-state", Assert.Single(after.Effects).Value); + Assert.Equal(1, after.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + private static JournaledTestOutbox GetOutbox(IGrainContext context) => + (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + + private static void AssertStopped(object extension, JournaledTestOutbox outbox) + { + Assert.True(outbox.Stopping.IsCompleted); + Assert.False(CancellationCleanupProbe.CoordinatorIsActive(extension)); + Assert.Empty(CancellationCleanupProbe.Field(extension, "_pendingWrites")); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_metricsActive")); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_reportedDepth")); + var results = CancellationCleanupProbe.Field(extension, "_pumpResults"); + var entries = (IDictionary)results.GetType().GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(results)!; + Assert.DoesNotContain(entries.Keys.Cast(), + key => (string)key.GetType().GetProperty("JobName")!.GetValue(key)! == ReceiverTestServices.InboxJobName); + } + + private static IList GetPending(IGrainContext context) + { + var type = ReceiverTestServices.GetImplementationType("InboxExtension"); + return (IList)type.GetField("_pendingWrites", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(context.ActivationServices.GetRequiredService(type))!; + } + + private static DurableJob CreateJob(IMessagingTestGrain receiver, string name, string? ownershipId = null) => new() + { + Id = Guid.NewGuid().ToString("N"), + ShardId = "quiescence-tests", + Name = name, + TargetGrainId = receiver.GetGrainId(), + DueTime = DateTimeOffset.UtcNow, + Metadata = ownershipId is null ? null : new Dictionary { ["orleans.messaging.ownership-id"] = ownershipId } + }; + + private static async Task InvokeJobAsync(IMessagingTestGrain receiver, DurableJob job, int dequeueCount = 1) + { + var assembly = typeof(DurableJob).Assembly; + var type = assembly.GetType("Orleans.DurableJobs.IDurableJobReceiverExtension", throwOnError: true)!; + var reference = receiver.AsReference(type); + var run = Activator.CreateInstance(assembly.GetType("Orleans.DurableJobs.JobRunContext", throwOnError: true)!, + job, Guid.NewGuid().ToString("N"), dequeueCount)!; + return await (ValueTask)type.GetMethod("HandleDurableJobAsync")! + .Invoke(reference, [run, TestContext.Current.CancellationToken])!; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs new file mode 100644 index 00000000000..2909d0cbaac --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs @@ -0,0 +1,92 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateEncodingTests : MessagingBehaviorTestBase +{ + public InboxStateEncodingTests() : base(new FaultingInboxCodecFixture()) { } + + [Fact] + public async Task CompleteCommandCodecFailure_PreservesOriginalCauseAndOwnedCleanup() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/complete-codec-fault"); + var envelope = CreateEnvelope(receiver, NewMessage(204, "complete-codec-fault"), "messages/complete-codec-fault"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Completion remove command failed."); + ((FaultingInboxCodecFixture)Fixture).NextFailure = failure; + handler.Release(); + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(CancellationCleanupProbe.Field(extension, "_pendingWrites")); + Assert.Equal(1, CancellationCleanupProbe.Field(extension, "_gate").CurrentCount); + Assert.False(CancellationCleanupProbe.CoordinatorIsActive(extension)); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_metricsActive")); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_reportedDepth")); + Assert.Empty(outbox.Messages); + _ = await receiver.GetSnapshotAsync(); + var fresh = Assert.IsType(Fixture.GetGrainContext(receiver).GrainInstance); + var replayed = Assert.IsType(fresh.ReplayedSnapshot); + Assert.Equal(1, replayed.InboxCount); + Assert.Equal(0, replayed.ProcessedMessageCount); + Assert.Empty(replayed.Effects); + Assert.Equal(0, replayed.OutboxCount); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InboxCodecFailure_DeactivatesBeforeStorageAndFreshScopeReplaysOnlyAcknowledgedState(bool snapshot) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Injected inbox command codec failure."); + var faulting = (FaultingInboxCodecFixture)Fixture; + faulting.NextFailure = failure; + faulting.FailOnSnapshot = snapshot; + if (snapshot) Fixture.Storage.RequestSnapshot(journal); + var envelope = CreateEnvelope(receiver, NewMessage(203, "codec")); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Same(failure, await grain.DeactivationFailure.Task); + Assert.Equal(snapshot, faulting.FailedSnapshot); + Assert.Empty(outbox.Messages); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + // Entry encoding precedes dictionary insertion; snapshot encoding observes staged values. + Assert.Equal(snapshot ? 1 : 0, grain.GetSnapshotForTest().InboxCount); + Assert.Empty(grain.GetSnapshotForTest().Effects); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs new file mode 100644 index 00000000000..94abb264c38 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs @@ -0,0 +1,228 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Messaging.Tests.Support; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Journaling.Json; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateManagerBoundaryTests +{ + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ActivationStandardState_UsesScopedCodecAndSingleOwnerEnrollment(bool keyedFirst) + { + const string selectedFormat = "orleans-binary"; + var builder = CreateBuilder(selectedFormat); + var codecType = GetDictionaryCodecType(builder.Services, selectedFormat); + builder.Services.AddKeyedScoped>(selectedFormat, (sp, _) => + new TrackingCodec((IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, codecType))); + builder.Services.AddScoped(); + var grainType = GrainType.Create("codec-scope"); + var manifest = new GrainManifest( + ImmutableDictionary.Empty.Add( + grainType, new GrainProperties( + ImmutableDictionary.Empty.WithComparers(StringComparer.Ordinal, StringComparer.Ordinal))), + ImmutableDictionary.Empty); + var manifestProvider = Substitute.For(); + manifestProvider.LocalGrainManifest.Returns(manifest); + manifestProvider.Current.Returns(new ClusterManifest( + new MajorMinorVersion(1, 0), + ImmutableDictionary.Empty, + [manifest])); + builder.Services.AddSingleton(manifestProvider); + builder.Services.AddSingleton(); + builder.Services.AddScoped(sp => + { + var context = Substitute.For(); + context.ActivationServices.Returns(sp); + context.GrainId.Returns(GrainId.Create(grainType, IdSpan.Create(Guid.NewGuid().ToString("N")))); + context.ObservableLifecycle.Returns(Substitute.For()); + return context; + }); + builder.Services.AddScoped(sp => new OwnedComponent( + sp.GetRequiredKeyedService>("state"), sp.GetRequiredService())); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + IDurableDictionary state; + OwnedComponent component; + Dependency dependency; + IJournaledStateManager journalOwner; + await using (var scope = services.CreateAsyncScope()) + { + var sp = scope.ServiceProvider; + Assert.Same(manifest.Grains[grainType], + sp.GetRequiredService().GetGrainProperties(sp.GetRequiredService().GrainId.Type)); + var app = sp.GetRequiredService(); + journalOwner = sp.GetRequiredService(); + Assert.Same(app, journalOwner); + var codec = Assert.IsType(sp.GetRequiredKeyedService>(selectedFormat)); + Assert.Throws(() => services.GetRequiredKeyedService>(selectedFormat)); + state = keyedFirst + ? sp.GetRequiredKeyedService>("state") + : app.GetOrAddState>("state"); + component = sp.GetRequiredService(); + dependency = sp.GetRequiredService(); + Assert.Same(state, component.State); + Assert.Same(dependency, component.Dependency); + Assert.Same(state, sp.GetRequiredKeyedService>("state")); + Assert.Same(state, app.GetOrAddState>("state")); + Assert.True(app.TryGetState>("state", out var found)); + Assert.Same(state, found); + Assert.True(journalOwner.TryGetStateMachine("state", out var machine)); + Assert.Same(state, machine); + sp.GetRequiredService().ObservableLifecycle.Received(1) + .Subscribe(Arg.Any(), GrainLifecycleStage.SetupState, Arg.Any()); + await journalOwner.InitializeAsync(Cancellation); + Assert.Equal(0, codec.Applies); + state.Add("committed", 17); + Assert.Equal(1, codec.SetsWritten); + await app.WriteStateAsync(Cancellation); + Assert.Equal(1, codec.SetsWritten); + Assert.Equal(17, state["committed"]); + Assert.Throws(() => app.GetOrAddState>("late")); + Assert.Throws(() => sp.GetRequiredKeyedService>("late")); + Assert.Same(state, app.GetOrAddState>("state")); + await journalOwner.DeleteStateAsync(Cancellation); + Assert.Empty(state); + Assert.False(dependency.Disposed); + Assert.Equal(0, component.Disposals); + await using var other = services.CreateAsyncScope(); + var otherOwner = other.ServiceProvider.GetRequiredService(); + Assert.NotSame(journalOwner, otherOwner); + Assert.NotSame(codec, other.ServiceProvider.GetRequiredKeyedService>(selectedFormat)); + } + Assert.True(dependency.Disposed); + Assert.Equal(1, component.Disposals); + await Assert.ThrowsAsync(() => journalOwner.WriteStateAsync(Cancellation).AsTask()); + } + + [Fact] + public async Task NamedStandaloneOwner_UsesSelectedFormatAndCallerOwnedComponentsAcrossReplay() + { + const string selectedFormat = "json"; + const string storageName = "caller-owned-store"; + var builder = CreateBuilder(selectedFormat); + builder.AddVolatileJournalStorage(storageName); + builder.UseJsonJournalFormat(StandardStateJsonContext.Default); + var codecType = GetDictionaryCodecType(builder.Services, selectedFormat); + builder.Services.AddKeyedSingleton>(selectedFormat, (sp, _) => + new TrackingCodec((IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, codecType))); + var dependenciesCreated = 0; + builder.Services.AddScoped(_ => { dependenciesCreated++; return new Dependency(); }); + var id = new JournalId("inbox-state-lifetime/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredKeyedService(storageName).CreateStandalone(id)); + builder.Services.AddScoped(sp => new OwnedComponent( + sp.GetRequiredKeyedService>("state"), sp.GetRequiredService())); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + Assert.Equal(0, dependenciesCreated); + var codec = Assert.IsType(services.GetRequiredKeyedService>(selectedFormat)); + Dependency dependency; + OwnedComponent component; + await using (var scope = services.CreateAsyncScope()) + { + var first = scope.ServiceProvider.GetRequiredService(); + Assert.False(first is IDurableStateManager); + dependency = scope.ServiceProvider.GetRequiredService(); + Assert.False(first.TryGetStateMachine("state", out _)); + component = scope.ServiceProvider.GetRequiredService(); + var state = component.State; + Assert.Same(scope.ServiceProvider.GetRequiredKeyedService>("state"), state); + Assert.True(first.TryGetStateMachine("state", out var registered)); + Assert.Same(state, registered); + await first.InitializeAsync(Cancellation); + Assert.Equal(0, codec.Applies); + state.Add("persisted", 23); + await first.WriteStateAsync(Cancellation); + Assert.Equal(1, codec.SetsWritten); + await first.DisposeAsync(); + Assert.Equal(0, component.Disposals); + Assert.False(dependency.Disposed); + Assert.Equal(1, dependenciesCreated); + await using var recoveryScope = services.CreateAsyncScope(); + var recovered = recoveryScope.ServiceProvider.GetRequiredService(); + Assert.NotSame(first, recovered); + Assert.False(recovered is IDurableStateManager); + var recoveredCodec = recoveryScope.ServiceProvider.GetRequiredKeyedService>(selectedFormat); + Assert.Same(codec, recoveredCodec); + var fresh = recoveryScope.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(Cancellation); + Assert.NotSame(state, fresh); + Assert.Equal(new KeyValuePair("persisted", 23), Assert.Single(fresh)); + Assert.Equal(1, codec.Applies); + await recovered.DeleteStateAsync(Cancellation); + Assert.Empty(fresh); + await recovered.DisposeAsync(); + Assert.Equal(0, component.Disposals); + Assert.False(dependency.Disposed); + Assert.Equal(1, dependenciesCreated); + } + + Assert.True(dependency.Disposed); + Assert.Equal(1, component.Disposals); + } + + private static Type GetDictionaryCodecType(IServiceCollection services, string selectedFormat) => services.Last(entry => + entry.ServiceType == typeof(IDurableDictionaryCommandCodec<,>) && Equals(entry.ServiceKey, selectedFormat)) + .KeyedImplementationType!.MakeGenericType(typeof(string), typeof(int)); + + internal static TestSiloBuilder CreateBuilder(string selectedFormat) + { + var builder = new TestSiloBuilder(); + builder.Services.AddSerializer(); + builder.Services.AddLogging(); + builder.Services.AddSingleton(TimeProvider.System); + builder.Services.AddKeyedSingleton(KeyedService.AnyKey, static (sp, _) => sp.GetRequiredService()); + builder.AddVolatileJournalStorage(); + builder.Services.Configure(options => options.JournalFormatKey = selectedFormat); + return builder; + } + + internal sealed class TestSiloBuilder : ISiloBuilder + { + public IServiceCollection Services { get; } = new ServiceCollection(); + public IConfiguration Configuration { get; } = new ConfigurationBuilder().Build(); + } + + private sealed class Dependency : IDisposable + { + public bool Disposed { get; private set; } + public void Dispose() => Disposed = true; + } + + private sealed class OwnedComponent(IDurableDictionary state, Dependency dependency) : IDisposable + { + public IDurableDictionary State { get; } = state; + public Dependency Dependency { get; } = dependency; + public int Disposals { get; private set; } + public void Dispose() => Disposals++; + } + + private sealed class TrackingCodec(IDurableDictionaryCommandCodec inner) : IDurableDictionaryCommandCodec + { + public int SetsWritten { get; private set; } + public int Applies { get; private set; } + public void WriteSet(string key, int value, JournalStreamWriter writer) { SetsWritten++; inner.WriteSet(key, value, writer); } + public void WriteRemove(string key, JournalStreamWriter writer) => inner.WriteRemove(key, writer); + public void WriteClear(JournalStreamWriter writer) => inner.WriteClear(writer); + public void WriteSnapshot(IReadOnlyCollection> items, JournalStreamWriter writer) => inner.WriteSnapshot(items, writer); + public void Apply(JournalBufferReader input, IDurableDictionaryCommandHandler consumer) + { + Applies++; + inner.Apply(input, consumer); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs new file mode 100644 index 00000000000..f94da664978 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs @@ -0,0 +1,146 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateProtocolTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task RawWriteDuringLocalPreparation_CapturesNoHandlerEffects() + { + var receiver = NewGrain(); + const string route = "messages/local-preparation"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(201, "local"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + await OnTurnAsync(context, () => context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "raw-write"); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.False(grain.ApplyAttempted.Task.IsCompleted); + var beforeApply = grain.GetSnapshotForTest(); + Assert.Empty(beforeApply.Effects); + Assert.Equal(1, beforeApply.InboxCount); + Assert.Equal(0, beforeApply.ProcessedMessageCount); + var captured = grain.Captures[^1]; + Assert.Empty(captured.Effects); + Assert.Equal(1, captured.InboxCount); + Assert.Equal(0, captured.ProcessedMessageCount); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + await receiver.RequestDeactivationAsync(); + var replayed = await receiver.GetSnapshotAsync(); + Assert.Equal(completed.Effects, replayed.Effects); + Assert.Equal(1, replayed.ProcessedMessageCount); + } + + [Fact] + public async Task HandlerFailureBeforeComplete_AccountsPreparationOutcomeWithoutMutationInference() + { + var receiver = NewGrain(); + const string route = "messages/no-earlier-write"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(203, "no-earlier-write"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Local preparation failed before Complete."); + await OnTurnAsync(context, () => + { + Assert.Equal(0, manager.PendingWriteByteCount); + grain.NextApplyFailure = failure; + }); + + handler.Release(); + var completed = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Contains(failure.Message, Assert.Single(completed.InboxDeadLetters).Reason, StringComparison.Ordinal); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(completed.Effects); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Equal(completed.InboxDeadLetters, recovered.InboxDeadLetters); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task CanceledOwnerDeleteWait_RetainsStoppedWorkflowThroughResetAndDeactivation() + { + var receiver = NewGrain(); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 204, "before-delete")); + await receiver.RetryWriteStateAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var inbox = (IInboxExtension)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + var stoppedToken = CancellationCleanupProbe.Field(inbox, "_shutdownCts").Token; + using var storage = Fixture.Storage.BlockDelete(JournalId.FromGrainId(receiver.GetGrainId())); + using var cancellation = new CancellationTokenSource(); + var deleting = receiver.DeleteStateAndDeactivateAsync(); + var caller = deleting.WaitAsync(cancellation.Token); + await storage.WaitUntilEnteredAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => caller); + Assert.True(outbox.Stopping.IsCompleted); + Assert.True(stoppedToken.IsCancellationRequested); + Assert.False(deleting.IsCompleted); + Assert.False(context.Deactivated.IsCompleted); + var envelope = CreateEnvelope(receiver, NewMessage(205, "after-delete")); + Task rejected = null!; + await OnTurnAsync(context, () => rejected = inbox.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); + await Assert.ThrowsAnyAsync(() => rejected); + var outgoing = TestApplicationProtocol.Create(Sessions, receiver.GetGrainId(), receiver.GetGrainId(), + "messages/record", NewMessage(205, "after-delete")); + Assert.Throws(() => outbox.Send(outgoing)); + Assert.Single(grain.GetSnapshotForTest().Effects); + storage.Release(); + await deleting; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.True(outbox.Stopping.IsCompleted); + Assert.True(stoppedToken.IsCancellationRequested); + var fresh = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, fresh.ActivationId); + Assert.Empty(fresh.Effects); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(fresh.ActivationId, completed.ActivationId); + Assert.Equal("after-delete", Assert.Single(completed.Effects).Value); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completion.SetResult(); } + catch (Exception exception) { completion.SetException(exception); } + }); + return completion.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs new file mode 100644 index 00000000000..0a113ba6e50 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs @@ -0,0 +1,117 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class JournaledTestOutboxBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task EquivalentDuplicates_BeforeAndAfterCommit_PreserveOneOutput() + { + var owner = NewGrain(); + var original = CreateOutput(owner); + var copied = original; + var serializer = Fixture.Client.ServiceProvider.GetRequiredService>(); + var copy = serializer.Deserialize(serializer.SerializeToArray(original)); + Assert.Equal(original.Envelope.EncodedHeaders, copied.Envelope.EncodedHeaders); + Assert.NotEqual(original.Envelope.EncodedHeaders, copy.Envelope.EncodedHeaders); + + await owner.StageOutputAsync(original); + var retained = Assert.Single(Fixture.GetStagedOutput(owner)); + await owner.StageOutputAsync(copied); + Assert.Equal(retained.Envelope.EncodedHeaders, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.EncodedHeaders); + await owner.RetryWriteStateAsync(); + await owner.StageOutputAsync(copy); + Assert.Equal(retained.Envelope.EncodedHeaders, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.EncodedHeaders); + var before = await owner.GetSnapshotAsync(); + await owner.RequestDeactivationAsync(); + var recovered = await owner.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + await owner.StageOutputAsync(copy); + + var output = Assert.Single(Fixture.GetStagedOutput(owner)); + Assert.Equal(original.Envelope.MessageId, output.Envelope.MessageId); + Assert.Equal(0, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), output).Body); + Assert.Equal(1, recovered.OutboxCount); + } + + [Theory] + [InlineData("sender")] + [InlineData("receiver")] + [InlineData("body-bytes")] + public async Task ConflictingDuplicate_RejectsChangedEnvelopeAndPreservesCommittedOutput(string difference) + { + var owner = NewGrain(); + var original = CreateOutput(owner); + await owner.StageOutputAsync(original); + await owner.RetryWriteStateAsync(); + var retained = Assert.Single(Fixture.GetStagedOutput(owner)); + var conflicting = difference switch + { + "sender" => original with { Envelope = TestApplicationProtocol.With(original.ToInboxMessage(), Sessions, + sender: GrainId.Create("other-sender", "1")).Envelope }, + "receiver" => original with { ReceiverId = GrainId.Create("other-receiver", "1") }, + _ => original with { Envelope = TestApplicationProtocol.With(original.ToInboxMessage(), Sessions, + payload: CreateOutput(owner, changed: true).Envelope.Payload).Envelope } + }; + + var exception = await Assert.ThrowsAsync(() => owner.StageOutputAsync(conflicting)); + + Assert.Contains(original.Envelope.MessageId.ToString(), exception.Message, StringComparison.Ordinal); + Assert.Equal(retained.Envelope.EncodedHeaders, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.EncodedHeaders); + await owner.RetryWriteStateAsync(); + await owner.RequestDeactivationAsync(); + var recovered = await owner.GetSnapshotAsync(); + Assert.Equal(1, recovered.OutboxCount); + var output = Assert.Single(Fixture.GetStagedOutput(owner)); + Assert.Equal(original.Envelope.MessageId, output.Envelope.MessageId); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, output.Envelope)); + Assert.Equal(original.ReceiverId, output.ReceiverId); + Assert.Equal(0, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), output).Body); + } + + [Fact] + public async Task HandlerEquivalentDuplicateOutput_CommitsOneEffectAndOneOutput() + { + var receiver = NewGrain(); + var target = GrainId.Create("output-target", "handler"); + var envelope = CreateEnvelope( + receiver, + NewMessage(97, "duplicate-output") with { ForwardTo = target }, + "messages/duplicate-output"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && (snapshot.Effects.Count > 0 || snapshot.InboxDeadLetters.Count > 0)); + + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(1, completed.OutboxCount); + var output = Assert.Single(Fixture.GetStagedOutput(receiver)); + Assert.Equal(target, output.ReceiverId); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(output.Envelope.MessageId, Assert.Single(Fixture.GetStagedOutput(receiver)).Envelope.MessageId); + Assert.Empty(recovered.InboxDeadLetters); + } + + + + + + private OutboxMessage CreateOutput(IMessagingTestGrain owner, bool changed = false) => + TestApplicationProtocol.Create(Fixture.Client.ServiceProvider.GetRequiredService(), + owner.GetGrainId(), owner.GetGrainId(), "output", changed ? 1 : 0); +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs b/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs new file mode 100644 index 00000000000..39e60402481 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs @@ -0,0 +1,9 @@ +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class MessagingClusterCollection +{ + public const string Name = "Messaging cluster"; +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs new file mode 100644 index 00000000000..de8f35124ec --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs @@ -0,0 +1,380 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Placement; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingGrainTypeConfiguratorTests() : MessagingBehaviorTestBase(new BootstrapClusterFixture()) +{ + private BootstrapProbe Probe => ((BootstrapClusterFixture)Fixture).Probe; + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(typeof(PlainBootstrapGrain))] + [InlineData(typeof(ApplicationBootstrapGrain))] + [InlineData(typeof(InterfaceBootstrapGrain))] + [InlineData(typeof(GenericBootstrapGrain))] + [InlineData(typeof(DurableBootstrapGrain))] + [InlineData(typeof(MarkedDurableBootstrapGrain))] + public async Task SelectedComposition_BindsOnceAndReplaysFreshScopedState(Type grainClass) + { + var grain = CreateGrain(grainClass); + Assert.Equal(0, await grain.GetValueAsync()); + var first = Assert.Single(Probe.Get(grain.GetGrainId())); + var state = first.Context.ActivationServices.GetRequiredService(); + AssertComposition(first, state, grainClass, expectedActivationValue: 0); + var setup = GetSetup(first.Context); + Assert.Single(setup.GetInvocationList()); + await grain.SetValueAsync(41); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + using var handler = Fixture.HandlerProbe.Arm(grain.GetGrainId(), BootstrapState.Route); + var envelope = CreateEnvelope(grain); + Assert.Equal(DeliveryStatus.Accepted, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + await handler.WaitUntilEnteredAsync(); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Assert.Equal(41, first.Value!.Value); + Assert.Equal(1, first.Inbox!.Count); + Assert.Equal(0, first.Outbox!.Count); + Assert.Empty(GetProcessed(first.Context)); + Assert.Equal(0, state.HandlerCalls); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var storage = Fixture.Storage.BlockWrite(journal); + handler.Release(); + await storage.WaitUntilEnteredAsync(); + Assert.Equal(42, first.Value!.Value); + Assert.Equal(0, first.Inbox!.Count); + Assert.Equal(1, first.Outbox!.Count); + Assert.Single(GetProcessed(first.Context)); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + storage.Release(); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(0, first.Inbox.Count); + Assert.Single(GetProcessed(first.Context)); + var output = Assert.Single(first.Outbox.Messages); + Assert.Equal(1, state.HandlerCalls); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(1, state.HandlerCalls); + + var other = CreateGrain(grainClass); + Assert.Equal(0, await other.GetValueAsync()); + var isolated = Assert.Single(Probe.Get(other.GetGrainId())); + Assert.NotSame(first.Manager, isolated.Manager); + Assert.NotSame(first.Inbox, isolated.Inbox); + Assert.NotSame(first.Outbox, isolated.Outbox); + Assert.Empty(isolated.Outbox!.Messages); + Assert.Equal(42, await grain.GetValueAsync()); + await other.DeactivateAsync(); + await isolated.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + + await grain.DeactivateAsync(); + await first.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(1, first.Disposals); + Assert.Equal(1, first.GrainDisposals); + Assert.Equal(1, state.Disposals); + Assert.Equal(42, await grain.GetValueAsync()); + var observations = Probe.Get(grain.GetGrainId()); + Assert.Equal(2, observations.Length); + var recovered = observations[1]; + var recoveredState = recovered.Context.ActivationServices.GetRequiredService(); + AssertComposition(recovered, recoveredState, grainClass, expectedActivationValue: 42); + Assert.NotSame(first.Context, recovered.Context); + Assert.NotSame(first.Manager, recovered.Manager); + Assert.NotSame(first.Value, recovered.Value); + Assert.NotSame(first.Inbox, recovered.Inbox); + Assert.NotSame(first.Outbox, recovered.Outbox); + Assert.Same(setup, GetSetup(recovered.Context)); + Assert.Equal(output.Envelope.MessageId, Assert.Single(recovered.Outbox!.Messages).Envelope.MessageId); + Assert.Single(GetProcessed(recovered.Context)); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(0, recoveredState.HandlerCalls); + Assert.Equal(2, Fixture.Storage.GetReadCount(journal)); + } + + [Fact] + public async Task SelectedWithoutConstructorDependencies_MaterializesPrimaryStateBeforeRead() + { + var grain = Control(); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var read = Fixture.Storage.BlockRead(journal); + var activation = grain.PingAsync(); + try + { + await read.WaitUntilEnteredAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Same(observation.ConstructedGrain, observation.Context.GrainInstance); + var manager = observation.Context.ActivationServices.GetRequiredService(); + Assert.Equal(8, BootstrapState.ReadMessagingStates(manager).Count()); + Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox", out _)); + Assert.True(manager.TryGetStateMachine("test-handler-output", out _)); + Assert.Equal(0, observation.Activations); + Assert.Single(GetSetup(observation.Context).GetInvocationList()); + } + finally + { + read.Release(); + } + await activation; + Assert.Equal(1, Fixture.Storage.GetReadCount(journal)); + } + + [Theory] + [InlineData(typeof(UnselectedBootstrapGrain), false)] + [InlineData(typeof(JournalOnlyBootstrapGrain), true)] + public async Task UnselectedComposition_LeavesMessagingUnresolved(Type grainClass, bool journaled) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + await grain.PingAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Equal(1, observation.Activations); + Assert.Null(observation.Inbox); + Assert.Null(observation.Outbox); + Assert.Null(GetSetupOrDefault(observation.Context)); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + Assert.Equal(journaled ? 1 : 0, Fixture.Storage.GetCreationCount(journal)); + Assert.Equal(journaled ? 1 : 0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + if (journaled) + { + Assert.NotNull(observation.Manager); + Assert.True(observation.Manager.TryGetStateMachine("bootstrap-journal-only", out var state)); + Assert.Same(observation.Value, state); + Assert.False(observation.Manager.TryGetStateMachine("__orleans.messaging.inbox", out _)); + Assert.False(observation.Manager.TryGetStateMachine("test-handler-output", out _)); + Assert.Empty(BootstrapState.ReadMessagingStates(observation.Manager)); + } + else + { + Assert.Null(observation.Manager); + } + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(ReentrantBootstrapGrain), "non-reentrant")] + [InlineData(typeof(StatelessBootstrapGrain), "one activation")] + [InlineData(typeof(MayInterleaveBootstrapGrain), "non-reentrant")] + [InlineData(typeof(AlwaysInterleaveBootstrapGrain), "interleavable method")] + [InlineData(typeof(MetadataReentrantBootstrapGrain), "non-reentrant")] + [InlineData(typeof(MetadataMayInterleaveBootstrapGrain), "non-reentrant")] + public async Task UnsupportedMarkedModel_FailsBeforeStorageAndCleansScope(Type grainClass, string diagnostic) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains(diagnostic, exception.ToString(), StringComparison.Ordinal); + Assert.Contains(grainClass.Name, exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.NotNull(observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.Equal(0, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(MetadataStatelessPlacementBootstrapGrain), "StatelessWorkerPlacement")] + [InlineData(typeof(AliasedStatelessPlacementBootstrapGrain), BootstrapClusterFixture.StatelessPlacementAlias)] + public async Task ResolvedStatelessPlacement_FailsBeforeStorageAndCleansScope(Type grainClass, string placementKey) + { + Assert.False(grainClass.IsDefined(typeof(StatelessWorkerAttribute), inherit: true)); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var services = Fixture.Cluster.Silos[0].ServiceProvider; + var properties = services.GetRequiredService().GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(placementKey, properties.Properties[WellKnownGrainTypeProperties.PlacementStrategy]); + var placement = services.GetRequiredService().GetPlacementStrategy(grain.GetGrainId().Type); + Assert.Equal(new StatelessWorkerAttribute().PlacementStrategy.GetType(), placement.GetType()); + Assert.False(placement.IsUsingGrainDirectory); + if (placementKey == BootstrapClusterFixture.StatelessPlacementAlias) + { + Assert.Same(services.GetRequiredKeyedService(placementKey), placement); + } + + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("one activation", exception.ToString(), StringComparison.Ordinal); + Assert.Contains(grainClass.Name, exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.IsType(grainClass, observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.Equal(0, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(MetadataOrdinaryPlacementBootstrapGrain), nameof(RandomPlacement))] + [InlineData(typeof(AliasedOrdinaryPlacementBootstrapGrain), BootstrapClusterFixture.OrdinaryPlacementAlias)] + public async Task ResolvedOrdinaryPlacement_WithoutMessagingConstructorDependencies_InitializesNormally(Type grainClass, string placementKey) + { + Assert.False(grainClass.IsDefined(typeof(StatelessWorkerAttribute), inherit: true)); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var services = Fixture.Cluster.Silos[0].ServiceProvider; + var properties = services.GetRequiredService().GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(placementKey, properties.Properties[WellKnownGrainTypeProperties.PlacementStrategy]); + var placement = services.GetRequiredService().GetPlacementStrategy(grain.GetGrainId().Type); + Assert.IsType(placement); + Assert.True(placement.IsUsingGrainDirectory); + if (placementKey == BootstrapClusterFixture.OrdinaryPlacementAlias) + { + Assert.Same(services.GetRequiredKeyedService(placementKey), placement); + } + + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var read = Fixture.Storage.BlockRead(journal); + var activation = grain.PingAsync(); + BootstrapObservation observation; + try + { + await read.WaitUntilEnteredAsync(); + observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.IsType(grainClass, observation.ConstructedGrain); + Assert.Same(observation.ConstructedGrain, observation.Context.GrainInstance); + Assert.Equal(0, observation.Activations); + var manager = observation.Context.ActivationServices.GetRequiredService(); + Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox", out var inbox)); + Assert.Same(observation.Context.ActivationServices.GetRequiredKeyedService< + IDurableDictionary>("__orleans.messaging.inbox"), inbox); + Assert.True(manager.TryGetStateMachine("test-handler-output", out var outbox)); + Assert.Same(((JournaledTestOutbox)observation.Context.ActivationServices.GetRequiredService()).StoredMessages, outbox); + Assert.Single(GetSetup(observation.Context).GetInvocationList()); + } + finally + { + read.Release(); + } + + await activation; + Assert.Equal(1, observation.Activations); + Assert.Equal(1, Fixture.Storage.GetCreationCount(journal)); + Assert.Equal(1, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public async Task SetupFailure_DisposesScopeAndPreservesOriginalError() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.IsType(observation.ExpectedFailure); + Assert.Contains(observation.ExpectedFailure.Message, exception.ToString(), StringComparison.Ordinal); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.NotNull(observation.Inbox); + Assert.NotNull(observation.Outbox); + var extension = Assert.IsAssignableFrom(observation.Extension); + var shutdown = (CancellationTokenSource)extension.GetType().GetField("_shutdownCts", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + Assert.Throws(() => shutdown.Token); + extension.Dispose(); + Assert.Equal(1, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public async Task UnselectedEndpointDependency_FailsExplicitly() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("Inbox activation requires IMessagingGrain or DurableGrain", exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(1, observation.Disposals); + Assert.Equal(1, observation.GrainDisposals); + Assert.Null(GetSetupOrDefault(observation.Context)); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public void RepeatedRegistration_InstallsOneTypeConfigurator() + { + var services = new ServiceCollection(); + ReceiverTestServices.Add(services, static _ => { }); + ReceiverTestServices.Add(services, static _ => { }); + var descriptor = Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IConfigureGrainTypeComponents)); + Assert.Equal(ServiceLifetime.Singleton, descriptor.Lifetime); + Assert.Equal(ReceiverTestServices.GetImplementationType("MessagingGrainTypeConfigurator"), descriptor.ImplementationType); + Assert.Empty(typeof(IMessagingGrain).GetInterfaces()); + Assert.Empty(typeof(IMessagingGrain).GetMethods()); + Assert.False(typeof(IAddressable).IsAssignableFrom(typeof(IMessagingGrain))); + } + + private static void AssertComposition(BootstrapObservation observation, BootstrapState state, Type grainClass, int expectedActivationValue) + { + Assert.Equal(grainClass, observation.ConstructedGrain!.GetType()); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.Same(observation.ConstructedGrain, state.GrainAtActivation); + Assert.Equal(8, state.MessagingStateCountAtActivation); + Assert.Equal(1, observation.Activations); + Assert.Equal(expectedActivationValue, state.ActivationValue); + var services = observation.Context.ActivationServices; + Assert.Same(observation.Manager, services.GetRequiredService()); + var applicationManager = services.GetRequiredService(); + Assert.Same(observation.Manager, applicationManager); + Assert.Same(observation.Value, applicationManager.GetOrAddState>("bootstrap-value")); + Assert.True(applicationManager.TryGetState>("test-handler-output", out var applicationOutbox)); + Assert.Same(((JournaledTestOutbox)observation.Outbox!).StoredMessages, applicationOutbox); + Assert.Same(applicationOutbox, applicationManager.GetOrAddState>("test-handler-output")); + Assert.Same(observation.Value, services.GetRequiredKeyedService>("bootstrap-value")); + Assert.Same(observation.Inbox, services.GetRequiredService()); + Assert.Same(observation.Outbox, services.GetRequiredService()); + var tryGetHandler = observation.Inbox!.GetType().GetMethod("TryGetHandler", BindingFlags.Instance | BindingFlags.NonPublic)!; + object?[] arguments = [null]; + Assert.True((bool)tryGetHandler.Invoke(observation.Inbox, arguments)!); + var handler = arguments[0]; + Assert.Same(state, handler); + Assert.Equal(8, BootstrapState.ReadMessagingStates(observation.Manager!).Count()); + var primary = services.GetRequiredKeyedService>("__orleans.messaging.inbox"); + Assert.Same(primary, applicationManager.GetOrAddState>("__orleans.messaging.inbox")); + Assert.True(observation.Manager!.TryGetStateMachine("test-handler-output", out var output)); + Assert.Same(applicationOutbox, output); + } + private IBootstrapTestGrain CreateGrain(Type grainClass) => grainClass == typeof(GenericBootstrapGrain) + ? Fixture.Client.GetGrain>(Guid.NewGuid()) + : Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + private IBootstrapControlGrain Control() => Fixture.Client.GetGrain(Guid.NewGuid(), typeof(T).FullName!); + private InboxMessage CreateEnvelope(IBootstrapTestGrain grain) => + TestApplicationProtocol.CreateInbox(Fixture.Client.ServiceProvider.GetRequiredService(), + GrainId.Create("bootstrap-sender", "external"), BootstrapState.Route, 1); + private static IDurableDictionary GetProcessed(IGrainContext context) => + context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed"); + private static Delegate GetSetup(IGrainContext context) => Assert.IsAssignableFrom(GetSetupOrDefault(context)); + private static object? GetSetupOrDefault(IGrainContext context) + { + var shared = context.GetType().GetField("_shared", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(context)!; + return shared.GetType().GetField("_activationSetup", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(shared); + } + private void AssertNoStorageWork(GrainId grainId) + { + var journal = JournalId.FromGrainId(grainId); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + Assert.Equal(0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + } + private void AssertNoScheduledJobs(GrainId grainId) + { + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, grainId)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount("orleans.messaging.outbox-drain", grainId)); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs new file mode 100644 index 00000000000..9df6c9aedf0 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs @@ -0,0 +1,279 @@ +using System.Diagnostics.Metrics; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingMetricCardinalityTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task PendingAndProcessedDuplicates_RecordOneBoundedReceiptEach() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var services = context.ActivationServices; + var extension = (IInboxExtension)services.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-metric-pending"); + var turn = receiver.HoldPumpTurnAsync("hold-metric-pending", deactivate: false); + await hold.WaitUntilEnteredAsync(); + var envelope = CreateEnvelope(receiver, NewMessage(451, "duplicate-receipts")); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery()).Status); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + var pending = Fixture.GetSnapshot(receiver); + Assert.Equal(1, pending.InboxCount); + Assert.Equal(0, pending.ProcessedMessageCount); + Assert.Empty(pending.Effects); + Assert.Equal(job.Id, pending.InboxJob!.Id); + Assert.Equal(job.ShardId, pending.InboxJob.ShardId); + var instruments = services.GetRequiredService(ReceiverTestServices.GetImplementationType("MessagingInstruments")); + var received = (Instrument)instruments.GetType().GetField("_inboxMessagesReceived", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instruments)!; + using var probe = new MessageMetricListener(received.Meter); + var grainType = receiver.GetGrainId().Type.ToString(); + + Assert.Equal(DeliveryStatus.Duplicate, (await StartDelivery()).Status); + Assert.Single(probe.Read("inbox-messages-received")).AssertCounter(grainType, "duplicate"); + hold.Release(); + await turn; + var processed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(0, processed.InboxCount); + Assert.Equal(1, processed.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(processed.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + var receipts = probe.Read("inbox-messages-received"); + Assert.Equal(2, receipts.Length); + Assert.All(receipts, receipt => receipt.AssertCounter(grainType, "duplicate")); + Assert.Single(receipts.Select(receipt => receipt.Series).Distinct()); + + Task StartDelivery() + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + } + + [Theory] + [InlineData(1)] + [InlineData(64)] + public async Task DistinctApplicationRoutes_KeepFixedMetricSeriesAndMissingHandlerOutcomes(int routeCount) + { + var receiver = NewGrain(); + var missingHandler = NewGrain(); + await missingHandler.ConfigureHandlerAsync(false); + await RefreshSeededOwnerAsync(missingHandler); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var instruments = context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("MessagingInstruments")); + var received = (Instrument)instruments.GetType().GetField("_inboxMessagesReceived", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instruments)!; + using var probe = new MessageMetricListener(received.Meter); + var grainType = receiver.GetGrainId().Type.ToString(); + var routes = new HashSet(); + for (var index = 0; index < routeCount; index++) + { + var unknown = $"unknown/{Guid.NewGuid():N}"; + routes.Add(unknown); + var rejected = CreateEnvelope(missingHandler, NewMessage(index, "unknown"), unknown); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(missingHandler, rejected)).Status); + Assert.Equal(unknown, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), rejected).Route); + var valid = $"messages/cardinality/{Guid.NewGuid():N}"; + routes.Add(valid); + var accepted = CreateEnvelope(receiver, NewMessage(index, "accepted"), valid); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, accepted)).Status); + Assert.Equal(valid, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), accepted).Route); + await Fixture.WaitForEffectCountAsync(receiver, index + 1); + } + + await probe.WaitForCountAsync("inbox-processing-duration", routeCount); + var receivedRows = probe.Read("inbox-messages-received"); + Assert.Equal(routeCount * 2, routes.Count); + Assert.Equal(routeCount * 2, receivedRows.Length); + Assert.Equal(2, receivedRows.Select(row => row.Series).Distinct().Count()); + var acceptedRows = receivedRows.Where(row => row.Tags.Any(tag => tag.Key == "status" && Equals(tag.Value, "accepted"))).ToArray(); + var rejectedRows = receivedRows.Where(row => row.Tags.Any(tag => tag.Key == "status" && Equals(tag.Value, "handler_not_found"))).ToArray(); + Assert.Equal(routeCount, acceptedRows.Length); + Assert.Equal(routeCount, rejectedRows.Length); + Assert.All(acceptedRows, row => row.AssertCounter(grainType, "accepted")); + Assert.All(rejectedRows, row => row.AssertCounter(grainType, "handler_not_found")); + var processed = probe.Read("inbox-messages-processed"); + Assert.Equal(routeCount, processed.Length); + Assert.Single(processed.Select(row => row.Series).Distinct()); + Assert.All(processed, row => row.AssertCounter(grainType, "success")); + var durations = probe.Read("inbox-processing-duration"); + Assert.Equal(routeCount, durations.Length); + Assert.Single(durations.Select(row => row.Series).Distinct()); + Assert.All(durations, row => + { + Assert.Equal("ms", row.Unit); + Assert.True(row.Value >= 0); + Assert.Equal(new[] { new KeyValuePair("grain_type", grainType) }, row.Tags); + }); + var completed = await receiver.GetSnapshotAsync(); + Assert.Equal(routeCount, completed.Effects.Count); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + Assert.Equal(routeCount, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Empty(completed.InboxDeadLetters); + } +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingInstrumentTagTests +{ + [Theory] + [InlineData("OnInboxMessageReceived", "inbox-messages-received", "accepted", false)] + [InlineData("OnInboxMessageProcessed", "inbox-messages-processed", "success", false)] + [InlineData("OnOutboxMessageSent", "outbox-messages-sent", null, false)] + [InlineData("OnOutboxMessageDelivered", "outbox-messages-delivered", "duplicate", false)] + [InlineData("OnInboxProcessingDuration", "inbox-processing-duration", null, true)] + [InlineData("OnOutboxDeliveryDuration", "outbox-delivery-duration", null, true)] + public void MessageInstruments_EmitExactBoundedTagsAndUnchangedValues(string method, string instrumentName, string? status, bool duration) + { + using var factory = new TestMeterFactory(); + var instruments = Activator.CreateInstance(ReceiverTestServices.GetImplementationType("MessagingInstruments"), + new OrleansInstruments(factory))!; + using var probe = new MessageMetricListener(factory.Meter); + const string grainType = "instrument-test"; + object[] arguments = duration ? [TimeSpan.FromMilliseconds(12.5), grainType] + : status is not null ? [grainType, status] : [grainType]; + Invoke(instruments, method, arguments); + var measurement = Assert.Single(probe.Read(instrumentName)); + Assert.Equal(MessageMetricListener.Prefix + instrumentName, measurement.Name); + if (duration) + { + Assert.Equal("ms", measurement.Unit); + Assert.Equal(12.5, measurement.Value); + Assert.Equal(new[] { new KeyValuePair("grain_type", grainType) }, measurement.Tags); + Invoke(instruments, method, TimeSpan.FromMilliseconds(-10), grainType); + var values = probe.Read(instrumentName); + Assert.Equal(2, values.Length); + Assert.Equal(0, values[1].Value); + Assert.Equal(measurement.Tags, values[1].Tags); + } + else + { + measurement.AssertCounter(grainType, status); + } + } + + [Fact] + public void OrphanAndDepthInstruments_RetainExistingTagsAndValues() + { + using var factory = new TestMeterFactory(); + var instruments = Activator.CreateInstance(ReceiverTestServices.GetImplementationType("MessagingInstruments"), + new OrleansInstruments(factory))!; + using var probe = new MessageMetricListener(factory.Meter); + Invoke(instruments, "OnOrphanedJobReclaimed", "instrument-test", ReceiverTestServices.InboxJobName); + Invoke(instruments, "OnInboxDepthChanged", 3); + Invoke(instruments, "OnInboxDepthChanged", -1); + Invoke(instruments, "OnOutboxDepthChanged", 4); + Invoke(instruments, "OnOutboxDepthChanged", -1); + probe.ObserveGauges(); + var orphan = Assert.Single(probe.Read("orphaned-jobs-reclaimed")); + Assert.Equal(1, orphan.Value); + Assert.Null(orphan.Unit); + Assert.Equal(new[] { new KeyValuePair("grain_type", "instrument-test"), new("job_name", ReceiverTestServices.InboxJobName) }, orphan.Tags); + var inboxDepth = Assert.Single(probe.Read("inbox-depth")); + var outboxDepth = Assert.Single(probe.Read("outbox-depth")); + Assert.Equal(2, inboxDepth.Value); + Assert.Equal(3, outboxDepth.Value); + Assert.Empty(inboxDepth.Tags); + Assert.Empty(outboxDepth.Tags); + Assert.Null(inboxDepth.Unit); + Assert.Null(outboxDepth.Unit); + } + + private static void Invoke(object instruments, string method, params object[] arguments) => + instruments.GetType().GetMethod(method, BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(instruments, arguments); + + private sealed class TestMeterFactory : IMeterFactory + { + public Meter Meter { get; private set; } = null!; + public Meter Create(MeterOptions options) => Meter = new Meter(options); + public void Dispose() => Meter.Dispose(); + } +} + +internal sealed class MessageMetricListener : IDisposable +{ + internal const string Prefix = "orleans-messaging-"; + private readonly MeterListener _listener; + private readonly object _lock = new(); + private readonly List _measurements = []; + private readonly List<(string Name, int Count, TaskCompletionSource Completion)> _waiters = []; + + public MessageMetricListener(Meter meter) + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (ReferenceEquals(instrument.Meter, meter) && instrument.Name.StartsWith(Prefix, StringComparison.Ordinal)) + listener.EnableMeasurementEvents(instrument); + } + }; + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.Start(); + } + + private void Record(Instrument instrument, double value, ReadOnlySpan> tags) + { + lock (_lock) + { + _measurements.Add(new(instrument.Name, instrument.Unit, value, tags.ToArray())); + foreach (var waiter in _waiters.ToArray()) + { + if (_measurements.Count(row => row.Name == waiter.Name) >= waiter.Count) + { + _waiters.Remove(waiter); + waiter.Completion.TrySetResult(); + } + } + } + } + public Task WaitForCountAsync(string name, int count) + { + lock (_lock) + { + if (_measurements.Count(row => row.Name == Prefix + name) >= count) return Task.CompletedTask; + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add((Prefix + name, count, completion)); + return completion.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + } + public Measurement[] Read(string name) + { + lock (_lock) return _measurements.Where(row => row.Name == Prefix + name).ToArray(); + } + public void ObserveGauges() => _listener.RecordObservableInstruments(); + public void Dispose() => _listener.Dispose(); + internal sealed record Measurement(string Name, string? Unit, double Value, KeyValuePair[] Tags) + { + public string Series => string.Join(";", Tags.Select(tag => $"{tag.Key}={tag.Value}")); + public void AssertCounter(string grainType, string? status = null) + { + Assert.Equal(1, Value); + Assert.Null(Unit); + KeyValuePair[] expected = status is null + ? [new("grain_type", grainType)] + : [new("grain_type", grainType), new("status", status)]; + Assert.Equal(expected, Tags); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingOwnershipRecoveryTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingOwnershipRecoveryTests.cs new file mode 100644 index 00000000000..b2fec9c19af --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingOwnershipRecoveryTests.cs @@ -0,0 +1,299 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingOwnershipRecoveryTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task JournalFailureDuringLocalScheduling_RejectsLaterWriteAndFreshOwnerReplaysAcknowledgedState() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 77, "uncommitted")); + using var schedule = Fixture.JobManagerProbe.BlockNext("orleans.messaging.inbox-drain"); + var envelope = CreateEnvelope(receiver, NewMessage(77, "failed-during-schedule")); + var delivery = DeliverAsync(receiver, envelope); + await schedule.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var failure = await Assert.ThrowsAsync(() => oldManager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(failure, await oldGrain.DeactivationFailure.Task); + schedule.Continue(); + var rejected = await Assert.ThrowsAsync(() => delivery); + Assert.Contains("fenced", rejected.Message, StringComparison.Ordinal); + Assert.Equal(failure.Message, Assert.IsType(rejected.InnerException).Message); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var oldState = oldGrain.GetSnapshotForTest(); + Assert.Equal(1, oldState.InboxCount); + Assert.NotNull(oldState.InboxJobId); + Assert.NotNull(oldState.InboxJob); + Assert.Single(oldState.Effects); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(oldState.ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task FailedInboxAcceptance_FencesOldObjectsAndFreshReplayOmitsEnvelope() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var envelope = CreateEnvelope(receiver, NewMessage(2, "failed-acceptance")); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var failed = oldGrain.GetSnapshotForTest(); + Assert.Equal(1, failed.InboxCount); + Assert.NotNull(failed.InboxJob); + Assert.Empty(failed.Effects); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.NotSame(oldManager, Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService()); + Assert.Equal(0, recovered.InboxCount); + Assert.Null(recovered.InboxJob); + Assert.Empty(recovered.Effects); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } + + [Fact] + public async Task AmbiguousInboxAcceptanceCommit_PreservesAndProcessesRecoveredEnvelope() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + Fixture.Storage.FailAfterWrite(journalId); + var envelope = CreateEnvelope(receiver, NewMessage(76, "ambiguous-acceptance")); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, completed.ActivationId); + Assert.Equal("ambiguous-acceptance", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task Inbox_PrecommitCrash_ReclaimsScheduledOrphanAfterRecovery() + { + const string jobName = "orleans.messaging.inbox-drain"; + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var barrier = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var attemptBaseline = Fixture.Metrics.GetCount("orleans-durablejobs-job-attempts-started"); + var completionBaseline = Fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed"); + var orphanBaseline = Fixture.Metrics.GetCount( + "orleans-messaging-orphaned-jobs-reclaimed", + jobName); + Fixture.JobManagerProbe.DuplicateNext(jobName); + var envelope = CreateEnvelope(receiver, NewMessage(3, "inbox-orphan")); + + var delivery = DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + await Fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-job-attempts-started", + attemptBaseline + 2); + + Assert.Equal( + orphanBaseline, + Fixture.Metrics.GetCount("orleans-messaging-orphaned-jobs-reclaimed", jobName)); + barrier.Fail(); + await Assert.ThrowsAnyAsync(() => delivery); + + var recovered = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId); + await Fixture.Metrics.WaitForCountAsync( + "orleans-messaging-orphaned-jobs-reclaimed", + orphanBaseline + 2, + jobName); + await Fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-jobs-completed", + completionBaseline + 2); + + Assert.Equal(0, recovered.InboxCount); + Assert.Null(recovered.InboxJobId); + Assert.Null(recovered.InboxJob); + Assert.Empty(recovered.Effects); + Assert.Equal( + orphanBaseline + 2, + Fixture.Metrics.GetCount("orleans-messaging-orphaned-jobs-reclaimed", jobName)); + } + + [Fact] + public async Task InboxJobClearWriteFailure_FencesThenRecoversInFreshActivation() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/inbox-clear-retry"); + var envelope = CreateEnvelope(receiver, NewMessage(57, "inbox-clear-retry"), "messages/inbox-clear-retry"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId()), matchingWrite: 2); + handler.Release(); + + var recovered = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId); + var cleaned = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && snapshot.InboxJobId is null); + + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(cleaned.Effects).Count); + Assert.Empty(cleaned.InboxDeadLetters); + Assert.Null(cleaned.InboxJobId); + } + + [Fact] + public async Task DeliveryIntoEmptyInbox_ReplacesStalePersistedJobOwnership() + { + var receiver = NewGrain(); + var staleJobId = $"stale-{Guid.NewGuid():N}"; + await receiver.SetInboxOwnershipAsync( + staleJobId, + new DurableJob + { + Id = $"stale-job-{Guid.NewGuid():N}", + Name = "orleans.messaging.inbox-drain", + DueTime = DateTimeOffset.UtcNow, + TargetGrainId = receiver.GetGrainId(), + ShardId = $"stale-shard-{Guid.NewGuid():N}", + Metadata = new Dictionary + { + ["orleans.messaging.ownership-id"] = staleJobId + } + }); + await RefreshSeededOwnerAsync(receiver); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/stale-owner"); + var envelope = CreateEnvelope(receiver, NewMessage(58, "stale-owner"), "messages/stale-owner"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var accepted = Fixture.GetSnapshot(receiver); + + Assert.NotNull(accepted.InboxJobId); + Assert.NotEqual(staleJobId, accepted.InboxJobId); + + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("stale-owner", Assert.Single(completed.Effects).Value); + } + + [Fact] + public async Task Inbox_StaleGenerationCompletesWithoutClearingNewerOwner() + { + var receiver = NewGrain(); + const string route = "messages/stale-inbox-generation"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(60, "newer-inbox-owner"), route); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var owned = Fixture.GetSnapshot(receiver); + Assert.False(string.IsNullOrEmpty(owned.InboxJobId)); + var completionBaseline = Fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed"); + var manager = Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService(); + + await manager.ScheduleJobAsync( + new ScheduleJobRequest + { + Target = receiver.GetGrainId(), + JobName = "orleans.messaging.inbox-drain", + DueTime = DateTimeOffset.UtcNow, + Metadata = new Dictionary + { + ["orleans.messaging.ownership-id"] = "0" + } + }, + TestContext.Current.CancellationToken); + await Fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-jobs-completed", + completionBaseline + 1); + + Assert.Equal(owned.InboxJobId, Fixture.GetSnapshot(receiver).InboxJobId); + handler.Release(); + Assert.Equal("newer-inbox-owner", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + } + + [Fact] + public async Task SchedulingFailureBeforeApply_PreservesStateAndRetrySchedulesOnce() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(61, "schedule-failure")); + Fixture.JobManagerProbe.FailNext(ReceiverTestServices.InboxJobName); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + var unchanged = await receiver.GetSnapshotAsync(); + Assert.Equal(0, unchanged.InboxCount); + Assert.Null(unchanged.InboxJobId); + Assert.Null(unchanged.InboxJob); + Assert.Empty(unchanged.Effects); + Assert.Empty(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(2, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + } + + [Fact] + public async Task AmbiguousSchedulingFailure_LeavesAcceptanceLocalAndRetryUsesNewToken() + { + const string jobName = "orleans.messaging.inbox-drain"; + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(59, "inbox-schedule-retry")); + var scheduledBaseline = Fixture.JobManagerProbe.GetScheduledJobs(jobName, receiver.GetGrainId()).Count; + Fixture.JobManagerProbe.FailAfterNext(jobName); + + await Assert.ThrowsAsync( + () => DeliverAsync(receiver, envelope)); + Assert.Equal(0, (await receiver.GetSnapshotAsync()).InboxCount); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + + var effect = Assert.Single(completed.Effects); + Assert.Equal("inbox-schedule-retry", effect.Value); + Assert.Equal(1, effect.Count); + Assert.Equal( + 2, + Fixture.JobManagerProbe.GetAttemptCount( + "orleans.messaging.inbox-drain", + receiver.GetGrainId())); + Assert.Equal( + 2, + Fixture.JobManagerProbe.GetSuccessCount(jobName, receiver.GetGrainId())); + var scheduled = Fixture.JobManagerProbe.GetScheduledJobs(jobName, receiver.GetGrainId()) + .Skip(scheduledBaseline) + .ToArray(); + Assert.Equal(2, scheduled.Length); + Assert.NotEqual(scheduled[0].Id, scheduled[1].Id); + Assert.NotEqual( + scheduled[0].Metadata!["orleans.messaging.ownership-id"], + scheduled[1].Metadata!["orleans.messaging.ownership-id"]); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs new file mode 100644 index 00000000000..0072eca521e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs @@ -0,0 +1,147 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingRoutingAndDeadLetterTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task InboxDeadLettersRetainNewestEntriesWithinConfiguredCapacity() + { + var receiver = NewGrain(); + var messageIds = new List(); + for (var sequence = 0; sequence < 3; sequence++) + { + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), + 20 + sequence, + $"dead-letter-{sequence}", + ThrowDuringPreparation: true)); + messageIds.Add(envelope.Envelope.MessageId); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.InboxDeadLetters.Any(entry => entry.MessageId == envelope.Envelope.MessageId)); + } + + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(2, state.InboxDeadLetters.Count); + Assert.DoesNotContain(state.InboxDeadLetters, entry => entry.MessageId == messageIds[0]); + Assert.Contains(state.InboxDeadLetters, entry => entry.MessageId == messageIds[1]); + Assert.Contains(state.InboxDeadLetters, entry => entry.MessageId == messageIds[2]); + } + + [Fact] + public async Task ActivationRemovesExpiredInboxDeadLetters() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), + 30, + "expired-dead-letter", + ThrowDuringPreparation: true)); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var before = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Fixture.Clock.Advance(TimeSpan.FromHours(2)); + await receiver.RequestDeactivationAsync(); + var after = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId + && snapshot.InboxDeadLetters.Count == 0); + + Assert.Empty(after.InboxDeadLetters); + } + + [Fact] + public async Task MalformedTypedBody_DeadLettersAndDoesNotBlockLaterValidMessage() + { + var receiver = NewGrain(); + var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed)).Status); + var poisoned = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Empty(poisoned.Effects); + var deadLetter = Assert.Single(poisoned.InboxDeadLetters); + Assert.Equal(malformed.Envelope.MessageId, deadLetter.MessageId); + Assert.Contains(nameof(TestMessage), deadLetter.Reason, StringComparison.Ordinal); + + var valid = CreateEnvelope(receiver, NewMessage(41, "valid-after-poison"), "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, valid)).Status); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("valid-after-poison", Assert.Single(recovered.Effects).Value); + Assert.Single(recovered.InboxDeadLetters); + } + + [Fact] + public async Task InboxDeadLetterRemoval_IsDurable() + { + var receiver = NewGrain(); + var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed)).Status); + _ = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.True(await receiver.RemoveInboxDeadLetterAsync( + malformed.Envelope.MessageId)); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + Assert.False(await receiver.RemoveInboxDeadLetterAsync( + malformed.Envelope.MessageId)); + + await receiver.RequestDeactivationAsync(); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + } + + [Fact] + public async Task HandlerNotFound_IsRejectedWithoutInboxPersistence() + { + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + await RefreshSeededOwnerAsync(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(71, "missing"), "unknown/route"); + + var result = await DeliverAsync(receiver, envelope); + + Assert.Equal(DeliveryStatus.HandlerNotFound, result.Status); + Assert.Equal("No inbox handler is registered.", result.Message); + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(0, state.InboxCount); + Assert.Empty(state.Effects); + } + + [Fact] + public async Task Deliver_ImplicitReceiverNamespacesTheSameIncomingCommand() + { + var receiver = NewGrain(); + var declaredReceiver = NewGrain(); + var envelope = CreateEnvelope( + declaredReceiver, + NewMessage(74, "wrong-receiver")); + + Assert.Null(typeof(InboxMessage).GetProperty("ReceiverId")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var first = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(first.Effects).LogicalId); + Assert.Equal(1, Assert.Single(first.Effects).Count); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(declaredReceiver, envelope)).Status); + var delivered = await Fixture.WaitForEffectCountAsync(declaredReceiver, 1); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(delivered.Effects).LogicalId); + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(declaredReceiver, envelope)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).ProcessedMessageCount); + Assert.Equal(1, (await declaredReceiver.GetSnapshotAsync()).ProcessedMessageCount); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs new file mode 100644 index 00000000000..2ac6559c263 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs @@ -0,0 +1,270 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ProcessedRetentionMaintenanceTests : MessagingBehaviorTestBase +{ + public ProcessedRetentionMaintenanceTests() : base(new RetentionFixture()) { } + + [Fact] + public async Task SustainedNonemptyInbox_CompactsAtExpiryAndPreservesCurrentRecords() + { + var receiver = await StartBacklogAsync(); + var owner = (await receiver.GetSnapshotAsync()).InboxJobId; + var old = CreateEnvelope(receiver, NewMessage(120, "old")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Fixture.Clock.Advance(TimeSpan.FromMinutes(5)); + var current = CreateEnvelope(receiver, NewMessage(121, "current")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, current)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 2); + _ = await receiver.GetSnapshotAsync(); + Fixture.Clock.Advance(TimeSpan.FromMinutes(5) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, old)).Status); + var processed = GetProcessed(receiver); + Assert.Equal(2, processed.Count); + + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + var fresh = CreateEnvelope(receiver, NewMessage(122, "fresh")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, fresh)).Status); + var maintained = await Fixture.WaitForEffectCountAsync(receiver, 3); + _ = await receiver.GetSnapshotAsync(); + + Assert.Equal(1, maintained.InboxCount); + Assert.Equal(owner, maintained.InboxJobId); + Assert.Empty(maintained.InboxDeadLetters); + Assert.Equal(2, processed.Count); + Assert.False(processed.ContainsKey(old.Envelope.MessageId)); + Assert.True(processed.ContainsKey(current.Envelope.MessageId)); + Assert.True(processed.ContainsKey(fresh.Envelope.MessageId)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, current)).Status); + } + + [Fact] + public async Task FreshActivation_CompactsProcessedRecordsWithoutDeadLettersOrOwnerClear() + { + var receiver = await StartBacklogAsync(); + var old = CreateEnvelope(receiver, NewMessage(123, "initial-expiry")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + var before = await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + await receiver.RequestDeactivationAsync(); + var after = await receiver.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, after.ActivationId); + Assert.Equal(1, after.InboxCount); + Assert.Equal(before.InboxJobId, after.InboxJobId); + Assert.Empty(after.InboxDeadLetters); + Assert.Empty(GetProcessed(receiver)); + Assert.Equal(1, Assert.Single(after.Effects).Count); + } + + [Fact] + public async Task BusyInbox_AmortizesScansAcrossRetentionCadence() + { + var receiver = await StartBacklogAsync(); + var context = Fixture.GetGrainContext(receiver); + var extension = context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var counted = new CountingProcessedDictionary(GetProcessed(receiver)); + extension.GetType().GetField("_processed", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(extension, counted); + var oldest = CreateEnvelope(receiver, NewMessage(124, "oldest")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, oldest)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + var next = CreateEnvelope(receiver, NewMessage(125, "next")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, next)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 2); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(0, counted.Enumerations); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + var trigger = CreateEnvelope(receiver, NewMessage(126, "sweep")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, trigger)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 3); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(1, counted.Enumerations); + Assert.False(counted.ContainsKey(oldest.Envelope.MessageId)); + Assert.True(counted.ContainsKey(next.Envelope.MessageId)); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + for (var i = 0; i < 8; i++) + { + var message = CreateEnvelope(receiver, NewMessage(130 + i, "within-cadence")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 4 + i); + _ = await receiver.GetSnapshotAsync(); + } + Assert.Equal(1, counted.Enumerations); + Assert.True(counted.ContainsKey(next.Envelope.MessageId)); + Fixture.Clock.Advance(TimeSpan.FromMinutes(2.5) - TimeSpan.FromTicks(1)); + var following = CreateEnvelope(receiver, NewMessage(140, "following-sweep")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, following)).Status); + var after = await Fixture.WaitForEffectCountAsync(receiver, 12); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(2, counted.Enumerations); + Assert.False(counted.ContainsKey(next.Envelope.MessageId)); + Assert.True(counted.ContainsKey(trigger.Envelope.MessageId)); + Assert.Equal(1, after.InboxCount); + } + + [Fact] + public async Task PumpMaintenance_WritesOnlyWhenExpiredRecordsExist() + { + var receiver = await StartBacklogAsync(); + var old = CreateEnvelope(receiver, NewMessage(141, "pump-expiry")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + var owned = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await RunPumpAsync(receiver)).Status); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await RunPumpAsync(receiver)).Status); + + var maintained = await receiver.GetSnapshotAsync(); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(GetProcessed(receiver)); + Assert.Equal(1, maintained.InboxCount); + Assert.Equal(owned.InboxJobId, maintained.InboxJobId); + Assert.Empty(maintained.InboxDeadLetters); + Assert.Equal(1, Assert.Single(maintained.Effects).Count); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await RunPumpAsync(receiver)).Status); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + } + + [Fact] + public async Task MaintenanceWriteFailure_FencesOldStateAndFreshReplayRetriesCompaction() + { + var receiver = await StartBacklogAsync(); + var old = CreateEnvelope(receiver, NewMessage(142, "failed-maintenance")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + Fixture.Storage.FailWrite(journal); + + var failure = await Assert.ThrowsAsync(() => RunPumpAsync(receiver)); + + Assert.Same(failure, await oldGrain.DeactivationFailure.Task); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, oldGrain.GetSnapshotForTest().ProcessedMessageCount); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + var newGrain = Assert.IsType(Fixture.GetGrainContext(receiver).GrainInstance); + Assert.NotEqual(oldGrain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.IsType(newGrain.ReplayedSnapshot).ProcessedMessageCount); + Assert.Equal(0, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.InboxCount); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + } + + private async Task RunPumpAsync(IMessagingTestGrain receiver) + { + var snapshot = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var extension = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var run = new PumpContext(Assert.IsType(snapshot.InboxJob)); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.GrainContext, context), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + DurableJobRunResult result = null!; + await OnTurnAsync(context, () => result = extension.ExecuteJobAsync(run, TestContext.Current.CancellationToken).GetAwaiter().GetResult()); + Assert.Equal(DurableJobRunStatus.InProgress, result.Status); + Assert.Null(Assert.IsType((await stopped).Payload).Exception); + await OnTurnAsync(context, () => result = extension.ExecuteJobAsync(run, TestContext.Current.CancellationToken).GetAwaiter().GetResult()); + return result; + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completed.SetResult(); } + catch (Exception exception) { completed.SetException(exception); } + }); + return completed.Task; + } + + private sealed class PumpContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + + private sealed class CountingProcessedDictionary(IDictionary inner) : IDictionary + { + public int Enumerations { get; private set; } + public DateTimeOffset this[HierarchicalKey key] { get => inner[key]; set => inner[key] = value; } + public ICollection Keys => inner.Keys; + public ICollection Values { get { Enumerations++; return inner.Values; } } + public int Count => inner.Count; + public bool IsReadOnly => inner.IsReadOnly; + public void Add(HierarchicalKey key, DateTimeOffset value) => inner.Add(key, value); + public void Add(KeyValuePair item) => inner.Add(item); + public void Clear() => inner.Clear(); + public bool Contains(KeyValuePair item) => inner.Contains(item); + public bool ContainsKey(HierarchicalKey key) => inner.ContainsKey(key); + public void CopyTo(KeyValuePair[] array, int index) => inner.CopyTo(array, index); + public bool Remove(HierarchicalKey key) => inner.Remove(key); + public bool Remove(KeyValuePair item) => inner.Remove(item); + public bool TryGetValue(HierarchicalKey key, out DateTimeOffset value) => inner.TryGetValue(key, out value); + public IEnumerator> GetEnumerator() { Enumerations++; return inner.GetEnumerator(); } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + private async Task StartBacklogAsync() + { + var receiver = NewGrain(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/retained-backlog"); + var backlog = CreateEnvelope(receiver, NewMessage(119, "backlog") with { ThrowDuringPreparation = true }, "messages/retained-backlog"); + var rescheduled = Fixture.Metrics.GetCount("orleans-durablejobs-jobs-rescheduled"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, backlog)).Status); + await handler.WaitUntilEnteredAsync(); + handler.Release(); + await Fixture.Metrics.WaitForCountAsync("orleans-durablejobs-jobs-rescheduled", rescheduled + 1); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.Effects); + Assert.Empty(pending.InboxDeadLetters); + return receiver; + } + + private IDurableDictionary GetProcessed(IMessagingTestGrain receiver) => + Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredKeyedService>( + "__orleans.messaging.inbox-processed"); + + private sealed class RetentionFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxProcessingAttempts = 10; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs b/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs new file mode 100644 index 00000000000..a8bc38600e9 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace Orleans.Messaging.Tests.Functional; + +[JsonSerializable(typeof(string))] +[JsonSerializable(typeof(int))] +[JsonSerializable(typeof(uint))] +[JsonSerializable(typeof(DateTime))] +internal partial class StandardStateJsonContext : JsonSerializerContext; diff --git a/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj new file mode 100644 index 00000000000..6eb416e53f3 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj @@ -0,0 +1,18 @@ + + + $(TestTargetFrameworks) + true + enable + enable + $(NoWarn);ORLEANSEXP005 + + + + + + + + + + + diff --git a/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs b/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs new file mode 100644 index 00000000000..47e0d937673 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs @@ -0,0 +1,137 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class ActivationValidationProbe +{ + private readonly ConcurrentDictionary> _observations = new(); + public Observation[] Get(GrainId grainId) => _observations.TryGetValue(grainId, out var values) ? values.ToArray() : []; + public Observation Track(IGrainContext context) + { + var observation = new Observation(context); + _observations.GetOrAdd(context.GrainId, static _ => new()).Enqueue(observation); + return observation; + } + + public sealed class Observation(IGrainContext context) + { + public IGrainContext Context { get; } = context; + public bool InstanceAvailableDuringConstruction { get; } = context.GrainInstance is not null; + public int Activated; + public int Calls; + } +} + +public interface IActivationValidationTestGrain : IGrainWithGuidKey +{ + Task IncrementAsync(); + Task DeactivateAsync(); +} + +public interface IAlwaysInterleaveValidationTestGrain : IActivationValidationTestGrain +{ + [AlwaysInterleave] + Task InterleaveAsync(); +} + +public abstract class ActivationValidationTestGrain : DurableGrain, IActivationValidationTestGrain +{ + private readonly IDurableValue _value; + private readonly ActivationValidationProbe.Observation _observation; + + protected ActivationValidationTestGrain(IGrainContext context, ActivationValidationProbe probe, + IDurableValue value) + { + _value = value; + _observation = probe.Track(context); + } + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _observation.Activated); + return base.OnActivateAsync(cancellationToken); + } + + public async Task IncrementAsync() + { + Interlocked.Increment(ref _observation.Calls); + _value.Value++; + await WriteStateAsync(); + return _value.Value; + } + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + +} + +public sealed class SupportedActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[Reentrant] +public sealed class ReentrantActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[StatelessWorker] +public sealed class StatelessActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[MayInterleave(nameof(Interleave))] +public sealed class MayInterleaveActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value) +{ + public static bool Interleave(IInvokable request) => true; +} + +public sealed class AlwaysInterleaveActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value), IAlwaysInterleaveValidationTestGrain +{ + public Task InterleaveAsync() => Task.CompletedTask; +} + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] +public sealed class ExecutionPropertyAttribute(string key, string value) : Attribute, IGrainPropertiesProviderAttribute +{ + public void Populate(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) => + properties[key] = value; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "TrUe")] +public sealed class MetadataReentrantActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[ExecutionProperty(WellKnownGrainTypeProperties.MayInterleavePredicate, nameof(Interleave))] +public sealed class MetadataMayInterleaveActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value) +{ + public static bool Interleave(IInvokable request) => true; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "false")] +public sealed class MetadataNonReentrantActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs new file mode 100644 index 00000000000..3884a741f81 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs @@ -0,0 +1,306 @@ +using System.Collections.Concurrent; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class BootstrapProbe +{ + private readonly ConcurrentDictionary> _activations = new(); + public void Add(BootstrapObservation observation) => + _activations.GetOrAdd(observation.Context.GrainId, static _ => new()).Enqueue(observation); + public BootstrapObservation[] Get(GrainId id) => _activations[id].ToArray(); +} + +public sealed class BootstrapObservation : IDisposable +{ + public BootstrapObservation(IGrainContext context, BootstrapProbe probe) + { + Context = context; + probe.Add(this); + } + + public IGrainContext Context { get; } + public object? ConstructedGrain { get; private set; } + public bool InstanceAvailableInConstructor { get; private set; } + public IJournaledStateManager? Manager { get; set; } + public IDurableValue? Value { get; set; } + public IInbox? Inbox { get; set; } + public IOutbox? Outbox { get; set; } + public object? Extension { get; set; } + public Exception? ExpectedFailure { get; set; } + public int Activations { get; set; } + public int GrainDisposals { get; set; } + public int Disposals { get; private set; } + + public void Constructed(object grain) + { + ConstructedGrain = grain; + InstanceAvailableInConstructor = Context.GrainInstance is not null; + } + + public void Dispose() => Disposals++; +} + +public sealed class BootstrapState : IInboxHandler, IDisposable +{ + public const string Route = "bootstrap"; + public static GrainId OutputTarget { get; } = GrainId.Create("bootstrap-output", "capture"); + private readonly HandlerProbe _handlers; + public BootstrapState(BootstrapObservation observation, IJournaledStateManager manager, HandlerProbe handlers, + [FromKeyedServices("bootstrap-value")] IDurableValue value, IInbox inbox, IOutbox outbox) + { + _handlers = handlers; + Observation = observation; + observation.Manager = manager; + observation.Value = value; + observation.Inbox = inbox; + observation.Outbox = outbox; + inbox.RegisterHandler(this); + } + + public BootstrapObservation Observation { get; } + public int MessagingStateCountAtActivation { get; private set; } + public object? GrainAtActivation { get; private set; } + public int ActivationValue { get; private set; } + public int HandlerCalls { get; private set; } + public IInboxHandler? HandlerOverride { get; set; } + public int Disposals { get; private set; } + public Task Read() => Task.FromResult(Observation.Value!.Value); + public async Task Set(int value) + { + Observation.Value!.Value = value; + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public async Task SendValue(int value) + { + var context = Observation.Context; + var envelope = TestApplicationProtocol.Create(context.ActivationServices.GetRequiredService(), context.GrainId, OutputTarget, "output", value); + Observation.Value!.Value = value; + Observation.Outbox!.Send(envelope); + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public async Task SendSynchronousValue(int value) + { + var context = Observation.Context; + var envelope = TestApplicationProtocol.Create(context.ActivationServices.GetRequiredService(), context.GrainId, OutputTarget, "output", value); + Observation.Value!.Value = value; + Observation.Outbox!.Send(envelope); + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public Task Persist() => Observation.Manager!.WriteStateAsync(CancellationToken.None).AsTask(); + public Task Activate() + { + Observation.Activations++; + GrainAtActivation = Observation.Context.GrainInstance; + MessagingStateCountAtActivation = ReadMessagingStates(Observation.Manager!).Count(); + ActivationValue = Observation.Value!.Value; + return Task.CompletedTask; + } + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (HandlerOverride is { } handler) + { + await handler.HandleAsync(context, cancellationToken); + return; + } + if (_handlers.TryGet(Observation.Context.GrainId, Route, out var barrier)) + { + barrier.Entered.TrySetResult(); + await barrier.Continue.Task.WaitAsync(cancellationToken); + } + var value = Observation.Value!.Value + 1; + var outgoing = TestApplicationProtocol.Create(Observation.Context.ActivationServices.GetRequiredService(), Observation.Context.GrainId, OutputTarget, "output", value); + Observation.Value.Value = value; + Observation.Outbox!.Send(outgoing); + HandlerCalls++; + context.Complete(); + } + public void Dispose() => Disposals++; + public static IEnumerable ReadMessagingStates(IJournaledStateManager manager) + { + foreach (var name in new[] { "inbox", "inbox-processed", "inbox-message-state", "inbox-dead-letters", + "inbox-job-id", "inbox-job-handle", "inbox-completed-job-id", "inbox-job-sequence" }) + { + if (manager.TryGetStateMachine("__orleans.messaging." + name, out var state)) yield return state; + } + } + +} + +public interface IBootstrapTestGrain : IGrainWithGuidKey +{ + Task GetValueAsync(); + Task SetValueAsync(int value); + Task SendValueAsync(int value); + Task SendSynchronousValueAsync(int value); + Task PersistAsync(); + Task DeactivateAsync(); +} +public interface IMarkedBootstrapTestGrain : IBootstrapTestGrain, IMessagingGrain; +public interface IGenericBootstrapTestGrain : IBootstrapTestGrain; + +public abstract class BootstrapGrainBase : Grain, IBootstrapTestGrain, IDisposable +{ + private readonly BootstrapState _state; + protected BootstrapGrainBase(BootstrapState state) + { + _state = state; + state.Observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) => _state.Activate(); + public Task GetValueAsync() => _state.Read(); + public Task SetValueAsync(int value) => _state.Set(value); + public Task SendValueAsync(int value) => _state.SendValue(value); + public Task SendSynchronousValueAsync(int value) => _state.SendSynchronousValue(value); + public Task PersistAsync() => _state.Persist(); + public Task DeactivateAsync() { DeactivateOnIdle(); return Task.CompletedTask; } + public void Dispose() => _state.Observation.GrainDisposals++; +} +public sealed class PlainBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IMessagingGrain; +public abstract class ApplicationBootstrapBase(BootstrapState state) : BootstrapGrainBase(state), IMessagingGrain; +public sealed class ApplicationBootstrapGrain(BootstrapState state) : ApplicationBootstrapBase(state); +public sealed class InterfaceBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IMarkedBootstrapTestGrain; +public sealed class GenericBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IGenericBootstrapTestGrain, IMessagingGrain; + +public class DurableBootstrapGrain : DurableGrain, IBootstrapTestGrain, IDisposable +{ + private readonly BootstrapState _state; + public DurableBootstrapGrain(BootstrapState state) + { + _state = state; + state.Observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) => _state.Activate(); + public Task GetValueAsync() => _state.Read(); + public Task SetValueAsync(int value) => _state.Set(value); + public Task SendValueAsync(int value) => _state.SendValue(value); + public Task SendSynchronousValueAsync(int value) => _state.SendSynchronousValue(value); + public Task PersistAsync() => _state.Persist(); + public Task DeactivateAsync() { DeactivateOnIdle(); return Task.CompletedTask; } + public void Dispose() => _state.Observation.GrainDisposals++; +} + +public sealed class MarkedDurableBootstrapGrain(BootstrapState state) : DurableBootstrapGrain(state), IMessagingGrain; + +public interface IBootstrapControlGrain : IGrainWithGuidKey { Task PingAsync(); } +public interface IInterleavingBootstrapControlGrain : IBootstrapControlGrain +{ + [AlwaysInterleave] Task InterleaveAsync(); +} +public abstract class BootstrapControlGrain : Grain, IBootstrapControlGrain, IDisposable +{ + protected BootstrapObservation Observation { get; } + protected BootstrapControlGrain(BootstrapObservation observation) + { + Observation = observation; + observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Observation.Activations++; + return Task.CompletedTask; + } + public Task PingAsync() => Task.CompletedTask; + public void Dispose() => Observation.GrainDisposals++; +} +public sealed class EagerBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +public sealed class UnselectedBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation); +public sealed class FailingBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +public sealed class UnselectedEndpointBootstrapGrain : BootstrapControlGrain +{ + public UnselectedEndpointBootstrapGrain(BootstrapObservation observation, + [FromKeyedServices(typeof(IInboxExtension))] IGrainExtension extension) : base(observation) => observation.Extension = extension; +} +public sealed class JournalOnlyBootstrapGrain : BootstrapControlGrain +{ + public JournalOnlyBootstrapGrain(BootstrapObservation observation, IJournaledStateManager manager, + [FromKeyedServices("bootstrap-journal-only")] IDurableValue value) : base(observation) + { + observation.Manager = manager; + observation.Value = value; + } +} +[Reentrant] +public sealed class ReentrantBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +[StatelessWorker] +public sealed class StatelessBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +[MayInterleave(nameof(Interleave))] +public sealed class MayInterleaveBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain +{ + public static bool Interleave(IInvokable request) => true; +} +public sealed class AlwaysInterleaveBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IInterleavingBootstrapControlGrain, IMessagingGrain +{ + public Task InterleaveAsync() => Task.CompletedTask; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "true")] +public sealed class MetadataReentrantBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.MayInterleavePredicate, nameof(Interleave))] +public sealed class MetadataMayInterleaveBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain +{ + public static bool Interleave(IInvokable request) => true; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, "StatelessWorkerPlacement")] +public sealed class MetadataStatelessPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, BootstrapClusterFixture.StatelessPlacementAlias)] +public sealed class AliasedStatelessPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, nameof(RandomPlacement))] +public sealed class MetadataOrdinaryPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, BootstrapClusterFixture.OrdinaryPlacementAlias)] +public sealed class AliasedOrdinaryPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +public sealed class BootstrapClusterFixture : MessagingClusterFixture +{ + public const string StatelessPlacementAlias = "bootstrap-worker-alias"; + public const string OrdinaryPlacementAlias = "bootstrap-directory-alias"; + + public BootstrapProbe Probe { get; } = new(); + protected override void ConfigureServices(IServiceCollection services) + { + services.AddKeyedSingleton(StatelessPlacementAlias, new StatelessWorkerAttribute(1).PlacementStrategy); + services.AddKeyedSingleton(OrdinaryPlacementAlias, new RandomPlacement()); + services.AddSingleton(Probe); + services.AddScoped(); + services.AddScoped(); + var extensionType = ReceiverTestServices.GetImplementationType("InboxExtension"); + var descriptor = services.Last(entry => entry.ServiceType == extensionType); + services.AddScoped(extensionType, provider => + { + var extension = descriptor.ImplementationFactory!(provider); + var context = provider.GetRequiredService(); + if (context.GrainInstance is FailingBootstrapGrain) + { + var observation = provider.GetRequiredService(); + observation.Manager = provider.GetRequiredService(); + observation.Inbox = provider.GetRequiredService(); + observation.Outbox = provider.GetRequiredService(); + observation.Extension = extension; + observation.ExpectedFailure = new IOException("Expected bootstrap runtime construction failure."); + ((IDisposable)extension).Dispose(); + throw observation.ExpectedFailure; + } + return extension; + }); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs new file mode 100644 index 00000000000..77edf5e3dd8 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs @@ -0,0 +1,151 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class DurableJobManagerProbe +{ + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _attempts = []; + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _successes = []; + private readonly ConcurrentDictionary _failures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _postScheduleFailures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _duplicates = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary<(string JobName, GrainId Target), ConcurrentQueue> _scheduledJobs = []; + private readonly ConcurrentDictionary _scheduleBarriers = new(StringComparer.Ordinal); + + public void FailNext(string jobName) => + _failures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public void FailAfterNext(string jobName) => + _postScheduleFailures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public void DuplicateNext(string jobName) => + _duplicates.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public int GetAttemptCount(string jobName, GrainId target) => + _attempts.TryGetValue((jobName, target), out var count) ? count : 0; + + public int GetSuccessCount(string jobName, GrainId target) => + _successes.TryGetValue((jobName, target), out var count) ? count : 0; + + public IReadOnlyList GetScheduledJobs(string jobName, GrainId target) => + _scheduledJobs.TryGetValue((jobName, target), out var jobs) ? jobs.ToArray() : []; + + public ScheduleBarrier BlockNext(string jobName) + { + var barrier = new ScheduleBarrier(); + if (!_scheduleBarriers.TryAdd(jobName, barrier)) + { + throw new InvalidOperationException($"A scheduling barrier is already armed for '{jobName}'."); + } + + return barrier; + } + + internal void OnAttempt(ScheduleJobRequest request) => + _attempts.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal void OnSuccess(ScheduleJobRequest request) => + _successes.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal void OnScheduled(ScheduleJobRequest request, DurableJob job) => + _scheduledJobs.GetOrAdd((request.JobName, request.Target), static _ => new()).Enqueue(job); + + internal bool ShouldFail(string jobName) + => TryConsumeFailure(_failures, jobName); + + internal bool ShouldFailAfterSchedule(string jobName) + => TryConsumeFailure(_postScheduleFailures, jobName); + + internal bool ShouldDuplicate(string jobName) + => TryConsumeFailure(_duplicates, jobName); + + internal async Task WaitIfBlockedAsync(string jobName, CancellationToken cancellationToken) + { + if (!_scheduleBarriers.TryRemove(jobName, out var barrier)) + { + return; + } + + barrier.Entered.TrySetResult(); + await barrier.Release.Task.WaitAsync(cancellationToken); + } + + private static bool TryConsumeFailure( + ConcurrentDictionary failures, + string jobName) + { + while (failures.TryGetValue(jobName, out var remaining) && remaining > 0) + { + if (failures.TryUpdate(jobName, remaining - 1, remaining)) + { + return true; + } + } + + return false; + } + + public sealed class ScheduleBarrier : IDisposable + { + internal TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + public void Continue() => Release.TrySetResult(); + + public void Dispose() => Continue(); + } +} + +internal sealed class ControlledDurableJobManager( + ILocalDurableJobManager inner, + DurableJobManagerProbe probe) : ILocalDurableJobManager +{ + public async Task ScheduleJobAsync( + ScheduleJobRequest request, + CancellationToken cancellationToken) + { + probe.OnAttempt(request); + if (probe.ShouldFail(request.JobName)) + { + throw new IOException($"Injected durable job scheduling failure for '{request.JobName}'."); + } + + await probe.WaitIfBlockedAsync(request.JobName, cancellationToken); + var result = await inner.ScheduleJobAsync(request, cancellationToken); + probe.OnScheduled(request, result); + if (probe.ShouldDuplicate(request.JobName)) + { + var duplicate = await inner.ScheduleJobAsync(request, cancellationToken); + probe.OnScheduled(request, duplicate); + } + + probe.OnSuccess(request); + if (probe.ShouldFailAfterSchedule(request.JobName)) + { + throw new IOException( + $"Injected durable job scheduling response failure for '{request.JobName}'."); + } + + return result; + } + + public Task CancelAsync(DurableJob job, CancellationToken cancellationToken) => + inner.CancelAsync(job, cancellationToken); + + public static void Decorate(IServiceCollection services, DurableJobManagerProbe probe) + { + var descriptor = services.Last(service => service.ServiceType == typeof(ILocalDurableJobManager)); + var factory = descriptor.ImplementationFactory + ?? throw new InvalidOperationException("The durable job manager registration must use an implementation factory."); + services.Remove(descriptor); + services.AddSingleton( + serviceProvider => new ControlledDurableJobManager( + (ILocalDurableJobManager)factory(serviceProvider), + probe)); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs b/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs new file mode 100644 index 00000000000..e9694f58927 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs @@ -0,0 +1,276 @@ +using System.Buffers; +using System.Collections.Concurrent; +using Microsoft.Extensions.Options; +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class ControlledJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog +{ + private VolatileJournalStorageProvider? _inner; + private readonly ConcurrentDictionary _readPlans = new(); + private readonly ConcurrentDictionary _deletePlans = new(); + private readonly ConcurrentDictionary _writePlans = new(); + private readonly ConcurrentDictionary _postWritePlans = new(); + private readonly ConcurrentDictionary _postDeleteFailures = new(); + private readonly ConcurrentDictionary _successfulWrites = new(); + private readonly ConcurrentDictionary _reads = new(); + private readonly ConcurrentDictionary _creations = new(); + private readonly ConcurrentDictionary _initializations = new(); + + private readonly ConcurrentDictionary _snapshots = new(); + public void RequestSnapshot(JournalId journalId) => _snapshots[journalId] = 0; + + public string? JournalFormatKey { get; private set; } + + public void Configure(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + JournalFormatKey = options.Value.JournalFormatKey; + _inner ??= new VolatileJournalStorageProvider(options); + } + + public IJournalStorage CreateStorage(JournalId journalId) + { + _creations.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + return new ControlledJournalStorage(this, journalId, Inner.CreateStorage(journalId)); + } + + public IAsyncEnumerable ListAsync( + JournalCatalogListOptions? options = null, + CancellationToken cancellationToken = default) => + Inner.ListAsync(options, cancellationToken); + + private VolatileJournalStorageProvider Inner => + _inner ?? throw new InvalidOperationException("The controlled journal storage provider has not been configured."); + + public WriteBarrier BlockWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + var plan = new WritePlan(matchingWrite, fail: false); + if (!_writePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + public WriteBarrier BlockDelete(JournalId journalId) + { + var plan = new WritePlan(1, fail: false); + if (!_deletePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A delete plan is already armed for journal '{journalId}'."); + } + return new WriteBarrier(plan); + } + + public WriteBarrier BlockRead(JournalId journalId, int matchingRead = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingRead); + var plan = new WritePlan(matchingRead, fail: false); + if (!_readPlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A read plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + public void FailWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_writePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + } + + public void FailAfterWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_postWritePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A post-write plan is already armed for journal '{journalId}'."); + } + } + + public WriteBarrier BlockAcknowledgement(JournalId journalId) + { + var plan = new WritePlan(1, fail: false); + if (!_postWritePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A post-write plan is already armed for journal '{journalId}'."); + } + return new WriteBarrier(plan); + } + + public void FailAfterDelete(JournalId journalId) + { + if (!_postDeleteFailures.TryAdd(journalId, 0)) + { + throw new InvalidOperationException($"A post-delete failure is already armed for journal '{journalId}'."); + } + } + + public int GetSuccessfulWriteCount(JournalId journalId) => + _successfulWrites.TryGetValue(journalId, out var count) ? count : 0; + + public int GetCreationCount(JournalId journalId) => _creations.TryGetValue(journalId, out var count) ? count : 0; + public int GetReadCount(JournalId journalId) => _reads.TryGetValue(journalId, out var count) ? count : 0; + public int GetInitializationCount(JournalId journalId) => _initializations.TryGetValue(journalId, out var count) ? count : 0; + + private async ValueTask BeforeWriteAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_writePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _writePlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.EntryScheduler = TaskScheduler.Current; + plan.EntryContext = ReceiverTestServices.CurrentGrainContext; + plan.Entered.TrySetResult(); + if (plan.Fail) + { + throw new IOException($"Injected journal write failure for '{journalId}'."); + } + + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private async ValueTask BeforeReadAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_readPlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _readPlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private void OnWriteSucceeded(JournalId journalId) => + _successfulWrites.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + + private async ValueTask AfterWriteAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_postWritePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _postWritePlans.TryRemove(new KeyValuePair(journalId, plan)); + if (plan.Fail) + { + throw new IOException($"Injected post-commit journal response failure for '{journalId}'."); + } + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + internal sealed class WritePlan(int target, bool fail) + { + public int Target { get; } = target; + public bool Fail { get; } = fail; + public int Seen; + public TaskScheduler? EntryScheduler; + public IGrainContext? EntryContext; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class WriteBarrier : IDisposable + { + private readonly WritePlan _plan; + + internal WriteBarrier(WritePlan plan) => _plan = plan; + + public TaskScheduler? EntryScheduler => _plan.EntryScheduler; + public IGrainContext? EntryContext => _plan.EntryContext; + public Task WaitUntilEnteredAsync() => _plan.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => _plan.Release.TrySetResult(); + public void Dispose() => Release(); + public void Fail() => Fail(new IOException("Injected blocked journal write failure.")); + public void Fail(Exception exception) => _plan.Release.TrySetException(exception); + } + + private sealed class ControlledJournalStorage( + ControlledJournalStorageProvider owner, + JournalId journalId, + IJournalStorage inner) : IJournalStorage + { + public bool IsCompactionRequested => owner._snapshots.ContainsKey(journalId) || inner.IsCompactionRequested; + + public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + var grain = ReceiverTestServices.CurrentGrainContext?.GrainInstance as MessagingTestGrain; + owner._reads.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + await owner.BeforeReadAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReadAsync(consumer, cancellationToken).ConfigureAwait(false); + grain?.CaptureStorageRead(); + } + + public ValueTask CreateIfNotExistsAsync( + IReadOnlyDictionary? metadata = null, + CancellationToken cancellationToken = default) + { + owner._initializations.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + return inner.CreateIfNotExistsAsync(metadata, cancellationToken); + } + + public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) => + inner.GetMetadataAsync(cancellationToken); + + public ValueTask UpdateMetadataAsync( + IReadOnlyDictionary? set = null, + IEnumerable? remove = null, + string? expectedETag = null, + CancellationToken cancellationToken = default) => + inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + + public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var instance = ReceiverTestServices.CurrentGrainContext?.GrainInstance; + var grain = instance as MessagingTestGrain; + var captured = grain?.CaptureStorageWrite(); + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReplaceAsync(value, cancellationToken).ConfigureAwait(false); + owner._snapshots.TryRemove(journalId, out _); + owner.OnWriteSucceeded(journalId); + await owner.AfterWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + if (captured is not null) grain!.PublishStoredSnapshot(captured); + } + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var instance = ReceiverTestServices.CurrentGrainContext?.GrainInstance; + var grain = instance as MessagingTestGrain; + var captured = grain?.CaptureStorageWrite(); + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.AppendAsync(value, cancellationToken).ConfigureAwait(false); + owner.OnWriteSucceeded(journalId); + await owner.AfterWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + if (captured is not null) grain!.PublishStoredSnapshot(captured); + } + + public async ValueTask DeleteAsync(CancellationToken cancellationToken) + { + if (owner._deletePlans.TryRemove(journalId, out var plan)) + { + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + await inner.DeleteAsync(cancellationToken).ConfigureAwait(false); + if (owner._postDeleteFailures.TryRemove(journalId, out _)) + { + throw new IOException($"Injected post-delete journal response failure for '{journalId}'."); + } + } + } +} diff --git a/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs b/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs new file mode 100644 index 00000000000..8cbd86e17c7 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs @@ -0,0 +1,48 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class FaultingInboxCodecFixture : MessagingClusterFixture +{ + public Exception? NextFailure { get; set; } + public bool FailOnSnapshot { get; set; } + public bool? FailedSnapshot { get; private set; } + protected override void ConfigureServices(IServiceCollection services) + { + var descriptor = services.Last(entry => entry.ServiceType == typeof(IDurableDictionaryCommandCodec<,>) + && Equals(entry.ServiceKey, "orleans-binary")); + var implementation = descriptor.KeyedImplementationType!.MakeGenericType(typeof(HierarchicalKey), typeof(InboxMessage)); + services.AddKeyedSingleton>("orleans-binary", (sp, _) => + new FaultingCodec(this, (IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, implementation))); + } + + private sealed class FaultingCodec(FaultingInboxCodecFixture owner, + IDurableDictionaryCommandCodec inner) + : IDurableDictionaryCommandCodec + { + private void Check(bool snapshot = false) + { + if (snapshot == owner.FailOnSnapshot && owner.NextFailure is { } failure) + { + owner.NextFailure = null; + owner.FailedSnapshot = snapshot; + throw failure; + } + } + public void WriteSet(HierarchicalKey key, InboxMessage value, JournalStreamWriter writer) + { + Check(); + inner.WriteSet(key, value, writer); + } + public void WriteRemove(HierarchicalKey key, JournalStreamWriter writer) { Check(); inner.WriteRemove(key, writer); } + public void WriteClear(JournalStreamWriter writer) { Check(); inner.WriteClear(writer); } + public void WriteSnapshot(IReadOnlyCollection> items, JournalStreamWriter writer) + { + Check(snapshot: true); + inner.WriteSnapshot(items, writer); + } + public void Apply(JournalBufferReader input, IDurableDictionaryCommandHandler consumer) => inner.Apply(input, consumer); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs new file mode 100644 index 00000000000..0ecb2480c55 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs @@ -0,0 +1,53 @@ +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class HandlerProbe +{ + private readonly ConcurrentDictionary<(GrainId GrainId, string Route), Barrier> _barriers = new(); + + public Barrier Arm(GrainId grainId, string route) + { + var barrier = new Barrier(this, (grainId, route)); + if (!_barriers.TryAdd((grainId, route), barrier)) + { + throw new InvalidOperationException($"A handler barrier is already armed for '{grainId}' and route '{route}'."); + } + + return barrier; + } + + public bool TryGet( + GrainId grainId, + string route, + [NotNullWhen(true)] out Barrier? barrier) => + _barriers.TryGetValue((grainId, route), out barrier); + + public sealed class Barrier : IDisposable + { + private readonly HandlerProbe _owner; + private readonly (GrainId GrainId, string Route) _key; + + internal Barrier(HandlerProbe owner, (GrainId GrainId, string Route) key) + { + _owner = owner; + _key = key; + } + + internal TaskCompletionSource Entered { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Continue { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => Continue.TrySetResult(); + + public void Dispose() + { + Release(); + _owner._barriers.TryRemove(new KeyValuePair<(GrainId GrainId, string Route), Barrier>(_key, this)); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs new file mode 100644 index 00000000000..75e558c8daf --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs @@ -0,0 +1,70 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +// Captures opaque handler output in the same journal as inbox effects. Dispatch belongs to the outbox layer. +internal sealed class JournaledTestOutbox(IDurableDictionary messages, IGrainContext context) + : IOutbox, ILifecycleObserver, IEnumerable> +{ + private bool _stopped; + private readonly TaskCompletionSource _stopping = new(TaskCreationOptions.RunContinuationsAsynchronously); + public GrainId SenderId => context.GrainId; + public Task Stopping => _stopping.Task; + public int SendCalls { get; private set; } + public Exception? NextSendFailure { get; set; } + public IDurableDictionary StoredMessages { get; } = messages; + public int Count => StoredMessages.Count; + public IEnumerable Messages => StoredMessages.Values; + public IEnumerator> GetEnumerator() => StoredMessages.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public void Send(OutboxMessage envelope) + { + SendCalls++; + if (_stopped) throw new InvalidOperationException("The test outbox owner is stopped."); + if (envelope.Envelope.MessageId.IsDefault || envelope.ReceiverId.IsDefault + || !envelope.Envelope.TryGetBytes(MessageHeaders.Payload, out _)) + { + throw new ArgumentException("An outgoing envelope requires identities and opaque payload bytes.", nameof(envelope)); + } + var sessions = context.ActivationServices.GetRequiredService(); + if (envelope.Envelope.TryGetValue(MessageHeaders.Sender, new Serializer(sessions), out var sender) + && sender != context.GrainId) + { + throw new InvalidOperationException($"The outgoing sender for '{envelope.Envelope.MessageId}' differs from the test outbox owner."); + } + if (TryGetMessage(envelope.Envelope.MessageId, out var existing)) + { + if (existing.ReceiverId != envelope.ReceiverId + || !string.Equals(TestApplicationProtocol.Subject(existing.Envelope), + TestApplicationProtocol.Subject(envelope.Envelope), StringComparison.Ordinal) + || !existing.Envelope.Payload.Span.SequenceEqual(envelope.Envelope.Payload.Span)) + { + throw new InvalidOperationException($"The outbox already contains a different envelope with message ID '{envelope.Envelope.MessageId}'."); + } + return; + } + if (NextSendFailure is { } failure) + { + NextSendFailure = null; + throw failure; + } + StoredMessages.Add(envelope.Envelope.MessageId, envelope); + } + + public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) => + StoredMessages.TryGetValue(messageId, out envelope); + public Task OnStart(CancellationToken cancellationToken) => Task.CompletedTask; + public Task OnStop(CancellationToken cancellationToken) + { + _stopped = true; + _stopping.TrySetResult(); + return Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs new file mode 100644 index 00000000000..15a2fd6af35 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs @@ -0,0 +1,83 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public abstract class MessagingBehaviorTestBase : IAsyncLifetime +{ + protected MessagingBehaviorTestBase() + : this(new MessagingClusterFixture()) + { + } + + protected MessagingBehaviorTestBase(MessagingClusterFixture fixture) => Fixture = fixture; + + protected MessagingClusterFixture Fixture { get; } + protected SerializerSessionPool Sessions => Fixture.Client.ServiceProvider.GetRequiredService(); + + public ValueTask InitializeAsync() => Fixture.InitializeAsync(); + public ValueTask DisposeAsync() => Fixture.DisposeAsync(); + + protected IMessagingTestGrain NewGrain() => + Fixture.Client.GetGrain(Guid.NewGuid()); + + protected static TestMessage NewMessage(int sequence, string value) => + new(TestApplicationProtocol.NewMessageId(), sequence, value); + + protected async Task RefreshSeededOwnerAsync(IMessagingTestGrain receiver) + { + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + Assert.NotSame(previous, Fixture.GetGrainContext(receiver)); + } + + protected static Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + DeliverWithCancellationAsync(receiver, envelope, TestContext.Current.CancellationToken); + + protected static async Task DeliverWithCancellationAsync( + IMessagingTestGrain receiver, + InboxMessage envelope, + CancellationToken cancellationToken) => + await receiver.AsReference().DeliverAsync(envelope, cancellationToken); + + protected static async Task WaitForBarrierAsync( + IMessagingTestGrain receiver, + HandlerProbe.Barrier barrier) + { + try + { + await barrier.WaitUntilEnteredAsync(); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Handler did not start. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, maxHandlers={snapshot.MaxConcurrentHandlers}, deadLetters={string.Join(" | ", snapshot.InboxDeadLetters.Select(static item => item.Reason))}.", + exception); + } + } + + protected InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + TestMessage message, + string route = "messages/record") => + CreateEnvelope(receiver, (object)message, route); + + protected InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + object body, + string route) + { + var sessions = Fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("external-test-sender", Guid.NewGuid().ToString("N")); + return TestApplicationProtocol.CreateInbox(sessions, sender, route, body); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs new file mode 100644 index 00000000000..a48111fbb18 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -0,0 +1,159 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public class MessagingClusterFixture : IAsyncLifetime +{ + public MessagingClusterFixture() : this(1) + { + } + + private MessagingClusterFixture(int initialSilos) + { + Clock = new FakeTimeProvider(DateTimeOffset.UtcNow); + Storage = new ControlledJournalStorageProvider(); + Metrics = new MessagingMetricProbe(); + JobManagerProbe = new DurableJobManagerProbe(); + HandlerProbe = new HandlerProbe(); + SnapshotProbe = new SnapshotProbe(); + ActivationProbe = new ActivationValidationProbe(); + var clusterId = $"messaging-{Guid.NewGuid():N}"; + var serviceId = $"messaging-service-{Guid.NewGuid():N}"; + var builder = new InProcessTestClusterBuilder((short)initialSilos); + builder.ConfigureClient(clientBuilder => + clientBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + })); + builder.ConfigureSilo((_, siloBuilder) => + { + siloBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + }); + siloBuilder.Services.AddSingleton(Clock); + siloBuilder.Services.UseTimeProviderForBackgroundAreas(TimeProvider.System); + siloBuilder.Services.AddSingleton(HandlerProbe); + siloBuilder.Services.AddSingleton(SnapshotProbe); + siloBuilder.Services.AddSingleton(ActivationProbe); + siloBuilder.Services.AddSingleton(); + siloBuilder.UseInMemoryDurableJobs(); + ReceiverTestServices.Add(siloBuilder.Services, ConfigureOptions); + siloBuilder.ConfigureServices(services => + ControlledDurableJobManager.Decorate(services, JobManagerProbe)); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.AddSingleton(Storage); + siloBuilder.Services.AddSingleton(serviceProvider => + { + Storage.Configure(serviceProvider.GetRequiredService>()); + return Storage; + }); + siloBuilder.Services.AddSingleton( + serviceProvider => (IJournalStorageCatalog)serviceProvider.GetRequiredService()); + ConfigureServices(siloBuilder.Services); + }); + Cluster = builder.Build(); + } + + public InProcessTestCluster Cluster { get; } + public IClusterClient Client => Cluster.Client!; + public FakeTimeProvider Clock { get; } + public ControlledJournalStorageProvider Storage { get; } + public MessagingMetricProbe Metrics { get; } + public DurableJobManagerProbe JobManagerProbe { get; } + public HandlerProbe HandlerProbe { get; } + public SnapshotProbe SnapshotProbe { get; } + public ActivationValidationProbe ActivationProbe { get; } + + public Task WaitForEffectCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.Effects.Sum(static effect => effect.Count) >= expected); + + public Task WaitForInboxCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.InboxCount == expected); + + public Task WaitForDeadLetterCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.InboxDeadLetters.Count + snapshot.OutboxDeadLetters.Count >= expected); + + public Task WaitForOutboxCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.OutboxCount == expected); + + public EndpointSnapshot GetSnapshot(IMessagingTestGrain grain) => + GetGrainInstance(grain).GetSnapshotForTest(); + + public OutboxMessage[] GetStagedOutput(IMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices.GetRequiredService().Messages.ToArray(); + + public ValueTask WriteStateAsync(IMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices + .GetRequiredService() + .WriteStateAsync(TestContext.Current.CancellationToken); + + internal IGrainContext GetGrainContext(IMessagingTestGrain grain) + { + if (!Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)) + { + throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' is not active."); + } + + return context; + } + + private MessagingTestGrain GetGrainInstance(IMessagingTestGrain grain) => + GetGrainContext(grain).GrainInstance as MessagingTestGrain + ?? throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' has an unexpected implementation."); + + protected virtual void ConfigureServices(IServiceCollection services) { } + + protected virtual void ConfigureOptions(InboxOptions options) + { + options.MaxCapacity = 2; + options.DeduplicationWindow = TimeSpan.FromMinutes(10); + options.MaxOutboxRetryAge = TimeSpan.FromMinutes(5); + options.MaxProcessingAttempts = 1; + options.MaxDeliveryAttempts = 3; + options.MaxRetainedDeadLetters = 2; + options.DeadLetterRetentionPeriod = TimeSpan.FromHours(1); + options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(25); + options.InboxBatchSize = 8; + options.OutboxBatchSize = 8; + } + + public ValueTask InitializeAsync() => new(Cluster.DeployAsync()); + public async ValueTask DisposeAsync() + { + await Cluster.DisposeAsync(); + Metrics.Dispose(); + } +} + +public sealed class DedupeExpiryClusterFixture : MessagingClusterFixture +{ +} + +public sealed class InboxCapacityClusterFixture : MessagingClusterFixture +{ + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxCapacity = 1; + options.MaxProcessingAttempts = 2; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs new file mode 100644 index 00000000000..c07cf9cc4a4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs @@ -0,0 +1,121 @@ +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class MessagingMetricProbe : IDisposable +{ + private readonly ConcurrentDictionary<(string Instrument, string TagValue), long> _measurements = []; + private readonly ConcurrentDictionary _gauges = []; + private readonly object _lock = new(); + private TaskCompletionSource _changed = CreateSignal(); + private readonly MeterListener _listener; + + public MessagingMetricProbe() + { + _listener = new MeterListener + { + InstrumentPublished = static (instrument, listener) => + { + if (instrument.Meter.Name == "Microsoft.Orleans" + && instrument.Name is "orleans-messaging-orphaned-jobs-reclaimed" + or "orleans-messaging-inbox-messages-processed" + or "orleans-durablejobs-job-attempts-started" + or "orleans-durablejobs-handler-executions-started" + or "orleans-durablejobs-jobs-completed" + or "orleans-durablejobs-jobs-rescheduled" + or "orleans-messaging-inbox-depth" + or "orleans-messaging-outbox-depth") + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + _listener.SetMeasurementEventCallback(OnMeasurement); + _listener.Start(); + } + + public long GetCount(string instrument, string tagValue = "") => + _measurements.TryGetValue((instrument, tagValue), out var count) ? count : 0; + + public long GetDepth(string instrument) + { + _listener.RecordObservableInstruments(); + return _gauges.TryGetValue(instrument, out var value) ? value : 0; + } + + public Task WaitForCountAsync( + string instrument, + long expected, + string tagValue = "") => + WaitForCountWithCancellationAsync( + instrument, + expected, + tagValue, + TestContext.Current.CancellationToken); + + public async Task WaitForCountWithCancellationAsync( + string instrument, + long expected, + string tagValue, + CancellationToken cancellationToken) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + while (GetCount(instrument, tagValue) < expected) + { + Task changed; + lock (_lock) + { + if (GetCount(instrument, tagValue) >= expected) + { + return; + } + + changed = _changed.Task; + } + + await changed.WaitAsync(timeout.Token); + } + } + + public void Dispose() => _listener.Dispose(); + + private void OnMeasurement( + Instrument instrument, + long measurement, + ReadOnlySpan> tags, + object? state) + { + if (instrument.Name is "orleans-messaging-inbox-depth" or "orleans-messaging-outbox-depth") + { + _gauges[instrument.Name] = measurement; + return; + } + + var tagValue = ""; + var tagName = instrument.Name == "orleans-messaging-inbox-messages-processed" ? "status" : "job_name"; + foreach (var tag in tags) + { + if (tag.Key == tagName) + { + tagValue = tag.Value as string ?? ""; + break; + } + } + + _measurements.AddOrUpdate( + (instrument.Name, tagValue), + measurement, + (_, current) => current + measurement); + lock (_lock) + { + _changed.TrySetResult(); + _changed = CreateSignal(); + } + } + + private static TaskCompletionSource CreateSignal() => + new(TaskCreationOptions.RunContinuationsAsynchronously); +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs new file mode 100644 index 00000000000..4fa64fd0fc9 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -0,0 +1,502 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Messaging; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +public interface IMessagingTestGrain : IGrainWithGuidKey +{ + Task SendAsync(GrainId target, string route, TestMessage message); + Task SendDuplicateAsync(GrainId target, string route, TestMessage message); + Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message); + Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message); + Task RetryWriteStateAsync(); + Task StageEffectAsync(MessageEffect effect); + Task StageOutputAsync(OutboxMessage envelope); + Task AcceptAndDeactivateAsync(InboxMessage envelope); + Task SetInboxOwnershipAsync(string ownershipId, DurableJob job); + Task SeedInboxStateAsync(InboxMessage envelope, string? ownershipId, DurableJob? job); + Task ConfigureHandlerAsync(bool enabled); + Task RemoveInboxDeadLetterAsync(HierarchicalKey messageId); + Task RemoveOutboxDeadLetterAsync(HierarchicalKey messageId); + Task GetSnapshotAsync(); + Task RequestDeactivationAsync(); + Task SetControlEnvelopeAsync(InboxMessage envelope); + Task DeleteStateAndDeactivateAsync(); + Task HoldPumpTurnAsync(string barrierRoute, bool deactivate); + Task HoldPumpTurnAsync(string barrierRoute, InboxMessage replacement, bool deactivate); +} + +[GenerateSerializer, Immutable] +public sealed record TestMessage( + [property: Id(0)] HierarchicalKey LogicalId, + [property: Id(1)] int Sequence, + [property: Id(2)] string Value, + [property: Id(3)] GrainId? ForwardTo = null, + [property: Id(8)] bool ThrowDuringPreparation = false, + [property: Id(5)] bool CommitDuringHandling = false, + [property: Id(6)] bool DeleteDuringHandling = false, + [property: Id(9)] bool ThrowOnceDuringPreparation = false); + +[GenerateSerializer, Immutable] +public sealed record MessageEffect( + [property: Id(0)] HierarchicalKey LogicalId, + [property: Id(1)] int Count, + [property: Id(2)] int Sequence, + [property: Id(3)] string Value); + +[GenerateSerializer, Immutable] +public sealed record EndpointSnapshot( + [property: Id(0)] Guid ActivationId, + [property: Id(1)] string SiloAddress, + [property: Id(2)] int InboxCount, + [property: Id(3)] int OutboxCount, + [property: Id(4)] int MaxConcurrentHandlers, + [property: Id(5)] IReadOnlyList Effects, + [property: Id(6)] IReadOnlyList InboxDeadLetters, + [property: Id(7)] IReadOnlyList OutboxDeadLetters, + [property: Id(8)] string? InboxJobId, + [property: Id(9)] int ProcessedMessageCount, + [property: Id(12)] string? OutboxJobId, + [property: Id(16)] DurableJob? InboxJob, + [property: Id(17)] DurableJob? OutboxJob); + +[GenerateSerializer, Immutable] +public sealed record DeadLetterSnapshot( + [property: Id(0)] HierarchicalKey MessageId, + [property: Id(1)] string Route, + [property: Id(2)] string Reason, + [property: Id(3)] int AttemptCount, + [property: Id(4)] DateTimeOffset DeadLetteredAt); + +[GrainType("messaging-inbox-test")] +public sealed class MessagingTestGrain : DurableGrain, IMessagingTestGrain, IDurableJobHandler, + IObserver, IDisposable +{ + private readonly IGrainContext _grainContext; + private readonly IDisposable _lifecycleSubscription; + private readonly IJournaledStateManager _journalOwner; + private readonly IInbox _inbox; + private readonly IOutbox _outbox; + private readonly IMessagingDiagnostics _diagnostics; + private readonly IDurableDictionary _effects; + private readonly IDurableDictionary _processedMessages; + private readonly SerializerSessionPool _sessions; + private readonly TestHandlerConfiguration _handlerConfiguration; + internal IInboxHandler? HandlerOverride { get; set; } + private readonly IDurableValue _inboxJobId; + private readonly IDurableValue _inboxJob; + private readonly IDurableValue _outboxJobId; + private readonly IDurableValue _outboxJob; + private readonly ILocalSiloDetails _siloDetails; + private readonly HandlerProbe _handlerProbe; + private readonly SnapshotProbe _snapshotProbe; + private readonly Guid _activationId = Guid.NewGuid(); + private int _activeHandlers; + private int _maxConcurrentHandlers; + private readonly HashSet _failedOnce = []; + + public MessagingTestGrain( + IGrainContext grainContext, + IJournaledStateManager journalOwner, + IInbox inbox, + IOutbox outbox, + IMessagingDiagnostics diagnostics, + [FromKeyedServices("test-effects")] IDurableDictionary effects, + [FromKeyedServices("inbox")] IDurableValue applicationInboxState, + [FromKeyedServices("__orleans.messaging.inbox-processed")] IDurableDictionary processedMessages, + [FromKeyedServices("__orleans.messaging.inbox-job-id")] IDurableValue inboxJobId, + [FromKeyedServices("__orleans.messaging.inbox-job-handle")] IDurableValue inboxJob, + [FromKeyedServices("__orleans.messaging.outbox-job-id")] IDurableValue outboxJobId, + [FromKeyedServices("__orleans.messaging.outbox-job-handle")] IDurableValue outboxJob, + SerializerSessionPool sessions, + TestHandlerConfiguration handlerConfiguration, + ILocalSiloDetails siloDetails, + HandlerProbe handlerProbe, + SnapshotProbe snapshotProbe) + { + _grainContext = grainContext; + _journalOwner = journalOwner; + _inbox = inbox; + _outbox = outbox; + _diagnostics = diagnostics; + _effects = effects; + ArgumentNullException.ThrowIfNull(applicationInboxState); + _processedMessages = processedMessages; + _inboxJobId = inboxJobId; + _inboxJob = inboxJob; + _outboxJobId = outboxJobId; + _outboxJob = outboxJob; + _sessions = sessions; + _handlerConfiguration = handlerConfiguration; + _siloDetails = siloDetails; + _handlerProbe = handlerProbe; + _snapshotProbe = snapshotProbe; + _lifecycleSubscription = GrainLifecycleEvents.AllEvents.Subscribe(this); + } + + public override async Task OnActivateAsync(CancellationToken cancellationToken) + { + if (_handlerConfiguration.IsEnabled(this.GetGrainId())) + { + _inbox.RegisterHandler(new ApplicationDispatcher(this)); + } + await base.OnActivateAsync(cancellationToken); + _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); + } + + public async Task SendAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.Envelope.MessageId; + } + + public async Task SendDuplicateAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.Envelope.MessageId; + } + + public async Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message) + { + var messageId = await SendAsync(target, route, message); + DeactivateOnIdle(); + return messageId; + } + + public Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + return Task.FromResult(envelope.Envelope.MessageId); + } + + public async Task RetryWriteStateAsync() => await WriteStateAsync(); + + public async Task AcceptAndDeactivateAsync(InboxMessage envelope) + { + var extension = (IInboxExtension)ServiceProvider.GetRequiredKeyedService(typeof(IInboxExtension)); + var result = await extension.DeliverAsync(envelope); + AcceptedSnapshot = CreateSnapshot(); + DeactivateOnIdle(); + return result; + } + + public Task StageOutputAsync(OutboxMessage envelope) + { + _outbox.Send(envelope); + return Task.CompletedTask; + } + + public Task StageEffectAsync(MessageEffect effect) + { + _effects[effect.LogicalId] = effect; + return Task.CompletedTask; + } + + public async Task SetInboxOwnershipAsync(string ownershipId, DurableJob job) + { + _inboxJobId.Value = ownershipId; + _inboxJob.Value = job; + await WriteStateAsync(); + } + + public async Task SeedInboxStateAsync(InboxMessage envelope, string? ownershipId, DurableJob? job) + { + var messages = ServiceProvider.GetRequiredKeyedService>( + "__orleans.messaging.inbox"); + messages.Add(envelope.Envelope.MessageId, envelope); + _inboxJobId.Value = ownershipId; + _inboxJob.Value = job; + await WriteStateAsync(); + } + + public Task ConfigureHandlerAsync(bool enabled) + { + _handlerConfiguration.Set(this.GetGrainId(), enabled); + return Task.CompletedTask; + } + + public async Task RemoveInboxDeadLetterAsync(HierarchicalKey messageId) + { + if (!_diagnostics.RemoveInboxDeadLetter(messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + + public async Task RemoveOutboxDeadLetterAsync(HierarchicalKey messageId) + { + if (!_diagnostics.RemoveOutboxDeadLetter(messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + + public Task GetSnapshotAsync() => Task.FromResult(CreateSnapshot()); + + internal EndpointSnapshot GetSnapshotForTest() => CreateSnapshot(); + + public Task RequestDeactivationAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + public async Task DeleteStateAndDeactivateAsync() + { + try + { + var inbox = (ILifecycleObserver)ServiceProvider.GetRequiredKeyedService(typeof(IInboxExtension)); + var outbox = (ILifecycleObserver)_outbox; + await Task.WhenAll(inbox.OnStop(CancellationToken.None), outbox.OnStop(CancellationToken.None)); + await _journalOwner.DeleteStateAsync(CancellationToken.None); + } + finally + { + DeactivateOnIdle(); + } + } + + internal EndpointSnapshot? AcceptedSnapshot { get; private set; } + + private InboxMessage? _controlEnvelope; + internal TaskCompletionSource ControlDeliveryEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task SetControlEnvelopeAsync(InboxMessage envelope) + { + _controlEnvelope = envelope; + return Task.CompletedTask; + } + + internal TaskScheduler? JobScheduler { get; private set; } + internal IGrainContext? JobGrainContext { get; private set; } + + public async Task ExecuteJobAsync(IJobRunContext context, CancellationToken attemptCancellationToken) + { + JobScheduler = TaskScheduler.Current; + JobGrainContext = ReceiverTestServices.CurrentGrainContext; + switch (context.Job.Name) + { + case "test/deliver-envelope": + ControlDeliveryEntered.TrySetResult(); + var extension = (IInboxExtension)ServiceProvider.GetRequiredKeyedService(typeof(IInboxExtension)); + await extension.DeliverAsync(_controlEnvelope ?? throw new InvalidOperationException("A control envelope must be configured."), attemptCancellationToken); + break; + case "test/write-journal": + await StateManager.WriteStateAsync(attemptCancellationToken); + break; + case "test/probe-scheduler": + break; + default: + throw new NotSupportedException($"Unknown test job '{context.Job.Name}'."); + } + } + + public Task HoldPumpTurnAsync(string barrierRoute, bool deactivate) => + HoldPumpTurnCoreAsync(barrierRoute, null, deactivate); + + public Task HoldPumpTurnAsync(string barrierRoute, InboxMessage replacement, bool deactivate) => + HoldPumpTurnCoreAsync(barrierRoute, replacement, deactivate); + + private async Task HoldPumpTurnCoreAsync(string barrierRoute, InboxMessage? replacement, bool deactivate) + { + if (!_handlerProbe.TryGet(this.GetGrainId(), barrierRoute, out var barrier)) + { + throw new InvalidOperationException("The pump-turn barrier must be armed."); + } + barrier.Entered.TrySetResult(); + await barrier.Continue.Task; + if (replacement is { } envelope) + { + var extension = (IInboxExtension)ServiceProvider.GetRequiredKeyedService(typeof(IInboxExtension)); + await extension.DeliverAsync(envelope); + } + if (deactivate) + { + DeactivateOnIdle(); + } + } + + internal TaskCompletionSource DeactivationFailure { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + void IObserver.OnNext(GrainLifecycleEvents.LifecycleEvent value) + { + if (value is GrainLifecycleEvents.Deactivating { Reason.Exception: { } exception } deactivating + && ReferenceEquals(deactivating.GrainContext, _grainContext)) + { + DeactivationFailure.TrySetResult(exception); + } + } + + void IObserver.OnError(Exception error) => DeactivationFailure.TrySetException(error); + void IObserver.OnCompleted() { } + public void Dispose() + { + _controlEnvelope = null; + _lifecycleSubscription.Dispose(); + } + + private readonly ConcurrentQueue _captures = new(); + private readonly ConcurrentQueue _outputCaptures = new(); + internal IReadOnlyList Captures => _captures.ToArray(); + internal IReadOnlyList OutputCaptures => _outputCaptures.ToArray(); + internal Exception? NextApplyFailure { get; set; } + internal TaskCompletionSource ApplyAttempted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal void ClearCaptures() => _captures.Clear(); + + internal EndpointSnapshot CaptureStorageWrite() + { + var snapshot = CreateSnapshot(); + _captures.Enqueue(snapshot); + _outputCaptures.Enqueue(_outbox.Messages.Select(static envelope => envelope.Envelope.MessageId).ToArray()); + return snapshot; + } + + internal void PublishStoredSnapshot(EndpointSnapshot snapshot) => _snapshotProbe.Publish(this.GetGrainId(), snapshot); + internal void CaptureStorageRead() => ReplayedSnapshot = CreateSnapshot(); + internal EndpointSnapshot? ReplayedSnapshot { get; private set; } + + private OutboxMessage CreateEnvelope(GrainId target, string route, TestMessage message) => + TestApplicationProtocol.Create(_sessions, this.GetGrainId(), target, route, message); + + private async ValueTask HandleAsync( + TestMessage message, + IInboxHandlerContext context, + CancellationToken cancellationToken) + { + var active = Interlocked.Increment(ref _activeHandlers); + _maxConcurrentHandlers = Math.Max(_maxConcurrentHandlers, active); + try + { + if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route, out var gate)) + { + gate.Entered.TrySetResult(); + await gate.Continue.Task.WaitAsync(cancellationToken); + } + + if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route + "/application-preparation", out var preparation)) + { + preparation.Entered.TrySetResult(); + await preparation.Continue.Task.WaitAsync(cancellationToken); + } + cancellationToken.ThrowIfCancellationRequested(); + if (message.CommitDuringHandling) + { + await WriteStateAsync(cancellationToken); + } + if (message.ThrowDuringPreparation || (message.ThrowOnceDuringPreparation && _failedOnce.Add(message.LogicalId))) + { + throw new InvalidOperationException($"Injected handler preparation failure for {message.LogicalId}."); + } + + var outgoing = message.ForwardTo is { } destination + ? TestApplicationProtocol.Create(_sessions, this.GetGrainId(), destination, "messages/forwarded", + message with { ForwardTo = null, ThrowDuringPreparation = false }, + context.Message.Envelope.MessageId.CreateChildKey("forwarded")) + : (OutboxMessage?)null; + if (NextApplyFailure is { } failure) + { + NextApplyFailure = null; + throw failure; + } + _effects.TryGetValue(message.LogicalId, out var prior); + _effects[message.LogicalId] = new MessageEffect(message.LogicalId, (prior?.Count ?? 0) + 1, message.Sequence, message.Value); + ApplyAttempted.TrySetResult(); + if (outgoing is { } output) + { + _outbox.Send(output); + if (TestApplicationProtocol.Read(_sessions, context.Message).Route == "messages/duplicate-output") + { + _outbox.Send(output); + } + } + context.Complete(); + } + finally + { + Interlocked.Decrement(ref _activeHandlers); + } + } + + private void PublishSnapshot() => _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); + + private EndpointSnapshot CreateSnapshot() => + new( + _activationId, + _siloDetails.SiloAddress.ToParsableString(), + _inbox.Count, + _outbox.Count, + _maxConcurrentHandlers, + _effects.Values.OrderBy(static effect => effect.Sequence).ToArray(), + _diagnostics.InboxDeadLetters.Select(ToSnapshot).ToArray(), + _diagnostics.OutboxDeadLetters.Select(ToSnapshot).ToArray(), + _inboxJobId.Value, + _processedMessages.Count, + _outboxJobId.Value, + _inboxJob.Value, + _outboxJob.Value); + + private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => + new( + deadLetter.Message.Envelope.MessageId, + TestApplicationProtocol.Read(_sessions, deadLetter.Message).Route, + deadLetter.Reason, + deadLetter.AttemptCount, + deadLetter.DeadLetteredAt); + + private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => + new( + deadLetter.Message.Envelope.MessageId, + TestApplicationProtocol.Read(_sessions, deadLetter.Message).Route, + deadLetter.Reason, + deadLetter.AttemptCount, + deadLetter.DeadLetteredAt); + + private sealed class ApplicationDispatcher(MessagingTestGrain owner) : IInboxHandler + { + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (owner.HandlerOverride is { } handler) + { + return handler.HandleAsync(context, cancellationToken); + } + + var application = TestApplicationProtocol.Read(owner._sessions, context.Message); + if (!application.Route.StartsWith("messages/", StringComparison.Ordinal) && application.Route != "typed") + { + throw new InvalidOperationException($"Unknown application route '{application.Route}'."); + } + return owner.HandleAsync(application.Body as TestMessage + ?? throw new InvalidOperationException($"Expected {nameof(TestMessage)} application payload."), + context, cancellationToken); + } + } +} + +// Tests can change registration between real activations without adding a production registry API. +public sealed class TestHandlerConfiguration +{ + private readonly ConcurrentDictionary _enabled = new(); + public bool IsEnabled(GrainId id) => !_enabled.TryGetValue(id, out var enabled) || enabled; + public void Set(GrainId id, bool enabled) => _enabled[id] = enabled; +} diff --git a/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs new file mode 100644 index 00000000000..43baddcd16a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs @@ -0,0 +1,116 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Orleans.Timers; + +namespace Orleans.Messaging.Tests.Support; + +internal static class ReceiverTestServices +{ + public const string InboxJobName = "orleans.messaging.inbox-drain"; + + private static readonly Func ReadCurrentContext = typeof(IGrainContext).Assembly + .GetType("Orleans.Runtime.RuntimeContext", throwOnError: true)! + .GetProperty("Current")!.GetMethod!.CreateDelegate>(); + + public static IGrainContext? CurrentGrainContext => ReadCurrentContext(); + + public static Type GetImplementationType(string name) => + typeof(IInbox).Assembly.GetType($"Orleans.Messaging.{name}", throwOnError: true)!; + + public static void Add(IServiceCollection services, Action configure) + { + var inboxType = GetImplementationType("Inbox"); + var extensionType = GetImplementationType("InboxExtension"); + var instrumentsType = GetImplementationType("MessagingInstruments"); + var pumpResultsType = GetImplementationType("MessagingPumpResults"); + var configuratorType = GetImplementationType("MessagingGrainTypeConfigurator"); + services.Configure(options => options.JournalFormatKey = "orleans-binary"); + services.AddOptions().Configure(configure); + services.TryAddSingleton(instrumentsType); + services.TryAddScoped(extensionType, sp => CreateInstance( + extensionType, + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(typeof(ILogger<>).MakeGenericType(extensionType)), + sp.GetRequiredService(instrumentsType), + sp.GetRequiredService(inboxType), + GetDictionary(sp, "inbox"), + GetDictionary(sp, "inbox-processed"), + GetInternalDictionary(sp, "InboxMessageState", "inbox-message-state"), + GetInternalDictionary(sp, "InboxDeadLetter", "inbox-dead-letters"), + GetValue(sp, "inbox-job-id"), + GetValue(sp, "inbox-job-handle"), + GetValue(sp, "inbox-completed-job-id"), + GetValue(sp, "inbox-job-sequence"), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(pumpResultsType), + sp.GetRequiredService(), + sp.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs), + sp.GetRequiredService>().Value)); + + services.TryAddKeyedScoped( + typeof(IInboxExtension), + (sp, _) => (IGrainExtension)sp.GetRequiredService(extensionType)); + services.TryAddScoped(inboxType, sp => + { + var options = sp.GetRequiredService>().Value; + _ = GetDictionary(sp, "inbox-processed"); + _ = GetInternalDictionary(sp, "InboxMessageState", "inbox-message-state"); + _ = GetInternalDictionary(sp, "InboxDeadLetter", "inbox-dead-letters"); + _ = GetValue(sp, "inbox-job-id"); + _ = GetValue(sp, "inbox-job-handle"); + _ = GetValue(sp, "inbox-completed-job-id"); + _ = GetValue(sp, "inbox-job-sequence"); + return CreateInstance(inboxType, + GetDictionary(sp, "inbox"), + sp.GetServices(), options.MaxCapacity); + }); + services.TryAddScoped(sp => (IInbox)sp.GetRequiredService(inboxType)); + + services.AddScoped(static sp => + new JournaledTestOutbox( + sp.GetRequiredKeyedService>("test-handler-output"), + sp.GetRequiredService())); + services.TryAddScoped(typeof(IMessagingDiagnostics), GetImplementationType("MessagingDiagnostics")); + services.TryAddScoped(pumpResultsType, sp => + { + var options = sp.GetRequiredService>().Value; + var completedRetentionPeriod = TimeSpan.FromMinutes(10); + var abandonedRetentionPeriod = options.JobStatusPollInterval <= TimeSpan.MaxValue / 4 + ? options.JobStatusPollInterval * 4 + : TimeSpan.MaxValue; + return CreateInstance(pumpResultsType, + sp.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs), + completedRetentionPeriod, + TimeSpan.FromTicks(Math.Max(completedRetentionPeriod.Ticks, abandonedRetentionPeriod.Ticks)), + 65_536); + }); + services.TryAddEnumerable(ServiceDescriptor.Singleton(typeof(IConfigureGrainTypeComponents), configuratorType)); + } + + private static IDurableDictionary GetDictionary(IServiceProvider services, string stateName) + where TKey : notnull => + services.GetRequiredKeyedService>($"__orleans.messaging.{stateName}"); + + private static object GetInternalDictionary(IServiceProvider services, string valueType, string stateName) => + services.GetRequiredKeyedService(typeof(IDurableDictionary<,>).MakeGenericType(typeof(TKey), GetImplementationType(valueType)), + $"__orleans.messaging.{stateName}"); + + private static IDurableValue GetValue(IServiceProvider services, string stateName) => + services.GetRequiredKeyedService>($"__orleans.messaging.{stateName}"); + + private static object CreateInstance(Type type, params object?[] arguments) => + Activator.CreateInstance(type, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DoNotWrapExceptions, + binder: null, arguments, culture: null)!; +} diff --git a/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs new file mode 100644 index 00000000000..81be9cbe4f3 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs @@ -0,0 +1,98 @@ +using System.Collections.Concurrent; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class SnapshotProbe +{ + private readonly ConcurrentDictionary _latest = new(); + private readonly ConcurrentDictionary> _waiters = new(); + + internal int WaiterListCount => _waiters.Count; + + public async Task WaitAsync( + GrainId grainId, + Func predicate, + TimeSpan? timeout = null) + { + if (_latest.TryGetValue(grainId, out var current) && predicate(current)) + { + return current; + } + + var waiter = new Waiter(predicate); + List waiters; + while (true) + { + waiters = _waiters.GetOrAdd(grainId, static _ => []); + lock (waiters) + { + if (!_waiters.TryGetValue(grainId, out var currentWaiters) + || !ReferenceEquals(waiters, currentWaiters)) + { + continue; + } + + if (_latest.TryGetValue(grainId, out current) && predicate(current)) + { + RemoveWaiterListIfEmpty(grainId, waiters); + return current; + } + + waiters.Add(waiter); + break; + } + } + + try + { + return await waiter.Completion.Task.WaitAsync(timeout ?? TimeSpan.FromSeconds(30)); + } + finally + { + lock (waiters) + { + waiters.Remove(waiter); + RemoveWaiterListIfEmpty(grainId, waiters); + } + } + } + + public void Publish(GrainId grainId, EndpointSnapshot snapshot) + { + _latest[grainId] = snapshot; + if (!_waiters.TryGetValue(grainId, out var waiters)) + { + return; + } + + lock (waiters) + { + foreach (var waiter in waiters.ToArray()) + { + if (waiter.Predicate(snapshot)) + { + waiters.Remove(waiter); + waiter.Completion.TrySetResult(snapshot); + } + } + + RemoveWaiterListIfEmpty(grainId, waiters); + } + } + + private void RemoveWaiterListIfEmpty(GrainId grainId, List waiters) + { + if (waiters.Count == 0) + { + _waiters.TryRemove(new KeyValuePair>(grainId, waiters)); + } + } + + private sealed class Waiter(Func predicate) + { + public Func Predicate { get; } = predicate; + public TaskCompletionSource Completion { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs b/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs new file mode 100644 index 00000000000..03c412d3113 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs @@ -0,0 +1,86 @@ +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class SnapshotProbeTests +{ + [Fact] + public async Task WaitAsync_TimeoutRemovesWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "timeout"); + var predicateCalls = 0; + + await Assert.ThrowsAsync( + () => probe.WaitAsync( + grainId, + _ => + { + predicateCalls++; + return false; + }, + TimeSpan.Zero)); + + Assert.Equal(0, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 0)); + + Assert.Equal(0, predicateCalls); + } + + [Fact] + public async Task WaitAsync_SuccessRemovesWaiterList() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "success"); + var wait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + var snapshot = CreateSnapshot(inboxCount: 1); + + probe.Publish(grainId, snapshot); + + Assert.Equal(0, probe.WaiterListCount); + Assert.Same(snapshot, await wait); + } + + [Fact] + public async Task WaitAsync_TimeoutRetainsListWithActiveWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "shared-list"); + var activeWait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + + await Assert.ThrowsAsync( + () => probe.WaitAsync(grainId, static _ => false, TimeSpan.Zero)); + + Assert.Equal(1, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 1)); + + await activeWait; + Assert.Equal(0, probe.WaiterListCount); + } + + private static EndpointSnapshot CreateSnapshot(int inboxCount) => + new( + Guid.Empty, + string.Empty, + inboxCount, + 0, + 0, + [], + [], + [], + null, + 0, + null, + null, + null); +} diff --git a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs new file mode 100644 index 00000000000..6d7a98a15a4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs @@ -0,0 +1,101 @@ +using System.Buffers; +using System.Text; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging.Tests.Support; + +// Routes and typed values belong to this test application, not to the messaging transport. +[GenerateSerializer] +public sealed record TestApplicationMessage( + [property: Id(0)] string Route, + [property: Id(1)] object? Body); + +internal static class TestApplicationProtocol +{ + public static HierarchicalKey NewMessageId() => HierarchicalKey.Create("tests", Guid.NewGuid().ToString("N")); + + public static OutboxMessage Create(SerializerSessionPool sessions, GrainId sender, GrainId receiver, + string route, object? body, HierarchicalKey messageId = default) => + new() + { + ReceiverId = receiver, + Envelope = CreateInbox(sessions, sender, route, body, messageId).Envelope + }; + + public static InboxMessage CreateInbox(SerializerSessionPool sessions, GrainId sender, + string route, object? body, HierarchicalKey messageId = default) + { + var key = messageId.IsDefault + ? body is TestMessage message ? message.LogicalId : NewMessageId() + : messageId; + using var builder = new EnvelopeBuilder(key); + builder.AddBytes(MessageHeaders.Payload, Encode(sessions, new TestApplicationMessage(route, body))); + builder.AddBytes(MessageHeaders.Subject, Encoding.UTF8.GetBytes(route)); + builder.AddValue(MessageHeaders.Sender, sender, new Serializer(sessions)); + return new InboxMessage { Envelope = builder.Build() }; + } + + public static byte[] Encode(SerializerSessionPool sessions, T value) + { + var output = new ArrayBufferWriter(); + using var session = sessions.GetSession(); + var writer = Writer.Create(output, session); + sessions.CodecProvider.GetCodec().WriteField(ref writer, 0, typeof(T), value); + writer.Commit(); + return output.WrittenSpan.ToArray(); + } + + public static T Decode(SerializerSessionPool sessions, ReadOnlyMemory payload) + { + using var session = sessions.GetSession(); + var reader = Reader.Create(payload.Span, session); + var field = reader.ReadFieldHeader(); + return sessions.CodecProvider.GetCodec().ReadValue(ref reader, field)!; + } + + public static TestApplicationMessage Read(SerializerSessionPool sessions, InboxMessage message) => + Decode(sessions, message.Envelope.Payload); + + public static TestApplicationMessage Read(SerializerSessionPool sessions, OutboxMessage message) => + Decode(sessions, message.Envelope.Payload); + + public static GrainId Sender(SerializerSessionPool sessions, Envelope envelope) => + envelope.TryGetValue(MessageHeaders.Sender, new Serializer(sessions), out var sender) + ? sender : default; + + public static string? Subject(Envelope envelope) => + envelope.TryGetSubject(out var subject) ? subject : null; + + public static InboxMessage With(InboxMessage message, SerializerSessionPool sessions, + HierarchicalKey? messageId = null, GrainId? sender = null, string? subject = null, + ReadOnlyMemory? payload = null, bool removeSender = false, bool removeSubject = false) + { + using var builder = new EnvelopeBuilder(messageId ?? message.Envelope.MessageId); + foreach (var key in message.Envelope.Keys) + { + if (key == MessageHeaders.Sender && (sender.HasValue || removeSender) + || key == MessageHeaders.Subject && (subject is not null || removeSubject) + || key == MessageHeaders.Payload && payload.HasValue) + { + continue; + } + if (!message.Envelope.TryGetBytes(key, out var bytes)) + { + throw new InvalidOperationException($"Missing enumerated header '{key}'."); + } + builder.AddBytes(key, bytes.Span); + } + if (sender is { } value) + { + builder.AddValue(MessageHeaders.Sender, value, new Serializer(sessions)); + } + if (subject is not null) builder.AddBytes(MessageHeaders.Subject, Encoding.UTF8.GetBytes(subject)); + if (payload is { } body) builder.AddBytes(MessageHeaders.Payload, body.Span); + return new() { Envelope = builder.Build() }; + } +}