From 8d98f37ec58b388fc4e8d6bb2c3ade0a99c201f5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 08:43:15 -0700 Subject: [PATCH 01/31] feat(journaling): add operation prerequisite and completion hooks --- .../grains/journaling/runtime-behavior.md | 35 + .../IJournaledStateCaptureHook.cs | 15 + src/Orleans.Journaling/IJournaledStateHook.cs | 56 ++ .../IJournaledStateManager.cs | 23 +- src/Orleans.Journaling/JournaledStateHook.cs | 37 + .../JournaledStateManager.cs | 295 +++++++- .../JournaledStatePostCommitException.cs | 27 + .../JournaledStatePreCommitException.cs | 27 + src/Orleans.Journaling/README.md | 33 + .../Orleans.Journaling/Orleans.Journaling.cs | 108 +++ .../StateManagerHookTests.cs | 697 ++++++++++++++++++ 11 files changed, 1344 insertions(+), 9 deletions(-) create mode 100644 src/Orleans.Journaling/IJournaledStateCaptureHook.cs create mode 100644 src/Orleans.Journaling/IJournaledStateHook.cs create mode 100644 src/Orleans.Journaling/JournaledStateHook.cs create mode 100644 src/Orleans.Journaling/JournaledStatePostCommitException.cs create mode 100644 src/Orleans.Journaling/JournaledStatePreCommitException.cs create mode 100644 test/Orleans.Journaling.Tests/StateManagerHookTests.cs diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index e513f837fc9..eff4032fb4d 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -68,6 +68,41 @@ Orleans executes that synchronous block on a single activation thread. Another g the operation awaits, so keep shared state safe to commit at each await. Any caller's write can include staged mutations from other calls. +## Journal operation hooks + +Activation-scoped features coordinate prerequisites and completion through +. The owner exposes a stable, lazily +allocated list of registrations. Inspect and +deduplicate feature registrations on the owner's logical execution context while persistence +is quiescent. Registration survives recovery and deletion; the standard manager rejects hook-list +mutation throughout each operation. + +Each actual append, snapshot, or deletion runs ordinary before callbacks in list order, outside +the manager lock. At most one supplies the +final prerequisite. Its before callback runs last, and the work loop awaits it directly before +synchronous capture or storage deletion. Prerequisites cover changes staged during asynchronous +preparation, including the final hook's own I/O wait. Preserve operation-local bookkeeping for +the captured batch separately from changes staged later. + +Storage acknowledgement and registered-state acknowledgement or reset precede after callbacks. +All after callbacks run in list order, including for successful zero-byte writes. Coalesced callers +share callbacks for the actual operation. adapts delegates, +executing the synchronous delegate before the asynchronous delegate in each phase. + +A failed prerequisite reports with pending +state retained for an explicit persistence retry after the prerequisite is restored. A failed after +callback reports with persistence completed. +Remaining after callbacks run, multiple failures are aggregated, and the manager stays usable. +The feature's durable recovery protocol resumes interrupted post-persistence work. Storage and +state-processing failures retain the manager's fencing and fresh-recovery behavior. + +Hook callbacks receive the owner's shutdown token. Cancelling a caller's wait leaves the owned +operation running through its actual outcome. Disposal drains owned hooks and storage before releasing +journal resources, including when cancellation callbacks or cleanup fail. Concurrent disposal callers +share this completion. Recursive initialization, persistence, or disposal on the same owner from a +hook is rejected. For deletion, the feature owner stops admission and drains feature operations before +queuing the whole-journal reset. + ## Consistency and competing writers Orleans grain placement normally supplies a single active writer for a grain identity. Journal storage providers also use optimistic concurrency to protect the journal when a stale or competing writer reaches storage. diff --git a/src/Orleans.Journaling/IJournaledStateCaptureHook.cs b/src/Orleans.Journaling/IJournaledStateCaptureHook.cs new file mode 100644 index 00000000000..cd6d0b42693 --- /dev/null +++ b/src/Orleans.Journaling/IJournaledStateCaptureHook.cs @@ -0,0 +1,15 @@ +namespace Orleans.Journaling; + +/// +/// Establishes the final prerequisite immediately before journal capture or deletion. +/// +/// +/// An owner admits at most one capture hook in . +/// Its before callback runs after all ordinary before callbacks. The work loop awaits it directly, +/// then captures state or starts deletion without a further asynchronous phase. +/// Its prerequisites cover changes staged while its own I/O awaited. After callbacks retain +/// normal list order. Features inspect and deduplicate this registration using the same hook list. +/// +public interface IJournaledStateCaptureHook : IJournaledStateHook +{ +} diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs new file mode 100644 index 00000000000..d0ffaae24f3 --- /dev/null +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -0,0 +1,56 @@ +namespace Orleans.Journaling; + +/// +/// Identifies a journal persistence operation. +/// +public enum JournaledStateOperation +{ + /// Appends pending journal entries. + Write, + + /// Replaces storage with a snapshot. + Snapshot, + + /// Deletes storage and resets registered states. + Delete +} + +/// +/// Participates in the prerequisites and completion of actual journal operations. +/// +/// +/// Ordinary before hooks run in list order on the owner's logical execution context, outside its lock. +/// An optional runs last immediately before capture or deletion. +/// Prerequisites must cover changes staged during asynchronous preparation. After hooks run after +/// storage acknowledgement and state acknowledgement or reset, including successful writes which +/// produce no storage bytes. Hooks retain operation-local data across these boundaries and keep later +/// pending changes separate. Recursive operations on the same journal owner are rejected. +/// +public interface IJournaledStateHook +{ + /// + /// Establishes prerequisites before capture or storage deletion. + /// + /// The operation about to execute. + /// The token for the owned operation's lifetime. + /// A completion representing the prerequisite work. + /// + /// Failure reports and retains pending changes + /// for an explicit retry. All staged changes remain safe to commit. Full deletion requires the + /// owner to stop admission and drain feature operations before queuing deletion. + /// + ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; + + /// + /// Completes post-persistence work after the operation succeeds. + /// + /// The completed operation. + /// The token for the owned operation's lifetime. + /// A completion representing post-persistence work. + /// + /// Every after hook is invoked even when an earlier after hook fails. Failures are surfaced as + /// and leave the successfully persisted owner usable. + /// Durable feature state supplies recovery for interrupted post-persistence work. + /// + ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; +} diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index b6363a1b4cc..4f2af34920e 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -9,9 +9,24 @@ namespace Orleans.Journaling; /// The owner registers state machines and initializes the journal before using recovered state. /// State machine instances and their dependencies retain the lifetime assigned by their caller. /// Disposing this manager stops journal processing and releases its journal resources. +/// Disposal drains owned storage and hook operations before releasing resources. /// public interface IJournaledStateManager : IAsyncDisposable { + /// + /// Gets the mutable, lazily allocated list of journal operation hooks. + /// + /// + /// Inspect, add, remove, and deduplicate hooks on the owner's logical execution context while + /// persistence is quiescent. Mutation during an operation is rejected. Ordinary before hooks + /// and all after hooks execute in list order. The optional single + /// supplies the final prerequisite. + /// Registration is independent of state-machine registration and persists through recovery and deletion. + /// Custom owners implement this property to support hooks. + /// + /// The custom owner does not support hooks. + IList Hooks => throw new NotSupportedException("This journal owner does not support operation hooks."); + /// ValueTask IAsyncDisposable.DisposeAsync() => default; @@ -51,8 +66,11 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// Stage mutations only after establishing that they are safe to commit. Pending changes are shared /// by all callers using this manager. Storage acknowledgement establishes durability. - /// A failed journal operation fences the manager; recovery requires a new manager and state machine instances. + /// A storage or state-processing failure fences the manager; recovery requires a new manager and state machine instances. /// Cancellation stops the caller's wait; an already queued write continues to its storage outcome. + /// Before-hook failure reports and retains pending + /// changes for an explicit retry. After-hook failure reports + /// after successful persistence. /// /// The token used to cancel the caller's wait. /// A task representing the write acknowledgement. @@ -65,6 +83,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// The caller keeps other operations quiescent through completion: deletion resets every registered state machine. /// Cancellation ends the caller's wait; an already queued deletion continues to its storage and reset outcome. /// A failed deletion permanently fences the manager and requests deactivation of its owning grain. + /// Before-hook failure reports with state retained. + /// After-hook failure reports after storage deletion + /// and state reset succeed. /// /// The cancellation token. /// A which represents the operation. diff --git a/src/Orleans.Journaling/JournaledStateHook.cs b/src/Orleans.Journaling/JournaledStateHook.cs new file mode 100644 index 00000000000..31e12d601f8 --- /dev/null +++ b/src/Orleans.Journaling/JournaledStateHook.cs @@ -0,0 +1,37 @@ +namespace Orleans.Journaling; + +/// +/// Adapts synchronous and asynchronous delegates to journal operation hooks. +/// +/// +/// When both delegates for a phase are supplied, the synchronous delegate runs first. +/// Asynchronous delegates await their I/O outcome before completing their returned value task. +/// Custom implementations can carry feature identity and state +/// for inspection and deduplication in the owner's hook list. +/// +public sealed class JournaledStateHook : IJournaledStateHook +{ + /// Gets the synchronous prerequisite callback. + public Action? BeforeOperation { get; init; } + + /// Gets the asynchronous prerequisite callback. + public Func? BeforeOperationAsync { get; init; } + + /// Gets the synchronous completion callback. + public Action? AfterOperation { get; init; } + + /// Gets the asynchronous completion callback. + public Func? AfterOperationAsync { get; init; } + + ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + BeforeOperation?.Invoke(operation, cancellationToken); + return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + + ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + AfterOperation?.Invoke(operation, cancellationToken); + return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } +} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index cbf6be752bc..4a852b65396 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -1,6 +1,8 @@ using System.Buffers; +using System.Collections.ObjectModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; using Microsoft.Extensions.Logging; using Orleans.Diagnostics; using Orleans.Serialization.Buffers; @@ -11,6 +13,7 @@ namespace Orleans.Journaling; internal partial class JournaledStateManager : IJournaledStateManager, IJournalStorageConsumer, ILifecycleParticipant, ILifecycleObserver, IDisposable { private const uint MinApplicationJournalStreamId = 8u; + private static readonly AsyncLocal CurrentHookOwner = new(); #if NET9_0_OR_GREATER private readonly Lock _lock = new(); #else @@ -33,6 +36,20 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private Exception? _failure; private bool _migrationSnapshotRequired; private int _disposed; + private Task? _disposeTask; + private HookCollection? _hooks; + private bool _hookOperationRunning; + + public IList Hooks + { + get + { + lock (_lock) + { + return _hooks ??= new(this); + } + } + } public JournaledStateManager(JournaledStateManagerShared shared, IJournalStorageProvider storageProvider, IGrainContext grainContext) : this(shared, CreateStorage(storageProvider, CreateJournalId(grainContext)), grainContext) @@ -155,6 +172,7 @@ public void RegisterStateMachine(string name, IStateMachine stateMachine) public async ValueTask InitializeAsync(CancellationToken cancellationToken = default) { + ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; @@ -252,6 +270,7 @@ private async Task WorkLoop() } workItem = dequeuedWorkItem; + _hookOperationRunning = workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem; } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -273,6 +292,9 @@ private async Task WorkLoop() storageActivity.SetTag(ActivityTagKeys.JournalStorageOperation, queueOperation); } + JournaledStateOperation? hookOperation = null; + var beforeHookRunning = false; + var afterHookRunning = false; try { if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem @@ -282,6 +304,42 @@ private async Task WorkLoop() "The journaled state operation was queued before deletion reset its state."); } + hookOperation = workItem switch + { + AppendJournalWorkItem or WriteSnapshotWorkItem => + workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCompactionRequested + ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write, + DeleteStateWorkItem => JournaledStateOperation.Delete, + _ => null + }; + if (hookOperation is { } operation && _hooks is { Count: > 0 }) + { + beforeHookRunning = true; + var captureHook = GetCaptureHook(); + if (captureHook is null || _hooks.Count > 1) + { + await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + } + + if (captureHook is not null) + { + // Await the final prerequisite in this frame so capture follows its completion directly. + var previous = CurrentHookOwner.Value; + CurrentHookOwner.Value = this; + try + { + await captureHook.BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + } + finally + { + CurrentHookOwner.Value = previous; + } + } + + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + beforeHookRunning = false; + } + // Note that the implementation of each command is inlined to avoid allocating unnecessary async states. // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) @@ -291,9 +349,7 @@ private async Task WorkLoop() { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. - var isSnapshot = workItem is WriteSnapshotWorkItem - || _migrationSnapshotRequired - || _storage.IsCompactionRequested; + var isSnapshot = hookOperation == JournaledStateOperation.Snapshot; var operationLabel = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; @@ -540,13 +596,47 @@ private async Task WorkLoop() } } + if (hookOperation is { } completedOperation && _hooks is { Count: > 0 }) + { + afterHookRunning = true; + await InvokeAfterHooksAsync(completedOperation, _shutdownCancellation.Token).ConfigureAwait(true); + afterHookRunning = false; + } + if (recordQueueDuration && queueOperation is not null) { _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: true); } storageActivity?.SetStatus(ActivityStatusCode.Ok); - workItem.SetResult(); + lock (_lock) + { + _hookOperationRunning = false; + workItem.SetResult(); + } + } + catch (Exception exception) when (beforeHookRunning || afterHookRunning) + { + if (beforeHookRunning && exception is OperationCanceledException && _shutdownCancellation.IsCancellationRequested) + { + workItem.TrySetCanceled(_shutdownCancellation.Token); + continue; + } + + LogOperationHookFailed(_shared.Logger, exception, hookOperation!.Value, afterHookRunning); + storageActivity?.SetStatus(ActivityStatusCode.Error, "Journal operation hook failed."); + if (recordQueueDuration && queueOperation is not null) + { + _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: afterHookRunning); + } + + lock (_lock) + { + _hookOperationRunning = false; + workItem.SetException(afterHookRunning + ? new JournaledStatePostCommitException(hookOperation.Value, exception) + : new JournaledStatePreCommitException(hookOperation.Value, exception)); + } } catch (Exception exception) { @@ -580,6 +670,11 @@ private async Task WorkLoop() } finally { + lock (_lock) + { + _hookOperationRunning = false; + } + storageActivity?.Dispose(); } } @@ -596,6 +691,154 @@ private async Task WorkLoop() } } + private IJournaledStateCaptureHook? GetCaptureHook() + { + foreach (var hook in _hooks!) + { + if (hook is IJournaledStateCaptureHook captureHook) + { + return captureHook; + } + } + + return null; + } + + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + var previous = CurrentHookOwner.Value; + CurrentHookOwner.Value = this; + try + { + foreach (var hook in _hooks!) + { + if (hook is IJournaledStateCaptureHook) + { + continue; + } + + cancellationToken.ThrowIfCancellationRequested(); + await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + + cancellationToken.ThrowIfCancellationRequested(); + } + finally + { + CurrentHookOwner.Value = previous; + } + } + + private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + var previous = CurrentHookOwner.Value; + CurrentHookOwner.Value = this; + List? failures = null; + try + { + foreach (var hook in _hooks!) + { + try + { + await hook.AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + catch (Exception exception) + { + (failures ??= []).Add(exception); + } + } + } + finally + { + CurrentHookOwner.Value = previous; + } + + if (failures is { Count: 1 }) + { + ExceptionDispatchInfo.Capture(failures[0]).Throw(); + } + + if (failures is not null) + { + throw new AggregateException(failures); + } + } + + private void ThrowIfHookReentry() + { + if (ReferenceEquals(CurrentHookOwner.Value, this)) + { + throw new InvalidOperationException("Journal operation hooks cannot enqueue another operation on the same journal owner."); + } + } + + private sealed class HookCollection(JournaledStateManager owner) : Collection + { + protected override void InsertItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + EnsureMutationAllowed(); + ValidateHook(item, replacingIndex: -1); + base.InsertItem(index, item); + } + } + + protected override void SetItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + EnsureMutationAllowed(); + ValidateHook(item, index); + base.SetItem(index, item); + } + } + + protected override void RemoveItem(int index) + { + lock (owner._lock) + { + EnsureMutationAllowed(); + base.RemoveItem(index); + } + } + + protected override void ClearItems() + { + lock (owner._lock) + { + EnsureMutationAllowed(); + base.ClearItems(); + } + } + + private void EnsureMutationAllowed() + { + ObjectDisposedException.ThrowIf(owner._disposed != 0, owner); + owner._shutdownCancellation.Token.ThrowIfCancellationRequested(); + owner.ThrowIfFenced(); + if (owner._hookOperationRunning) + { + throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); + } + } + + private void ValidateHook(IJournaledStateHook item, int replacingIndex) + { + ArgumentNullException.ThrowIfNull(item); + if (item is IJournaledStateCaptureHook) + { + for (var i = 0; i < Count; i++) + { + if (i != replacingIndex && this[i] is IJournaledStateCaptureHook) + { + throw new InvalidOperationException("A journal owner supports one final capture prerequisite hook. Inspect and deduplicate the hook list before registration."); + } + } + } + } + } + private void Fence(Exception exception) { lock (_lock) @@ -699,6 +942,7 @@ private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWrit public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = default) { + ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; @@ -886,6 +1130,7 @@ private InvalidOperationException CreateRecoveryFormatException(Exception except public async ValueTask WriteStateAsync(CancellationToken cancellationToken = default) { + ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task pendingWrite; @@ -1062,9 +1307,23 @@ void ILifecycleParticipant.Participate(IGrainLifecycle observer private async Task StopAsync(CancellationToken cancellationToken) { + ThrowIfHookReentry(); + AggregateException? cancellationFailure = null; lock (_lock) { - _shutdownCancellation.Cancel(); + try + { + _shutdownCancellation.Cancel(); + } + catch (AggregateException exception) + { + cancellationFailure = exception; + } + } + + if (cancellationFailure is not null) + { + LogShutdownCancellationFailed(_shared.Logger, cancellationFailure); } _workSignal.Signal(); @@ -1079,6 +1338,11 @@ private async Task StopAsync(CancellationToken cancellationToken) { CancelQueuedWorkItems(_shutdownCancellation.Token); } + + if (cancellationFailure is not null) + { + ExceptionDispatchInfo.Capture(cancellationFailure).Throw(); + } } private void CancelQueuedWorkItems(CancellationToken cancellationToken) @@ -1097,13 +1361,18 @@ void IDisposable.Dispose() DisposeAsync().AsTask().GetAwaiter().GetResult(); } - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - if (Interlocked.Exchange(ref _disposed, 1) != 0) + ThrowIfHookReentry(); + lock (_lock) { - return; + return new(_disposeTask ??= DisposeCoreAsync()); } + } + private async Task DisposeCoreAsync() + { + _disposed = 1; try { await StopAsync(CancellationToken.None).ConfigureAwait(false); @@ -1352,6 +1621,16 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal {Operation} hook failed. Persistence completed: {Committed}.")] + private static partial void LogOperationHookFailed(ILogger logger, Exception exception, JournaledStateOperation operation, bool committed); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal shutdown cancellation callback failed; owned operations are drained before resources are released.")] + private static partial void LogShutdownCancellationFailed(ILogger logger, Exception exception); + [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/Orleans.Journaling/JournaledStatePostCommitException.cs b/src/Orleans.Journaling/JournaledStatePostCommitException.cs new file mode 100644 index 00000000000..c10a39e92b6 --- /dev/null +++ b/src/Orleans.Journaling/JournaledStatePostCommitException.cs @@ -0,0 +1,27 @@ +namespace Orleans.Journaling; + +/// +/// Reports a hook failure after the journal operation and its state acknowledgement or reset succeeded. +/// +/// +/// The journal owner remains usable. The caller handles the failed post-persistence work using +/// the feature's durable recovery protocol, preserving the completed business operation. +/// +[GenerateSerializer] +public sealed class JournaledStatePostCommitException : Exception +{ + /// + /// Initializes a post-persistence hook failure. + /// + /// The successfully completed journal operation. + /// The failure or aggregate of failures from after hooks. + public JournaledStatePostCommitException(JournaledStateOperation operation, Exception innerException) + : base($"Journal operation '{operation}' completed, but a post-persistence hook failed.", innerException) + { + Operation = operation; + } + + /// Gets the successfully completed journal operation. + [Id(0)] + public JournaledStateOperation Operation { get; } +} diff --git a/src/Orleans.Journaling/JournaledStatePreCommitException.cs b/src/Orleans.Journaling/JournaledStatePreCommitException.cs new file mode 100644 index 00000000000..1fba60d1d3e --- /dev/null +++ b/src/Orleans.Journaling/JournaledStatePreCommitException.cs @@ -0,0 +1,27 @@ +namespace Orleans.Journaling; + +/// +/// Reports a prerequisite hook failure which prevented the journal storage operation. +/// +/// +/// Pending changes remain staged and safe to commit. The caller can restore the prerequisite +/// and explicitly retry persistence, or retire its owner and recover from durable state. +/// +[GenerateSerializer] +public sealed class JournaledStatePreCommitException : Exception +{ + /// + /// Initializes a prerequisite hook failure. + /// + /// The journal operation prevented by the failed prerequisite. + /// The original prerequisite failure. + public JournaledStatePreCommitException(JournaledStateOperation operation, Exception innerException) + : base($"Journal operation '{operation}' was prevented by a prerequisite hook failure.", innerException) + { + Operation = operation; + } + + /// Gets the journal operation which was prevented. + [Id(0)] + public JournaledStateOperation Operation { get; } +} diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 2f5046a30be..7c4f4ed865a 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -264,6 +264,39 @@ the operation's token before use, or deliberately supply them through a registra lifecycle ownership. Creation through the explicit-journal factory keeps failure handling independent of the ambient grain context, including when the caller subsequently enrolls the manager in a lifecycle. +## Journal operation hooks + +`IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` +registrations. Features inspect and deduplicate their registrations on the owner's logical +execution context while persistence is quiescent. Registration persists through recovery and +whole-journal deletion. The standard manager rejects mutation of the list throughout an +operation and admits at most one `IJournaledStateCaptureHook`. + +For each actual write, snapshot, or deletion, ordinary before hooks run in list order outside +the manager lock. The capture hook runs last: the work loop awaits it directly, then synchronously +captures the registered states or starts deletion. Prerequisites cover all changes staged during +asynchronous preparation, including changes arriving while the capture hook awaits its own I/O. +Keep operation-local bookkeeping for the captured batch separate from later pending changes. + +After hooks run in list order after storage acknowledgement and state acknowledgement or reset. +They also run for a successful zero-byte write. Coalesced callers share the hooks for their actual +operation. `JournaledStateHook` adapts synchronous and asynchronous delegates; within each phase +the synchronous delegate executes first. + +| Outcome | Owner and caller behavior | +| --- | --- | +| Before hook fails | `JournaledStatePreCommitException` retains pending state for an explicit retry after restoring the prerequisite. | +| Storage or state processing fails | The original failure fences the manager; create a fresh owner and recover the durable outcome. | +| After hook fails | `JournaledStatePostCommitException` reports successful persistence. Every remaining after hook runs, failures are aggregated, and the manager stays usable. The feature's durable recovery protocol resumes interrupted post-persistence work. | + +Hooks receive the owner's shutdown token. Caller cancellation ends the caller's wait while +the owned prerequisite, capture, storage, and completion phases continue. Disposal cancels the +owner token and drains owned work before releasing journal resources, including when cancellation +callbacks or after-hook cleanup fail. Concurrent disposal callers share that drain and its outcome. +Recursive initialization, persistence, or disposal on the same owner from a hook is rejected. +Before whole-journal deletion, the feature owner stops admission and drains its own operations; +deletion completion follows storage deletion and registered-state reset. + ## State identity and retirement Preserve state names across activations and deployments. A stream absent from the setup declarations diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 396a1692cad..84a6099c001 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -219,8 +219,20 @@ public partial interface IDurableValue T? Value { get; set; } } + public partial interface IJournaledStateCaptureHook : IJournaledStateHook + { + } + + public partial interface IJournaledStateHook + { + System.Threading.Tasks.ValueTask AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.ValueTask BeforeOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); + } + public partial interface IJournaledStateManager : System.IAsyncDisposable { + System.Collections.Generic.IList Hooks { get; } + long PendingWriteByteCount { get; } System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken = default); @@ -416,6 +428,21 @@ public sealed partial class JournalCatalogListOptions public JournalId Prefix { get { throw null; } set { } } } + public sealed partial class JournaledStateHook : IJournaledStateHook + { + public System.Action? AfterOperation { get { throw null; } init { } } + + public System.Func? AfterOperationAsync { get { throw null; } init { } } + + public System.Action? BeforeOperation { get { throw null; } init { } } + + public System.Func? BeforeOperationAsync { get { throw null; } init { } } + + System.Threading.Tasks.ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken) { throw null; } + + System.Threading.Tasks.ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken) { throw null; } + } + public sealed partial class JournaledStateManagerOptions { public static readonly System.TimeSpan DEFAULT_RETIREMENT_GRACE_PERIOD; @@ -424,6 +451,31 @@ public sealed partial class JournaledStateManagerOptions public System.TimeSpan RetirementGracePeriod { get { throw null; } set { } } } + public enum JournaledStateOperation + { + Write = 0, + Snapshot = 1, + Delete = 2 + } + + [GenerateSerializer] + public sealed partial class JournaledStatePostCommitException : System.Exception + { + public JournaledStatePostCommitException(JournaledStateOperation operation, System.Exception innerException) { } + + [Id(0)] + public JournaledStateOperation Operation { get { throw null; } } + } + + [GenerateSerializer] + public sealed partial class JournaledStatePreCommitException : System.Exception + { + public JournaledStatePreCommitException(JournaledStateOperation operation, System.Exception innerException) { } + + [Id(0)] + public JournaledStateOperation Operation { get { throw null; } } + } + public readonly ref partial struct JournalEntry { private readonly object _dummy; @@ -800,10 +852,66 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. where TBufferWriter : System.Buffers.IBufferWriter { } } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_JournaledStatePostCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePostCommitException instance) { } + + public global::Orleans.Journaling.JournaledStatePostCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePostCommitException instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePostCommitException value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_JournaledStatePreCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePreCommitException instance) { } + + public global::Orleans.Journaling.JournaledStatePreCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePreCommitException instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePreCommitException value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] public sealed partial class Copier_DurableTaskCompletionSourceState : global::Orleans.Serialization.Cloning.ShallowCopier> { } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_JournaledStatePostCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier + { + public Copier_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } + + public override void DeepCopy(global::Orleans.Journaling.JournaledStatePostCommitException input, global::Orleans.Journaling.JournaledStatePostCommitException output, global::Orleans.Serialization.Cloning.CopyContext context) { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_JournaledStatePreCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier + { + public Copier_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } + + public override void DeepCopy(global::Orleans.Journaling.JournaledStatePreCommitException input, global::Orleans.Journaling.JournaledStatePreCommitException output, global::Orleans.Serialization.Cloning.CopyContext context) { } + } } \ No newline at end of file diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs new file mode 100644 index 00000000000..29e2d9d60e5 --- /dev/null +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -0,0 +1,697 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Journaling.Tests; + +public partial class StateManagerTests +{ + [Fact] + public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() + { + await using var manager = CreateTestSystem().Manager; + var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; + Assert.Null(field.GetValue(manager)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Null(field.GetValue(manager)); + + var hook = new JournaledStateHook(); + var hooks = manager.Hooks; + hooks.Add(hook); + Assert.True(hooks.Contains(hook)); + Assert.Same(hooks, manager.Hooks); + Assert.Same(hook, Assert.Single(hooks)); + Assert.True(hooks.Remove(hook)); + Assert.Empty(manager.Hooks); + Assert.Throws(() => hooks.Add(null!)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot, BlockNextAppend = !snapshot, BlockNextReplace = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; + CancellationToken ownedToken = default; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperation = (operation, token) => + { + Assert.Equal(expected, operation); + Assert.True(token.CanBeCanceled); + ownedToken = token; + events.Add("before-sync"); + state["hook"] = 2; + }, + BeforeOperationAsync = async (operation, token) => + { + Assert.Equal(expected, operation); + Assert.Equal(ownedToken, token); + await Task.Yield(); + events.Add("before-async"); + }, + AfterOperation = (operation, token) => + { + Assert.Equal(expected, operation); + Assert.Equal(ownedToken, token); + Assert.Single(snapshot ? storage.Replaces : storage.Appends); + events.Add("after-sync"); + }, + AfterOperationAsync = async (_, token) => + { + Assert.Equal(ownedToken, token); + await Task.Yield(); + events.Add("after-async"); + } + }); + state["business"] = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await (snapshot ? storage.ReplaceEntered : storage.BlockedAppendStarted).Task + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "before-sync", "before-async", "capture" }, events); + Assert.False(write.IsCompleted); + (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "before-sync", "before-async", "capture", "ack", "after-sync", "after-async" }, events); + } + finally + { + (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); + } + + await using var replayManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["hook"]); + } + + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledStateOperation operation) + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var capture = new AlwaysWritingState(); + manager.RegisterStateMachine("capture", capture); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var pendingBytes = manager.PendingWriteByteCount; + var failure = new IOException("Prerequisite failed."); + var after = 0; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperation = (_, _) => throw failure, + AfterOperation = (_, _) => after++ + }); + var caught = await Assert.ThrowsAsync(() => InvokeAsync()); + Assert.Same(failure, caught.InnerException); + Assert.Equal(operation, caught.Operation); + Assert.Equal(1, state["business"]); + Assert.Equal(pendingBytes, manager.PendingWriteByteCount); + Assert.Equal(0, capture.AppendEntriesCount); + Assert.Equal(0, capture.WriteCompletedCount); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, after); + manager.Hooks.Clear(); + await InvokeAsync(); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); + Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); + + Task InvokeAsync() => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) + : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask(); + } + + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHooks(JournaledStateOperation operation) + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var first = new IOException("First cleanup failed."); + var second = new IOException("Second cleanup failed."); + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => throw first }); + manager.Hooks.Add(new JournaledStateHook + { + AfterOperationAsync = async (_, _) => + { + await Task.Yield(); + events.Add("second"); + throw second; + } + }); + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => events.Add("last") }); + var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) + : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); + Assert.Equal(operation, caught.Operation); + Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); + Assert.Equal(operation == JournaledStateOperation.Delete + ? new[] { "second", "last" } + : new[] { "capture", "ack", "second", "last" }, events); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + manager.Hooks.Clear(); + + await using var replayManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, recovered.Count); + if (operation != JournaledStateOperation.Delete) + { + Assert.Equal(1, recovered["business"]); + } + + state["later"] = 2; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, state["later"]); + } + + [Fact] + public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() + { + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage { NextAppendException = failure }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var after = 0; + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => after++ }); + state["business"] = 1; + Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); + Assert.Equal(0, after); + var fenced = await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(failure, fenced.InnerException); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinaryHook(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + manager.Hooks.Add(new CaptureTestHook( + (_, _) => + { + Assert.Equal(3, state["late"]); + state["owner"] = 2; + events.Add("capture-prerequisite"); + return default; + }, + (_, _) => events.Add("capture-after"))); + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperationAsync = async (_, token) => + { + events.Add("ordinary-before"); + entered.TrySetResult(); + await release.Task.WaitAsync(token); + events.Add("ordinary-complete"); + }, + AfterOperation = (_, _) => events.Add("ordinary-after") + }); + state["business"] = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "ordinary-before" }, events); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + state["late"] = 3; + release.TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "ordinary-before", "ordinary-complete", "capture-prerequisite", "capture", "ack", "capture-after", "ordinary-after" }, events); + } + finally + { + release.TrySetResult(); + } + + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["owner"]); + Assert.Equal(3, recovered["late"]); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(bool captureHook) + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken ownedToken = default; + Func before = async (_, token) => + { + ownedToken = token; + entered.TrySetResult(); + await release.Task.WaitAsync(token); + state["scheduled"] = 2; + }; + Action after = (_, token) => + { + Assert.Equal(ownedToken, token); + Assert.False(token.IsCancellationRequested); + Assert.Single(storage.Appends); + completed.TrySetResult(); + }; + manager.Hooks.Add(captureHook + ? new CaptureTestHook(before, after) + : new JournaledStateHook { BeforeOperationAsync = before, AfterOperation = after }); + using var caller = new CancellationTokenSource(); + state["business"] = 1; + var write = manager.WriteStateAsync(caller.Token).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.NotEqual(caller.Token, ownedToken); + state["late"] = 3; + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => write); + Assert.False(ownedToken.IsCancellationRequested); + Assert.Empty(storage.Appends); + Assert.False(completed.Task.IsCompleted); + release.TrySetResult(); + await completed.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(2, state["scheduled"]); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["scheduled"]); + Assert.Equal(3, recovered["late"]); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_SameOwnerReentryIsRejectedWithoutDeadlock(bool after) + { + await using var manager = CreateTestSystem().Manager; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Func callback = async (_, token) => + { + await Assert.ThrowsAsync(() => manager.InitializeAsync(token).AsTask()); + await Assert.ThrowsAsync(() => manager.DeleteStateAsync(token).AsTask()); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + await manager.WriteStateAsync(token); + }; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperationAsync = after ? null : callback, + AfterOperationAsync = after ? callback : null + }); + var failure = await Record.ExceptionAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken) + .AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + var cause = after + ? Assert.IsType(failure).InnerException + : Assert.IsType(failure).InnerException; + Assert.Contains("same journal owner", Assert.IsType(cause).Message, StringComparison.Ordinal); + manager.Hooks.Clear(); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task Hooks_FinalPrerequisiteRejectsSameOwnerReentry() + { + await using var manager = CreateTestSystem().Manager; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + manager.Hooks.Add(new CaptureTestHook((_, token) => manager.DeleteStateAsync(token))); + var failure = await Assert.ThrowsAsync(() => + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Contains("same journal owner", Assert.IsType(failure.InnerException).Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Hooks_RegistrationEnforcesSingleCapturePrerequisite() + { + await using var manager = CreateTestSystem().Manager; + var first = new CaptureTestHook((_, _) => default); + var second = new CaptureTestHook((_, _) => default); + var ordinary = new JournaledStateHook(); + manager.Hooks.Add(first); + manager.Hooks.Add(ordinary); + Assert.Throws(() => manager.Hooks.Add(second)); + Assert.Throws(() => manager.Hooks[1] = second); + Assert.Equal(new IJournaledStateHook[] { first, ordinary }, manager.Hooks); + manager.Hooks[0] = second; + Assert.Same(second, manager.Hooks[0]); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var after = 0; + manager.Hooks.Add(new CaptureTestHook( + async (operation, token) => + { + Assert.Equal(JournaledStateOperation.Delete, operation); + Assert.Equal(1, state.Value); + entered.TrySetResult(); + await release.Task.WaitAsync(token); + }, + (operation, _) => + { + Assert.Equal(JournaledStateOperation.Delete, operation); + Assert.Equal(0, state.Value); + Assert.Equal(1, storage.DeleteCount); + after++; + })); + var deletion = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, after); + Assert.False(deletion.IsCompleted); + release.TrySetResult(); + await deletion.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, after); + Assert.Single(manager.Hooks); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_MutationIsRejectedThroughoutOwnedOperation(bool after) + { + var storage = new CapturingStorage { BlockNextAppend = !after }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hook = new JournaledStateHook + { + BeforeOperation = (_, _) => Assert.Throws(() => manager.Hooks.Clear()), + AfterOperationAsync = after ? async (_, _) => + { + entered.TrySetResult(); + await release.Task; + } : null + }; + manager.Hooks.Add(hook); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await (after ? entered : storage.BlockedAppendStarted).Task + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Throws(() => manager.Hooks.Add(new JournaledStateHook())); + Assert.Throws(() => manager.Hooks[0] = new JournaledStateHook()); + Assert.Throws(() => manager.Hooks.Remove(hook)); + Assert.Throws(() => manager.Hooks.Clear()); + Assert.Same(hook, Assert.Single(manager.Hooks)); + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + manager.Hooks.Clear(); + Assert.Empty(manager.Hooks); + } + finally + { + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var drained = false; + var failure = new IOException("Cancellation callback failed."); + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperationAsync = async (_, token) => + { + using var registration = token.Register(() => throw failure); + entered.TrySetResult(); + await release.Task; + drained = true; + } + }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + var concurrentShutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, concurrentShutdown); + Assert.False(shutdown.IsCompleted); + Assert.False(drained); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => + shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Same(failure, Assert.Single(caught.InnerExceptions)); + await Assert.ThrowsAnyAsync(() => write); + await Assert.ThrowsAnyAsync(() => queued); + Assert.True(drained); + Assert.Empty(storage.Appends); + Assert.Throws(() => manager.Hooks.Clear()); + } + finally + { + release.TrySetResult(); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + } + } + + [Fact] + public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var first = new IOException("First cleanup failed."); + var second = new IOException("Second cleanup failed."); + List events = []; + manager.Hooks.Add(new JournaledStateHook + { + AfterOperationAsync = async (_, token) => + { + Assert.Single(storage.Appends); + entered.TrySetResult(); + await release.Task; + Assert.True(token.IsCancellationRequested); + events.Add("first"); + throw first; + } + }); + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => throw second }); + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => events.Add("last") }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var shutdown = manager.DisposeAsync().AsTask(); + var concurrentShutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, concurrentShutdown); + Assert.False(shutdown.IsCompleted); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => + write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "first", "last" }, events); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Fact] + public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var before = 0; + var after = 0; + manager.Hooks.Add(new JournaledStateHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + state["first"] = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await storage.BlockedAppendStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(0, after); + state["second"] = 2; + var second = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + state["third"] = 3; + var third = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(2, before); + Assert.Equal(2, after); + Assert.Equal(2, storage.Appends.Count); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + var before = 0; + var after = 0; + manager.Hooks.Add(new JournaledStateHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + Assert.Single(storage.Appends); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Hooks_FailureOutcomesRoundTripWithOperationAndOriginalCause(bool after) + { + Exception original = after + ? new JournaledStatePostCommitException(JournaledStateOperation.Delete, new IOException("Cleanup failed.")) + : new JournaledStatePreCommitException(JournaledStateOperation.Snapshot, new IOException("Scheduling failed.")); + var serializer = ServiceProvider.GetRequiredService(); + var copy = serializer.Deserialize(serializer.SerializeToArray(original)); + if (after) + { + Assert.Equal(JournaledStateOperation.Delete, Assert.IsType(copy).Operation); + } + else + { + Assert.Equal(JournaledStateOperation.Snapshot, Assert.IsType(copy).Operation); + } + + Assert.Equal(original.InnerException!.Message, Assert.IsType(copy.InnerException).Message); + Assert.Equal(original.Message, copy.Message); + } + + [Fact] + public async Task Hooks_LegacyCustomOwnerKeepsExistingOperationsAndReportsUnsupportedHooks() + { + await using IJournaledStateManager manager = new LegacyHookTestOwner(); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Throws(() => manager.Hooks); + } + + private sealed class LegacyHookTestOwner : IJournaledStateManager + { + public ValueTask InitializeAsync(CancellationToken cancellationToken = default) => default; + public ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken = default) => default; + public void RegisterStateMachine(string name, IStateMachine stateMachine) { } + public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? stateMachine) + { + stateMachine = null; + return false; + } + } + + private sealed class CaptureTestHook( + Func before, + Action? after = null) : IJournaledStateCaptureHook + { + public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => + before(operation, cancellationToken); + + public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + after?.Invoke(operation, cancellationToken); + return default; + } + } + + private sealed class HookAcknowledgementState(List events) : IStateMachine + { + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) { } + public void Reset(JournalStreamWriter writer) { } + public void WritePendingEntries(JournalStreamWriter writer) => events.Add("capture"); + public void WriteSnapshot(JournalStreamWriter writer) => events.Add("capture"); + public void OnWriteCompleted() => events.Add("ack"); + } +} From 2c301564bff4df0ad7f877279b360f022b3b1ec2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 10:55:30 -0700 Subject: [PATCH 02/31] fix(journaling): close work admission during hook shutdown --- .../grains/journaling/runtime-behavior.md | 4 +- .../IJournaledStateManager.cs | 3 +- .../JournaledStateManager.cs | 16 +- src/Orleans.Journaling/README.md | 2 + .../StateManagerHookTests.cs | 146 ++++++++++++++++++ 5 files changed, 163 insertions(+), 8 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index eff4032fb4d..00d88df1c0e 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -100,7 +100,9 @@ Hook callbacks receive the owner's shutdown token. Cancelling a caller's wait le operation running through its actual outcome. Disposal drains owned hooks and storage before releasing journal resources, including when cancellation callbacks or cleanup fail. Concurrent disposal callers share this completion. Recursive initialization, persistence, or disposal on the same owner from a -hook is rejected. For deletion, the feature owner stops admission and drains feature operations before +hook is rejected. Shutdown closes work admission and cancels queued operations while the current +operation drains to its actual storage and hook outcome. For deletion, the feature owner stops +admission and drains feature operations before queuing the whole-journal reset. ## Consistency and competing writers diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 4f2af34920e..4b5598cdfce 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -8,7 +8,6 @@ namespace Orleans.Journaling; /// /// The owner registers state machines and initializes the journal before using recovered state. /// State machine instances and their dependencies retain the lifetime assigned by their caller. -/// Disposing this manager stops journal processing and releases its journal resources. /// Disposal drains owned storage and hook operations before releasing resources. /// public interface IJournaledStateManager : IAsyncDisposable @@ -82,7 +81,7 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// The caller keeps other operations quiescent through completion: deletion resets every registered state machine. /// Cancellation ends the caller's wait; an already queued deletion continues to its storage and reset outcome. - /// A failed deletion permanently fences the manager and requests deactivation of its owning grain. + /// A storage or state-reset failure permanently fences the manager and requests deactivation of its owning grain. /// Before-hook failure reports with state retained. /// After-hook failure reports after storage deletion /// and state reset succeed. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 4a852b65396..7cd2d160607 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -263,6 +263,11 @@ private async Task WorkLoop() WorkItem workItem; lock (_lock) { + if (_shutdownCancellation.IsCancellationRequested) + { + return; + } + if (!_workQueue.TryDequeue(out var dequeuedWorkItem)) { // Wait for the queue to be signaled again. @@ -693,9 +698,9 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo private IJournaledStateCaptureHook? GetCaptureHook() { - foreach (var hook in _hooks!) + for (var i = 0; i < _hooks!.Count; i++) { - if (hook is IJournaledStateCaptureHook captureHook) + if (_hooks[i] is IJournaledStateCaptureHook captureHook) { return captureHook; } @@ -710,8 +715,9 @@ private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation CurrentHookOwner.Value = this; try { - foreach (var hook in _hooks!) + for (var i = 0; i < _hooks!.Count; i++) { + var hook = _hooks[i]; if (hook is IJournaledStateCaptureHook) { continue; @@ -736,11 +742,11 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, List? failures = null; try { - foreach (var hook in _hooks!) + for (var i = 0; i < _hooks!.Count; i++) { try { - await hook.AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); } catch (Exception exception) { diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 7c4f4ed865a..ce02135b40c 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -293,6 +293,8 @@ Hooks receive the owner's shutdown token. Caller cancellation ends the caller's the owned prerequisite, capture, storage, and completion phases continue. Disposal cancels the owner token and drains owned work before releasing journal resources, including when cancellation callbacks or after-hook cleanup fail. Concurrent disposal callers share that drain and its outcome. +Shutdown closes work admission and cancels queued operations while an already running operation +drains to its actual storage and hook outcome. Recursive initialization, persistence, or disposal on the same owner from a hook is rejected. Before whole-journal deletion, the feature owner stops admission and drains its own operations; deletion completion follows storage deletion and registered-state reset. diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 29e2d9d60e5..607878d692a 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -1,5 +1,7 @@ +using System.Buffers; using System.Reflection; using Microsoft.Extensions.DependencyInjection; +using NSubstitute; using Orleans.Serialization; using Xunit; @@ -526,6 +528,150 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() } } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDrains(bool after) + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = NewSignal(); + var release = NewSignal(); + Func callback = async (_, _) => + { + entered.TrySetResult(); + await release.Task; + }; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperationAsync = after ? null : callback, + AfterOperationAsync = after ? callback : null + }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var initializing = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var deleting = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.False(shutdown.IsCompleted); + Assert.False(initializing.IsCompleted); + Assert.False(deleting.IsCompleted); + release.TrySetResult(); + await WaitFor(shutdown); + if (after) + { + await WaitFor(write); + } + else + { + await Assert.ThrowsAnyAsync(() => WaitFor(write)); + } + + await Assert.ThrowsAnyAsync(() => WaitFor(initializing)); + await Assert.ThrowsAnyAsync(() => WaitFor(deleting)); + Assert.True(initializing.IsCanceled); + Assert.True(deleting.IsCanceled); + Assert.Equal(after ? 1 : 0, state.CaptureCount); + Assert.Equal(after ? 1 : 0, state.WriteCompletedCount); + Assert.Equal(after ? 1 : 0, storage.Appends.Count); + Assert.Equal(0, storage.DeleteCount); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesItsOutcome(bool snapshot, bool storageFails) + { + var storage = Substitute.For(); + storage.IsCompactionRequested.Returns(snapshot); + storage.ReadAsync(Arg.Any(), Arg.Any()).Returns(call => + { + call.Arg().Complete(metadata: null); + return ValueTask.CompletedTask; + }); + storage.AppendAsync(Arg.Any>(), Arg.Any()) + .Returns(call => WriteAsync(call.Arg>(), call.Arg())); + storage.ReplaceAsync(Arg.Any>(), Arg.Any()) + .Returns(call => WriteAsync(call.Arg>(), call.Arg())); + var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = NewSignal(); + var release = NewSignal(); + var cancellationFailure = new IOException("Cancellation callback failed."); + var storageFailure = new IOException("Actual storage failure."); + CancellationTokenRegistration registration = default; + var completed = false; + var after = 0; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), + AfterOperation = (_, _) => after++ + }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, manager.DisposeAsync().AsTask()); + Assert.False(shutdown.IsCompleted); + Assert.False(completed); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Equal(0, after); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => WaitFor(shutdown)); + Assert.Same(cancellationFailure, Assert.Single(caught.InnerExceptions)); + Assert.True(completed); + if (storageFails) + { + Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(write))); + Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(queued))); + } + else + { + await WaitFor(write); + await Assert.ThrowsAnyAsync(() => WaitFor(queued)); + } + + Assert.Equal(storageFails ? 0 : 1, state.WriteCompletedCount); + Assert.Equal(storageFails ? 0 : 1, after); + } + finally + { + release.TrySetResult(); + registration.Dispose(); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + } + + async ValueTask WriteAsync(ReadOnlySequence bytes, CancellationToken token) + { + var original = bytes.ToArray(); + entered.TrySetResult(); + await release.Task; + Assert.True(token.IsCancellationRequested); + Assert.Equal(original, bytes.ToArray()); + completed = true; + if (storageFails) + { + throw storageFailure; + } + } + } + [Fact] public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() { From 8b5b3e9e59542ab61fbbb0e4dcb60c412d7a5e41 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 11:45:21 -0700 Subject: [PATCH 03/31] test(journaling): initialize storage callback barriers before registration --- test/Orleans.Journaling.Tests/StateManagerHookTests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 607878d692a..332a18c51cf 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -594,6 +594,11 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra [InlineData(true, true)] public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesItsOutcome(bool snapshot, bool storageFails) { + var entered = NewSignal(); + var release = NewSignal(); + var cancellationFailure = new IOException("Cancellation callback failed."); + var storageFailure = new IOException("Actual storage failure."); + var completed = false; var storage = Substitute.For(); storage.IsCompactionRequested.Returns(snapshot); storage.ReadAsync(Arg.Any(), Arg.Any()).Returns(call => @@ -609,12 +614,7 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt var state = new LifecycleState(); manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); - var entered = NewSignal(); - var release = NewSignal(); - var cancellationFailure = new IOException("Cancellation callback failed."); - var storageFailure = new IOException("Actual storage failure."); CancellationTokenRegistration registration = default; - var completed = false; var after = 0; manager.Hooks.Add(new JournaledStateHook { From 2f21229ab95e30e52e81b126faaed5dd1e918fe9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 17:07:56 -0700 Subject: [PATCH 04/31] refactor(journaling): trim operation hook API and preparation --- .../grains/journaling/runtime-behavior.md | 4 +- src/Orleans.Journaling/JournaledStateHook.cs | 37 ---------- .../JournaledStateManager.cs | 26 ++----- src/Orleans.Journaling/README.md | 4 +- .../Orleans.Journaling/Orleans.Journaling.cs | 15 ----- .../StateManagerHookTests.cs | 67 ++++++++++++------- 6 files changed, 54 insertions(+), 99 deletions(-) delete mode 100644 src/Orleans.Journaling/JournaledStateHook.cs diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 00d88df1c0e..68398471d80 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -86,8 +86,8 @@ the captured batch separately from changes staged later. Storage acknowledgement and registered-state acknowledgement or reset precede after callbacks. All after callbacks run in list order, including for successful zero-byte writes. Coalesced callers -share callbacks for the actual operation. adapts delegates, -executing the synchronous delegate before the asynchronous delegate in each phase. +share callbacks for the actual operation. Features implement the before and after callbacks on +their own hook, retaining feature identity and operation-local bookkeeping there. A failed prerequisite reports with pending state retained for an explicit persistence retry after the prerequisite is restored. A failed after diff --git a/src/Orleans.Journaling/JournaledStateHook.cs b/src/Orleans.Journaling/JournaledStateHook.cs deleted file mode 100644 index 31e12d601f8..00000000000 --- a/src/Orleans.Journaling/JournaledStateHook.cs +++ /dev/null @@ -1,37 +0,0 @@ -namespace Orleans.Journaling; - -/// -/// Adapts synchronous and asynchronous delegates to journal operation hooks. -/// -/// -/// When both delegates for a phase are supplied, the synchronous delegate runs first. -/// Asynchronous delegates await their I/O outcome before completing their returned value task. -/// Custom implementations can carry feature identity and state -/// for inspection and deduplication in the owner's hook list. -/// -public sealed class JournaledStateHook : IJournaledStateHook -{ - /// Gets the synchronous prerequisite callback. - public Action? BeforeOperation { get; init; } - - /// Gets the asynchronous prerequisite callback. - public Func? BeforeOperationAsync { get; init; } - - /// Gets the synchronous completion callback. - public Action? AfterOperation { get; init; } - - /// Gets the asynchronous completion callback. - public Func? AfterOperationAsync { get; init; } - - ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - BeforeOperation?.Invoke(operation, cancellationToken); - return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; - } - - ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - AfterOperation?.Invoke(operation, cancellationToken); - return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; - } -} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 7cd2d160607..f7c22804500 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -320,11 +320,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo if (hookOperation is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; - var captureHook = GetCaptureHook(); - if (captureHook is null || _hooks.Count > 1) - { - await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); - } + var captureHook = await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); if (captureHook is not null) { @@ -696,30 +692,19 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } } - private IJournaledStateCaptureHook? GetCaptureHook() - { - for (var i = 0; i < _hooks!.Count; i++) - { - if (_hooks[i] is IJournaledStateCaptureHook captureHook) - { - return captureHook; - } - } - - return null; - } - - private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { var previous = CurrentHookOwner.Value; CurrentHookOwner.Value = this; + IJournaledStateCaptureHook? captureHook = null; try { for (var i = 0; i < _hooks!.Count; i++) { var hook = _hooks[i]; - if (hook is IJournaledStateCaptureHook) + if (hook is IJournaledStateCaptureHook capture) { + captureHook = capture; continue; } @@ -728,6 +713,7 @@ private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation } cancellationToken.ThrowIfCancellationRequested(); + return captureHook; } finally { diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index ce02135b40c..938b86ad946 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -280,8 +280,8 @@ Keep operation-local bookkeeping for the captured batch separate from later pend After hooks run in list order after storage acknowledgement and state acknowledgement or reset. They also run for a successful zero-byte write. Coalesced callers share the hooks for their actual -operation. `JournaledStateHook` adapts synchronous and asynchronous delegates; within each phase -the synchronous delegate executes first. +operation. Features implement the before and after callbacks on their own hook, retaining their +identity and operation-local bookkeeping there. | Outcome | Owner and caller behavior | | --- | --- | diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 84a6099c001..11dc0ac3001 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -428,21 +428,6 @@ public sealed partial class JournalCatalogListOptions public JournalId Prefix { get { throw null; } set { } } } - public sealed partial class JournaledStateHook : IJournaledStateHook - { - public System.Action? AfterOperation { get { throw null; } init { } } - - public System.Func? AfterOperationAsync { get { throw null; } init { } } - - public System.Action? BeforeOperation { get { throw null; } init { } } - - public System.Func? BeforeOperationAsync { get { throw null; } init { } } - - System.Threading.Tasks.ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken) { throw null; } - - System.Threading.Tasks.ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken) { throw null; } - } - public sealed partial class JournaledStateManagerOptions { public static readonly System.TimeSpan DEFAULT_RETIREMENT_GRACE_PERIOD; diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 332a18c51cf..9e81a5e5604 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -19,7 +19,7 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Null(field.GetValue(manager)); - var hook = new JournaledStateHook(); + var hook = new DelegateTestHook(); var hooks = manager.Hooks; hooks.Add(hook); Assert.True(hooks.Contains(hook)); @@ -43,7 +43,7 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo await manager.InitializeAsync(TestContext.Current.CancellationToken); var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; CancellationToken ownedToken = default; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (operation, token) => { @@ -115,7 +115,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta var pendingBytes = manager.PendingWriteByteCount; var failure = new IOException("Prerequisite failed."); var after = 0; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => throw failure, AfterOperation = (_, _) => after++ @@ -158,8 +158,8 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo state["business"] = 1; var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => throw first }); - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw first }); + manager.Hooks.Add(new DelegateTestHook { AfterOperationAsync = async (_, _) => { @@ -168,7 +168,7 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo throw second; } }); - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => events.Add("last") }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); @@ -204,7 +204,7 @@ public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); await manager.InitializeAsync(TestContext.Current.CancellationToken); var after = 0; - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => after++ }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => after++ }); state["business"] = 1; Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); Assert.Equal(0, after); @@ -234,7 +234,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar return default; }, (_, _) => events.Add("capture-after"))); - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -301,7 +301,7 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b }; manager.Hooks.Add(captureHook ? new CaptureTestHook(before, after) - : new JournaledStateHook { BeforeOperationAsync = before, AfterOperation = after }); + : new DelegateTestHook { BeforeOperationAsync = before, AfterOperation = after }); using var caller = new CancellationTokenSource(); state["business"] = 1; var write = manager.WriteStateAsync(caller.Token).AsTask(); @@ -346,7 +346,7 @@ public async Task Hooks_SameOwnerReentryIsRejectedWithoutDeadlock(bool after) await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); await manager.WriteStateAsync(token); }; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperationAsync = after ? null : callback, AfterOperationAsync = after ? callback : null @@ -379,7 +379,7 @@ public async Task Hooks_RegistrationEnforcesSingleCapturePrerequisite() await using var manager = CreateTestSystem().Manager; var first = new CaptureTestHook((_, _) => default); var second = new CaptureTestHook((_, _) => default); - var ordinary = new JournaledStateHook(); + var ordinary = new DelegateTestHook(); manager.Hooks.Add(first); manager.Hooks.Add(ordinary); Assert.Throws(() => manager.Hooks.Add(second)); @@ -448,14 +448,15 @@ public async Task Hooks_MutationIsRejectedThroughoutOwnedOperation(bool after) state.Value = 1; var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var hook = new JournaledStateHook + var hook = new DelegateTestHook { BeforeOperation = (_, _) => Assert.Throws(() => manager.Hooks.Clear()), AfterOperationAsync = after ? async (_, _) => { entered.TrySetResult(); await release.Task; - } : null + } + : null }; manager.Hooks.Add(hook); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -463,8 +464,8 @@ public async Task Hooks_MutationIsRejectedThroughoutOwnedOperation(bool after) { await (after ? entered : storage.BlockedAppendStarted).Task .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Throws(() => manager.Hooks.Add(new JournaledStateHook())); - Assert.Throws(() => manager.Hooks[0] = new JournaledStateHook()); + Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); + Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); Assert.Throws(() => manager.Hooks.Remove(hook)); Assert.Throws(() => manager.Hooks.Clear()); Assert.Same(hook, Assert.Single(manager.Hooks)); @@ -491,7 +492,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var drained = false; var failure = new IOException("Cancellation callback failed."); - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -545,7 +546,7 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra entered.TrySetResult(); await release.Task; }; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperationAsync = after ? null : callback, AfterOperationAsync = after ? callback : null @@ -616,7 +617,7 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt await manager.InitializeAsync(TestContext.Current.CancellationToken); CancellationTokenRegistration registration = default; var after = 0; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), AfterOperation = (_, _) => after++ @@ -685,7 +686,7 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); List events = []; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { AfterOperationAsync = async (_, token) => { @@ -697,8 +698,8 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() throw first; } }); - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => throw second }); - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => events.Add("last") }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw second }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -731,7 +732,7 @@ public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() await manager.InitializeAsync(TestContext.Current.CancellationToken); var before = 0; var after = 0; - manager.Hooks.Add(new JournaledStateHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); state["first"] = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -766,7 +767,7 @@ public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() await manager.WriteStateAsync(TestContext.Current.CancellationToken); var before = 0; var after = 0; - manager.Hooks.Add(new JournaledStateHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(1, before); Assert.Equal(1, after); @@ -832,6 +833,26 @@ public ValueTask AfterOperationAsync(JournaledStateOperation operation, Cancella } } + private sealed class DelegateTestHook : IJournaledStateHook + { + public Action? BeforeOperation { get; init; } + public Func? BeforeOperationAsync { get; init; } + public Action? AfterOperation { get; init; } + public Func? AfterOperationAsync { get; init; } + + ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + BeforeOperation?.Invoke(operation, cancellationToken); + return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + + ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + AfterOperation?.Invoke(operation, cancellationToken); + return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + } + private sealed class HookAcknowledgementState(List events) : IStateMachine { public void ReplayEntry(JournalEntry entry, JournalReplayContext context) { } From 33749ac593c6376dcde1600ff38064a235025e0e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 17:46:43 -0700 Subject: [PATCH 05/31] fix(journaling): preserve hooks for queued persistence --- .../grains/journaling/runtime-behavior.md | 2 +- .../IJournaledStateManager.cs | 2 +- .../JournaledStateManager.cs | 8 +++ src/Orleans.Journaling/README.md | 4 +- .../StateManagerHookTests.cs | 62 +++++++++++++++++++ 5 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 68398471d80..7b4433ed7e1 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -75,7 +75,7 @@ Activation-scoped features coordinate prerequisites and completion through allocated list of registrations. Inspect and deduplicate feature registrations on the owner's logical execution context while persistence is quiescent. Registration survives recovery and deletion; the standard manager rejects hook-list -mutation throughout each operation. +mutation while persistence is queued or running. Each actual append, snapshot, or deletion runs ordinary before callbacks in list order, outside the manager lock. At most one supplies the diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 4b5598cdfce..4b31a647282 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -17,7 +17,7 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// /// Inspect, add, remove, and deduplicate hooks on the owner's logical execution context while - /// persistence is quiescent. Mutation during an operation is rejected. Ordinary before hooks + /// persistence is quiescent. Mutation while persistence is queued or running is rejected. Ordinary before hooks /// and all after hooks execute in list order. The optional single /// supplies the final prerequisite. /// Registration is independent of state-machine registration and persists through recovery and deletion. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index f7c22804500..47784d90a86 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -813,6 +813,14 @@ private void EnsureMutationAllowed() { throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); } + + foreach (var workItem in owner._workQueue) + { + if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem) + { + throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); + } + } } private void ValidateHook(IJournaledStateHook item, int replacingIndex) diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 938b86ad946..37d2f763d6e 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -269,8 +269,8 @@ of the ambient grain context, including when the caller subsequently enrolls the `IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` registrations. Features inspect and deduplicate their registrations on the owner's logical execution context while persistence is quiescent. Registration persists through recovery and -whole-journal deletion. The standard manager rejects mutation of the list throughout an -operation and admits at most one `IJournaledStateCaptureHook`. +whole-journal deletion. The standard manager rejects mutation of the list while persistence is +queued or running and admits at most one `IJournaledStateCaptureHook`. For each actual write, snapshot, or deletion, ordinary before hooks run in list order outside the manager lock. The capture hook runs last: the work loop awaits it directly, then synchronously diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 9e81a5e5604..cbbca0546a5 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -436,6 +436,68 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() } } + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_MutationIsRejectedWhilePersistenceIsQueued(JournaledStateOperation operation) + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + var before = 0; + var after = 0; + var hook = new CaptureTestHook( + (actual, _) => + { + Assert.Equal(operation, actual); + before++; + return default; + }, + (actual, _) => + { + Assert.Equal(operation, actual); + after++; + }); + manager.Hooks.Add(hook); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + using var caller = new CancellationTokenSource(); + var canceledWaiter = InvokeAsync(caller.Token); + var remainingWaiter = InvokeAsync(TestContext.Current.CancellationToken); + caller.Cancel(); + + Assert.Equal(0, state.CaptureCount); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, before); + Assert.False(remainingWaiter.IsCompleted); + Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); + Assert.Throws(() => manager.Hooks.Insert(0, new DelegateTestHook())); + Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); + Assert.Throws(() => manager.Hooks.Remove(hook)); + Assert.Throws(() => manager.Hooks.Clear()); + Assert.Same(hook, Assert.Single(manager.Hooks)); + + await Assert.ThrowsAnyAsync(() => canceledWaiter); + await remainingWaiter; + Assert.Equal(1, before); + Assert.Equal(1, after); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.CaptureCount); + Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); + Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + manager.Hooks.Clear(); + Assert.Empty(manager.Hooks); + + Task InvokeAsync(CancellationToken token) => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(token) + : manager.WriteStateAsync(token)).AsTask(); + }); + } + [Theory] [InlineData(false)] [InlineData(true)] From 43b25a0b356e977a87104bf3de447e2d83eed48e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 18:31:13 -0700 Subject: [PATCH 06/31] feat(journaling)!: require journal operation hook support --- .../grains/journaling/runtime-behavior.md | 2 ++ .../CompatibilitySuppressions.xml | 14 ++++++++++++ .../IJournaledStateManager.cs | 5 ++--- src/Orleans.Journaling/README.md | 2 ++ .../DurableListJournalBenchmarks.cs | 2 ++ .../JournaledJobShardDiscoveryTests.cs | 1 + .../DurableCollectionDirectWriteTests.cs | 2 ++ .../DurableListDirectWriteTests.cs | 2 ++ .../KeyedJournalingRegistrationTests.cs | 1 + .../StateManagerHookTests.cs | 22 ------------------- 10 files changed, 28 insertions(+), 25 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 7b4433ed7e1..0b0d25201b7 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -76,6 +76,8 @@ allocated list of registrations. I deduplicate feature registrations on the owner's logical execution context while persistence is quiescent. Registration survives recovery and deletion; the standard manager rejects hook-list mutation while persistence is queued or running. +Every journal owner implementation provides this list and runs its registered callbacks at the +operation boundaries below. Delegating owners forward the list to the inner owner. Each actual append, snapshot, or deletion runs ordinary before callbacks in list order, outside the manager lock. At most one supplies the diff --git a/src/Orleans.Journaling/CompatibilitySuppressions.xml b/src/Orleans.Journaling/CompatibilitySuppressions.xml index eb687e7cf91..ed87a41fc9b 100644 --- a/src/Orleans.Journaling/CompatibilitySuppressions.xml +++ b/src/Orleans.Journaling/CompatibilitySuppressions.xml @@ -267,6 +267,13 @@ lib/net10.0/Orleans.Journaling.dll true + + CP0006 + P:Orleans.Journaling.IJournaledStateManager.Hooks + lib/net10.0/Orleans.Journaling.dll + lib/net10.0/Orleans.Journaling.dll + true + CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey @@ -302,6 +309,13 @@ lib/net8.0/Orleans.Journaling.dll true + + CP0006 + P:Orleans.Journaling.IJournaledStateManager.Hooks + lib/net8.0/Orleans.Journaling.dll + lib/net8.0/Orleans.Journaling.dll + true + CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 4b31a647282..ffd59075f22 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -21,10 +21,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// and all after hooks execute in list order. The optional single /// supplies the final prerequisite. /// Registration is independent of state-machine registration and persists through recovery and deletion. - /// Custom owners implement this property to support hooks. + /// Every implementation provides this list and invokes its hooks at the documented operation boundaries. /// - /// The custom owner does not support hooks. - IList Hooks => throw new NotSupportedException("This journal owner does not support operation hooks."); + IList Hooks { get; } /// ValueTask IAsyncDisposable.DisposeAsync() => default; diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 37d2f763d6e..45ae95fcf62 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -271,6 +271,8 @@ registrations. Features inspect and deduplicate their registrations on the owner execution context while persistence is quiescent. Registration persists through recovery and whole-journal deletion. The standard manager rejects mutation of the list while persistence is queued or running and admits at most one `IJournaledStateCaptureHook`. +Every journal owner implementation provides the hook list and invokes registered callbacks at +the operation boundaries described below. Delegating owners forward the list to their inner owner. For each actual write, snapshot, or deletion, ordinary before hooks run in list order outside the manager lock. The capture hook runs last: the work loop awaits it directly, then synchronously diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index c94bd6045e2..f5931b7e85b 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -126,6 +126,8 @@ private void ReplayEncodedJournalData() private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter buffer, JournalStreamId streamId) : IJournaledStateManager { + public IList Hooks { get; } = []; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(buffer.CreateJournalStreamWriter(streamId)); diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index 7a9862c5cab..c6829908e9a 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -727,6 +727,7 @@ public IJournaledStateManager CreateStandalone(JournalId journalId) private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager { + public IList Hooks => inner.Hooks; public int DisposeCalls { get; private set; } public long PendingWriteByteCount => inner.PendingWriteByteCount; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 26cf16aac46..4e46d24cde7 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -193,6 +193,8 @@ public void Collections_UseDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { + public IList Hooks { get; } = []; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index e1cc3708672..bb3648d69fa 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -92,6 +92,8 @@ public void Add_UsesDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { + public IList Hooks { get; } = []; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index e9822dd9557..97b65742677 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -746,6 +746,7 @@ public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { + public IList Hooks => inner.Hooks; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); public void RegisterStateMachine(string name, IStateMachine state) => inner.RegisterStateMachine(name, state); public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index cbbca0546a5..7cb585e922e 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -859,28 +859,6 @@ public void Hooks_FailureOutcomesRoundTripWithOperationAndOriginalCause(bool aft Assert.Equal(original.Message, copy.Message); } - [Fact] - public async Task Hooks_LegacyCustomOwnerKeepsExistingOperationsAndReportsUnsupportedHooks() - { - await using IJournaledStateManager manager = new LegacyHookTestOwner(); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Throws(() => manager.Hooks); - } - - private sealed class LegacyHookTestOwner : IJournaledStateManager - { - public ValueTask InitializeAsync(CancellationToken cancellationToken = default) => default; - public ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => default; - public ValueTask DeleteStateAsync(CancellationToken cancellationToken = default) => default; - public void RegisterStateMachine(string name, IStateMachine stateMachine) { } - public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? stateMachine) - { - stateMachine = null; - return false; - } - } - private sealed class CaptureTestHook( Func before, Action? after = null) : IJournaledStateCaptureHook From 659766b01433e1493bd2c9f2f5c399a1ace7735e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 19:40:35 -0700 Subject: [PATCH 07/31] refactor(journaling): simplify hook execution and failure state --- .../grains/journaling/runtime-behavior.md | 5 +- src/Orleans.Journaling/IJournaledStateHook.cs | 4 +- .../JournaledStateManager.cs | 75 +++++-------------- .../JournaledStatePostCommitException.cs | 5 -- .../JournaledStatePreCommitException.cs | 5 -- src/Orleans.Journaling/README.md | 3 +- .../Orleans.Journaling/Orleans.Journaling.cs | 14 +--- .../StateManagerHookTests.cs | 51 ++----------- 8 files changed, 32 insertions(+), 130 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 0b0d25201b7..6b245288bb5 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -101,8 +101,9 @@ state-processing failures retain the manager's fencing and fresh-recovery behavi Hook callbacks receive the owner's shutdown token. Cancelling a caller's wait leaves the owned operation running through its actual outcome. Disposal drains owned hooks and storage before releasing journal resources, including when cancellation callbacks or cleanup fail. Concurrent disposal callers -share this completion. Recursive initialization, persistence, or disposal on the same owner from a -hook is rejected. Shutdown closes work admission and cancels queued operations while the current +share this completion. Hook implementations complete without calling initialization, persistence, +or disposal on their own owner. Awaiting an operation serialized behind the current callback creates +a circular dependency. Shutdown closes work admission and cancels queued operations while the current operation drains to its actual storage and hook outcome. For deletion, the feature owner stops admission and drains feature operations before queuing the whole-journal reset. diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index d0ffaae24f3..3639b6c658a 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -24,7 +24,9 @@ public enum JournaledStateOperation /// Prerequisites must cover changes staged during asynchronous preparation. After hooks run after /// storage acknowledgement and state acknowledgement or reset, including successful writes which /// produce no storage bytes. Hooks retain operation-local data across these boundaries and keep later -/// pending changes separate. Recursive operations on the same journal owner are rejected. +/// pending changes separate. Hook implementations must complete without calling initialization, +/// persistence, or disposal on their own journal owner. Those operations are serialized behind the +/// current operation, so awaiting them from a hook would create a circular dependency. /// public interface IJournaledStateHook { diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 47784d90a86..6f8d8479d36 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -13,7 +13,6 @@ namespace Orleans.Journaling; internal partial class JournaledStateManager : IJournaledStateManager, IJournalStorageConsumer, ILifecycleParticipant, ILifecycleObserver, IDisposable { private const uint MinApplicationJournalStreamId = 8u; - private static readonly AsyncLocal CurrentHookOwner = new(); #if NET9_0_OR_GREATER private readonly Lock _lock = new(); #else @@ -172,7 +171,6 @@ public void RegisterStateMachine(string name, IStateMachine stateMachine) public async ValueTask InitializeAsync(CancellationToken cancellationToken = default) { - ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; @@ -325,16 +323,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo if (captureHook is not null) { // Await the final prerequisite in this frame so capture follows its completion directly. - var previous = CurrentHookOwner.Value; - CurrentHookOwner.Value = this; - try - { - await captureHook.BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); - } - finally - { - CurrentHookOwner.Value = previous; - } + await captureHook.BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); } _shutdownCancellation.Token.ThrowIfCancellationRequested(); @@ -694,55 +683,37 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { - var previous = CurrentHookOwner.Value; - CurrentHookOwner.Value = this; IJournaledStateCaptureHook? captureHook = null; - try + for (var i = 0; i < _hooks!.Count; i++) { - for (var i = 0; i < _hooks!.Count; i++) + var hook = _hooks[i]; + if (hook is IJournaledStateCaptureHook capture) { - var hook = _hooks[i]; - if (hook is IJournaledStateCaptureHook capture) - { - captureHook = capture; - continue; - } - - cancellationToken.ThrowIfCancellationRequested(); - await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); + captureHook = capture; + continue; } cancellationToken.ThrowIfCancellationRequested(); - return captureHook; - } - finally - { - CurrentHookOwner.Value = previous; + await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); } + + cancellationToken.ThrowIfCancellationRequested(); + return captureHook; } private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { - var previous = CurrentHookOwner.Value; - CurrentHookOwner.Value = this; List? failures = null; - try + for (var i = 0; i < _hooks!.Count; i++) { - for (var i = 0; i < _hooks!.Count; i++) + try { - try - { - await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); - } - catch (Exception exception) - { - (failures ??= []).Add(exception); - } + await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + catch (Exception exception) + { + (failures ??= []).Add(exception); } - } - finally - { - CurrentHookOwner.Value = previous; } if (failures is { Count: 1 }) @@ -756,14 +727,6 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, } } - private void ThrowIfHookReentry() - { - if (ReferenceEquals(CurrentHookOwner.Value, this)) - { - throw new InvalidOperationException("Journal operation hooks cannot enqueue another operation on the same journal owner."); - } - } - private sealed class HookCollection(JournaledStateManager owner) : Collection { protected override void InsertItem(int index, IJournaledStateHook item) @@ -942,7 +905,6 @@ private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWrit public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = default) { - ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; @@ -1130,7 +1092,6 @@ private InvalidOperationException CreateRecoveryFormatException(Exception except public async ValueTask WriteStateAsync(CancellationToken cancellationToken = default) { - ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task pendingWrite; @@ -1307,7 +1268,6 @@ void ILifecycleParticipant.Participate(IGrainLifecycle observer private async Task StopAsync(CancellationToken cancellationToken) { - ThrowIfHookReentry(); AggregateException? cancellationFailure = null; lock (_lock) { @@ -1363,7 +1323,6 @@ void IDisposable.Dispose() public ValueTask DisposeAsync() { - ThrowIfHookReentry(); lock (_lock) { return new(_disposeTask ??= DisposeCoreAsync()); diff --git a/src/Orleans.Journaling/JournaledStatePostCommitException.cs b/src/Orleans.Journaling/JournaledStatePostCommitException.cs index c10a39e92b6..8274c8726d6 100644 --- a/src/Orleans.Journaling/JournaledStatePostCommitException.cs +++ b/src/Orleans.Journaling/JournaledStatePostCommitException.cs @@ -18,10 +18,5 @@ public sealed class JournaledStatePostCommitException : Exception public JournaledStatePostCommitException(JournaledStateOperation operation, Exception innerException) : base($"Journal operation '{operation}' completed, but a post-persistence hook failed.", innerException) { - Operation = operation; } - - /// Gets the successfully completed journal operation. - [Id(0)] - public JournaledStateOperation Operation { get; } } diff --git a/src/Orleans.Journaling/JournaledStatePreCommitException.cs b/src/Orleans.Journaling/JournaledStatePreCommitException.cs index 1fba60d1d3e..fe8d2eb79d4 100644 --- a/src/Orleans.Journaling/JournaledStatePreCommitException.cs +++ b/src/Orleans.Journaling/JournaledStatePreCommitException.cs @@ -18,10 +18,5 @@ public sealed class JournaledStatePreCommitException : Exception public JournaledStatePreCommitException(JournaledStateOperation operation, Exception innerException) : base($"Journal operation '{operation}' was prevented by a prerequisite hook failure.", innerException) { - Operation = operation; } - - /// Gets the journal operation which was prevented. - [Id(0)] - public JournaledStateOperation Operation { get; } } diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 45ae95fcf62..0444dfcb9e3 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -297,7 +297,8 @@ owner token and drains owned work before releasing journal resources, including callbacks or after-hook cleanup fail. Concurrent disposal callers share that drain and its outcome. Shutdown closes work admission and cancels queued operations while an already running operation drains to its actual storage and hook outcome. -Recursive initialization, persistence, or disposal on the same owner from a hook is rejected. +Hook implementations complete without calling initialization, persistence, or disposal on their +own owner: awaiting an operation serialized behind the current callback creates a circular dependency. Before whole-journal deletion, the feature owner stops admission and drains its own operations; deletion completion follows storage deletion and registered-state reset. diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 11dc0ac3001..76ff42769dd 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -447,18 +447,12 @@ public enum JournaledStateOperation public sealed partial class JournaledStatePostCommitException : System.Exception { public JournaledStatePostCommitException(JournaledStateOperation operation, System.Exception innerException) { } - - [Id(0)] - public JournaledStateOperation Operation { get { throw null; } } } [GenerateSerializer] public sealed partial class JournaledStatePreCommitException : System.Exception { public JournaledStatePreCommitException(JournaledStateOperation operation, System.Exception innerException) { } - - [Id(0)] - public JournaledStateOperation Operation { get { throw null; } } } public readonly ref partial struct JournalEntry @@ -842,7 +836,7 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] public sealed partial class Codec_JournaledStatePostCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec { - public Codec_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + public Codec_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePostCommitException instance) { } @@ -860,7 +854,7 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] public sealed partial class Codec_JournaledStatePreCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec { - public Codec_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + public Codec_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePreCommitException instance) { } @@ -886,8 +880,6 @@ public sealed partial class Copier_DurableTaskCompletionSourceState : global: public sealed partial class Copier_JournaledStatePostCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier { public Copier_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } - - public override void DeepCopy(global::Orleans.Journaling.JournaledStatePostCommitException input, global::Orleans.Journaling.JournaledStatePostCommitException output, global::Orleans.Serialization.Cloning.CopyContext context) { } } [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] @@ -896,7 +888,5 @@ public override void DeepCopy(global::Orleans.Journaling.JournaledStatePostCommi public sealed partial class Copier_JournaledStatePreCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier { public Copier_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } - - public override void DeepCopy(global::Orleans.Journaling.JournaledStatePreCommitException input, global::Orleans.Journaling.JournaledStatePreCommitException output, global::Orleans.Serialization.Cloning.CopyContext context) { } } } \ No newline at end of file diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 7cb585e922e..dff2a601174 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -122,7 +122,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta }); var caught = await Assert.ThrowsAsync(() => InvokeAsync()); Assert.Same(failure, caught.InnerException); - Assert.Equal(operation, caught.Operation); + Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); Assert.Equal(1, state["business"]); Assert.Equal(pendingBytes, manager.PendingWriteByteCount); Assert.Equal(0, capture.AppendEntriesCount); @@ -172,7 +172,7 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); - Assert.Equal(operation, caught.Operation); + Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); Assert.Equal(operation == JournaledStateOperation.Delete ? new[] { "second", "last" } @@ -332,47 +332,6 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b } } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_SameOwnerReentryIsRejectedWithoutDeadlock(bool after) - { - await using var manager = CreateTestSystem().Manager; - await manager.InitializeAsync(TestContext.Current.CancellationToken); - Func callback = async (_, token) => - { - await Assert.ThrowsAsync(() => manager.InitializeAsync(token).AsTask()); - await Assert.ThrowsAsync(() => manager.DeleteStateAsync(token).AsTask()); - await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); - await manager.WriteStateAsync(token); - }; - manager.Hooks.Add(new DelegateTestHook - { - BeforeOperationAsync = after ? null : callback, - AfterOperationAsync = after ? callback : null - }); - var failure = await Record.ExceptionAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken) - .AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); - var cause = after - ? Assert.IsType(failure).InnerException - : Assert.IsType(failure).InnerException; - Assert.Contains("same journal owner", Assert.IsType(cause).Message, StringComparison.Ordinal); - manager.Hooks.Clear(); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task Hooks_FinalPrerequisiteRejectsSameOwnerReentry() - { - await using var manager = CreateTestSystem().Manager; - await manager.InitializeAsync(TestContext.Current.CancellationToken); - manager.Hooks.Add(new CaptureTestHook((_, token) => manager.DeleteStateAsync(token))); - var failure = await Assert.ThrowsAsync(() => - manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() - .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); - Assert.Contains("same journal owner", Assert.IsType(failure.InnerException).Message, StringComparison.Ordinal); - } - [Fact] public async Task Hooks_RegistrationEnforcesSingleCapturePrerequisite() { @@ -839,7 +798,7 @@ public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() [Theory] [InlineData(false)] [InlineData(true)] - public void Hooks_FailureOutcomesRoundTripWithOperationAndOriginalCause(bool after) + public void Hooks_FailureOutcomesRoundTripWithTypeAndOriginalCause(bool after) { Exception original = after ? new JournaledStatePostCommitException(JournaledStateOperation.Delete, new IOException("Cleanup failed.")) @@ -848,11 +807,11 @@ public void Hooks_FailureOutcomesRoundTripWithOperationAndOriginalCause(bool aft var copy = serializer.Deserialize(serializer.SerializeToArray(original)); if (after) { - Assert.Equal(JournaledStateOperation.Delete, Assert.IsType(copy).Operation); + Assert.IsType(copy); } else { - Assert.Equal(JournaledStateOperation.Snapshot, Assert.IsType(copy).Operation); + Assert.IsType(copy); } Assert.Equal(original.InnerException!.Message, Assert.IsType(copy.InnerException).Message); From 104fdab6811c042db77b1746181828a05ac47fb7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 08:37:20 -0700 Subject: [PATCH 08/31] refactor(journaling): fix hook registration at initialization --- .../grains/journaling/runtime-behavior.md | 10 +- src/Orleans.Journaling/IJournaledStateHook.cs | 1 + .../IJournaledStateManager.cs | 4 +- .../JournaledStateManager.Hooks.cs | 122 ++++++ .../JournaledStateManager.cs | 187 ++------- src/Orleans.Journaling/README.md | 8 +- .../DurableStateManagerTests.cs | 2 +- .../StateManagerHookTests.cs | 361 +++++++++++------- 8 files changed, 387 insertions(+), 308 deletions(-) create mode 100644 src/Orleans.Journaling/JournaledStateManager.Hooks.cs diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 6b245288bb5..4f739a24284 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -72,10 +72,9 @@ staged mutations from other calls. Activation-scoped features coordinate prerequisites and completion through . The owner exposes a stable, lazily -allocated list of registrations. Inspect and -deduplicate feature registrations on the owner's logical execution context while persistence -is quiescent. Registration survives recovery and deletion; the standard manager rejects hook-list -mutation while persistence is queued or running. +allocated list of registrations. Register and +deduplicate feature hooks during setup before initialization begins, alongside state machines. +The hook list then remains fixed through recovery, persistence and deletion. Every journal owner implementation provides this list and runs its registered callbacks at the operation boundaries below. Delegating owners forward the list to the inner owner. @@ -103,7 +102,8 @@ operation running through its actual outcome. Disposal drains owned hooks and st journal resources, including when cancellation callbacks or cleanup fail. Concurrent disposal callers share this completion. Hook implementations complete without calling initialization, persistence, or disposal on their own owner. Awaiting an operation serialized behind the current callback creates -a circular dependency. Shutdown closes work admission and cancels queued operations while the current +a circular dependency. Dependencies between journal owners also remain acyclic. +Shutdown closes work admission and cancels queued operations while the current operation drains to its actual storage and hook outcome. For deletion, the feature owner stops admission and drains feature operations before queuing the whole-journal reset. diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index 3639b6c658a..29d245799d6 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -27,6 +27,7 @@ public enum JournaledStateOperation /// pending changes separate. Hook implementations must complete without calling initialization, /// persistence, or disposal on their own journal owner. Those operations are serialized behind the /// current operation, so awaiting them from a hook would create a circular dependency. +/// Dependencies between journal owners must also remain acyclic. /// public interface IJournaledStateHook { diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index ffd59075f22..77149dc9500 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -16,8 +16,8 @@ public interface IJournaledStateManager : IAsyncDisposable /// Gets the mutable, lazily allocated list of journal operation hooks. /// /// - /// Inspect, add, remove, and deduplicate hooks on the owner's logical execution context while - /// persistence is quiescent. Mutation while persistence is queued or running is rejected. Ordinary before hooks + /// Inspect, add, remove, and deduplicate hooks during setup, before initialization begins. + /// Registration remains fixed for this manager's initialized lifetime. Ordinary before hooks /// and all after hooks execute in list order. The optional single /// supplies the final prerequisite. /// Registration is independent of state-machine registration and persists through recovery and deletion. diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs new file mode 100644 index 00000000000..1c9502a772f --- /dev/null +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -0,0 +1,122 @@ +using System.Collections.ObjectModel; +using System.Runtime.ExceptionServices; + +namespace Orleans.Journaling; + +internal partial class JournaledStateManager +{ + private HookCollection? _hooks; + + public IList Hooks + { + get + { + lock (_lock) + { + return _hooks ??= new(this); + } + } + } + + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + IJournaledStateCaptureHook? captureHook = null; + for (var i = 0; i < _hooks!.Count; i++) + { + var hook = _hooks[i]; + if (hook is IJournaledStateCaptureHook capture) + { + captureHook = capture; + continue; + } + + cancellationToken.ThrowIfCancellationRequested(); + await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + + cancellationToken.ThrowIfCancellationRequested(); + return captureHook; + } + + private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + List? failures = null; + for (var i = 0; i < _hooks!.Count; i++) + { + try + { + await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + catch (Exception exception) + { + (failures ??= []).Add(exception); + } + } + + if (failures is { Count: 1 }) + { + ExceptionDispatchInfo.Capture(failures[0]).Throw(); + } + + if (failures is not null) + { + throw new AggregateException(failures); + } + } + + private sealed class HookCollection(JournaledStateManager owner) : Collection + { + protected override void InsertItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + owner.EnsureRegistrationAllowed(); + ValidateHook(item, replacingIndex: -1); + base.InsertItem(index, item); + } + } + + protected override void SetItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + owner.EnsureRegistrationAllowed(); + ValidateHook(item, index); + base.SetItem(index, item); + } + } + + protected override void RemoveItem(int index) + { + lock (owner._lock) + { + owner.EnsureRegistrationAllowed(); + base.RemoveItem(index); + } + } + + protected override void ClearItems() + { + lock (owner._lock) + { + owner.EnsureRegistrationAllowed(); + base.ClearItems(); + } + } + + private void ValidateHook(IJournaledStateHook item, int replacingIndex) + { + ArgumentNullException.ThrowIfNull(item); + if (item is IJournaledStateCaptureHook) + { + for (var i = 0; i < Count; i++) + { + if (i != replacingIndex && this[i] is IJournaledStateCaptureHook) + { + throw new InvalidOperationException("A journal owner supports one final capture prerequisite hook. Inspect and deduplicate the hook list before registration."); + } + } + } + } + } +} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 6f8d8479d36..0260219f6be 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.Collections.ObjectModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.ExceptionServices; @@ -36,19 +35,6 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private bool _migrationSnapshotRequired; private int _disposed; private Task? _disposeTask; - private HookCollection? _hooks; - private bool _hookOperationRunning; - - public IList Hooks - { - get - { - lock (_lock) - { - return _hooks ??= new(this); - } - } - } public JournaledStateManager(JournaledStateManagerShared shared, IJournalStorageProvider storageProvider, IGrainContext grainContext) : this(shared, CreateStorage(storageProvider, CreateJournalId(grainContext)), grainContext) @@ -143,7 +129,7 @@ protected void EnsureRegistrationAllowed() _shutdownCancellation.Token.ThrowIfCancellationRequested(); if (_workLoop is not null) { - throw new InvalidOperationException("New states cannot be registered after journaled state manager initialization has begun."); + throw new InvalidOperationException("Journal registrations cannot be changed after journaled state manager initialization has begun."); } } } @@ -273,7 +259,6 @@ private async Task WorkLoop() } workItem = dequeuedWorkItem; - _hookOperationRunning = workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem; } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -599,11 +584,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } storageActivity?.SetStatus(ActivityStatusCode.Ok); - lock (_lock) - { - _hookOperationRunning = false; - workItem.SetResult(); - } + workItem.SetResult(); } catch (Exception exception) when (beforeHookRunning || afterHookRunning) { @@ -620,13 +601,9 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: afterHookRunning); } - lock (_lock) - { - _hookOperationRunning = false; - workItem.SetException(afterHookRunning - ? new JournaledStatePostCommitException(hookOperation.Value, exception) - : new JournaledStatePreCommitException(hookOperation.Value, exception)); - } + workItem.SetException(afterHookRunning + ? new JournaledStatePostCommitException(hookOperation.Value, exception) + : new JournaledStatePreCommitException(hookOperation.Value, exception)); } catch (Exception exception) { @@ -660,11 +637,6 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } finally { - lock (_lock) - { - _hookOperationRunning = false; - } - storageActivity?.Dispose(); } } @@ -681,127 +653,6 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } } - private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - IJournaledStateCaptureHook? captureHook = null; - for (var i = 0; i < _hooks!.Count; i++) - { - var hook = _hooks[i]; - if (hook is IJournaledStateCaptureHook capture) - { - captureHook = capture; - continue; - } - - cancellationToken.ThrowIfCancellationRequested(); - await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); - } - - cancellationToken.ThrowIfCancellationRequested(); - return captureHook; - } - - private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - List? failures = null; - for (var i = 0; i < _hooks!.Count; i++) - { - try - { - await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); - } - catch (Exception exception) - { - (failures ??= []).Add(exception); - } - } - - if (failures is { Count: 1 }) - { - ExceptionDispatchInfo.Capture(failures[0]).Throw(); - } - - if (failures is not null) - { - throw new AggregateException(failures); - } - } - - private sealed class HookCollection(JournaledStateManager owner) : Collection - { - protected override void InsertItem(int index, IJournaledStateHook item) - { - lock (owner._lock) - { - EnsureMutationAllowed(); - ValidateHook(item, replacingIndex: -1); - base.InsertItem(index, item); - } - } - - protected override void SetItem(int index, IJournaledStateHook item) - { - lock (owner._lock) - { - EnsureMutationAllowed(); - ValidateHook(item, index); - base.SetItem(index, item); - } - } - - protected override void RemoveItem(int index) - { - lock (owner._lock) - { - EnsureMutationAllowed(); - base.RemoveItem(index); - } - } - - protected override void ClearItems() - { - lock (owner._lock) - { - EnsureMutationAllowed(); - base.ClearItems(); - } - } - - private void EnsureMutationAllowed() - { - ObjectDisposedException.ThrowIf(owner._disposed != 0, owner); - owner._shutdownCancellation.Token.ThrowIfCancellationRequested(); - owner.ThrowIfFenced(); - if (owner._hookOperationRunning) - { - throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); - } - - foreach (var workItem in owner._workQueue) - { - if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem) - { - throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); - } - } - } - - private void ValidateHook(IJournaledStateHook item, int replacingIndex) - { - ArgumentNullException.ThrowIfNull(item); - if (item is IJournaledStateCaptureHook) - { - for (var i = 0; i < Count; i++) - { - if (i != replacingIndex && this[i] is IJournaledStateCaptureHook) - { - throw new InvalidOperationException("A journal owner supports one final capture prerequisite hook. Inspect and deduplicate the hook list before registration."); - } - } - } - } - } - private void Fence(Exception exception) { lock (_lock) @@ -1325,21 +1176,37 @@ public ValueTask DisposeAsync() { lock (_lock) { - return new(_disposeTask ??= DisposeCoreAsync()); + if (_disposeTask is null) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _disposeTask = completion.Task; + _ = DisposeCoreAsync(completion); + } + + return new(_disposeTask); } } - private async Task DisposeCoreAsync() + private async Task DisposeCoreAsync(TaskCompletionSource completion) { _disposed = 1; try { - await StopAsync(CancellationToken.None).ConfigureAwait(false); + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + finally + { + _shutdownCancellation.Dispose(); + _journalWriter.Dispose(); + } + + completion.SetResult(); } - finally + catch (Exception exception) { - _shutdownCancellation.Dispose(); - _journalWriter.Dispose(); + completion.SetException(exception); } } diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 0444dfcb9e3..2d7f0f36200 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -267,10 +267,9 @@ of the ambient grain context, including when the caller subsequently enrolls the ## Journal operation hooks `IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` -registrations. Features inspect and deduplicate their registrations on the owner's logical -execution context while persistence is quiescent. Registration persists through recovery and -whole-journal deletion. The standard manager rejects mutation of the list while persistence is -queued or running and admits at most one `IJournaledStateCaptureHook`. +registrations. Features register and deduplicate their hooks during setup before initialization +begins, alongside state-machine registration. The hook list then remains fixed through recovery, +persistence and whole-journal deletion. The standard manager admits at most one `IJournaledStateCaptureHook`. Every journal owner implementation provides the hook list and invokes registered callbacks at the operation boundaries described below. Delegating owners forward the list to their inner owner. @@ -299,6 +298,7 @@ Shutdown closes work admission and cancels queued operations while an already ru drains to its actual storage and hook outcome. Hook implementations complete without calling initialization, persistence, or disposal on their own owner: awaiting an operation serialized behind the current callback creates a circular dependency. +Dependencies between journal owners also remain acyclic. Before whole-journal deletion, the feature owner stops admission and drains its own operations; deletion completion follows storage deletion and registered-state reset. diff --git a/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs b/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs index d1b0b4b797f..e85899b2bdb 100644 --- a/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs @@ -592,7 +592,7 @@ private static void AssertIncompatible( private static void AssertAdmissionRejected(Action action) { var error = Assert.Throws(action); - Assert.Equal("New states cannot be registered after journaled state manager initialization has begun.", error.Message); + Assert.Equal("Journal registrations cannot be changed after journaled state manager initialization has begun.", error.Message); } private static void AssertFenced(InvalidOperationException error, IOException cause) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index dff2a601174..6c49ce2ede1 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -15,9 +15,6 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() await using var manager = CreateTestSystem().Manager; var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; Assert.Null(field.GetValue(manager)); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Null(field.GetValue(manager)); var hook = new DelegateTestHook(); var hooks = manager.Hooks; @@ -28,6 +25,67 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() Assert.True(hooks.Remove(hook)); Assert.Empty(manager.Hooks); Assert.Throws(() => hooks.Add(null!)); + hooks.Add(hook); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Same(hooks, manager.Hooks); + Assert.Same(hook, Assert.Single(hooks)); + Assert.Throws(() => hooks.Clear()); + + await using var withoutHooks = CreateTestSystem().Manager; + await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); + await withoutHooks.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Null(field.GetValue(withoutHooks)); + } + + [Fact] + public async Task Hooks_RegistrationClosesBeforeRecoveryAndRemainsClosedAfterFailure() + { + var entered = NewSignal(); + var release = NewSignal(); + var storage = Substitute.For(); + var failure = new IOException("Recovery failed."); + var fail = true; + storage.ReadAsync(Arg.Any(), Arg.Any()) + .Returns(call => ReadAsync(call.Arg(), call.Arg())); + await using var manager = CreateTestSystem(storage).Manager; + var hook = new DelegateTestHook(); + var hooks = manager.Hooks; + hooks.Add(hook); + var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + Assert.Throws(() => hooks.Add(new DelegateTestHook())); + Assert.Throws(() => hooks.Insert(0, new DelegateTestHook())); + Assert.Throws(() => hooks[0] = new DelegateTestHook()); + Assert.Throws(() => hooks.Remove(hook)); + Assert.Throws(() => hooks.Clear()); + await WaitFor(entered.Task); + Assert.False(initialization.IsCompleted); + release.TrySetResult(); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); + Assert.Throws(() => hooks.Clear()); + fail = false; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Throws(() => hooks.Clear()); + Assert.Same(hook, Assert.Single(hooks)); + } + finally + { + release.TrySetResult(); + } + + async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + if (fail) + { + throw failure; + } + + consumer.Complete(metadata: null); + } } [Theory] @@ -40,7 +98,6 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); List events = []; manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; CancellationToken ownedToken = default; manager.Hooks.Add(new DelegateTestHook @@ -74,6 +131,7 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo events.Add("after-async"); } }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -110,16 +168,23 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); var capture = new AlwaysWritingState(); manager.RegisterStateMachine("capture", capture); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["business"] = 1; - var pendingBytes = manager.PendingWriteByteCount; var failure = new IOException("Prerequisite failed."); + var fail = true; var after = 0; manager.Hooks.Add(new DelegateTestHook { - BeforeOperation = (_, _) => throw failure, + BeforeOperation = (_, _) => + { + if (fail) + { + throw failure; + } + }, AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var pendingBytes = manager.PendingWriteByteCount; var caught = await Assert.ThrowsAsync(() => InvokeAsync()); Assert.Same(failure, caught.InnerException); Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); @@ -131,8 +196,9 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta Assert.Empty(storage.Replaces); Assert.Equal(0, storage.DeleteCount); Assert.Equal(0, after); - manager.Hooks.Clear(); + fail = false; await InvokeAsync(); + Assert.Equal(1, after); Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); @@ -154,21 +220,34 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); List events = []; manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["business"] = 1; var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw first }); + var fail = true; + manager.Hooks.Add(new DelegateTestHook + { + AfterOperation = (_, _) => + { + if (fail) + { + throw first; + } + } + }); manager.Hooks.Add(new DelegateTestHook { AfterOperationAsync = async (_, _) => { await Task.Yield(); events.Add("second"); - throw second; + if (fail) + { + throw second; + } } }); manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); @@ -179,7 +258,7 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo : new[] { "capture", "ack", "second", "last" }, events); Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); - manager.Hooks.Clear(); + fail = false; await using var replayManager = CreateTestSystem(storage).Manager; var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); @@ -202,9 +281,9 @@ public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() var storage = new CapturingStorage { NextAppendException = failure }; await using var manager = CreateTestSystem(storage).Manager; var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var after = 0; manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); Assert.Equal(0, after); @@ -222,7 +301,6 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); List events = []; manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); manager.Hooks.Add(new CaptureTestHook( @@ -245,6 +323,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar }, AfterOperation = (_, _) => events.Add("ordinary-after") }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -280,7 +359,6 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -302,6 +380,7 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b manager.Hooks.Add(captureHook ? new CaptureTestHook(before, after) : new DelegateTestHook { BeforeOperationAsync = before, AfterOperation = after }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); using var caller = new CancellationTokenSource(); state["business"] = 1; var write = manager.WriteStateAsync(caller.Token).AsTask(); @@ -356,15 +435,19 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; var state = new DurableValue("state", manager, CreateValueCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); + var deleting = false; var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var after = 0; manager.Hooks.Add(new CaptureTestHook( async (operation, token) => { + if (!deleting) + { + Assert.Equal(JournaledStateOperation.Write, operation); + return; + } + Assert.Equal(JournaledStateOperation.Delete, operation); Assert.Equal(1, state.Value); entered.TrySetResult(); @@ -372,11 +455,20 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() }, (operation, _) => { + if (!deleting) + { + return; + } + Assert.Equal(JournaledStateOperation.Delete, operation); Assert.Equal(0, state.Value); Assert.Equal(1, storage.DeleteCount); after++; })); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + deleting = true; var deletion = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -395,120 +487,11 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() } } - [Theory] - [InlineData(JournaledStateOperation.Write)] - [InlineData(JournaledStateOperation.Snapshot)] - [InlineData(JournaledStateOperation.Delete)] - public async Task Hooks_MutationIsRejectedWhilePersistenceIsQueued(JournaledStateOperation operation) - { - var context = new QueuedSynchronizationContext(); - await context.Run(async () => - { - var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new LifecycleState(); - manager.RegisterStateMachine("state", state); - var before = 0; - var after = 0; - var hook = new CaptureTestHook( - (actual, _) => - { - Assert.Equal(operation, actual); - before++; - return default; - }, - (actual, _) => - { - Assert.Equal(operation, actual); - after++; - }); - manager.Hooks.Add(hook); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - using var caller = new CancellationTokenSource(); - var canceledWaiter = InvokeAsync(caller.Token); - var remainingWaiter = InvokeAsync(TestContext.Current.CancellationToken); - caller.Cancel(); - - Assert.Equal(0, state.CaptureCount); - Assert.Equal(0, storage.DeleteCount); - Assert.Equal(0, before); - Assert.False(remainingWaiter.IsCompleted); - Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); - Assert.Throws(() => manager.Hooks.Insert(0, new DelegateTestHook())); - Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); - Assert.Throws(() => manager.Hooks.Remove(hook)); - Assert.Throws(() => manager.Hooks.Clear()); - Assert.Same(hook, Assert.Single(manager.Hooks)); - - await Assert.ThrowsAnyAsync(() => canceledWaiter); - await remainingWaiter; - Assert.Equal(1, before); - Assert.Equal(1, after); - Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.CaptureCount); - Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); - Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); - Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); - manager.Hooks.Clear(); - Assert.Empty(manager.Hooks); - - Task InvokeAsync(CancellationToken token) => (operation == JournaledStateOperation.Delete - ? manager.DeleteStateAsync(token) - : manager.WriteStateAsync(token)).AsTask(); - }); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_MutationIsRejectedThroughoutOwnedOperation(bool after) - { - var storage = new CapturingStorage { BlockNextAppend = !after }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var hook = new DelegateTestHook - { - BeforeOperation = (_, _) => Assert.Throws(() => manager.Hooks.Clear()), - AfterOperationAsync = after ? async (_, _) => - { - entered.TrySetResult(); - await release.Task; - } - : null - }; - manager.Hooks.Add(hook); - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await (after ? entered : storage.BlockedAppendStarted).Task - .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); - Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); - Assert.Throws(() => manager.Hooks.Remove(hook)); - Assert.Throws(() => manager.Hooks.Clear()); - Assert.Same(hook, Assert.Single(manager.Hooks)); - release.TrySetResult(); - storage.ReleaseAppend.TrySetResult(); - await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - manager.Hooks.Clear(); - Assert.Empty(manager.Hooks); - } - finally - { - release.TrySetResult(); - storage.ReleaseAppend.TrySetResult(); - } - } - [Fact] public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() { var storage = new CapturingStorage(); var manager = CreateTestSystem(storage).Manager; - await manager.InitializeAsync(TestContext.Current.CancellationToken); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var drained = false; @@ -523,6 +506,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() drained = true; } }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -559,7 +543,6 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra var manager = CreateTestSystem(storage).Manager; var state = new LifecycleState(); manager.RegisterStateMachine("state", state); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var entered = NewSignal(); var release = NewSignal(); Func callback = async (_, _) => @@ -572,6 +555,7 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra BeforeOperationAsync = after ? null : callback, AfterOperationAsync = after ? callback : null }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -635,7 +619,6 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt var manager = CreateTestSystem(storage).Manager; var state = new LifecycleState(); manager.RegisterStateMachine("state", state); - await manager.InitializeAsync(TestContext.Current.CancellationToken); CancellationTokenRegistration registration = default; var after = 0; manager.Hooks.Add(new DelegateTestHook @@ -643,6 +626,7 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -695,13 +679,56 @@ async ValueTask WriteAsync(ReadOnlySequence bytes, CancellationToken token } [Fact] - public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() + public async Task Hooks_ShutdownCallbackReentrySharesDisposalCompletion() { var storage = new CapturingStorage(); var manager = CreateTestSystem(storage).Manager; var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + Task? reenteredDisposal = null; + var callbackCount = 0; + manager.Hooks.Add(new DelegateTestHook + { + AfterOperationAsync = async (_, token) => + { + using var registration = token.Register(() => + { + callbackCount++; + reenteredDisposal = manager.DisposeAsync().AsTask(); + }); + entered.TrySetResult(); + await release.Task; + } + }); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var disposal = manager.DisposeAsync().AsTask(); + Assert.Equal(1, callbackCount); + Assert.Same(disposal, reenteredDisposal); + Assert.False(disposal.IsCompleted); + release.TrySetResult(); + await WaitFor(Task.WhenAll(write, disposal)); + Assert.Same(disposal, manager.DisposeAsync().AsTask()); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Fact] + public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var first = new IOException("First cleanup failed."); @@ -721,6 +748,8 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() }); manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw second }); manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -750,10 +779,10 @@ public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() var storage = new CapturingStorage { BlockNextAppend = true }; await using var manager = CreateTestSystem(storage).Manager; var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var before = 0; var after = 0; manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); state["first"] = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -777,18 +806,78 @@ public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() } } + [Fact] + public async Task Hooks_StateChangesDuringAfterCallbackPersistInNextOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var after = 0; + manager.Hooks.Add(new DelegateTestHook + { + AfterOperationAsync = async (_, _) => + { + if (++after == 1) + { + entered.TrySetResult(); + await release.Task; + } + } + }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["first"] = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + Assert.Single(storage.Appends); + Assert.False(first.IsCompleted); + state["later"] = 2; + var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(next.IsCompleted); + + await using (var beforeNext = CreateTestSystem(storage).Manager) + { + var recovered = new DurableDictionary("state", beforeNext, CreateDictionaryCodec()); + await beforeNext.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(recovered).Value); + } + + release.TrySetResult(); + await WaitFor(Task.WhenAll(first, next)); + Assert.Equal(2, after); + Assert.Equal(2, storage.Appends.Count); + await using var afterNext = CreateTestSystem(storage).Manager; + var persisted = new DurableDictionary("state", afterNext, CreateDictionaryCodec()); + await afterNext.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, persisted.Count); + Assert.Equal(1, persisted["first"]); + Assert.Equal(2, persisted["later"]); + } + finally + { + release.TrySetResult(); + } + } + [Fact] public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() { var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; var state = new DurableValue("state", manager, CreateValueCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); var before = 0; var after = 0; manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + before = 0; + after = 0; await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(1, before); Assert.Equal(1, after); From 16843f0438bc497e3c2778ae8ff70332517fdab1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 08:39:57 -0700 Subject: [PATCH 09/31] refactor(journaling): name the persistence operation type --- src/Orleans.Journaling/JournaledStateManager.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 0260219f6be..5bf644a328c 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -280,7 +280,7 @@ private async Task WorkLoop() storageActivity.SetTag(ActivityTagKeys.JournalStorageOperation, queueOperation); } - JournaledStateOperation? hookOperation = null; + JournaledStateOperation? operationType = null; var beforeHookRunning = false; var afterHookRunning = false; try @@ -292,7 +292,7 @@ private async Task WorkLoop() "The journaled state operation was queued before deletion reset its state."); } - hookOperation = workItem switch + operationType = workItem switch { AppendJournalWorkItem or WriteSnapshotWorkItem => workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCompactionRequested @@ -300,7 +300,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null }; - if (hookOperation is { } operation && _hooks is { Count: > 0 }) + if (operationType is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; var captureHook = await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); @@ -324,7 +324,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. - var isSnapshot = hookOperation == JournaledStateOperation.Snapshot; + var isSnapshot = operationType == JournaledStateOperation.Snapshot; var operationLabel = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; @@ -571,7 +571,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } } - if (hookOperation is { } completedOperation && _hooks is { Count: > 0 }) + if (operationType is { } completedOperation && _hooks is { Count: > 0 }) { afterHookRunning = true; await InvokeAfterHooksAsync(completedOperation, _shutdownCancellation.Token).ConfigureAwait(true); @@ -594,7 +594,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo continue; } - LogOperationHookFailed(_shared.Logger, exception, hookOperation!.Value, afterHookRunning); + LogOperationHookFailed(_shared.Logger, exception, operationType!.Value, afterHookRunning); storageActivity?.SetStatus(ActivityStatusCode.Error, "Journal operation hook failed."); if (recordQueueDuration && queueOperation is not null) { @@ -602,8 +602,8 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } workItem.SetException(afterHookRunning - ? new JournaledStatePostCommitException(hookOperation.Value, exception) - : new JournaledStatePreCommitException(hookOperation.Value, exception)); + ? new JournaledStatePostCommitException(operationType.Value, exception) + : new JournaledStatePreCommitException(operationType.Value, exception)); } catch (Exception exception) { From a3523e77ffbfa0e5d4b9305a1ad75315c0ba3586 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 09:15:59 -0700 Subject: [PATCH 10/31] refactor(journaling): select explicit snapshot work directly --- src/Orleans.Journaling/JournaledStateManager.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 5bf644a328c..a5a5314da64 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -294,8 +294,9 @@ private async Task WorkLoop() operationType = workItem switch { - AppendJournalWorkItem or WriteSnapshotWorkItem => - workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCompactionRequested + WriteSnapshotWorkItem => JournaledStateOperation.Snapshot, + AppendJournalWorkItem => + _migrationSnapshotRequired || _storage.IsCompactionRequested ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write, DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null From 363c04831cb4a698e4507b27ae1543b84f395e9a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 09:17:06 -0700 Subject: [PATCH 11/31] refactor(journaling): guard snapshot selection with when --- src/Orleans.Journaling/JournaledStateManager.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index a5a5314da64..d59c7546eb1 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -295,9 +295,8 @@ private async Task WorkLoop() operationType = workItem switch { WriteSnapshotWorkItem => JournaledStateOperation.Snapshot, - AppendJournalWorkItem => - _migrationSnapshotRequired || _storage.IsCompactionRequested - ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write, + AppendJournalWorkItem when _migrationSnapshotRequired || _storage.IsCompactionRequested => JournaledStateOperation.Snapshot, + AppendJournalWorkItem => JournaledStateOperation.Write, DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null }; From 07be377d0a55834846c85e86f15049e035be0431 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 09:17:56 -0700 Subject: [PATCH 12/31] refactor(journaling): centralize snapshot requirement --- src/Orleans.Journaling/JournaledStateManager.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index d59c7546eb1..36e2d3767ff 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -100,6 +100,8 @@ internal static IJournalStorage CreateStorage(IJournalStorageProvider storagePro internal string WriteJournalFormatKey => _shared.JournalFormatKey; + private bool IsSnapshotRequired => _migrationSnapshotRequired || _storage.IsCompactionRequested; + internal IServiceProvider ServiceProvider => _grainContext is { } context ? context.ActivationServices : _shared.ServiceProvider; public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? stateMachine) @@ -295,7 +297,7 @@ private async Task WorkLoop() operationType = workItem switch { WriteSnapshotWorkItem => JournaledStateOperation.Snapshot, - AppendJournalWorkItem when _migrationSnapshotRequired || _storage.IsCompactionRequested => JournaledStateOperation.Snapshot, + AppendJournalWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, AppendJournalWorkItem => JournaledStateOperation.Write, DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null @@ -951,7 +953,7 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def lock (_lock) { ThrowIfStateOperationsUnavailable(); - var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; + var isSnapshot = IsSnapshotRequired; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; pendingWrite = isSnapshot ? EnqueueOrGetPendingWorkItem(out didEnqueue) From 46f1e99d8a9f8382bbdaede3f434129bbdec27e6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 09:45:18 -0700 Subject: [PATCH 13/31] refactor(journaling): coalesce pending journal writes --- .../grains/journaling/runtime-behavior.md | 5 +- .../JournaledStateManager.cs | 58 ++++++------- src/Orleans.Journaling/README.md | 2 + .../StateManagerHookTests.cs | 84 +++++++++++++++++++ 4 files changed, 119 insertions(+), 30 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 4f739a24284..d23ad5f0141 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -54,7 +54,10 @@ Durable collections encode their operation before applying it to the in-memory c - **Append** adds the encoded operation batch atomically. - **Snapshot replacement** writes the current state of every registered stream and atomically publishes it as the new journal generation. -Concurrent calls made while the same kind of write is queued can share that queued operation. Each caller observes its completion or failure. Calls made after a storage operation starts are processed by a later operation. +Concurrent calls made while a write is queued share that queued operation. A snapshot requirement +from any caller upgrades the queued write to a snapshot and remains set until that operation runs. +Each caller observes its completion or failure. Calls made after a storage operation starts are +processed by a later operation with its own snapshot requirement. > [!IMPORTANT] > In-memory mutation is visible before storage acknowledgement. Return success to a caller only after the required `WriteStateAsync` completes. Recovery reconstructs durable state in a new activation. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 36e2d3767ff..f685a41c337 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -32,7 +32,8 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private ManagerState _state; private long _deletionGeneration; private Exception? _failure; - private bool _migrationSnapshotRequired; + // Recovery format changes require a snapshot; queued writes retain their own snapshot requests. + private bool _snapshotRequired; private int _disposed; private Task? _disposeTask; @@ -100,7 +101,7 @@ internal static IJournalStorage CreateStorage(IJournalStorageProvider storagePro internal string WriteJournalFormatKey => _shared.JournalFormatKey; - private bool IsSnapshotRequired => _migrationSnapshotRequired || _storage.IsCompactionRequested; + private bool IsSnapshotRequired => _snapshotRequired || _storage.IsCompactionRequested; internal IServiceProvider ServiceProvider => _grainContext is { } context ? context.ActivationServices : _shared.ServiceProvider; @@ -171,7 +172,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def _workLoop = Start(); } - task = EnqueueOrGetPendingWorkItem(out didEnqueue); + task = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; } if (didEnqueue) @@ -287,7 +288,7 @@ private async Task WorkLoop() var afterHookRunning = false; try { - if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem + if (workItem is WriteStateWorkItem && workItem.DeletionGeneration != Volatile.Read(ref _deletionGeneration)) { throw new InvalidOperationException( @@ -296,9 +297,8 @@ private async Task WorkLoop() operationType = workItem switch { - WriteSnapshotWorkItem => JournaledStateOperation.Snapshot, - AppendJournalWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, - AppendJournalWorkItem => JournaledStateOperation.Write, + WriteStateWorkItem write when write.SnapshotRequired || IsSnapshotRequired => JournaledStateOperation.Snapshot, + WriteStateWorkItem => JournaledStateOperation.Write, DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null }; @@ -321,8 +321,7 @@ private async Task WorkLoop() // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) { - case AppendJournalWorkItem: - case WriteSnapshotWorkItem: + case WriteStateWorkItem writeState: { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. @@ -332,9 +331,9 @@ private async Task WorkLoop() : JournalingInstruments.OperationAppend; if (isSnapshot) { - var compactionReason = workItem is WriteSnapshotWorkItem + var compactionReason = writeState.SnapshotRequired ? JournalingInstruments.CompactionReasonUserSnapshot - : _migrationSnapshotRequired + : _snapshotRequired ? JournalingInstruments.CompactionReasonMigration : JournalingInstruments.CompactionReasonStorageRequested; _shared.Instruments.OnCompactionTriggered(compactionReason); @@ -358,7 +357,7 @@ private async Task WorkLoop() RetireOrResurectStates(); } - if (_migrationSnapshotRequired) + if (_snapshotRequired) { ThrowIfMigrationBlockedByRetiredStates(); } @@ -506,7 +505,7 @@ private async Task WorkLoop() if (isSnapshot) { - _migrationSnapshotRequired = false; + _snapshotRequired = false; } } } @@ -764,7 +763,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = de lock (_lock) { ThrowIfStateOperationsUnavailable(); - task = EnqueueOrGetPendingWorkItem(out didEnqueue); + task = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; } if (didEnqueue) @@ -839,7 +838,7 @@ private async Task RecoverAsync(CancellationToken cancellationToken) private void ResetForRecovery() { _journalWriter.Reset(); - _migrationSnapshotRequired = false; + _snapshotRequired = false; _statesMap.Clear(); _statesMap[StateDirectory.Id] = _journalStreamDirectory; _statesMap[RetiredStateTracker.Id] = _retirementTracker; @@ -910,7 +909,7 @@ private void ProcessRecoveryBuffer(JournalBufferReader buffer, IJournalMetadata? { if (!string.Equals(journalFormatKey, _shared.JournalFormatKey, StringComparison.Ordinal)) { - _migrationSnapshotRequired = true; + _snapshotRequired = true; } var journalFormat = GetJournalFormat(journalFormatKey); @@ -955,9 +954,9 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def ThrowIfStateOperationsUnavailable(); var isSnapshot = IsSnapshotRequired; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; - pendingWrite = isSnapshot - ? EnqueueOrGetPendingWorkItem(out didEnqueue) - : EnqueueOrGetPendingWorkItem(out didEnqueue); + var workItem = EnqueueOrGetPendingWorkItem(out didEnqueue); + workItem.SnapshotRequired |= isSnapshot; + pendingWrite = workItem.Task; } if (didEnqueue) @@ -1058,17 +1057,17 @@ private async ValueTask DeleteStorageAsync(CancellationToken cancellationToken) } } - private Task EnqueueOrGetPendingWorkItem(out bool didEnqueue) + private TWorkItem EnqueueOrGetPendingWorkItem(out bool didEnqueue) where TWorkItem : WorkItem, new() { foreach (var workItem in _workQueue) { - if (workItem.GetType() != typeof(TWorkItem)) + if (workItem is not TWorkItem pending) { continue; } - if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem + if (workItem is WriteStateWorkItem && workItem.DeletionGeneration != _deletionGeneration) { continue; @@ -1077,7 +1076,7 @@ private Task EnqueueOrGetPendingWorkItem(out bool didEnqueue) workItem.RecordTraceContext(); workItem.AddCaller(); didEnqueue = false; - return workItem.Task; + return pending; } var newWorkItem = new TWorkItem(); @@ -1086,7 +1085,7 @@ private Task EnqueueOrGetPendingWorkItem(out bool didEnqueue) newWorkItem.RecordTraceContext(); _workQueue.Enqueue(newWorkItem); didEnqueue = true; - return newWorkItem.Task; + return newWorkItem; } private void BindState(string name, uint id) @@ -1293,17 +1292,18 @@ public void RecordTraceContext() private static string? GetStorageQueueOperation(WorkItem workItem) => workItem switch { - AppendJournalWorkItem => JournalingInstruments.OperationAppend, - WriteSnapshotWorkItem => JournalingInstruments.OperationSnapshot, + WriteStateWorkItem { SnapshotRequired: true } => JournalingInstruments.OperationSnapshot, + WriteStateWorkItem => JournalingInstruments.OperationAppend, DeleteStateWorkItem => JournalingInstruments.OperationDelete, _ => null }; private sealed class InitializeWorkItem : WorkItem; - private sealed class AppendJournalWorkItem : WorkItem; - - private sealed class WriteSnapshotWorkItem : WorkItem; + private sealed class WriteStateWorkItem : WorkItem + { + public bool SnapshotRequired { get; set; } + } private sealed class DeleteStateWorkItem : WorkItem; diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 2d7f0f36200..745c04d707a 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -283,6 +283,8 @@ After hooks run in list order after storage acknowledgement and state acknowledg They also run for a successful zero-byte write. Coalesced callers share the hooks for their actual operation. Features implement the before and after callbacks on their own hook, retaining their identity and operation-local bookkeeping there. +Queued writes coalesce into one operation. A snapshot requirement from any caller upgrades that +queued operation to a snapshot, even if the storage compaction signal clears before it runs. | Outcome | Owner and caller behavior | | --- | --- | diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 6c49ce2ede1..a9a455cd91d 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -773,6 +773,90 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() } } + [Fact] + public async Task Hooks_QueuedWritesCoalesceAndRetainSnapshotRequirement() + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + List before = []; + List after = []; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (operation, _) => before.Add(operation), + AfterOperation = (operation, _) => after.Add(operation) + }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var append = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.IsCompactionRequested = true; + state.Value = 2; + var snapshot = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.IsCompactionRequested = false; + state.Value = 3; + var anotherAppend = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + Assert.Empty(before); + await Task.WhenAll(append, snapshot, anotherAppend); + Assert.Equal([JournaledStateOperation.Snapshot], before); + Assert.Equal([JournaledStateOperation.Snapshot], after); + Assert.Empty(storage.Appends); + Assert.Single(storage.Replaces); + + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Value); + }); + } + + [Fact] + public async Task Hooks_SnapshotAcknowledgementPreservesLaterQueuedSnapshot() + { + var storage = new CapturingStorage { IsCompactionRequested = true, BlockNextReplace = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + List before = []; + List after = []; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (operation, _) => before.Add(operation), + AfterOperation = (operation, _) => after.Add(operation) + }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.ReplaceEntered.Task); + state.Value = 2; + var second = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.IsCompactionRequested = false; + Assert.Empty(storage.Replaces); + Assert.Equal([JournaledStateOperation.Snapshot], before); + Assert.Empty(after); + storage.ReleaseReplace.TrySetResult(); + await WaitFor(Task.WhenAll(first, second)); + Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], before); + Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], after); + Assert.Empty(storage.Appends); + Assert.Equal(2, storage.Replaces.Count); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Value); + } + finally + { + storage.ReleaseReplace.TrySetResult(); + } + } + [Fact] public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() { From 6b77e5578386c7d914b7619f0761e7981200205a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 13:08:13 -0700 Subject: [PATCH 14/31] refactor(journaling): consume pending snapshot requirements --- .../grains/journaling/runtime-behavior.md | 6 +- .../JournaledStateManager.cs | 142 +++++++++--------- src/Orleans.Journaling/README.md | 4 +- .../CodecRecoveryTests.cs | 24 +-- .../StateManagerHookTests.cs | 111 ++++++++++++++ 5 files changed, 199 insertions(+), 88 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index d23ad5f0141..36435a498e7 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -55,9 +55,11 @@ Durable collections encode their operation before applying it to the in-memory c - **Snapshot replacement** writes the current state of every registered stream and atomically publishes it as the new journal generation. Concurrent calls made while a write is queued share that queued operation. A snapshot requirement -from any caller upgrades the queued write to a snapshot and remains set until that operation runs. +from any caller upgrades pending work to a snapshot. The manager consumes the requirement when +selecting a write and restores it if a prerequisite hook fails. Storage acknowledgement leaves +newer pending requirements untouched. Each caller observes its completion or failure. Calls made after a storage operation starts are -processed by a later operation with its own snapshot requirement. +processed by a later operation. > [!IMPORTANT] > In-memory mutation is visible before storage acknowledgement. Return success to a caller only after the required `WriteStateAsync` completes. Recovery reconstructs durable state in a new activation. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index f685a41c337..89791f81443 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -32,7 +32,7 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private ManagerState _state; private long _deletionGeneration; private Exception? _failure; - // Recovery format changes require a snapshot; queued writes retain their own snapshot requests. + // A snapshot requirement not yet claimed by a write. Consume under the lock before preparation awaits. private bool _snapshotRequired; private int _disposed; private Task? _disposeTask; @@ -265,43 +265,44 @@ private async Task WorkLoop() } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); - var queueOperation = GetStorageQueueOperation(workItem); - var queueDuration = queueOperation is null - ? default - : _shared.TimeProvider.GetElapsedTime(workItem.EnqueuedTimestamp, processingTimestamp); + string? queueOperation = null; + TimeSpan queueDuration = default; var recordQueueDuration = workItem is DeleteStateWorkItem; - Activity? storageActivity = queueOperation is null - ? null - : ActivitySources.StorageGrainSource.StartActivity( - $"journal {queueOperation}", - ActivityKind.Internal, - workItem.TraceParent, - tags: null, - links: workItem.TraceLinks); - if (storageActivity is not null) - { - storageActivity.SetTag(ActivityTagKeys.JournalStorageOperation, queueOperation); - } + Activity? storageActivity = null; JournaledStateOperation? operationType = null; + string? compactionReason = null; var beforeHookRunning = false; var afterHookRunning = false; try { - if (workItem is WriteStateWorkItem - && workItem.DeletionGeneration != Volatile.Read(ref _deletionGeneration)) + lock (_lock) { - throw new InvalidOperationException( - "The journaled state operation was queued before deletion reset its state."); + if (workItem is WriteStateWorkItem + && workItem.DeletionGeneration != _deletionGeneration) + { + throw new InvalidOperationException( + "The journaled state operation was queued before deletion reset its state."); + } + + operationType = workItem switch + { + WriteStateWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, + WriteStateWorkItem => JournaledStateOperation.Write, + DeleteStateWorkItem => JournaledStateOperation.Delete, + _ => null + }; + if (operationType == JournaledStateOperation.Snapshot) + { + compactionReason = _snapshotRequired + ? JournalingInstruments.CompactionReasonUserSnapshot + : JournalingInstruments.CompactionReasonStorageRequested; + _snapshotRequired = false; + } } - operationType = workItem switch - { - WriteStateWorkItem write when write.SnapshotRequired || IsSnapshotRequired => JournaledStateOperation.Snapshot, - WriteStateWorkItem => JournaledStateOperation.Write, - DeleteStateWorkItem => JournaledStateOperation.Delete, - _ => null - }; + (queueOperation, queueDuration, storageActivity) = GetStorageQueueOperation(workItem, operationType, processingTimestamp); + if (operationType is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; @@ -321,7 +322,7 @@ private async Task WorkLoop() // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) { - case WriteStateWorkItem writeState: + case WriteStateWorkItem: { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. @@ -331,12 +332,7 @@ private async Task WorkLoop() : JournalingInstruments.OperationAppend; if (isSnapshot) { - var compactionReason = writeState.SnapshotRequired - ? JournalingInstruments.CompactionReasonUserSnapshot - : _snapshotRequired - ? JournalingInstruments.CompactionReasonMigration - : JournalingInstruments.CompactionReasonStorageRequested; - _shared.Instruments.OnCompactionTriggered(compactionReason); + _shared.Instruments.OnCompactionTriggered(compactionReason!); } _shared.Instruments.OnWriteCoalesced(operationLabel, workItem.CallerCount); var gatherStartTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -357,11 +353,6 @@ private async Task WorkLoop() RetireOrResurectStates(); } - if (_snapshotRequired) - { - ThrowIfMigrationBlockedByRetiredStates(); - } - // The map of state ids is itself stored as a durable state with the id 0. // This must be stored first, since it includes the identities of all other states, which are needed when replaying the journal. // If we removed retired states, this snapshot will persist that change. @@ -502,11 +493,6 @@ private async Task WorkLoop() { state.OnWriteCompleted(); } - - if (isSnapshot) - { - _snapshotRequired = false; - } } } @@ -589,6 +575,14 @@ private async Task WorkLoop() } catch (Exception exception) when (beforeHookRunning || afterHookRunning) { + if (beforeHookRunning && operationType == JournaledStateOperation.Snapshot) + { + lock (_lock) + { + _snapshotRequired = true; + } + } + if (beforeHookRunning && exception is OperationCanceledException && _shutdownCancellation.IsCancellationRequested) { workItem.TrySetCanceled(_shutdownCancellation.Token); @@ -726,20 +720,6 @@ private void RetireOrResurectStates() } } - private void ThrowIfMigrationBlockedByRetiredStates() - { - foreach (var state in _statesMap.Values) - { - if (state is RetiredStateMachine { PreservedEntries.Count: > 0 } retiredState) - { - throw new InvalidOperationException( - $"Cannot migrate journal to format key '{_shared.JournalFormatKey}' because stream " + - $"{retiredState.StreamId.Value} belongs to a state which is not currently registered. Register the state so it can be decoded " + - "and snapshotted in the configured format, or keep the previous journal format configured until the state can be retired."); - } - } - } - private JournalStreamWriter CreateJournalStreamWriter(JournalStreamId streamId) => _journalWriter.CreateJournalStreamWriter(streamId); private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWriter, bool isSnapshot, uint id, IStateMachine state) @@ -952,11 +932,9 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def lock (_lock) { ThrowIfStateOperationsUnavailable(); - var isSnapshot = IsSnapshotRequired; - operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; - var workItem = EnqueueOrGetPendingWorkItem(out didEnqueue); - workItem.SnapshotRequired |= isSnapshot; - pendingWrite = workItem.Task; + _snapshotRequired = IsSnapshotRequired; + operation = _snapshotRequired ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; + pendingWrite = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; } if (didEnqueue) @@ -1289,21 +1267,37 @@ public void RecordTraceContext() } } - private static string? GetStorageQueueOperation(WorkItem workItem) => - workItem switch + private (string? Operation, TimeSpan QueueDuration, Activity? Activity) GetStorageQueueOperation( + WorkItem workItem, + JournaledStateOperation? operationType, + long processingTimestamp) + { + var operation = operationType switch { - WriteStateWorkItem { SnapshotRequired: true } => JournalingInstruments.OperationSnapshot, - WriteStateWorkItem => JournalingInstruments.OperationAppend, - DeleteStateWorkItem => JournalingInstruments.OperationDelete, + JournaledStateOperation.Snapshot => JournalingInstruments.OperationSnapshot, + JournaledStateOperation.Write => JournalingInstruments.OperationAppend, + JournaledStateOperation.Delete => JournalingInstruments.OperationDelete, _ => null }; + if (operation is null) + { + return default; + } + + var queueDuration = _shared.TimeProvider.GetElapsedTime(workItem.EnqueuedTimestamp, processingTimestamp); + var activity = ActivitySources.StorageGrainSource.StartActivity( + $"journal {operation}", + ActivityKind.Internal, + workItem.TraceParent, + tags: null, + links: workItem.TraceLinks); + activity?.SetTag(ActivityTagKeys.JournalStorageOperation, operation); + return (operation, queueDuration, activity); + } private sealed class InitializeWorkItem : WorkItem; - private sealed class WriteStateWorkItem : WorkItem - { - public bool SnapshotRequired { get; set; } - } + private sealed class WriteStateWorkItem : WorkItem; private sealed class DeleteStateWorkItem : WorkItem; @@ -1427,8 +1421,6 @@ private sealed class RetiredStateMachine(JournalStreamId streamId) : IStateMachi public JournalStreamId StreamId { get; } = streamId; - public IReadOnlyList PreservedEntries => _preservedEntries; - void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => _preservedEntries.Add(new PreservedJournalEntry(entry.FormatKey, entry.Reader)); diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 745c04d707a..46b80082885 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -284,7 +284,9 @@ They also run for a successful zero-byte write. Coalesced callers share the hook operation. Features implement the before and after callbacks on their own hook, retaining their identity and operation-local bookkeeping there. Queued writes coalesce into one operation. A snapshot requirement from any caller upgrades that -queued operation to a snapshot, even if the storage compaction signal clears before it runs. +pending work to a snapshot, even if the storage compaction signal clears before it runs. +The manager consumes the pending requirement when selecting a write and restores it when a before +hook fails. Storage acknowledgement leaves newer snapshot requests untouched. | Outcome | Owner and caller behavior | | --- | --- | diff --git a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs index f1cd56ccf37..5217532cad1 100644 --- a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs +++ b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs @@ -293,27 +293,31 @@ public async Task Recovery_EmptyJournalWithStaleMetadata_WritesConfiguredFormat( Assert.Contains("""[8,["set","alpha",1]]""", Encoding.UTF8.GetString(storage.Segments.Single()), StringComparison.Ordinal); } - [Fact] - public async Task Migration_WithUnregisteredRetiredState_ThrowsClearError() + [Theory] + [InlineData(OrleansBinaryJournalFormat.JournalFormatKey, JsonLinesJournalFormat.JournalFormatKey)] + [InlineData(JsonLinesJournalFormat.JournalFormatKey, OrleansBinaryJournalFormat.JournalFormatKey)] + public async Task Migration_WithUnregisteredRetiredState_PreservesStorage(string sourceFormat, string targetFormat) { - var storage = new VolatileJournalStorage(OrleansBinaryJournalFormat.JournalFormatKey); - using var first = CreateFormatAwareTestSystem(storage, OrleansBinaryJournalFormat.JournalFormatKey); - var dict = CreateFormatAwareDictionary(first, OrleansBinaryJournalFormat.JournalFormatKey, "dict"); + var storage = new VolatileJournalStorage(sourceFormat); + using var first = CreateFormatAwareTestSystem(storage, sourceFormat); + var dict = CreateFormatAwareDictionary(first, sourceFormat, "dict"); await first.Lifecycle.OnStart(TestContext.Current.CancellationToken); dict.Add("alpha", 1); await first.Manager.WriteStateAsync(CancellationToken.None); - storage.SetConfiguredJournalFormatKey(JsonLinesJournalFormat.JournalFormatKey); - using var recovered = CreateFormatAwareTestSystem(storage, JsonLinesJournalFormat.JournalFormatKey); - var other = CreateFormatAwareDictionary(recovered, JsonLinesJournalFormat.JournalFormatKey, "other"); + var originalBytes = Assert.Single(storage.Segments).ToArray(); + storage.SetConfiguredJournalFormatKey(targetFormat); + using var recovered = CreateFormatAwareTestSystem(storage, targetFormat); + var other = CreateFormatAwareDictionary(recovered, targetFormat, "other"); await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); other.Add("beta", 2); var exception = await Assert.ThrowsAsync( () => recovered.Manager.WriteStateAsync(CancellationToken.None).AsTask()); - Assert.Contains("Cannot migrate journal", exception.Message, StringComparison.Ordinal); - Assert.Contains("not currently registered", exception.Message, StringComparison.Ordinal); + Assert.Contains("cannot append preserved entry", exception.Message, StringComparison.Ordinal); + Assert.Equal(sourceFormat, storage.StoredJournalFormatKey); + Assert.Equal(originalBytes, Assert.Single(storage.Segments)); } [Theory] diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index a9a455cd91d..b8426953b24 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -1,7 +1,10 @@ using System.Buffers; +using System.Collections.Concurrent; +using System.Diagnostics; using System.Reflection; using Microsoft.Extensions.DependencyInjection; using NSubstitute; +using Orleans.Diagnostics; using Orleans.Serialization; using Xunit; @@ -9,6 +12,58 @@ namespace Orleans.Journaling.Tests; public partial class StateManagerTests { + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Hooks_QueueActivityUsesSelectedOperationAndCompletesWithStorageOutcome(bool snapshot, bool storageFails) + { + using var parent = new Activity("journal queue test").SetIdFormat(ActivityIdFormat.W3C).Start(); + var stopped = new ConcurrentBag(); + using var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == ActivitySources.StorageActivitySourceName, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStopped = activity => + { + if (activity.ParentSpanId == parent.SpanId) + { + stopped.Add(activity); + } + } + }; + ActivitySource.AddActivityListener(listener); + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage + { + IsCompactionRequested = snapshot, + NextAppendException = storageFails && !snapshot ? failure : null, + NextReplaceException = storageFails && snapshot ? failure : null + }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + if (storageFails) + { + Assert.Same(failure, await Assert.ThrowsAsync(() => + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); + } + else + { + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + await manager.DisposeAsync(); + var activity = Assert.Single(stopped); + var operation = snapshot ? "snapshot" : "append"; + Assert.Equal($"journal {operation}", activity.OperationName); + Assert.Equal(operation, activity.GetTagItem(ActivityTagKeys.JournalStorageOperation)); + Assert.Equal(storageFails ? ActivityStatusCode.Error : ActivityStatusCode.Ok, activity.Status); + Assert.Equal(storageFails ? failure.Message : null, activity.StatusDescription); + } + [Fact] public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() { @@ -196,6 +251,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta Assert.Empty(storage.Replaces); Assert.Equal(0, storage.DeleteCount); Assert.Equal(0, after); + storage.IsCompactionRequested = false; fail = false; await InvokeAsync(); Assert.Equal(1, after); @@ -815,6 +871,61 @@ await context.Run(async () => }); } + [Fact] + public async Task Hooks_BeforeFailureRestoresSnapshotForAlreadyQueuedWrite() + { + var storage = new CapturingStorage { IsCompactionRequested = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var failure = new IOException("Snapshot prerequisite failed."); + List before = []; + List after = []; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperationAsync = async (operation, _) => + { + before.Add(operation); + if (before.Count == 1) + { + entered.TrySetResult(); + await release.Task; + throw failure; + } + }, + AfterOperation = (operation, _) => after.Add(operation) + }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + storage.IsCompactionRequested = false; + state.Value = 2; + var retry = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => WaitFor(first)); + Assert.Same(failure, caught.InnerException); + await WaitFor(retry); + Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], before); + Assert.Equal([JournaledStateOperation.Snapshot], after); + Assert.Empty(storage.Appends); + Assert.Single(storage.Replaces); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Value); + } + finally + { + release.TrySetResult(); + } + } + [Fact] public async Task Hooks_SnapshotAcknowledgementPreservesLaterQueuedSnapshot() { From 158fcf35da8209f2ca9bb643773ee477b92951dc Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 13:58:10 -0700 Subject: [PATCH 15/31] refactor(journaling): order hook preparation by registration --- .../grains/journaling/runtime-behavior.md | 9 ++- .../IJournaledStateCaptureHook.cs | 15 ---- src/Orleans.Journaling/IJournaledStateHook.cs | 7 +- .../IJournaledStateManager.cs | 6 +- .../JournaledStateManager.Hooks.cs | 33 ++------- .../JournaledStateManager.cs | 8 +-- src/Orleans.Journaling/README.md | 10 +-- .../Orleans.Journaling/Orleans.Journaling.cs | 4 -- .../StateManagerHookTests.cs | 68 +++++++++---------- 9 files changed, 53 insertions(+), 107 deletions(-) delete mode 100644 src/Orleans.Journaling/IJournaledStateCaptureHook.cs diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 36435a498e7..c8b8554f3ce 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -83,11 +83,10 @@ The hook list then remains fixed through recovery, persistence and deletion. Every journal owner implementation provides this list and runs its registered callbacks at the operation boundaries below. Delegating owners forward the list to the inner owner. -Each actual append, snapshot, or deletion runs ordinary before callbacks in list order, outside -the manager lock. At most one supplies the -final prerequisite. Its before callback runs last, and the work loop awaits it directly before -synchronous capture or storage deletion. Prerequisites cover changes staged during asynchronous -preparation, including the final hook's own I/O wait. Preserve operation-local bookkeeping for +Each actual append, snapshot, or deletion runs before callbacks in reverse list order, outside +the manager lock. Register the final prerequisite at index zero so its callback runs last before +synchronous capture or storage deletion. Prerequisites remain valid through capture or deletion, covering changes staged +during asynchronous preparation and while the work loop resumes. Preserve operation-local bookkeeping for the captured batch separately from changes staged later. Storage acknowledgement and registered-state acknowledgement or reset precede after callbacks. diff --git a/src/Orleans.Journaling/IJournaledStateCaptureHook.cs b/src/Orleans.Journaling/IJournaledStateCaptureHook.cs deleted file mode 100644 index cd6d0b42693..00000000000 --- a/src/Orleans.Journaling/IJournaledStateCaptureHook.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Orleans.Journaling; - -/// -/// Establishes the final prerequisite immediately before journal capture or deletion. -/// -/// -/// An owner admits at most one capture hook in . -/// Its before callback runs after all ordinary before callbacks. The work loop awaits it directly, -/// then captures state or starts deletion without a further asynchronous phase. -/// Its prerequisites cover changes staged while its own I/O awaited. After callbacks retain -/// normal list order. Features inspect and deduplicate this registration using the same hook list. -/// -public interface IJournaledStateCaptureHook : IJournaledStateHook -{ -} diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index 29d245799d6..95b8cb4b3b1 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -19,9 +19,10 @@ public enum JournaledStateOperation /// Participates in the prerequisites and completion of actual journal operations. /// /// -/// Ordinary before hooks run in list order on the owner's logical execution context, outside its lock. -/// An optional runs last immediately before capture or deletion. -/// Prerequisites must cover changes staged during asynchronous preparation. After hooks run after +/// Before hooks run in reverse list order on the owner's logical execution context, outside its lock. +/// The hook at index zero supplies the final prerequisite before capture or deletion. +/// Prerequisites remain valid until capture or deletion, covering changes staged during asynchronous +/// preparation and while the journal work loop resumes. After hooks run in forward list order after /// storage acknowledgement and state acknowledgement or reset, including successful writes which /// produce no storage bytes. Hooks retain operation-local data across these boundaries and keep later /// pending changes separate. Hook implementations must complete without calling initialization, diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 77149dc9500..c5f169d6d3d 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -17,9 +17,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// /// Inspect, add, remove, and deduplicate hooks during setup, before initialization begins. - /// Registration remains fixed for this manager's initialized lifetime. Ordinary before hooks - /// and all after hooks execute in list order. The optional single - /// supplies the final prerequisite. + /// Registration remains fixed for this manager's initialized lifetime. Before hooks execute in + /// reverse list order; after hooks execute in forward list order. Register the final capture + /// prerequisite at index zero so it runs last before capture or deletion. /// Registration is independent of state-machine registration and persists through recovery and deletion. /// Every implementation provides this list and invokes its hooks at the documented operation boundaries. /// diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 1c9502a772f..789a06ead9f 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -18,24 +18,15 @@ public IList Hooks } } - private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { - IJournaledStateCaptureHook? captureHook = null; - for (var i = 0; i < _hooks!.Count; i++) + for (var i = _hooks!.Count - 1; i >= 0; i--) { - var hook = _hooks[i]; - if (hook is IJournaledStateCaptureHook capture) - { - captureHook = capture; - continue; - } - cancellationToken.ThrowIfCancellationRequested(); - await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); + await _hooks[i].BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); } cancellationToken.ThrowIfCancellationRequested(); - return captureHook; } private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) @@ -71,7 +62,7 @@ protected override void InsertItem(int index, IJournaledStateHook item) lock (owner._lock) { owner.EnsureRegistrationAllowed(); - ValidateHook(item, replacingIndex: -1); + ArgumentNullException.ThrowIfNull(item); base.InsertItem(index, item); } } @@ -81,7 +72,7 @@ protected override void SetItem(int index, IJournaledStateHook item) lock (owner._lock) { owner.EnsureRegistrationAllowed(); - ValidateHook(item, index); + ArgumentNullException.ThrowIfNull(item); base.SetItem(index, item); } } @@ -104,19 +95,5 @@ protected override void ClearItems() } } - private void ValidateHook(IJournaledStateHook item, int replacingIndex) - { - ArgumentNullException.ThrowIfNull(item); - if (item is IJournaledStateCaptureHook) - { - for (var i = 0; i < Count; i++) - { - if (i != replacingIndex && this[i] is IJournaledStateCaptureHook) - { - throw new InvalidOperationException("A journal owner supports one final capture prerequisite hook. Inspect and deduplicate the hook list before registration."); - } - } - } - } } } diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 89791f81443..6790b90e51f 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -306,13 +306,7 @@ private async Task WorkLoop() if (operationType is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; - var captureHook = await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); - - if (captureHook is not null) - { - // Await the final prerequisite in this frame so capture follows its completion directly. - await captureHook.BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); - } + await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); _shutdownCancellation.Token.ThrowIfCancellationRequested(); beforeHookRunning = false; diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 46b80082885..51eb00c3f6b 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -269,14 +269,14 @@ of the ambient grain context, including when the caller subsequently enrolls the `IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` registrations. Features register and deduplicate their hooks during setup before initialization begins, alongside state-machine registration. The hook list then remains fixed through recovery, -persistence and whole-journal deletion. The standard manager admits at most one `IJournaledStateCaptureHook`. +persistence and whole-journal deletion. Every journal owner implementation provides the hook list and invokes registered callbacks at the operation boundaries described below. Delegating owners forward the list to their inner owner. -For each actual write, snapshot, or deletion, ordinary before hooks run in list order outside -the manager lock. The capture hook runs last: the work loop awaits it directly, then synchronously -captures the registered states or starts deletion. Prerequisites cover all changes staged during -asynchronous preparation, including changes arriving while the capture hook awaits its own I/O. +For each actual write, snapshot, or deletion, before hooks run in reverse list order outside +the manager lock. Register the final prerequisite at index zero so it runs last before the work loop +captures the registered states or starts deletion. Prerequisites remain valid through capture or +deletion, covering changes staged during asynchronous preparation and while the work loop resumes. Keep operation-local bookkeeping for the captured batch separate from later pending changes. After hooks run in list order after storage acknowledgement and state acknowledgement or reset. diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 76ff42769dd..7502cdeb322 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -219,10 +219,6 @@ public partial interface IDurableValue T? Value { get; set; } } - public partial interface IJournaledStateCaptureHook : IJournaledStateHook - { - } - public partial interface IJournaledStateHook { System.Threading.Tasks.ValueTask AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index b8426953b24..649a3fac00e 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -359,7 +359,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - manager.Hooks.Add(new CaptureTestHook( + manager.Hooks.Add(new DelegateTestHook( (_, _) => { Assert.Equal(3, state["late"]); @@ -407,10 +407,8 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar Assert.Equal(3, recovered["late"]); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(bool captureHook) + [Fact] + public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion() { var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; @@ -433,9 +431,7 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b Assert.Single(storage.Appends); completed.TrySetResult(); }; - manager.Hooks.Add(captureHook - ? new CaptureTestHook(before, after) - : new DelegateTestHook { BeforeOperationAsync = before, AfterOperation = after }); + manager.Hooks.Add(new DelegateTestHook(before, after)); await manager.InitializeAsync(TestContext.Current.CancellationToken); using var caller = new CancellationTokenSource(); state["business"] = 1; @@ -468,21 +464,31 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b } [Fact] - public async Task Hooks_RegistrationEnforcesSingleCapturePrerequisite() + public async Task Hooks_BeforeCallbacksRunInReverseAndAfterCallbacksRunInListOrder() { await using var manager = CreateTestSystem().Manager; - var first = new CaptureTestHook((_, _) => default); - var second = new CaptureTestHook((_, _) => default); - var ordinary = new DelegateTestHook(); - manager.Hooks.Add(first); - manager.Hooks.Add(ordinary); - Assert.Throws(() => manager.Hooks.Add(second)); - Assert.Throws(() => manager.Hooks[1] = second); - Assert.Equal(new IJournaledStateHook[] { first, ordinary }, manager.Hooks); - manager.Hooks[0] = second; - Assert.Same(second, manager.Hooks[0]); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (_, _) => events.Add("second-before"), + AfterOperation = (_, _) => events.Add("second-after") + }); + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (_, _) => events.Add("third-before"), + AfterOperation = (_, _) => events.Add("third-after") + }); + manager.Hooks.Insert(0, new DelegateTestHook + { + BeforeOperation = (_, _) => events.Add("first-before"), + AfterOperation = (_, _) => events.Add("first-after") + }); await manager.InitializeAsync(TestContext.Current.CancellationToken); await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal( + ["third-before", "second-before", "first-before", "capture", "ack", "first-after", "second-after", "third-after"], + events); } [Fact] @@ -495,7 +501,7 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var after = 0; - manager.Hooks.Add(new CaptureTestHook( + manager.Hooks.Add(new DelegateTestHook( async (operation, token) => { if (!deleting) @@ -1102,25 +1108,13 @@ public void Hooks_FailureOutcomesRoundTripWithTypeAndOriginalCause(bool after) Assert.Equal(original.Message, copy.Message); } - private sealed class CaptureTestHook( - Func before, - Action? after = null) : IJournaledStateCaptureHook - { - public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => - before(operation, cancellationToken); - - public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - after?.Invoke(operation, cancellationToken); - return default; - } - } - - private sealed class DelegateTestHook : IJournaledStateHook + private sealed class DelegateTestHook( + Func? before = null, + Action? after = null) : IJournaledStateHook { public Action? BeforeOperation { get; init; } - public Func? BeforeOperationAsync { get; init; } - public Action? AfterOperation { get; init; } + public Func? BeforeOperationAsync { get; init; } = before; + public Action? AfterOperation { get; init; } = after; public Func? AfterOperationAsync { get; init; } ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) From 470ec86827a3d478470e0735445ff33ead1010ef Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:08:33 -0700 Subject: [PATCH 16/31] refactor(journaling): decouple hook registration freeze --- .../grains/journaling/runtime-behavior.md | 2 + .../IJournaledStateManager.cs | 2 + .../JournaledStateManager.Hooks.cs | 47 ++++++++++--------- .../JournaledStateManager.cs | 2 + src/Orleans.Journaling/README.md | 2 + .../StateManagerHookTests.cs | 46 ++++++++++++++---- 6 files changed, 69 insertions(+), 32 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index c8b8554f3ce..1533355c815 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -80,6 +80,8 @@ Activation-scoped features coordinate prerequisites and completion through allocated list of registrations. Register and deduplicate feature hooks during setup before initialization begins, alongside state machines. The hook list then remains fixed through recovery, persistence and deletion. +Keep registration sequential with initialization and shutdown. Both transitions make the list +read-only, including when the hook list is first accessed afterward. Every journal owner implementation provides this list and runs its registered callbacks at the operation boundaries below. Delegating owners forward the list to the inner owner. diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index c5f169d6d3d..f6c25ecc739 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -17,6 +17,8 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// /// Inspect, add, remove, and deduplicate hooks during setup, before initialization begins. + /// The caller keeps hook registration sequential with initialization and shutdown. + /// Initialization or shutdown makes the list read-only. /// Registration remains fixed for this manager's initialized lifetime. Before hooks execute in /// reverse list order; after hooks execute in forward list order. Register the final capture /// prerequisite at index zero so it runs last before capture or deletion. diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 789a06ead9f..f9b61c08791 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -13,7 +13,7 @@ public IList Hooks { lock (_lock) { - return _hooks ??= new(this); + return _hooks ??= new(_workLoop is not null || _shutdownCancellation.IsCancellationRequested); } } } @@ -55,45 +55,46 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, } } - private sealed class HookCollection(JournaledStateManager owner) : Collection + private sealed class HookCollection(bool isReadOnly) : Collection, IList { + private bool _isReadOnly = isReadOnly; + + bool ICollection.IsReadOnly => _isReadOnly; + + public void Freeze() => _isReadOnly = true; + protected override void InsertItem(int index, IJournaledStateHook item) { - lock (owner._lock) - { - owner.EnsureRegistrationAllowed(); - ArgumentNullException.ThrowIfNull(item); - base.InsertItem(index, item); - } + ThrowIfReadOnly(); + ArgumentNullException.ThrowIfNull(item); + base.InsertItem(index, item); } protected override void SetItem(int index, IJournaledStateHook item) { - lock (owner._lock) - { - owner.EnsureRegistrationAllowed(); - ArgumentNullException.ThrowIfNull(item); - base.SetItem(index, item); - } + ThrowIfReadOnly(); + ArgumentNullException.ThrowIfNull(item); + base.SetItem(index, item); } protected override void RemoveItem(int index) { - lock (owner._lock) - { - owner.EnsureRegistrationAllowed(); - base.RemoveItem(index); - } + ThrowIfReadOnly(); + base.RemoveItem(index); } protected override void ClearItems() { - lock (owner._lock) + ThrowIfReadOnly(); + base.ClearItems(); + } + + private void ThrowIfReadOnly() + { + if (_isReadOnly) { - owner.EnsureRegistrationAllowed(); - base.ClearItems(); + throw new NotSupportedException("Journal operation hook registrations are read-only."); } } - } } diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 6790b90e51f..6cdf5ebf108 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -169,6 +169,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def ThrowIfFenced(); if (_workLoop is null) { + _hooks?.Freeze(); _workLoop = Start(); } @@ -1095,6 +1096,7 @@ private async Task StopAsync(CancellationToken cancellationToken) AggregateException? cancellationFailure = null; lock (_lock) { + _hooks?.Freeze(); try { _shutdownCancellation.Cancel(); diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 51eb00c3f6b..df2af6af17a 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -270,6 +270,8 @@ of the ambient grain context, including when the caller subsequently enrolls the registrations. Features register and deduplicate their hooks during setup before initialization begins, alongside state-machine registration. The hook list then remains fixed through recovery, persistence and whole-journal deletion. +Keep hook registration sequential with initialization and shutdown. Either lifecycle transition +makes the list read-only; its collection owns this state independently of the journal owner. Every journal owner implementation provides the hook list and invokes registered callbacks at the operation boundaries described below. Delegating owners forward the list to their inner owner. diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 649a3fac00e..b0c1a84b39e 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -73,6 +73,7 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() var hook = new DelegateTestHook(); var hooks = manager.Hooks; + Assert.False(hooks.IsReadOnly); hooks.Add(hook); Assert.True(hooks.Contains(hook)); Assert.Same(hooks, manager.Hooks); @@ -85,12 +86,38 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Same(hooks, manager.Hooks); Assert.Same(hook, Assert.Single(hooks)); - Assert.Throws(() => hooks.Clear()); + Assert.True(hooks.IsReadOnly); + Assert.Throws(() => hooks.Clear()); await using var withoutHooks = CreateTestSystem().Manager; await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); await withoutHooks.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Null(field.GetValue(withoutHooks)); + Assert.True(withoutHooks.Hooks.IsReadOnly); + Assert.Throws(() => withoutHooks.Hooks.Add(hook)); + } + + [Fact] + public async Task Hooks_ShutdownMakesSetupRegistrationsReadOnlyBeforeInitialization() + { + var manager = CreateTestSystem().Manager; + var hooks = manager.Hooks; + var hook = new DelegateTestHook(); + hooks.Add(hook); + Assert.False(hooks.IsReadOnly); + + await manager.DisposeAsync(); + + Assert.True(hooks.IsReadOnly); + Assert.Same(hooks, manager.Hooks); + Assert.Same(hook, Assert.Single(hooks)); + Assert.Throws(() => hooks.Add(hook)); + Assert.Throws(() => hooks.Clear()); + + var withoutHooks = CreateTestSystem().Manager; + await withoutHooks.DisposeAsync(); + Assert.True(withoutHooks.Hooks.IsReadOnly); + Assert.Throws(() => withoutHooks.Hooks.Add(hook)); } [Fact] @@ -110,19 +137,20 @@ public async Task Hooks_RegistrationClosesBeforeRecoveryAndRemainsClosedAfterFai var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); try { - Assert.Throws(() => hooks.Add(new DelegateTestHook())); - Assert.Throws(() => hooks.Insert(0, new DelegateTestHook())); - Assert.Throws(() => hooks[0] = new DelegateTestHook()); - Assert.Throws(() => hooks.Remove(hook)); - Assert.Throws(() => hooks.Clear()); + Assert.True(hooks.IsReadOnly); + Assert.Throws(() => hooks.Add(new DelegateTestHook())); + Assert.Throws(() => hooks.Insert(0, new DelegateTestHook())); + Assert.Throws(() => hooks[0] = new DelegateTestHook()); + Assert.Throws(() => hooks.Remove(hook)); + Assert.Throws(() => hooks.Clear()); await WaitFor(entered.Task); Assert.False(initialization.IsCompleted); release.TrySetResult(); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); - Assert.Throws(() => hooks.Clear()); + Assert.Throws(() => hooks.Clear()); fail = false; await manager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Throws(() => hooks.Clear()); + Assert.Throws(() => hooks.Clear()); Assert.Same(hook, Assert.Single(hooks)); } finally @@ -587,7 +615,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() await Assert.ThrowsAnyAsync(() => queued); Assert.True(drained); Assert.Empty(storage.Appends); - Assert.Throws(() => manager.Hooks.Clear()); + Assert.Throws(() => manager.Hooks.Clear()); } finally { From 065c3da253dfb47f1b809ab6adf7c188f75cd77f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:43:52 -0700 Subject: [PATCH 17/31] feat(journaling)!: serialize dynamic hook registration --- .../grains/journaling/runtime-behavior.md | 19 +- .../CompatibilitySuppressions.xml | 28 ++ src/Orleans.Journaling/IJournaledStateHook.cs | 2 + .../IJournaledStateManager.cs | 39 +- .../JournaledStateManager.Hooks.cs | 100 ++-- .../JournaledStateManager.cs | 94 +++- src/Orleans.Journaling/README.md | 20 +- .../Orleans.Journaling/Orleans.Journaling.cs | 4 +- .../DurableListJournalBenchmarks.cs | 19 +- .../JournaledJobShardDiscoveryTests.cs | 6 +- .../DurableCollectionDirectWriteTests.cs | 19 +- .../DurableListDirectWriteTests.cs | 19 +- .../KeyedJournalingRegistrationTests.cs | 6 +- .../StateManagerHookTests.cs | 457 ++++++++++++++---- 14 files changed, 675 insertions(+), 157 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 1533355c815..7230ffd7315 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -77,16 +77,19 @@ staged mutations from other calls. Activation-scoped features coordinate prerequisites and completion through . The owner exposes a stable, lazily -allocated list of registrations. Register and -deduplicate feature hooks during setup before initialization begins, alongside state machines. -The hook list then remains fixed through recovery, persistence and deletion. -Keep registration sequential with initialization and shutdown. Both transitions make the list -read-only, including when the hook list is first accessed afterward. -Every journal owner implementation provides this list and runs its registered callbacks at the -operation boundaries below. Delegating owners forward the list to the inner owner. +allocated read-only view of registrations. +Use and + to change the set. Idle changes +apply synchronously; busy changes queue between complete operations, after current after callbacks. +The hook set stays stable throughout each operation. Queued changes also separate coalesced writes, +so writes admitted after a change observe that change. +Cancellation ends only the caller's wait while the manager owns admitted changes. +Every owner implements the registration methods and callback boundaries; delegating owners forward +them to their inner owner. Synchronous state callbacks can request a queued change and observe its +completion outside the callback. Standalone callers coordinate view inspection with changes. Each actual append, snapshot, or deletion runs before callbacks in reverse list order, outside -the manager lock. Register the final prerequisite at index zero so its callback runs last before +the manager lock. Add the final prerequisite at index zero so its callback runs last before synchronous capture or storage deletion. Prerequisites remain valid through capture or deletion, covering changes staged during asynchronous preparation and while the work loop resumes. Preserve operation-local bookkeeping for the captured batch separately from changes staged later. diff --git a/src/Orleans.Journaling/CompatibilitySuppressions.xml b/src/Orleans.Journaling/CompatibilitySuppressions.xml index ed87a41fc9b..3b379c6860f 100644 --- a/src/Orleans.Journaling/CompatibilitySuppressions.xml +++ b/src/Orleans.Journaling/CompatibilitySuppressions.xml @@ -239,6 +239,13 @@ lib/net8.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.AddHookAsync(Orleans.Journaling.IJournaledStateHook,System.Nullable{System.Int32},System.Threading.CancellationToken) + lib/net10.0/Orleans.Journaling.dll + lib/net10.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.RegisterStateMachine(System.String,Orleans.Journaling.IStateMachine) @@ -246,6 +253,13 @@ lib/net10.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.RemoveHookAsync(Orleans.Journaling.IJournaledStateHook,System.Threading.CancellationToken) + lib/net10.0/Orleans.Journaling.dll + lib/net10.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.TryGetStateMachine(System.String,Orleans.Journaling.IStateMachine@) @@ -281,6 +295,13 @@ lib/net10.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.AddHookAsync(Orleans.Journaling.IJournaledStateHook,System.Nullable{System.Int32},System.Threading.CancellationToken) + lib/net8.0/Orleans.Journaling.dll + lib/net8.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.RegisterStateMachine(System.String,Orleans.Journaling.IStateMachine) @@ -288,6 +309,13 @@ lib/net8.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.RemoveHookAsync(Orleans.Journaling.IJournaledStateHook,System.Threading.CancellationToken) + lib/net8.0/Orleans.Journaling.dll + lib/net8.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.TryGetStateMachine(System.String,Orleans.Journaling.IStateMachine@) diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index 95b8cb4b3b1..10c8d5eb9f1 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -29,6 +29,8 @@ public enum JournaledStateOperation /// persistence, or disposal on their own journal owner. Those operations are serialized behind the /// current operation, so awaiting them from a hook would create a circular dependency. /// Dependencies between journal owners must also remain acyclic. +/// Callbacks can request a hook registration change, observing its queued completion outside the +/// callback rather than awaiting work which follows the current operation. /// public interface IJournaledStateHook { diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index f6c25ecc739..761eb1d50cd 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -13,19 +13,46 @@ namespace Orleans.Journaling; public interface IJournaledStateManager : IAsyncDisposable { /// - /// Gets the mutable, lazily allocated list of journal operation hooks. + /// Gets the read-only view of registered journal operation hooks. /// /// - /// Inspect, add, remove, and deduplicate hooks during setup, before initialization begins. - /// The caller keeps hook registration sequential with initialization and shutdown. - /// Initialization or shutdown makes the list read-only. - /// Registration remains fixed for this manager's initialized lifetime. Before hooks execute in + /// Inspect hooks on the owner's logical execution context, coordinating standalone inspection + /// with registration changes. Use and + /// to change registrations. Before hooks execute in /// reverse list order; after hooks execute in forward list order. Register the final capture /// prerequisite at index zero so it runs last before capture or deletion. /// Registration is independent of state-machine registration and persists through recovery and deletion. /// Every implementation provides this list and invokes its hooks at the documented operation boundaries. /// - IList Hooks { get; } + IReadOnlyList Hooks { get; } + + /// + /// Adds a hook before the next persistence operation which follows this registration. + /// + /// The hook to register. + /// The insertion index, or to append. + /// The token used to cancel the caller's wait. + /// A completion representing application of the registration. + /// + /// Idle registration applies synchronously. Otherwise it is queued between owned operations, + /// after the current operation's after hooks finish. Queued registration is a write-coalescing barrier. + /// Cancellation ends the caller's wait while admitted registration remains owned by the manager. + /// Synchronous state callbacks can request registration and observe its completion outside the callback. + /// Await registration before staging changes whose safety depends on the added hook. + /// + ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default); + + /// + /// Removes a hook between persistence operations. + /// + /// The registered hook to remove. + /// The token used to cancel the caller's wait. + /// A completion representing application of the removal. + /// The hook is not registered when the removal is applied. + /// + /// Removal has the same ownership and ordering guarantees as . + /// + ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default); /// ValueTask IAsyncDisposable.DisposeAsync() => default; diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index f9b61c08791..8cb1fe842cf 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -1,4 +1,4 @@ -using System.Collections.ObjectModel; +using System.Collections; using System.Runtime.ExceptionServices; namespace Orleans.Journaling; @@ -7,17 +7,61 @@ internal partial class JournaledStateManager { private HookCollection? _hooks; - public IList Hooks + public IReadOnlyList Hooks { get { lock (_lock) { - return _hooks ??= new(_workLoop is not null || _shutdownCancellation.IsCancellationRequested); + return _hooks ??= new(); } } } + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + MutateHookAsync(hook, index, remove: false, cancellationToken); + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + MutateHookAsync(hook, index: null, remove: true, cancellationToken); + + private ValueTask MutateHookAsync(IJournaledStateHook hook, int? index, bool remove, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(hook); + cancellationToken.ThrowIfCancellationRequested(); + Task task; + lock (_lock) + { + ThrowIfFenced(); + ObjectDisposedException.ThrowIf(_disposed != 0, this); + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + if (_workLoop is null || (!_workItemRunning && _workQueue.Count == 0)) + { + ApplyHookMutation(hook, index, remove); + return default; + } + + var workItem = new HookMutationWorkItem(hook, index, remove); + _workQueue.Enqueue(workItem); + task = workItem.Task; + } + + _workSignal.Signal(); + return new(task.WaitAsync(cancellationToken)); + } + + private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove) + { + var hooks = _hooks ??= new(); + if (remove) + { + hooks.Remove(hook); + } + else + { + hooks.Insert(index, hook); + } + } + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { for (var i = _hooks!.Count - 1; i >= 0; i--) @@ -55,46 +99,30 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, } } - private sealed class HookCollection(bool isReadOnly) : Collection, IList + private sealed class HookCollection : IReadOnlyList { - private bool _isReadOnly = isReadOnly; - - bool ICollection.IsReadOnly => _isReadOnly; + private readonly List _items = []; - public void Freeze() => _isReadOnly = true; + public int Count => _items.Count; + public IJournaledStateHook this[int index] => _items[index]; + public IEnumerator GetEnumerator() => _items.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - protected override void InsertItem(int index, IJournaledStateHook item) - { - ThrowIfReadOnly(); - ArgumentNullException.ThrowIfNull(item); - base.InsertItem(index, item); - } + public void Insert(int? index, IJournaledStateHook hook) => _items.Insert(index ?? _items.Count, hook); - protected override void SetItem(int index, IJournaledStateHook item) + public void Remove(IJournaledStateHook hook) { - ThrowIfReadOnly(); - ArgumentNullException.ThrowIfNull(item); - base.SetItem(index, item); - } - - protected override void RemoveItem(int index) - { - ThrowIfReadOnly(); - base.RemoveItem(index); - } - - protected override void ClearItems() - { - ThrowIfReadOnly(); - base.ClearItems(); - } - - private void ThrowIfReadOnly() - { - if (_isReadOnly) + if (!_items.Remove(hook)) { - throw new NotSupportedException("Journal operation hook registrations are read-only."); + throw new InvalidOperationException("The journal operation hook is not registered."); } } } + + private sealed class HookMutationWorkItem(IJournaledStateHook hook, int? index, bool remove) : WorkItem + { + public IJournaledStateHook Hook { get; } = hook; + public int? Index { get; } = index; + public bool Remove { get; } = remove; + } } diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 6cdf5ebf108..1f3e6c18d62 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -29,6 +29,8 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private readonly StateDirectory _journalStreamDirectory; private readonly RetiredStateTracker _retirementTracker; private Task? _workLoop; + // Under _lock, covers recovery and dequeued work through the last after callback. + private bool _workItemRunning; private ManagerState _state; private long _deletionGeneration; private Exception? _failure; @@ -169,7 +171,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def ThrowIfFenced(); if (_workLoop is null) { - _hooks?.Freeze(); + _workItemRunning = true; _workLoop = Start(); } @@ -195,6 +197,11 @@ private async Task WorkLoop() await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.ForceYielding); while (!_shutdownCancellation.IsCancellationRequested) { + lock (_lock) + { + _workItemRunning = true; + } + try { await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); @@ -216,9 +223,17 @@ private async Task WorkLoop() lock (_lock) { FaultQueuedWorkItemsUnderLock(exception); + _workItemRunning = false; } } } + finally + { + lock (_lock) + { + _workItemRunning = false; + } + } // Signals can remain from the failed attempt. Retry only for newly queued initialization work. while (true) @@ -231,7 +246,17 @@ private async Task WorkLoop() lock (_lock) { - if (_workQueue.Count > 0) + var retryRequested = false; + foreach (var workItem in _workQueue) + { + if (workItem is InitializeWorkItem) + { + retryRequested = true; + break; + } + } + + if (retryRequested) { break; } @@ -263,6 +288,7 @@ private async Task WorkLoop() } workItem = dequeuedWorkItem; + _workItemRunning = true; } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -317,6 +343,15 @@ private async Task WorkLoop() // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) { + case HookMutationWorkItem mutation: + { + lock (_lock) + { + ApplyHookMutation(mutation.Hook, mutation.Index, mutation.Remove); + } + break; + } + case WriteStateWorkItem: { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. @@ -566,7 +601,11 @@ private async Task WorkLoop() } storageActivity?.SetStatus(ActivityStatusCode.Ok); - workItem.SetResult(); + lock (_lock) + { + _workItemRunning = false; + workItem.SetResult(); + } } catch (Exception exception) when (beforeHookRunning || afterHookRunning) { @@ -591,9 +630,22 @@ private async Task WorkLoop() _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: afterHookRunning); } - workItem.SetException(afterHookRunning - ? new JournaledStatePostCommitException(operationType.Value, exception) - : new JournaledStatePreCommitException(operationType.Value, exception)); + lock (_lock) + { + _workItemRunning = false; + workItem.SetException(afterHookRunning + ? new JournaledStatePostCommitException(operationType.Value, exception) + : new JournaledStatePreCommitException(operationType.Value, exception)); + } + } + catch (Exception exception) when (workItem is HookMutationWorkItem) + { + LogHookMutationFailed(_shared.Logger, exception); + lock (_lock) + { + _workItemRunning = false; + workItem.SetException(exception); + } } catch (Exception exception) { @@ -627,6 +679,11 @@ private async Task WorkLoop() } finally { + lock (_lock) + { + _workItemRunning = false; + } + storageActivity?.Dispose(); } } @@ -1033,9 +1090,17 @@ private async ValueTask DeleteStorageAsync(CancellationToken cancellationToken) private TWorkItem EnqueueOrGetPendingWorkItem(out bool didEnqueue) where TWorkItem : WorkItem, new() { + TWorkItem? pending = null; foreach (var workItem in _workQueue) { - if (workItem is not TWorkItem pending) + if (workItem is HookMutationWorkItem) + { + // Later callers must not coalesce into work which precedes a registration change. + pending = null; + continue; + } + + if (pending is not null || workItem is not TWorkItem candidate) { continue; } @@ -1046,8 +1111,13 @@ private TWorkItem EnqueueOrGetPendingWorkItem(out bool didEnqueue) continue; } - workItem.RecordTraceContext(); - workItem.AddCaller(); + pending = candidate; + } + + if (pending is not null) + { + pending.RecordTraceContext(); + pending.AddCaller(); didEnqueue = false; return pending; } @@ -1096,7 +1166,6 @@ private async Task StopAsync(CancellationToken cancellationToken) AggregateException? cancellationFailure = null; lock (_lock) { - _hooks?.Freeze(); try { _shutdownCancellation.Cancel(); @@ -1442,6 +1511,11 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } Message = "Journal {Operation} hook failed. Persistence completed: {Committed}.")] private static partial void LogOperationHookFailed(ILogger logger, Exception exception, JournaledStateOperation operation, bool committed); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal operation hook registration change failed.")] + private static partial void LogHookMutationFailed(ILogger logger, Exception exception); + [LoggerMessage( Level = LogLevel.Error, Message = "Journal shutdown cancellation callback failed; owned operations are drained before resources are released.")] diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index df2af6af17a..4e43c8c8334 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -266,17 +266,21 @@ of the ambient grain context, including when the caller subsequently enrolls the ## Journal operation hooks -`IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` -registrations. Features register and deduplicate their hooks during setup before initialization -begins, alongside state-machine registration. The hook list then remains fixed through recovery, -persistence and whole-journal deletion. -Keep hook registration sequential with initialization and shutdown. Either lifecycle transition -makes the list read-only; its collection owns this state independently of the journal owner. +`IJournaledStateManager.Hooks` is a lazily allocated, stable read-only view of `IJournaledStateHook` +registrations. Use `AddHookAsync(hook, index)` and `RemoveHookAsync(hook)` to change registrations. +Changes apply synchronously when idle, or queue between complete journal operations while busy. +Each operation keeps its hook set through preparation, capture, storage ACK and after callbacks. +Queued changes are write-coalescing barriers: writes admitted afterward observe the changed set. +Caller cancellation ends only its wait for an admitted change; the manager retains ownership. +Await registration before staging changes whose safety depends on the new hook. Every journal owner implementation provides the hook list and invokes registered callbacks at -the operation boundaries described below. Delegating owners forward the list to their inner owner. +the operation boundaries described below. Delegating owners forward registration methods and the +read-only view to their inner owner. Synchronous state callbacks can request a queued registration +and observe completion outside the callback. Standalone callers coordinate view inspection with +registration changes. For each actual write, snapshot, or deletion, before hooks run in reverse list order outside -the manager lock. Register the final prerequisite at index zero so it runs last before the work loop +the manager lock. Add the final prerequisite with `index: 0` so it runs last before the work loop captures the registered states or starts deletion. Prerequisites remain valid through capture or deletion, covering changes staged during asynchronous preparation and while the work loop resumes. Keep operation-local bookkeeping for the captured batch separate from later pending changes. diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 7502cdeb322..969a8c672d1 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -227,13 +227,15 @@ public partial interface IJournaledStateHook public partial interface IJournaledStateManager : System.IAsyncDisposable { - System.Collections.Generic.IList Hooks { get; } + System.Collections.Generic.IReadOnlyList Hooks { get; } long PendingWriteByteCount { get; } + System.Threading.Tasks.ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken = default); void RegisterStateMachine(string name, IStateMachine stateMachine); + System.Threading.Tasks.ValueTask RemoveHookAsync(IJournaledStateHook hook, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); bool TryGetStateMachine(string name, out IStateMachine? stateMachine); System.Threading.Tasks.ValueTask WriteStateAsync(System.Threading.CancellationToken cancellationToken = default); diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index f5931b7e85b..fa2f4658687 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -126,7 +126,24 @@ private void ReplayEncodedJournalData() private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter buffer, JournalStreamId streamId) : IJournaledStateManager { - public IList Hooks { get; } = []; + private readonly List _hooks = []; + + public IReadOnlyList Hooks => _hooks; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index c6829908e9a..c06a4576ec7 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -727,7 +727,11 @@ public IJournaledStateManager CreateStandalone(JournalId journalId) private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public IList Hooks => inner.Hooks; + public IReadOnlyList Hooks => inner.Hooks; + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + inner.AddHookAsync(hook, index, cancellationToken); + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + inner.RemoveHookAsync(hook, cancellationToken); public int DisposeCalls { get; private set; } public long PendingWriteByteCount => inner.PendingWriteByteCount; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 4e46d24cde7..94fc6b4ff09 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -193,7 +193,24 @@ public void Collections_UseDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { - public IList Hooks { get; } = []; + private readonly List _hooks = []; + + public IReadOnlyList Hooks => _hooks; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index bb3648d69fa..bd4176f6403 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -92,7 +92,24 @@ public void Add_UsesDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { - public IList Hooks { get; } = []; + private readonly List _hooks = []; + + public IReadOnlyList Hooks => _hooks; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index 97b65742677..5d91cafdd8b 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -746,7 +746,11 @@ public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public IList Hooks => inner.Hooks; + public IReadOnlyList Hooks => inner.Hooks; + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + inner.AddHookAsync(hook, index, cancellationToken); + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + inner.RemoveHookAsync(hook, cancellationToken); public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); public void RegisterStateMachine(string name, IStateMachine state) => inner.RegisterStateMachine(name, state); public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index b0c1a84b39e..4166f54ab9a 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -65,93 +65,125 @@ public async Task Hooks_QueueActivityUsesSelectedOperationAndCompletesWithStorag } [Fact] - public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() + public async Task Hooks_IdleRegistrationAppliesSynchronouslyAndExposesStableView() { await using var manager = CreateTestSystem().Manager; + _ = new DurableValue("state", manager, CreateValueCodec()); var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; Assert.Null(field.GetValue(manager)); var hook = new DelegateTestHook(); + Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); var hooks = manager.Hooks; - Assert.False(hooks.IsReadOnly); - hooks.Add(hook); - Assert.True(hooks.Contains(hook)); + Assert.False(hooks is IList); + Assert.Contains(hook, hooks); Assert.Same(hooks, manager.Hooks); Assert.Same(hook, Assert.Single(hooks)); - Assert.True(hooks.Remove(hook)); + Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); Assert.Empty(manager.Hooks); - Assert.Throws(() => hooks.Add(null!)); - hooks.Add(hook); + await Assert.ThrowsAsync(() => manager.AddHookAsync(null!, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.True(manager.AddHookAsync(hook, index: 0, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Same(hooks, manager.Hooks); Assert.Same(hook, Assert.Single(hooks)); - Assert.True(hooks.IsReadOnly); - Assert.Throws(() => hooks.Clear()); + Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); await using var withoutHooks = CreateTestSystem().Manager; await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); await withoutHooks.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Null(field.GetValue(withoutHooks)); - Assert.True(withoutHooks.Hooks.IsReadOnly); - Assert.Throws(() => withoutHooks.Hooks.Add(hook)); } [Fact] - public async Task Hooks_ShutdownMakesSetupRegistrationsReadOnlyBeforeInitialization() + public async Task Hooks_ShutdownRejectsRegistrationAndRemoval() { var manager = CreateTestSystem().Manager; - var hooks = manager.Hooks; var hook = new DelegateTestHook(); - hooks.Add(hook); - Assert.False(hooks.IsReadOnly); + await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); + var hooks = manager.Hooks; await manager.DisposeAsync(); - Assert.True(hooks.IsReadOnly); Assert.Same(hooks, manager.Hooks); Assert.Same(hook, Assert.Single(hooks)); - Assert.Throws(() => hooks.Add(hook)); - Assert.Throws(() => hooks.Clear()); + await Assert.ThrowsAsync(() => manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + } - var withoutHooks = CreateTestSystem().Manager; - await withoutHooks.DisposeAsync(); - Assert.True(withoutHooks.Hooks.IsReadOnly); - Assert.Throws(() => withoutHooks.Hooks.Add(hook)); + [Fact] + public async Task Hooks_RegistrationDuringRecoveryAppliesBeforeLaterPersistence() + { + var entered = NewSignal(); + var release = NewSignal(); + var storage = Substitute.For(); + storage.ReadAsync(Arg.Any(), Arg.Any()) + .Returns(call => ReadAsync(call.Arg(), call.Arg())); + await using var manager = CreateTestSystem(storage).Manager; + var before = 0; + var after = 0; + var hook = new DelegateTestHook + { + BeforeOperation = (_, _) => before++, + AfterOperation = (_, _) => after++ + }; + var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var registration = manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + Assert.False(initialization.IsCompleted); + Assert.False(registration.IsCompleted); + Assert.Empty(manager.Hooks); + release.TrySetResult(); + await WaitFor(Task.WhenAll(initialization, registration)); + Assert.Same(hook, Assert.Single(manager.Hooks)); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + } + finally + { + release.TrySetResult(); + } + + async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + consumer.Complete(metadata: null); + } } [Fact] - public async Task Hooks_RegistrationClosesBeforeRecoveryAndRemainsClosedAfterFailure() + public async Task Hooks_QueuedRegistrationReportsRecoveryFailureAndIdleRegistrationDoesNotRetryRecovery() { var entered = NewSignal(); var release = NewSignal(); var storage = Substitute.For(); var failure = new IOException("Recovery failed."); var fail = true; + var reads = 0; storage.ReadAsync(Arg.Any(), Arg.Any()) .Returns(call => ReadAsync(call.Arg(), call.Arg())); await using var manager = CreateTestSystem(storage).Manager; - var hook = new DelegateTestHook(); - var hooks = manager.Hooks; - hooks.Add(hook); var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var failedRegistration = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); try { - Assert.True(hooks.IsReadOnly); - Assert.Throws(() => hooks.Add(new DelegateTestHook())); - Assert.Throws(() => hooks.Insert(0, new DelegateTestHook())); - Assert.Throws(() => hooks[0] = new DelegateTestHook()); - Assert.Throws(() => hooks.Remove(hook)); - Assert.Throws(() => hooks.Clear()); await WaitFor(entered.Task); - Assert.False(initialization.IsCompleted); release.TrySetResult(); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); - Assert.Throws(() => hooks.Clear()); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(failedRegistration))); + Assert.Empty(manager.Hooks); + var hook = new DelegateTestHook(); + Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); + Assert.Equal(1, reads); fail = false; await manager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Throws(() => hooks.Clear()); - Assert.Same(hook, Assert.Single(hooks)); + Assert.Equal(2, reads); + Assert.Same(hook, Assert.Single(manager.Hooks)); } finally { @@ -160,17 +192,275 @@ public async Task Hooks_RegistrationClosesBeforeRecoveryAndRemainsClosedAfterFai async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) { + reads++; entered.TrySetResult(); await release.Task.WaitAsync(token); if (fail) { throw failure; } - consumer.Complete(metadata: null); } } + [Theory] + [InlineData("before")] + [InlineData("storage")] + [InlineData("after")] + public async Task Hooks_BusyMutationWaitsForCompleteOperationAndSurvivesCallerCancellation(string phase) + { + var storage = new CapturingStorage { BlockNextAppend = phase == "storage" }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var oldBefore = 0; + var oldAfter = 0; + var newBefore = 0; + var newAfter = 0; + var original = new DelegateTestHook + { + BeforeOperationAsync = async (_, _) => + { + oldBefore++; + if (phase == "before") + { + entered.TrySetResult(); + await release.Task; + } + }, + AfterOperationAsync = async (_, _) => + { + oldAfter++; + if (phase == "after") + { + entered.TrySetResult(); + await release.Task; + } + } + }; + var replacement = new DelegateTestHook + { + BeforeOperation = (_, _) => newBefore++, + AfterOperation = (_, _) => newAfter++ + }; + await manager.AddHookAsync(original, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(phase == "storage" ? storage.BlockedAppendStarted.Task : entered.Task); + using var caller = new CancellationTokenSource(); + var added = manager.AddHookAsync(replacement, index: 0, caller.Token).AsTask(); + var removed = manager.RemoveHookAsync(original, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + state.Value = 2; + var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(added.IsCompleted); + Assert.False(removed.IsCompleted); + Assert.Same(original, Assert.Single(manager.Hooks)); + Assert.Equal(0, newBefore); + Assert.Equal(0, newAfter); + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => added); + Assert.Same(original, Assert.Single(manager.Hooks)); + + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + await WaitFor(Task.WhenAll(first, removed, next)); + Assert.Same(replacement, Assert.Single(manager.Hooks)); + Assert.Equal(1, oldBefore); + Assert.Equal(1, oldAfter); + Assert.Equal(1, newBefore); + Assert.Equal(1, newAfter); + Assert.Equal(2, state.Value); + } + finally + { + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_QueuedMutationSeparatesWriteCoalescingGroups() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + var oldBefore = 0; + var oldAfter = 0; + var newBefore = 0; + var newAfter = 0; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => oldBefore++, + AfterOperation = (_, _) => oldAfter++ + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.BlockedAppendStarted.Task); + var beforeChange = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var change = manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => newBefore++, + AfterOperation = (_, _) => newAfter++ + }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var afterChange = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var coalesced = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await WaitFor(Task.WhenAll(first, beforeChange, change, afterChange, coalesced)); + Assert.Equal(3, state.CaptureCount); + Assert.Equal(3, storage.Appends.Count); + Assert.Equal(3, oldBefore); + Assert.Equal(3, oldAfter); + Assert.Equal(1, newBefore); + Assert.Equal(1, newAfter); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ResetCanQueueRegistrationWithoutChangingCurrentAfterHooks() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + Task? registration = null; + var newBefore = 0; + var newAfter = 0; + state.ResetAction = () => + { + if (state.ResetCount == 2) + { + registration = manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => newBefore++, + AfterOperation = (_, _) => newAfter++ + }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + } + }; + manager.RegisterStateMachine("state", state); + await manager.AddHookAsync(new DelegateTestHook + { + AfterOperation = (operation, _) => + { + if (operation == JournaledStateOperation.Delete) + { + Assert.NotNull(registration); + Assert.False(registration.IsCompleted); + Assert.Equal(0, newBefore); + Assert.Equal(0, newAfter); + } + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.DeleteStateAsync(TestContext.Current.CancellationToken); + Assert.NotNull(registration); + await WaitFor(registration); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, newBefore); + Assert.Equal(1, newAfter); + } + + [Fact] + public async Task Hooks_FailedQueuedMutationReportsErrorAndLeavesOwnerUsable() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.BlockedAppendStarted.Task); + var invalid = manager.AddHookAsync(new DelegateTestHook(), index: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var valid = new DelegateTestHook(); + var accepted = manager.AddHookAsync(valid, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await Assert.ThrowsAsync(() => WaitFor(invalid)); + await WaitFor(Task.WhenAll(first, accepted)); + Assert.Same(valid, Assert.Single(manager.Hooks)); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_StorageFailureFaultsQueuedRegistrationWithOriginalCause() + { + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage { BlockNextAppend = true, NextAppendException = failure }; + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.BlockedAppendStarted.Task); + var registration = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(write))); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(registration))); + Assert.Empty(manager.Hooks); + var fenced = await Assert.ThrowsAsync(() => + manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask()); + Assert.Same(failure, fenced.InnerException); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ShutdownCancelsQueuedMutationWithoutChangingCommittedOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + var entered = NewSignal(); + var release = NewSignal(); + var original = new DelegateTestHook + { + AfterOperationAsync = async (_, _) => + { + entered.TrySetResult(); + await release.Task; + } + }; + await manager.AddHookAsync(original, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var added = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var removed = manager.RemoveHookAsync(original, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.False(shutdown.IsCompleted); + release.TrySetResult(); + await WaitFor(Task.WhenAll(write, shutdown)); + await Assert.ThrowsAnyAsync(() => WaitFor(added)); + await Assert.ThrowsAnyAsync(() => WaitFor(removed)); + Assert.Same(original, Assert.Single(manager.Hooks)); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + } + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -183,7 +473,7 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; CancellationToken ownedToken = default; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (operation, token) => { @@ -213,7 +503,7 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo await Task.Yield(); events.Add("after-async"); } - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -254,7 +544,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta var failure = new IOException("Prerequisite failed."); var fail = true; var after = 0; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => { @@ -264,7 +554,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta } }, AfterOperation = (_, _) => after++ - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var pendingBytes = manager.PendingWriteByteCount; @@ -307,7 +597,7 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); var fail = true; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => { @@ -316,8 +606,8 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo throw first; } } - }); - manager.Hooks.Add(new DelegateTestHook + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperationAsync = async (_, _) => { @@ -328,8 +618,8 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo throw second; } } - }); - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete @@ -366,7 +656,7 @@ public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() await using var manager = CreateTestSystem(storage).Manager; var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); var after = 0; - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => after++ }); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); @@ -387,7 +677,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - manager.Hooks.Add(new DelegateTestHook( + await manager.AddHookAsync(new DelegateTestHook( (_, _) => { Assert.Equal(3, state["late"]); @@ -395,8 +685,8 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar events.Add("capture-prerequisite"); return default; }, - (_, _) => events.Add("capture-after"))); - manager.Hooks.Add(new DelegateTestHook + (_, _) => events.Add("capture-after")), cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -406,7 +696,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar events.Add("ordinary-complete"); }, AfterOperation = (_, _) => events.Add("ordinary-after") - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -459,7 +749,7 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion() Assert.Single(storage.Appends); completed.TrySetResult(); }; - manager.Hooks.Add(new DelegateTestHook(before, after)); + await manager.AddHookAsync(new DelegateTestHook(before, after), cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); using var caller = new CancellationTokenSource(); state["business"] = 1; @@ -497,21 +787,21 @@ public async Task Hooks_BeforeCallbacksRunInReverseAndAfterCallbacksRunInListOrd await using var manager = CreateTestSystem().Manager; List events = []; manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => events.Add("second-before"), AfterOperation = (_, _) => events.Add("second-after") - }); - manager.Hooks.Add(new DelegateTestHook + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => events.Add("third-before"), AfterOperation = (_, _) => events.Add("third-after") - }); - manager.Hooks.Insert(0, new DelegateTestHook + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => events.Add("first-before"), AfterOperation = (_, _) => events.Add("first-after") - }); + }, index: 0, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal( @@ -529,7 +819,7 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var after = 0; - manager.Hooks.Add(new DelegateTestHook( + await manager.AddHookAsync(new DelegateTestHook( async (operation, token) => { if (!deleting) @@ -554,7 +844,7 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() Assert.Equal(0, state.Value); Assert.Equal(1, storage.DeleteCount); after++; - })); + }), cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; await manager.WriteStateAsync(TestContext.Current.CancellationToken); @@ -586,7 +876,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var drained = false; var failure = new IOException("Cancellation callback failed."); - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -595,7 +885,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() await release.Task; drained = true; } - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -615,7 +905,8 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() await Assert.ThrowsAnyAsync(() => queued); Assert.True(drained); Assert.Empty(storage.Appends); - Assert.Throws(() => manager.Hooks.Clear()); + await Assert.ThrowsAsync(() => + manager.RemoveHookAsync(Assert.Single(manager.Hooks), cancellationToken: TestContext.Current.CancellationToken).AsTask()); } finally { @@ -640,11 +931,11 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra entered.TrySetResult(); await release.Task; }; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperationAsync = after ? null : callback, AfterOperationAsync = after ? callback : null - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -711,11 +1002,11 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt manager.RegisterStateMachine("state", state); CancellationTokenRegistration registration = default; var after = 0; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), AfterOperation = (_, _) => after++ - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -778,7 +1069,7 @@ public async Task Hooks_ShutdownCallbackReentrySharesDisposalCompletion() var release = NewSignal(); Task? reenteredDisposal = null; var callbackCount = 0; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { AfterOperationAsync = async (_, token) => { @@ -790,7 +1081,7 @@ public async Task Hooks_ShutdownCallbackReentrySharesDisposalCompletion() entered.TrySetResult(); await release.Task; } - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -824,7 +1115,7 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); List events = []; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { AfterOperationAsync = async (_, token) => { @@ -835,9 +1126,9 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() events.Add("first"); throw first; } - }); - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw second }); - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => throw second }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -874,11 +1165,11 @@ await context.Run(async () => var state = new DurableValue("state", manager, CreateValueCodec()); List before = []; List after = []; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (operation, _) => before.Add(operation), AfterOperation = (operation, _) => after.Add(operation) - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var append = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -916,7 +1207,7 @@ public async Task Hooks_BeforeFailureRestoresSnapshotForAlreadyQueuedWrite() var failure = new IOException("Snapshot prerequisite failed."); List before = []; List after = []; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperationAsync = async (operation, _) => { @@ -929,7 +1220,7 @@ public async Task Hooks_BeforeFailureRestoresSnapshotForAlreadyQueuedWrite() } }, AfterOperation = (operation, _) => after.Add(operation) - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -968,11 +1259,11 @@ public async Task Hooks_SnapshotAcknowledgementPreservesLaterQueuedSnapshot() var state = new DurableValue("state", manager, CreateValueCodec()); List before = []; List after = []; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (operation, _) => before.Add(operation), AfterOperation = (operation, _) => after.Add(operation) - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -1010,7 +1301,7 @@ public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); var before = 0; var after = 0; - manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["first"] = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -1044,7 +1335,7 @@ public async Task Hooks_StateChangesDuringAfterCallbackPersistInNextOperation() var entered = NewSignal(); var release = NewSignal(); var after = 0; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { AfterOperationAsync = async (_, _) => { @@ -1054,7 +1345,7 @@ public async Task Hooks_StateChangesDuringAfterCallbackPersistInNextOperation() await release.Task; } } - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["first"] = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -1099,7 +1390,7 @@ public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() var state = new DurableValue("state", manager, CreateValueCodec()); var before = 0; var after = 0; - manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; await manager.WriteStateAsync(TestContext.Current.CancellationToken); From 5ab481ca2658b660d88d65da463ea1b771454ff5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 17:25:32 -0700 Subject: [PATCH 18/31] refactor(journaling)!: keep operation hooks private --- .../grains/journaling/runtime-behavior.md | 15 ++--- .../CompatibilitySuppressions.xml | 14 ----- .../IJournaledStateManager.cs | 24 +++----- .../JournaledStateManager.Hooks.cs | 41 ++----------- .../JournaledStateManager.cs | 25 +++----- src/Orleans.Journaling/README.md | 15 +++-- .../Orleans.Journaling/Orleans.Journaling.cs | 2 - .../DurableListJournalBenchmarks.cs | 2 - .../JournaledJobShardDiscoveryTests.cs | 1 - .../DurableCollectionDirectWriteTests.cs | 2 - .../DurableListDirectWriteTests.cs | 2 - .../KeyedJournalingRegistrationTests.cs | 1 - .../StateManagerHookTests.cs | 60 +++++++++---------- 13 files changed, 68 insertions(+), 136 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 7230ffd7315..70d297fcb49 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -75,18 +75,19 @@ staged mutations from other calls. ## Journal operation hooks -Activation-scoped features coordinate prerequisites and completion through -. The owner exposes a stable, lazily -allocated read-only view of registrations. -Use and - to change the set. Idle changes -apply synchronously; busy changes queue between complete operations, after current after callbacks. +Features coordinate prerequisites and completion by registering + callbacks through + and +. +Registration completes synchronously before initialization, supporting constructors. +After initialization begins, idle changes apply synchronously; busy changes queue between +complete operations, after current after callbacks. The hook set stays stable throughout each operation. Queued changes also separate coalesced writes, so writes admitted after a change observe that change. Cancellation ends only the caller's wait while the manager owns admitted changes. Every owner implements the registration methods and callback boundaries; delegating owners forward them to their inner owner. Synchronous state callbacks can request a queued change and observe its -completion outside the callback. Standalone callers coordinate view inspection with changes. +completion outside the callback. Each actual append, snapshot, or deletion runs before callbacks in reverse list order, outside the manager lock. Add the final prerequisite at index zero so its callback runs last before diff --git a/src/Orleans.Journaling/CompatibilitySuppressions.xml b/src/Orleans.Journaling/CompatibilitySuppressions.xml index 3b379c6860f..5e86005fc7b 100644 --- a/src/Orleans.Journaling/CompatibilitySuppressions.xml +++ b/src/Orleans.Journaling/CompatibilitySuppressions.xml @@ -281,13 +281,6 @@ lib/net10.0/Orleans.Journaling.dll true - - CP0006 - P:Orleans.Journaling.IJournaledStateManager.Hooks - lib/net10.0/Orleans.Journaling.dll - lib/net10.0/Orleans.Journaling.dll - true - CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey @@ -337,13 +330,6 @@ lib/net8.0/Orleans.Journaling.dll true - - CP0006 - P:Orleans.Journaling.IJournaledStateManager.Hooks - lib/net8.0/Orleans.Journaling.dll - lib/net8.0/Orleans.Journaling.dll - true - CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 761eb1d50cd..5c18162570f 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -12,20 +12,6 @@ namespace Orleans.Journaling; /// public interface IJournaledStateManager : IAsyncDisposable { - /// - /// Gets the read-only view of registered journal operation hooks. - /// - /// - /// Inspect hooks on the owner's logical execution context, coordinating standalone inspection - /// with registration changes. Use and - /// to change registrations. Before hooks execute in - /// reverse list order; after hooks execute in forward list order. Register the final capture - /// prerequisite at index zero so it runs last before capture or deletion. - /// Registration is independent of state-machine registration and persists through recovery and deletion. - /// Every implementation provides this list and invokes its hooks at the documented operation boundaries. - /// - IReadOnlyList Hooks { get; } - /// /// Adds a hook before the next persistence operation which follows this registration. /// @@ -34,8 +20,14 @@ public interface IJournaledStateManager : IAsyncDisposable /// The token used to cancel the caller's wait. /// A completion representing application of the registration. /// - /// Idle registration applies synchronously. Otherwise it is queued between owned operations, - /// after the current operation's after hooks finish. Queued registration is a write-coalescing barrier. + /// Registration completes synchronously before initialization begins, supporting constructor registration. + /// After initialization begins, idle registration applies synchronously; otherwise it is queued + /// between owned operations, after the current operation's after hooks finish. + /// Before hooks execute in reverse registration order; after hooks execute in forward order. + /// Register the final capture prerequisite at index zero so it runs last before capture or deletion. + /// Registration is independent of state-machine registration and persists through recovery and deletion. + /// Every implementation provides registration and invokes hooks at the documented operation boundaries. + /// Queued registration is a write-coalescing barrier. /// Cancellation ends the caller's wait while admitted registration remains owned by the manager. /// Synchronous state callbacks can request registration and observe its completion outside the callback. /// Await registration before staging changes whose safety depends on the added hook. diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 8cb1fe842cf..751ece9e5a3 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -1,22 +1,10 @@ -using System.Collections; using System.Runtime.ExceptionServices; namespace Orleans.Journaling; internal partial class JournaledStateManager { - private HookCollection? _hooks; - - public IReadOnlyList Hooks - { - get - { - lock (_lock) - { - return _hooks ??= new(); - } - } - } + private List? _hooks; public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => MutateHookAsync(hook, index, remove: false, cancellationToken); @@ -54,11 +42,14 @@ private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove var hooks = _hooks ??= new(); if (remove) { - hooks.Remove(hook); + if (!hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } } else { - hooks.Insert(index, hook); + hooks.Insert(index ?? hooks.Count, hook); } } @@ -99,26 +90,6 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, } } - private sealed class HookCollection : IReadOnlyList - { - private readonly List _items = []; - - public int Count => _items.Count; - public IJournaledStateHook this[int index] => _items[index]; - public IEnumerator GetEnumerator() => _items.GetEnumerator(); - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - - public void Insert(int? index, IJournaledStateHook hook) => _items.Insert(index ?? _items.Count, hook); - - public void Remove(IJournaledStateHook hook) - { - if (!_items.Remove(hook)) - { - throw new InvalidOperationException("The journal operation hook is not registered."); - } - } - } - private sealed class HookMutationWorkItem(IJournaledStateHook hook, int? index, bool remove) : WorkItem { public IJournaledStateHook Hook { get; } = hook; diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 1f3e6c18d62..fb634b4f169 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -202,6 +202,7 @@ private async Task WorkLoop() _workItemRunning = true; } + Exception? recoveryFailure = null; try { await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); @@ -214,24 +215,18 @@ private async Task WorkLoop() } catch (Exception exception) { - try - { - LogErrorProcessingWorkItems(_shared.Logger, exception); - } - finally - { - lock (_lock) - { - FaultQueuedWorkItemsUnderLock(exception); - _workItemRunning = false; - } - } + recoveryFailure = exception; + LogErrorProcessingWorkItems(_shared.Logger, exception); } finally { lock (_lock) { _workItemRunning = false; + if (recoveryFailure is not null) + { + FaultQueuedWorkItemsUnderLock(recoveryFailure); + } } } @@ -345,10 +340,8 @@ private async Task WorkLoop() { case HookMutationWorkItem mutation: { - lock (_lock) - { - ApplyHookMutation(mutation.Hook, mutation.Index, mutation.Remove); - } + // The work loop owns the hook set while _workItemRunning is true. + ApplyHookMutation(mutation.Hook, mutation.Index, mutation.Remove); break; } diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 4e43c8c8334..ebba60fad84 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -266,18 +266,17 @@ of the ambient grain context, including when the caller subsequently enrolls the ## Journal operation hooks -`IJournaledStateManager.Hooks` is a lazily allocated, stable read-only view of `IJournaledStateHook` -registrations. Use `AddHookAsync(hook, index)` and `RemoveHookAsync(hook)` to change registrations. -Changes apply synchronously when idle, or queue between complete journal operations while busy. +Use `IJournaledStateManager.AddHookAsync(hook, index)` and `RemoveHookAsync(hook)` to register +`IJournaledStateHook` callbacks. Registration completes synchronously before initialization, +supporting constructors. After initialization begins, changes apply synchronously when idle, +or queue between complete journal operations while busy. Each operation keeps its hook set through preparation, capture, storage ACK and after callbacks. Queued changes are write-coalescing barriers: writes admitted afterward observe the changed set. Caller cancellation ends only its wait for an admitted change; the manager retains ownership. Await registration before staging changes whose safety depends on the new hook. -Every journal owner implementation provides the hook list and invokes registered callbacks at -the operation boundaries described below. Delegating owners forward registration methods and the -read-only view to their inner owner. Synchronous state callbacks can request a queued registration -and observe completion outside the callback. Standalone callers coordinate view inspection with -registration changes. +Every journal owner implementation provides registration and invokes callbacks at the operation +boundaries described below. Delegating owners forward registration methods to their inner owner. +Synchronous state callbacks can request a queued registration and observe completion outside the callback. For each actual write, snapshot, or deletion, before hooks run in reverse list order outside the manager lock. Add the final prerequisite with `index: 0` so it runs last before the work loop diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 969a8c672d1..cf2ecdf72e7 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -227,8 +227,6 @@ public partial interface IJournaledStateHook public partial interface IJournaledStateManager : System.IAsyncDisposable { - System.Collections.Generic.IReadOnlyList Hooks { get; } - long PendingWriteByteCount { get; } System.Threading.Tasks.ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, System.Threading.CancellationToken cancellationToken = default); diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index fa2f4658687..5cb36fbffcf 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -128,8 +128,6 @@ private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter bu { private readonly List _hooks = []; - public IReadOnlyList Hooks => _hooks; - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) { _hooks.Insert(index ?? _hooks.Count, hook); diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index c06a4576ec7..39e4213c5c1 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -727,7 +727,6 @@ public IJournaledStateManager CreateStandalone(JournalId journalId) private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public IReadOnlyList Hooks => inner.Hooks; public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => inner.AddHookAsync(hook, index, cancellationToken); public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 94fc6b4ff09..ede9edb2f56 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -195,8 +195,6 @@ private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJourn { private readonly List _hooks = []; - public IReadOnlyList Hooks => _hooks; - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) { _hooks.Insert(index ?? _hooks.Count, hook); diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index bd4176f6403..245561f8c50 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -94,8 +94,6 @@ private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJourn { private readonly List _hooks = []; - public IReadOnlyList Hooks => _hooks; - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) { _hooks.Insert(index ?? _hooks.Count, hook); diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index 5d91cafdd8b..dd7945ec7cb 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -746,7 +746,6 @@ public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public IReadOnlyList Hooks => inner.Hooks; public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => inner.AddHookAsync(hook, index, cancellationToken); public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 4166f54ab9a..3918d093fed 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -65,30 +65,28 @@ public async Task Hooks_QueueActivityUsesSelectedOperationAndCompletesWithStorag } [Fact] - public async Task Hooks_IdleRegistrationAppliesSynchronouslyAndExposesStableView() + public async Task Hooks_IdleRegistrationAppliesSynchronously() { await using var manager = CreateTestSystem().Manager; _ = new DurableValue("state", manager, CreateValueCodec()); var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; Assert.Null(field.GetValue(manager)); - var hook = new DelegateTestHook(); + var before = 0; + var hook = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); - var hooks = manager.Hooks; - Assert.False(hooks is IList); - Assert.Contains(hook, hooks); - Assert.Same(hooks, manager.Hooks); - Assert.Same(hook, Assert.Single(hooks)); Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); - Assert.Empty(manager.Hooks); await Assert.ThrowsAsync(() => manager.AddHookAsync(null!, cancellationToken: TestContext.Current.CancellationToken).AsTask()); await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, before); Assert.True(manager.AddHookAsync(hook, index: 0, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Same(hooks, manager.Hooks); - Assert.Same(hook, Assert.Single(hooks)); + Assert.Equal(1, before); Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); await using var withoutHooks = CreateTestSystem().Manager; await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); @@ -102,12 +100,9 @@ public async Task Hooks_ShutdownRejectsRegistrationAndRemoval() var manager = CreateTestSystem().Manager; var hook = new DelegateTestHook(); await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); - var hooks = manager.Hooks; await manager.DisposeAsync(); - Assert.Same(hooks, manager.Hooks); - Assert.Same(hook, Assert.Single(hooks)); await Assert.ThrowsAsync(() => manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); } @@ -135,10 +130,8 @@ public async Task Hooks_RegistrationDuringRecoveryAppliesBeforeLaterPersistence( await WaitFor(entered.Task); Assert.False(initialization.IsCompleted); Assert.False(registration.IsCompleted); - Assert.Empty(manager.Hooks); release.TrySetResult(); await WaitFor(Task.WhenAll(initialization, registration)); - Assert.Same(hook, Assert.Single(manager.Hooks)); await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(1, before); Assert.Equal(1, after); @@ -169,21 +162,27 @@ public async Task Hooks_QueuedRegistrationReportsRecoveryFailureAndIdleRegistrat .Returns(call => ReadAsync(call.Arg(), call.Arg())); await using var manager = CreateTestSystem(storage).Manager; var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); - var failedRegistration = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var failedBefore = 0; + var failedRegistration = manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => failedBefore++ + }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); try { await WaitFor(entered.Task); release.TrySetResult(); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(failedRegistration))); - Assert.Empty(manager.Hooks); - var hook = new DelegateTestHook(); + var before = 0; + var hook = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); Assert.Equal(1, reads); fail = false; await manager.InitializeAsync(TestContext.Current.CancellationToken); Assert.Equal(2, reads); - Assert.Same(hook, Assert.Single(manager.Hooks)); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, failedBefore); + Assert.Equal(1, before); } finally { @@ -258,17 +257,14 @@ public async Task Hooks_BusyMutationWaitsForCompleteOperationAndSurvivesCallerCa var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); Assert.False(added.IsCompleted); Assert.False(removed.IsCompleted); - Assert.Same(original, Assert.Single(manager.Hooks)); Assert.Equal(0, newBefore); Assert.Equal(0, newAfter); caller.Cancel(); await Assert.ThrowsAnyAsync(() => added); - Assert.Same(original, Assert.Single(manager.Hooks)); release.TrySetResult(); storage.ReleaseAppend.TrySetResult(); await WaitFor(Task.WhenAll(first, removed, next)); - Assert.Same(replacement, Assert.Single(manager.Hooks)); Assert.Equal(1, oldBefore); Assert.Equal(1, oldAfter); Assert.Equal(1, newBefore); @@ -381,13 +377,14 @@ public async Task Hooks_FailedQueuedMutationReportsErrorAndLeavesOwnerUsable() { await WaitFor(storage.BlockedAppendStarted.Task); var invalid = manager.AddHookAsync(new DelegateTestHook(), index: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - var valid = new DelegateTestHook(); + var before = 0; + var valid = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; var accepted = manager.AddHookAsync(valid, cancellationToken: TestContext.Current.CancellationToken).AsTask(); storage.ReleaseAppend.TrySetResult(); await Assert.ThrowsAsync(() => WaitFor(invalid)); await WaitFor(Task.WhenAll(first, accepted)); - Assert.Same(valid, Assert.Single(manager.Hooks)); await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); } finally { @@ -411,7 +408,6 @@ public async Task Hooks_StorageFailureFaultsQueuedRegistrationWithOriginalCause( storage.ReleaseAppend.TrySetResult(); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(write))); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(registration))); - Assert.Empty(manager.Hooks); var fenced = await Assert.ThrowsAsync(() => manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask()); Assert.Same(failure, fenced.InnerException); @@ -452,7 +448,6 @@ public async Task Hooks_ShutdownCancelsQueuedMutationWithoutChangingCommittedOpe await WaitFor(Task.WhenAll(write, shutdown)); await Assert.ThrowsAnyAsync(() => WaitFor(added)); await Assert.ThrowsAnyAsync(() => WaitFor(removed)); - Assert.Same(original, Assert.Single(manager.Hooks)); Assert.Single(storage.Appends); } finally @@ -818,10 +813,12 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() var deleting = false; var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var before = 0; var after = 0; await manager.AddHookAsync(new DelegateTestHook( async (operation, token) => { + before++; if (!deleting) { Assert.Equal(JournaledStateOperation.Write, operation); @@ -859,7 +856,9 @@ await manager.AddHookAsync(new DelegateTestHook( release.TrySetResult(); await deletion.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); Assert.Equal(1, after); - Assert.Single(manager.Hooks); + deleting = false; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, before); } finally { @@ -876,7 +875,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var drained = false; var failure = new IOException("Cancellation callback failed."); - await manager.AddHookAsync(new DelegateTestHook + var hook = new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -885,7 +884,8 @@ await manager.AddHookAsync(new DelegateTestHook await release.Task; drained = true; } - }, cancellationToken: TestContext.Current.CancellationToken); + }; + await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -906,7 +906,7 @@ await manager.AddHookAsync(new DelegateTestHook Assert.True(drained); Assert.Empty(storage.Appends); await Assert.ThrowsAsync(() => - manager.RemoveHookAsync(Assert.Single(manager.Hooks), cancellationToken: TestContext.Current.CancellationToken).AsTask()); + manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); } finally { From 8d58d858f3340bbbf1846ea05f73f71c3b358370 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 17:52:04 -0700 Subject: [PATCH 19/31] fix(journaling): capture directly after synchronous final hooks --- .../grains/journaling/runtime-behavior.md | 3 + src/Orleans.Journaling/IJournaledStateHook.cs | 2 + .../JournaledStateManager.Hooks.cs | 11 -- .../JournaledStateManager.cs | 7 +- src/Orleans.Journaling/README.md | 3 + .../StateManagerHookTests.cs | 106 ++++++++++++++++++ 6 files changed, 120 insertions(+), 12 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 70d297fcb49..90d8eb3e3b2 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -94,6 +94,9 @@ the manager lock. Add the final prerequisite at index zero so its callback runs synchronous capture or storage deletion. Prerequisites remain valid through capture or deletion, covering changes staged during asynchronous preparation and while the work loop resumes. Preserve operation-local bookkeeping for the captured batch separately from changes staged later. +When the final callback completes synchronously, capture or the start of deletion follows in the +same work-loop continuation. Other owner turns can stage changes while an incomplete callback is awaited, +so its prerequisite covers those changes through capture or deletion. Storage acknowledgement and registered-state acknowledgement or reset precede after callbacks. All after callbacks run in list order, including for successful zero-byte writes. Coalesced callers diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index 10c8d5eb9f1..c2e187254ff 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -21,6 +21,8 @@ public enum JournaledStateOperation /// /// Before hooks run in reverse list order on the owner's logical execution context, outside its lock. /// The hook at index zero supplies the final prerequisite before capture or deletion. +/// When the final callback completes synchronously, capture or the start of deletion follows in the +/// same work-loop continuation. /// Prerequisites remain valid until capture or deletion, covering changes staged during asynchronous /// preparation and while the journal work loop resumes. After hooks run in forward list order after /// storage acknowledgement and state acknowledgement or reset, including successful writes which diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 751ece9e5a3..e02c5f583b4 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -53,17 +53,6 @@ private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove } } - private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - for (var i = _hooks!.Count - 1; i >= 0; i--) - { - cancellationToken.ThrowIfCancellationRequested(); - await _hooks[i].BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); - } - - cancellationToken.ThrowIfCancellationRequested(); - } - private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { List? failures = null; diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index fb634b4f169..e0a8b676ba2 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -328,7 +328,12 @@ private async Task WorkLoop() if (operationType is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; - await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + // Keep synchronous final completion and capture in the same work-loop continuation. + for (var i = _hooks.Count - 1; i >= 0; i--) + { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + await _hooks[i].BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + } _shutdownCancellation.Token.ThrowIfCancellationRequested(); beforeHookRunning = false; diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index ebba60fad84..d50c3b34275 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -282,6 +282,9 @@ For each actual write, snapshot, or deletion, before hooks run in reverse list o the manager lock. Add the final prerequisite with `index: 0` so it runs last before the work loop captures the registered states or starts deletion. Prerequisites remain valid through capture or deletion, covering changes staged during asynchronous preparation and while the work loop resumes. +When the final callback completes synchronously, capture or the start of deletion follows in the +same work-loop continuation. Other owner turns can stage changes while an incomplete callback is awaited, +so its prerequisite covers those changes through capture or deletion. Keep operation-local bookkeeping for the captured batch separate from later pending changes. After hooks run in list order after storage acknowledgement and state acknowledgement or reset. diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 3918d093fed..8833f87cb85 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Diagnostics; using System.Reflection; +using System.Threading.Channels; using Microsoft.Extensions.DependencyInjection; using NSubstitute; using Orleans.Diagnostics; @@ -776,6 +777,77 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion() } } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_SynchronousFinalPrerequisiteAndCaptureShareContinuationAfterAsyncEarlierHook(bool snapshot) + { + var scheduler = new QueuedHookTaskScheduler(); + await scheduler.Run(async () => + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + List prepared = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + var queueMutation = true; + Task? mutation = null; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => + { + events.Add("final-before"); + prepared.AddRange(state.Keys); + if (queueMutation) + { + Assert.Empty(state); + queueMutation = false; + mutation = scheduler.Queue(() => + { + state["intent"] = 1; + events.Add("queued-mutation"); + }); + } + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperationAsync = async (_, _) => + { + events.Add("ordinary-before"); + await Task.Yield(); + events.Add("ordinary-complete"); + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.NotNull(mutation); + await mutation; + Assert.Empty(prepared); + Assert.Equal(1, state["intent"]); + await using (var recoveredManager = CreateTestSystem(storage).Manager) + { + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Empty(recovered); + } + + Assert.Equal( + ["ordinary-before", "ordinary-complete", "final-before", "capture", "ack", "queued-mutation"], + events); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(["intent"], prepared); + await using var replayManager = CreateTestSystem(storage).Manager; + var replayed = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + var intent = Assert.Single(replayed); + Assert.Equal("intent", intent.Key); + Assert.Equal(1, intent.Value); + }); + } + [Fact] public async Task Hooks_BeforeCallbacksRunInReverseAndAfterCallbacksRunInListOrder() { @@ -1427,6 +1499,40 @@ public void Hooks_FailureOutcomesRoundTripWithTypeAndOriginalCause(bool after) Assert.Equal(original.Message, copy.Message); } + private sealed class QueuedHookTaskScheduler : TaskScheduler + { + private readonly Channel _queue = Channel.CreateUnbounded(); + + public override int MaximumConcurrencyLevel => 1; + protected override IEnumerable? GetScheduledTasks() => null; + protected override void QueueTask(Task task) => Assert.True(_queue.Writer.TryWrite(task)); + protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) => false; + + public Task Queue(Action action) => + Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this); + + public async Task Run(Func action) + { + var task = Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this).Unwrap(); + while (!task.IsCompleted) + { + var next = await _queue.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); + var previous = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(null); + try + { + Assert.True(TryExecuteTask(next)); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previous); + } + } + + await task; + } + } + private sealed class DelegateTestHook( Func? before = null, Action? after = null) : IJournaledStateHook From c9158123e28bdf3337bfbdeb2a8de7ec0ab3dcef Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:10:56 -0700 Subject: [PATCH 20/31] feat(messaging): add simple contracts above current journal hooks --- Orleans.slnx | 2 + .../Configuration/InboxOptions.cs | 238 +++++++++++ src/Orleans.Messaging/DeliveryResult.cs | 55 +++ src/Orleans.Messaging/DeliveryStatus.cs | 32 ++ src/Orleans.Messaging/Envelope.cs | 32 ++ src/Orleans.Messaging/EnvelopeEquivalence.cs | 17 + src/Orleans.Messaging/EnvelopeValidation.cs | 52 +++ src/Orleans.Messaging/HierarchicalKey.cs | 289 +++++++++++++ src/Orleans.Messaging/HierarchicalKeyCodec.cs | 48 +++ src/Orleans.Messaging/IInbox.cs | 43 ++ src/Orleans.Messaging/IInboxExtension.cs | 36 ++ src/Orleans.Messaging/IInboxHandler.cs | 41 ++ src/Orleans.Messaging/IInboxHandlerContext.cs | 31 ++ src/Orleans.Messaging/IOutbox.cs | 62 +++ src/Orleans.Messaging/InboxHandlerContext.cs | 12 + src/Orleans.Messaging/InboxMessage.cs | 10 + src/Orleans.Messaging/MessageState.cs | 65 +++ .../Orleans.Messaging.csproj | 28 ++ src/Orleans.Messaging/OrleansContracts.txt | 14 + src/Orleans.Messaging/OutboxMessage.cs | 20 + src/Orleans.Messaging/README.md | 33 ++ .../Orleans.Messaging/Orleans.Messaging.cs | 393 ++++++++++++++++++ .../DeliveryAndOptionsContractTests.cs | 105 +++++ .../Contracts/EnvelopeContractTests.cs | 293 +++++++++++++ .../Contracts/HandlerRoutingContractTests.cs | 204 +++++++++ .../Contracts/HierarchicalKeyTests.cs | 349 ++++++++++++++++ .../Orleans.Messaging.Tests.csproj | 14 + 27 files changed, 2518 insertions(+) create mode 100644 src/Orleans.Messaging/Configuration/InboxOptions.cs create mode 100644 src/Orleans.Messaging/DeliveryResult.cs create mode 100644 src/Orleans.Messaging/DeliveryStatus.cs create mode 100644 src/Orleans.Messaging/Envelope.cs create mode 100644 src/Orleans.Messaging/EnvelopeEquivalence.cs create mode 100644 src/Orleans.Messaging/EnvelopeValidation.cs create mode 100644 src/Orleans.Messaging/HierarchicalKey.cs create mode 100644 src/Orleans.Messaging/HierarchicalKeyCodec.cs create mode 100644 src/Orleans.Messaging/IInbox.cs create mode 100644 src/Orleans.Messaging/IInboxExtension.cs create mode 100644 src/Orleans.Messaging/IInboxHandler.cs create mode 100644 src/Orleans.Messaging/IInboxHandlerContext.cs create mode 100644 src/Orleans.Messaging/IOutbox.cs create mode 100644 src/Orleans.Messaging/InboxHandlerContext.cs create mode 100644 src/Orleans.Messaging/InboxMessage.cs create mode 100644 src/Orleans.Messaging/MessageState.cs create mode 100644 src/Orleans.Messaging/Orleans.Messaging.csproj create mode 100644 src/Orleans.Messaging/OrleansContracts.txt create mode 100644 src/Orleans.Messaging/OutboxMessage.cs create mode 100644 src/Orleans.Messaging/README.md create mode 100644 src/api/Orleans.Messaging/Orleans.Messaging.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs create mode 100644 test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj diff --git a/Orleans.slnx b/Orleans.slnx index e4047b5f194..877afac409d 100644 --- a/Orleans.slnx +++ b/Orleans.slnx @@ -42,6 +42,7 @@ + @@ -152,6 +153,7 @@ + diff --git a/src/Orleans.Messaging/Configuration/InboxOptions.cs b/src/Orleans.Messaging/Configuration/InboxOptions.cs new file mode 100644 index 00000000000..90beb3a2c5c --- /dev/null +++ b/src/Orleans.Messaging/Configuration/InboxOptions.cs @@ -0,0 +1,238 @@ +using System; + +namespace Orleans.Messaging.Configuration; + +/// +/// Configuration options for the inbox messaging system. +/// +/// +/// +/// These options control the behavior of the inbox, including capacity limits, +/// deduplication tracking, retry behavior, and pump batch sizes. +/// +/// +/// Transport is at-least-once. Deduplication provides effectively-once handler effects only +/// while the processed-message record is retained. Configuration values affect memory usage, +/// throughput, and recovery characteristics. +/// +/// +public class InboxOptions +{ + internal const int MaximumBackoffExponent = 6; + internal const int MaximumBackoffMultiplier = 1 << MaximumBackoffExponent; + + // The runtime multiplies this base delay by at most MaximumBackoffMultiplier. + // Keep the expanded delay within the timer implementation's uint-millisecond limit. + private static readonly TimeSpan MaxSupportedRetryDelay = + TimeSpan.FromTicks( + TimeSpan.FromMilliseconds(uint.MaxValue - 1).Ticks + / MaximumBackoffMultiplier); + + /// + /// Gets or sets the maximum number of pending messages in the inbox. + /// When this limit is reached, new message deliveries will return DeliveryResult.Backpressured(). + /// + /// + /// + /// A lower value (e.g., 100) provides stronger backpressure but may reduce throughput. + /// A higher value (e.g., 10,000) allows more buffering but increases memory usage and recovery time. + /// + /// + /// The inbox capacity is checked before accepting new messages. Messages are persisted to durable + /// storage, so capacity limits affect both in-memory state and storage I/O during recovery. + /// + /// + /// + /// The maximum inbox capacity. Must be greater than zero. Defaults to 1000. + /// + public int MaxCapacity { get; set; } = 1000; + + /// + /// Gets or sets the time window for tracking processed messages to prevent duplicates. + /// Messages that were processed within this window will be rejected with DeliveryResult.Duplicate(). + /// + /// + /// + /// A longer window (e.g., 30 days) provides stronger deduplication guarantees but increases + /// memory usage and storage I/O. A shorter window (e.g., 1 hour) reduces overhead but may + /// allow duplicate processing if retries are delayed. + /// + /// + /// Processed message tracking uses the exact message ID within the receiving inbox, with timestamps. + /// Expired entries are removed atomically when a replay is accepted and are also eligible for + /// compaction during inbox pump maintenance. + /// + /// + /// Consider your retry policies when setting this value. For example, if senders retry for + /// up to 24 hours, set the window to at least 48 hours to ensure deduplication coverage. + /// + /// + /// + /// The deduplication window. Must be greater than zero. Defaults to 7 days. + /// + public TimeSpan DeduplicationWindow { get; set; } = TimeSpan.FromDays(7); + + /// + /// Gets or sets the base delay between retry attempts when delivery encounters backpressure. + /// + /// + /// + /// When the target inbox is at capacity and returns DeliveryResult.Backpressured(), + /// the outbox delivery pump applies exponential backoff from this duration before retrying. + /// + /// + /// A shorter delay (e.g., 100ms) enables faster recovery when the target processes messages quickly, + /// but may increase CPU usage during sustained backpressure. A longer delay (e.g., 5 seconds) + /// reduces retry overhead but increases latency for message delivery. + /// + /// + /// For high-throughput scenarios where quick recovery from backpressure is important, + /// consider values between 100-500ms. For less time-sensitive workloads, 1-5 seconds is appropriate. + /// + /// + /// + /// The base backpressure retry delay. Must be greater than zero. Defaults to 1 second. + /// + public TimeSpan BackpressureRetryDelay { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Gets or sets the maximum number of attempts before an inbox message is dead-lettered. + /// + public int MaxProcessingAttempts { get; set; } = 5; + + /// + /// Gets or sets the maximum number of attempts before an outbox message is dead-lettered. + /// + public int MaxDeliveryAttempts { get; set; } = 100; + + /// + /// Gets or sets the maximum age of an outbox message. + /// + public TimeSpan MaxOutboxRetryAge { get; set; } = TimeSpan.FromDays(1); + + /// + /// Gets or sets how long inbox and outbox dead letters are retained. + /// + public TimeSpan DeadLetterRetentionPeriod { get; set; } = TimeSpan.FromDays(30); + + /// + /// Gets or sets the maximum number of dead letters retained per inbox and per outbox. + /// + public int MaxRetainedDeadLetters { get; set; } = 1000; + + /// + /// Gets or sets the maximum number of inbox messages processed by one durable job attempt. + /// + public int InboxBatchSize { get; set; } = 32; + + /// + /// Gets or sets the maximum number of outbox messages processed by one durable job attempt. + /// + public int OutboxBatchSize { get; set; } = 32; + + /// + /// Gets or sets how long an empty outbox retains its acknowledged physical recovery job. + /// + /// + /// The default is 100 milliseconds. Zero selects immediate retirement. Ready local work + /// wakes its pump immediately, while delivery, acceptance, and acknowledgement keep their + /// normal durability boundaries. Once idle, the job is rescheduled at the idle deadline; + /// provider polling and activation latency also contribute to recovery timing. + /// A durable wakeup is scheduled before outbound work can be acknowledged. + /// + /// A non-negative interval within the supported timer range. + public TimeSpan OutboxIdleRetirementGracePeriod { get; set; } = TimeSpan.FromMilliseconds(100); + + /// + /// Validates the configuration values and throws if any are invalid. + /// + /// + /// Thrown if is less than or equal to zero, + /// or if is less than or equal to , + /// or if a retry, retention, or batch option is outside its supported range. + /// + /// + /// This method is typically called by the dependency injection container during service registration + /// to ensure configuration values are valid before the system starts. + /// + public void Validate() + { + if (OutboxIdleRetirementGracePeriod < TimeSpan.Zero + || OutboxIdleRetirementGracePeriod > TimeSpan.FromMilliseconds(uint.MaxValue - 1)) + { + throw new ArgumentOutOfRangeException(nameof(OutboxIdleRetirementGracePeriod), OutboxIdleRetirementGracePeriod, + "OutboxIdleRetirementGracePeriod must be non-negative and within the supported timer range."); + } + + if (MaxCapacity <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxCapacity), MaxCapacity, "MaxCapacity must be greater than zero."); + } + + if (DeduplicationWindow <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(DeduplicationWindow), DeduplicationWindow, "DeduplicationWindow must be greater than TimeSpan.Zero."); + } + + if (BackpressureRetryDelay <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(BackpressureRetryDelay), BackpressureRetryDelay, "BackpressureRetryDelay must be greater than TimeSpan.Zero."); + } + if (BackpressureRetryDelay > MaxSupportedRetryDelay) + { + throw new ArgumentOutOfRangeException( + nameof(BackpressureRetryDelay), + BackpressureRetryDelay, + $"BackpressureRetryDelay must be less than or equal to {MaxSupportedRetryDelay}."); + } + + if (MaxProcessingAttempts <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxProcessingAttempts), MaxProcessingAttempts, "MaxProcessingAttempts must be greater than zero."); + } + + if (MaxDeliveryAttempts <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxDeliveryAttempts), MaxDeliveryAttempts, "MaxDeliveryAttempts must be greater than zero."); + } + + if (MaxOutboxRetryAge <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(MaxOutboxRetryAge), MaxOutboxRetryAge, "MaxOutboxRetryAge must be greater than TimeSpan.Zero."); + } + + if (MaxOutboxRetryAge >= DeduplicationWindow) + { + throw new ArgumentOutOfRangeException( + nameof(MaxOutboxRetryAge), + MaxOutboxRetryAge, + "MaxOutboxRetryAge must be less than DeduplicationWindow."); + } + + if (DeadLetterRetentionPeriod <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(DeadLetterRetentionPeriod), + DeadLetterRetentionPeriod, + "DeadLetterRetentionPeriod must be greater than TimeSpan.Zero."); + } + + if (MaxRetainedDeadLetters <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(MaxRetainedDeadLetters), + MaxRetainedDeadLetters, + "MaxRetainedDeadLetters must be greater than zero."); + } + + if (InboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(InboxBatchSize), InboxBatchSize, "InboxBatchSize must be greater than zero."); + } + + if (OutboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(OutboxBatchSize), OutboxBatchSize, "OutboxBatchSize must be greater than zero."); + } + } +} diff --git a/src/Orleans.Messaging/DeliveryResult.cs b/src/Orleans.Messaging/DeliveryResult.cs new file mode 100644 index 00000000000..6124b87548c --- /dev/null +++ b/src/Orleans.Messaging/DeliveryResult.cs @@ -0,0 +1,55 @@ +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// Result of attempting to deliver a message to an inbox. +/// +[GenerateSerializer, Alias("Orleans.Messaging.DeliveryResult")] +public readonly struct DeliveryResult +{ + /// + /// The status of the delivery attempt. + /// + [Id(0)] + public DeliveryStatus Status { get; init; } + + /// + /// Optional diagnostic message (e.g., reason for rejection). + /// + [Id(1)] + public string? Message { get; init; } + + /// + /// Creates a result indicating the message was accepted and persisted to inbox. + /// + public static DeliveryResult Accepted() => new() { Status = DeliveryStatus.Accepted }; + + /// + /// Creates a result indicating the message was a duplicate. + /// + public static DeliveryResult Duplicate() => new() { Status = DeliveryStatus.Duplicate }; + + /// + /// Creates a result indicating the inbox is at capacity. + /// + public static DeliveryResult Backpressured() => new() { Status = DeliveryStatus.Backpressured }; + + /// + /// Creates a result indicating the receiving inbox has no registered handler. + /// + public static DeliveryResult HandlerNotFound() => new() + { + Status = DeliveryStatus.HandlerNotFound, + Message = "No inbox handler is registered." + }; + + /// + /// Creates a result indicating the message was dead-lettered. + /// + public static DeliveryResult DeadLettered(string reason) => new() + { + Status = DeliveryStatus.DeadLettered, + Message = reason + }; +} diff --git a/src/Orleans.Messaging/DeliveryStatus.cs b/src/Orleans.Messaging/DeliveryStatus.cs new file mode 100644 index 00000000000..d4d24333e2d --- /dev/null +++ b/src/Orleans.Messaging/DeliveryStatus.cs @@ -0,0 +1,32 @@ +namespace Orleans.Messaging; + +/// +/// Status codes for delivery attempts. +/// +public enum DeliveryStatus +{ + /// + /// Message was accepted and persisted to inbox. + /// + Accepted = 0, + + /// + /// Message was a duplicate (already processed or in inbox). + /// + Duplicate = 1, + + /// + /// Inbox is at capacity; sender should retry later. + /// + Backpressured = 2, + + /// + /// The receiving inbox has no registered handler. + /// + HandlerNotFound = 3, + + /// + /// The message was moved to the receiver's dead-letter store. + /// + DeadLettered = 4 +} diff --git a/src/Orleans.Messaging/Envelope.cs b/src/Orleans.Messaging/Envelope.cs new file mode 100644 index 00000000000..96b6e545c05 --- /dev/null +++ b/src/Orleans.Messaging/Envelope.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; + +namespace Orleans.Messaging; + +/// Contains a command identity, payload, subject, and optional application headers. +/// +/// Message identity and payload are required; an empty payload is valid. Raw protocols can omit +/// subject and headers. Prepare the payload and headers before publishing a message and keep them +/// unchanged afterward. Struct assignment shares these objects; ordinary Orleans deep copying +/// and deserialization produce independent payload arrays, header dictionaries, and header value arrays. +/// +[GenerateSerializer, Alias("Orleans.Messaging.Envelope")] +public readonly struct Envelope +{ + /// Gets the exact application-defined command identity within the receiving inbox. + [Id(0)] + public required HierarchicalKey MessageId { get; init; } + + /// Gets the required application payload bytes, including an empty payload. + [Id(1)] + public required byte[] Payload { get; init; } + + /// Gets the optional, nonempty ordinal application protocol subject. + [Id(2)] + public string? Subject { get; init; } + + /// Gets optional application-defined named byte-array headers. + /// Use an ordinal dictionary for case-sensitive header names. Keys are nonempty and arrays are non-null. + /// The optional "sender" header contains an Orleans-serialized grain identity, decoded by the consuming protocol. + [Id(3)] + public Dictionary? Headers { get; init; } +} diff --git a/src/Orleans.Messaging/EnvelopeEquivalence.cs b/src/Orleans.Messaging/EnvelopeEquivalence.cs new file mode 100644 index 00000000000..4710813bacd --- /dev/null +++ b/src/Orleans.Messaging/EnvelopeEquivalence.cs @@ -0,0 +1,17 @@ +using System; + +namespace Orleans.Messaging; + +internal static class EnvelopeEquivalence +{ + public static bool AreEquivalent(OutboxMessage left, OutboxMessage right) => + left.ReceiverId == right.ReceiverId && AreSameCommand(left.Envelope, right.Envelope); + + public static bool AreSameCommand(InboxMessage left, InboxMessage right) => AreSameCommand(left.Envelope, right.Envelope); + + public static bool AreSameCommand(Envelope left, Envelope right) => + left.MessageId == right.MessageId + && left.Payload.Length == right.Payload.Length + && string.Equals(left.Subject, right.Subject, StringComparison.Ordinal) + && left.Payload.AsSpan().SequenceEqual(right.Payload); +} diff --git a/src/Orleans.Messaging/EnvelopeValidation.cs b/src/Orleans.Messaging/EnvelopeValidation.cs new file mode 100644 index 00000000000..6fef6d27aba --- /dev/null +++ b/src/Orleans.Messaging/EnvelopeValidation.cs @@ -0,0 +1,52 @@ +using System; +using System.Text; + +namespace Orleans.Messaging; + +internal static class EnvelopeValidation +{ + internal const int MaxMessageIdBytes = 1024; + internal const int MaxMessageIdSegments = 32; + internal const int MaxSubjectBytes = 256; + private static readonly UTF8Encoding Utf8 = new(false, true); + + // External admission and recovery validate generated values before publishing shared mutations. + public static void Validate(Envelope envelope) + { + ValidateMessageId(envelope.MessageId); + if (envelope.Payload is null) throw new ArgumentException("The payload must not be null.", nameof(envelope)); + if (envelope.Subject is not null) ValidateSubject(envelope.Subject); + if (envelope.Headers is { } headers) + { + foreach (var header in headers) + { + if (string.IsNullOrEmpty(header.Key)) throw new ArgumentException("Header keys must not be empty.", nameof(envelope)); + if (header.Value is null) throw new ArgumentException("Header values must not be null.", nameof(envelope)); + } + } + } + + public static void Validate(InboxMessage message) => Validate(message.Envelope); + + public static void Validate(OutboxMessage message) + { + if (message.ReceiverId.IsDefault) throw new ArgumentException("The outgoing receiver must not be default.", nameof(message)); + Validate(message.Envelope); + } + + internal static void ValidateMessageId(HierarchicalKey messageId) + { + if (messageId.IsDefault) throw new ArgumentException("The message ID must not be unset.", nameof(messageId)); + if (messageId.SegmentCount > MaxMessageIdSegments) + throw new ArgumentException($"The message ID exceeds {MaxMessageIdSegments} segments.", nameof(messageId)); + if (Utf8.GetByteCount(messageId.ToString()) > MaxMessageIdBytes) + throw new ArgumentException($"The message ID exceeds {MaxMessageIdBytes} UTF-8 bytes.", nameof(messageId)); + } + + public static void ValidateSubject(string subject) + { + ArgumentException.ThrowIfNullOrEmpty(subject); + if (Utf8.GetByteCount(subject) > MaxSubjectBytes) + throw new ArgumentException($"The subject exceeds {MaxSubjectBytes} UTF-8 bytes.", nameof(subject)); + } +} diff --git a/src/Orleans.Messaging/HierarchicalKey.cs b/src/Orleans.Messaging/HierarchicalKey.cs new file mode 100644 index 00000000000..0debc8647be --- /dev/null +++ b/src/Orleans.Messaging/HierarchicalKey.cs @@ -0,0 +1,289 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text; + +namespace Orleans.Messaging; + +/// +/// An immutable, ordinal application identity formed from nonempty hierarchical segments. +/// +/// +/// Create constructs literal segments and escapes slash and backslash characters exactly once. +/// Parse reads the canonical escaped path. Assignment shares immutable backing data; equality and +/// hashing use the full canonical identity. Applications preserve the identity across retries. +/// +[Immutable, Alias("Orleans.Messaging.HierarchicalKey")] +public readonly struct HierarchicalKey : ISpanFormattable, IEquatable, IParsable, ISpanParsable +{ + /// The escape character used within canonical segments. + public const char EscapeCharacter = '\\'; + + /// The separator between canonical segments. + public const char SegmentSeparator = '/'; + + private readonly KeyData? _data; + + private HierarchicalKey(string canonical, int segmentCount) => _data = new(canonical, segmentCount); + + /// Gets whether this value is unset. + public bool IsDefault => _data is null; + + /// Gets the canonical path length in UTF-16 characters. + public int Length => _data?.Canonical.Length ?? 0; + + /// Gets the number of segments, or zero for an unset key. + public int SegmentCount => _data?.SegmentCount ?? 0; + + /// Creates one literal segment, escaping slash and backslash characters. + /// The nonempty literal segment. + /// The segment identity. + /// is null. + /// is empty. + public static HierarchicalKey Create(string value) + { + ArgumentException.ThrowIfNullOrEmpty(value); + return new(Escape(value), 1); + } + + /// Creates a hierarchy from literal segments in root-first order. + /// The nonempty literal segments. + /// A flat canonical identity which owns its immutable backing string. + /// A segment is null. + /// The input or a segment is empty. + public static HierarchicalKey Create(params ReadOnlySpan values) + { + if (values.IsEmpty) + { + throw new ArgumentException("Values must not be empty.", nameof(values)); + } + var builder = new StringBuilder(); + foreach (var value in values) + { + ArgumentException.ThrowIfNullOrEmpty(value); + if (builder.Length > 0) builder.Append(SegmentSeparator); + AppendEscaped(builder, value); + } + return new(builder.ToString(), values.Length); + } + + /// Appends one literal child segment. + /// The nonempty literal child segment. + /// The child identity. + /// This key is unset. + /// is null. + /// is empty. + public HierarchicalKey CreateChildKey(string value) + { + EnsureSet(); + ArgumentException.ThrowIfNullOrEmpty(value); + return new(string.Concat(_data!.Canonical, "/", Escape(value)), checked(SegmentCount + 1)); + } + + /// Composes this hierarchy with an already constructed suffix hierarchy. + /// The constructed suffix. + /// A flat concatenated identity preserving both paths' segment boundaries. + /// This key is unset. + /// is unset. + public HierarchicalKey Append(HierarchicalKey suffix) + { + EnsureSet(); + if (suffix.IsDefault) throw new ArgumentException("The suffix must not be unset.", nameof(suffix)); + return new(string.Concat(_data!.Canonical, "/", suffix._data!.Canonical), checked(SegmentCount + suffix.SegmentCount)); + } + + /// Gets the immediate parent, or null for a root or unset key. + /// The parent identity when this key has more than one segment. + public HierarchicalKey? GetParent() + { + if (SegmentCount < 2) return null; + var lastSeparator = 0; + var path = _data!.Canonical.AsSpan(); + for (var i = 0; i < path.Length; i++) + { + if (path[i] == EscapeCharacter) i++; + else if (path[i] == SegmentSeparator) lastSeparator = i; + } + return new HierarchicalKey(_data.Canonical[..lastSeparator], SegmentCount - 1); + } + + /// Tests whether this key is the other's immediate child. + /// The potential parent. + /// Whether there is exactly one additional segment. + public bool IsChildOf(HierarchicalKey other) => other.IsParentOf(this); + + /// Tests whether this key is the other's immediate parent. + /// The potential child. + /// Whether the other key extends this key by exactly one segment. + public bool IsParentOf(HierarchicalKey other) => !IsDefault && other.SegmentCount == SegmentCount + 1 && IsAncestorOf(other); + + /// Tests whether this key is equal to or an ancestor of the other key. + /// The identity to inspect. + /// Whether the full prefix consists of equal ordinal segments. Unset keys return false. + public bool IsAncestorOf(HierarchicalKey other) => !IsDefault && !other.IsDefault + && (Equals(other) || (other.Length > Length && other._data!.Canonical[Length] == SegmentSeparator + && other._data.Canonical.StartsWith(_data!.Canonical, StringComparison.Ordinal))); + + /// + public static HierarchicalKey Parse(string s, IFormatProvider? provider = null) + { + ArgumentNullException.ThrowIfNull(s); + return TryParse(s, provider, out var result) ? result : throw new FormatException("The value is not a valid canonical hierarchical key."); + } + + /// + public static HierarchicalKey Parse(ReadOnlySpan s, IFormatProvider? provider = null) => + TryParse(s, provider, out var result) ? result : throw new FormatException("The value is not a valid canonical hierarchical key."); + + /// + public static bool TryParse([NotNullWhen(true)] string? s, IFormatProvider? provider, out HierarchicalKey result) + { + if (s is not null && TryCountSegments(s, out var count)) + { + result = new(s, count); + return true; + } + result = default; + return false; + } + + /// + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, out HierarchicalKey result) + { + if (TryCountSegments(s, out var count)) + { + result = new(new string(s), count); + return true; + } + result = default; + return false; + } + + /// + public bool Equals(HierarchicalKey other) => ReferenceEquals(_data, other._data) + || string.Equals(_data?.Canonical, other._data?.Canonical, StringComparison.Ordinal); + + /// + public override bool Equals(object? obj) => obj is HierarchicalKey other && Equals(other); + + /// + public override int GetHashCode() => _data?.Hash ?? 0; + + /// Compares complete ordinal key values. + /// The first identity. + /// The second identity. + /// Whether the identities are equal. + public static bool operator ==(HierarchicalKey left, HierarchicalKey right) => left.Equals(right); + + /// Compares complete ordinal key values for inequality. + /// The first identity. + /// The second identity. + /// Whether the identities differ. + public static bool operator !=(HierarchicalKey left, HierarchicalKey right) => !left.Equals(right); + + /// + public override string ToString() => _data?.Canonical ?? string.Empty; + + /// + public string ToString(string? format, IFormatProvider? formatProvider) => ToString(); + + /// + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + if (ToString().AsSpan().TryCopyTo(destination)) + { + charsWritten = Length; + return true; + } + charsWritten = 0; + return false; + } + + /// Enumerates escaped canonical segment spans in root-first order. + /// An allocation-free segment enumerator. + public SegmentEnumerator GetEnumerator() => new(ToString().AsSpan()); + + private void EnsureSet() + { + if (IsDefault) throw new InvalidOperationException("The key must not be unset."); + } + + private static string Escape(string value) + { + if (value.AsSpan().IndexOfAny(EscapeCharacter, SegmentSeparator) < 0) return value; + var builder = new StringBuilder(value.Length); + AppendEscaped(builder, value); + return builder.ToString(); + } + + private static void AppendEscaped(StringBuilder builder, string value) + { + foreach (var character in value) + { + if (character is EscapeCharacter or SegmentSeparator) builder.Append(EscapeCharacter); + builder.Append(character); + } + } + + private static bool TryCountSegments(ReadOnlySpan path, out int count) + { + count = 0; + var segmentLength = 0; + for (var i = 0; i < path.Length; i++) + { + if (path[i] == EscapeCharacter) + { + if (++i == path.Length || path[i] is not (EscapeCharacter or SegmentSeparator)) return false; + } + else if (path[i] == SegmentSeparator) + { + if (segmentLength == 0) return false; + count++; + segmentLength = 0; + continue; + } + segmentLength++; + } + if (segmentLength == 0) return false; + count++; + return true; + } + + private sealed class KeyData(string canonical, int segmentCount) + { + public string Canonical { get; } = canonical; + public int Hash { get; } = canonical.GetHashCode(StringComparison.Ordinal); + public int SegmentCount { get; } = segmentCount; + } + + /// Enumerates borrowed spans of the immutable canonical key. + public ref struct SegmentEnumerator + { + private readonly ReadOnlySpan _path; + private int _next; + + internal SegmentEnumerator(ReadOnlySpan path) => _path = path; + + /// Gets the current escaped canonical segment. + public ReadOnlySpan Current { get; private set; } + + /// Advances to the next segment. + /// Whether a segment is available. + public bool MoveNext() + { + if (_next >= _path.Length) + { + Current = default; + return false; + } + var start = _next; + for (; _next < _path.Length; _next++) + { + if (_path[_next] == EscapeCharacter) _next++; + else if (_path[_next] == SegmentSeparator) break; + } + Current = _path[start.._next]; + _next++; + return true; + } + } +} diff --git a/src/Orleans.Messaging/HierarchicalKeyCodec.cs b/src/Orleans.Messaging/HierarchicalKeyCodec.cs new file mode 100644 index 00000000000..98a43ecea81 --- /dev/null +++ b/src/Orleans.Messaging/HierarchicalKeyCodec.cs @@ -0,0 +1,48 @@ +using System; +using System.Buffers; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Cloning; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging; + +[RegisterSerializer] +internal sealed class HierarchicalKeyCodec(IFieldCodec strings) : IFieldCodec +{ + public void WriteField(ref Writer writer, uint fieldIdDelta, + Type? expectedType, HierarchicalKey value) where TBufferWriter : IBufferWriter + { + ReferenceCodec.MarkValueField(writer.Session); + writer.WriteFieldHeader(fieldIdDelta, expectedType, typeof(HierarchicalKey), WireType.TagDelimited); + strings.WriteField(ref writer, 0, typeof(string), value.IsDefault ? null! : value.ToString()); + writer.WriteEndObject(); + } + + public HierarchicalKey ReadValue(ref Reader reader, Field field) + { + field.EnsureWireTypeTagDelimited(); + ReferenceCodec.MarkValueField(reader.Session); + HierarchicalKey result = default; + uint fieldId = 0; + while (true) + { + var header = reader.ReadFieldHeader(); + if (header.IsEndBaseOrEndObject) return result; + fieldId += header.FieldIdDelta; + if (fieldId == 0) + { + var canonical = strings.ReadValue(ref reader, header); + result = canonical is null ? default : HierarchicalKey.Parse(canonical); + } + else reader.ConsumeUnknownField(header); + } + } +} + +[RegisterCopier] +internal sealed class HierarchicalKeyCopier : IDeepCopier +{ + public HierarchicalKey DeepCopy(HierarchicalKey input, CopyContext context) => input; +} diff --git a/src/Orleans.Messaging/IInbox.cs b/src/Orleans.Messaging/IInbox.cs new file mode 100644 index 00000000000..ddd09ad0b5c --- /dev/null +++ b/src/Orleans.Messaging/IInbox.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.Messaging; + +/// +/// Registers a grain's opaque-message handler and exposes pending inbox state. +/// +public interface IInbox +{ + /// + /// Gets the number of unprocessed messages. + /// + int Count { get; } + + /// + /// Gets the capacity at which delivery returns . + /// + int Capacity { get; } + + /// + /// Gets pending messages in unspecified order. + /// + /// Messages share their payload arrays, header dictionaries, and header value arrays, which remain unchanged after publication. + IEnumerable Messages { get; } + + /// + /// Looks up a pending command by its application identity within this inbox. + /// + /// The exact command identity, independent of its immediate sender and subject. + /// The matching stored message when found. + /// Whether the message is pending. + bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out InboxMessage message); + + /// + /// Registers the handler for this inbox. + /// + /// The handler responsible for application decoding and dispatch. + /// is null. + /// A handler has already been registered. + void RegisterHandler(IInboxHandler handler); +} diff --git a/src/Orleans.Messaging/IInboxExtension.cs b/src/Orleans.Messaging/IInboxExtension.cs new file mode 100644 index 00000000000..396706e8fda --- /dev/null +++ b/src/Orleans.Messaging/IInboxExtension.cs @@ -0,0 +1,36 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Orleans; +using Orleans.Runtime; +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// Non-generic grain extension for inbox message delivery. +/// +[Alias("IInboxExtension")] +public interface IInboxExtension : IGrainExtension +{ + /// + /// Delivers a message to this grain's inbox. + /// + /// The received command with a required identity and payload; its receiver is this grain. + /// Cancels the caller's wait for delivery. + /// + /// Direct admission shares the payload array, header dictionary, and header value arrays under the immutable-publication contract. + /// RPC copying and deserialization isolate payload arrays, header dictionaries, and header value arrays using ordinary Orleans serialization. + /// Once delivery owns inbox admission, it retains its gate and ownership reservation until + /// its operation completes. Caller cancellation leaves that operation running to its durable outcome. + /// The grain owner keeps delivery quiescent during journal deletion and resumes delivery + /// after the deletion task completes successfully. + /// + /// Result indicating delivery/processing status. + /// + /// has an unset message ID, exceeds identity admission limits, or has invalid payload or headers. + /// Optional standard metadata is validated where its contract is consumed. + /// + [Alias("DeliverAsync")] + ValueTask DeliverAsync(InboxMessage message, CancellationToken cancellationToken = default); +} diff --git a/src/Orleans.Messaging/IInboxHandler.cs b/src/Orleans.Messaging/IInboxHandler.cs new file mode 100644 index 00000000000..f06bf473ccb --- /dev/null +++ b/src/Orleans.Messaging/IInboxHandler.cs @@ -0,0 +1,41 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Messaging; + +/// +/// Processes opaque messages accepted by a grain's inbox. +/// +/// +/// Register one handler per inbox. Applications perform payload decoding and dispatch in their handler +/// and stage outbound messages through an injected . +/// +public interface IInboxHandler +{ + /// + /// Handles a message and stages its logical completion. + /// + /// The received envelope and attempt-scoped completion operation. + /// The token to check during preparation and before shared mutation. + /// The handler's method outcome, awaited by the inbox runtime. + /// + /// + /// Decode the payload, perform asynchronous I/O, validate local results, build outgoing envelopes, + /// and check cancellation before the first shared business or journaled mutation. + /// + /// + /// From the first shared mutation through completion of this method, execute synchronously. + /// Apply complete, safe-to-commit business changes, stage outgoing messages through + /// , call , and return. + /// This final block includes the method's return after Complete and relies on the trusted + /// handler contract. Every successful outcome calls Complete, including outcomes with no business effects. + /// + /// + /// Complete stages inbox completion and transport deduplication alongside the business changes + /// and outgoing intents. The runtime owns the subsequent journal write, persistence acknowledgement, + /// and retirement. Failures during local preparation follow the inbox retry and dead-letter policy. + /// An error after Complete preserves the completed logical outcome and is reported by the runtime. + /// + /// + ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken); +} diff --git a/src/Orleans.Messaging/IInboxHandlerContext.cs b/src/Orleans.Messaging/IInboxHandlerContext.cs new file mode 100644 index 00000000000..f729a38349f --- /dev/null +++ b/src/Orleans.Messaging/IInboxHandlerContext.cs @@ -0,0 +1,31 @@ +namespace Orleans.Messaging; + +/// +/// Exposes the received message and its attempt-scoped logical completion. +/// +public interface IInboxHandlerContext +{ + /// + /// Gets the received message and its finalized application bytes. + /// + /// + /// The context keeps the message available through actual handler completion, including + /// after Complete removes the pending message. Keep its payload and headers unchanged while handling the command. + /// + InboxMessage Message { get; } + + /// + /// Synchronously stages inbox completion and transport deduplication for the active attempt. + /// + /// + /// Apply safe-to-commit business mutations and stage outgoing messages before calling Complete + /// in the same synchronous final block. Return from the handler without further awaits. + /// The runtime owns the subsequent journal write and acknowledgement. + /// Repeated completion within the same active attempt coalesces. Completion retains its + /// logical outcome when cancellation arrives after the final block starts. + /// + /// + /// The context is retired or belongs to another activation or handler attempt. + /// + void Complete(); +} diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs new file mode 100644 index 00000000000..5d42d1fe902 --- /dev/null +++ b/src/Orleans.Messaging/IOutbox.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Stages opaque durable messages alongside a grain's journaled business state. +/// +/// +/// The journal capture hook establishes a durable self-wakeup before capturing pending state. +/// Dispatch begins after persistence acknowledgement. Messages remain pending until the destination +/// acknowledges durable acceptance, recognizes a duplicate, or reports a terminal delivery outcome. +/// Applications namespace command identities and define ordering in their protocols. +/// +public interface IOutbox +{ + /// + /// Gets the grain identity which owns this outbox and sends its messages. + /// + GrainId SenderId { get; } + + /// + /// Gets the number of pending outbound messages. + /// + int Count { get; } + + /// + /// Gets pending messages in unspecified order. + /// + /// Messages share their payload arrays, header dictionaries, and header value arrays, which remain unchanged after publication. + IEnumerable Messages { get; } + + /// + /// Synchronously stages an outgoing command for the grain's next journal write. + /// + /// The fully built outgoing message and required receiver. + /// + /// Direct sends share the payload array, header dictionary, and header value arrays. Keep these objects unchanged after publication. + /// A supplied sender must identify this outbox's grain. Equivalent repeated identities retain + /// the original message and headers; payload, optional subject, and destination must match. + /// Custom headers and sender provenance do not change pending intent equality. Conflicts fail explicitly. + /// Inbox handlers stage outgoing + /// messages in their synchronous final block before calling . + /// Ordinary callers persist staged messages using their journaled state manager. + /// An explicit write retry retains pending business changes and messages after a scheduling failure. + /// + /// The envelope has an unset identity, invalid subject or destination, or exceeds identity/subject limits. + /// + /// The sender differs from the owning grain, the identity conflicts, or the outbox is unavailable. + /// + void Send(OutboxMessage message); + + /// + /// Looks up a pending outbound message. + /// + /// The message identifier. + /// The matching stored message when found. + /// Whether the message is pending. + bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage message); +} diff --git a/src/Orleans.Messaging/InboxHandlerContext.cs b/src/Orleans.Messaging/InboxHandlerContext.cs new file mode 100644 index 00000000000..a9278c2d532 --- /dev/null +++ b/src/Orleans.Messaging/InboxHandlerContext.cs @@ -0,0 +1,12 @@ +using System; + +namespace Orleans.Messaging; + +internal sealed class InboxHandlerContext(InboxMessage message, Action complete) : IInboxHandlerContext +{ + private readonly Action _complete = complete ?? throw new ArgumentNullException(nameof(complete)); + + public InboxMessage Message { get; } = message; + + public void Complete() => _complete(); +} diff --git a/src/Orleans.Messaging/InboxMessage.cs b/src/Orleans.Messaging/InboxMessage.cs new file mode 100644 index 00000000000..b2d6b6004ca --- /dev/null +++ b/src/Orleans.Messaging/InboxMessage.cs @@ -0,0 +1,10 @@ +namespace Orleans.Messaging; + +/// A received command whose destination is the grain owning the inbox. +[GenerateSerializer, Alias("Orleans.Messaging.InboxMessage")] +public readonly struct InboxMessage +{ + /// Gets the command identity, payload, and application metadata. + [Id(0)] + public required Envelope Envelope { get; init; } +} diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs new file mode 100644 index 00000000000..6cb3e4ed292 --- /dev/null +++ b/src/Orleans.Messaging/MessageState.cs @@ -0,0 +1,65 @@ +using System; + +namespace Orleans.Messaging; + +[GenerateSerializer, Alias("Orleans.Messaging.InboxMessageState")] +internal sealed class InboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessageState")] +internal sealed class OutboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + + [Id(3)] + public DateTimeOffset? EnqueuedAt { get; set; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxDeadLetter")] +internal sealed class InboxDeadLetter +{ + [Id(0)] + public required InboxMessage Message { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxDeadLetter")] +internal sealed class OutboxDeadLetter +{ + [Id(0)] + public required OutboxMessage Message { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } +} diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj new file mode 100644 index 00000000000..58a2ac7ff4a --- /dev/null +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -0,0 +1,28 @@ + + + Microsoft.Orleans.Messaging + Microsoft Orleans Messaging + Messaging envelope, delivery, and handler-routing contracts for Microsoft Orleans. + $(PackageTags) Messaging Inbox Outbox + false + + true + true + $(DefaultTargetFrameworks) + disable + enable + $(NoWarn);ORLEANSEXP005 + $(VersionSuffix).alpha.1 + alpha.1 + + + + + + + + + + + + diff --git a/src/Orleans.Messaging/OrleansContracts.txt b/src/Orleans.Messaging/OrleansContracts.txt new file mode 100644 index 00000000000..4a090bf615d --- /dev/null +++ b/src/Orleans.Messaging/OrleansContracts.txt @@ -0,0 +1,14 @@ +# This file is generated by the Orleans contract analyzer. +# To regenerate this project from the repository root: +# dotnet format PATH_TO_PROJECT.csproj analyzers --severity info --diagnostics ORLEANS0016 ORLEANS0017 ORLEANS0018 ORLEANS0019 ORLEANS0020 ORLEANS0022 ORLEANS0023 ORLEANS0024 +# Run the command once per contract project; do not pass a .sln or .slnx path. +# Verify with: dotnet build PATH_TO_PROJECT.csproj +# The regeneration command edits this manifest only; it does not change source attributes. +# OrleansContracts format: 2 +# Method lines use: wire-identity: CLR-signature. +# The identity is the identifier Orleans uses at runtime, whether generated or declared in source. +# Review every diff: identity or signature changes can break wire compatibility during rolling upgrades. +# Details: https://aka.ms/orleans/OrleansContracts.txt + +interface [GrainInterfaceType("Orleans.Messaging.IInboxExtension")] Orleans.Messaging.IInboxExtension [Version(0)] + DeliverAsync: DeliverAsync(Orleans.Messaging.InboxMessage, System.Threading.CancellationToken) -> ValueTask diff --git a/src/Orleans.Messaging/OutboxMessage.cs b/src/Orleans.Messaging/OutboxMessage.cs new file mode 100644 index 00000000000..4a4ee561998 --- /dev/null +++ b/src/Orleans.Messaging/OutboxMessage.cs @@ -0,0 +1,20 @@ +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// An outgoing command with destination routing separate from its common envelope. +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessage")] +public readonly struct OutboxMessage +{ + /// Gets the command identity, payload, and application metadata. + [Id(0)] + public required Envelope Envelope { get; init; } + + /// Gets the required destination grain identity. + [Id(1)] + public required GrainId ReceiverId { get; init; } + + /// Creates the incoming view by sharing the envelope, payload, and headers. + /// A received message whose destination is implied by the receiving grain. + public InboxMessage ToInboxMessage() => new() { Envelope = Envelope }; +} diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md new file mode 100644 index 00000000000..71c4a2bb8d2 --- /dev/null +++ b/src/Orleans.Messaging/README.md @@ -0,0 +1,33 @@ +# Microsoft Orleans Messaging + +This intermediate project supplies application command identities, envelopes, distinct incoming and outgoing +messages, and handler contracts. It remains non-packable while runtime and hosting layers assemble +`Microsoft.Orleans.Messaging`. + +`Envelope` has a required `HierarchicalKey MessageId` and non-null `byte[] Payload`, which can be empty. +`Subject` is first-class optional metadata. Raw handlers accept identity and payload alone; typed protocols +supply their subjects. `Headers` is an optional `Dictionary` for application-defined metadata, +with nonempty keys and non-null arrays. The optional `"sender"` header stores a grain identity encoded using +an Orleans serializer, supplied and decoded by the consuming protocol. Use an ordinal dictionary for case-sensitive header names. Applications define their payload format and +can use ordinary Orleans serializers to encode and decode payloads and header values. + +Prepare payload arrays, header dictionaries, and header value arrays before publication and keep them unchanged +afterward. Direct staging, struct assignment, and local delivery share those objects. Ordinary Orleans deep copying +and deserialization produce independent payload arrays, dictionaries, and header value arrays. + +`OutboxMessage` adds the required destination `ReceiverId`. `ToInboxMessage()` shares its envelope with an +`InboxMessage`, whose receiver is the grain owning the inbox. Pending incoming equality compares exact ID, +payload, and optional ordinal subject; outgoing equality also compares destination. Equivalent submissions +retain the first message and its original headers, including sender provenance. Completed records deduplicate by +receiver-local message ID for their configured lifetime. + +Admission validates the identity's 1,024 UTF-8 byte and 32 segment limits, a non-null payload, extra header +keys and values, and any supplied subject before shared mutation. Present subjects have 1 to 256 UTF-8 bytes. +`HierarchicalKey` is a readonly ordinal value with an immutable canonical path and cached hash. `Create` and +`CreateChildKey` accept literal segments; `Parse` reads escaped canonical paths; `Append` composes hierarchies. +The default key is unset. Serialization stores the canonical path and reconstructs navigation and hash state. + +`IInboxHandler.HandleAsync` receives an `InboxMessage` through its context and explicitly calls `Complete()`. +Decode local results, perform asynchronous preparation, build outgoing messages, and check cancellation before +the first shared mutation. From that mutation through method return, execute synchronously: apply safe-to-commit +changes, send through `IOutbox`, complete, and return. The runtime owns persistence and acknowledgement. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs new file mode 100644 index 00000000000..746510e9f65 --- /dev/null +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -0,0 +1,393 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ +namespace Orleans.Messaging +{ + [GenerateSerializer] + [Alias("Orleans.Messaging.DeliveryResult")] + public readonly partial struct DeliveryResult + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + [Id(1)] + public string? Message { get { throw null; } init { } } + + [Id(0)] + public DeliveryStatus Status { get { throw null; } init { } } + + public static DeliveryResult Accepted() { throw null; } + + public static DeliveryResult Backpressured() { throw null; } + + public static DeliveryResult DeadLettered(string reason) { throw null; } + + public static DeliveryResult Duplicate() { throw null; } + + public static DeliveryResult HandlerNotFound() { throw null; } + } + + public enum DeliveryStatus + { + Accepted = 0, + Duplicate = 1, + Backpressured = 2, + HandlerNotFound = 3, + DeadLettered = 4 + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.Envelope")] + public readonly partial struct Envelope + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + [Id(3)] + public System.Collections.Generic.Dictionary? Headers { get { throw null; } init { } } + + [Id(0)] + public required HierarchicalKey MessageId { get { throw null; } init { } } + + [Id(1)] + public required byte[] Payload { get { throw null; } init { } } + + [Id(2)] + public string? Subject { get { throw null; } init { } } + } + + [Immutable] + [Alias("Orleans.Messaging.HierarchicalKey")] + public readonly partial struct HierarchicalKey : System.ISpanFormattable, System.IFormattable, System.IEquatable, System.IParsable, System.ISpanParsable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public const char EscapeCharacter = '\\'; + public const char SegmentSeparator = '/'; + public bool IsDefault { get { throw null; } } + + public int Length { get { throw null; } } + + public int SegmentCount { get { throw null; } } + + public readonly HierarchicalKey Append(HierarchicalKey suffix) { throw null; } + + public static HierarchicalKey Create(scoped params System.ReadOnlySpan values) { throw null; } + + public static HierarchicalKey Create(string value) { throw null; } + + public readonly HierarchicalKey CreateChildKey(string value) { throw null; } + + public readonly bool Equals(HierarchicalKey other) { throw null; } + + public override readonly bool Equals(object? obj) { throw null; } + + public readonly SegmentEnumerator GetEnumerator() { throw null; } + + public override readonly int GetHashCode() { throw null; } + + public readonly HierarchicalKey? GetParent() { throw null; } + + public readonly bool IsAncestorOf(HierarchicalKey other) { throw null; } + + public readonly bool IsChildOf(HierarchicalKey other) { throw null; } + + public readonly bool IsParentOf(HierarchicalKey other) { throw null; } + + public static bool operator ==(HierarchicalKey left, HierarchicalKey right) { throw null; } + + public static bool operator !=(HierarchicalKey left, HierarchicalKey right) { throw null; } + + static HierarchicalKey System.ISpanParsable.Parse(System.ReadOnlySpan s, System.IFormatProvider? provider) { throw null; } + + static HierarchicalKey System.IParsable.Parse(string s, System.IFormatProvider? provider) { throw null; } + + public override readonly string ToString() { throw null; } + + public readonly string ToString(string? format, System.IFormatProvider? formatProvider) { throw null; } + + public readonly bool TryFormat(System.Span destination, out int charsWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } + + static bool System.ISpanParsable.TryParse(System.ReadOnlySpan s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + static bool System.IParsable.TryParse(string? s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + public ref partial struct SegmentEnumerator + { + private object _dummy; + private int _dummyPrimitive; + public System.ReadOnlySpan Current { get { throw null; } } + + public bool MoveNext() { throw null; } + } + } + + public partial interface IInbox + { + int Capacity { get; } + + int Count { get; } + + System.Collections.Generic.IEnumerable Messages { get; } + + void RegisterHandler(IInboxHandler handler); + bool TryGetMessage(HierarchicalKey messageId, out InboxMessage message); + } + + [Alias("IInboxExtension")] + public partial interface IInboxExtension : Runtime.IGrainExtension, Runtime.IAddressable + { + [Alias("DeliverAsync")] + System.Threading.Tasks.ValueTask DeliverAsync(InboxMessage message, System.Threading.CancellationToken cancellationToken = default); + } + + public partial interface IInboxHandler + { + System.Threading.Tasks.ValueTask HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + } + + public partial interface IInboxHandlerContext + { + InboxMessage Message { get; } + + void Complete(); + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.InboxMessage")] + public readonly partial struct InboxMessage + { + [Id(0)] + public required Envelope Envelope { get { throw null; } init { } } + } + + public partial interface IOutbox + { + int Count { get; } + + System.Collections.Generic.IEnumerable Messages { get; } + + Runtime.GrainId SenderId { get; } + + void Send(OutboxMessage message); + bool TryGetMessage(HierarchicalKey messageId, out OutboxMessage message); + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.OutboxMessage")] + public readonly partial struct OutboxMessage + { + [Id(0)] + public required Envelope Envelope { get { throw null; } init { } } + + [Id(1)] + public required Runtime.GrainId ReceiverId { get { throw null; } init { } } + + public readonly InboxMessage ToInboxMessage() { throw null; } + } +} + +namespace Orleans.Messaging.Configuration +{ + public partial class InboxOptions + { + public System.TimeSpan BackpressureRetryDelay { get { throw null; } set { } } + + public System.TimeSpan DeadLetterRetentionPeriod { get { throw null; } set { } } + + public System.TimeSpan DeduplicationWindow { get { throw null; } set { } } + + public int InboxBatchSize { get { throw null; } set { } } + + public int MaxCapacity { get { throw null; } set { } } + + public int MaxDeliveryAttempts { get { throw null; } set { } } + + public System.TimeSpan MaxOutboxRetryAge { get { throw null; } set { } } + + public int MaxProcessingAttempts { get { throw null; } set { } } + + public int MaxRetainedDeadLetters { get { throw null; } set { } } + + public int OutboxBatchSize { get { throw null; } set { } } + + public System.TimeSpan OutboxIdleRetirementGracePeriod { get { throw null; } set { } } + + public void Validate() { } + } +} + +namespace OrleansCodeGen.Orleans.Messaging +{ + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_DeliveryResult : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_DeliveryResult(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.DeliveryResult instance) { } + + public global::Orleans.Messaging.DeliveryResult ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.DeliveryResult instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.DeliveryResult value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_Envelope : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_Envelope(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.Envelope instance) { } + + public global::Orleans.Messaging.Envelope ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.Envelope instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.Envelope value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_InboxMessage : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_InboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.InboxMessage instance) { } + + public global::Orleans.Messaging.InboxMessage ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.InboxMessage instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.InboxMessage value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 instance) { } + + public Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_OutboxMessage : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_OutboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.OutboxMessage instance) { } + + public global::Orleans.Messaging.OutboxMessage ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.OutboxMessage instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.OutboxMessage value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_Envelope : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_Envelope(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.Envelope DeepCopy(global::Orleans.Messaging.Envelope original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_InboxMessage : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_InboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.InboxMessage DeepCopy(global::Orleans.Messaging.InboxMessage original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 DeepCopy(Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_OutboxMessage : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_OutboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.OutboxMessage DeepCopy(global::Orleans.Messaging.OutboxMessage original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + [global::Orleans.CompoundTypeAlias(new[] { "inv", typeof(global::Orleans.Runtime.GrainReference), "Ext", typeof(global::Orleans.Messaging.IInboxExtension), typeof(global::Orleans.Messaging.IInboxExtension), "DeliverAsync" })] + public sealed partial class Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Runtime.Request + { + public global::Orleans.Messaging.InboxMessage arg0; + public System.Threading.CancellationToken arg1; + public override bool IsCancellable { get { throw null; } } + + public override void Dispose() { } + + public override string GetActivityName() { throw null; } + + public override object GetArgument(int index) { throw null; } + + public override int GetArgumentCount() { throw null; } + + public override System.Threading.CancellationToken GetCancellationToken() { throw null; } + + public override string GetInterfaceName() { throw null; } + + public override System.Type GetInterfaceType() { throw null; } + + public override System.Reflection.MethodInfo GetMethod() { throw null; } + + public override string GetMethodName() { throw null; } + + public override object GetTarget() { throw null; } + + protected override System.Threading.Tasks.ValueTask InvokeInner() { throw null; } + + public override void SetArgument(int index, object value) { } + + public override void SetTarget(global::Orleans.Serialization.Invocation.ITargetHolder holder) { } + + public override bool TryCancel() { throw null; } + } +} \ No newline at end of file diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs new file mode 100644 index 00000000000..5d6af665177 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -0,0 +1,105 @@ +using Orleans.Messaging.Configuration; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DeliveryAndOptionsContractTests +{ + [Fact] + public void DeliveryResult_EachFactory_PreservesStatusAndPayload() + { + var handlerMissing = DeliveryResult.HandlerNotFound(); + var deadLettered = DeliveryResult.DeadLettered("poison body"); + + Assert.Equal(DeliveryStatus.Accepted, DeliveryResult.Accepted().Status); + Assert.Equal(DeliveryStatus.Duplicate, DeliveryResult.Duplicate().Status); + Assert.Equal(DeliveryStatus.Backpressured, DeliveryResult.Backpressured().Status); + Assert.Equal(DeliveryStatus.HandlerNotFound, handlerMissing.Status); + Assert.Equal("No inbox handler is registered.", handlerMissing.Message); + Assert.Equal(DeliveryStatus.DeadLettered, deadLettered.Status); + Assert.Equal("poison body", deadLettered.Message); + } + + [Fact] + public void DeliveryStatus_AllValues_HaveContiguousDistinctValues() + { + Assert.Equal( + [ + DeliveryStatus.Accepted, + DeliveryStatus.Duplicate, + DeliveryStatus.Backpressured, + DeliveryStatus.HandlerNotFound, + DeliveryStatus.DeadLettered + ], + Enum.GetValues()); + Assert.Equal([0, 1, 2, 3, 4], Enum.GetValues().Select(static value => (int)value)); + } + + [Fact] + public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() + { + var options = new InboxOptions(); + + options.Validate(); + + Assert.Equal(1000, options.MaxCapacity); + Assert.Equal(TimeSpan.FromDays(7), options.DeduplicationWindow); + Assert.Equal(TimeSpan.FromDays(1), options.MaxOutboxRetryAge); + Assert.Equal(TimeSpan.FromDays(30), options.DeadLetterRetentionPeriod); + Assert.Equal(5, options.MaxProcessingAttempts); + Assert.Equal(100, options.MaxDeliveryAttempts); + Assert.Equal(1000, options.MaxRetainedDeadLetters); + Assert.Equal(32, options.InboxBatchSize); + Assert.Equal(32, options.OutboxBatchSize); + Assert.Equal(TimeSpan.FromMilliseconds(100), options.OutboxIdleRetirementGracePeriod); + } + + [Fact] + public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContract() + { + var invalidCases = new (string Parameter, Action Mutate)[] + { + (nameof(InboxOptions.MaxCapacity), options => options.MaxCapacity = 0), + (nameof(InboxOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.Zero), + (nameof(InboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.Zero), + (nameof(InboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.MaxValue), + (nameof(InboxOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), + (nameof(InboxOptions.MaxDeliveryAttempts), options => options.MaxDeliveryAttempts = 0), + (nameof(InboxOptions.MaxOutboxRetryAge), options => options.MaxOutboxRetryAge = TimeSpan.Zero), + (nameof(InboxOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), + (nameof(InboxOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), + (nameof(InboxOptions.InboxBatchSize), options => options.InboxBatchSize = 0), + (nameof(InboxOptions.OutboxBatchSize), options => options.OutboxBatchSize = 0), + (nameof(InboxOptions.OutboxIdleRetirementGracePeriod), options => options.OutboxIdleRetirementGracePeriod = TimeSpan.FromTicks(-1)), + (nameof(InboxOptions.OutboxIdleRetirementGracePeriod), options => options.OutboxIdleRetirementGracePeriod = TimeSpan.MaxValue), + }; + + foreach (var (parameter, mutate) in invalidCases) + { + var options = new InboxOptions(); + mutate(options); + var exception = Assert.Throws(options.Validate); + Assert.Equal(parameter, exception.ParamName); + } + } + + [Fact] + public void Validate_MaxOutboxRetryAgeNotLessThanDeduplicationWindow_FailsAtBoundaryAndAbove() + { + foreach (var retryAge in new[] { TimeSpan.FromHours(2), TimeSpan.FromHours(3) }) + { + var options = new InboxOptions + { + DeduplicationWindow = TimeSpan.FromHours(2), + MaxOutboxRetryAge = retryAge, + }; + + var exception = Assert.Throws(options.Validate); + Assert.Equal(nameof(InboxOptions.MaxOutboxRetryAge), exception.ParamName); + Assert.Contains("less than DeduplicationWindow", exception.Message, StringComparison.Ordinal); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs new file mode 100644 index 00000000000..432079782aa --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -0,0 +1,293 @@ +using System.Reflection; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class EnvelopeContractTests +{ + private static HierarchicalKey Id => HierarchicalKey.Create("orders", "123"); + private static GrainId Receiver => GrainId.Create("receiver", "123"); + + [Fact] + public void RequiredPayloadAndOptionalMetadata() + { + var envelope = Create([]); + Validate(envelope); + Validate(new InboxMessage { Envelope = envelope }); + Assert.Equal(Id, envelope.MessageId); + Assert.Empty(envelope.Payload); + Assert.Null(envelope.Subject); + Assert.Null(envelope.Headers); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var decoded = serializer.Deserialize(serializer.SerializeToArray(envelope)); + Validate(decoded); + Assert.Empty(decoded.Payload); + Assert.Null(decoded.Subject); + Assert.Null(decoded.Headers); + Assert.Throws(() => Validate(new OutboxMessage { Envelope = envelope, ReceiverId = default })); + Assert.Throws(() => Validate(default(Envelope))); + Assert.Throws(() => Validate(envelope with { Payload = null! })); + } + + [Fact] + public void FirstClassSubjectAndEncodedSenderUseByteHeaders() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var senderSerializer = services.GetRequiredService>(); + var sender = GrainId.Create("sender", "123"); + var envelope = Create([1]) with + { + Subject = "orders.v1", + Headers = new(StringComparer.Ordinal) + { + ["subject"] = "extra subject"u8.ToArray(), + ["sender"] = senderSerializer.SerializeToArray(sender), + ["custom"] = "first"u8.ToArray(), + ["CUSTOM"] = "second"u8.ToArray() + } + }; + Validate(envelope); + Assert.Equal("orders.v1", envelope.Subject); + Assert.Equal(sender, senderSerializer.Deserialize(envelope.Headers["sender"])); + Assert.Equal("extra subject"u8.ToArray(), envelope.Headers["subject"]); + Assert.Equal("first"u8.ToArray(), envelope.Headers["custom"]); + Assert.Equal("second"u8.ToArray(), envelope.Headers["CUSTOM"]); + Assert.False(envelope.Headers.TryAdd("custom", "replacement"u8.ToArray())); + Assert.Equal("first"u8.ToArray(), envelope.Headers["custom"]); + var serializer = services.GetRequiredService>(); + var decoded = serializer.Deserialize(serializer.SerializeToArray(envelope)); + Assert.NotNull(decoded.Headers); + Assert.Equal("orders.v1", decoded.Subject); + Assert.Equal(sender, senderSerializer.Deserialize(decoded.Headers["sender"])); + Assert.NotSame(envelope.Headers["sender"], decoded.Headers["sender"]); + Assert.Equal("extra subject"u8.ToArray(), decoded.Headers["subject"]); + } + + [Fact] + public void StructAssignmentAndLocalConversionSharePublishedObjects() + { + var envelope = Create([0, 255, 128]) with { Headers = new(StringComparer.Ordinal) { ["trace"] = "original"u8.ToArray() } }; + var assigned = envelope; + var outgoing = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + var incoming = outgoing.ToInboxMessage(); + Assert.Same(envelope.Payload, assigned.Payload); + Assert.Same(envelope.Headers, assigned.Headers); + Assert.Same(envelope.Payload, incoming.Envelope.Payload); + Assert.Same(envelope.Headers, incoming.Envelope.Headers); + Assert.Equal(Id, incoming.Envelope.MessageId); + Assert.Null(typeof(InboxMessage).GetProperty("ReceiverId")); + Assert.Null(typeof(Envelope).GetProperty("ReceiverId")); + } + + [Theory] + [InlineData(0)] [InlineData(7)] [InlineData(100000)] + public void OrdinaryCopyAndIndependentWireRoundTrip(int length) + { + using var sending = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var receiving = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var payload = Enumerable.Range(0, length).Select(i => (byte)i).ToArray(); + var envelope = Create(payload) with + { + Subject = "orders.v1", + Headers = new(StringComparer.Ordinal) { ["custom"] = "retained"u8.ToArray(), ["CUSTOM"] = "distinct"u8.ToArray(), ["empty"] = [] } + }; + var serializer = sending.GetRequiredService>(); + var wire = serializer.SerializeToArray(envelope); + var decoded = receiving.GetRequiredService>().Deserialize(wire); + var copied = sending.GetRequiredService().Copy(envelope); + foreach (var actual in new[] { decoded, copied }) + { + Validate(actual); + Assert.Equal(Id, actual.MessageId); + Assert.Equal(payload, actual.Payload); + Assert.Equal("orders.v1", actual.Subject); + Assert.NotNull(actual.Headers); + Assert.NotSame(envelope.Headers, actual.Headers); + Assert.Equal("retained"u8.ToArray(), actual.Headers["custom"]); + Assert.Equal("distinct"u8.ToArray(), actual.Headers["CUSTOM"]); + Assert.Empty(actual.Headers["empty"]); + Assert.NotSame(envelope.Headers["custom"], actual.Headers["custom"]); + actual.Headers["custom"][0] = 0; + Assert.Equal("retained"u8.ToArray(), envelope.Headers["custom"]); + if (length > 0) + { + Assert.NotSame(envelope.Payload, actual.Payload); + actual.Payload[0] = 42; + Assert.Equal((byte)0, envelope.Payload[0]); + } + } + Assert.Equal(wire, serializer.SerializeToArray(envelope)); + } + + [Fact] + public void PayloadUsesOrdinaryExternallyBoundOrleansSerializer() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var envelope = Create(serializer.SerializeToArray("typed application payload")) with { Subject = "typed.v1" }; + Assert.Equal("typed application payload", serializer.Deserialize(envelope.Payload)); + Assert.Equal("typed.v1", envelope.Subject); + Assert.Null(envelope.Headers); + } + + [Theory] + [InlineData("InboxMessage")] [InlineData("OutboxMessage")] + [InlineData("InboxDeadLetter")] [InlineData("OutboxDeadLetter")] + public void MessageAndDeadLetterCopyPreservesMetadataAndIsolation(string kind) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var envelope = Create([0, 255]) with { Subject = "subject.v1", Headers = new() { ["custom"] = "value"u8.ToArray() } }; + object value = kind.StartsWith("Inbox", StringComparison.Ordinal) ? new InboxMessage { Envelope = envelope } + : new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + if (kind.EndsWith("DeadLetter", StringComparison.Ordinal)) + { + var type = typeof(Envelope).Assembly.GetType("Orleans.Messaging." + kind, true)!; + var record = Activator.CreateInstance(type)!; + type.GetProperty("Message")!.SetValue(record, value); + type.GetProperty("DeadLetteredAt")!.SetValue(record, DateTimeOffset.UnixEpoch); + type.GetProperty("Reason")!.SetValue(record, "failure"); + type.GetProperty("AttemptCount")!.SetValue(record, 3); + value = record; + } + var serializer = services.GetRequiredService(); + foreach (var result in new[] { serializer.Deserialize(serializer.SerializeToArray(value)), services.GetRequiredService().Copy(value) }) + { + Assert.NotNull(result); + var message = kind.EndsWith("DeadLetter", StringComparison.Ordinal) ? result.GetType().GetProperty("Message")!.GetValue(result)! : result; + var actual = message is InboxMessage incoming ? incoming.Envelope : Assert.IsType(message).Envelope; + Assert.Equal(new byte[] { 0, 255 }, actual.Payload); + Assert.NotSame(envelope.Payload, actual.Payload); + Assert.Equal("subject.v1", actual.Subject); + Assert.NotNull(actual.Headers); + Assert.NotSame(envelope.Headers, actual.Headers); + Assert.Equal("value"u8.ToArray(), actual.Headers["custom"]); + Assert.NotSame(envelope.Headers!["custom"], actual.Headers["custom"]); + if (message is OutboxMessage outgoing) Assert.Equal(Receiver, outgoing.ReceiverId); + if (kind.EndsWith("DeadLetter", StringComparison.Ordinal)) Assert.Equal(3, result.GetType().GetProperty("AttemptCount")!.GetValue(result)); + } + } + + [Fact] + public void SimpleEnvelopeSurfaceUsesGeneratedOrdinaryFields() + { + AssertIds(typeof(Envelope), ("MessageId", 0u), ("Payload", 1u), ("Subject", 2u), ("Headers", 3u)); + AssertIds(typeof(InboxMessage), ("Envelope", 0u)); + AssertIds(typeof(OutboxMessage), ("Envelope", 0u), ("ReceiverId", 1u)); + Assert.Equal(typeof(Dictionary), typeof(Envelope).GetProperty("Headers")!.PropertyType); + Assert.Equal(typeof(byte[]), typeof(Envelope).GetProperty("Payload")!.PropertyType); + Assert.Null(typeof(Envelope).GetCustomAttribute()); + Assert.Single(typeof(Envelope).GetCustomAttributes()); + Assert.Equal("Orleans.Messaging.Envelope", typeof(Envelope).GetCustomAttribute()!.Alias); + Assert.Equal(new[] { "Headers", "MessageId", "Payload", "Subject" }, typeof(Envelope).GetProperties().Select(p => p.Name).Order(StringComparer.Ordinal)); + Assert.Null(typeof(Envelope).Assembly.GetType("Orleans.Messaging.PackedEnvelopeHeaders")); + Assert.Null(typeof(Envelope).Assembly.GetType("Orleans.Messaging.EnvelopeBuilder")); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var outgoing = new OutboxMessage { Envelope = Create([7]), ReceiverId = Receiver }; + var other = outgoing with { ReceiverId = GrainId.Create("receiver", "other") }; + var serializer = services.GetRequiredService>(); + Assert.Equal(serializer.SerializeToArray(outgoing.ToInboxMessage()), serializer.SerializeToArray(other.ToInboxMessage())); + } + + [Theory] + [InlineData("same", true, true)] [InlineData("headers", true, true)] [InlineData("sender", true, true)] + [InlineData("receiver", false, true)] [InlineData("id", false, false)] [InlineData("child", false, false)] + [InlineData("subject", false, false)] [InlineData("subject-case", false, false)] [InlineData("no-subject", false, false)] + [InlineData("payload", false, false)] [InlineData("payload-length", false, false)] + public void PendingCommandEqualityIgnoresExtraHeadersAndSender(string change, bool outbox, bool inbox) + { + var envelope = Create([0, 255, 128]) with { Subject = "subject.v1", Headers = new() { ["custom"] = "first"u8.ToArray() } }; + var original = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + var repeated = new OutboxMessage + { + ReceiverId = change == "receiver" ? GrainId.Create("receiver", "other") : Receiver, + Envelope = envelope with + { + MessageId = change == "id" ? HierarchicalKey.Create("other") : change == "child" ? Id.CreateChildKey("child") : Id, + Payload = change == "payload" ? [0, 255, 129] : change == "payload-length" ? [0, 255] : [0, 255, 128], + Subject = change == "subject" ? "other" : change == "subject-case" ? "Subject.v1" : change == "no-subject" ? null : "subject.v1", + Headers = new() { ["custom"] = Encoding.UTF8.GetBytes(change == "headers" ? "second" : "first"), ["sender"] = change == "sender" ? [2] : [1] } + } + }; + Assert.Equal(outbox, Invoke("EnvelopeEquivalence", "AreEquivalent", original, repeated)); + Assert.Equal(inbox, Invoke("EnvelopeEquivalence", "AreSameCommand", original.ToInboxMessage(), repeated.ToInboxMessage())); + Assert.Equal(inbox, Invoke("EnvelopeEquivalence", "AreSameCommand", original.Envelope, repeated.Envelope)); + Assert.Equal("first"u8.ToArray(), original.Envelope.Headers!["custom"]); + Assert.Equal(new byte[] { 0, 255, 128 }, original.Envelope.Payload); + } + + [Fact] + public void PendingRawCommandsAllowAbsentSubject() + { + var first = Create([]); + var repeated = first with { Headers = new() { ["custom"] = "different"u8.ToArray() } }; + Assert.True(Invoke("EnvelopeEquivalence", "AreSameCommand", first, repeated)); + Assert.Null(first.Subject); + Assert.Null(first.Headers); + } + + [Theory] + [InlineData("id", 1024, true)] [InlineData("id", 1025, false)] + [InlineData("depth", 32, true)] [InlineData("depth", 33, false)] + [InlineData("unicode-id", 512, true)] [InlineData("unicode-id", 513, false)] + [InlineData("subject", 256, true)] [InlineData("subject", 257, false)] + [InlineData("unicode-subject", 128, true)] [InlineData("unicode-subject", 129, false)] + public void IdentityAndSubjectAdmissionBoundaries(string field, int length, bool valid) + { + var envelope = Create([]) with + { + MessageId = field switch + { + "id" => HierarchicalKey.Create(new string('x', length)), + "depth" => HierarchicalKey.Create(Enumerable.Repeat("x", length).ToArray()), + "unicode-id" => HierarchicalKey.Create(new string('\u00e9', length)), + _ => Id + }, + Subject = field.Contains("subject", StringComparison.Ordinal) ? new string(field == "subject" ? 's' : '\u00e9', length) : null + }; + if (valid) { Validate(envelope); Assert.Empty(envelope.Payload); } + else Assert.Throws(() => Validate(envelope)); + } + + [Theory] + [InlineData("empty-subject")] [InlineData("invalid-subject")] [InlineData("empty-header-key")] [InlineData("null-header-value")] + public void InvalidMetadataIsRejectedBeforeAdmission(string invalid) + { + var envelope = Create([1]) with + { + Subject = invalid == "empty-subject" ? "" : invalid == "invalid-subject" ? "\ud800" : null, + Headers = invalid == "empty-header-key" ? new() { [""] = "value"u8.ToArray() } : invalid == "null-header-value" ? new() { ["key"] = null! } : null + }; + Assert.ThrowsAny(() => Validate(envelope)); + Assert.ThrowsAny(() => Validate(new InboxMessage { Envelope = envelope })); + Assert.ThrowsAny(() => Validate(new OutboxMessage { Envelope = envelope, ReceiverId = Receiver })); + Assert.Equal(new byte[] { 1 }, envelope.Payload); + } + + private static Envelope Create(byte[] payload) => new() { MessageId = Id, Payload = payload }; + private static void Validate(T message) => Invoke("EnvelopeValidation", "Validate", message!); + private static T Invoke(string type, string method, params object[] args) + { + var helper = typeof(Envelope).Assembly.GetType("Orleans.Messaging." + type, true)!; + try { return (T)helper.GetMethod(method, args.Select(a => a.GetType()).ToArray())!.Invoke(null, args)!; } + catch (TargetInvocationException exception) when (exception.InnerException is { } cause) + { + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(cause).Throw(); + throw; + } + } + private static void AssertIds(Type type, params (string Name, uint Id)[] expected) + { + var actual = type.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .SelectMany(member => member.GetCustomAttributes().Select(id => (member.Name, id.Id))) + .OrderBy(p => p.Id).ToArray(); + Assert.Equal(expected.OrderBy(p => p.Id), actual); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs new file mode 100644 index 00000000000..9a254f1ebf8 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs @@ -0,0 +1,204 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class HandlerRoutingContractTests +{ + [Theory] + [InlineData(1)] + [InlineData(2)] + public void HandlerContext_Complete_ForwardsToOwnerSynchronously(int calls) + { + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => count++); + + for (var i = 0; i < calls; i++) + { + context.Complete(); + Assert.Equal(i + 1, count); + } + + Assert.Equal(calls, count); + AssertEnvelope(envelope, context.Message.Envelope); + } + + [Fact] + public void HandlerContext_Complete_PropagatesOriginalCallbackFailureWithoutChangingEnvelope() + { + var sentinel = new InvalidOperationException("owner failure"); + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => + { + count++; + throw sentinel; + }); + + var exception = Assert.Throws(context.Complete); + + Assert.Same(sentinel, exception); + Assert.Equal(1, count); + AssertEnvelope(envelope, context.Message.Envelope); + } + + [Fact] + public void HandlerContext_Envelope_PreservesIdentityAndPayload() + { + byte[] source = [0x00, 0xff, 0x80]; + var envelope = Envelope(source.ToArray()); + var count = 0; + var context = CreateContext(envelope, () => count++); + Array.Fill(source, (byte)0x42); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + + var decoded = services.GetRequiredService>() + .Deserialize(services.GetRequiredService>().SerializeToArray(context.Message.Envelope)); + context.Complete(); + + Assert.Equal(1, count); + AssertEnvelope(envelope, context.Message.Envelope); + Assert.Equal(envelope.MessageId, decoded.MessageId); + Assert.Equal(envelope.Subject, decoded.Subject); + Assert.Equal(new byte[] { 0x00, 0xff, 0x80 }, decoded.Payload.ToArray()); + Assert.Equal(new byte[] { 0x00, 0xff, 0x80 }, context.Message.Envelope.Payload.ToArray()); + } + + [Fact] + public async Task HandleAsync_SynchronousHandling_PreservesContextTokenAndExplicitCompletion() + { + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => count++); + using var cancellation = new CancellationTokenSource(); + var handler = Substitute.For(); + handler.HandleAsync(context, cancellation.Token).Returns(_ => + { + context.Complete(); + return ValueTask.CompletedTask; + }); + + var result = handler.HandleAsync(context, cancellation.Token); + Assert.True(result.IsCompletedSuccessfully); + await result; + + Assert.Equal(1, count); + AssertEnvelope(envelope, context.Message.Envelope); + await handler.Received(1).HandleAsync(context, cancellation.Token); + } + + [Fact] + public async Task HandleAsync_DeferredWork_PreservesContextAndCompletesOnlyAfterRelease() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => count++); + using var cancellation = new CancellationTokenSource(); + var handler = Substitute.For(); + async ValueTask Handle() + { + entered.SetResult(); + await release.Task; + context.Complete(); + } + handler.HandleAsync(context, cancellation.Token).Returns(_ => Handle()); + + var pending = handler.HandleAsync(context, cancellation.Token).AsTask(); + await entered.Task; + Assert.False(pending.IsCompleted); + Assert.Equal(0, count); + release.SetResult(); + await pending; + + Assert.Equal(1, count); + AssertEnvelope(envelope, context.Message.Envelope); + await handler.Received(1).HandleAsync(context, cancellation.Token); + } + + [Fact] + public async Task HandleAsync_ReturnWithoutComplete_DoesNotInvokeCompletion() + { + var count = 0; + var envelope = Envelope(); + var context = CreateContext(envelope, () => count++); + var handler = Substitute.For(); + handler.HandleAsync(context, CancellationToken.None).Returns(ValueTask.CompletedTask); + + await handler.HandleAsync(context, CancellationToken.None); + + Assert.Equal(0, count); + AssertEnvelope(envelope, context.Message.Envelope); + await handler.Received(1).HandleAsync(context, CancellationToken.None); + } + + [Fact] + public void HandlerNotFound_RoundTripsStatusAndDiagnostic() + { + var result = DeliveryResult.HandlerNotFound(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + + var decoded = serializer.Deserialize(serializer.SerializeToArray(result)); + + Assert.Equal(DeliveryStatus.HandlerNotFound, result.Status); + Assert.Equal(DeliveryStatus.HandlerNotFound, decoded.Status); + Assert.Equal("No inbox handler is registered.", result.Message); + Assert.Equal("No inbox handler is registered.", decoded.Message); + } + + [Fact] + public void ExternalConsumerAssembly_HasNoFriendAccessToMessaging() + { + var consumer = typeof(HandlerRoutingContractTests).Assembly.GetName().Name!; + Assert.DoesNotContain(typeof(IInboxHandler).Assembly.GetCustomAttributes(), + attribute => attribute.AssemblyName.Split(',')[0] == consumer); + } + + [Fact] + public void HandlerContext_Constructor_RequiresCompletionCallback() + { + var envelope = Envelope(); + var type = typeof(IInboxHandlerContext).Assembly.GetType("Orleans.Messaging.InboxHandlerContext", throwOnError: true)!; + var exception = Assert.Throws(() => Activator.CreateInstance(type, [new InboxMessage { Envelope = envelope }, null])); + Assert.Equal("complete", Assert.IsType(exception.InnerException).ParamName); + var parameters = Assert.Single(type.GetConstructors()).GetParameters(); + Assert.Equal(2, parameters.Length); + Assert.Equal(typeof(Action), parameters[1].ParameterType); + Assert.False(parameters[1].IsOptional); + } + + private static IInboxHandlerContext CreateContext(Envelope envelope, Action complete) + { + var type = typeof(IInboxHandlerContext).Assembly.GetType("Orleans.Messaging.InboxHandlerContext", throwOnError: true)!; + return Assert.IsAssignableFrom(Activator.CreateInstance(type, new InboxMessage { Envelope = envelope }, complete)); + } + + private static Envelope Envelope(byte[]? payload = null) + { + return new() + { + MessageId = HierarchicalKey.Create("44444444-4444-4444-4444-444444444444"), + Payload = payload ?? [0x00, 0xff, 0x80] + }; + } + + private static void AssertEnvelope(Envelope expected, Envelope actual) + { + Assert.Equal(expected.MessageId, actual.MessageId); + Assert.Same(expected.Headers, actual.Headers); + Assert.Equal(expected.Payload, actual.Payload); + Assert.Equal(new byte[] { 0x00, 0xff, 0x80 }, actual.Payload.ToArray()); + Assert.Null(actual.Subject); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs new file mode 100644 index 00000000000..8f1f82cc1c2 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs @@ -0,0 +1,349 @@ +using System.Buffers; +using System.Reflection; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class HierarchicalKeyTests +{ + [Fact] + public void SerializationContract_IdentifiesCurrentType() + { + var type = typeof(HierarchicalKey); + Assert.Equal("Orleans.Messaging.HierarchicalKey", Assert.Single(type.GetCustomAttributes()).Alias); + Assert.True(type.IsValueType); + Assert.Single(type.GetCustomAttributes()); + Assert.Empty(type.GetCustomAttributes()); + var field = Assert.Single(type.GetFields(BindingFlags.Instance | BindingFlags.NonPublic)); + Assert.True(field.IsInitOnly); + Assert.True(field.FieldType.IsSealed); + Assert.Equal(typeof(string), field.FieldType.GetProperty("Canonical")!.PropertyType); + Assert.Equal(typeof(int), field.FieldType.GetProperty("Hash")!.PropertyType); + Assert.DoesNotContain(field.FieldType.GetFields(BindingFlags.Instance | BindingFlags.NonPublic), + value => value.FieldType == typeof(HierarchicalKey) || value.FieldType == field.FieldType); + } + + [Fact] + public void Default_IsDistinguishedUnsetValue() + { + var key = default(HierarchicalKey); + Assert.True(key.IsDefault); + Assert.Equal(0, key.Length); + Assert.Equal(0, key.SegmentCount); + Assert.Equal(0, key.GetHashCode()); + Assert.Equal(string.Empty, key.ToString()); + Assert.True(key == default); + Assert.False(key != default); + Assert.False(key.Equals(null)); + Assert.Null(key.GetParent()); + Assert.False(key.GetEnumerator().MoveNext()); + Assert.False(key.IsAncestorOf(default)); + Assert.False(key.IsParentOf(default)); + Assert.False(key.IsChildOf(default)); + Assert.False(HierarchicalKey.Create("a").IsAncestorOf(key)); + Assert.Throws(() => key.CreateChildKey("a")); + Assert.Throws(() => key.Append(HierarchicalKey.Create("a"))); + Assert.Throws(() => HierarchicalKey.Create("a").Append(default)); + Span empty = []; + Assert.True(key.TryFormat(empty, out var written, default, null)); + Assert.Equal(0, written); + } + + [Theory] + [InlineData("foo", "foo")] + [InlineData("foo/bar", @"foo\/bar")] + [InlineData(@"foo\bar", @"foo\\bar")] + [InlineData(@"foo\/bar/baz", @"foo\\\/bar\/baz")] + [InlineData("/", @"\/")] + [InlineData(@"\", @"\\")] + [InlineData("/a//b/", @"\/a\/\/b\/")] + [InlineData(" ", " ")] + public void Create_LiteralSegment_EscapesExactlyOnce(string literal, string canonical) + { + var key = HierarchicalKey.Create(literal); + Assert.Equal(canonical, key.ToString()); + Assert.Equal(canonical.Length, key.Length); + Assert.Equal(1, key.SegmentCount); + Assert.Null(key.GetParent()); + Assert.Equal(key, HierarchicalKey.Parse(canonical)); + Assert.Equal([canonical], Segments(key)); + Assert.Equal(key, HierarchicalKey.Create(new[] { literal })); + } + + [Fact] + public void Create_WithValues_CreatesRootFirstHierarchy() + { + var key = HierarchicalKey.Create("orders", "42", "payment"); + var root = HierarchicalKey.Create("orders"); + var parent = root.CreateChildKey("42"); + Assert.Equal("orders/42/payment", key.ToString()); + Assert.Equal(["orders", "42", "payment"], Segments(key)); + Assert.Equal(3, key.SegmentCount); + Assert.Equal(17, key.Length); + Assert.Equal(parent.CreateChildKey("payment"), key); + Assert.Equal(HierarchicalKey.Parse("orders/42/payment"), key); + Assert.Equal(parent, key.GetParent()); + Assert.Equal(root, key.GetParent()!.Value.GetParent()); + Assert.Null(key.GetParent()!.Value.GetParent()!.Value.GetParent()); + Assert.True(parent.IsParentOf(key)); + Assert.True(key.IsChildOf(parent)); + Assert.True(root.IsAncestorOf(key)); + Assert.False(root.IsParentOf(key)); + } + + [Fact] + public void Create_WithValues_ArrayAndSpanSlicesUseOnlyProvidedFragments() + { + string[] input = ["ignored", "orders", "42", "payment", "ignored"]; + var expected = HierarchicalKey.Parse("orders/42/payment"); + Assert.Equal(expected, HierarchicalKey.Create(input.AsSpan(1, 3))); + ReadOnlySpan slice = input.AsSpan(1, 3); + Assert.Equal(expected, HierarchicalKey.Create(slice)); + Assert.Equal(expected, HierarchicalKey.Create(["orders", "42", "payment"])); + } + + [Fact] + public void Create_WithValues_ArrayMutationDoesNotChangeKey() + { + string[] input = ["orders", "42", "payment"]; + var key = HierarchicalKey.Create(input); + var hash = key.GetHashCode(); + input[0] = "other"; + input[1] = null!; + Assert.Equal("orders/42/payment", key.ToString()); + Assert.Equal(hash, key.GetHashCode()); + Assert.Equal(HierarchicalKey.Parse("orders/42"), key.GetParent()); + } + + [Fact] + public void Create_WithValues_EmptyInputThrowsArgumentException() + { + Assert.Equal("values", Assert.Throws(() => HierarchicalKey.Create()).ParamName); + Assert.Throws(() => HierarchicalKey.Create(ReadOnlySpan.Empty)); + Assert.Throws(() => HierarchicalKey.Create(Array.Empty())); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void Create_WithValues_NullFragmentThrowsArgumentNullException(int position) + { + string[] input = ["a", "b", "c"]; + input[position] = null!; + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(input)).ParamName); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void Create_WithValues_EmptyLiteralSegmentRejects(int position) + { + string[] input = ["a", "b", "c"]; + input[position] = ""; + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(input)).ParamName); + } + + [Fact] + public void Create_LiteralSegment_RejectsOnlyNullAndEmpty() + { + Assert.Throws(() => HierarchicalKey.Create((string)null!)); + Assert.Throws(() => HierarchicalKey.Create("")); + var parent = HierarchicalKey.Create("root"); + Assert.Throws(() => parent.CreateChildKey(null!)); + Assert.Throws(() => parent.CreateChildKey("")); + } + + [Theory] + [InlineData("")] + [InlineData("/a")] + [InlineData("a/")] + [InlineData("a//b")] + [InlineData(@"a\")] + [InlineData(@"a\q")] + [InlineData(@"a\/b/")] + public void Parse_InvalidCanonicalPath_RejectsStringAndSpan(string path) + { + Assert.Throws(() => HierarchicalKey.Parse(path)); + Assert.Throws(() => HierarchicalKey.Parse(path.AsSpan())); + Assert.False(HierarchicalKey.TryParse(path, null, out var first)); + Assert.True(first.IsDefault); + Assert.False(HierarchicalKey.TryParse(path.AsSpan(), null, out var second)); + Assert.True(second.IsDefault); + } + + [Fact] + public void Parse_WithNullString_ThrowsArgumentNullException() + { + Assert.Throws(() => HierarchicalKey.Parse((string)null!)); + Assert.False(HierarchicalKey.TryParse(null, null, out var key)); + Assert.True(key.IsDefault); + } + + [Theory] + [InlineData("a/b/c", 3)] + [InlineData(@"a\/b/c", 2)] + [InlineData(@"a\\/b/c", 3)] + [InlineData(@"a\\\/b/c", 2)] + public void Parse_CanonicalEscapes_PreserveSegmentBoundaries(string path, int count) + { + var key = HierarchicalKey.Parse(path); + Assert.Equal(path, key.ToString()); + Assert.Equal(count, key.SegmentCount); + Assert.Equal(count, Segments(key).Count); + Assert.True(HierarchicalKey.TryParse(path.AsSpan(), null, out var copy)); + Assert.True(key == copy); + Assert.Equal(key.GetHashCode(), copy.GetHashCode()); + var characters = path.ToCharArray(); + var fromCaller = HierarchicalKey.Parse(characters.AsSpan()); + characters[0] = 'z'; + Assert.Equal(key, fromCaller); + } + + [Fact] + public void Append_ComposesConstructedPaths_ChildAppendsOneLiteral() + { + var parent = HierarchicalKey.Parse(@"root/fo\/o"); + var suffix = HierarchicalKey.Parse(@"ba\\r/baz"); + var composed = parent.Append(suffix); + Assert.Equal(@"root/fo\/o/ba\\r/baz", composed.ToString()); + Assert.Equal(4, composed.SegmentCount); + Assert.Equal(["root", @"fo\/o", @"ba\\r", "baz"], Segments(composed)); + Assert.Equal(parent.Append(HierarchicalKey.Create(@"ba\r")), composed.GetParent()); + Assert.True(parent.IsAncestorOf(composed)); + Assert.False(parent.IsParentOf(composed)); + var literal = parent.CreateChildKey("ba/r/baz"); + Assert.Equal(@"root/fo\/o/ba\/r\/baz", literal.ToString()); + Assert.Equal(3, literal.SegmentCount); + Assert.Equal(parent, literal.GetParent()); + Assert.True(parent.IsParentOf(literal)); + Assert.NotEqual(composed, literal); + } + + [Theory] + [InlineData("foo", "foo/bar", true, true)] + [InlineData("foo", "foo/bar/baz", false, true)] + [InlineData("foo/bar", "foo/bar", false, true)] + [InlineData("foo", "foobar/child", false, false)] + [InlineData("foo", @"foo\/bar/child", false, false)] + [InlineData("foo/bar", "foo/baz", false, false)] + [InlineData("Foo", "foo/bar", false, false)] + public void Navigation_UsesExactOrdinalSegments(string root, string path, bool immediate, bool ancestor) + { + var parent = HierarchicalKey.Parse(root); + var child = HierarchicalKey.Parse(path); + Assert.Equal(immediate, parent.IsParentOf(child)); + Assert.Equal(immediate, child.IsChildOf(parent)); + Assert.Equal(ancestor, parent.IsAncestorOf(child)); + if (immediate) Assert.Equal(parent, child.GetParent()); + } + + [Fact] + public void Equality_HashOperatorsAndDictionary_UseCanonicalOrdinalIdentity() + { + var direct = HierarchicalKey.Parse(@"tenant/acme/orders\/42"); + var composed = HierarchicalKey.Create("tenant", "acme", "orders/42"); + Assert.True(direct == composed); + Assert.False(direct != composed); + Assert.True(direct.Equals((object)composed)); + Assert.Equal(direct.GetHashCode(), composed.GetHashCode()); + Assert.Equal(direct.ToString().GetHashCode(StringComparison.Ordinal), direct.GetHashCode()); + var other = HierarchicalKey.Create("Tenant", "acme", "orders/42"); + Assert.True(other != direct); + Assert.False(other == direct); + Assert.False(direct.Equals("tenant/acme/orders\\/42")); + var dictionary = new Dictionary { [direct] = 42, [other] = 81, [default] = 5 }; + Assert.Equal(42, dictionary[composed]); + Assert.Equal(81, dictionary[other]); + Assert.Equal(5, dictionary[default]); + Assert.Equal(3, dictionary.Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SerializationAndCopy_RebuildCanonicalCacheAndPreserveValue(bool unset) + { + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var key = unset ? default : HierarchicalKey.Create("tenant", @"a\/b", "42"); + var serializer = first.GetRequiredService>(); + var bytes = serializer.SerializeToArray(key); + var decoded = second.GetRequiredService>().Deserialize(bytes); + var copy = first.GetRequiredService().Copy(key); + Assert.Equal(key, decoded); + Assert.Equal(key.IsDefault, decoded.IsDefault); + Assert.Equal(key.GetHashCode(), decoded.GetHashCode()); + Assert.Equal(key.SegmentCount, decoded.SegmentCount); + Assert.Equal(key, copy); + var data = Assert.Single(typeof(HierarchicalKey).GetFields(BindingFlags.Instance | BindingFlags.NonPublic)); + Assert.Same(data.GetValue(key), data.GetValue(copy)); + if (!unset) Assert.NotSame(data.GetValue(key), data.GetValue(decoded)); + Assert.Equal(bytes, serializer.SerializeToArray(decoded)); + var map = first.GetRequiredService>>(); + var dictionary = Assert.IsType>(map.Deserialize(map.SerializeToArray(new() { [key] = 42 }))); + Assert.Equal(42, dictionary[decoded]); + } + + [Theory] + [InlineData("")] + [InlineData("a//b")] + [InlineData(@"a\q")] + public void Codec_InvalidCanonicalWire_RejectsInsteadOfConstructingKey(string canonical) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var session = services.GetRequiredService().GetSession(); + var output = new ArrayBufferWriter(); + var writer = Writer.Create(output, session); + ReferenceCodec.MarkValueField(session); + writer.WriteFieldHeader(0, typeof(HierarchicalKey), typeof(HierarchicalKey), WireType.TagDelimited); + services.GetRequiredService>().WriteField(ref writer, 0, typeof(string), canonical); + writer.WriteEndObject(); + writer.Commit(); + Assert.Throws(() => services.GetRequiredService>().Deserialize(output.WrittenSpan)); + } + + [Fact] + public void LongHierarchy_ConstructionNavigationAndFormatting_AreIterative() + { + var key = HierarchicalKey.Create(Enumerable.Repeat("x", 10000).ToArray()); + Assert.Equal(10000, key.SegmentCount); + Assert.Equal(19999, key.Length); + Assert.Equal(10000, Segments(key).Count); + Assert.Equal(9999, key.GetParent()!.Value.SegmentCount); + Assert.Equal(key, HierarchicalKey.Parse(key.ToString())); + Assert.True(HierarchicalKey.Create("x").IsAncestorOf(key)); + var span = new char[key.Length]; + Assert.True(key.TryFormat(span, out var written, default, null)); + Assert.Equal(key.Length, written); + Assert.Equal(key.ToString(), new string(span)); + Array.Fill(span, 'z'); + Assert.False(key.TryFormat(span.AsSpan(0, key.Length - 1), out written, default, null)); + Assert.Equal(0, written); + Assert.All(span, character => Assert.Equal('z', character)); + } + + [Fact] + public void EscapeCharacter_IsBackslash() => Assert.Equal('\\', HierarchicalKey.EscapeCharacter); + + [Fact] + public void SegmentSeparator_IsForwardSlash() => Assert.Equal('/', HierarchicalKey.SegmentSeparator); + + private static List Segments(HierarchicalKey key) + { + var values = new List(); + foreach (var segment in key) values.Add(segment.ToString()); + return values; + } +} diff --git a/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj new file mode 100644 index 00000000000..b6e3c992322 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj @@ -0,0 +1,14 @@ + + + $(TestTargetFrameworks) + true + enable + enable + + + + + + + + From fde9c0d65777c0e4a00f6415ee6b93cf84ff445f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 8 Oct 2026 22:06:27 -0700 Subject: [PATCH 21/31] feat(messaging): process extensible inbox messages --- src/Orleans.Messaging/DeadLetterRetention.cs | 41 + .../IMessagingDiagnostics.cs | 104 ++ src/Orleans.Messaging/IMessagingGrain.cs | 14 + src/Orleans.Messaging/Inbox.cs | 90 ++ src/Orleans.Messaging/InboxExtension.cs | 1409 +++++++++++++++++ .../MessagingActivationValidator.cs | 49 + .../MessagingCancellation.cs | 44 + .../MessagingGrainTypeConfigurator.cs | 32 + src/Orleans.Messaging/MessagingInstruments.cs | 114 ++ .../MessagingJobOwnership.cs | 127 ++ .../MessagingPumpCoordinator.cs | 82 + src/Orleans.Messaging/MessagingPumpResults.cs | 428 +++++ src/Orleans.Messaging/MessagingStateNames.cs | 22 + src/Orleans.Messaging/MessagingTime.cs | 21 + src/Orleans.Messaging/MessagingTurn.cs | 75 + .../Orleans.Messaging.csproj | 3 +- .../Contracts/ActivationValidationTests.cs | 148 ++ .../DeliveryAndOptionsContractTests.cs | 201 +++ .../Contracts/HandlerRoutingContractTests.cs | 88 +- .../Contracts/MessagingPumpResultsTests.cs | 356 +++++ .../Contracts/StandardJournaledStateTests.cs | 105 ++ .../Functional/ByteMessagingPayloadTests.cs | 150 ++ .../Functional/DedupeExpiryBehaviorTests.cs | 106 ++ .../Functional/DeliveryCancellationTests.cs | 227 +++ .../EarlyActivationValidationTests.cs | 91 ++ .../InboxAcceptanceBehaviorTests.cs | 234 +++ .../Functional/InboxAdmissionTests.cs | 372 +++++ .../Functional/InboxCallbackOwnershipTests.cs | 295 ++++ .../InboxCancellationCleanupTests.cs | 306 ++++ .../Functional/InboxCapacityBehaviorTests.cs | 113 ++ .../Functional/InboxCommittedCountTests.cs | 382 +++++ .../Functional/InboxEagerManagerTests.cs | 196 +++ .../InboxEnvelopeValidationTests.cs | 75 + .../Functional/InboxHandlerCompletionTests.cs | 698 ++++++++ .../InboxHandlerTransactionTests.cs | 150 ++ .../Functional/InboxHeaderBehaviorTests.cs | 280 ++++ .../Functional/InboxIdentityTests.cs | 473 ++++++ .../Functional/InboxLifecycleBehaviorTests.cs | 118 ++ .../Functional/InboxMissingHandlerTests.cs | 492 ++++++ .../InboxPumpResultLifetimeTests.cs | 241 +++ .../Functional/InboxQuiescenceTests.cs | 332 ++++ .../Functional/InboxStateEncodingTests.cs | 92 ++ .../InboxStateManagerBoundaryTests.cs | 228 +++ .../Functional/InboxStateProtocolTests.cs | 146 ++ .../JournaledTestOutboxBehaviorTests.cs | 117 ++ .../Functional/MessagingClusterCollection.cs | 9 + .../MessagingGrainTypeConfiguratorTests.cs | 380 +++++ .../MessagingMetricCardinalityTests.cs | 279 ++++ .../MessagingOwnershipRecoveryTests.cs | 299 ++++ .../MessagingRoutingAndDeadLetterTests.cs | 147 ++ .../ProcessedRetentionMaintenanceTests.cs | 270 ++++ .../Functional/StandardStateJsonContext.cs | 9 + .../Orleans.Messaging.Tests.csproj | 4 + .../Support/ActivationValidationTestGrains.cs | 137 ++ .../Support/BootstrapTestGrains.cs | 306 ++++ .../Support/ControlledDurableJobManager.cs | 151 ++ .../ControlledJournalStorageProvider.cs | 276 ++++ .../Support/FaultingInboxCodecFixture.cs | 48 + .../Support/HandlerProbe.cs | 53 + .../Support/JournaledTestOutbox.cs | 70 + .../Support/MessagingBehaviorTestBase.cs | 83 + .../Support/MessagingClusterFixture.cs | 159 ++ .../Support/MessagingMetricProbe.cs | 121 ++ .../Support/MessagingTestGrains.cs | 502 ++++++ .../Support/ReceiverTestServices.cs | 116 ++ .../Support/SnapshotProbe.cs | 98 ++ .../Support/SnapshotProbeTests.cs | 86 + .../Support/TestApplicationProtocol.cs | 101 ++ 68 files changed, 13167 insertions(+), 4 deletions(-) create mode 100644 src/Orleans.Messaging/DeadLetterRetention.cs create mode 100644 src/Orleans.Messaging/IMessagingDiagnostics.cs create mode 100644 src/Orleans.Messaging/IMessagingGrain.cs create mode 100644 src/Orleans.Messaging/Inbox.cs create mode 100644 src/Orleans.Messaging/InboxExtension.cs create mode 100644 src/Orleans.Messaging/MessagingActivationValidator.cs create mode 100644 src/Orleans.Messaging/MessagingCancellation.cs create mode 100644 src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs create mode 100644 src/Orleans.Messaging/MessagingInstruments.cs create mode 100644 src/Orleans.Messaging/MessagingJobOwnership.cs create mode 100644 src/Orleans.Messaging/MessagingPumpCoordinator.cs create mode 100644 src/Orleans.Messaging/MessagingPumpResults.cs create mode 100644 src/Orleans.Messaging/MessagingStateNames.cs create mode 100644 src/Orleans.Messaging/MessagingTime.cs create mode 100644 src/Orleans.Messaging/MessagingTurn.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/MessagingPumpResultsTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/DeliveryCancellationTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxCallbackOwnershipTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxCancellationCleanupTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxCommittedCountTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxMissingHandlerTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxPumpResultLifetimeTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingOwnershipRecoveryTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs create mode 100644 test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs create mode 100644 test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs create mode 100644 test/Orleans.Messaging.Tests/Support/HandlerProbe.cs create mode 100644 test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs create mode 100644 test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs create mode 100644 test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs create mode 100644 test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs create mode 100644 test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs create mode 100644 test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs create mode 100644 test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs create mode 100644 test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs diff --git a/src/Orleans.Messaging/DeadLetterRetention.cs b/src/Orleans.Messaging/DeadLetterRetention.cs new file mode 100644 index 00000000000..e2388a608e9 --- /dev/null +++ b/src/Orleans.Messaging/DeadLetterRetention.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Orleans.Messaging; + +internal static class DeadLetterRetention +{ + public static bool Compact( + IDictionary entries, + DateTimeOffset now, + TimeSpan retentionPeriod, + int maxRetainedEntries, + Func getTimestamp, + int reservedCapacity = 0) + where TKey : notnull + { + var removed = false; + foreach (var entry in entries + .Where(entry => MessagingTime.IsExpired(now, getTimestamp(entry.Value), retentionPeriod)) + .ToList()) + { + entries.Remove(entry.Key); + removed = true; + } + + var removeCount = entries.Count + reservedCapacity - maxRetainedEntries; + if (removeCount <= 0) + { + return removed; + } + + foreach (var entry in entries.OrderBy(entry => getTimestamp(entry.Value)).Take(removeCount).ToList()) + { + entries.Remove(entry.Key); + removed = true; + } + + return removed; + } +} diff --git a/src/Orleans.Messaging/IMessagingDiagnostics.cs b/src/Orleans.Messaging/IMessagingDiagnostics.cs new file mode 100644 index 00000000000..fb73bd880ea --- /dev/null +++ b/src/Orleans.Messaging/IMessagingDiagnostics.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Provides operational access to a grain's messaging state. +/// +public interface IMessagingDiagnostics +{ + /// + /// Gets messages which failed during inbox processing. + /// + IReadOnlyList> InboxDeadLetters { get; } + + /// + /// Gets messages which could not be delivered from the outbox. + /// + IReadOnlyList> OutboxDeadLetters { get; } + + /// + /// Stages removal of an inbox dead letter. + /// + /// The receiver-local command identity. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveInboxDeadLetter(HierarchicalKey messageId); + + /// + /// Stages removal of an outbox dead letter. + /// + /// The message identifier. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveOutboxDeadLetter(HierarchicalKey messageId); +} + +/// +/// Describes a dead-lettered durable message. +/// +/// The incoming command or outgoing routed intent type. +public sealed class DeadLetter +{ + /// + /// Gets the message. + /// + /// + /// Treat the message payload as immutable. Ordinary RPC serialization copies the payload + /// when returning diagnostic results. + /// + public required TMessage Message { get; init; } + + /// + /// Gets when the message was dead-lettered. + /// + public DateTimeOffset DeadLetteredAt { get; init; } + + /// + /// Gets the terminal failure reason. + /// + public required string Reason { get; init; } + + /// + /// Gets the number of attempts made. + /// + public int AttemptCount { get; init; } +} + +internal sealed class MessagingDiagnostics( + [Microsoft.Extensions.DependencyInjection.FromKeyedServices(MessagingStateNames.InboxDeadLetters)] + IDurableDictionary inbox, + [Microsoft.Extensions.DependencyInjection.FromKeyedServices(MessagingStateNames.OutboxDeadLetters)] + IDurableDictionary outbox) : IMessagingDiagnostics +{ + public IReadOnlyList> InboxDeadLetters => + inbox.Values.Select(static entry => new DeadLetter + { + Message = entry.Message, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount + }).ToList(); + + public IReadOnlyList> OutboxDeadLetters => + outbox.Values.Select(static entry => new DeadLetter + { + Message = entry.Message, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount + }).ToList(); + + public bool RemoveInboxDeadLetter(HierarchicalKey messageId) => + inbox.Remove(messageId); + + public bool RemoveOutboxDeadLetter(HierarchicalKey messageId) => outbox.Remove(messageId); +} diff --git a/src/Orleans.Messaging/IMessagingGrain.cs b/src/Orleans.Messaging/IMessagingGrain.cs new file mode 100644 index 00000000000..718d6952c0f --- /dev/null +++ b/src/Orleans.Messaging/IMessagingGrain.cs @@ -0,0 +1,14 @@ +namespace Orleans.Messaging; + +/// +/// Identifies grain implementations which initialize inbox and outbox services during activation setup. +/// +/// +/// Implement this local capability on a grain class, an application base class, or an application grain interface. +/// With messaging services registered, selected activations validate their execution model and bind +/// messaging state before journal recovery. Grains deriving from +/// receive the same setup automatically. +/// +public interface IMessagingGrain +{ +} diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs new file mode 100644 index 00000000000..edbfa198acd --- /dev/null +++ b/src/Orleans.Messaging/Inbox.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Provides access to journaled pending messages and their capacity limit. +/// Registers the single handler for this inbox. +/// +internal sealed class Inbox : IInbox +{ + private readonly IDurableDictionary _inbox; + private IInboxHandler? _handler; + private readonly int _capacity; + + /// + /// Creates an inbox over journaled message storage. + /// + /// Durable dictionary for storing unprocessed messages. + /// Maximum inbox capacity (default: 1000). + public Inbox( + IDurableDictionary inbox, + int capacity = 1000) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity); + + _inbox = inbox; + _capacity = capacity; + } + + internal Inbox( + IDurableDictionary inbox, + IEnumerable handlers, + int capacity) + : this(inbox, capacity) + { + foreach (var handler in handlers) + { + RegisterHandler(handler); + } + } + + /// + /// Number of unprocessed messages. + /// + public int Count => _inbox.Count; + + /// + /// Gets the maximum inbox capacity. + /// + public int Capacity => _capacity; + + /// + /// Gets all pending messages (no ordering guarantee). + /// + public IEnumerable Messages => _inbox.Values; + + /// + /// Tries to get a specific message by its key. + /// + /// The receiver-local command identity. + /// The envelope if found. + /// True if the message exists in the inbox; otherwise, false. + public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out InboxMessage envelope) => + _inbox.TryGetValue(messageId, out envelope); + + /// + /// Registers the single handler for this inbox. + /// + public void RegisterHandler(IInboxHandler handler) + { + ArgumentNullException.ThrowIfNull(handler); + if (_handler is not null) + { + throw new InvalidOperationException("A handler is already registered for this inbox."); + } + + _handler = handler; + } + + internal bool TryGetHandler([MaybeNullWhen(false)] out IInboxHandler handler) + { + handler = _handler; + return handler is not null; + } +} diff --git a/src/Orleans.Messaging/InboxExtension.cs b/src/Orleans.Messaging/InboxExtension.cs new file mode 100644 index 00000000000..1a04ddfc353 --- /dev/null +++ b/src/Orleans.Messaging/InboxExtension.cs @@ -0,0 +1,1409 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.TypeSystem; +using Orleans.Timers; + +namespace Orleans.Messaging; + +/// +/// Implementation of inbox extension for grain message delivery. +/// Handles message persistence, deduplication, and processing. +/// +internal sealed partial class InboxExtension : + IInboxExtension, + IDurableJobFeatureHandler, + ILifecycleObserver, + IDisposable +{ + internal const string JobName = "orleans.messaging.inbox-drain"; + + // Nested handler requests and independent interleaved calls have different logical execution contexts. + private static readonly AsyncLocal _handlerExecution = new(); + + public bool CanHandle(string jobName) => string.Equals(jobName, JobName, StringComparison.Ordinal); + + private readonly IGrainContext _grainContext; + private readonly string _grainType; + private readonly ITimerRegistry _timerRegistry; + private readonly IJournaledStateManager _stateManager; + private readonly ILogger _logger; + private readonly MessagingInstruments _instruments; + private readonly Inbox _inbox; + private readonly IDictionary _inboxDict; + private readonly IDictionary _processed; + private readonly IDictionary _messageStates; + private readonly IDictionary _deadLetters; + private readonly IDurableValue _jobId; + private readonly IDurableValue _job; + private readonly IDurableValue _completedJobId; + private readonly IDurableValue _jobSequence; + private readonly ILocalDurableJobManager _jobManager; + private readonly TimeProvider _timeProvider; + private readonly TimeProvider _jobTimeProvider; + private readonly HashSet _provisionalAcceptances = []; + private readonly MessagingPumpResults _pumpResults; + private readonly MessagingPumpCoordinator _pumpCoordinator = new(); + private readonly int _maxCapacity; + private readonly TimeSpan _deduplicationWindow; + private readonly TimeSpan _processedCompactionInterval; + private readonly int _maxProcessingAttempts; + private readonly int _batchSize; + private readonly TimeSpan _retryDelay; + private readonly TimeSpan _deadLetterRetentionPeriod; + private readonly int _maxRetainedDeadLetters; + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly CancellationTokenSource _shutdownCts = new(); + private readonly CancellationToken _shutdownToken; + private Task _activeDelivery = Task.CompletedTask; + private PumpTimerState? _pumpTimer; + private LocalDrainTimerState? _localDrainTimer; + private bool _localDrainRequested; + private int _disposed; + private int _metricsActive; + private int _reportedDepth; + private string? _durableOwnershipId; + private DurableJob? _durableJob; + private string _ownershipEpoch = Guid.NewGuid().ToString("N"); + private long _stateGeneration; + private bool _recoveryCompleted; + private string? _ownershipStateError; + private long _reservedSequence; + private ExceptionDispatchInfo? _failure; + private readonly HashSet _pendingOwnershipIds = new(StringComparer.Ordinal); + private readonly List _pendingWrites = []; + private string? _durableCompletedJobId; + private DateTimeOffset? _nextProcessedExpiry; + private DateTimeOffset? _lastProcessedCompaction; + + /// + /// Creates a new inbox extension instance. + /// + /// The grain context for this extension. + /// State manager for atomic persistence. + /// Logger for diagnostics. + /// Journaling metrics. + /// The grain's inbox (shared with grain DI). + /// Durable dictionary for inbox messages. + /// Durable dictionary for processed message tracking. + /// Messaging options. + public InboxExtension( + IGrainContext grainContext, + ITimerRegistry timerRegistry, + IJournaledStateManager stateManager, + ILogger logger, + MessagingInstruments instruments, + Inbox inbox, + IDictionary inboxDict, + IDictionary processed, + IDictionary messageStates, + IDictionary deadLetters, + IDurableValue jobId, + IDurableValue job, + IDurableValue completedJobId, + IDurableValue jobSequence, + ILocalDurableJobManager jobManager, + IDurableJobHandlerRegistry jobHandlers, + MessagingPumpResults pumpResults, + TimeProvider timeProvider, + TimeProvider jobTimeProvider, + InboxOptions options) + { + ArgumentNullException.ThrowIfNull(grainContext); + ArgumentNullException.ThrowIfNull(timerRegistry); + ArgumentNullException.ThrowIfNull(stateManager); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(instruments); + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(inboxDict); + ArgumentNullException.ThrowIfNull(processed); + ArgumentNullException.ThrowIfNull(messageStates); + ArgumentNullException.ThrowIfNull(deadLetters); + ArgumentNullException.ThrowIfNull(jobId); + ArgumentNullException.ThrowIfNull(job); + ArgumentNullException.ThrowIfNull(completedJobId); + ArgumentNullException.ThrowIfNull(jobSequence); + ArgumentNullException.ThrowIfNull(jobManager); + ArgumentNullException.ThrowIfNull(jobHandlers); + ArgumentNullException.ThrowIfNull(pumpResults); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentNullException.ThrowIfNull(jobTimeProvider); + ArgumentNullException.ThrowIfNull(options); + _shutdownToken = _shutdownCts.Token; + _grainContext = grainContext; + _grainType = grainContext.GrainId.Type.ToString(); + _timerRegistry = timerRegistry; + _stateManager = stateManager; + _logger = logger; + _instruments = instruments; + _inbox = inbox; + _inboxDict = inboxDict; + _processed = processed; + _messageStates = messageStates; + _deadLetters = deadLetters; + _jobId = jobId; + _job = job; + _completedJobId = completedJobId; + _jobSequence = jobSequence; + _jobManager = jobManager; + _pumpResults = pumpResults; + _timeProvider = timeProvider; + _jobTimeProvider = jobTimeProvider; + _maxCapacity = options.MaxCapacity; + _deduplicationWindow = options.DeduplicationWindow; + _processedCompactionInterval = TimeSpan.FromTicks(Math.Max(1, _deduplicationWindow.Ticks / 4)); + _maxProcessingAttempts = options.MaxProcessingAttempts; + _batchSize = options.InboxBatchSize; + _retryDelay = options.BackpressureRetryDelay; + _deadLetterRetentionPeriod = options.DeadLetterRetentionPeriod; + _maxRetainedDeadLetters = options.MaxRetainedDeadLetters; + jobHandlers.Register(this); + grainContext.ObservableLifecycle.Subscribe( + RuntimeTypeNameFormatter.Format(GetType()), + GrainLifecycleStage.Activate, + this); + } + + private static void ThrowIfHandlerOperationRejected(HandlerExecution execution) => execution.RejectionFailure?.Throw(); + + public int Count => _inboxDict.Count; + public int Capacity => _maxCapacity; + + public async ValueTask DeliverAsync(InboxMessage envelope, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + ValidateReady(); + EnvelopeValidation.Validate(envelope); + + EnsureMetricsActive(); + // Direct publication shares immutable payload bytes; the admitted task keeps the + // envelope reachable independently of cancellation of its caller's wait. + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + var delivery = DeliverUnderGateAsync(envelope); + _activeDelivery = delivery; + delivery.Ignore(); + return await delivery.WaitAsync(cancellationToken).ConfigureAwait(true); + } + + private async Task DeliverUnderGateAsync(InboxMessage envelope) + { + try + { + ValidateReady(); + var key = envelope.Envelope.MessageId; + if (_processed.TryGetValue(key, out var processedAt) + && !MessagingTime.IsExpired(_timeProvider.GetUtcNow(), processedAt, _deduplicationWindow)) + { + _instruments.OnInboxMessageReceived(_grainType, "duplicate"); + return DeliveryResult.Duplicate(); + } + + if (_inboxDict.TryGetValue(key, out var pending)) + { + if (!EnvelopeEquivalence.AreSameCommand(pending, envelope)) + { + throw new InvalidOperationException( + $"The inbox already contains a different command with message ID '{key}'."); + } + await EnsureJobScheduledUnderGateAsync(CancellationToken.None).ConfigureAwait(true); + ScheduleLocalDrain(); + _instruments.OnInboxMessageReceived(_grainType, "duplicate"); + return DeliveryResult.Duplicate(); + } + + if (_inboxDict.Count >= _maxCapacity) + { + _instruments.OnInboxMessageReceived(_grainType, "backpressured"); + return DeliveryResult.Backpressured(); + } + + if (!_inbox.TryGetHandler(out _)) + { + _instruments.OnInboxMessageReceived(_grainType, "handler_not_found"); + LogHandlerNotFound(_logger, envelope.Envelope.MessageId, _grainContext.GrainId); + return DeliveryResult.HandlerNotFound(); + } + + var generation = _stateGeneration; + OwnershipProposal? proposal = null; + try + { + if (GetCommittedInboxCount() == 0 || !HasCommittedOwnership()) + { + proposal = await PrepareOwnershipAsync(CancellationToken.None).ConfigureAwait(true); + } + + ValidateReady(); + var operation = new AcceptanceWrite(generation, envelope, proposal); + await SubmitAsync(operation).ConfigureAwait(true); + ValidateReady(); + ScheduleLocalDrain(); + _instruments.OnInboxMessageReceived(_grainType, "accepted"); + LogMessageAccepted(_logger, envelope.Envelope.MessageId, _grainContext.GrainId); + return DeliveryResult.Accepted(); + } + finally + { + if (proposal is not null) + { + _pendingOwnershipIds.Remove(proposal.Id); + } + } + } + catch (Exception exception) + { + LogDeliveryOperationFailed(_logger, exception, envelope.Envelope.MessageId, _grainContext.GrainId); + throw; + } + finally + { + _gate.Release(); + } + } + + private async Task PrepareOwnershipAsync(CancellationToken cancellationToken) + { + ValidateReady(); + var generation = _stateGeneration; + var previousId = _jobId.Value; + var previousJob = _job.Value; + var sequence = checked(++_reservedSequence); + var id = MessagingJobOwnership.CreateId(_ownershipEpoch, sequence); + _pendingOwnershipIds.Add(id); + try + { + using var cancellation = MessagingCancellation.Combine(cancellationToken, _shutdownCts.Token, out var combinedToken); + var job = await _jobManager.ScheduleJobAsync(new ScheduleJobRequest + { + Target = _grainContext.GrainId, + JobName = JobName, + DueTime = _jobTimeProvider.GetUtcNow(), + Metadata = MessagingJobOwnership.CreateMetadata(id) + }, combinedToken).ConfigureAwait(true); + ValidateReady(); + return new(id, sequence, job, generation, previousId, previousJob); + } + catch + { + _pendingOwnershipIds.Remove(id); + _failure?.Throw(); + throw; + } + } + + private async ValueTask EnsureJobScheduledUnderGateAsync(CancellationToken cancellationToken) + { + ValidateReady(); + if (GetCommittedInboxCount() == 0 || HasCommittedOwnership()) + { + return; + } + + var proposal = await PrepareOwnershipAsync(cancellationToken).ConfigureAwait(true); + try + { + await SubmitAsync(new OwnershipWrite(_stateGeneration, proposal)).ConfigureAwait(true); + } + finally + { + _pendingOwnershipIds.Remove(proposal.Id); + } + } + + private async ValueTask SubmitAsync(InboxWrite operation) + { + ValidateReady(); + _pendingWrites.Add(operation); + try + { + if (operation is HandlerWrite handler) + { + await InvokeHandlerAsync(handler).ConfigureAwait(true); + if (handler.Skipped) + { + return; + } + } + + if (operation is not HandlerWrite { Completed: true } && !StageWrite(operation)) + { + return; + } + try + { + await _stateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + } + catch (JournaledStatePostCommitException) + { + AcknowledgeWrite(operation); + throw; + } + catch (Exception exception) + { + LatchFailure(exception); + throw; + } + AcknowledgeWrite(operation); + } + catch (OperationCanceledException) when (_failure is null + && operation is HandlerWrite { Completed: false } handler + && handler.Cancellation.IsCancellationRequested + && handler.Execution?.RejectionFailure is null) + { + throw; + } + catch (Exception exception) when (exception is not JournaledStatePostCommitException + && (exception is not OperationCanceledException || _failure is not null || !_shutdownToken.IsCancellationRequested + || operation is HandlerWrite { Completed: true } + || operation is HandlerWrite { Execution.RejectionFailure: not null })) + { + LatchFailure(exception); + try + { + _grainContext.Deactivate(new DeactivationReason(DeactivationReasonCode.ApplicationError, _failure!.SourceException, "Inbox operation failed.")); + } + catch (Exception deactivationException) + { + LogDeactivationRequestFailure(_logger, deactivationException); + } + _failure!.Throw(); + throw; + } + finally + { + _pendingWrites.Remove(operation); + operation.SignalFinished(); + } + + if (operation is HandlerWrite { PostCompletionFailure: { } failure }) + { + failure.Throw(); + } + } + + private void ValidateReady() + { + _failure?.Throw(); + ThrowIfOwnershipStateInvalid(); + _shutdownToken.ThrowIfCancellationRequested(); + if (!_recoveryCompleted) + { + throw new InvalidOperationException("Inbox initialization must complete before processing."); + } + } + + private void ThrowIfOwnershipStateInvalid() + { + var error = _ownershipStateError ?? MessagingJobOwnership.GetPairError(_jobId.Value, _job.Value); + if (error is not null) + { + throw new InvalidOperationException(error); + } + } + + private void ValidateGeneration(long generation) + { + ValidateReady(); + if (generation != _stateGeneration) + { + throw new InvalidOperationException("The admitted inbox operation belongs to an obsolete activation or deletion generation."); + } + } + + private bool HasCommittedOwnership() => + MessagingJobOwnership.HasOwner(_jobId.Value, _job.Value) + && string.Equals(_durableOwnershipId, _jobId.Value, StringComparison.Ordinal) + && MessagingJobOwnership.IsSamePhysicalJob(_durableJob, _job.Value); + + private bool IsCurrentOwner(PumpOwner owner) => owner.Generation == _stateGeneration + && string.Equals(owner.Id, _jobId.Value, StringComparison.Ordinal) + && MessagingJobOwnership.IsSamePhysicalJob(owner.Job, _job.Value) + && HasCommittedOwnership(); + + private void ValidateOwner(PumpOwner owner) + { + ValidateGeneration(owner.Generation); + if (!IsCurrentOwner(owner)) + { + throw new InvalidOperationException("The admitted inbox operation no longer owns the acknowledged physical job."); + } + } + + // Every provisional key is an inbox key; acknowledgement removes only its provisional marker. + private int GetCommittedInboxCount() => _inboxDict.Count - _provisionalAcceptances.Count; + + private async ValueTask InvokeHandlerAsync(HandlerWrite operation) + { + ValidateOwner(operation.Owner); + if (!_inboxDict.ContainsKey(operation.Key)) + { + throw new InvalidOperationException("The admitted inbox message is no longer pending."); + } + + if (operation.Cancellation.IsCancellationRequested) + { + operation.Skipped = true; + return; + } + + // Both pump entry points already combine activation shutdown with their attempt scope. + // Borrow that scope through handler completion and the owned journal write. + var combinedToken = operation.Cancellation; + var previous = _handlerExecution.Value; + var execution = operation.Execution = new HandlerExecution(this, operation); + _handlerExecution.Value = execution; + try + { + var found = _inbox.TryGetHandler(out var handler); + if (!found) + { + operation.Error = new InvalidOperationException("No inbox handler is registered."); + operation.DeadLetter = true; + return; + } + + execution.Active = true; + await handler!.HandleAsync(new InboxHandlerContext( + operation.Envelope, execution.Complete), + combinedToken).ConfigureAwait(true); + ThrowIfHandlerOperationRejected(execution); + if (!operation.Completed) + { + execution.RejectOperation(new InvalidOperationException( + "Inbox handlers must call Complete before returning successfully.")); + } + } + catch (Exception exception) when (operation.Completed) + { + operation.PostCompletionFailure = execution.RejectionFailure ?? ExceptionDispatchInfo.Capture(exception); + LogHandlerException(_logger, operation.PostCompletionFailure.SourceException, + operation.Envelope.Envelope.MessageId, _grainContext.GrainId); + } + catch (Exception) when (execution.RejectionFailure is not null) + { + execution.RejectionFailure.Throw(); + throw; + } + catch (Exception exception) when (!combinedToken.IsCancellationRequested && _failure is null) + { + // Before Complete, handler failures retain the message for retry or dead-lettering. + operation.Error = exception; + LogHandlerException(_logger, exception, operation.Envelope.Envelope.MessageId, _grainContext.GrainId); + } + finally + { + execution.Active = false; + _handlerExecution.Value = previous; + } + + if (operation.Completed) + { + return; + } + ValidateOwner(operation.Owner); + + if (operation.Error is not null) + { + var attempts = _messageStates.TryGetValue(operation.Key, out var state) ? state.AttemptCount : 0; + var count = checked(attempts + 1); + operation.DeadLetter = count >= _maxProcessingAttempts; + operation.Retry = new InboxMessageState + { + AttemptCount = count, + LastError = operation.Error.ToString(), + NextAttemptAt = MessagingTime.AddClamped(_timeProvider.GetUtcNow(), + TimeSpan.FromTicks(_retryDelay.Ticks * (1L << Math.Min(count - 1, InboxOptions.MaximumBackoffExponent)))) + }; + } + } + + private bool StageWrite(InboxWrite operation) + { + ValidateGeneration(operation.Generation); + var processedBefore = operation is CompactWrite ? _processed.Count : 0; + var deadLettersBefore = operation is CompactWrite ? _deadLetters.Count : 0; + switch (operation) + { + case AcceptanceWrite acceptance: + if (_inboxDict.ContainsKey(acceptance.Key) || _inboxDict.Count >= _maxCapacity) + { + throw new InvalidOperationException("The admitted inbox acceptance no longer matches available capacity or message identity."); + } + + if (acceptance.Owner is { } acceptanceOwner) + { + ApplyOwnership(acceptanceOwner); + } + else if (!HasCommittedOwnership()) + { + throw new InvalidOperationException("The admitted inbox acceptance requires acknowledged job ownership."); + } + + _processed.Remove(acceptance.Key); + _inboxDict.Add(acceptance.Key, acceptance.Envelope); + _messageStates.Add(acceptance.Key, new InboxMessageState()); + _provisionalAcceptances.Add(acceptance.Key); + UpdateInboxDepth(1); + break; + case OwnershipWrite ownership: + if (GetCommittedInboxCount() == 0) + { + throw new InvalidOperationException("The admitted ownership repair has no pending inbox work."); + } + ApplyOwnership(ownership.Owner); + break; + case HandlerWrite handler: + ValidateOwner(handler.Owner); + handler.Cancellation.ThrowIfCancellationRequested(); + if (handler.Skipped) + { + break; + } + + if (!_inboxDict.ContainsKey(handler.Key)) + { + throw new InvalidOperationException("The invoked inbox handler lost its pending message before capture."); + } + + if (handler.Error is not null && !handler.DeadLetter) + { + _messageStates[handler.Key] = handler.Retry!; + } + else + { + var now = _timeProvider.GetUtcNow(); + if (handler.DeadLetter) + { + DeadLetterRetention.Compact(_deadLetters, now, _deadLetterRetentionPeriod, + _maxRetainedDeadLetters, static entry => entry.DeadLetteredAt, + reservedCapacity: _deadLetters.ContainsKey(handler.Key) ? 0 : 1); + _deadLetters[handler.Key] = new InboxDeadLetter + { + Message = handler.Envelope, + DeadLetteredAt = now, + Reason = handler.Error!.Message, + AttemptCount = handler.Retry?.AttemptCount ?? 0 + }; + } + StageHandlerCompletion(handler); + } + break; + case ClearOwnerWrite clear: + ValidateOwner(clear.Owner); + if (_inboxDict.Count != 0) + { + throw new InvalidOperationException("The admitted inbox owner cannot be cleared while work is pending."); + } + _completedJobId.Value = clear.Owner.Id; + _jobId.Value = null; + _job.Value = null; + break; + case CompactWrite: + DeadLetterRetention.Compact(_deadLetters, _timeProvider.GetUtcNow(), _deadLetterRetentionPeriod, + _maxRetainedDeadLetters, static entry => entry.DeadLetteredAt); + break; + } + + var maintenanceTime = _timeProvider.GetUtcNow(); + if (IsProcessedMaintenanceDue(maintenanceTime)) + { + CompactProcessedMessages(maintenanceTime); + } + + return operation is not CompactWrite || _processed.Count != processedBefore || _deadLetters.Count != deadLettersBefore; + } + + private void StageHandlerCompletion(HandlerWrite operation) + { + var now = _timeProvider.GetUtcNow(); + RemoveMessage(operation.Key); + _messageStates.Remove(operation.Key); + _processed[operation.Key] = now; + TrackProcessedExpiry(now); + operation.Completed = true; + } + + private void ApplyOwnership(OwnershipProposal proposal) + { + ValidateGeneration(proposal.Generation); + if (!string.Equals(_jobId.Value, proposal.PreviousId, StringComparison.Ordinal) + || !(proposal.PreviousJob is null && _job.Value is null + || MessagingJobOwnership.IsSamePhysicalJob(proposal.PreviousJob, _job.Value))) + { + throw new InvalidOperationException("The admitted inbox ownership proposal no longer matches the preceding owner."); + } + _jobId.Value = proposal.Id; + _job.Value = proposal.Job; + _jobSequence.Value = proposal.Sequence; + } + + private void AcknowledgeWrite(InboxWrite operation) + { + // Ownership-changing operations retain the inbox gate through their own write acknowledgement. + switch (operation) + { + case AcceptanceWrite acceptance: + if (acceptance.Owner is { } owner) + { + _durableOwnershipId = owner.Id; + _durableJob = owner.Job; + } + _provisionalAcceptances.Remove(acceptance.Key); + break; + case OwnershipWrite ownership: + _durableOwnershipId = ownership.Owner.Id; + _durableJob = ownership.Owner.Job; + break; + case ClearOwnerWrite clear: + _durableCompletedJobId = clear.Owner.Id; + _durableOwnershipId = null; + _durableJob = null; + break; + } + } + + private void LatchFailure(Exception exception) + { + _failure ??= ExceptionDispatchInfo.Capture(exception); + _pumpCoordinator.Reset(); + _pumpResults.Clear(JobName); + CancelProcessing(); + } + + private void CancelProcessing() + { + _localDrainRequested = false; + try + { + _shutdownCts.Cancel(); + } + catch (AggregateException cancellationException) + { + LogCancellationCallbackFailure(_logger, cancellationException); + } + finally + { + // Activation cancellation owns callback failures; disposing timers first would + // transfer those errors to the runtime timer logger instead of preserving them here. + _pumpTimer?.Dispose(); + _localDrainTimer?.Dispose(); + } + } + + [LoggerMessage(Level = LogLevel.Error, Message = "An inbox cancellation callback failed while stopping processing.")] + private static partial void LogCancellationCallbackFailure(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "Requesting deactivation after an inbox persistence failure failed.")] + private static partial void LogDeactivationRequestFailure(ILogger logger, Exception exception); + + private void InitializeRecoveredState() + { + _stateGeneration++; + _reservedSequence = _jobSequence.Value; + _durableOwnershipId = _jobId.Value; + _durableJob = _job.Value; + _durableCompletedJobId = _completedJobId.Value; + _ownershipStateError = MessagingJobOwnership.GetPairError(_jobId.Value, _job.Value); + _lastProcessedCompaction = null; + RebuildProcessedExpiry(); + _recoveryCompleted = true; + ReconcileInboxDepth(); + } + + internal async Task ResumeProcessingAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + ValidateReady(); + EnsureMetricsActive(); + await EnsureJobScheduledUnderGateAsync(cancellationToken).ConfigureAwait(true); + ScheduleLocalDrain(); + } + finally + { + _gate.Release(); + } + } + + public async Task OnStart(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_grainContext.GrainInstance is not DurableGrain and not IMessagingGrain) + { + throw new InvalidOperationException("Inbox activation requires IMessagingGrain or DurableGrain."); + } + foreach (var (key, message) in _inboxDict) + { + EnvelopeValidation.Validate(message); + if (key != message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered inbox key does not match its command identity."); + } + } + foreach (var entry in _deadLetters.Values) + { + EnvelopeValidation.Validate(entry.Message); + } + InitializeRecoveredState(); + await ResumeProcessingAsync(cancellationToken).ConfigureAwait(true); + if (_deadLetters.Values.Any(entry => MessagingTime.IsExpired(_timeProvider.GetUtcNow(), entry.DeadLetteredAt, _deadLetterRetentionPeriod)) + || _deadLetters.Count > _maxRetainedDeadLetters + || HasExpiredProcessedMessages(_timeProvider.GetUtcNow())) + { + await SubmitAsync(new CompactWrite(_stateGeneration)).ConfigureAwait(true); + } + } + + public async Task OnStop(CancellationToken cancellationToken) + { + var operations = _pendingWrites.Select(static operation => operation.WaitForFinished()).Append(_activeDelivery).ToArray(); + StopProcessing(); + // Owned work outlives caller cancellation and drains through handler completion and actual persistence. + await Task.WhenAll(operations).ConfigureAwait( + ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + } + + internal void StopProcessing() + { + try + { + CancelProcessing(); + } + finally + { + _pumpCoordinator.Reset(); + _pumpResults.Clear(JobName); + if (Interlocked.Exchange(ref _metricsActive, 0) != 0) + { + _instruments.OnInboxDepthChanged(-Interlocked.Exchange(ref _reportedDepth, 0)); + } + } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + try + { + StopProcessing(); + } + finally + { + // Scope teardown has drained actual operations and releases durable values. + // Retired admission metadata must not continue reporting provisional work. + _provisionalAcceptances.Clear(); + _shutdownCts.Dispose(); + } + } + } + + private sealed class HandlerExecution(InboxExtension owner, HandlerWrite operation) + { + public InboxExtension Owner { get; } = owner; + public bool Active { get; set; } + public ExceptionDispatchInfo? RejectionFailure { get; private set; } + + public void Complete() + { + ValidateAttempt(); + if (operation.Completed) + { + return; + } + try + { + // Business state and directly injected outbox sends have already been staged. + // Stage inbox removal and dedupe synchronously, before the owned write can await. + Owner.StageHandlerCompletion(operation); + } + catch (Exception exception) + { + RejectOperation(exception); + throw; + } + } + + private void ValidateAttempt() + { + if (!Active || !ReferenceEquals(_handlerExecution.Value, this)) + { + RejectOperation(new InvalidOperationException( + "The inbox handler context belongs to an inactive or different attempt.")); + } + ThrowIfHandlerOperationRejected(this); + } + + [DoesNotReturn] + public void RejectOperation(Exception exception) + { + RejectionFailure ??= ExceptionDispatchInfo.Capture(exception); + if (_handlerExecution.Value is { } current && ReferenceEquals(current.Owner, Owner)) + { + current.RejectionFailure ??= RejectionFailure; + } + RejectionFailure.Throw(); + } + } + + private readonly record struct PumpOwner(string Id, DurableJob Job, long Generation); + private sealed record OwnershipProposal(string Id, long Sequence, DurableJob Job, long Generation, string? PreviousId, DurableJob? PreviousJob); + + private abstract class InboxWrite(long generation) + { + public long Generation { get; } = generation; + private TaskCompletionSource? _finished; + private bool _retired; + + public TaskCompletionSource Finished + { + get + { + var finished = _finished ??= new(TaskCreationOptions.RunContinuationsAsynchronously); + if (_retired) finished.TrySetResult(); + return finished; + } + } + public Task WaitForFinished() => _retired ? Task.CompletedTask : Finished.Task; + + public void SignalFinished() + { + _retired = true; + _finished?.TrySetResult(); + } + } + + private sealed class AcceptanceWrite(long generation, InboxMessage envelope, OwnershipProposal? owner) : InboxWrite(generation) + { + public InboxMessage Envelope { get; } = envelope; + public HierarchicalKey Key => Envelope.Envelope.MessageId; + public OwnershipProposal? Owner { get; } = owner; + } + + private sealed class OwnershipWrite(long generation, OwnershipProposal owner) : InboxWrite(generation) + { + public OwnershipProposal Owner { get; } = owner; + } + + private sealed class HandlerWrite(PumpOwner owner, InboxMessage envelope, CancellationToken cancellation) : InboxWrite(owner.Generation) + { + public PumpOwner Owner { get; } = owner; + public InboxMessage Envelope { get; } = envelope; + public HierarchicalKey Key => Envelope.Envelope.MessageId; + public CancellationToken Cancellation { get; } = cancellation; + public HandlerExecution? Execution { get; set; } + public bool Completed { get; set; } + public ExceptionDispatchInfo? PostCompletionFailure { get; set; } + public bool Skipped { get; set; } + public bool DeadLetter { get; set; } + public Exception? Error { get; set; } + public InboxMessageState? Retry { get; set; } + } + + private sealed class ClearOwnerWrite(PumpOwner owner) : InboxWrite(owner.Generation) + { + public PumpOwner Owner { get; } = owner; + } + + private sealed class CompactWrite(long generation) : InboxWrite(generation); + + public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) + { + _failure?.Throw(); + _shutdownToken.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); + if (!_recoveryCompleted) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + ThrowIfOwnershipStateInvalid(); + if (!MessagingJobOwnership.TryGetOwnershipId(context.Job, out var ownershipId)) + { + return DurableJobRunResult.Completed; + } + + if (_pendingOwnershipIds.Count != 0 || _pendingWrites.Any(static operation => operation is ClearOwnerWrite)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + var key = new MessagingPumpExecutionKey( + JobName, + context.Job.Id, + context.RunId, + Volatile.Read(ref _stateGeneration)); + if (!string.Equals(_jobId.Value, ownershipId, StringComparison.Ordinal)) + { + var disposition = MessagingJobOwnership.ResolveMismatch( + _recoveryCompleted, + HasCommittedOwnership(), + MessagingJobOwnership.IsCompleted(_durableCompletedJobId, ownershipId), + _inboxDict.Count > 0); + if (disposition == OwnershipMismatchDisposition.ReclaimOrphan) + { + LogOrphanedJobReclaimed(_logger, ownershipId, _grainContext.GrainId); + _instruments.OnOrphanedJobReclaimed(_grainContext.GrainId.Type.ToString(), JobName); + return CompleteObsoleteExecution(key); + } + + if (disposition == OwnershipMismatchDisposition.CompleteStale) + { + return CompleteObsoleteExecution(key); + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!HasCommittedOwnership()) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!MessagingJobOwnership.IsSamePhysicalJob(_job.Value, context.Job)) + { + return DurableJobRunResult.Completed; + } + + if (_pumpResults.TryTake(key, out var result, out var exception)) + { + if (exception is not null) + { + throw exception; + } + + return result!; + } + + if (!_pumpCoordinator.TryAcquire(ownershipId, cancellationToken, out var lease)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!_pumpResults.TryStart(key, cancellationToken, out var execution)) + { + _pumpCoordinator.Release(lease); + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + try + { + (_pumpTimer ??= new(this)).Queue(new(execution, lease, new(ownershipId, context.Job, key.StateGeneration), cancellationToken)); + } + catch (Exception registrationException) + { + _pumpCoordinator.Release(lease); + _pumpResults.Fail(execution, registrationException); + throw; + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + private DurableJobRunResult CompleteObsoleteExecution(MessagingPumpExecutionKey key) + { + // Committed ownership establishes retirement; this run's retained result will no longer be polled. + _pumpResults.TryTake(key, out _, out _); + return DurableJobRunResult.Completed; + } + + private async Task RunPumpTimerAsync( + MessagingPumpExecution execution, + MessagingPumpLease lease, + PumpOwner pumpOwner, + CancellationToken jobCancellation, + CancellationToken timerCancellation) + { + if (!_pumpCoordinator.IsCurrent(lease) || !IsCurrentOwner(pumpOwner)) + { + _pumpResults.Discard(execution); + return; + } + + if (!_pumpResults.TryBegin(execution)) + { + return; + } + + DurableJobRunResult? result = null; + Exception? failure = null; + try + { + using var linkedCancellation = MessagingCancellation.Combine( + jobCancellation, timerCancellation, _shutdownCts.Token, out var combinedToken); + result = await ExecuteJobCoreAsync( + pumpOwner, + clearOwnershipWhenEmpty: true, + combinedToken); + } + catch (Exception exception) + { + failure = exception; + } + finally + { + if (failure is null) + { + _pumpResults.Complete(execution, result!); + } + else + { + _pumpResults.Fail(execution, failure); + } + } + } + + private async ValueTask ExecuteJobCoreAsync(PumpOwner owner, bool clearOwnershipWhenEmpty, CancellationToken cancellationToken) + { + ValidateReady(); + if (!IsCurrentOwner(owner)) + { + return DurableJobRunResult.Completed; + } + + var now = _timeProvider.GetUtcNow(); + var pending = new List(Math.Min(_inboxDict.Count, _batchSize)); + foreach (var pair in _inboxDict) + { + if (!_provisionalAcceptances.Contains(pair.Key) + && (!_messageStates.TryGetValue(pair.Key, out var state) || state.NextAttemptAt is null || state.NextAttemptAt <= now)) + { + pending.Add(pair.Value); + if (pending.Count == _batchSize) + { + break; + } + } + } + foreach (var envelope in pending) + { + cancellationToken.ThrowIfCancellationRequested(); + ValidateOwner(owner); + var start = Stopwatch.GetTimestamp(); + var operation = new HandlerWrite(owner, envelope, cancellationToken); + await SubmitAsync(operation).ConfigureAwait(true); + if (operation.Skipped) + { + cancellationToken.ThrowIfCancellationRequested(); + } + var status = operation.Error is null ? "success" : operation.DeadLetter ? "dead_lettered" : "retry"; + _instruments.OnInboxMessageProcessed(_grainType, status); + _instruments.OnInboxProcessingDuration(Stopwatch.GetElapsedTime(start), _grainType); + } + + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + ValidateReady(); + if (!IsCurrentOwner(owner)) + { + return DurableJobRunResult.Completed; + } + if (_inboxDict.Count == 0) + { + if (clearOwnershipWhenEmpty) + { + await SubmitAsync(new ClearOwnerWrite(owner)).ConfigureAwait(true); + } + return DurableJobRunResult.Completed; + } + } + finally + { + _gate.Release(); + } + + if (HasExpiredProcessedMessages(_timeProvider.GetUtcNow())) + { + await SubmitAsync(new CompactWrite(_stateGeneration)).ConfigureAwait(true); + } + + var nextAttempt = GetNextAttemptAt(); + if (GetNextProcessedMaintenance() is { } maintenance && maintenance < nextAttempt) + { + nextAttempt = maintenance; + } + var delay = nextAttempt - _timeProvider.GetUtcNow(); + return DurableJobRunResult.RescheduleAt(MessagingTime.AddClamped( + _jobTimeProvider.GetUtcNow(), delay > TimeSpan.Zero ? delay : TimeSpan.Zero)); + } + + private DateTimeOffset GetNextAttemptAt() + { + var now = _timeProvider.GetUtcNow(); + var next = DateTimeOffset.MaxValue; + foreach (var key in _inboxDict.Keys) + { + if (!_messageStates.TryGetValue(key, out var state) || state.NextAttemptAt is not { } at || at <= now) + { + return now; + } + if (at < next) + { + next = at; + } + } + return next; + } + + private DateTimeOffset? GetNextProcessedMaintenance() + { + if (_nextProcessedExpiry is not { } next) + { + return null; + } + if (_lastProcessedCompaction is { } last) + { + if (_processedCompactionInterval.Ticks > DateTimeOffset.MaxValue.UtcTicks - last.UtcTicks) + { + return null; + } + var earliestScan = MessagingTime.AddClamped(last, _processedCompactionInterval); + if (earliestScan > next) + { + next = earliestScan; + } + } + return next; + } + + private bool IsProcessedMaintenanceDue(DateTimeOffset now) => + GetNextProcessedMaintenance() is { } next && now >= next; + + private bool HasExpiredProcessedMessages(DateTimeOffset now) + { + if (!IsProcessedMaintenanceDue(now)) + { + return false; + } + if (_processed.Any(pair => MessagingTime.IsExpired(now, pair.Value, _deduplicationWindow))) + { + return true; + } + + _lastProcessedCompaction = now; + RebuildProcessedExpiry(); + return false; + } + + private void TrackProcessedExpiry(DateTimeOffset processedAt) + { + if (_deduplicationWindow.Ticks > DateTimeOffset.MaxValue.UtcTicks - processedAt.UtcTicks) + { + return; + } + var expiry = MessagingTime.AddClamped(processedAt, _deduplicationWindow); + if (_nextProcessedExpiry is null || expiry < _nextProcessedExpiry) + { + _nextProcessedExpiry = expiry; + } + } + + private void RebuildProcessedExpiry() + { + _nextProcessedExpiry = null; + foreach (var entry in _processed) + { + TrackProcessedExpiry(entry.Value); + } + } + + private void CompactProcessedMessages(DateTimeOffset now) + { + // Amortize dictionary scans over retention time; ordinary completion and owner-clear writes only check the deadline. + List? expired = null; + _nextProcessedExpiry = null; + foreach (var entry in _processed) + { + if (MessagingTime.IsExpired(now, entry.Value, _deduplicationWindow)) + { + (expired ??= []).Add(entry.Key); + } + else + { + TrackProcessedExpiry(entry.Value); + } + } + if (expired is not null) + { + foreach (var key in expired) + { + _processed.Remove(key); + } + } + _lastProcessedCompaction = now; + } + + private void EnsureMetricsActive() + { + if (Interlocked.Exchange(ref _metricsActive, 1) == 0) + { + Volatile.Write(ref _reportedDepth, _inboxDict.Count); + _instruments.OnInboxDepthChanged(_inboxDict.Count); + } + } + + private void UpdateInboxDepth(int delta) + { + if (Volatile.Read(ref _metricsActive) != 0) + { + Interlocked.Add(ref _reportedDepth, delta); + _instruments.OnInboxDepthChanged(delta); + } + } + + private void ReconcileInboxDepth() + { + if (Volatile.Read(ref _metricsActive) == 0) + { + return; + } + + var count = _inboxDict.Count; + var delta = count - Interlocked.Exchange(ref _reportedDepth, count); + if (delta != 0) + { + _instruments.OnInboxDepthChanged(delta); + } + } + + private bool RemoveMessage(HierarchicalKey key) + { + if (!_inboxDict.Remove(key)) + { + return false; + } + + UpdateInboxDepth(-1); + + return true; + } + + // Structured logging using LoggerMessage source generator + + [LoggerMessage(Level = LogLevel.Error, EventName = "DeliveryOperationFailed", + Message = "Inbox delivery of message {MessageId} to {GrainId} failed")] + private static partial void LogDeliveryOperationFailed(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId grainId); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "No inbox handler registered for message {MessageId} to {ReceiverId}")] + private static partial void LogHandlerNotFound(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Accepted message {MessageId} to {ReceiverId}")] + private static partial void LogMessageAccepted(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Handler threw exception for message {MessageId} to {ReceiverId}")] + private static partial void LogHandlerException(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Reclaimed orphaned inbox job ownership {OwnershipId} for grain {GrainId}")] + private static partial void LogOrphanedJobReclaimed(ILogger logger, string ownershipId, GrainId grainId); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Error scheduling inbox recovery for grain {GrainId}")] + private static partial void LogRecoverySchedulingError(ILogger logger, Exception exception, GrainId grainId); + + private readonly record struct PumpTurn(MessagingPumpExecution Execution, + MessagingPumpLease Lease, PumpOwner Owner, CancellationToken JobCancellation); + private readonly record struct LocalDrainTurn(MessagingPumpLease Lease, PumpOwner Owner); + + private sealed class PumpTimerState(InboxExtension owner) : MessagingTurn + { + protected override IGrainTimer RegisterTimer(long registrationGeneration) => owner._timerRegistry.RegisterGrainTimer( + owner._grainContext, (state, token) => state.RunAsync(registrationGeneration, token), this, + new GrainTimerCreationOptions(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan) { Interleave = false, KeepAlive = true }); + + protected override async Task ExecuteAsync(PumpTurn turn, CancellationToken timerCancellation) + { + try + { + await owner.RunPumpTimerAsync(turn.Execution, turn.Lease, turn.Owner, turn.JobCancellation, timerCancellation); + } + finally + { + owner._pumpCoordinator.Release(turn.Lease); + if (owner._localDrainRequested) owner.ScheduleLocalDrain(); + } + } + + protected override void Discard(PumpTurn turn) + { + owner._pumpResults.Discard(turn.Execution); + owner._pumpCoordinator.Release(turn.Lease); + } + } + + private void ScheduleLocalDrain() + { + if (_shutdownToken.IsCancellationRequested || _failure is not null + || _jobId.Value is not { Length: > 0 } jobId + || GetCommittedInboxCount() == 0 + || !HasCommittedOwnership()) + { + return; + } + + _localDrainRequested = true; + if (_pumpCoordinator.IsActive || !_pumpCoordinator.TryAcquire(jobId, _shutdownToken, out var lease)) + { + return; + } + + _localDrainRequested = false; + try + { + (_localDrainTimer ??= new(this)).Queue(new(lease, new PumpOwner(jobId, _job.Value!, _stateGeneration))); + } + catch + { + _pumpCoordinator.Release(lease); + throw; + } + } + + private sealed class LocalDrainTimerState(InboxExtension owner) : MessagingTurn + { + protected override IGrainTimer RegisterTimer(long registrationGeneration) => owner._timerRegistry.RegisterGrainTimer( + owner._grainContext, (state, token) => state.RunAsync(registrationGeneration, token), this, + new GrainTimerCreationOptions(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan) { Interleave = false, KeepAlive = true }); + + protected override async Task ExecuteAsync(LocalDrainTurn turn, CancellationToken cancellationToken) + { + try + { + if (owner._pumpCoordinator.IsCurrent(turn.Lease)) + { + using var cancellation = MessagingCancellation.Combine( + cancellationToken, owner._shutdownToken, out var combinedToken); + _ = await owner.ExecuteJobCoreAsync(turn.Owner, clearOwnershipWhenEmpty: false, combinedToken); + } + } + finally + { + owner._pumpCoordinator.Release(turn.Lease); + if (owner._localDrainRequested) owner.ScheduleLocalDrain(); + } + } + + protected override void Discard(LocalDrainTurn turn) => owner._pumpCoordinator.Release(turn.Lease); + } + +} diff --git a/src/Orleans.Messaging/MessagingActivationValidator.cs b/src/Orleans.Messaging/MessagingActivationValidator.cs new file mode 100644 index 00000000000..cfdfaf65056 --- /dev/null +++ b/src/Orleans.Messaging/MessagingActivationValidator.cs @@ -0,0 +1,49 @@ +using System; +using System.Linq; +using Orleans; +using Orleans.Concurrency; +using Orleans.Metadata; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal static class MessagingActivationValidator +{ + // The public attribute exposes the runtime's internal, sealed strategy type. + private static readonly Type _statelessWorkerPlacementType = new StatelessWorkerAttribute().PlacementStrategy.GetType(); + + public static void Validate(IGrainContext grainContext, GrainProperties properties, PlacementStrategy placementStrategy) + { + var grain = grainContext.GrainInstance + ?? throw new InvalidOperationException("Messaging activation requires an initialized grain instance."); + var grainType = grain.GetType(); + if (placementStrategy.GetType() == _statelessWorkerPlacementType) + { + throw new InvalidOperationException( + $"Messaging requires one activation per grain identity, but grain type '{grainType}' is a stateless worker."); + } + + if (properties.Properties.TryGetValue(WellKnownGrainTypeProperties.Reentrant, out var reentrant) && bool.Parse(reentrant) + || properties.Properties.ContainsKey(WellKnownGrainTypeProperties.MayInterleavePredicate)) + { + throw new InvalidOperationException( + $"Messaging requires non-reentrant grain execution, but grain type '{grainType}' enables interleaving."); + } + + var grainInterfaces = grainType + .GetInterfaces() + .Where(static type => typeof(IGrain).IsAssignableFrom(type)) + .ToArray(); + var interleavableMethod = grainInterfaces + .SelectMany(static type => type.GetInterfaces().Append(type)) + .Distinct() + .SelectMany(static type => type.GetMethods()) + .FirstOrDefault(static method => method.IsDefined(typeof(AlwaysInterleaveAttribute), inherit: true)); + if (interleavableMethod is not null) + { + throw new InvalidOperationException( + $"Messaging grain type '{grainType}' implements interleavable method " + + $"'{interleavableMethod.DeclaringType}.{interleavableMethod.Name}'."); + } + } +} diff --git a/src/Orleans.Messaging/MessagingCancellation.cs b/src/Orleans.Messaging/MessagingCancellation.cs new file mode 100644 index 00000000000..c868da367f7 --- /dev/null +++ b/src/Orleans.Messaging/MessagingCancellation.cs @@ -0,0 +1,44 @@ +using System.Threading; + +namespace Orleans.Messaging; + +internal static class MessagingCancellation +{ + // A borrowed token needs no resource; distinct scopes still require a link owned by the caller. + public static CancellationTokenSource? Combine(CancellationToken first, CancellationToken second, out CancellationToken token) + { + if (!first.CanBeCanceled || first == second) + { + token = second; + return null; + } + if (!second.CanBeCanceled) + { + token = first; + return null; + } + var source = CancellationTokenSource.CreateLinkedTokenSource(first, second); + token = source.Token; + return source; + } + + public static CancellationTokenSource? Combine( + CancellationToken first, CancellationToken second, CancellationToken third, out CancellationToken token) + { + if (!first.CanBeCanceled || first == second || first == third) + { + return Combine(second, third, out token); + } + if (!second.CanBeCanceled || second == third) + { + return Combine(first, third, out token); + } + if (!third.CanBeCanceled) + { + return Combine(first, second, out token); + } + var source = CancellationTokenSource.CreateLinkedTokenSource(first, second, third); + token = source.Token; + return source; + } +} diff --git a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs new file mode 100644 index 00000000000..bd0187a2a4f --- /dev/null +++ b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs @@ -0,0 +1,32 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal sealed class MessagingGrainTypeConfigurator(GrainClassMap grainClasses) : IConfigureGrainTypeComponents +{ + public void Configure(GrainType grainType, GrainProperties properties, GrainTypeSharedContext shared) + { + if (!grainClasses.TryGetGrainClass(grainType, out var grainClass)) + { + throw new InvalidOperationException($"No grain implementation is registered for '{grainType}'."); + } + + if (typeof(DurableGrain).IsAssignableFrom(grainClass) || typeof(IMessagingGrain).IsAssignableFrom(grainClass)) + { + var placementStrategy = shared.PlacementStrategy; + shared.AddActivationSetup(context => + { + MessagingActivationValidator.Validate(context, properties, placementStrategy); + var services = context.ActivationServices; + _ = services.GetRequiredService(); + _ = services.GetRequiredService(); + _ = services.GetRequiredService(); + _ = services.GetRequiredService(); + }); + } + } +} diff --git a/src/Orleans.Messaging/MessagingInstruments.cs b/src/Orleans.Messaging/MessagingInstruments.cs new file mode 100644 index 00000000000..01af41137fb --- /dev/null +++ b/src/Orleans.Messaging/MessagingInstruments.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Threading; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal sealed class MessagingInstruments(OrleansInstruments instruments) +{ + private const string MillisecondsUnit = "ms"; + private const string GrainTypeTagName = "grain_type"; + private const string StatusTagName = "status"; + + private readonly Counter _inboxMessagesReceived = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-received"); + private readonly Counter _inboxMessagesProcessed = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-processed"); + private readonly Counter _outboxMessagesSent = instruments.Meter.CreateCounter("orleans-messaging-outbox-messages-sent"); + private readonly Counter _outboxMessagesDelivered = instruments.Meter.CreateCounter("orleans-messaging-outbox-messages-delivered"); + private readonly Counter _orphanedJobsReclaimed = instruments.Meter.CreateCounter("orleans-messaging-orphaned-jobs-reclaimed"); + private readonly Histogram _inboxProcessingDuration = instruments.Meter.CreateHistogram("orleans-messaging-inbox-processing-duration", MillisecondsUnit); + private readonly Histogram _outboxDeliveryDuration = instruments.Meter.CreateHistogram("orleans-messaging-outbox-delivery-duration", MillisecondsUnit); + private readonly DepthTracker _inboxDepth = new(instruments.Meter, "orleans-messaging-inbox-depth"); + private readonly DepthTracker _outboxDepth = new(instruments.Meter, "orleans-messaging-outbox-depth"); + + internal static MessagingInstruments CreateForDirectConstruction() => new(new OrleansInstruments(new DirectMeterFactory())); + + internal void OnInboxDepthChanged(int delta) => _inboxDepth.Adjust(delta); + + internal void OnOutboxDepthChanged(int delta) => _outboxDepth.Adjust(delta); + + internal void OnInboxMessageReceived(string grainType, string status) => + Add(_inboxMessagesReceived, grainType, status); + + internal void OnInboxMessageProcessed(string grainType, string status) => + Add(_inboxMessagesProcessed, grainType, status); + + internal void OnInboxProcessingDuration(TimeSpan duration, string grainType) => + Record(_inboxProcessingDuration, duration, grainType); + + internal void OnOutboxMessageSent(string grainType) + { + if (_outboxMessagesSent.Enabled) + { + _outboxMessagesSent.Add(1, CreateTags(grainType)); + } + } + + internal void OnOutboxMessageDelivered(string grainType, string status) => + Add(_outboxMessagesDelivered, grainType, status); + + internal void OnOutboxDeliveryDuration(TimeSpan duration, string grainType) => + Record(_outboxDeliveryDuration, duration, grainType); + + internal void OnOrphanedJobReclaimed(string grainType, string jobName) + { + if (_orphanedJobsReclaimed.Enabled) + { + _orphanedJobsReclaimed.Add( + 1, + [ + new(GrainTypeTagName, grainType), + new("job_name", jobName) + ]); + } + } + + private static void Add(Counter counter, string grainType, string status) + { + if (counter.Enabled) + { + counter.Add( + 1, + [ + new(GrainTypeTagName, grainType), + new(StatusTagName, status) + ]); + } + } + + private static void Record(Histogram histogram, TimeSpan duration, string grainType) + { + if (histogram.Enabled) + { + histogram.Record(Math.Max(0, duration.TotalMilliseconds), CreateTags(grainType)); + } + } + + private static KeyValuePair[] CreateTags(string grainType) => + [ + new(GrainTypeTagName, grainType) + ]; + + private sealed class DirectMeterFactory : IMeterFactory + { + public Meter Create(MeterOptions options) => new(options); + + public void Dispose() + { + } + } + + private sealed class DepthTracker + { + private readonly ObservableGauge _gauge; + private long _value; + + public DepthTracker(Meter meter, string name) + { + _gauge = meter.CreateObservableGauge(name, () => Volatile.Read(ref _value)); + } + + public void Adjust(int delta) => Interlocked.Add(ref _value, delta); + } +} diff --git a/src/Orleans.Messaging/MessagingJobOwnership.cs b/src/Orleans.Messaging/MessagingJobOwnership.cs new file mode 100644 index 00000000000..a73ae264e66 --- /dev/null +++ b/src/Orleans.Messaging/MessagingJobOwnership.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal static class MessagingJobOwnership +{ + private const string MetadataKey = "orleans.messaging.ownership-id"; + + public static IReadOnlyDictionary CreateMetadata(string ownershipId) => + new Dictionary(1, StringComparer.Ordinal) + { + [MetadataKey] = ownershipId + }; + + public static bool TryGetOwnershipId(DurableJob job, out string ownershipId) + { + if (job.Metadata is not null + && job.Metadata.TryGetValue(MetadataKey, out var value) + && !string.IsNullOrWhiteSpace(value)) + { + ownershipId = value; + return true; + } + + ownershipId = string.Empty; + return false; + } + + + public static bool HasOwner(string? ownershipId, DurableJob? job) => + !string.IsNullOrWhiteSpace(ownershipId) && job is not null; + + public static string? GetPairError(string? ownershipId, DurableJob? job) + { + var hasOwnershipId = !string.IsNullOrWhiteSpace(ownershipId); + if (hasOwnershipId != (job is not null)) + { + return "The messaging ownership generation and job handle must either both be present or both be absent."; + } + + if (hasOwnershipId + && (!TryGetOwnershipId(job!, out var jobOwnershipId) + || !string.Equals(jobOwnershipId, ownershipId, StringComparison.Ordinal))) + { + return "The messaging job handle metadata does not match its ownership generation."; + } + + return null; + } + + public static bool IsSamePhysicalJob(DurableJob? expected, DurableJob? actual) => + expected is not null + && actual is not null + && string.Equals(expected.Id, actual.Id, StringComparison.Ordinal) + && string.Equals(expected.ShardId, actual.ShardId, StringComparison.Ordinal); + + public static string CreateId(string epoch, long sequence) + { + ArgumentException.ThrowIfNullOrWhiteSpace(epoch); + return $"{epoch}:{sequence.ToString(CultureInfo.InvariantCulture)}"; + } + + public static bool IsCompleted(string? completedOwnershipId, string ownershipId) + { + if (string.Equals(completedOwnershipId, ownershipId, StringComparison.Ordinal)) + { + return true; + } + + return TryParse(completedOwnershipId, out var completedEpoch, out var completed) + && TryParse(ownershipId, out var currentEpoch, out var current) + && string.Equals(completedEpoch, currentEpoch, StringComparison.Ordinal) + && current <= completed; + } + + private static bool TryParse(string? value, out string epoch, out long sequence) + { + sequence = 0; + var separator = value?.LastIndexOf(':') ?? -1; + if (separator <= 0 + || !long.TryParse( + value.AsSpan(separator + 1), + NumberStyles.None, + CultureInfo.InvariantCulture, + out sequence)) + { + epoch = string.Empty; + return false; + } + + epoch = value![..separator]; + return true; + } + + public static OwnershipMismatchDisposition ResolveMismatch( + bool recoveryCompleted, + bool hasCurrentOwner, + bool ownershipCompleted, + bool hasWork) + { + if (!recoveryCompleted) + { + return OwnershipMismatchDisposition.WaitForRecovery; + } + + if (hasCurrentOwner || ownershipCompleted) + { + return OwnershipMismatchDisposition.CompleteStale; + } + + return hasWork + ? OwnershipMismatchDisposition.WaitForReplacement + : OwnershipMismatchDisposition.ReclaimOrphan; + } +} + +internal enum OwnershipMismatchDisposition +{ + WaitForRecovery, + WaitForReplacement, + CompleteStale, + ReclaimOrphan +} diff --git a/src/Orleans.Messaging/MessagingPumpCoordinator.cs b/src/Orleans.Messaging/MessagingPumpCoordinator.cs new file mode 100644 index 00000000000..c4d57c76f83 --- /dev/null +++ b/src/Orleans.Messaging/MessagingPumpCoordinator.cs @@ -0,0 +1,82 @@ +using System; +using System.Threading; + +namespace Orleans.Messaging; + +/// +/// Coalesces physical durable-job callbacks which represent the same logical pump ownership. +/// +internal sealed class MessagingPumpCoordinator +{ + private readonly object _lock = new(); + private string? _activeOwnershipId; + private CancellationToken _activeCancellationToken; + private long _activeGeneration; + + public bool IsActive + { + get + { + lock (_lock) + { + return _activeOwnershipId is not null; + } + } + } + + public bool TryAcquire( + string ownershipId, + CancellationToken cancellationToken, + out MessagingPumpLease lease) + { + ArgumentException.ThrowIfNullOrWhiteSpace(ownershipId); + + lock (_lock) + { + if (string.Equals(_activeOwnershipId, ownershipId, StringComparison.Ordinal) + && !_activeCancellationToken.IsCancellationRequested) + { + lease = default; + return false; + } + + lease = new MessagingPumpLease(ownershipId, ++_activeGeneration); + _activeOwnershipId = ownershipId; + _activeCancellationToken = cancellationToken; + return true; + } + } + + public bool IsCurrent(MessagingPumpLease lease) + { + lock (_lock) + { + return _activeGeneration == lease.Generation + && string.Equals(_activeOwnershipId, lease.OwnershipId, StringComparison.Ordinal); + } + } + + public void Release(MessagingPumpLease lease) + { + lock (_lock) + { + if (_activeGeneration == lease.Generation + && string.Equals(_activeOwnershipId, lease.OwnershipId, StringComparison.Ordinal)) + { + _activeOwnershipId = null; + _activeCancellationToken = default; + } + } + } + + public void Reset() + { + lock (_lock) + { + _activeOwnershipId = null; + _activeCancellationToken = default; + } + } +} + +internal readonly record struct MessagingPumpLease(string OwnershipId, long Generation); diff --git a/src/Orleans.Messaging/MessagingPumpResults.cs b/src/Orleans.Messaging/MessagingPumpResults.cs new file mode 100644 index 00000000000..16a3bdbe3d0 --- /dev/null +++ b/src/Orleans.Messaging/MessagingPumpResults.cs @@ -0,0 +1,428 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal readonly record struct MessagingPumpExecutionKey(string JobName, string JobId, string RunId, long StateGeneration); + +internal readonly record struct MessagingPumpExecution(MessagingPumpExecutionKey Key, long Generation); + +internal sealed class MessagingPumpResults +{ + private const int DefaultMaxRetainedEntries = 65_536; + private static readonly TimeSpan DefaultRetentionPeriod = TimeSpan.FromMinutes(10); + + private readonly object _lock = new(); + private readonly Dictionary _entries = []; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _completedRetentionPeriod; + private readonly TimeSpan _abandonedRetentionPeriod; + private readonly TimeSpan _cleanupInterval; + private readonly int _maxRetainedEntries; + private DateTimeOffset _nextCleanup; + private long _generation; + + internal MessagingPumpResults() + : this(TimeProvider.System, DefaultRetentionPeriod, DefaultRetentionPeriod, DefaultMaxRetainedEntries) + { + } + + internal MessagingPumpResults( + TimeProvider timeProvider, + TimeSpan completedRetentionPeriod, + TimeSpan abandonedRetentionPeriod, + int maxRetainedEntries) + { + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(completedRetentionPeriod, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(abandonedRetentionPeriod, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxRetainedEntries); + + _timeProvider = timeProvider; + _completedRetentionPeriod = completedRetentionPeriod; + _abandonedRetentionPeriod = abandonedRetentionPeriod; + _maxRetainedEntries = maxRetainedEntries; + _cleanupInterval = TimeSpan.FromTicks(Math.Max( + TimeSpan.FromSeconds(1).Ticks, + Math.Min(TimeSpan.FromMinutes(1).Ticks, Math.Min(completedRetentionPeriod.Ticks, abandonedRetentionPeriod.Ticks) / 4))); + _nextCleanup = MessagingTime.AddClamped(timeProvider.GetUtcNow(), _cleanupInterval); + } + + public bool TryStart( + MessagingPumpExecutionKey key, + CancellationToken cancellationToken, + out MessagingPumpExecution execution) + { + List? removed; + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + removed = Prune(now, force: _entries.Count >= _maxRetainedEntries); + if (_entries.ContainsKey(key)) + { + execution = default; + } + else + { + if (_entries.Count >= _maxRetainedEntries) + { + MessagingPumpExecutionKey? candidateKey = null; + Entry? candidateEntry = null; + DateTimeOffset candidateTime = DateTimeOffset.MaxValue; + + foreach (var (entryKey, entryValue) in _entries) + { + if (entryValue.State != EntryState.Running) + { + var timestamp = entryValue.State == EntryState.Completed ? entryValue.CompletedAt : entryValue.CreatedAt; + if (candidateEntry is null || timestamp < candidateTime) + { + candidateKey = entryKey; + candidateEntry = entryValue; + candidateTime = timestamp; + } + } + } + + if (candidateKey is not null && _entries.Remove(candidateKey.Value)) + { + (removed ??= []).Add(candidateEntry!); + } + } + + if (_entries.Count >= _maxRetainedEntries) + { + execution = default; + } + else + { + execution = new(key, ++_generation); + _entries.Add(key, new Entry(execution.Generation, now)); + } + } + } + + DisposeRegistrations(removed); + if (execution == default) + { + return false; + } + + if (!cancellationToken.CanBeCanceled) + { + return true; + } + + var registration = cancellationToken.UnsafeRegister( + static state => + { + var cancellation = (CancellationState)state!; + cancellation.Owner.CancelWaiting(cancellation.Execution, cancellation.Token); + }, + new CancellationState(this, execution, cancellationToken)); + + var disposeRegistration = false; + lock (_lock) + { + if (_entries.TryGetValue(key, out var current) + && current.Generation == execution.Generation + && current.State == EntryState.Waiting) + { + current.CancellationRegistration = registration; + } + else + { + disposeRegistration = true; + } + } + + if (disposeRegistration) + { + registration.Dispose(); + } + + return true; + } + + public bool TryBegin(MessagingPumpExecution execution) + { + CancellationTokenRegistration registration = default; + lock (_lock) + { + if (!_entries.TryGetValue(execution.Key, out var entry) + || entry.Generation != execution.Generation + || entry.State != EntryState.Waiting) + { + return false; + } + + entry.State = EntryState.Running; + registration = entry.CancellationRegistration; + entry.CancellationRegistration = default; + } + + registration.Dispose(); + return true; + } + + public void Complete(MessagingPumpExecution execution, DurableJobRunResult result) + { + ArgumentNullException.ThrowIfNull(result); + Finish(execution, result, exception: null); + } + + public void Fail(MessagingPumpExecution execution, Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + Finish(execution, result: null, exception); + } + + public void Discard(MessagingPumpExecution execution) + { + Entry? removed = null; + lock (_lock) + { + if (_entries.TryGetValue(execution.Key, out var entry) + && entry.Generation == execution.Generation + && entry.State != EntryState.Running) + { + _entries.Remove(execution.Key); + removed = entry; + } + } + + if (removed is not null) + { + DisposeRegistration(removed); + } + } + + public void Clear(string jobName) + { + List? removed = null; + lock (_lock) + { + foreach (var pair in _entries.ToArray()) + { + if (string.Equals(pair.Key.JobName, jobName, StringComparison.Ordinal)) + { + _entries.Remove(pair.Key); + (removed ??= []).Add(pair.Value); + } + } + } + + DisposeRegistrations(removed); + } + + public bool TryTake( + MessagingPumpExecutionKey key, + out DurableJobRunResult? result, + out Exception? exception) + { + Entry? removedEntry = null; + List? pruned; + lock (_lock) + { + pruned = Prune(_timeProvider.GetUtcNow(), force: false); + if (!_entries.TryGetValue(key, out var entry) || entry.State != EntryState.Completed) + { + result = null; + exception = null; + } + else + { + _entries.Remove(key); + removedEntry = entry; + result = entry.Result; + exception = entry.Exception; + } + } + + DisposeRegistrations(pruned); + if (removedEntry is null) + { + return false; + } + + removedEntry.CancellationRegistration.Dispose(); + return true; + } + + private void Finish( + MessagingPumpExecution execution, + DurableJobRunResult? result, + Exception? exception) + { + CancellationTokenRegistration registration = default; + List? removed; + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + if (_entries.TryGetValue(execution.Key, out var entry) + && entry.Generation == execution.Generation + && entry.State != EntryState.Completed) + { + entry.Result = result; + entry.Exception = exception; + entry.State = EntryState.Completed; + entry.CompletedAt = now; + registration = entry.CancellationRegistration; + entry.CancellationRegistration = default; + } + + removed = Prune(now, force: _entries.Count > _maxRetainedEntries); + } + + registration.Dispose(); + DisposeRegistrations(removed); + } + + private void CancelWaiting(MessagingPumpExecution execution, CancellationToken cancellationToken) + { + CancellationTokenRegistration registration = default; + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + if (_entries.TryGetValue(execution.Key, out var entry) + && entry.Generation == execution.Generation + && entry.State == EntryState.Waiting) + { + entry.Exception = new OperationCanceledException(cancellationToken); + entry.State = EntryState.Completed; + entry.CompletedAt = now; + registration = entry.CancellationRegistration; + entry.CancellationRegistration = default; + } + } + + registration.Dispose(); + } + + private List? Prune(DateTimeOffset now, bool force) + { + if (!force && now < _nextCleanup) + { + return null; + } + + _nextCleanup = MessagingTime.AddClamped(now, _cleanupInterval); + List? removed = null; + foreach (var pair in _entries.ToArray()) + { + var entry = pair.Value; + var expired = entry.State switch + { + EntryState.Completed => now - entry.CompletedAt >= _completedRetentionPeriod, + EntryState.Waiting => now - entry.CreatedAt >= _abandonedRetentionPeriod, + _ => false + }; + if (expired && _entries.Remove(pair.Key)) + { + (removed ??= []).Add(entry); + } + } + + if (_entries.Count <= _maxRetainedEntries) + { + return removed; + } + + foreach (var pair in _entries + .Where(static pair => pair.Value.State != EntryState.Running) + .OrderBy(static pair => pair.Value.State == EntryState.Completed ? pair.Value.CompletedAt : pair.Value.CreatedAt) + .ToArray()) + { + if (_entries.Count <= _maxRetainedEntries) + { + break; + } + + if (_entries.Remove(pair.Key)) + { + (removed ??= []).Add(pair.Value); + } + } + + return removed; + } + + private static void DisposeRegistrations(List? entries) + { + if (entries is null) + { + return; + } + + foreach (var entry in entries) + { + DisposeRegistration(entry); + } + } + + private static void DisposeRegistration(Entry entry) + { + var registration = entry.CancellationRegistration; + entry.CancellationRegistration = default; + registration.Dispose(); + } + + private sealed class Entry(long generation, DateTimeOffset createdAt) + { + public long Generation { get; } = generation; + public DateTimeOffset CreatedAt { get; } = createdAt; + public DateTimeOffset CompletedAt { get; set; } + public EntryState State { get; set; } + public DurableJobRunResult? Result { get; set; } + public Exception? Exception { get; set; } + public CancellationTokenRegistration CancellationRegistration { get; set; } + } + + private sealed record CancellationState( + MessagingPumpResults Owner, + MessagingPumpExecution Execution, + CancellationToken Token); + + private enum EntryState + { + Waiting, + Running, + Completed + } +} + +internal sealed class OneShotTimerHandle +{ + private readonly object _lock = new(); + private IGrainTimer? _timer; + private bool _completed; + + public void Attach(IGrainTimer timer) + { + lock (_lock) + { + if (_completed) + { + timer.Dispose(); + } + else + { + _timer = timer; + } + } + } + + public void Complete() + { + lock (_lock) + { + _completed = true; + _timer?.Dispose(); + _timer = null; + } + } +} diff --git a/src/Orleans.Messaging/MessagingStateNames.cs b/src/Orleans.Messaging/MessagingStateNames.cs new file mode 100644 index 00000000000..b4a99609f08 --- /dev/null +++ b/src/Orleans.Messaging/MessagingStateNames.cs @@ -0,0 +1,22 @@ +namespace Orleans.Messaging; + +internal static class MessagingStateNames +{ + private const string Prefix = "__orleans.messaging."; + + public const string Inbox = Prefix + "inbox"; + public const string InboxProcessed = Prefix + "inbox-processed"; + public const string InboxMessageState = Prefix + "inbox-message-state"; + public const string InboxDeadLetters = Prefix + "inbox-dead-letters"; + public const string InboxJobId = Prefix + "inbox-job-id"; + public const string InboxJobHandle = Prefix + "inbox-job-handle"; + public const string InboxCompletedJobId = Prefix + "inbox-completed-job-id"; + public const string InboxJobSequence = Prefix + "inbox-job-sequence"; + public const string Outbox = Prefix + "outbox"; + public const string OutboxMessageState = Prefix + "outbox-message-state"; + public const string OutboxDeadLetters = Prefix + "outbox-dead-letters"; + public const string OutboxJobId = Prefix + "outbox-job-id"; + public const string OutboxJobHandle = Prefix + "outbox-job-handle"; + public const string OutboxCompletedJobId = Prefix + "outbox-completed-job-id"; + public const string OutboxJobSequence = Prefix + "outbox-job-sequence"; +} diff --git a/src/Orleans.Messaging/MessagingTime.cs b/src/Orleans.Messaging/MessagingTime.cs new file mode 100644 index 00000000000..c185c232926 --- /dev/null +++ b/src/Orleans.Messaging/MessagingTime.cs @@ -0,0 +1,21 @@ +using System; + +namespace Orleans.Messaging; + +internal static class MessagingTime +{ + public static bool IsExpired(DateTimeOffset now, DateTimeOffset timestamp, TimeSpan retention) + { + var nowTicks = now.UtcTicks; + var timestampTicks = timestamp.UtcTicks; + return nowTicks >= timestampTicks + && nowTicks - timestampTicks >= retention.Ticks; + } + + public static DateTimeOffset AddClamped(DateTimeOffset timestamp, TimeSpan duration) + { + var utcTicks = timestamp.UtcDateTime.Ticks; + var remainingTicks = DateTimeOffset.MaxValue.Ticks - utcTicks; + return new DateTimeOffset(utcTicks + Math.Min(duration.Ticks, remainingTicks), TimeSpan.Zero); + } +} diff --git a/src/Orleans.Messaging/MessagingTurn.cs b/src/Orleans.Messaging/MessagingTurn.cs new file mode 100644 index 00000000000..5030e326dea --- /dev/null +++ b/src/Orleans.Messaging/MessagingTurn.cs @@ -0,0 +1,75 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +// Accessed on the owning activation. Runtime admission stays non-interleaving and neutral. +// Only the timer/state is reused: every logical arm carries an immutable owner-bound payload. +internal abstract class MessagingTurn : IDisposable where T : struct +{ + private IGrainTimer? _timer; + private T? _pending; + private bool _disposed; + private long _registrationGeneration; + private CancellationToken _timerCancellation; + + protected abstract IGrainTimer RegisterTimer(long registrationGeneration); + protected abstract Task ExecuteAsync(T payload, CancellationToken cancellationToken); + protected abstract void Discard(T payload); + + public void Queue(T payload) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_pending is { } superseded) + { + _pending = null; + Discard(superseded); + } + _pending = payload; + try + { + // Register disarmed so no callback can observe a partially installed timer handle. + if (_timerCancellation.IsCancellationRequested) + { + // A disposed physical handle is never reanimated. The active callback keeps its + // original payload/token; only future arms receive a new registration generation. + _timer?.Dispose(); + _timer = null; + _timerCancellation = default; + } + _timer ??= RegisterTimer(++_registrationGeneration); + _timer.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); + } + catch + { + _pending = null; // Queue's caller still owns this failed admission and its cleanup. + throw; + } + } + + public Task RunAsync(long registrationGeneration, CancellationToken cancellationToken) + { + if (_disposed || registrationGeneration != _registrationGeneration || _pending is not { } payload) + { + return Task.CompletedTask; + } + _pending = null; + _timerCancellation = cancellationToken; + // The active payload is a value snapshot, never the mutable slot used by a subsequent arm. + return ExecuteAsync(payload, cancellationToken); + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + if (_pending is { } queued) + { + _pending = null; + Discard(queued); + } + _timer?.Dispose(); + } +} diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj index 58a2ac7ff4a..2069ef3a10b 100644 --- a/src/Orleans.Messaging/Orleans.Messaging.csproj +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -2,7 +2,7 @@ Microsoft.Orleans.Messaging Microsoft Orleans Messaging - Messaging envelope, delivery, and handler-routing contracts for Microsoft Orleans. + Opaque messaging contracts and journaled inbox processing for Microsoft Orleans. $(PackageTags) Messaging Inbox Outbox false @@ -20,6 +20,7 @@ + diff --git a/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs b/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs new file mode 100644 index 00000000000..a86d99f456b --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs @@ -0,0 +1,148 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using NSubstitute; +using Orleans.Concurrency; +using Orleans.Messaging.Tests.Support; +using Orleans.Messaging.Tests.Functional; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Placement; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ActivationValidationTests +{ + private static readonly Action ValidateActivation = ReceiverTestServices + .GetImplementationType("MessagingActivationValidator") + .GetMethod("Validate")! + .CreateDelegate>(); + + [Fact] + public void ExternalConsumerAssembly_HasNoFriendAccessToMessaging() + { + var sourceAssembly = typeof(IInbox).Assembly; + var consumerName = typeof(ActivationValidationTests).Assembly.GetName().Name; + var friendDeclarations = sourceAssembly + .GetCustomAttributesData() + .Where(attribute => attribute.AttributeType.FullName == "System.Runtime.CompilerServices.InternalsVisibleToAttribute") + .Select(attribute => attribute.ConstructorArguments[0].Value?.ToString()) + .ToArray(); + + Assert.DoesNotContain(friendDeclarations, declaration => + declaration?.StartsWith(consumerName!, StringComparison.Ordinal) == true); + } + + [Theory] + [InlineData("reentrant", "non-reentrant")] + [InlineData("stateless", "one activation")] + [InlineData("interleavable-method", "interleavable method")] + [InlineData("may-interleave", "non-reentrant")] + public void UnsupportedExecutionModel_FailsWithSpecificDiagnostic(string model, string expected) + { + object grain = model switch + { + "reentrant" => new ReentrantGrain(), + "stateless" => new StatelessGrain(), + "interleavable-method" => new InterleavableGrain(), + "may-interleave" => new MayInterleaveGrain(), + _ => throw new ArgumentOutOfRangeException(nameof(model)) + }; + var context = Substitute.For(); + context.GrainInstance.Returns(grain); + + var values = new Dictionary(); + new AttributeGrainPropertiesProvider(Substitute.For()) + .Populate(grain.GetType(), GrainType.Create("activation-validation"), values); + var properties = new GrainProperties(values.ToImmutableDictionary(StringComparer.Ordinal)); + var placement = model == "stateless" + ? new StatelessWorkerAttribute().PlacementStrategy + : new RandomPlacementAttribute().PlacementStrategy; + var exception = Assert.Throws(() => ValidateActivation(context, properties, placement)); + + Assert.Contains(expected, exception.Message, StringComparison.Ordinal); + Assert.Contains(grain.GetType().ToString(), exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void MissingGrainInstance_FailsBeforeExecutionModelValidation() + { + var context = Substitute.For(); + + var properties = new GrainProperties(ImmutableDictionary.Empty.WithComparers(StringComparer.Ordinal)); + var exception = Assert.Throws(() => + ValidateActivation(context, properties, new RandomPlacementAttribute().PlacementStrategy)); + + Assert.Contains("initialized grain instance", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task StandardStateConstruction_MissingSelectedCodecFailsBeforeEnrollment() + { + const string selectedFormat = "orleans-binary"; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(selectedFormat); + builder.Services.RemoveAllKeyed(typeof(IDurableValueCommandCodec<>), selectedFormat); + var id = new JournalId("missing-codec/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var exception = Assert.Throws(() => + scope.ServiceProvider.GetRequiredKeyedService>("state")); + Assert.Contains(selectedFormat, exception.Message, StringComparison.Ordinal); + Assert.False(owner.TryGetStateMachine("state", out _)); + } + + [Fact] + public async Task CodecResolution_PreservesUnrelatedFailure() + { + var cause = new InvalidOperationException("codec resolution failure"); + const string selectedFormat = "orleans-binary"; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(selectedFormat); + builder.Services.AddKeyedSingleton>(selectedFormat, (_, _) => throw cause); + var id = new JournalId("codec-resolution/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + + var exception = Assert.Throws(() => + scope.ServiceProvider.GetRequiredKeyedService>("state")); + + Assert.Same(cause, exception); + Assert.False(owner.TryGetStateMachine("state", out _)); + } + + [Reentrant] + private sealed class ReentrantGrain; + + [StatelessWorker] + private sealed class StatelessGrain; + + [MayInterleave(nameof(Interleave))] + private sealed class MayInterleaveGrain + { + public static bool Interleave(IInvokable request) => true; + } + + public interface IInterleavableBase + { + [AlwaysInterleave] + Task PingAsync(); + } + + public interface IInterleavableGrain : IGrain, IInterleavableBase; + + private sealed class InterleavableGrain : IInterleavableGrain + { + public Task PingAsync() => Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs index 5d6af665177..4ec4d1b583c 100644 --- a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -1,4 +1,14 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.DurableJobs; using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Functional; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Timers; using Xunit; namespace Orleans.Messaging.Tests.Contracts; @@ -8,6 +18,87 @@ namespace Orleans.Messaging.Tests.Contracts; [TestArea("Messaging")] public sealed class DeliveryAndOptionsContractTests { + [Fact] + public void RetentionTimeArithmetic_HandlesMaximumDurationsWithoutOverflow() + { + var timeType = typeof(IInbox).Assembly.GetType( + "Orleans.Messaging.MessagingTime", + throwOnError: true)!; + var isExpired = timeType.GetMethod( + "IsExpired", + BindingFlags.Static | BindingFlags.Public)!; + var addClamped = timeType.GetMethod( + "AddClamped", + BindingFlags.Static | BindingFlags.Public)!; + var timestamp = DateTimeOffset.MaxValue - TimeSpan.FromTicks(1); + + Assert.False((bool)isExpired.Invoke( + null, + [DateTimeOffset.MaxValue, timestamp, TimeSpan.MaxValue])!); + var fullDateTimeRange = TimeSpan.FromTicks( + DateTimeOffset.MaxValue.UtcTicks - DateTimeOffset.MinValue.UtcTicks); + Assert.True((bool)isExpired.Invoke( + null, + [DateTimeOffset.MaxValue, DateTimeOffset.MinValue, fullDateTimeRange])!); + Assert.False((bool)isExpired.Invoke( + null, + [DateTimeOffset.MaxValue, DateTimeOffset.MinValue, TimeSpan.MaxValue])!); + Assert.False((bool)isExpired.Invoke( + null, + [DateTimeOffset.MinValue, DateTimeOffset.MaxValue, TimeSpan.FromTicks(1)])!); + Assert.Equal( + DateTimeOffset.MaxValue, + (DateTimeOffset)addClamped.Invoke(null, [timestamp, TimeSpan.MaxValue])!); + } + + [Fact] + public void DeadLetterCompaction_SaturatesMaximumRetentionWithoutOverflow() + { + var retentionType = typeof(IInbox).Assembly.GetType( + "Orleans.Messaging.DeadLetterRetention", + throwOnError: true)!; + var compact = retentionType.GetMethod( + "Compact", + BindingFlags.Static | BindingFlags.Public)! + .MakeGenericMethod(typeof(string), typeof(DateTimeOffset)); + var entries = new Dictionary + { + ["oldest"] = DateTimeOffset.MinValue, + ["newest"] = DateTimeOffset.MaxValue + }; + + var removed = (bool)compact.Invoke( + null, + [ + entries, + DateTimeOffset.MaxValue, + TimeSpan.MaxValue, + int.MaxValue, + (Func)(static timestamp => timestamp), + 0 + ])!; + + Assert.False(removed); + Assert.Equal(2, entries.Count); + + var fullDateTimeRange = TimeSpan.FromTicks( + DateTimeOffset.MaxValue.UtcTicks - DateTimeOffset.MinValue.UtcTicks); + removed = (bool)compact.Invoke( + null, + [ + entries, + DateTimeOffset.MaxValue, + fullDateTimeRange, + int.MaxValue, + (Func)(static timestamp => timestamp), + 0 + ])!; + + Assert.True(removed); + Assert.DoesNotContain("oldest", entries); + Assert.Contains("newest", entries); + } + [Fact] public void DeliveryResult_EachFactory_PreservesStatusAndPayload() { @@ -86,6 +177,15 @@ public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContrac } } + [Theory] + [InlineData(0)] + [InlineData(100)] + [InlineData(4294967294)] + public void Validate_OutboxIdleGrace_SupportsImmediateAndMaximumTimerBoundary(long milliseconds) + { + new InboxOptions { OutboxIdleRetirementGracePeriod = TimeSpan.FromMilliseconds(milliseconds) }.Validate(); + } + [Fact] public void Validate_MaxOutboxRetryAgeNotLessThanDeduplicationWindow_FailsAtBoundaryAndAbove() { @@ -102,4 +202,105 @@ public void Validate_MaxOutboxRetryAgeNotLessThanDeduplicationWindow_FailsAtBoun Assert.Contains("less than DeduplicationWindow", exception.Message, StringComparison.Ordinal); } } + + [Fact] + public void InboxDispose_CancelsWorkOnceAndSupportsRepeatedDisposal() + { + var assembly = typeof(IInbox).Assembly; + var extensionType = assembly.GetType("Orleans.Messaging.InboxExtension", throwOnError: true)!; + var coordinatorType = assembly.GetType("Orleans.Messaging.MessagingPumpCoordinator", throwOnError: true)!; + var extension = (IDisposable)RuntimeHelpers.GetUninitializedObject(extensionType); + extensionType.GetField("_provisionalAcceptances", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(extension, new HashSet()); + var coordinator = Activator.CreateInstance(coordinatorType)!; + var results = Activator.CreateInstance(assembly.GetType("Orleans.Messaging.MessagingPumpResults", throwOnError: true)!, nonPublic: true)!; + extensionType.GetField("_pumpResults", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(extension, results); + using var shutdown = new CancellationTokenSource(); + var token = shutdown.Token; + var cancellationCount = 0; + using var registration = token.Register(() => cancellationCount++); + extensionType.GetField("_shutdownCts", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(extension, shutdown); + extensionType.GetField("_pumpCoordinator", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(extension, coordinator); + object?[] acquireArguments = ["owner", token, null]; + Assert.True((bool)coordinatorType.GetMethod("TryAcquire")!.Invoke(coordinator, acquireArguments)!); + + extension.Dispose(); + + Assert.True(token.IsCancellationRequested); + Assert.Equal(1, cancellationCount); + Assert.False((bool)coordinatorType.GetMethod("IsCurrent")!.Invoke(coordinator, [acquireArguments[2]])!); + Assert.Throws(() => shutdown.Token); + + extension.Dispose(); + + Assert.Equal(1, cancellationCount); + } + + [Fact] + public async Task InboxLifecycleStart_ObservesPreCanceledLifecycleToken() + { + var extensionType = typeof(IInbox).Assembly.GetType( + "Orleans.Messaging.InboxExtension", + throwOnError: true)!; + var extension = (ILifecycleObserver)RuntimeHelpers.GetUninitializedObject(extensionType); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + () => extension.OnStart(cancellation.Token)); + } + + [Fact] + public async Task InboxLifecycleStart_CancellationInterruptsBlockedResume() + { + var builder = InboxStateManagerBoundaryTests.CreateBuilder("orleans-binary"); + var id = new JournalId("lifecycle-start/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + builder.Services.AddScoped(sp => + { + var context = Substitute.For(); + context.GrainId.Returns(GrainId.Create("lifecycle-start", "blocked")); + context.GrainInstance.Returns(Substitute.For()); + context.ActivationServices.Returns(sp); + context.ObservableLifecycle.Returns(Substitute.For()); + return context; + }); + var jobs = Substitute.For(); + var timers = Substitute.For(); + builder.Services.AddSingleton(jobs); + builder.Services.AddSingleton(timers); + builder.Services.AddSingleton(Substitute.For()); + var instrumentsType = ReceiverTestServices.GetImplementationType("MessagingInstruments"); + builder.Services.AddSingleton(instrumentsType, + instrumentsType.GetMethod("CreateForDirectConstruction", BindingFlags.NonPublic | BindingFlags.Static)!.Invoke(null, null)!); + ReceiverTestServices.Add(builder.Services, static _ => { }); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var extensionType = ReceiverTestServices.GetImplementationType("InboxExtension"); + var extension = (ILifecycleObserver)scope.ServiceProvider.GetRequiredService(extensionType); + var owner = scope.ServiceProvider.GetRequiredService(); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + var gate = (SemaphoreSlim)extensionType.GetField("_gate", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + await gate.WaitAsync(TestContext.Current.CancellationToken); + try + { + using var cancellation = new CancellationTokenSource(); + var start = extension.OnStart(cancellation.Token); + Assert.False(start.IsCompleted); + cancellation.Cancel(); + + var canceled = await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(cancellation.Token, canceled.CancellationToken); + Assert.Equal(0, gate.CurrentCount); + Assert.Empty(jobs.ReceivedCalls()); + Assert.Empty(timers.ReceivedCalls()); + Assert.Empty(scope.ServiceProvider.GetRequiredKeyedService>( + "__orleans.messaging.inbox")); + } + finally + { + gate.Release(); + } + } } diff --git a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs index 9a254f1ebf8..c7bfed4fe54 100644 --- a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs @@ -2,6 +2,7 @@ using System.Runtime.CompilerServices; using Microsoft.Extensions.DependencyInjection; using NSubstitute; +using Orleans.Journaling; using Orleans.Runtime; using Orleans.Serialization; using Xunit; @@ -13,6 +14,61 @@ namespace Orleans.Messaging.Tests.Contracts; [TestArea("Messaging")] public sealed class HandlerRoutingContractTests { + [Fact] + public void RegisterHandler_FirstRegistration_PreservesHandlerIdentity() + { + var inbox = CreateInbox(out var storage); + Assert.Null(RegisteredHandler(inbox)); + var handler = Substitute.For(); + + inbox.RegisterHandler(handler); + + Assert.Same(handler, RegisteredHandler(inbox)); + Assert.Equal(1000, inbox.Capacity); + storage.DidNotReceiveWithAnyArgs().TryGetValue(default, out _); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterHandler_SecondRegistration_RejectsWithoutReplacingOriginal(bool sameInstance) + { + var inbox = CreateInbox(out _); + var first = Substitute.For(); + inbox.RegisterHandler(first); + var replacement = sameInstance ? first : Substitute.For(); + + var exception = Assert.Throws(() => inbox.RegisterHandler(replacement)); + + Assert.Equal("A handler is already registered for this inbox.", exception.Message); + Assert.Same(first, RegisteredHandler(inbox)); + Assert.Equal(1000, inbox.Capacity); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterHandler_NullRegistration_RejectsWithoutChangingHandler(bool alreadyRegistered) + { + var inbox = CreateInbox(out _); + var handler = Substitute.For(); + if (alreadyRegistered) inbox.RegisterHandler(handler); + + var exception = Assert.Throws(() => inbox.RegisterHandler(null!)); + + Assert.Equal("handler", exception.ParamName); + if (alreadyRegistered) + { + Assert.Same(handler, RegisteredHandler(inbox)); + } + else + { + Assert.Null(RegisteredHandler(inbox)); + inbox.RegisterHandler(handler); + Assert.Same(handler, RegisteredHandler(inbox)); + } + } + [Theory] [InlineData(1)] [InlineData(2)] @@ -86,8 +142,10 @@ public async Task HandleAsync_SynchronousHandling_PreservesContextTokenAndExplic context.Complete(); return ValueTask.CompletedTask; }); + var inbox = CreateInbox(out _); + inbox.RegisterHandler(handler); - var result = handler.HandleAsync(context, cancellation.Token); + var result = RegisteredHandler(inbox)!.HandleAsync(context, cancellation.Token); Assert.True(result.IsCompletedSuccessfully); await result; @@ -113,8 +171,10 @@ async ValueTask Handle() context.Complete(); } handler.HandleAsync(context, cancellation.Token).Returns(_ => Handle()); + var inbox = CreateInbox(out _); + inbox.RegisterHandler(handler); - var pending = handler.HandleAsync(context, cancellation.Token).AsTask(); + var pending = RegisteredHandler(inbox)!.HandleAsync(context, cancellation.Token).AsTask(); await entered.Task; Assert.False(pending.IsCompleted); Assert.Equal(0, count); @@ -134,8 +194,10 @@ public async Task HandleAsync_ReturnWithoutComplete_DoesNotInvokeCompletion() var context = CreateContext(envelope, () => count++); var handler = Substitute.For(); handler.HandleAsync(context, CancellationToken.None).Returns(ValueTask.CompletedTask); + var inbox = CreateInbox(out _); + inbox.RegisterHandler(handler); - await handler.HandleAsync(context, CancellationToken.None); + await RegisteredHandler(inbox)!.HandleAsync(context, CancellationToken.None); Assert.Equal(0, count); AssertEnvelope(envelope, context.Message.Envelope); @@ -165,6 +227,26 @@ public void ExternalConsumerAssembly_HasNoFriendAccessToMessaging() attribute => attribute.AssemblyName.Split(',')[0] == consumer); } + private static IInbox CreateInbox(out IDurableDictionary storage) + { + var type = typeof(IInbox).Assembly.GetType("Orleans.Messaging.Inbox", throwOnError: true)!; + storage = Substitute.For>(); + return Assert.IsAssignableFrom(Activator.CreateInstance(type, storage, 1000)); + } + + private static IInboxHandler? RegisteredHandler(IInbox inbox) + { + var method = inbox.GetType().GetMethod("TryGetHandler", BindingFlags.NonPublic | BindingFlags.Instance)!; + object?[] arguments = [null]; + var found = Assert.IsType(method.Invoke(inbox, arguments)); + if (!found) + { + Assert.Null(arguments[0]); + return null; + } + return Assert.IsAssignableFrom(arguments[0]); + } + [Fact] public void HandlerContext_Constructor_RequiresCompletionCallback() { diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingPumpResultsTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingPumpResultsTests.cs new file mode 100644 index 00000000000..ec9b8270d5a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingPumpResultsTests.cs @@ -0,0 +1,356 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.Time.Testing; +using Orleans.DurableJobs; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingPumpResultsTests +{ + [Fact] + public void ConcurrentStarts_SuppressDuplicateExecution() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + var starts = new bool[64]; + + Parallel.For(0, starts.Length, index => starts[index] = results.TryStart(key, out _)); + + Assert.Equal(1, starts.Count(static started => started)); + } + + [Fact] + public void CanceledWaitingExecution_BecomesTakeableAndDoesNotRun() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + using var cancellation = new CancellationTokenSource(); + + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + cancellation.Cancel(); + + Assert.False(results.HasCancellationRegistration(key)); + Assert.False(results.TryBegin(execution)); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Null(result); + Assert.IsType(exception); + } + + + [Fact] + public void AlreadyCanceledToken_CompletesBeforeRegistrationInstallWithoutRetainingRegistration() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + + Assert.False(results.HasCancellationRegistration(key)); + Assert.False(results.TryBegin(execution)); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Null(result); + Assert.IsType(exception); + } + + [Fact] + public void RunningExecution_DoesNotRetainAttemptCancellationRegistration() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + using var cancellation = new CancellationTokenSource(); + + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + Assert.True(results.HasCancellationRegistration(key)); + Assert.True(results.TryBegin(execution)); + + Assert.False(results.HasCancellationRegistration(key)); + } + + + [Fact] + public void CompletedWaitingExecution_DoesNotRetainAttemptCancellationRegistration() + { + var results = new PumpResults(); + var key = results.CreateKey("job", "id", "run"); + using var cancellation = new CancellationTokenSource(); + + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + Assert.True(results.HasCancellationRegistration(key)); + results.Complete(execution); + + Assert.False(results.HasCancellationRegistration(key)); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Fact] + public void CompletedResultWithoutSecondPoll_Expires() + { + var clock = new FakeTimeProvider(); + var results = new PumpResults(clock, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 16); + var key = results.CreateKey("job", "id", "run"); + Assert.True(results.TryStart(key, out var execution)); + Assert.True(results.TryBegin(execution)); + results.Complete(execution); + + clock.Advance(TimeSpan.FromMinutes(2)); + _ = results.TryStart(results.CreateKey("job", "other", "run"), out _); + + Assert.False(results.TryTake(key, out _, out _)); + } + + [Fact] + public void RetainedEntries_AreBounded() + { + var results = new PumpResults(new FakeTimeProvider(), TimeSpan.FromHours(1), TimeSpan.FromHours(1), 4); + + for (var index = 0; index < 100; index++) + { + Assert.True(results.TryStart(results.CreateKey("job", index.ToString(), "run"), out _)); + } + + Assert.InRange(results.Count, 0, 4); + } + + [Fact] + public void RunningExecution_IsNotExpiredOrDuplicated() + { + var clock = new FakeTimeProvider(); + var results = new PumpResults(clock, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 4); + var key = results.CreateKey("job", "id", "run"); + Assert.True(results.TryStart(key, out var execution)); + Assert.True(results.TryBegin(execution)); + + clock.Advance(TimeSpan.FromHours(1)); + + Assert.False(results.TryStart(key, out _)); + results.Complete(execution); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Fact] + public void CapacityExhaustedByRunningExecution_RejectsNewStartWithoutLosingRunningExecution() + { + var results = new PumpResults(new FakeTimeProvider(), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 1); + var runningKey = results.CreateKey("job", "running", "run"); + var rejectedKey = results.CreateKey("job", "rejected", "run"); + Assert.True(results.TryStart(runningKey, out var execution)); + Assert.True(results.TryBegin(execution)); + + Assert.False(results.TryStart(rejectedKey, out _)); + results.Complete(execution); + + Assert.True(results.TryTake(runningKey, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Fact] + public void DifferentRunId_DoesNotObserveOlderResult() + { + var results = new PumpResults(); + var firstKey = results.CreateKey("job", "id", "run-1"); + var secondKey = results.CreateKey("job", "id", "run-2"); + Assert.True(results.TryStart(firstKey, out var firstExecution)); + Assert.True(results.TryBegin(firstExecution)); + results.Complete(firstExecution); + + Assert.True(results.TryStart(secondKey, out var secondExecution)); + Assert.False(results.TryTake(secondKey, out _, out _)); + Assert.True(results.TryTake(firstKey, out var firstResult, out _)); + Assert.Same(DurableJobRunResult.Completed, firstResult); + Assert.True(results.TryBegin(secondExecution)); + } + + [Fact] + public void DifferentStateGeneration_DoesNotObservePreRecoveryResult() + { + var results = new PumpResults(); + var beforeRecovery = results.CreateKey("job", "id", "run", stateGeneration: 1); + var afterRecovery = results.CreateKey("job", "id", "run", stateGeneration: 2); + Assert.True(results.TryStart(beforeRecovery, out var execution)); + Assert.True(results.TryBegin(execution)); + results.Complete(execution); + + Assert.False(results.TryTake(afterRecovery, out _, out _)); + Assert.True(results.TryStart(afterRecovery, out _)); + Assert.True(results.TryTake(beforeRecovery, out var result, out _)); + Assert.Same(DurableJobRunResult.Completed, result); + } + + [Fact] + public void DiscardWaitingExecution_ReleasesRegistrationAndCapacity() + { + var results = new PumpResults(new FakeTimeProvider(), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), 1); + var key = results.CreateKey("inbox", "id", "run"); + using var cancellation = new CancellationTokenSource(); + Assert.True(results.TryStartWithCancellation(key, out var execution, cancellation.Token)); + var entry = results.GetEntry(key); + Assert.True(results.HasCancellationRegistration(key)); + + results.Discard(execution); + + Assert.Equal(0, results.Count); + Assert.Equal(default, PumpResults.GetRegistration(entry)); + Assert.False(results.TryBegin(execution)); + cancellation.Cancel(); + Assert.False(results.TryTake(key, out _, out _)); + Assert.True(results.TryStart(results.CreateKey("inbox", "other", "run"), out var next)); + Assert.True(results.TryBegin(next)); + } + + [Fact] + public void DiscardOldExecution_PreservesNewGenerationWithSameKey() + { + var results = new PumpResults(); + var key = results.CreateKey("inbox", "id", "run"); + Assert.True(results.TryStart(key, out var previous)); + results.Discard(previous); + using var cancellation = new CancellationTokenSource(); + Assert.True(results.TryStartWithCancellation(key, out var current, cancellation.Token)); + + results.Discard(previous); + + Assert.Equal(1, results.Count); + Assert.True(results.HasCancellationRegistration(key)); + Assert.True(results.TryBegin(current)); + } + + [Fact] + public void DiscardRunningExecution_PreservesInFlightResult() + { + var results = new PumpResults(); + var key = results.CreateKey("inbox", "id", "run"); + Assert.True(results.TryStart(key, out var execution)); + Assert.True(results.TryBegin(execution)); + + results.Discard(execution); + + Assert.Equal(1, results.Count); + Assert.False(results.TryStart(key, out _)); + results.Complete(execution); + Assert.True(results.TryTake(key, out var result, out var exception)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Null(exception); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ClearJob_ReleasesOnlyThatFeaturesEntriesAndFencesLateCompletion(bool running) + { + var results = new PumpResults(); + var inbox = results.CreateKey("inbox", "id", "run"); + var outbox = results.CreateKey("outbox", "id", "run"); + using var cancellation = new CancellationTokenSource(); + Assert.True(results.TryStartWithCancellation(inbox, out var oldInbox, cancellation.Token)); + var removed = results.GetEntry(inbox); + if (running) + { + Assert.True(results.TryBegin(oldInbox)); + } + Assert.True(results.TryStartWithCancellation(outbox, out var liveOutbox, cancellation.Token)); + + results.Clear("inbox"); + + Assert.Equal(1, results.Count); + Assert.Equal(default, PumpResults.GetRegistration(removed)); + Assert.True(results.HasCancellationRegistration(outbox)); + Assert.False(results.TryBegin(oldInbox)); + Assert.True(results.TryStart(inbox, out var newInbox)); + results.Complete(oldInbox); + Assert.False(results.TryTake(inbox, out _, out _)); + Assert.True(results.TryBegin(newInbox)); + Assert.True(results.TryBegin(liveOutbox)); + } + + private sealed class PumpResults + { + private static readonly Assembly Assembly = typeof(IOutbox).Assembly; + private static readonly Type ResultsType = Assembly.GetType("Orleans.Messaging.MessagingPumpResults", throwOnError: true)!; + private static readonly Type KeyType = Assembly.GetType("Orleans.Messaging.MessagingPumpExecutionKey", throwOnError: true)!; + private readonly object _instance; + + public PumpResults() + { + _instance = Activator.CreateInstance(ResultsType, nonPublic: true)!; + } + + public PumpResults(TimeProvider timeProvider, TimeSpan completedRetention, TimeSpan abandonedRetention, int maxEntries) + { + _instance = Activator.CreateInstance( + ResultsType, + BindingFlags.Instance | BindingFlags.NonPublic, + binder: null, + [timeProvider, completedRetention, abandonedRetention, maxEntries], + culture: null)!; + } + + public int Count => ((IDictionary)ResultsType + .GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_instance)!).Count; + + public object GetEntry(object key) => ((IDictionary)ResultsType + .GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_instance)!)[key]!; + + public static CancellationTokenRegistration GetRegistration(object entry) => + (CancellationTokenRegistration)entry.GetType().GetProperty("CancellationRegistration")!.GetValue(entry)!; + + public void Discard(object execution) => ResultsType.GetMethod("Discard")!.Invoke(_instance, [execution]); + public void Clear(string jobName) => ResultsType.GetMethod("Clear")!.Invoke(_instance, [jobName]); + + public bool HasCancellationRegistration(object key) + { + var entries = (IDictionary)ResultsType + .GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_instance)!; + var entry = entries[key]!; + var registration = (CancellationTokenRegistration)entry.GetType() + .GetProperty("CancellationRegistration")! + .GetValue(entry)!; + return registration != default; + } + + public object CreateKey(string jobName, string jobId, string runId, long stateGeneration = 0) => + Activator.CreateInstance(KeyType, [jobName, jobId, runId, stateGeneration])!; + + public bool TryStart(object key, out object execution) => + TryStartWithCancellation(key, out execution, TestContext.Current.CancellationToken); + + public bool TryStartWithCancellation( + object key, + out object execution, + CancellationToken cancellationToken) + { + object?[] arguments = [key, cancellationToken, null]; + var result = (bool)ResultsType.GetMethod("TryStart")!.Invoke(_instance, arguments)!; + execution = arguments[2]!; + return result; + } + + public bool TryBegin(object execution) => + (bool)ResultsType.GetMethod("TryBegin")!.Invoke(_instance, [execution])!; + + public void Complete(object execution) => + ResultsType.GetMethod("Complete")!.Invoke(_instance, [execution, DurableJobRunResult.Completed]); + + public bool TryTake(object key, out DurableJobRunResult? result, out Exception? exception) + { + object?[] arguments = [key, null, null]; + var taken = (bool)ResultsType.GetMethod("TryTake")!.Invoke(_instance, arguments)!; + result = arguments[1] as DurableJobRunResult; + exception = arguments[2] as Exception; + return taken; + } + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs b/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs new file mode 100644 index 00000000000..1b4de3c3296 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs @@ -0,0 +1,105 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Messaging.Tests.Functional; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class StandardJournaledStateTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Dictionary_WriteAcknowledgementPreservesLaterMutations(bool snapshot) + { + var id = new JournalId("standard-dictionary/" + Guid.NewGuid().ToString("N")); + var storage = new ControlledJournalStorageProvider(); + await using var provider = CreateProvider(storage, id); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var items = scope.ServiceProvider.GetRequiredKeyedService>("state"); + Assert.False(owner is IDurableStateManager); + Assert.True(owner.TryGetStateMachine("state", out var registered)); + Assert.Same(items, registered); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + items.Add("first", 1); + if (snapshot) storage.RequestSnapshot(id); + using var blocked = storage.BlockWrite(id); + var first = owner.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await blocked.WaitUntilEnteredAsync(); + items["later"] = 2; + Assert.False(first.IsCompleted); + blocked.Release(); + await first; + await using (var replay = provider.CreateAsyncScope()) + { + var recovered = replay.ServiceProvider.GetRequiredService(); + var persisted = replay.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(new KeyValuePair("first", 1), Assert.Single(persisted)); + } + Assert.Equal(2, items["later"]); + await owner.WriteStateAsync(TestContext.Current.CancellationToken); + await using var finalScope = provider.CreateAsyncScope(); + var finalOwner = finalScope.ServiceProvider.GetRequiredService(); + var final = finalScope.ServiceProvider.GetRequiredKeyedService>("state"); + await finalOwner.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(new[] { new KeyValuePair("first", 1), new KeyValuePair("later", 2) }, final.OrderBy(pair => pair.Key)); + Assert.Equal(2, storage.GetSuccessfulWriteCount(id)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Value_WriteAcknowledgementPreservesNewerPendingValue(bool snapshot) + { + var id = new JournalId("standard-value/" + Guid.NewGuid().ToString("N")); + var storage = new ControlledJournalStorageProvider(); + await using var provider = CreateProvider(storage, id); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var value = scope.ServiceProvider.GetRequiredKeyedService>("state"); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + value.Value = 1; + if (snapshot) storage.RequestSnapshot(id); + using var blocked = storage.BlockWrite(id); + var first = owner.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await blocked.WaitUntilEnteredAsync(); + value.Value = 2; + blocked.Release(); + await first; + await using (var replay = provider.CreateAsyncScope()) + { + var recovered = replay.ServiceProvider.GetRequiredService(); + var persisted = replay.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, persisted.Value); + } + Assert.Equal(2, value.Value); + await owner.WriteStateAsync(TestContext.Current.CancellationToken); + await using var finalScope = provider.CreateAsyncScope(); + var finalOwner = finalScope.ServiceProvider.GetRequiredService(); + var final = finalScope.ServiceProvider.GetRequiredKeyedService>("state"); + await finalOwner.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, final.Value); + Assert.Equal(2, storage.GetSuccessfulWriteCount(id)); + } + + private static ServiceProvider CreateProvider(ControlledJournalStorageProvider storage, JournalId id) + { + var builder = InboxStateManagerBoundaryTests.CreateBuilder("orleans-binary"); + builder.Services.AddSingleton(sp => + { + storage.Configure(sp.GetRequiredService>()); + return storage; + }); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + return builder.Services.BuildServiceProvider(validateScopes: true); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs new file mode 100644 index 00000000000..e9151a1a184 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs @@ -0,0 +1,150 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ByteMessagingPayloadTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task DirectAdmission_CallerCancellationLeavesActualAcceptanceAndHandlerPayloadAvailable() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + var envelope = CreateEnvelope(receiver, NewMessage(300, "gc-admission")); + var publishedPayload = envelope.Envelope.Payload; + var expected = envelope.Envelope.Payload.ToArray(); + using var handler = new PayloadProbe(expected); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + using var scheduling = Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName); + using var cancellation = new CancellationTokenSource(); + Task waiting = null!; + await OnTurnAsync(context, () => waiting = extension.DeliverAsync(envelope, cancellation.Token).AsTask()); + await scheduling.WaitUntilEnteredAsync(); + var actual = (Task)extension.GetType().GetField("_activeDelivery", + BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + envelope = default; + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => waiting); + Assert.False(actual.IsCompleted); + scheduling.Continue(); + await actual.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(publishedPayload, handler.Payload); + Assert.Equal(expected, handler.Payload.ToArray()); + handler.Release.TrySetResult(); + await handler.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.InboxCount == 0 && snapshot.ProcessedMessageCount == 1); + Assert.Empty(completed.InboxDeadLetters); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, handler.Payload.ToArray()); + } + + [Fact] + public async Task RpcAcceptanceAndDuplicate_CopyPayloadAndPreserveCallerBytesAcrossReplay() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(302, "rpc-bytes")); + var expected = envelope.Envelope.Payload.ToArray(); + using var handler = new PayloadProbe(expected); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.True(MemoryMarshal.TryGetArray(envelope.Envelope.Payload, out var caller)); + Assert.True(MemoryMarshal.TryGetArray(handler.Payload, out var received)); + Assert.NotSame(caller.Array, received.Array); + caller.Array![caller.Offset] ^= 0xff; + Assert.Equal(expected, handler.Payload.ToArray()); + caller.Array[caller.Offset] ^= 0xff; + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + handler.Release.TrySetResult(); + await handler.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.InboxCount == 0 && snapshot.ProcessedMessageCount == 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task RpcRejection_PreservesCallerBytesWithoutStagingState() + { + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + var original = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await original.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(301, "rejected-bytes")); + var expected = envelope.Envelope.Payload.ToArray(); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(receiver, envelope)).Status); + var rejected = await receiver.GetSnapshotAsync(); + Assert.Equal(0, rejected.InboxCount); + Assert.Equal(0, rejected.ProcessedMessageCount); + Assert.Empty(rejected.Effects); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completed.SetResult(); } + catch (Exception exception) { completed.SetException(exception); } + }); + return completed.Task; + } + + private sealed class PayloadProbe(byte[] expected) : IInboxHandler, IDisposable + { + public ReadOnlyMemory Payload { get; private set; } + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + try + { + Payload = context.Message.Envelope.Payload; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + Assert.Equal(expected, Payload.ToArray()); + context.Complete(); + Assert.Equal(Payload, context.Message.Envelope.Payload); + Assert.Equal(expected, context.Message.Envelope.Payload.ToArray()); + Completed.TrySetResult(); + } + catch (Exception exception) + { + Completed.TrySetException(exception); + throw; + } + } + + public void Dispose() => Release.TrySetResult(); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs new file mode 100644 index 00000000000..c7ebf76f74d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs @@ -0,0 +1,106 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DedupeExpiryClusterCollection : ICollectionFixture +{ + public const string Name = "Messaging dedupe expiry cluster"; +} + +[Collection(DedupeExpiryClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DedupeExpiryBehaviorTests(DedupeExpiryClusterFixture fixture) +{ + [Fact] + public async Task IdleReplay_AtDeduplicationBoundary_IsAcceptedWithoutCompactionTrigger() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var original = new TestMessage(TestApplicationProtocol.NewMessageId(), 15, "expires"); + var first = CreateEnvelope(receiver, original); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, first)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + + fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + var state = await fixture.WaitForEffectCountAsync(receiver, 2); + + Assert.Equal(2, state.Effects.Single(effect => effect.LogicalId == original.LogicalId).Count); + } + + [Fact] + public async Task ExpiryReplacementFailure_ReplaysDedupeRecordBeforeExpiryMaintenance() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), 16, "failed-expiry-replacement")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + var oldContext = fixture.GetGrainContext(receiver); + fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await Assert.ThrowsAnyAsync(() => DeliverAsync(receiver, envelope)); + + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var failed = await receiver.GetSnapshotAsync(); + Assert.Equal(0, failed.InboxCount); + Assert.Equal(1, Assert.Single(failed.Effects).Count); + var replayed = Assert.IsType(fixture.GetGrainContext(receiver).GrainInstance).ReplayedSnapshot; + Assert.Equal(1, Assert.IsType(replayed).ProcessedMessageCount); + Assert.Equal(0, failed.ProcessedMessageCount); + + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(failed.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(0, recovered.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var retried = await fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(2, Assert.Single(retried.Effects).Count); + } + + private Task WaitForIdleInboxAsync(IMessagingTestGrain receiver) => + fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && string.IsNullOrEmpty(snapshot.InboxJobId)); + + private static async Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + await receiver.AsReference().DeliverAsync(envelope); + + private InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + TestMessage message, + HierarchicalKey? messageId = null) + { + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("expiry-test-sender", "stable"); + var built = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/expiry", message); + if (messageId is { } id) + { + built = TestApplicationProtocol.With(built, sessions, messageId: id); + } + + return built; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/DeliveryCancellationTests.cs b/test/Orleans.Messaging.Tests/Functional/DeliveryCancellationTests.cs new file mode 100644 index 00000000000..021791b4d19 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/DeliveryCancellationTests.cs @@ -0,0 +1,227 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DeliveryCancellationTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false, "schedule")] + [InlineData(true, "schedule")] + [InlineData(false, "acknowledgement")] + [InlineData(true, "acknowledgement")] + [InlineData(false, "storage")] + [InlineData(true, "storage")] + public async Task CallerCancellation_RetainsOwnedDeliveryUntilCommit(bool proxy, string phase) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var extension = GetExtension(context); + using var blocked = Block(receiver, phase); + var envelope = CreateEnvelope(receiver, NewMessage(150, phase)); + using var cancellation = new CancellationTokenSource(); + var delivery = StartDelivery(proxy, receiver, context, extension, envelope, cancellation.Token); + await blocked.Entered(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => delivery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + + Assert.Equal(0, GetGate(extension).CurrentCount); + Assert.Single(GetPendingOwners(extension)); + var duplicate = StartDelivery(false, receiver, context, extension, envelope, TestContext.Current.CancellationToken); + await OnTurnAsync(context, static () => { }); + Assert.False(duplicate.IsCompleted); + Assert.Equal(1, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + blocked.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(3, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.Equal(1, GetGate(extension).CurrentCount); + Assert.Empty(GetPendingOwners(extension)); + Assert.False(((MessagingTestGrain)context.GrainInstance!).DeactivationFailure.Task.IsCompleted); + } + + [Theory] + [InlineData(false, "acknowledgement")] + [InlineData(true, "acknowledgement")] + [InlineData(false, "storage")] + [InlineData(true, "storage")] + public async Task CanceledCaller_LateFailureIsLoggedAndReleasesOwnership(bool proxy, string phase) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = GetExtension(context); + var envelope = CreateEnvelope(receiver, NewMessage(151, phase)); + using var logs = new DeliveryLogProbe(envelope.Envelope.MessageId); + Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService().AddProvider(logs); + using var blocked = Block(receiver, phase); + using var cancellation = new CancellationTokenSource(); + var delivery = StartDelivery(proxy, receiver, context, extension, envelope, cancellation.Token); + await blocked.Entered(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Equal(0, GetGate(extension).CurrentCount); + Assert.Single(GetPendingOwners(extension)); + + Assert.IsType(blocked.Fail)(); + var failure = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var logged = await logs.Failure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + Assert.IsType(failure); + Assert.Same(failure, logged); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(1, GetGate(extension).CurrentCount); + Assert.Empty(GetPendingOwners(extension)); + Assert.Empty(grain.GetSnapshotForTest().Effects); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + if (phase == "acknowledgement") + { + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + else + { + Assert.Equal(0, recovered.InboxCount); + Assert.Null(recovered.InboxJobId); + Assert.Empty(recovered.Effects); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeactivationAfterCallerCancellation_DrainsOwnedDelivery(bool proxy) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = GetExtension(context); + var shutdown = (CancellationTokenSource)extension.GetType().GetField("_shutdownCts", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + var stopping = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = shutdown.Token.Register(() => stopping.TrySetResult()); + using var blocked = Block(receiver, "storage"); + var envelope = CreateEnvelope(receiver, NewMessage(152, "deactivation")); + using var cancellation = new CancellationTokenSource(); + var delivery = StartDelivery(proxy, receiver, context, extension, envelope, cancellation.Token); + await blocked.Entered(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + + _ = await receiver.GetSnapshotAsync().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Drain canceled delivery waiter."), TestContext.Current.CancellationToken); + await stopping.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.False(context.Deactivated.IsCompleted); + Assert.Equal(0, GetGate(extension).CurrentCount); + Assert.Single(GetPendingOwners(extension)); + Assert.True(shutdown.Token.CanBeCanceled); + blocked.Release(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(1, GetGate(extension).CurrentCount); + Assert.Empty(GetPendingOwners(extension)); + Assert.Throws(() => shutdown.Token); + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + private BlockedPhase Block(IMessagingTestGrain receiver, string phase) + { + switch (phase) + { + case "schedule": + var schedule = Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName); + return new(schedule.WaitUntilEnteredAsync, schedule.Continue, null); + case "acknowledgement": + var acknowledgement = Fixture.Storage.BlockAcknowledgement(JournalId.FromGrainId(receiver.GetGrainId())); + return new(acknowledgement.WaitUntilEnteredAsync, acknowledgement.Release, acknowledgement.Fail); + case "storage": + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + return new(storage.WaitUntilEnteredAsync, storage.Release, storage.Fail); + default: + throw new ArgumentOutOfRangeException(nameof(phase)); + } + } + + private static Task StartDelivery(bool proxy, IMessagingTestGrain receiver, IGrainContext context, + IInboxExtension extension, InboxMessage envelope, CancellationToken cancellationToken) + { + if (proxy) + { + return DeliverWithCancellationAsync(receiver, envelope, cancellationToken); + } + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(envelope, cancellationToken).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private static IInboxExtension GetExtension(IGrainContext context) => + (IInboxExtension)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + private static SemaphoreSlim GetGate(IInboxExtension extension) => + (SemaphoreSlim)extension.GetType().GetField("_gate", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + private static HashSet GetPendingOwners(IInboxExtension extension) => + (HashSet)extension.GetType().GetField("_pendingOwnershipIds", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completed.SetResult(); } + catch (Exception exception) { completed.SetException(exception); } + }); + return completed.Task; + } + + private sealed record BlockedPhase(Func Entered, Action Release, Action? Fail) : IDisposable + { + public void Dispose() => Release(); + } + + private sealed class DeliveryLogProbe(HierarchicalKey messageId) : ILoggerProvider + { + public TaskCompletionSource Failure { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public ILogger CreateLogger(string categoryName) => new Logger(this, categoryName, messageId.ToString()); + public void Dispose() { } + private sealed class Logger(DeliveryLogProbe owner, string category, string messageId) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel level) => level >= LogLevel.Error; + public void Log(LogLevel level, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (exception is not null && category == "Orleans.Messaging.InboxExtension" + && eventId.Name == "DeliveryOperationFailed" && formatter(state, exception).Contains(messageId, StringComparison.Ordinal)) + { + owner.Failure.TrySetResult(exception); + } + } + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs new file mode 100644 index 00000000000..174ac17952a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs @@ -0,0 +1,91 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Messaging.Tests.Support; +using Orleans.Metadata; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class EarlyActivationValidationTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(typeof(ReentrantActivationValidationTestGrain), "non-reentrant")] + [InlineData(typeof(StatelessActivationValidationTestGrain), "one activation")] + [InlineData(typeof(MayInterleaveActivationValidationTestGrain), "non-reentrant")] + [InlineData(typeof(AlwaysInterleaveActivationValidationTestGrain), "interleavable method")] + public async Task UnsupportedModel_FailsBeforeJournalInitializationReplayAndWork(Type grainType, string diagnostic) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + await AssertRejectedBeforeRecoveryAsync(grain, grainType, diagnostic); + } + + [Theory] + [InlineData(typeof(MetadataReentrantActivationValidationGrain), WellKnownGrainTypeProperties.Reentrant, "TrUe")] + [InlineData(typeof(MetadataMayInterleaveActivationValidationGrain), WellKnownGrainTypeProperties.MayInterleavePredicate, "Interleave")] + public async Task ResolvedInterleavingMetadata_FailsBeforeJournalRecovery(Type grainType, string key, string value) + { + Assert.False(grainType.IsDefined(typeof(ReentrantAttribute), inherit: true)); + Assert.False(grainType.IsDefined(typeof(MayInterleaveAttribute), inherit: true)); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + var properties = Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService() + .GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(value, properties.Properties[key]); + await AssertRejectedBeforeRecoveryAsync(grain, grainType, "non-reentrant"); + } + + private async Task AssertRejectedBeforeRecoveryAsync(IActivationValidationTestGrain grain, Type grainType, string diagnostic) + { + var failure = await Assert.ThrowsAnyAsync(() => grain.IncrementAsync()); + Assert.Contains(diagnostic, failure.ToString(), StringComparison.Ordinal); + Assert.Contains(grainType.Name, failure.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Fixture.ActivationProbe.Get(grain.GetGrainId())); + Assert.False(observation.InstanceAvailableDuringConstruction); + Assert.Equal(0, observation.Activated); + Assert.Equal(0, observation.Calls); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + Assert.Equal(0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, grain.GetGrainId())); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount("orleans.messaging.outbox-drain", grain.GetGrainId())); + } + + [Fact] + public async Task SupportedModel_InitializesAndReplaysFreshActivationNormally() + { + await AssertSupportedReplayAsync(typeof(SupportedActivationValidationTestGrain)); + } + + [Fact] + public async Task ResolvedNonReentrantMetadata_InitializesAndReplaysNormally() + { + await AssertSupportedReplayAsync(typeof(MetadataNonReentrantActivationValidationGrain)); + } + + private async Task AssertSupportedReplayAsync(Type grainType) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + Assert.Equal(1, await grain.IncrementAsync()); + var first = Assert.Single(Fixture.ActivationProbe.Get(grain.GetGrainId())); + Assert.False(first.InstanceAvailableDuringConstruction); + Assert.NotNull(first.Context.GrainInstance); + Assert.Equal(1, first.Activated); + Assert.Equal(1, first.Calls); + Assert.Equal(1, Fixture.Storage.GetReadCount(JournalId.FromGrainId(grain.GetGrainId()))); + await grain.DeactivateAsync(); + await first.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(2, await grain.IncrementAsync()); + var observations = Fixture.ActivationProbe.Get(grain.GetGrainId()); + Assert.Equal(2, observations.Length); + Assert.NotSame(first.Context, observations[1].Context); + Assert.Equal(1, observations[1].Activated); + Assert.Equal(1, observations[1].Calls); + Assert.Equal(2, Fixture.Storage.GetReadCount(JournalId.FromGrainId(grain.GetGrainId()))); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs new file mode 100644 index 00000000000..cc271252d8e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs @@ -0,0 +1,234 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxAcceptanceBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task Deliver_AcceptedOnlyAfterInboxAndStableJobOwnershipAreDurable() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var barrier = Fixture.Storage.BlockWrite(journalId); + var envelope = CreateEnvelope(receiver, NewMessage(1, "durability")); + + var delivery = DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + + Assert.False(delivery.IsCompleted); + var staged = Fixture.GetSnapshot(receiver); + Assert.Equal(1, staged.InboxCount); + Assert.Empty(staged.Effects); + var scheduledJob = Assert.Single( + Fixture.JobManagerProbe.GetScheduledJobs("orleans.messaging.inbox-drain", receiver.GetGrainId())); + Assert.Equal(staged.InboxJobId, scheduledJob.Metadata!["orleans.messaging.ownership-id"]); + Assert.Same(scheduledJob, staged.InboxJob); + Assert.Equal(scheduledJob.Id, staged.InboxJob?.Id); + Assert.Equal(scheduledJob.ShardId, staged.InboxJob?.ShardId); + + barrier.Release(); + var result = await delivery; + Assert.Equal(DeliveryStatus.Accepted, result.Status); + EndpointSnapshot completed; + try + { + completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Accepted message did not drain. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}, job={snapshot.InboxJobId}.", + exception); + } + Assert.Single(completed.Effects); + Assert.Equal(0, completed.InboxCount); + completed = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxJobId is null && snapshot.InboxJob is null); + Assert.Null(completed.InboxJob); + Assert.Equal(3, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + } + + [Fact] + public async Task Deliver_CancellationStopsWaitingForInboxGate() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var barrier = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var firstEnvelope = CreateEnvelope(receiver, NewMessage(72, "holds-gate")); + var secondEnvelope = CreateEnvelope(receiver, NewMessage(73, "canceled")); + var firstDelivery = DeliverAsync(receiver, firstEnvelope); + await barrier.WaitUntilEnteredAsync(); + using var cancellation = new CancellationTokenSource(); + + var canceledDelivery = DeliverWithCancellationAsync( + receiver, + secondEnvelope, + cancellation.Token); + Assert.False(canceledDelivery.IsCompleted); + cancellation.Cancel(); + + try + { + await Assert.ThrowsAnyAsync(() => canceledDelivery); + } + finally + { + barrier.Release(); + } + + Assert.Equal(DeliveryStatus.Accepted, (await firstDelivery).Status); + } + + [Fact] + public async Task ConcurrentWriteDuringLocalAcceptancePreparation_PreservesSafeStagingOnly() + { + var receiver = NewGrain(); + var effect = new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 74, "prior-safe-state"); + await receiver.StageEffectAsync(effect); + using var schedule = Fixture.JobManagerProbe.BlockNext("orleans.messaging.inbox-drain"); + var envelope = CreateEnvelope(receiver, NewMessage(75, "schedule-barrier")); + var delivery = DeliverAsync(receiver, envelope); + await schedule.WaitUntilEnteredAsync(); + var staged = Fixture.GetSnapshot(receiver); + Assert.Equal(0, staged.InboxCount); + Assert.Null(staged.InboxJobId); + Assert.Null(staged.InboxJob); + await Fixture.WriteStateAsync(receiver); + Assert.Equal(effect, Assert.Single(Fixture.GetSnapshot(receiver).Effects)); + Assert.False(delivery.IsCompleted); + schedule.Continue(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(new[] { "prior-safe-state", "schedule-barrier" }, completed.Effects.Select(static e => e.Value)); + } + + [Fact] + public async Task ConcurrentDuplicateDeliveries_ConvergeToOneEffectWithinRetention() + { + var receiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/blocked-duplicate"); + var envelope = CreateEnvelope(receiver, NewMessage(11, "duplicate"), "messages/blocked-duplicate"); + + var first = await DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + var second = DeliverAsync(receiver, envelope); + + Assert.Equal(DeliveryStatus.Accepted, first.Status); + Assert.False(second.IsCompleted); + Assert.Equal(1, Fixture.GetSnapshot(receiver).InboxCount); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await second).Status); + var state = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(state.Effects).Count); + Assert.Equal(1, state.MaxConcurrentHandlers); + } + + [Fact] + public async Task DuplicateAfterReactivationWithinRetention_RemainsEffectivelyOnce() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(13, "reactivation")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var before = await Fixture.WaitForEffectCountAsync(receiver, 1); + await receiver.RequestDeactivationAsync(); + var after = await receiver.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, after.ActivationId); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task SameMessageIdFromDistinctSenders_DeduplicatesOneReceiverCommand() + { + var receiver = NewGrain(); + var original = CreateEnvelope(receiver, NewMessage(14, "sender-scoped")); + var other = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("other-sender", "same-message-id")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, other)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, original)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, other)).Status); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task ReorderedDistinctAndDuplicateMessages_ConvergeByApplicationSequence() + { + var receiver = NewGrain(); + var messages = new[] + { + NewMessage(3, "third"), + NewMessage(1, "first"), + NewMessage(2, "second"), + }; + + foreach (var message in messages) + { + var envelope = CreateEnvelope(receiver, message); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + var state = await Fixture.WaitForEffectCountAsync(receiver, 3); + Assert.Equal([1, 2, 3], state.Effects.Select(static effect => effect.Sequence)); + Assert.All(state.Effects, static effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task ConcurrentDelivery_WaitsWhileHandlersRemainSequential() + { + var receiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/sequential"); + var first = CreateEnvelope(receiver, NewMessage(21, "first"), "messages/sequential"); + var second = CreateEnvelope(receiver, NewMessage(22, "second"), "messages/sequential"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await WaitForBarrierAsync(receiver, barrier); + var secondDelivery = DeliverAsync(receiver, second); + Assert.False(secondDelivery.IsCompleted); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await secondDelivery).Status); + EndpointSnapshot state; + try + { + state = await Fixture.WaitForEffectCountAsync(receiver, 2); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Second message did not complete. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + Assert.Equal(1, state.MaxConcurrentHandlers); + Assert.Equal([21, 22], state.Effects.Select(static effect => effect.Sequence)); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs new file mode 100644 index 00000000000..1f9aa7e260e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs @@ -0,0 +1,372 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxAdmissionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task ExplicitPreparation_AllowsPriorStateWriteThenCapturesAppliedCohortAtomically() + { + using var attempt = await PrepareAttemptAsync("atomic-admission"); + var state = attempt.Grain.GetSnapshotForTest(); + Assert.Equal(1, state.InboxCount); + Assert.Equal(0, state.ProcessedMessageCount); + Assert.Empty(state.Effects); + Assert.Empty(attempt.Outbox); + await OnTurnAsync(attempt.Context, () => + attempt.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "previous-state"); + await attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var priorCapture = attempt.Grain.Captures[^1]; + Assert.Empty(priorCapture.Effects); + Assert.Equal(1, priorCapture.InboxCount); + Assert.Equal(0, priorCapture.ProcessedMessageCount); + Assert.Equal(0, priorCapture.OutboxCount); + Assert.False(attempt.Grain.ApplyAttempted.Task.IsCompleted); + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + using var storage = Fixture.Storage.BlockWrite(attempt.JournalId); + attempt.Preparation.Release(); + await storage.WaitUntilEnteredAsync(); + var queued = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var coalesced = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(queued.IsCompleted); + Assert.False(coalesced.IsCompleted); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + + var captured = attempt.Grain.GetSnapshotForTest(); + Assert.Equal(0, captured.InboxCount); + Assert.Equal(1, captured.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(captured.Effects).Count); + Assert.Single(attempt.Grain.OutputCaptures[^1]); + Assert.False(queued.IsCompleted); + storage.Release(); + await Task.WhenAll(queued, coalesced); + await attempt.Receiver.RequestDeactivationAsync(); + var recovered = await attempt.Receiver.GetSnapshotAsync(); + Assert.NotEqual(state.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task HandlerStorageFailure_DeactivatesAndFailsQueuedWritesAfterPriorAcknowledgement() + { + using var attempt = await PrepareAttemptAsync("fault-ordering"); + var extension = attempt.Context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + await OnTurnAsync(attempt.Context, () => + { + attempt.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "prior-captured"; + }); + using var storage = Fixture.Storage.BlockWrite(attempt.JournalId); + var preceding = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(attempt.JournalId); + var queued = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + attempt.Preparation.Release(); + await attempt.Grain.ApplyAttempted.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await OnTurnAsync(attempt.Context, () => + { + Assert.Equal(1, attempt.Outbox.SendCalls); + Assert.Single(attempt.Grain.GetSnapshotForTest().Effects); + }); + Assert.False(queued.IsCompleted); + storage.Release(); + await preceding; + + var failure = await Assert.ThrowsAsync(() => queued); + Assert.Same(failure, await attempt.Grain.DeactivationFailure.Task); + await attempt.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Same(failure, await Assert.ThrowsAsync(async () => + await ((IInboxExtension)extension).DeliverAsync(attempt.Envelope, TestContext.Current.CancellationToken))); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + var failed = attempt.Grain.GetSnapshotForTest(); + Assert.Single(failed.Effects); + Assert.Equal(0, failed.InboxCount); + Assert.Equal(1, failed.ProcessedMessageCount); + _ = await attempt.Receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(attempt.Receiver, 1); + Assert.NotEqual(failed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task PhysicalCallbackMismatch_DuringPreparationPreservesCommittedOwner() + { + using var attempt = await PrepareAttemptAsync("owner-invalidation"); + var value = attempt.Context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-job-handle"); + var previous = value.Value!; + var mismatched = new DurableJob + { + Id = previous.Id, + ShardId = "different-physical-shard", + Metadata = previous.Metadata, + Name = previous.Name, + TargetGrainId = previous.TargetGrainId, + DueTime = previous.DueTime + }; + var feature = (IDurableJobFeatureHandler)attempt.Context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + DurableJobRunResult result = null!; + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + await OnTurnTaskAsync(attempt.Context, async () => + result = await feature.ExecuteJobAsync(new CallbackContext(mismatched), TestContext.Current.CancellationToken)); + Assert.Same(DurableJobRunResult.Completed, result); + Assert.Same(previous, value.Value); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + var pending = attempt.Grain.GetSnapshotForTest(); + Assert.Empty(pending.Effects); + Assert.Equal(1, pending.InboxCount); + Assert.Equal(0, pending.ProcessedMessageCount); + Assert.False(attempt.Grain.ApplyAttempted.Task.IsCompleted); + Assert.Empty(attempt.Outbox.Messages); + attempt.Preparation.Release(); + var recovered = await Fixture.WaitForEffectCountAsync(attempt.Receiver, 1); + Assert.Equal(pending.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Empty(recovered.InboxDeadLetters); + } + + [Fact] + public async Task LateOutgoingIntent_AfterCapture_WaitsForNextCapture() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var grain = Assert.IsType(context.GrainInstance); + var durable = context.ActivationServices.GetRequiredKeyedService>("test-handler-output"); + var sessions = Fixture.Client.ServiceProvider.GetRequiredService(); + var first = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 1); + var late = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 2); + await receiver.StageOutputAsync(first); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var write = Fixture.WriteStateAsync(receiver).AsTask(); + await storage.WaitUntilEnteredAsync(); + await OnTurnTaskAsync(context, async () => + { + outbox.Send(late); + }); + Assert.Equal(new[] { first.Envelope.MessageId }, grain.OutputCaptures[^1]); + Assert.Equal(2, outbox.Count); + storage.Release(); + await write; + Assert.Equal(new[] { first.Envelope.MessageId }, grain.OutputCaptures[^1]); + await receiver.RetryWriteStateAsync(); + Assert.Equal(new[] { first.Envelope.MessageId, late.Envelope.MessageId }.OrderBy(static key => key.ToString(), StringComparer.Ordinal), + grain.OutputCaptures[^1].OrderBy(static key => key.ToString(), StringComparer.Ordinal)); + Assert.Equal(2, durable.Count); + await receiver.RequestDeactivationAsync(); + Assert.Equal(2, (await receiver.GetSnapshotAsync()).OutboxCount); + } + + [Fact] + public async Task AcceptanceAfterCapture_IsAcknowledgedByLaterOperation() + { + var receiver = NewGrain(); + var seed = CreateEnvelope(receiver, NewMessage(110, "seed"), "messages/cutoff"); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-capture-cutoff"); + var turn = receiver.HoldPumpTurnAsync("hold-capture-cutoff", deactivate: false); + await hold.WaitUntilEnteredAsync(); + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + Task initial = null!; + await OnTurnAsync(context, () => initial = extension.DeliverAsync(seed, TestContext.Current.CancellationToken).AsTask()); + Assert.Equal(DeliveryStatus.Accepted, (await initial).Status); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + grain.ClearCaptures(); + await OnTurnAsync(context, () => context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "capture-cutoff"); + var preceding = Fixture.WriteStateAsync(receiver).AsTask(); + await storage.WaitUntilEnteredAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/cutoff"); + var incoming = CreateEnvelope(receiver, NewMessage(111, "late-acceptance"), "messages/cutoff"); + Task delivery = null!; + await OnTurnAsync(context, () => delivery = extension.DeliverAsync(incoming).AsTask()); + Assert.False(delivery.IsCompleted); + Assert.Equal(2, grain.GetSnapshotForTest().InboxCount); + storage.Release(); + await preceding; + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + Assert.Equal(1, grain.Captures[0].InboxCount); + Assert.Equal(2, grain.Captures[1].InboxCount); + Assert.Equal(2, grain.GetSnapshotForTest().InboxCount); + hold.Release(); + await turn; + await handler.WaitUntilEnteredAsync(); + handler.Release(); + Assert.Equal(2, (await Fixture.WaitForEffectCountAsync(receiver, 2)).ProcessedMessageCount); + } + + [Fact] + public async Task CallerWaitCancellation_DoesNotWithdrawCapturedAcceptance() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var storage = Fixture.Storage.BlockWrite(journal); + var envelope = CreateEnvelope(receiver, NewMessage(112, "cancel-wait")); + using var cancellation = new CancellationTokenSource(); + var delivery = DeliverWithCancellationAsync(receiver, envelope, cancellation.Token); + await storage.WaitUntilEnteredAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery); + storage.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task OwnedTimerCancellation_DuringHandlerPreparation_PreservesActivationAndRetries() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(113, "attempt-cancel"), "messages/attempt-cancel"); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/attempt-cancel"); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var timer = Assert.Single(events.Events.Select(item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, context) + && item.Timer.GetType().GenericTypeArguments is [var state] + && state.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension")).Timer; + await OnTurnAsync(context, timer.Dispose); + await events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + Assert.False(context.Deactivated.IsCompleted); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.Equal(1, grain.GetSnapshotForTest().InboxCount); + handler.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task MessagingPrimaryStates_AreTheRegisteredScopedDataInstances() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(receiver).ActivationServices; + var manager = services.GetRequiredService(); + Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox", out var inbox)); + Assert.Same(services.GetRequiredKeyedService>("__orleans.messaging.inbox"), inbox); + Assert.True(manager.TryGetStateMachine("test-handler-output", out var output)); + Assert.Same(((JournaledTestOutbox)services.GetRequiredService()).StoredMessages, output); + Assert.Same(services.GetRequiredKeyedService>("test-handler-output"), output); + } + + [Fact] + public async Task AcceptanceWriteFailure_DeactivatesAndFreshScopeRetries() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Fixture.Storage.FailWrite(journal); + var envelope = CreateEnvelope(receiver, NewMessage(114, "acceptance-write-failure")); + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Contains("Injected journal write failure", failure.Message, StringComparison.Ordinal); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var rejected = grain.GetSnapshotForTest(); + Assert.Equal(before.ActivationId, rejected.ActivationId); + Assert.Equal(1, rejected.InboxCount); + Assert.NotNull(rejected.InboxJobId); + Assert.NotNull(rejected.InboxJob); + Assert.Empty(rejected.Effects); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var deactivation = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.IsType(deactivation); + Assert.Equal(failure.Message, deactivation.Message); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(rejected.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Null(recovered.InboxJobId); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + var jobs = Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId()); + Assert.Equal(2, jobs.Count); + Assert.NotEqual(jobs[0].Metadata!["orleans.messaging.ownership-id"], jobs[1].Metadata!["orleans.messaging.ownership-id"]); + } + + private async Task PrepareAttemptAsync(string routeSuffix) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var route = "messages/" + routeSuffix; + var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(109, routeSuffix) with { ForwardTo = GrainId.Create("output", routeSuffix) }, route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var preparation = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route + "/application-preparation"); + handler.Release(); + await preparation.WaitUntilEnteredAsync(); + return new(receiver, context, outbox, handler, preparation, envelope); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completion.SetResult(); } + catch (Exception exception) { completion.SetException(exception); } + }); + return completion.Task; + } + + private static Task OnTurnTaskAsync(IGrainContext context, Func action) + { + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(action()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private sealed record Attempt(IMessagingTestGrain Receiver, IGrainContext Context, JournaledTestOutbox Outbox, + HandlerProbe.Barrier Handler, HandlerProbe.Barrier Preparation, InboxMessage Envelope) : IDisposable + { + public MessagingTestGrain Grain { get; } = Assert.IsType(Context.GrainInstance); + public IJournaledStateManager Manager => Context.ActivationServices.GetRequiredService(); + public JournalId JournalId => JournalId.FromGrainId(Receiver.GetGrainId()); + public void Dispose() { Preparation.Dispose(); Handler.Dispose(); } + } + + private sealed class CallbackContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCallbackOwnershipTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCallbackOwnershipTests.cs new file mode 100644 index 00000000000..fb5882609dc --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCallbackOwnershipTests.cs @@ -0,0 +1,295 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCallbackOwnershipTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task MissingOwnerWithWork_RepairsScheduleAndCommitBeforeOrphanRetires() + { + var receiver = NewGrain(); + const string route = "messages/repair-owner"; + var envelope = CreateEnvelope(receiver, NewMessage(90, "repaired"), route); + await receiver.SeedInboxStateAsync(envelope, null, null); + var oldContext = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var orphan = CreateJob(receiver, "orphan", "old-shard", "old:1"); + using var schedule = Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName); + using var write = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + + var activation = receiver.GetSnapshotAsync(); + await schedule.WaitUntilEnteredAsync(); + var extension = GetExtension(receiver); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, orphan)).Status); + Assert.Empty(Fixture.GetSnapshot(receiver).Effects); + Assert.Empty(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + + schedule.Continue(); + await write.WaitUntilEnteredAsync(); + var scheduled = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, orphan)).Status); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, scheduled)).Status); + Assert.Empty(Fixture.GetSnapshot(receiver).Effects); + + write.Release(); + await activation; + await handler.WaitUntilEnteredAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await ExecuteAsync(extension, orphan)).Status); + var owned = Fixture.GetSnapshot(receiver); + Assert.Same(scheduled, owned.InboxJob); + Assert.Equal(scheduled.Metadata!["orleans.messaging.ownership-id"], owned.InboxJobId); + Assert.Equal(1, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + handler.Release(); + + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + } + + [Theory] + [InlineData("generation-only")] + [InlineData("handle-only")] + [InlineData("mismatched-metadata")] + public async Task RecoveredInvalidPair_FailsStartupAndStopsCallbacksWithoutRepair(string fault) + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(91, fault)); + var handle = CreateJob(receiver, "physical", "shard", fault == "mismatched-metadata" ? "other:1" : "owner:1"); + await receiver.SeedInboxStateAsync( + envelope, + fault == "handle-only" ? null : "owner:1", + fault == "generation-only" ? null : handle); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journalId); + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + using var read = Fixture.Storage.BlockRead(journalId); + var activation = receiver.GetSnapshotAsync(); + await read.WaitUntilEnteredAsync(); + var extension = GetExtension(receiver); + var current = Fixture.GetGrainContext(receiver); + read.Release(); + var lifecycle = await Assert.ThrowsAnyAsync(() => activation); + await current.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Assert.ThrowsAnyAsync(async () => await ExecuteAsync(extension, handle)); + var delivery = await Assert.ThrowsAsync(() => + OnTurnAsync(extension.Context, () => + ((IInboxExtension)extension.Handler).DeliverAsync(envelope, TestContext.Current.CancellationToken))); + Assert.Contains(delivery.Message, lifecycle.ToString(), StringComparison.Ordinal); + Assert.Contains(fault == "mismatched-metadata" ? "metadata does not match" : "both be present or both be absent", delivery.Message, StringComparison.Ordinal); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + var replayFailure = await Assert.ThrowsAnyAsync(() => receiver.GetSnapshotAsync()); + Assert.Contains(delivery.Message, replayFailure.ToString(), StringComparison.Ordinal); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + } + + [Theory] + [InlineData("different-id")] + [InlineData("different-shard")] + [InlineData("missing-metadata")] + public async Task NonAuthoritativeCallback_RetiresWithoutChangingCommittedOwner(string fault) + { + var receiver = NewGrain(); + const string route = "messages/physical-owner"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(92, fault), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var owned = Fixture.GetSnapshot(receiver); + var handle = Assert.IsType(owned.InboxJob); + var callback = CreateJob( + receiver, + fault == "different-id" ? "other-id" : handle.Id, + fault == "different-shard" ? "other-shard" : handle.ShardId, + fault == "missing-metadata" ? null : owned.InboxJobId); + + var result = await ExecuteAsync(GetExtension(receiver), callback); + + Assert.Equal(DurableJobRunStatus.Completed, result.Status); + Assert.Same(handle, Fixture.GetSnapshot(receiver).InboxJob); + Assert.Equal(owned.InboxJobId, Fixture.GetSnapshot(receiver).InboxJobId); + Assert.Empty(Fixture.GetSnapshot(receiver).Effects); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.MaxConcurrentHandlers); + } + + [Fact] + public async Task CallbacksDuringFreshInitialization_WaitUntilCommittedOwnerIsRestored() + { + var receiver = NewGrain(); + const string route = "messages/callback-initialization"; + var envelope = CreateEnvelope(receiver, NewMessage(93, "initialization"), route); + var handle = CreateJob(receiver, "committed-physical", "committed-shard", "owner:1"); + await receiver.SeedInboxStateAsync(envelope, "owner:1", handle); + var oldContext = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + using var read = Fixture.Storage.BlockRead(JournalId.FromGrainId(receiver.GetGrainId())); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var activation = receiver.GetSnapshotAsync(); + await read.WaitUntilEnteredAsync(); + var extension = GetExtension(receiver); + var orphan = CreateJob(receiver, "orphan", "orphan-shard", "stale:1"); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, handle)).Status); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, orphan)).Status); + read.Release(); + await activation; + await handler.WaitUntilEnteredAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await ExecuteAsync(extension, orphan)).Status); + var recovered = Fixture.GetSnapshot(receiver); + Assert.Equal(handle.Id, recovered.InboxJob?.Id); + Assert.Equal(handle.ShardId, recovered.InboxJob?.ShardId); + handler.Release(); + await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(DurableJobRunStatus.Completed, (await ExecuteAsync(extension, orphan)).Status); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + } + + [Fact] + public async Task DuplicateScheduledJobs_RetireExtraHandleAfterAcceptanceCommits() + { + var receiver = NewGrain(); + const string route = "messages/duplicate-jobs"; + _ = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(94, "duplicate-jobs"), route); + Fixture.JobManagerProbe.DuplicateNext(ReceiverTestServices.InboxJobName); + using var write = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var delivery = DeliverAsync(receiver, envelope); + await write.WaitUntilEnteredAsync(); + var jobs = Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId()); + Assert.Equal(2, jobs.Count); + var extension = GetExtension(receiver); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, jobs[0])).Status); + Assert.Equal(DurableJobRunStatus.InProgress, (await ExecuteAsync(extension, jobs[1])).Status); + Assert.False(delivery.IsCompleted); + write.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await handler.WaitUntilEnteredAsync(); + + Assert.Equal(DurableJobRunStatus.Completed, (await ExecuteAsync(extension, jobs[1])).Status); + Assert.Same(jobs[0], Fixture.GetSnapshot(receiver).InboxJob); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.MaxConcurrentHandlers); + } + + [Fact] + public async Task PreviousOwnerCallback_DuringReplacementPreparation_QuiescesUntilAcceptanceCommits() + { + var receiver = NewGrain(); + var oldJob = CreateJob(receiver, "previous-physical", "previous-shard", "previous:1"); + await receiver.SetInboxOwnershipAsync("previous:1", oldJob); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId())); + using var schedule = Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/replacement-quiescence"); + var envelope = CreateEnvelope(receiver, NewMessage(116, "replacement-quiescence"), "messages/replacement-quiescence"); + using var timers = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var delivery = DeliverAsync(receiver, envelope); + await schedule.WaitUntilEnteredAsync(); + + var callback = await InvokeDurableCallbackAsync(receiver, oldJob); + + Assert.Equal(DurableJobRunStatus.InProgress, callback.Status); + Assert.DoesNotContain(timers.Events, item => item.Payload is GrainTimerEvents.Created created + && ReferenceEquals(created.GrainContext, context)); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.False(delivery.IsCompleted); + Assert.Equal(oldJob.Id, Fixture.GetSnapshot(receiver).InboxJob?.Id); + schedule.Continue(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await handler.WaitUntilEnteredAsync(); + var accepted = Fixture.GetSnapshot(receiver); + var replacement = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + Assert.Same(replacement, accepted.InboxJob); + Assert.Equal(replacement.Metadata!["orleans.messaging.ownership-id"], accepted.InboxJobId); + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeDurableCallbackAsync(receiver, oldJob, dequeueCount: 2)).Status); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + } + + private static async Task InvokeDurableCallbackAsync(IMessagingTestGrain receiver, DurableJob job, int dequeueCount = 1) + { + var assembly = typeof(DurableJob).Assembly; + var receiverType = assembly.GetType("Orleans.DurableJobs.IDurableJobReceiverExtension", throwOnError: true)!; + var reference = receiver.AsReference(receiverType); + var run = Activator.CreateInstance(assembly.GetType("Orleans.DurableJobs.JobRunContext", throwOnError: true)!, + job, Guid.NewGuid().ToString("N"), dequeueCount)!; + return await (ValueTask)receiverType.GetMethod("HandleDurableJobAsync")! + .Invoke(reference, [run, TestContext.Current.CancellationToken])!; + } + + private (IGrainContext Context, IDurableJobFeatureHandler Handler) GetExtension(IMessagingTestGrain receiver) + { + var context = Fixture.GetGrainContext(receiver); + return (context, (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension"))); + } + + private static Task ExecuteAsync( + (IGrainContext Context, IDurableJobFeatureHandler Handler) extension, DurableJob job) + { + var cancellationToken = TestContext.Current.CancellationToken; + return OnTurnAsync(extension.Context, () => + extension.Handler.ExecuteJobAsync(new CallbackContext(job), cancellationToken)); + } + + private static Task OnTurnAsync(IGrainContext context, Func> action) + { + var task = new Task>(async () => + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + var result = await action(); + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + return result; + }); + context.Scheduler.QueueTask(task); + return task.Unwrap(); + } + + private static DurableJob CreateJob(IMessagingTestGrain receiver, string id, string shardId, string? ownershipId) => + new() + { + Id = id, + ShardId = shardId, + Name = ReceiverTestServices.InboxJobName, + TargetGrainId = receiver.GetGrainId(), + DueTime = DateTimeOffset.UtcNow, + Metadata = ownershipId is null ? null : new Dictionary + { + ["orleans.messaging.ownership-id"] = ownershipId + } + }; + + private sealed class CallbackContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCancellationCleanupTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCancellationCleanupTests.cs new file mode 100644 index 00000000000..8d2a670eb50 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCancellationCleanupTests.cs @@ -0,0 +1,306 @@ +using System.Collections; +using System.Collections.Concurrent; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCancellationCleanupTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task HandlerCancellation_CallbackFailurePreservesCleanupAndOriginalCause(bool throws, bool terminalFailure) + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var shutdown = CancellationCleanupProbe.Field(extension, "_shutdownCts"); + var token = shutdown.Token; + using var logs = new CancellationCleanupProbe.Logs(); + Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService().AddProvider(logs); + using var handler = new CancelingHandler(throws); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + var envelope = CreateEnvelope(receiver, NewMessage(310, "cancellation"), "cancel-cleanup"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId())); + IOException? firstFailure = null; + using var seeded = await OnTurnAsync(context, () => CancellationCleanupProbe.SeedResults(extension)); + if (terminalFailure) + { + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var failingInput = CreateEnvelope(receiver, NewMessage(311, "capture-failure")); + var delivery = await OnTurnAsync(context, () => + ((IInboxExtension)extension).DeliverAsync(failingInput, TestContext.Current.CancellationToken).AsTask()); + firstFailure = await Assert.ThrowsAsync(() => delivery); + } + Task stopping = null!; + await OnTurnAsync(context, () => + { + if (!terminalFailure) Assert.True(CancellationCleanupProbe.CoordinatorIsActive(extension)); + stopping = ((ILifecycleObserver)extension).OnStop(CancellationToken.None); + Assert.False(stopping.IsCompleted); + CancellationCleanupProbe.AssertClean(extension, seeded); + Assert.True(shutdown.Token.IsCancellationRequested); + Assert.Equal(1, handler.CallbackCalls); + logs.AssertCallback(throws, handler.CallbackFailure); + if (terminalFailure) Assert.Same(firstFailure, CancellationCleanupProbe.Field(extension, "_failure").SourceException); + else Assert.Null(CancellationCleanupProbe.Field(extension, "_failure")); + }); + Assert.True(token.IsCancellationRequested); + Assert.False(handler.Finished); + handler.Release.TrySetResult(); + await stopping.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Exception failure; + if (terminalFailure) + { + failure = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Same(firstFailure, failure); + } + else + { + failure = Assert.IsType(handler.Cancellation); + Assert.Null(CancellationCleanupProbe.Field(extension, "_failure")); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + await receiver.RequestDeactivationAsync(); + } + Assert.DoesNotContain(handler.CallbackFailure, failure is AggregateException aggregate ? aggregate.Flatten().InnerExceptions : [failure]); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.True(handler.Finished); + Assert.Throws(() => shutdown.Token); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.Equal(1, CancellationCleanupProbe.Field(extension, "_gate").CurrentCount); + Assert.Empty(CancellationCleanupProbe.Field(extension, "_pendingWrites")); + ((IDisposable)extension).Dispose(); + Assert.Equal(1, handler.CallbackCalls); + logs.AssertCallback(throws, handler.CallbackFailure); + await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + private sealed class CancelingHandler(bool throws) : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Exception CallbackFailure { get; } = new InvalidOperationException("Application cancellation callback failed."); + public int CallbackCalls { get; private set; } + public bool Finished { get; private set; } + public OperationCanceledException? Cancellation { get; private set; } + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + using var registration = cancellationToken.Register(() => + { + CallbackCalls++; + if (throws) throw CallbackFailure; + }); + Entered.TrySetResult(); + try + { + await Release.Task; + cancellationToken.ThrowIfCancellationRequested(); + throw new InvalidOperationException("A canceled handler must never complete."); + } + catch (OperationCanceledException exception) + { + Cancellation = exception; + throw; + } + finally { Finished = true; } + } + public void Dispose() => Release.TrySetResult(); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => OnTurnAsync(context, () => { action(); return true; }); + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { done.SetResult(action()); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCancellationCleanupContractTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Dispose_CallbackFailureStillReleasesOwnedResourcesOnce(bool throws) + { + using var probe = new CancellationCleanupProbe(throws); + ((IDisposable)probe.Extension).Dispose(); + probe.AssertClean(); + Assert.Throws(() => probe.Shutdown.Token); + ((IDisposable)probe.Extension).Dispose(); + probe.AssertClean(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OnStop_CallbackFailureStillDrainsOwnedDelivery(bool throws) + { + using var probe = new CancellationCleanupProbe(throws); + var owned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationCleanupProbe.SetField(probe.Extension, "_activeDelivery", owned.Task); + var stopping = ((ILifecycleObserver)probe.Extension).OnStop(CancellationToken.None); + Assert.False(stopping.IsCompleted); + probe.AssertClean(); + Assert.True(probe.Shutdown.Token.CanBeCanceled); + owned.SetException(new IOException("Already observed delivery failure.")); + await stopping; + ((IDisposable)probe.Extension).Dispose(); + Assert.Throws(() => probe.Shutdown.Token); + probe.AssertClean(); + } +} + +internal sealed class CancellationCleanupProbe : IDisposable +{ + internal static readonly Type ExtensionType = ReceiverTestServices.GetImplementationType("InboxExtension"); + private readonly CancellationTokenSource _handlerCancellation; + private readonly CancellationTokenRegistration _registration; + private readonly ILoggerFactory _loggerFactory; + private readonly bool _throws; + private readonly Seeded _seeded; + private readonly Logs _logs = new(); + private readonly Exception _callbackFailure = new InvalidOperationException("Linked handler cancellation failed."); + private int _calls; + public object Extension { get; } = RuntimeHelpers.GetUninitializedObject(ExtensionType); + public CancellationTokenSource Shutdown { get; } = new(); + public CancellationCleanupProbe(bool throws) + { + _throws = throws; + _loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(_logs)); + SetField(Extension, "_logger", Activator.CreateInstance(typeof(Logger<>).MakeGenericType(ExtensionType), _loggerFactory)!); + SetField(Extension, "_shutdownCts", Shutdown); + SetField(Extension, "_shutdownToken", Shutdown.Token); + SetField(Extension, "_pumpCoordinator", Activator.CreateInstance(ReceiverTestServices.GetImplementationType("MessagingPumpCoordinator"))!); + SetField(Extension, "_pumpResults", Activator.CreateInstance(ReceiverTestServices.GetImplementationType("MessagingPumpResults"), nonPublic: true)!); + var instruments = ReceiverTestServices.GetImplementationType("MessagingInstruments") + .GetMethod("CreateForDirectConstruction", BindingFlags.NonPublic | BindingFlags.Static)!.Invoke(null, null)!; + SetField(Extension, "_instruments", instruments); + instruments.GetType().GetMethod("OnInboxDepthChanged", BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(instruments, [1]); + SetField(Extension, "_metricsActive", 1); + SetField(Extension, "_reportedDepth", 1); + SetField(Extension, "_activeDelivery", Task.CompletedTask); + SetField(Extension, "_provisionalAcceptances", Activator.CreateInstance( + ExtensionType.GetField("_provisionalAcceptances", BindingFlags.NonPublic | BindingFlags.Instance)!.FieldType)!); + SetField(Extension, "_pendingWrites", Activator.CreateInstance( + ExtensionType.GetField("_pendingWrites", BindingFlags.NonPublic | BindingFlags.Instance)!.FieldType)!); + var coordinator = Field(Extension, "_pumpCoordinator"); + object?[] arguments = ["owner", Shutdown.Token, null]; + Assert.True((bool)coordinator.GetType().GetMethod("TryAcquire")!.Invoke(coordinator, arguments)!); + _seeded = SeedResults(Extension); + _handlerCancellation = CancellationTokenSource.CreateLinkedTokenSource(Shutdown.Token); + _registration = _handlerCancellation.Token.Register(() => { _calls++; if (_throws) throw _callbackFailure; }); + } + public void AssertClean() + { + Assert.True(Shutdown.IsCancellationRequested); + Assert.True(_handlerCancellation.IsCancellationRequested); + Assert.Equal(1, _calls); + AssertClean(Extension, _seeded); + _logs.AssertCallback(_throws, _callbackFailure); + } + public static T Field(object instance, string name) => (T)ExtensionType.GetField(name, BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(instance)!; + public static void SetField(object instance, string name, object value) => ExtensionType.GetField(name, BindingFlags.NonPublic | BindingFlags.Instance)!.SetValue(instance, value); + public static bool CoordinatorIsActive(object extension) + { + var coordinator = Field(extension, "_pumpCoordinator"); + return (bool)coordinator.GetType().GetProperty("IsActive")!.GetValue(coordinator)!; + } + public static Seeded SeedResults(object extension) + { + var results = Field(extension, "_pumpResults"); + var keyType = ReceiverTestServices.GetImplementationType("MessagingPumpExecutionKey"); + var entries = (IDictionary)results.GetType().GetField("_entries", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(results)!; + object? captured = null; + var cancellation = new CancellationTokenSource(); + foreach (var name in new[] { ReceiverTestServices.InboxJobName, "test/other-pump" }) + { + var key = Activator.CreateInstance(keyType, name, "cleanup", "run", 1L)!; + object?[] arguments = [key, cancellation.Token, null]; + Assert.True((bool)results.GetType().GetMethod("TryStart")!.Invoke(results, arguments)!); + if (name == ReceiverTestServices.InboxJobName) captured = entries[key]; + } + var registration = (CancellationTokenRegistration)captured!.GetType().GetProperty("CancellationRegistration")!.GetValue(captured)!; + Assert.NotEqual(default, registration); + return new Seeded(cancellation, captured); + } + public static void AssertClean(object extension, Seeded seeded) + { + Assert.False(CoordinatorIsActive(extension)); + Assert.Equal(0, Field(extension, "_metricsActive")); + Assert.Equal(0, Field(extension, "_reportedDepth")); + var instruments = Field(extension, "_instruments"); + var depth = instruments.GetType().GetField("_inboxDepth", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instruments)!; + Assert.Equal(0L, depth.GetType().GetField("_value", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(depth)); + var results = Field(extension, "_pumpResults"); + var entries = (IDictionary)results.GetType().GetField("_entries", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(results)!; + var key = Assert.Single(entries.Keys.Cast()); + Assert.Equal("test/other-pump", key.GetType().GetProperty("JobName")!.GetValue(key)); + Assert.Equal(default(CancellationTokenRegistration), seeded.Entry.GetType().GetProperty("CancellationRegistration")!.GetValue(seeded.Entry)); + } + public void Dispose() + { + _seeded.Dispose(); + _registration.Dispose(); + _handlerCancellation.Dispose(); + Shutdown.Dispose(); + _loggerFactory.Dispose(); + } + internal sealed record Seeded(CancellationTokenSource Cancellation, object Entry) : IDisposable + { + public void Dispose() => Cancellation.Dispose(); + } + + internal sealed class Logs : ILoggerProvider + { + private readonly ConcurrentQueue _exceptions = new(); + public ILogger CreateLogger(string categoryName) => new Sink(this, categoryName); + public void Dispose() { } + public void AssertCallback(bool throws, Exception failure) + { + if (throws) Assert.Same(failure, Assert.Single(Assert.IsType(Assert.Single(_exceptions)).Flatten().InnerExceptions)); + else Assert.Empty(_exceptions); + } + private sealed class Sink(Logs owner, string category) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Error; + public void Log(LogLevel level, EventId id, TState state, Exception? exception, Func formatter) + { + if (category == "Orleans.Messaging.InboxExtension" && id.Name == "LogCancellationCallbackFailure" && exception is not null) + owner._exceptions.Enqueue(exception); + } + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs new file mode 100644 index 00000000000..abe8cb45fd4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs @@ -0,0 +1,113 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class InboxCapacityCollection : ICollectionFixture +{ + public const string Name = "Messaging inbox capacity"; +} + +[Collection(InboxCapacityCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCapacityBehaviorTests(InboxCapacityClusterFixture fixture) +{ + [Fact] + public async Task InboxAtCapacity_BackpressuresWithoutPersistenceAndRecoversWhenCapacityFrees() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("capacity-test-sender", Guid.NewGuid().ToString("N")); + var poison = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/capacity", new TestMessage(TestApplicationProtocol.NewMessageId(), 31, "poison", ThrowDuringPreparation: true)); + var rejected = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/capacity", new TestMessage(TestApplicationProtocol.NewMessageId(), 32, "accepted-after-capacity")); + const string processedInstrument = "orleans-messaging-inbox-messages-processed"; + var firstRetryAcknowledged = fixture.Metrics.WaitForCountAsync(processedInstrument, 1, "retry"); + using var timers = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, poison)).Status); + var full = await fixture.WaitForInboxCountAsync(receiver, 1); + Assert.Empty(full.Effects); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, rejected)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + + await firstRetryAcknowledged; + Assert.Equal(1, fixture.Metrics.GetCount(processedInstrument, "retry")); + var originalContext = fixture.GetGrainContext(receiver); + var timer = Assert.Single(timers.Events.Select(static item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, originalContext) + && item.Timer.GetType().GenericTypeArguments is [var state] + && state.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension") + && state.Name == "LocalDrainTimerState").Timer; + var stopped = await timers.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Null(Assert.IsType(stopped.Payload).Exception); + DateTimeOffset nextAttempt = default; + await OnTurnAsync(originalContext, () => + { + var stateType = ReceiverTestServices.GetImplementationType("InboxMessageState"); + var statesType = typeof(IDurableDictionary<,>).MakeGenericType(typeof(HierarchicalKey), stateType); + var states = originalContext.ActivationServices.GetRequiredKeyedService( + statesType, "__orleans.messaging.inbox-message-state"); + var dictionaryType = typeof(IDictionary<,>).MakeGenericType(typeof(HierarchicalKey), stateType); + var state = dictionaryType.GetProperty("Item")!.GetValue(states, [poison.Envelope.MessageId])!; + Assert.Equal(1, stateType.GetProperty("AttemptCount")!.GetValue(state)); + nextAttempt = Assert.IsType(stateType.GetProperty("NextAttemptAt")!.GetValue(state)); + Assert.Equal(fixture.Clock.GetUtcNow() + TimeSpan.FromHours(1), nextAttempt); + }); + fixture.Clock.Advance(TimeSpan.FromHours(2)); + Assert.True(nextAttempt <= fixture.Clock.GetUtcNow()); + var previous = fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + Assert.NotSame(previous, fixture.GetGrainContext(receiver)); + var current = fixture.GetGrainContext(receiver); + var currentGrain = Assert.IsType(current.GrainInstance); + try + { + await fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId == currentGrain.GetSnapshotForTest().ActivationId + && snapshot.InboxDeadLetters.Count == 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Recovered retry did not complete. Activation={snapshot.ActivationId}, inbox={snapshot.InboxCount}, " + + $"processed={snapshot.ProcessedMessageCount}, deadLetters={snapshot.InboxDeadLetters.Count}, " + + $"nextAttempt={nextAttempt:O}, now={fixture.Clock.GetUtcNow():O}, job={snapshot.InboxJobId}, " + + $"failureCompleted={currentGrain.DeactivationFailure.Task.IsCompleted}.", exception); + } + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, rejected)).Status); + var recovered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("accepted-after-capacity", Assert.Single(recovered.Effects).Value); + Assert.Equal(2, Assert.Single(recovered.InboxDeadLetters).AttemptCount); + } + + private static async Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + await receiver.AsReference().DeliverAsync(envelope); + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var finished = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); finished.SetResult(); } + catch (Exception exception) { finished.SetException(exception); } + }); + return finished.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCommittedCountTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCommittedCountTests.cs new file mode 100644 index 00000000000..e9264a683d1 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCommittedCountTests.cs @@ -0,0 +1,382 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCommittedCountTests() : MessagingBehaviorTestBase(new CountFixture()) +{ + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(1)] + [InlineData(32)] + [InlineData(128)] + public async Task QueuedDeliveryBurst_CountChecksPerformConstantCollectionWork(int messages) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var probe = new CountProbe(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-count-burst"); + var turn = receiver.HoldPumpTurnAsync("hold-count-burst", deactivate: false); + await hold.WaitUntilEnteredAsync(); + try + { + for (var i = 0; i < messages; i++) + { + var envelope = CreateEnvelope(receiver, NewMessage(i, "queued-count")); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery(context, probe.Extension, envelope, Cancellation)).Status); + } + await OnTurnAsync(context, () => + { + Assert.Equal(new Counts(messages, 0, messages), probe.Read()); + Assert.Equal(0, probe.Dictionary.KeyCollections); + Assert.Equal(0, probe.Dictionary.KeyVisits); + Assert.Equal(0, probe.Dictionary.EntryEnumerations); + var reads = probe.Dictionary.CountReads; + Assert.Equal(messages, probe.DurableCount()); + Assert.Equal(reads + 1, probe.Dictionary.CountReads); + }); + Assert.Equal(1, ScheduleCount(receiver)); + } + finally + { + hold.Release(); + } + await turn; + await Fixture.WaitForEffectCountAsync(receiver, messages); + _ = await receiver.GetSnapshotAsync(); + await AssertCountsAsync(context, probe, new(0, 0, 0)); + } + + [Theory] + [InlineData(0)] + [InlineData(2)] + public async Task AcceptanceCount_TracksPreparationCommitCancellationAndAcknowledgement(int existing) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var probe = new CountProbe(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-count-phases"); + var turn = receiver.HoldPumpTurnAsync("hold-count-phases", deactivate: false); + await hold.WaitUntilEnteredAsync(); + try + { + for (var i = 0; i < existing; i++) + { + var queued = CreateEnvelope(receiver, NewMessage(i, "existing")); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery(context, probe.Extension, queued, Cancellation)).Status); + } + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var storage = Fixture.Storage.BlockWrite(journal); + try + { + using var scheduling = existing == 0 ? Fixture.JobManagerProbe.BlockNext(ReceiverTestServices.InboxJobName) : null; + var envelope = CreateEnvelope(receiver, NewMessage(140, "pending-count")); + using var cancellation = new CancellationTokenSource(); + var delivery = StartDelivery(context, probe.Extension, envelope, cancellation.Token); + if (scheduling is not null) + { + await scheduling.WaitUntilEnteredAsync(); + await AssertCountsAsync(context, probe, new(0, 0, 0)); + scheduling.Continue(); + } + await storage.WaitUntilEnteredAsync(); + await AssertCountsAsync(context, probe, new(existing + 1, 1, existing)); + var duplicate = StartDelivery(context, probe.Extension, envelope, Cancellation); + Assert.False(delivery.IsCompleted); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery); + await AssertCountsAsync(context, probe, new(existing + 1, 1, existing)); + Assert.False(duplicate.IsCompleted); + storage.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + await AssertCountsAsync(context, probe, new(existing + 1, 0, existing + 1)); + Assert.Equal(1, ScheduleCount(receiver)); + } + finally + { + storage.Release(); + } + } + finally + { + hold.Release(); + } + await turn; + await Fixture.WaitForEffectCountAsync(receiver, existing + 1); + _ = await receiver.GetSnapshotAsync(); + await AssertCountsAsync(context, probe, new(0, 0, 0)); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + Assert.Equal(DurableJobRunStatus.Completed, (await RunPumpAsync(context, probe.Extension, job)).Status); + var completed = await receiver.GetSnapshotAsync(); + Assert.Null(completed.InboxJobId); + Assert.Equal(existing + 1, completed.ProcessedMessageCount); + await AssertCountsAsync(context, probe, new(0, 0, 0)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task FailedAcceptanceCount_PreservesOldStateAndUsesFreshReplay(bool committed) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var probe = new CountProbe(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-count-fault"); + using var handlers = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/count-replay"); + var turn = receiver.HoldPumpTurnAsync("hold-count-fault", deactivate: false); + await hold.WaitUntilEnteredAsync(); + try + { + var first = CreateEnvelope(receiver, NewMessage(150, "acknowledged"), "messages/count-replay"); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery(context, probe.Extension, first, Cancellation)).Status); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var storage = Fixture.Storage.BlockWrite(journal); + try + { + if (committed) Fixture.Storage.FailAfterWrite(journal); + var second = CreateEnvelope(receiver, NewMessage(151, "ambiguous"), "messages/count-replay"); + var delivery = StartDelivery(context, probe.Extension, second, Cancellation); + await storage.WaitUntilEnteredAsync(); + await AssertCountsAsync(context, probe, new(2, 1, 1)); + if (committed) storage.Release(); + else storage.Fail(); + var failure = await Assert.ThrowsAsync(() => delivery); + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation)); + await AssertCountsAsync(context, probe, new(2, 1, 1)); + } + finally + { + storage.Release(); + } + } + finally + { + hold.Release(); + } + await turn; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(new Counts(2, 0, 2), probe.Read()); + // The recovered drain turn can hold admission ahead of this snapshot request. + var recoverySnapshot = receiver.GetSnapshotAsync(); + await handlers.WaitUntilEnteredAsync(); + var freshContext = Fixture.GetGrainContext(receiver); + Assert.NotSame(context, freshContext); + var fresh = new CountProbe(freshContext); + var restored = committed ? 2 : 1; + await AssertCountsAsync(freshContext, fresh, new(restored, 0, restored)); + Assert.Equal(new Counts(2, 0, 2), probe.Read()); + handlers.Release(); + _ = await recoverySnapshot; + await Fixture.WaitForEffectCountAsync(receiver, restored); + _ = await receiver.GetSnapshotAsync(); + await AssertCountsAsync(freshContext, fresh, new(0, 0, 0)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task FailedAcceptanceCount_DistinguishesSchedulingFailureFromStagedWriteFailure(bool scheduling) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var probe = new CountProbe(context); + if (scheduling) Fixture.JobManagerProbe.FailNext(ReceiverTestServices.InboxJobName); + else Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var envelope = CreateEnvelope(receiver, NewMessage(152, "rejected")); + if (scheduling) + { + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Contains("Injected durable job scheduling failure", failure.Message, StringComparison.Ordinal); + } + else + { + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Contains("Injected journal write failure", failure.Message, StringComparison.Ordinal); + } + if (!scheduling) await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(scheduling ? new Counts(0, 0, 0) : new Counts(1, 0, 1), probe.Read()); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + if (!scheduling) + { + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + } + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + var current = Fixture.GetGrainContext(receiver); + await AssertCountsAsync(current, new CountProbe(current), new(0, 0, 0)); + if (!scheduling) Assert.Equal(new Counts(1, 0, 1), probe.Read()); + } + + [Fact] + public async Task StoppedOwnerDeletionCount_ResetsAndLaterAcceptanceUsesFreshGeneration() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(153, "delete-pending")); + var job = new DurableJob + { + Id = "count-delete-job", + ShardId = "count-delete-shard", + Name = ReceiverTestServices.InboxJobName, + TargetGrainId = receiver.GetGrainId(), + DueTime = Fixture.Clock.GetUtcNow(), + Metadata = new Dictionary { ["orleans.messaging.ownership-id"] = "count-delete:1" } + }; + await receiver.SeedInboxStateAsync(envelope, "count-delete:1", job); + var context = Fixture.GetGrainContext(receiver); + var probe = new CountProbe(context); + await AssertCountsAsync(context, probe, new(1, 0, 1)); + await receiver.DeleteStateAndDeactivateAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(new Counts(0, 0, 0), probe.Read()); + _ = await receiver.GetSnapshotAsync(); + var replacement = Fixture.GetGrainContext(receiver); + Assert.NotSame(context, replacement); + var replacementProbe = new CountProbe(replacement); + await AssertCountsAsync(replacement, replacementProbe, new(0, 0, 0)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + await AssertCountsAsync(replacement, replacementProbe, new(0, 0, 0)); + await receiver.RequestDeactivationAsync(); + await replacement.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + _ = await receiver.GetSnapshotAsync(); + var freshContext = Fixture.GetGrainContext(receiver); + Assert.NotSame(replacement, freshContext); + await AssertCountsAsync(freshContext, new CountProbe(freshContext), new(0, 0, 0)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + private int ScheduleCount(IMessagingTestGrain receiver) => + Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId()); + private static Task AssertCountsAsync(IGrainContext context, CountProbe probe, Counts expected) => + OnTurnAsync(context, () => Assert.Equal(expected, probe.Read())); + private static Task StartDelivery(IGrainContext context, IInboxExtension extension, InboxMessage envelope, CancellationToken cancellation) + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(envelope, cancellation).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } + private static async Task RunPumpAsync(IGrainContext context, IInboxExtension extension, DurableJob job) + { + var feature = (IDurableJobFeatureHandler)extension; + var run = new PumpContext(job); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + DurableJobRunResult result = null!; + await OnTurnAsync(context, () => result = feature.ExecuteJobAsync(run, Cancellation).GetAwaiter().GetResult()); + if (result.IsInProgress) + { + await events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.GrainContext, context), + TimeSpan.FromSeconds(30), Cancellation); + await OnTurnAsync(context, () => result = feature.ExecuteJobAsync(run, Cancellation).GetAwaiter().GetResult()); + } + return result; + } + private sealed record Counts(int Stored, int Provisional, int Durable, bool ProvisionalSubset = true); + private sealed class CountProbe + { + private readonly HashSet _provisional; + public CountProbe(IGrainContext context) + { + var type = ReceiverTestServices.GetImplementationType("InboxExtension"); + Extension = (IInboxExtension)context.ActivationServices.GetRequiredService(type); + var field = type.GetField("_inboxDict", BindingFlags.Instance | BindingFlags.NonPublic)!; + Dictionary = new CountingDictionary((IDictionary)field.GetValue(Extension)!); + field.SetValue(Extension, Dictionary); + _provisional = (HashSet)type.GetField("_provisionalAcceptances", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(Extension)!; + DurableCount = type.GetMethod("GetCommittedInboxCount", BindingFlags.Instance | BindingFlags.NonPublic)!.CreateDelegate>(Extension); + } + public IInboxExtension Extension { get; } + public CountingDictionary Dictionary { get; } + public Func DurableCount { get; } + public Counts Read() => new(Dictionary.Inner.Count, _provisional.Count, DurableCount(), _provisional.All(Dictionary.Inner.ContainsKey)); + } + private sealed class CountingDictionary(IDictionary inner) : IDictionary + { + public IDictionary Inner => inner; + public int CountReads { get; private set; } + public int KeyCollections { get; private set; } + public int KeyVisits { get; private set; } + public int EntryEnumerations { get; private set; } + public int Count { get { CountReads++; return inner.Count; } } + public ICollection Keys { get { KeyCollections++; return new CountingKeys(this); } } + public ICollection Values => inner.Values; + public InboxMessage this[HierarchicalKey key] { get => inner[key]; set => inner[key] = value; } + public bool IsReadOnly => inner.IsReadOnly; + public void Add(HierarchicalKey key, InboxMessage value) => inner.Add(key, value); + public void Add(KeyValuePair item) => inner.Add(item); + public void Clear() => inner.Clear(); + public bool Contains(KeyValuePair item) => inner.Contains(item); + public bool ContainsKey(HierarchicalKey key) => inner.ContainsKey(key); + public void CopyTo(KeyValuePair[] array, int index) => inner.CopyTo(array, index); + public bool Remove(HierarchicalKey key) => inner.Remove(key); + public bool Remove(KeyValuePair item) => inner.Remove(item); + public bool TryGetValue(HierarchicalKey key, out InboxMessage value) => inner.TryGetValue(key, out value); + public IEnumerator> GetEnumerator() { EntryEnumerations++; return inner.GetEnumerator(); } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + private sealed class CountingKeys(CountingDictionary owner) : ICollection + { + public int Count => owner.Inner.Keys.Count; + public bool IsReadOnly => true; + public bool Contains(HierarchicalKey item) => owner.Inner.Keys.Contains(item); + public void CopyTo(HierarchicalKey[] array, int index) => owner.Inner.Keys.CopyTo(array, index); + public IEnumerator GetEnumerator() + { + foreach (var key in owner.Inner.Keys) { owner.KeyVisits++; yield return key; } + } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + public void Add(HierarchicalKey item) => throw new NotSupportedException(); + public void Clear() => throw new NotSupportedException(); + public bool Remove(HierarchicalKey item) => throw new NotSupportedException(); + } + } + private sealed class PumpContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + private sealed class CountFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxCapacity = 256; + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs new file mode 100644 index 00000000000..dbbdbb57238 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs @@ -0,0 +1,196 @@ +using System.Buffers; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Orleans.Timers; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxEagerManagerTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + public async Task SynchronousManager_StagesBeforeWriteAndCompletesWithoutStateFaultNotifications(bool reverseStateOrder, bool failWrite) + { + var source = NewGrain(); + _ = await source.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(source).ActivationServices; + var format = services.GetRequiredService>().Value.JournalFormatKey; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(format); + builder.Services.AddScoped(sp => + new EagerManager(sp.GetRequiredKeyedService(format), reverseStateOrder)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var stateServices = scope.ServiceProvider; + var manager = Assert.IsType(stateServices.GetRequiredService()); + var messages = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox"); + var processed = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed"); + var attempts = stateServices.GetRequiredKeyedService(typeof(IDurableDictionary<,>) + .MakeGenericType(typeof(HierarchicalKey), ReceiverTestServices.GetImplementationType("InboxMessageState")), "__orleans.messaging.inbox-message-state"); + var deadLetters = stateServices.GetRequiredKeyedService(typeof(IDurableDictionary<,>) + .MakeGenericType(typeof(HierarchicalKey), ReceiverTestServices.GetImplementationType("InboxDeadLetter")), "__orleans.messaging.inbox-dead-letters"); + var ownerId = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-job-id"); + var ownerJob = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-job-handle"); + var completed = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-completed-job-id"); + var sequence = stateServices.GetRequiredKeyedService>("__orleans.messaging.inbox-job-sequence"); + var handler = Substitute.For(); + var inbox = Create("Inbox", messages, new[] { handler }, 10); + var context = Substitute.For(); + var grainId = GrainId.Create("eager-inbox", "standalone"); + context.GrainId.Returns(grainId); + context.ActivationServices.Returns(stateServices); + context.GrainInstance.Returns(Substitute.For()); + context.ObservableLifecycle.Returns(Substitute.For()); + var jobs = Substitute.For(); + DurableJob scheduled = null!; + jobs.ScheduleJobAsync(Arg.Any(), Arg.Any()).Returns(call => + { + var request = call.Arg(); + scheduled = new DurableJob + { + Id = "scheduler-returned-id", + ShardId = "scheduler-returned-shard", + Name = request.JobName, + TargetGrainId = request.Target, + DueTime = request.DueTime, + Metadata = request.Metadata + }; + return Task.FromResult(scheduled); + }); + var failure = new IOException("Owner failed while capturing the staged write."); + if (failWrite) + { + context.When(value => value.Deactivate(Arg.Any(), Arg.Any())) + .Do(_ => throw new IOException("Secondary deactivation failure.")); + } + var timers = Substitute.For(); + var type = ReceiverTestServices.GetImplementationType("InboxExtension"); + var extension = (IInboxExtension)Create("InboxExtension", context, + timers, manager, + services.GetRequiredService(typeof(ILogger<>).MakeGenericType(type)), + services.GetRequiredService(ReceiverTestServices.GetImplementationType("MessagingInstruments")), + inbox, messages, processed, attempts, deadLetters, ownerId, ownerJob, completed, sequence, + jobs, Substitute.For(), + Create("MessagingPumpResults"), TimeProvider.System, TimeProvider.System, + new InboxOptions { MaxCapacity = 10 }); + using var lifetime = (IDisposable)extension; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await ((ILifecycleObserver)extension).OnStart(TestContext.Current.CancellationToken); + var envelope = TestApplicationProtocol.CreateInbox(stateServices.GetRequiredService(), GrainId.Create("sender", "eager"), "route", 42); + manager.BeforeCapture = () => + { + Assert.Equal(envelope, Assert.Single(messages).Value); + Assert.Same(scheduled, ownerJob.Value); + Assert.Equal(scheduled.Metadata!["orleans.messaging.ownership-id"], ownerId.Value); + Assert.Equal(1, sequence.Value); + if (failWrite) throw failure; + }; + var delivery = extension.DeliverAsync(envelope, TestContext.Current.CancellationToken); + if (failWrite) + { + Assert.True(delivery.IsCompleted); + Assert.Same(failure, await Assert.ThrowsAsync(() => delivery.AsTask())); + var captured = (System.Runtime.ExceptionServices.ExceptionDispatchInfo)type + .GetField("_failure", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + Assert.Same(failure, captured.SourceException); + Assert.Equal(1, manager.Requests); + Assert.Equal(0, manager.Writes); + Assert.True(manager.IsFenced); + Assert.Empty(manager.Batches); + Assert.Single(messages); + Assert.Empty(timers.ReceivedCalls()); + } + else + { + Assert.True(delivery.IsCompletedSuccessfully); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + Assert.Equal(1, manager.Writes); + Assert.NotEmpty(Assert.Single(manager.Batches)); + Assert.Empty((IEnumerable)type.GetField("_provisionalAcceptances", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!); + } + await jobs.Received(1).ScheduleJobAsync(Arg.Any(), Arg.Any()); + } + + private static object Create(string name, params object[] args) => Activator.CreateInstance( + ReceiverTestServices.GetImplementationType(name), BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DoNotWrapExceptions, + binder: null, args, culture: null)!; + + private sealed class EagerManager(IJournalFormat format, bool reverseStateOrder) : IJournaledStateManager + { + private readonly Dictionary _states = []; + private readonly JournalBufferWriter _writer = format.CreateWriter(); + private readonly Dictionary _ids = []; + private IEnumerable States => reverseStateOrder ? _states.Values.Reverse() : _states.Values; + private System.Runtime.ExceptionServices.ExceptionDispatchInfo? _failure; + public IList Hooks { get; } = []; + public Action? BeforeCapture { get; set; } + public int Writes { get; private set; } + public int Requests { get; private set; } + public bool IsFenced => _failure is not null; + public List Batches { get; } = []; + public void RegisterStateMachine(string name, IStateMachine state) + { + _states.Add(name, state); + _ids.Add(state, checked((uint)(_ids.Count + 8))); + } + public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) => _states.TryGetValue(name, out state); + public ValueTask InitializeAsync(CancellationToken token) + { + _failure?.Throw(); + token.ThrowIfCancellationRequested(); + foreach (var state in States) state.Reset(_writer.CreateJournalStreamWriter(new(_ids[state]))); + foreach (var state in States) state.OnRecoveryCompleted(); + return default; + } + public ValueTask WriteStateAsync(CancellationToken token) + { + Requests++; + _failure?.Throw(); + token.ThrowIfCancellationRequested(); + try + { + BeforeCapture?.Invoke(); + foreach (var state in States) state.WritePendingEntries(_writer.CreateJournalStreamWriter(new(_ids[state]))); + using var buffer = _writer.GetBuffer(); + if (buffer.Length > 0) + { + Batches.Add(buffer.AsReadOnlySequence().ToArray()); + _writer.Reset(); + foreach (var state in States) state.OnWriteCompleted(); + } + Writes++; + return ValueTask.CompletedTask; + } + catch (Exception exception) + { + _failure ??= System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(exception); + throw; + } + } + public ValueTask DeleteStateAsync(CancellationToken token) + { + _failure?.Throw(); + token.ThrowIfCancellationRequested(); + _writer.Reset(); + foreach (var state in States) state.Reset(_writer.CreateJournalStreamWriter(new(_ids[state]))); + Batches.Clear(); + return default; + } + public ValueTask DisposeAsync() { _writer.Dispose(); return default; } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs new file mode 100644 index 00000000000..9e885113e92 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs @@ -0,0 +1,75 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxEnvelopeValidationTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DefaultMessage_RejectsBeforeAcceptanceOrDuplicate(bool existingKey) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + // External deserialization can materialize invalid default fields before admission. + var malformed = new InboxMessage { Envelope = default }; + var context = Fixture.GetGrainContext(receiver); + if (existingKey) + { + var processed = context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed"); + await OnTurnAsync(context, () => processed.Add(malformed.Envelope.MessageId, Fixture.Clock.GetUtcNow())); + await receiver.RetryWriteStateAsync(); + } + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + + var exception = await Assert.ThrowsAsync(() => DeliverAsync(receiver, malformed)); + + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + await OnTurnAsync(context, () => + { + var direct = extension.DeliverAsync(malformed, TestContext.Current.CancellationToken); + Assert.True(direct.IsCompleted); + Assert.Throws(() => direct.GetAwaiter().GetResult()); + }); + Assert.Contains("message", exception.Message, StringComparison.OrdinalIgnoreCase); + var after = await receiver.GetSnapshotAsync(); + Assert.Equal(0, after.InboxCount); + Assert.Null(after.InboxJobId); + Assert.Empty(after.Effects); + Assert.Equal(existingKey ? 1 : 0, after.ProcessedMessageCount); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + } + + [Fact] + public async Task DirectEnvelopeWithExplicitValidIdentity_PreservesDeliveryAndDedupe() + { + var receiver = NewGrain(); + var template = CreateEnvelope(receiver, NewMessage(187, "explicit-identity")); + var envelope = TestApplicationProtocol.With(template, Sessions, + messageId: HierarchicalKey.Parse("tenant/test/command/explicit", null)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs new file mode 100644 index 00000000000..39119f2a21f --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -0,0 +1,698 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHandlerCompletionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task LateAcknowledgedAcceptance_ImmediatelyRearmsSameLocalTimerWithoutClockAdvance() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + Assert.Null(RequestContext.Get("reusable-turn-parent")); + self.Mutate(); + self.Context.Complete(); + return default; + }; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + RequestContext.Set("reusable-turn-parent", "first-call-chain"); + InboxMessage first; + try { first = await DeliverAsync(rig); } + finally { RequestContext.Remove("reusable-turn-parent"); } + var timer = GetTimer(events, rig); + var second = CreateEnvelope(rig.Receiver, NewMessage(502, "late-local"), "async/handler"); + Task accepted = null!; + // The pump is deliberately non-interleaving. Inject the owned late admission on its + // scheduler, as capture/hook tests do, rather than waiting on a blocked ordinary RPC. + await OnTurnAsync(rig.Context, () => + { + RequestContext.Set("reusable-turn-parent", "second-call-chain"); + try + { + var extension = (IInboxExtension)rig.Context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + accepted = extension.DeliverAsync(second).AsTask(); + } + finally { RequestContext.Remove("reusable-turn-parent"); } + }); + Assert.Equal(DeliveryStatus.Accepted, (await accepted).Status); + Assert.Equal(2, rig.Inbox.Count); + var drained = events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer) + && rig.Inbox.Count == 0 && rig.Effects.Count == 2, + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + handler.Release.TrySetResult(); + await drained; + Assert.Equal(2, rig.Effects.Count); + Assert.Equal(2, rig.Processed.Count); + Assert.Single(events.Events.Select(item => item.Payload).OfType(), + created => ReferenceEquals(created.GrainContext, rig.Context) && IsLocalDrainTimer(created.Timer)); + Assert.Equal(2, events.Events.Select(item => item.Payload).OfType() + .Count(stopped => ReferenceEquals(stopped.Timer, timer))); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + [Fact] + public async Task Complete_StagesBusinessOutputAndDedupeBeforeHandlerReturn() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, token) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + AssertCompletedState(rig, self.Context.Message); + Assert.Equal(1, Assert.Single(rig.Effects).Value.Count); + Assert.Single(rig.Outbox); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockAcknowledgement(rig.Journal); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + Assert.False(finished.IsCompleted); + storage.Release(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 1); + await DeactivateAsync(rig); + var recovered = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + + [Fact] + public async Task Complete_PrecedingWriterCapturesCompletionWithBusinessAcrossReturnContinuation() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var writes = Writes(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var preceding = OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "prior"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + await storage.WaitUntilEnteredAsync(); + var priorCapture = rig.Grain.Captures[^1]; + Assert.Empty(priorCapture.Effects); + Assert.Equal(1, priorCapture.InboxCount); + Assert.Equal(0, priorCapture.ProcessedMessageCount); + Assert.Equal(0, priorCapture.OutboxCount); + var queued = OnTurnAsync(rig.Context, async () => await rig.Manager.WriteStateAsync(CancellationToken.None)); + var staged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var returnContinuation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + handler.Body = async (self, _) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + AssertCompletedState(rig, input); + staged.TrySetResult(); + await returnContinuation.Task; + }; + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + try + { + await WaitAsync(staged.Task); + Assert.False(finished.IsCompleted); + storage.Release(); + await WaitAsync(Task.WhenAll(preceding, queued)); + Assert.False(finished.IsCompleted); + Assert.Equal(writes + 2, Writes(rig)); + var capturedEffects = Assert.Single(rig.Grain.Captures, snapshot => snapshot.Effects.Count != 0); + Assert.Equal(0, capturedEffects.InboxCount); + Assert.Equal(1, capturedEffects.ProcessedMessageCount); + Assert.Equal(1, capturedEffects.OutboxCount); + Assert.Equal(new MessageEffect(input.Envelope.MessageId, 1, 501, "async-handler"), Assert.Single(capturedEffects.Effects)); + } + finally + { + returnContinuation.TrySetResult(); + } + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 1); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandlerFailure_BeforeVsAfterCompleteUsesActualLogicalOutcome(bool completed) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var error = new IOException("Handler outcome."); + handler.Body = (self, _) => + { + if (completed) + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + } + throw error; + }; + var input = await DeliverAsync(rig); + var timer = GetTimer(events, rig); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + var stop = (GrainTimerEvents.TickStop)(await TimerStoppedAsync(events, timer)).Payload!; + if (completed) + { + Assert.Same(error, stop.Exception); + AssertSuccess(rig, input, outputCount: 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + else + { + Assert.Null(stop.Exception); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters).MessageId); + } + await AssertHealthyAsync(rig); + await DeactivateAsync(rig); + var replay = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(completed ? 1 : 0, replay.Effects.Count); + Assert.Equal(1, replay.ProcessedMessageCount); + Assert.Equal(completed ? 0 : 1, replay.InboxDeadLetters.Count); + } + + [Fact] + public async Task SuccessfulReturnWithoutComplete_IsExplicitTerminalMisuse() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (_, _) => ValueTask.CompletedTask; + var input = await DeliverAsync(rig); + var writes = Writes(rig); + handler.Release.TrySetResult(); + var error = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); + Assert.Contains("must call Complete", error.Message, StringComparison.Ordinal); + Assert.Equal(writes, Writes(rig)); + await rig.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Inbox).Key); + Assert.Equal(input.Envelope.Payload, Assert.Single(rig.Inbox).Value.Envelope.Payload); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Effects); + await AssertFailureReplayAsync(rig, input); + } + + [Fact] + public async Task RepeatedComplete_SameActiveAttemptStagesExactlyOnce() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + self.Mutate(); + self.Context.Complete(); + var timestamp = Assert.Single(rig.Processed).Value; + self.Context.Complete(); + Assert.Equal(timestamp, Assert.Single(rig.Processed).Value); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData("complete")] + public async Task RetainedContext_AfterRetirementRejectsAndKeepsOwnerHealthy(string operation) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + var error = await Assert.ThrowsAsync(() => + OnTurnAsync(rig.Context, () => InvokeContextOperation(handler, operation))); + Assert.Contains("inactive or different attempt", error.Message, StringComparison.Ordinal); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData("complete")] + public async Task RetainedContext_DuringOtherAttemptRetainsFirstMisuse(string operation) + { + var rig = await CreateAsync(); + using var first = rig.Handler; + var one = await DeliverAsync(rig); + var finished = await FinishedAsync(rig); + first.Release.TrySetResult(); + await WaitAsync(finished); + using var second = new TestHandler(rig.Effects, rig.Context); + InvalidOperationException? rejection = null; + second.Body = (_, _) => + { + try { InvokeContextOperation(first, operation); } + catch (InvalidOperationException error) { rejection = error; } + throw new IOException("Replacement failure."); + }; + await OnTurnAsync(rig.Context, () => + rig.Grain.HandlerOverride = second); + var two = CreateEnvelope(rig.Receiver, NewMessage(502, "next"), "async/next"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, two)).Status); + await WaitAsync(second.Entered.Task); + second.Release.TrySetResult(); + var failure = await WaitAsync(rig.Grain.DeactivationFailure.Task); + Assert.Same(Assert.IsType(rejection), failure); + Assert.Equal(1, Assert.Single(rig.Effects).Value.Count); + await WaitAsync(rig.Context.Deactivated); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DurableJobCancellation_BeforeCompleteRetainsExactOwnerAndRetries(bool cancellationCheck) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var localEvents = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var input = await DeliverAsync(rig); + var localTimer = GetTimer(localEvents, rig); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var resume = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retry = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var attempts = 0; + OperationCanceledException? expected = null; + handler.Body = async (self, token) => + { + if (++attempts > 1) + { + await retry.Task.WaitAsync(token); + self.Mutate(); + self.Context.Complete(); + return; + } + entered.TrySetResult(); + try + { + if (cancellationCheck) + { + await resume.Task; + token.ThrowIfCancellationRequested(); + } + else await resume.Task.WaitAsync(token); + } + catch (OperationCanceledException error) { expected = error; throw; } + }; + var snapshot = rig.Grain.GetSnapshotForTest(); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, rig.Receiver.GetGrainId())); + var run = new JobContext(job); + var feature = (IDurableJobFeatureHandler)rig.Context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + using var jobEvents = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + using var cancellation = new CancellationTokenSource(); + var start = new StartAtTimerStop(rig, localTimer, feature, run, cancellation.Token, jobEvents); + using var subscription = GrainTimerEvents.AllEvents.Subscribe(start); + await OnTurnAsync(rig.Context, localTimer.Dispose); + var timer = await WaitAsync(start.Started.Task); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + handler.Release.TrySetResult(); + try + { + await WaitAsync(entered.Task); + await OnTurnAsync(rig.Context, cancellation.Cancel); + } + finally + { + resume.TrySetResult(); + } + await TimerStoppedAsync(jobEvents, timer); + var failure = await Assert.ThrowsAnyAsync(() => + OnTurnAsync(rig.Context, async () => await feature.ExecuteJobAsync(run, TestContext.Current.CancellationToken))); + Assert.Same(expected, failure); + Assert.Single(rig.Inbox); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Effects); + Assert.Equal(snapshot.InboxJobId, rig.Grain.GetSnapshotForTest().InboxJobId); + Assert.Same(snapshot.InboxJob, rig.Grain.GetSnapshotForTest().InboxJob); + await AssertHealthyAsync(rig); + var duplicate = DeliverAsync(rig.Receiver, input); + retry.TrySetResult(); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + AssertSuccess(rig, input, outputCount: 0); + Assert.Same(rig.Context, Fixture.GetGrainContext(rig.Receiver)); + } + + [Fact] + public async Task CanceledJobCallback_LeavesActiveHandlerAndCommittedInboxUndisturbed() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var snapshot = rig.Grain.GetSnapshotForTest(); + var writes = Writes(rig); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, rig.Receiver.GetGrainId())); + var feature = (IDurableJobFeatureHandler)rig.Context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var canceled = new CancellationToken(canceled: true); + var error = await Assert.ThrowsAnyAsync(() => OnTurnAsync(rig.Context, async () => + await feature.ExecuteJobAsync(new JobContext(job), canceled))); + Assert.Equal(canceled, error.CancellationToken); + Assert.Equal(writes, Writes(rig)); + Assert.Single(rig.Inbox); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Processed); + Assert.Equal(snapshot.InboxJobId, rig.Grain.GetSnapshotForTest().InboxJobId); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 0); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CancellationInFinalBlock_CompleteAndOwnedWritePreserveOutcome(bool failWrite) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + IGrainTimer timer = null!; + handler.Body = (self, token) => + { + self.Mutate(); + timer.Dispose(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + timer = GetTimer(events, rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + AssertCompletedState(rig, input); + Assert.False(finished.IsCompleted); + var failure = new OperationCanceledException("Actual storage canceled.", new CancellationToken(canceled: true)); + if (failWrite) storage.Fail(failure); + else storage.Release(); + await WaitAsync(finished); + if (failWrite) + { + Assert.Same(failure, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + } + else + { + AssertSuccess(rig, input, outputCount: 1); + await AssertHealthyAsync(rig); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletionWrite_HookOutcomePreservesActualCommit(bool postCommit) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var input = await DeliverAsync(rig); + var error = new IOException("Journal hook failed."); + await OnTurnAsync(rig.Context, () => rig.Manager.Hooks.Add(new FailingHook(postCommit, error))); + var timer = GetTimer(events, rig); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + if (postCommit) + { + Assert.IsType( + ((GrainTimerEvents.TickStop)(await TimerStoppedAsync(events, timer)).Payload!).Exception); + AssertSuccess(rig, input, outputCount: 0); + await OnTurnAsync(rig.Context, rig.Manager.Hooks.Clear); + await AssertHealthyAsync(rig); + } + else + { + var failure = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); + Assert.Same(error, failure.InnerException); + await AssertFailureReplayAsync(rig, input); + } + } + + [Fact] + public async Task PostCompleteHandlerFailure_WithStorageFailureKeepsStorageCauseAuthoritative() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var handlerError = new InvalidOperationException("After Complete."); + handler.Body = (self, _) => { self.Mutate(); self.Context.Complete(); throw handlerError; }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = await FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + var storageError = new IOException("Storage failed."); + storage.Fail(storageError); + await WaitAsync(finished); + Assert.Same(storageError, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + } + + private async Task CreateAsync() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var services = context.ActivationServices; + var effects = services.GetRequiredKeyedService>("test-effects"); + var handler = new TestHandler(effects, context); + var rig = new Rig(receiver, context, Assert.IsType(context.GrainInstance), + services.GetRequiredService(), (JournaledTestOutbox)services.GetRequiredService(), + effects, handler, + services.GetRequiredKeyedService>("__orleans.messaging.inbox"), + services.GetRequiredKeyedService>("__orleans.messaging.inbox-processed")); + await OnTurnAsync(context, () => rig.Grain.HandlerOverride = handler); + return rig; + } + + private async Task DeliverAsync(Rig rig) + { + var input = CreateEnvelope(rig.Receiver, NewMessage(501, "async-handler"), "async/handler"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, input)).Status); + await WaitAsync(rig.Handler.Entered.Task); + return input; + } + + private static void InvokeContextOperation(TestHandler handler, string operation) + { + switch (operation) + { + case "complete": handler.Context.Complete(); break; + default: throw new ArgumentException(nameof(operation)); + } + } + + private static void AssertCompletedState(Rig rig, InboxMessage input) + { + Assert.Empty(rig.Inbox); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Processed).Key); + } + + private static void AssertSuccess(Rig rig, InboxMessage input, int outputCount) + { + AssertCompletedState(rig, input); + Assert.Equal(new MessageEffect(input.Envelope.MessageId, 1, 501, "async-handler"), Assert.Single(rig.Effects).Value); + Assert.Equal(outputCount, rig.Outbox.Count); + Assert.Empty(rig.Grain.GetSnapshotForTest().InboxDeadLetters); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + private int Writes(Rig rig) => Fixture.Storage.GetSuccessfulWriteCount(rig.Journal); + private static async Task AssertHealthyAsync(Rig rig) + { + await OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "healthy"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + private static async Task DeactivateAsync(Rig rig) + { + await rig.Receiver.RequestDeactivationAsync(); + await WaitAsync(rig.Context.Deactivated); + } + + private async Task AssertFailureReplayAsync(Rig rig, InboxMessage input) + { + await WaitAsync(rig.Context.Deactivated); + await rig.Receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForDeadLetterCountAsync(rig.Receiver, 1); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(input.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + private static async Task FinishedAsync(Rig rig) + { + Task result = null!; + await OnTurnAsync(rig.Context, () => + { + var extension = rig.Context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var pending = CancellationCleanupProbe.Field(extension, "_pendingWrites"); + var operation = Assert.Single(pending.Cast(), item => item.GetType().Name == "HandlerWrite"); + result = ((TaskCompletionSource)operation.GetType().GetProperty("Finished")!.GetValue(operation)!).Task; + }); + return result; + } + + private static IGrainTimer GetTimer(DiagnosticEventCollector events, Rig rig) => + Assert.Single(events.Events.Select(item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, rig.Context) && IsLocalDrainTimer(item.Timer)).Timer; + private static IGrainTimer GetPumpTimer(DiagnosticEventCollector events, Rig rig) => + Assert.Single(events.Events.Select(item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, rig.Context) && IsInboxTimer(item.Timer) + && !IsLocalDrainTimer(item.Timer)).Timer; + private static bool IsLocalDrainTimer(IGrainTimer timer) => IsInboxTimer(timer) + && timer.GetType().GenericTypeArguments[0].Name == "LocalDrainTimerState"; + private static bool IsInboxTimer(IGrainTimer timer) => timer.GetType().GenericTypeArguments is [var type] + && type.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension"); + private static Task TimerStoppedAsync(DiagnosticEventCollector events, IGrainTimer timer) => + events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = CompleteAsync(); }); + return done.Task; + async Task CompleteAsync() + { + try + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + await action(); + done.SetResult(); + } + catch (Exception error) { done.SetException(error); } + } + } + + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, MessagingTestGrain Grain, + IJournaledStateManager Manager, JournaledTestOutbox Outbox, IDurableDictionary Effects, + TestHandler Handler, IDurableDictionary Inbox, + IDurableDictionary Processed) + { + public JournalId Journal => JournalId.FromGrainId(Receiver.GetGrainId()); + } + + private sealed class TestHandler(IDurableDictionary effects, IGrainContext grainContext) : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public IInboxHandlerContext Context { get; private set; } = null!; + public OutboxMessage Output { get; private set; } + public Func Body { get; set; } = + static (self, _) => { self.Mutate(); self.Context.Complete(); return default; }; + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + Context = context; + + Output = TestApplicationProtocol.Create(grainContext.ActivationServices.GetRequiredService(), grainContext.GrainId, grainContext.GrainId, "output", 41); + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + await Body(this, cancellationToken); + } + public void Mutate() + { + effects.TryGetValue(Context.Message.Envelope.MessageId, out var previous); + effects[Context.Message.Envelope.MessageId] = + new MessageEffect(Context.Message.Envelope.MessageId, (previous?.Count ?? 0) + 1, 501, "async-handler"); + } + public void Dispose() + { + Release.TrySetResult(); + + } + } + private sealed class JobContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + + private sealed class FailingHook(bool postCommit, Exception error) : IJournaledStateHook + { + public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (!postCommit) throw error; + return default; + } + + public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (postCommit) throw error; + return default; + } + } + + private sealed class StartAtTimerStop(Rig rig, IGrainTimer preceding, IDurableJobFeatureHandler feature, + JobContext run, CancellationToken cancellationToken, DiagnosticEventCollector events) + : IObserver + { + private bool _started; + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void OnNext(GrainTimerEvents.TimerEvent value) + { + if (_started || value is not GrainTimerEvents.TickStop stopped + || !ReferenceEquals(stopped.Timer, preceding)) + { + return; + } + _started = true; + try + { + Assert.Same(rig.Context, ReceiverTestServices.CurrentGrainContext); + var result = feature.ExecuteJobAsync(run, cancellationToken); + Assert.True(result.IsCompletedSuccessfully); + Assert.True(result.GetAwaiter().GetResult().IsInProgress); + Started.SetResult(GetPumpTimer(events, rig)); + } + catch (Exception error) + { + Started.SetException(error); + } + } + public void OnCompleted() { } + public void OnError(Exception error) => Started.TrySetException(error); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs new file mode 100644 index 00000000000..ec2d4ed9bd7 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs @@ -0,0 +1,150 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHandlerTransactionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task HandlerSuccess_CommitsEffectCompletionDedupeAndStagedOutputAtomically() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var logicalId = TestApplicationProtocol.NewMessageId(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(logicalId, 7, "atomic", sink.GetGrainId())); + + var result = await DeliverAsync(receiver, envelope); + var receiverState = await Fixture.WaitForEffectCountAsync(receiver, 1); + var output = Fixture.GetStagedOutput(receiver); + + Assert.Equal(DeliveryStatus.Accepted, result.Status); + var effect = Assert.Single(receiverState.Effects); + Assert.Equal(new MessageEffect(logicalId, 1, 7, "atomic"), effect); + Assert.Equal(0, receiverState.InboxCount); + Assert.Equal(1, receiverState.OutboxCount); + var outgoing = Assert.Single(output); + Assert.Equal(sink.GetGrainId(), outgoing.ReceiverId); + Assert.Equal("messages/forwarded", TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), outgoing).Route); + var body = TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), outgoing).Body; + Assert.Equal(new TestMessage(logicalId, 7, "atomic"), body); + Assert.Equal(1, receiverState.ProcessedMessageCount); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(receiverState.ActivationId, recovered.ActivationId); + Assert.Equal(effect, Assert.Single(recovered.Effects)); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(outgoing.Envelope.MessageId, Assert.Single(Fixture.GetStagedOutput(receiver)).Envelope.MessageId); + + var duplicate = await DeliverAsync(receiver, envelope); + Assert.Equal(DeliveryStatus.Duplicate, duplicate.Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + Assert.Single(Fixture.GetStagedOutput(receiver)); + } + + [Fact] + public async Task HandlerPreparationFailure_DeadLettersWithoutStagingEffectsOrOutput() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), 9, "preparation-failure", sink.GetGrainId(), ThrowDuringPreparation: true)); + + var accepted = await DeliverAsync(receiver, envelope); + var state = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Equal(DeliveryStatus.Accepted, accepted.Status); + Assert.Empty(state.Effects); + Assert.Equal(0, state.InboxCount); + Assert.Equal(0, state.OutboxCount); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal(1, deadLetter.AttemptCount); + Assert.Contains("Injected handler preparation failure", deadLetter.Reason, StringComparison.Ordinal); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(state.ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + [Fact] + public async Task HandlerPreparation_OrdinaryWriteLeavesEffectsLocalUntilApplyAndCompletion() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(10, "premature-commit") with { CommitDuringHandling = true }); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal("premature-commit", Assert.Single(completed.Effects).Value); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(oldGrain.DeactivationFailure.Task.IsCompleted); + Assert.Same(oldContext, Fixture.GetGrainContext(receiver)); + } + + [Fact] + public async Task HandlerCompletionWriteFailure_FencesOldManagerAndFreshActivationRetries() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/completion-failure"); + var envelope = CreateEnvelope(receiver, NewMessage(78, "completion-failure"), "messages/completion-failure"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Assert.ThrowsAnyAsync(() => oldManager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId != oldGrain.GetSnapshotForTest().ActivationId + && snapshot.Effects.Sum(static effect => effect.Count) == 1); + Assert.NotEqual(oldGrain.GetSnapshotForTest().ActivationId, completed.ActivationId); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task PreparationFailureAccountingWriteFailure_RecoversInFreshActivation() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/failure-accounting"); + var envelope = CreateEnvelope(receiver, NewMessage(79, "failure-accounting") with { ThrowDuringPreparation = true }, "messages/failure-accounting"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, completed.ActivationId); + Assert.Equal(1, Assert.Single(completed.InboxDeadLetters).AttemptCount); + Assert.Empty(completed.Effects); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs new file mode 100644 index 00000000000..7056ae69a10 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -0,0 +1,280 @@ +using System.Text; +using System.Runtime.InteropServices; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHeaderBehaviorTests : MessagingBehaviorTestBase +{ + public InboxHeaderBehaviorTests() : base(new HeaderFixture()) { } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RawMessage_AbsentOptionalMetadataCommitsAndDeduplicates(bool empty) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + byte[] payload = empty ? [] : [0, 255, 128, 42]; + var message = Raw(HierarchicalKey.Create("raw", Guid.NewGuid().ToString("N")), payload); + Assert.False(message.Envelope.TryGetSubject(out _)); + Assert.False(message.Envelope.TryGetBytes(MessageHeaders.Sender, out _)); + using var handler = new RawHandler(context); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + using var storage = Fixture.Storage.BlockAcknowledgement(JournalId.FromGrainId(context.GrainId)); + var delivery = DeliverAsync(receiver, message); + await storage.WaitUntilEnteredAsync(); + Assert.False(delivery.IsCompleted); + Assert.False(handler.Entered.Task.IsCompleted); + storage.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await WaitAsync(handler.Entered.Task); + Assert.Equal(payload, handler.Message.Envelope.Payload.ToArray()); + handler.Release.TrySetResult(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(message.Envelope.MessageId, Assert.Single(completed.Effects).LogicalId); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + await receiver.RequestDeactivationAsync(); + await WaitAsync(context.Deactivated); + var replayed = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, replayed.ActivationId); + Assert.Equal(1, Assert.Single(replayed.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + } + + [Fact] + public async Task PendingDuplicate_ChangedCustomHeadersAndSenderPreservesOriginal() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var key = HierarchicalKey.Create("custom", Guid.NewGuid().ToString("N")); + var original = Raw(key, [1, 2, 3], "original", GrainId.Create("producer", "first")); + var changed = Raw(key, [1, 2, 3], "changed", GrainId.Create("forwarder", "second")); + using var handler = new RawHandler(context); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var pending = context.ActivationServices.GetRequiredService(); + Envelope first = default; + await OnTurnAsync(context, () => first = Assert.Single(pending.Messages).Envelope); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); + var schedules = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, context.GrainId); + await OnTurnAsync(context, async () => + { + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(changed)).Status); + var retained = Assert.Single(pending.Messages).Envelope; + Assert.Equal(first.EncodedHeaders, retained.EncodedHeaders); + Assert.True(retained.TryGetBytes("custom.trace", out var trace)); + Assert.Equal("original", Encoding.UTF8.GetString(trace.Span)); + Assert.Equal(GrainId.Create("producer", "first"), TestApplicationProtocol.Sender(Sessions, retained)); + }); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); + Assert.Equal(schedules, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, context.GrainId)); + handler.Release.TrySetResult(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + [Fact] + public async Task UnknownHeaders_RecoverAndRemainAvailableAfterRetryAndDeadLetter() + { + var receiver = NewGrain(); + var key = HierarchicalKey.Create("headers", Guid.NewGuid().ToString("N")); + var body = NewMessage(810, "poison") with { LogicalId = key, ThrowDuringPreparation = true }; + var application = CreateEnvelope(receiver, body); + using var builder = new EnvelopeBuilder(key); + foreach (var header in application.Envelope.Keys) + { + Assert.True(application.Envelope.TryGetBytes(header, out var bytes)); + builder.AddBytes(header, bytes.Span); + } + byte[] opaque = [0xff, 0, 0x80, 0x7f]; + builder.AddBytes("future.opaque", opaque); + builder.AddValue("future.number", 8675309, new Serializer(Sessions)); + var message = new InboxMessage { Envelope = builder.Build() }; + var retry = Fixture.Metrics.WaitForCountAsync("orleans-messaging-inbox-messages-processed", 1, "retry"); + await receiver.SeedInboxStateAsync(message, null, null); + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(previous.Deactivated); + _ = await receiver.GetSnapshotAsync(); + var current = Fixture.GetGrainContext(receiver); + Assert.NotSame(previous, current); + await retry; + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.InboxDeadLetters); + await OnTurnAsync(current, () => + { + var stored = Assert.Single(current.ActivationServices.GetRequiredService().Messages).Envelope; + Assert.True(stored.TryGetBytes("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes.ToArray()); + Assert.True(stored.TryGetValue("future.number", new Serializer(Sessions), out var number)); + Assert.Equal(8675309, number); + }); + Fixture.Clock.Advance(TimeSpan.FromHours(1)); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + _ = await receiver.GetSnapshotAsync(); + current = Fixture.GetGrainContext(receiver); + var dead = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Equal(key, Assert.Single(dead.InboxDeadLetters).MessageId); + Assert.Equal(2, Assert.Single(dead.InboxDeadLetters).AttemptCount); + Assert.Empty(dead.Effects); + var diagnostics = current.ActivationServices.GetRequiredService(); + await OnTurnAsync(current, () => + { + var retained = Assert.Single(diagnostics.InboxDeadLetters).Message.Envelope; + Assert.True(retained.TryGetBytes("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes.ToArray()); + Assert.True(retained.TryGetValue("future.number", new Serializer(Sessions), out var number)); + Assert.Equal(8675309, number); + Assert.Equal(application.Envelope.Payload.ToArray(), retained.Payload.ToArray()); + }); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + _ = await receiver.GetSnapshotAsync(); + var recovered = Fixture.GetGrainContext(receiver); + await OnTurnAsync(recovered, () => + { + var retained = Assert.Single(recovered.ActivationServices.GetRequiredService() + .InboxDeadLetters).Message.Envelope; + Assert.True(retained.TryGetBytes("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes.ToArray()); + Assert.True(retained.TryGetValue("future.number", new Serializer(Sessions), out var number)); + Assert.Equal(8675309, number); + }); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, Raw(key, []))).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MalformedMessage_RejectsBeforeSchedulingOrMutation(bool completed) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var key = HierarchicalKey.Create("malformed", Guid.NewGuid().ToString("N")); + var message = Raw(key, [1, 2]); + if (completed) + { + await OnTurnAsync(context, async () => + { + context.ActivationServices.GetRequiredKeyedService>( + "__orleans.messaging.inbox-processed").Add(key, Fixture.Clock.GetUtcNow()); + await context.ActivationServices.GetRequiredService().WriteStateAsync(CancellationToken.None); + }); + } + Assert.True(MemoryMarshal.TryGetArray(message.Envelope.EncodedHeaders, out var encoded)); + encoded.Array![encoded.Offset] = 255; // Represents malformed external/generated input. + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); + await OnTurnAsync(context, async () => + { + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + await Assert.ThrowsAsync(() => extension.DeliverAsync(message, TestContext.Current.CancellationToken).AsTask()); + }); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, context.GrainId)); + var snapshot = await receiver.GetSnapshotAsync(); + Assert.Equal(0, snapshot.InboxCount); + Assert.Equal(completed ? 1 : 0, snapshot.ProcessedMessageCount); + Assert.Empty(snapshot.Effects); + } + + [Fact] + public async Task MalformedRecoveredHeaders_FailBeforeSchedulingOrBusinessMutation() + { + var receiver = NewGrain(); + var key = HierarchicalKey.Create("recovery", Guid.NewGuid().ToString("N")); + var message = Raw(key, []); + Assert.True(MemoryMarshal.TryGetArray(message.Envelope.EncodedHeaders, out var encoded)); + encoded.Array![encoded.Offset] = 255; + await receiver.SeedInboxStateAsync(message, null, null); + var previous = Fixture.GetGrainContext(receiver); + var journal = JournalId.FromGrainId(previous.GrainId); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + await receiver.RequestDeactivationAsync(); + await WaitAsync(previous.Deactivated); + var failure = await Assert.ThrowsAnyAsync(() => receiver.GetSnapshotAsync()); + Assert.Contains("envelope format", failure.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, previous.GrainId)); + } + + private InboxMessage Raw(HierarchicalKey key, byte[] payload, string? trace = null, GrainId? sender = null) + { + using var builder = new EnvelopeBuilder(key); + builder.AddBytes(MessageHeaders.Payload, payload); + if (trace is not null) builder.AddBytes("custom.trace", Encoding.UTF8.GetBytes(trace)); + if (sender is { } value) builder.AddValue(MessageHeaders.Sender, value, new Serializer(Sessions)); + return new() { Envelope = builder.Build() }; + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var result = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = Run(); }); + return result.Task; + async Task Run() + { + try { await action(); result.SetResult(); } + catch (Exception exception) { result.SetException(exception); } + } + } + + private static Task WaitAsync(Task task) => + task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + private sealed class RawHandler(IGrainContext context) : IInboxHandler, IDisposable + { + public InboxMessage Message { get; private set; } + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async ValueTask HandleAsync(IInboxHandlerContext handler, CancellationToken cancellationToken) + { + Message = handler.Message; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + var key = handler.Message.Envelope.MessageId; + var effects = context.ActivationServices.GetRequiredKeyedService>("test-effects"); + effects[key] = new(key, 1, 800, "raw"); + handler.Complete(); + } + + public void Dispose() => Release.TrySetResult(); + } + + private sealed class HeaderFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxProcessingAttempts = 2; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs new file mode 100644 index 00000000000..6ae704bf821 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs @@ -0,0 +1,473 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxIdentityTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task PendingCommand_NewSenderCoalescesAndPreservesOriginalEnvelope() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1042", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = TestApplicationProtocol.With(original, Sessions, sender: GrainId.Create("forwarder", "second")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId); + ReadOnlyMemory storedPayload = default; + await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Envelope.Payload); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, repeated)).Status); + await OnTurnAsync(rig.Context, () => + { + var inbox = rig.Context.ActivationServices.GetRequiredService(); + Assert.True(inbox.TryGetMessage(key, out var stored)); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, stored.Envelope)); + Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(stored.Envelope)); + Assert.Equal(storedPayload, stored.Envelope.Payload); + Assert.Equal(original.Envelope.Payload.ToArray(), stored.Envelope.Payload.ToArray()); + Assert.Single(rig.Pending); + Assert.Empty(rig.Processed); + }); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId)); + handler.Release.TrySetResult(); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, handler.Calls); + Assert.Equal(key, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).LogicalId); + Assert.Equal(1, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).Count); + } + + [Theory] + [InlineData("subject")] + [InlineData("subject-case")] + [InlineData("body")] + public async Task PendingCommand_ConflictingContentRejectsBeforeSchedulingOrMutation(string difference) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1043", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var conflictingSubject = difference switch + { + "subject" => "inventory.release.v1", + "subject-case" => "Inventory.reserve.v1", + _ => Assert.IsType(TestApplicationProtocol.Subject(original.Envelope)) + }; + var replacement = Create(rig, key, conflictingSubject, "different-parameters", + GrainId.Create("forwarder", "other")); + var conflict = difference == "body" ? replacement : TestApplicationProtocol.With(original, Sessions, + sender: TestApplicationProtocol.Sender(Sessions, replacement.Envelope), subject: conflictingSubject); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var writes = Writes(rig); + var job = rig.Grain.GetSnapshotForTest().InboxJob; + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId); + ReadOnlyMemory storedPayload = default; + await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Envelope.Payload); + var failure = await Assert.ThrowsAsync(() => DeliverOnTurnAsync(rig, conflict)); + Assert.Contains(key.ToString(), failure.Message, StringComparison.Ordinal); + Assert.Contains("different command", failure.Message, StringComparison.Ordinal); + await OnTurnAsync(rig.Context, () => + { + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, Assert.Single(rig.Pending).Value.Envelope)); + Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(Assert.Single(rig.Pending).Value.Envelope)); + Assert.Equal(storedPayload, Assert.Single(rig.Pending).Value.Envelope.Payload); + Assert.Equal(original.Envelope.Payload.ToArray(), Assert.Single(rig.Pending).Value.Envelope.Payload.ToArray()); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().Effects); + Assert.Same(job, rig.Grain.GetSnapshotForTest().InboxJob); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + }); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId)); + await OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "healthy"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, original)).Status); + handler.Release.TrySetResult(); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, handler.Calls); + Assert.Equal(1, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletedCommand_ChangedSenderSubjectAndBodyAcknowledgesWithoutFingerprint(bool restart) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1044", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = Create(rig, key, "completely.changed.subject.v2", "different-body", + GrainId.Create("forwarder", "new-owner")); + handler.Release.TrySetResult(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + await Fixture.SnapshotProbe.WaitAsync(rig.Receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + var prior = rig.Context; + if (restart) + { + await rig.Receiver.RequestDeactivationAsync(); + await WaitAsync(prior.Deactivated); + await rig.Receiver.GetSnapshotAsync(); + Assert.NotSame(prior, Fixture.GetGrainContext(rig.Receiver)); + } + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, repeated)).Status); + var completed = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId)); + } + + [Fact] + public async Task ExactParentChildAndSiblingCommandsHaveIndependentCompletions() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var parent = HierarchicalKey.Create("tenant", "orders", "1045"); + var keys = new[] { parent, parent.CreateChildKey("reserve"), parent.CreateChildKey("release") }; + for (var index = 0; index < keys.Length; index++) + { + var command = Create(rig, keys[index], "inventory.command.v1", $"command-{index}"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, command)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, index + 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, command)).Status); + } + var completed = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(3, completed.Effects.Count); + Assert.Equal(3, completed.ProcessedMessageCount); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + Assert.Equal(keys.OrderBy(static key => key.ToString(), StringComparer.Ordinal), + completed.Effects.Select(static effect => effect.LogicalId).OrderBy(static key => key.ToString(), StringComparer.Ordinal)); + } + + [Fact] + public async Task SameCommandKey_InDifferentReceiversCompletesIndependently() + { + var first = await CreateAsync(); + var second = await CreateAsync(); + using var firstHandler = first.Handler; + using var secondHandler = second.Handler; + firstHandler.Release.TrySetResult(); + secondHandler.Release.TrySetResult(); + var key = HierarchicalKey.Create("tenant", "orders", "1049", "reserve"); + var one = Create(first, key, "inventory.reserve.v1", "first-receiver"); + var two = Create(second, key, "inventory.reserve.v1", "second-receiver"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(first.Receiver, one)).Status); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(second.Receiver, two)).Status); + var firstOutcome = await Fixture.WaitForEffectCountAsync(first.Receiver, 1); + var secondOutcome = await Fixture.WaitForEffectCountAsync(second.Receiver, 1); + Assert.Equal(key, Assert.Single(firstOutcome.Effects).LogicalId); + Assert.Equal(key, Assert.Single(secondOutcome.Effects).LogicalId); + Assert.Equal(1, firstOutcome.ProcessedMessageCount); + Assert.Equal(1, secondOutcome.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(first.Receiver, one)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(second.Receiver, two)).Status); + Assert.Equal(1, firstHandler.Calls); + Assert.Equal(1, secondHandler.Calls); + } + + [Fact] + public async Task PendingIdentityAfterReplayStillRejectsConflictAndDeduplicatesNewSender() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + using var timers = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var key = HierarchicalKey.Create("tenant", "orders", "1046", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var timer = Assert.Single(timers.Events.Select(static item => item.Payload).OfType(), + item => ReferenceEquals(item.GrainContext, rig.Context) + && item.Timer.GetType().GenericTypeArguments is [var type] + && type.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension") + && type.Name == "LocalDrainTimerState").Timer; + await OnTurnAsync(rig.Context, () => + { + timer.Dispose(); + rig.Context.Deactivate(new DeactivationReason( + DeactivationReasonCode.ApplicationRequested, "Verify pending command identity replay."), + TestContext.Current.CancellationToken); + }); + await WaitAsync(rig.Context.Deactivated); + using var recoveredHandler = new IdentityHandler(); + using var read = Fixture.Storage.BlockRead(JournalId.FromGrainId(rig.Context.GrainId)); + var activation = rig.Receiver.GetSnapshotAsync(); + await read.WaitUntilEnteredAsync(); + var recoveredContext = Fixture.GetGrainContext(rig.Receiver); + var recoveredGrain = Assert.IsType(recoveredContext.GrainInstance); + await OnTurnAsync(recoveredContext, () => recoveredGrain.HandlerOverride = recoveredHandler); + read.Release(); + await WaitAsync(recoveredHandler.Entered.Task); + var recovered = CreateRig(rig.Receiver, recoveredContext, recoveredHandler); + Assert.NotSame(rig.Context, recovered.Context); + var repeated = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("forwarder", "after-replay")); + var conflict = Create(recovered, key, "inventory.reserve.v1", "changed", + GrainId.Create("forwarder", "after-replay")); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(recovered, repeated)).Status); + await Assert.ThrowsAsync(() => DeliverOnTurnAsync(recovered, conflict)); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, Assert.Single(recovered.Pending).Value.Envelope)); + Assert.Empty(recovered.Processed); + Assert.False(recovered.Grain.DeactivationFailure.Task.IsCompleted); + recoveredHandler.Release.TrySetResult(); + await activation; + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + } + + [Fact] + public async Task DeadLetterInspectionAndRemovalUseReceiverLocalKeyAcrossReplay() + { + var receiver = NewGrain(); + var key = HierarchicalKey.Create("tenant", "orders", "1048", "invalid-command"); + var body = new TestMessage(key, 611, "poison", ThrowDuringPreparation: true); + var original = CreateEnvelope(receiver, body); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + var deadLetter = Assert.Single((await Fixture.WaitForDeadLetterCountAsync(receiver, 1)).InboxDeadLetters); + Assert.Equal(key, deadLetter.MessageId); + var old = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(old.Deactivated); + var recovered = await receiver.GetSnapshotAsync(); + Assert.Equal(key, Assert.Single(recovered.InboxDeadLetters).MessageId); + var duplicate = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("forwarder", "deadletter"), subject: "another.subject.v1"); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, duplicate)).Status); + Assert.True(await receiver.RemoveInboxDeadLetterAsync(key)); + Assert.False(await receiver.RemoveInboxDeadLetterAsync(key)); + var current = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, duplicate)).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MissingOptionalMetadata_RawHandlerCompletesWithoutSubject(bool removeSender) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "invalid-subject", "command"); + var envelope = Create(rig, key, "valid.subject.v1", "missing-subject"); + var raw = TestApplicationProtocol.With(envelope, Sessions, removeSubject: true, removeSender: removeSender); + var bytes = envelope.Envelope.Payload.ToArray(); + handler.Release.TrySetResult(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverOnTurnAsync(rig, raw)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(raw.Envelope.TryGetSubject(out _)); + if (removeSender) Assert.False(raw.Envelope.TryGetBytes(MessageHeaders.Sender, out _)); + Assert.Equal(bytes, raw.Envelope.Payload.ToArray()); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, raw)).Status); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + [Theory] + [InlineData("key-bytes")] + [InlineData("key-utf8")] + [InlineData("key-depth")] + [InlineData("subject-bytes")] + [InlineData("subject-utf8")] + public async Task OversizedMetadata_RejectsBeforeSchedulingOrPersistence(string field) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = field switch + { + "key-bytes" => HierarchicalKey.Create(new string('x', 1025)), + "key-utf8" => HierarchicalKey.Create(new string('\u00e9', 513)), + "key-depth" => HierarchicalKey.Create(Enumerable.Repeat("segment", 33).ToArray()), + _ => HierarchicalKey.Create("tenant", "bounded", "command") + }; + var subject = field switch + { + "subject-bytes" => new string('x', 257), + "subject-utf8" => new string('\u00e9', 129), + _ => "inventory.reserve.v1" + }; + var writes = Writes(rig); + Assert.ThrowsAny(() => Create(rig, key, subject, "oversized")); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId)); + Assert.Empty(rig.Pending); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().Effects); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + Assert.False(handler.Entered.Task.IsCompleted); + } + + [Theory] + [InlineData("key-bytes")] + [InlineData("key-utf8")] + [InlineData("key-depth")] + [InlineData("subject-bytes")] + [InlineData("subject-utf8")] + [InlineData("subject-space")] + public async Task MetadataAtExactDefaultLimit_IsAccepted(string field) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var key = field switch + { + "key-bytes" => HierarchicalKey.Create(new string('x', 1024)), + "key-utf8" => HierarchicalKey.Create(new string('\u00e9', 512)), + "key-depth" => HierarchicalKey.Create(Enumerable.Repeat("segment", 32).ToArray()), + _ => HierarchicalKey.Create("tenant", "bounded", "command") + }; + var subject = field switch + { + "subject-bytes" => new string('x', 256), + "subject-utf8" => new string('\u00e9', 128), + "subject-space" => " ", + _ => "inventory.reserve.v1" + }; + var envelope = Create(rig, key, subject, "maximum"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, envelope)).Status); + } + + [Fact] + public async Task ExpiredIdentityCanAcceptChangedSenderSubjectAndBodyAtExactBoundary() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var key = HierarchicalKey.Create("tenant", "orders", "1047", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = Create(rig, key, "inventory.reserve.v2", "changed", + GrainId.Create("forwarder", "new")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + await Fixture.SnapshotProbe.WaitAsync(rig.Receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, repeated)).Status); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, repeated)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 2); + Assert.Equal(2, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(2, handler.Calls); + } + + private async Task CreateAsync() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var handler = new IdentityHandler(); + var rig = CreateRig(receiver, context, handler); + await OnTurnAsync(context, () => rig.Grain.HandlerOverride = handler); + return rig; + } + + private static Rig CreateRig(IMessagingTestGrain receiver, IGrainContext context, IdentityHandler handler) => + new(receiver, context, Assert.IsType(context.GrainInstance), handler, + context.ActivationServices.GetRequiredService(), + context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox"), + context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed")); + + private InboxMessage Create(Rig rig, HierarchicalKey key, string subject, string value, GrainId sender = default) => + TestApplicationProtocol.CreateInbox(Sessions, + sender.IsDefault ? GrainId.Create("producer", "first") : sender, + subject, new TestMessage(key, 610, value), key); + + private int Writes(Rig rig) => Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(rig.Context.GrainId)); + + private static Task DeliverOnTurnAsync(Rig rig, InboxMessage envelope) + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + rig.Context.Scheduler.QueueAction(() => + { + try + { + Assert.Same(rig.Context, ReceiverTestServices.CurrentGrainContext); + var extension = (IInboxExtension)rig.Context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + started.SetResult(extension.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); + } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = CompleteAsync(); }); + return done.Task; + async Task CompleteAsync() + { + try + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + await action(); + done.SetResult(); + } + catch (Exception exception) { done.SetException(exception); } + } + } + + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, + MessagingTestGrain Grain, IdentityHandler Handler, IJournaledStateManager Manager, + IDurableDictionary Pending, + IDurableDictionary Processed); + + private sealed class IdentityHandler : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int Calls { get; private set; } + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + var services = ReceiverTestServices.CurrentGrainContext!.ActivationServices; + var effects = services.GetRequiredKeyedService>("test-effects"); + var application = TestApplicationProtocol.Read(services.GetRequiredService(), context.Message); + var body = Assert.IsType(application.Body); + Calls++; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + effects.TryGetValue(context.Message.Envelope.MessageId, out var prior); + effects[context.Message.Envelope.MessageId] = new MessageEffect(context.Message.Envelope.MessageId, + (prior?.Count ?? 0) + 1, body.Sequence, body.Value); + context.Complete(); + } + public void Dispose() => Release.TrySetResult(); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs new file mode 100644 index 00000000000..b5e286df763 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs @@ -0,0 +1,118 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxLifecycleBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task Deactivation_WaitsForActiveInboxTimerBeforeDisposingShutdownSource() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var shutdown = GetShutdownSource(context); + var token = shutdown.Token; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName, GrainLifecycleEvents.ListenerName); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/teardown-active"); + var envelope = CreateEnvelope(receiver, NewMessage(95, "active-timer"), "messages/teardown-active"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var started = Assert.Single(events.Events.Select(static item => item.Payload) + .OfType(), item => ReferenceEquals(item.GrainContext, context)); + var tickStopped = events.WaitForEventAsync( + nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, started.Timer), + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + var stoppedAtCancellation = false; + var cancellations = 0; + using var registration = token.Register(() => + { + stoppedAtCancellation = events.Events.Any(item => + item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, started.Timer)); + Interlocked.Increment(ref cancellations); + }); + + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Verify active inbox timer teardown."), TestContext.Current.CancellationToken); + + Assert.False(context.Deactivated.IsCompleted); + Assert.False(tickStopped.IsCompleted); + Assert.False(token.IsCancellationRequested); + Assert.Equal(token, shutdown.Token); + handler.Release(); + var stopped = Assert.IsType((await tickStopped).Payload); + await WaitForDeactivationAsync(context); + + Assert.Null(stopped.Exception); + Assert.True(stoppedAtCancellation); + Assert.Equal(1, cancellations); + Assert.True(token.IsCancellationRequested); + Assert.Throws(() => shutdown.Token); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task Deactivation_DisposesQueuedInboxTimerBeforeScopeTeardownAndRecoveryDrainsOnce() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var shutdown = GetShutdownSource(context); + var token = shutdown.Token; + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName, GrainLifecycleEvents.ListenerName); + var envelope = CreateEnvelope(receiver, NewMessage(96, "queued-timer")); + + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync(envelope)).Status); + await WaitForDeactivationAsync(context); + + var captured = events.Events.Select(static item => item.Payload).ToList(); + var created = Assert.Single(captured.OfType(), + item => ReferenceEquals(item.GrainContext, context)); + Assert.DoesNotContain(captured.OfType(), item => ReferenceEquals(item.GrainContext, context)); + var disposed = Assert.Single(captured.OfType(), + item => ReferenceEquals(item.Timer, created.Timer)); + var deactivated = Assert.Single(captured.OfType(), + item => ReferenceEquals(item.GrainContext, context)); + Assert.True(captured.IndexOf(disposed) < captured.IndexOf(deactivated)); + Assert.True(token.IsCancellationRequested); + Assert.Throws(() => shutdown.Token); + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + } + + private static CancellationTokenSource GetShutdownSource(IGrainContext context) + { + var extensionType = ReceiverTestServices.GetImplementationType("InboxExtension"); + var extension = context.ActivationServices.GetRequiredService(extensionType); + return (CancellationTokenSource)extensionType.GetField("_shutdownCts", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(extension)!; + } + + private static async Task WaitForDeactivationAsync(IGrainContext context) + { + try + { + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + catch (TimeoutException exception) + { + throw new TimeoutException($"Inbox activation {context.GrainId} did not complete timer/lifecycle teardown.", exception); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxMissingHandlerTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxMissingHandlerTests.cs new file mode 100644 index 00000000000..4fa27922827 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxMissingHandlerTests.cs @@ -0,0 +1,492 @@ +using System.Collections; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Orleans.Timers; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxMissingHandlerTests() : MessagingBehaviorTestBase(new RetryFixture()) +{ + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandlerInvocationFailure_BeforeCompletePreservesOriginalCauseAndRetryState(bool deferred) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var failure = new IOException("Original application invocation failure."); + var armed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var handler = new InvocationFaultHandler(failure, deferred, () => armed.TrySetResult(Fixture.Storage.BlockWrite(journal))); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = WaitForPumpAsync(events, receiver.GetGrainId()); + var envelope = CreateEnvelope(receiver, NewMessage(192, "invocation-retry")); + var now = Fixture.Clock.GetUtcNow(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + var owner = grain.GetSnapshotForTest().InboxJob; + var ownershipId = grain.GetSnapshotForTest().InboxJobId; + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (deferred) + { + Assert.False(armed.Task.IsCompleted); + var initialEntry = Assert.Single(GetAttemptStates(context).Cast()); + var initialState = initialEntry.GetType().GetProperty("Value")!.GetValue(initialEntry)!; + Assert.Equal(0, initialState.GetType().GetProperty("AttemptCount")!.GetValue(initialState)); + Assert.Null(initialState.GetType().GetProperty("LastError")!.GetValue(initialState)); + Assert.Null(initialState.GetType().GetProperty("NextAttemptAt")!.GetValue(initialState)); + Assert.Equal(1, grain.GetSnapshotForTest().InboxCount); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.Equal(0, grain.GetSnapshotForTest().ProcessedMessageCount); + Assert.Equal(0, grain.GetSnapshotForTest().OutboxCount); + handler.Release.TrySetResult(); + } + using var accounting = await armed.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + await accounting.WaitUntilEnteredAsync(); + var pending = CancellationCleanupProbe.Field(extension, "_pendingWrites"); + var operation = Assert.Single(pending.Cast(), item => item.GetType().Name == "HandlerWrite"); + Assert.Same(failure, operation.GetType().GetProperty("Error")!.GetValue(operation)); + Assert.Equal(false, operation.GetType().GetProperty("Completed")!.GetValue(operation)); + AssertPendingRetry(context, grain.GetSnapshotForTest(), 1, now + TimeSpan.FromMinutes(1)); + Assert.Same(owner, grain.GetSnapshotForTest().InboxJob); + Assert.Equal(ownershipId, grain.GetSnapshotForTest().InboxJobId); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + await OnTurnAsync(context, () => grain.HandlerOverride = null); + accounting.Release(); + await stopped; + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + AssertPendingRetry(context, await receiver.GetSnapshotAsync(), 1, now + TimeSpan.FromMinutes(1)); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + using var retry = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/record"); + Fixture.Clock.Advance(TimeSpan.FromMinutes(1)); + // The grain-facing retry clock is fake while shard-executor scheduling uses + // real background time. Request the authoritative callback explicitly so this + // test does not depend on whether the original background dequeue happened + // before or after the fake-time advance. RunPumpAsync still runs the real + // non-interleaving inbox timer and awaits that exact requested run's outcome. + var retryPump = RunPumpAsync(receiver, Assert.IsType(owner)); + await retry.WaitUntilEnteredAsync(); + AssertPendingRetry(context, grain.GetSnapshotForTest(), 1, now + TimeSpan.FromMinutes(1)); + retry.Release(); + Assert.Equal(DurableJobRunStatus.Completed, (await retryPump).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(1, ScheduleCount(receiver)); + Assert.Same(context, Fixture.GetGrainContext(receiver)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + private sealed class InvocationFaultHandler(Exception failure, bool deferred, Action beforeThrow) : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Entered.TrySetResult(); + if (deferred) return ThrowAfterBarrierAsync(cancellationToken); + beforeThrow(); + throw failure; + } + private async ValueTask ThrowAfterBarrierAsync(CancellationToken cancellationToken) + { + await Release.Task.WaitAsync(cancellationToken); + beforeThrow(); + throw failure; + } + public void Dispose() => Release.TrySetResult(); + } + + [Fact] + public async Task AcceptedHandlerMissingOnFreshActivation_DeadLettersImmediatelyAndRetainsDedupe() + { + var receiver = NewGrain(); + const string route = "activation-only/handler"; + _ = await receiver.GetSnapshotAsync(); + await receiver.ConfigureHandlerAsync(false); + var originalContext = Fixture.GetGrainContext(receiver); + var originalGrain = Assert.IsType(originalContext.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(190, "missing-after-replay"), route); + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync(envelope)).Status); + await originalContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + var accepted = Assert.IsType(originalGrain.AcceptedSnapshot); + Assert.Equal(1, accepted.InboxCount); + Assert.Empty(accepted.Effects); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + var now = Fixture.Clock.GetUtcNow(); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = WaitForPumpAsync(events, receiver.GetGrainId()); + + _ = await receiver.GetSnapshotAsync(); + await stopped; + var completed = await receiver.GetSnapshotAsync(); + + Assert.NotEqual(accepted.ActivationId, completed.ActivationId); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.Effects); + Assert.Equal(0, completed.OutboxCount); + Assert.Equal(0, completed.MaxConcurrentHandlers); + var deadLetter = Assert.Single(completed.InboxDeadLetters); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal("No inbox handler is registered.", deadLetter.Reason); + Assert.Equal(0, deadLetter.AttemptCount); + Assert.Equal(now, deadLetter.DeadLetteredAt); + var context = Fixture.GetGrainContext(receiver); + var retained = Assert.Single(context.ActivationServices.GetRequiredService().InboxDeadLetters); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, envelope.Envelope), TestApplicationProtocol.Sender(Sessions, retained.Message.Envelope)); + Assert.Empty(GetAttemptStates(context)); + var processed = context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed"); + Assert.Equal(now, processed[envelope.Envelope.MessageId]); + Assert.Equal(job.Id, completed.InboxJob!.Id); + Assert.Equal(job.ShardId, completed.InboxJob.ShardId); + Assert.Equal(accepted.InboxJobId, completed.InboxJobId); + Assert.Equal(1, ScheduleCount(receiver)); + Assert.False(Assert.IsType(context.GrainInstance).DeactivationFailure.Task.IsCompleted); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + + Assert.Equal(DurableJobRunStatus.Completed, (await RunPumpAsync(receiver, job)).Status); + var retired = await receiver.GetSnapshotAsync(); + Assert.Null(retired.InboxJobId); + Assert.Null(retired.InboxJob); + Assert.Equal(1, ScheduleCount(receiver)); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(deadLetter, Assert.Single(recovered.InboxDeadLetters)); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Empty(recovered.Effects); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(deadLetter, Assert.Single((await receiver.GetSnapshotAsync()).InboxDeadLetters)); + Assert.Equal(1, ScheduleCount(receiver)); + } + + [Fact] + public Task OrdinaryHandlerFailures_RetryUntilExactConfiguredAttemptLimit() => + RetryUntilExactConfiguredAttemptLimitAsync(startBackgroundBeforeDuplicate: false); + + [Fact] + public Task RetryDuplicateDelivery_QueuedBehindActiveHandler_DoesNotDeadlock() => + RetryUntilExactConfiguredAttemptLimitAsync(startBackgroundBeforeDuplicate: true); + + private async Task RetryUntilExactConfiguredAttemptLimitAsync(bool startBackgroundBeforeDuplicate) + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(191, "retry-limit") with { ThrowDuringPreparation = true }); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = WaitForPumpAsync(events, receiver.GetGrainId()); + var now = Fixture.Clock.GetUtcNow(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await stopped; + var first = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + AssertPendingRetry(context, first, expectedAttempts: 1, now + TimeSpan.FromMinutes(1)); + var job = Assert.IsType(first.InboxJob); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await RunPumpAsync(receiver, job)).Status); + AssertPendingRetry(context, await receiver.GetSnapshotAsync(), expectedAttempts: 1, now + TimeSpan.FromMinutes(1)); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + + Task? background = null; + using var tick = startBackgroundBeforeDuplicate + ? Fixture.Clock.CreateTimer(_ => background = RunPumpAsync(receiver, job), null, TimeSpan.FromMinutes(1), Timeout.InfiniteTimeSpan) + : null; + using (var runningLocalDrain = ArmHandlerBeforeAdvance(receiver.GetGrainId(), TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), envelope).Route, TimeSpan.FromMinutes(1))) + { + if (startBackgroundBeforeDuplicate) + { + await runningLocalDrain.WaitUntilEnteredAsync(); + } + var duplicate = DeliverAsync(receiver, envelope); + await runningLocalDrain.WaitUntilEnteredAsync(); + var retry = startBackgroundBeforeDuplicate + ? Assert.IsAssignableFrom>(background) + : RunPumpAsync(receiver, job); + await OnTurnAsync(context, static () => { }); + await AssertUnrelatedTimerDoesNotCompleteAsync(context, events, retry); + if (startBackgroundBeforeDuplicate) + { + Assert.False(duplicate.IsCompleted); + } + runningLocalDrain.Release(); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await retry).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + } + AssertPendingRetry(context, await receiver.GetSnapshotAsync(), expectedAttempts: 2, now + TimeSpan.FromMinutes(3)); + using var runningRequestedPump = ArmHandlerBeforeAdvance(receiver.GetGrainId(), TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), envelope).Route, TimeSpan.FromMinutes(2)); + var terminal = RunPumpAsync(receiver, job); + await runningRequestedPump.WaitUntilEnteredAsync(); + await AssertUnrelatedTimerDoesNotCompleteAsync(context, events, terminal); + runningRequestedPump.Release(); + Assert.Equal(DurableJobRunStatus.Completed, (await terminal).Status); + var completed = await receiver.GetSnapshotAsync(); + var deadLetter = Assert.Single(completed.InboxDeadLetters); + Assert.Equal(3, deadLetter.AttemptCount); + Assert.Contains("Injected handler preparation failure", deadLetter.Reason, StringComparison.Ordinal); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Empty(completed.Effects); + Assert.Equal(0, completed.OutboxCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(GetAttemptStates(context)); + Assert.Null(completed.InboxJobId); + Assert.Equal(1, ScheduleCount(receiver)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.False(Assert.IsType(context.GrainInstance).DeactivationFailure.Task.IsCompleted); + } + + [Fact] + public async Task RetryBarrier_IsArmedWhenClockMakesBackgroundPumpEligible() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(192, "clock-boundary") with { ThrowDuringPreparation = true }); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = WaitForPumpAsync(events, receiver.GetGrainId()); + var now = Fixture.Clock.GetUtcNow(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await stopped; + var first = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var job = Assert.IsType(first.InboxJob); + AssertPendingRetry(context, first, expectedAttempts: 1, now + TimeSpan.FromMinutes(1)); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId())); + HandlerProbe.Barrier? observedBarrier = null; + Task? background = null; + using var tick = Fixture.Clock.CreateTimer(_ => + { + Fixture.HandlerProbe.TryGet(receiver.GetGrainId(), TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), envelope).Route, out observedBarrier); + background = RunPumpAsync(receiver, job); + }, null, TimeSpan.FromMinutes(1), Timeout.InfiniteTimeSpan); + + using var handler = ArmHandlerBeforeAdvance(receiver.GetGrainId(), TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), envelope).Route, TimeSpan.FromMinutes(1)); + Assert.Same(handler, observedBarrier); + Assert.NotNull(background); + await handler.WaitUntilEnteredAsync(); + await OnTurnAsync(context, () => AssertPendingRetry(context, grain.GetSnapshotForTest(), expectedAttempts: 1, now + TimeSpan.FromMinutes(1))); + Assert.False(background.IsCompleted); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + await AssertUnrelatedTimerDoesNotCompleteAsync(context, events, background); + handler.Release(); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await background).Status); + AssertPendingRetry(context, await receiver.GetSnapshotAsync(), expectedAttempts: 2, now + TimeSpan.FromMinutes(3)); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.Equal(1, ScheduleCount(receiver)); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + private HandlerProbe.Barrier ArmHandlerBeforeAdvance(GrainId grainId, string route, TimeSpan advance) + { + var handler = Fixture.HandlerProbe.Arm(grainId, route); + try + { + Fixture.Clock.Advance(advance); + return handler; + } + catch + { + handler.Dispose(); + throw; + } + } + + private static async Task AssertUnrelatedTimerDoesNotCompleteAsync(IGrainContext context, DiagnosticEventCollector events, Task pump) + { + IGrainTimer unrelated = null!; + await OnTurnAsync(context, () => unrelated = context.ActivationServices.GetRequiredService() + .RegisterGrainTimer(context, static (_, _) => Task.CompletedTask, 0, + new GrainTimerCreationOptions(TimeSpan.Zero, Timeout.InfiniteTimeSpan) { Interleave = true })); + using (unrelated) + { + await events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, unrelated), + TimeSpan.FromSeconds(30), Cancellation); + Assert.False(pump.IsCompleted); + } + } + + private static void AssertPendingRetry(IGrainContext context, EndpointSnapshot snapshot, int expectedAttempts, DateTimeOffset nextAttempt) + { + Assert.Equal(1, snapshot.InboxCount); + Assert.Equal(0, snapshot.ProcessedMessageCount); + Assert.Empty(snapshot.InboxDeadLetters); + Assert.Empty(snapshot.Effects); + Assert.Equal(0, snapshot.OutboxCount); + var entry = Assert.Single(GetAttemptStates(context).Cast()); + var state = entry.GetType().GetProperty("Value")!.GetValue(entry)!; + Assert.Equal(expectedAttempts, state.GetType().GetProperty("AttemptCount")!.GetValue(state)); + Assert.Equal(nextAttempt, state.GetType().GetProperty("NextAttemptAt")!.GetValue(state)); + } + private static IEnumerable GetAttemptStates(IGrainContext context) + { + var manager = context.ActivationServices.GetRequiredService(); + Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox-message-state", out var state)); + return Assert.IsAssignableFrom(state); + } + private int ScheduleCount(IMessagingTestGrain receiver) => + Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId()); + private static async Task WaitForPumpAsync(DiagnosticEventCollector events, GrainId grainId) + { + var created = await events.WaitForEventAsync(nameof(GrainTimerEvents.Created), + item => item.Payload is GrainTimerEvents.Created timer && timer.GrainContext.GrainId == grainId && IsInboxTimer(timer.Timer), + TimeSpan.FromSeconds(30), Cancellation); + var timer = Assert.IsType(created.Payload).Timer; + return await events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), Cancellation); + } + private static bool IsInboxTimer(IGrainTimer timer) => timer.GetType().GenericTypeArguments is [var state] + && state.DeclaringType == ReceiverTestServices.GetImplementationType("InboxExtension"); + private async Task RunPumpAsync(IMessagingTestGrain receiver, DurableJob job) + { + var context = Fixture.GetGrainContext(receiver); + var feature = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var completion = new PumpCompletion(context, feature, new PumpContext(job)); + using var subscription = GrainTimerEvents.AllEvents.Subscribe(completion); + await OnTurnAsync(context, completion.Start); + return await completion.Result.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + } + private sealed class PumpCompletion(IGrainContext context, IDurableJobFeatureHandler feature, IJobRunContext run) + : IObserver + { + private IGrainTimer? _timer; + private bool _capturing; + private bool _started; + public TaskCompletionSource Result { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Start() + { + _started = true; + StartRequestedPump(allowCoalescing: true); + } + private void StartRequestedPump(bool allowCoalescing) + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + DurableJobRunResult result; + _capturing = true; + try + { + result = feature.ExecuteJobAsync(run, Cancellation).GetAwaiter().GetResult(); + } + finally + { + _capturing = false; + } + if (result.IsInProgress && _timer is null) + { + // Bind only this exact requested payload, never an unrelated coalesced turn. + var state = feature.GetType().GetField("_pumpTimer", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!.GetValue(feature); + if (state is not null) + { + var pending = state.GetType().BaseType!.GetField("_pending", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!.GetValue(state); + if (pending is not null) + { + var execution = pending.GetType().GetProperty("Execution")!.GetValue(pending)!; + var key = execution.GetType().GetProperty("Key")!.GetValue(execution)!; + if (Equals(run.RunId, key.GetType().GetProperty("RunId")!.GetValue(key)) + && Equals(run.Job.Id, key.GetType().GetProperty("JobId")!.GetValue(key))) + { + _timer = Assert.IsAssignableFrom(state.GetType().BaseType! + .GetField("_timer", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!.GetValue(state)); + } + } + } + } + if (!result.IsInProgress) + { + Result.TrySetResult(result); + } + else if (!allowCoalescing) + { + Assert.NotNull(_timer); + } + } + public void OnNext(GrainTimerEvents.TimerEvent item) + { + if (!ReferenceEquals(item.GrainContext, context) || Result.Task.IsCompleted) return; + try + { + if (item is GrainTimerEvents.Created && _capturing) + { + Assert.Null(_timer); + _timer = item.Timer; + } + else if (item is GrainTimerEvents.TickStop stop && _started) + { + if (ReferenceEquals(stop.Timer, _timer)) + { + Assert.Null(stop.Exception); + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + var result = feature.ExecuteJobAsync(run, Cancellation).GetAwaiter().GetResult(); + Assert.False(result.IsInProgress); + Result.TrySetResult(result); + } + else if (_timer is null && IsInboxTimer(stop.Timer)) + { + // The coalesced pump released its lease before this event; start the requested run on this turn. + Assert.Null(stop.Exception); + StartRequestedPump(allowCoalescing: false); + } + } + } + catch (Exception exception) + { + Result.TrySetException(exception); + } + } + public void OnError(Exception error) => Result.TrySetException(error); + public void OnCompleted() { } + } + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } + private sealed class PumpContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + private sealed class RetryFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxProcessingAttempts = 3; + options.BackpressureRetryDelay = TimeSpan.FromMinutes(1); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxPumpResultLifetimeTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxPumpResultLifetimeTests.cs new file mode 100644 index 00000000000..80608adae59 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxPumpResultLifetimeTests.cs @@ -0,0 +1,241 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxPumpResultLifetimeTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task ReplacedQueuedPump_DiscardsUnconsumableExecutionAndRegistration() + { + var receiver = NewGrain(); + var oldJob = CreateJob(receiver, "replaced:1"); + await receiver.SetInboxOwnershipAsync("replaced:1", oldJob); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var results = GetEntries(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-replacement"); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/replacement"); + var envelope = CreateEnvelope(receiver, NewMessage(170, "replacement"), "messages/replacement"); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var turn = receiver.HoldPumpTurnAsync("hold-replacement", envelope, deactivate: false); + await hold.WaitUntilEnteredAsync(); + var run = CreateRun(oldJob); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeAsync(receiver, run)).Status); + var entry = Assert.Single(results.Values.Cast()); + Assert.NotEqual(default, GetRegistration(entry)); + var timer = Assert.Single(events.Events.Select(static e => e.Payload).OfType(), e => ReferenceEquals(e.GrainContext, context)).Timer; + var stopped = events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + e => e.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.Timer, timer), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + hold.Release(); + await turn; + await stopped; + await handler.WaitUntilEnteredAsync(); + + Assert.DoesNotContain(results.Keys.Cast(), key => GetJobId(key) == oldJob.Id); + Assert.Equal(default, GetRegistration(entry)); + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeAsync(receiver, CreateRun(oldJob, 2))).Status); + Assert.False(Assert.IsType(context.GrainInstance).DeactivationFailure.Task.IsCompleted); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + } + + [Fact] + public async Task OwnerClearCompletion_PollConsumesResultBeforeOwnershipMismatchReturns() + { + var receiver = NewGrain(); + var job = CreateJob(receiver, "clear:1"); + await receiver.SetInboxOwnershipAsync("clear:1", job); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var entries = GetEntries(context); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + e => e.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.GrainContext, context), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var run = CreateRun(job); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeAsync(receiver, run)).Status); + await stopped; + Assert.Null((await receiver.GetSnapshotAsync()).InboxJobId); + Assert.Single(entries); + var feature = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + DurableJobRunResult result = null!; + await OnTurnAsync(context, () => result = feature.ExecuteJobAsync(run, TestContext.Current.CancellationToken).GetAwaiter().GetResult()); + Assert.Equal(DurableJobRunStatus.Completed, result.Status); + Assert.Empty(entries); + } + + [Fact] + public async Task ShutdownWithQueuedPump_ReleasesRegistrationAndFreshActivationRetries() + { + var receiver = NewGrain(); + var job = CreateJob(receiver, "shutdown:1"); + await receiver.SetInboxOwnershipAsync("shutdown:1", job); + await RefreshSeededOwnerAsync(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(171, "fresh-after-stop")); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var entries = GetEntries(context); + var feature = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-stop"); + var turn = receiver.HoldPumpTurnAsync("hold-stop", deactivate: true); + await hold.WaitUntilEnteredAsync(); + var run = CreateRun(job); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeAsync(receiver, run)).Status); + var entry = Assert.Single(entries.Values.Cast()); + Assert.NotEqual(default, GetRegistration(entry)); + var messages = context.ActivationServices.GetRequiredKeyedService>( + "__orleans.messaging.inbox"); + var manager = context.ActivationServices.GetRequiredService(); + Task persisted = null!; + await OnTurnAsync(context, () => + { + messages.Add(envelope.Envelope.MessageId, envelope); + persisted = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + }); + await persisted; + hold.Release(); + await turn; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Empty(entries); + Assert.Equal(default, GetRegistration(entry)); + await Assert.ThrowsAnyAsync(async () => await feature.ExecuteJobAsync(run, TestContext.Current.CancellationToken)); + Assert.Empty(grain.GetSnapshotForTest().Effects); + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + } + + [Fact] + public async Task TerminalFaultWithQueuedPump_ReleasesEntryWithoutCompletingDurableJob() + { + var receiver = NewGrain(); + var job = CreateJob(receiver, "fault:1"); + await receiver.SetInboxOwnershipAsync("fault:1", job); + await RefreshSeededOwnerAsync(receiver); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 172, "uncommitted")); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var entries = GetEntries(context); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-fault"); + var turn = receiver.HoldPumpTurnAsync("hold-fault", deactivate: false); + await hold.WaitUntilEnteredAsync(); + var run = CreateRun(job); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeAsync(receiver, run)).Status); + var entry = Assert.Single(entries.Values.Cast()); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var failedWrite = new DurableJob + { + Id = Guid.NewGuid().ToString("N"), + ShardId = "pump-result-lifetime", + Name = "test/write-journal", + TargetGrainId = receiver.GetGrainId(), + DueTime = DateTimeOffset.UtcNow + }; + + var result = await InvokeAsync(receiver, CreateRun(failedWrite)); + + Assert.Equal(DurableJobRunStatus.Failed, result.Status); + Assert.IsType(await grain.DeactivationFailure.Task); + hold.Release(); + await turn; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Empty(entries); + Assert.Equal(default, GetRegistration(entry)); + Assert.Equal(job.Id, grain.GetSnapshotForTest().InboxJob!.Id); + Assert.Single(grain.GetSnapshotForTest().Effects); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + + [Fact] + public async Task CanceledQueuedAttempt_DiscardsOnlySupersededRunAndAllowsRetry() + { + var receiver = NewGrain(); + var job = CreateJob(receiver, "attempt:1"); + await receiver.SetInboxOwnershipAsync("attempt:1", job); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var entries = GetEntries(context); + var feature = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-attempt"); + var turn = receiver.HoldPumpTurnAsync("hold-attempt", deactivate: false); + await hold.WaitUntilEnteredAsync(); + using var cancellation = new CancellationTokenSource(); + var canceledRun = CreateRun(job); + var retryRun = CreateRun(job, 2); + await OnTurnAsync(context, () => Assert.Equal(DurableJobRunStatus.InProgress, + feature.ExecuteJobAsync(canceledRun, cancellation.Token).GetAwaiter().GetResult().Status)); + cancellation.Cancel(); + await OnTurnAsync(context, () => + { + Assert.Throws(() => feature.ExecuteJobAsync(canceledRun, cancellation.Token).GetAwaiter().GetResult()); + Assert.Equal(DurableJobRunStatus.InProgress, + feature.ExecuteJobAsync(retryRun, TestContext.Current.CancellationToken).GetAwaiter().GetResult().Status); + }); + var queuedRetry = Assert.Single(entries.Keys.Cast()); + Assert.Equal(retryRun.RunId, queuedRetry.GetType().GetProperty("RunId")!.GetValue(queuedRetry)); + hold.Release(); + await turn; + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + _ = await receiver.GetSnapshotAsync(); + var retained = Assert.Single(entries.Keys.Cast()); + Assert.Equal(retryRun.RunId, retained.GetType().GetProperty("RunId")!.GetValue(retained)); + await OnTurnAsync(context, () => Assert.Equal(DurableJobRunStatus.Completed, + feature.ExecuteJobAsync(retryRun, TestContext.Current.CancellationToken).GetAwaiter().GetResult().Status)); + Assert.Empty(entries); + } + + private static IDictionary GetEntries(IGrainContext context) + { + var type = ReceiverTestServices.GetImplementationType("MessagingPumpResults"); + return (IDictionary)type.GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(context.ActivationServices.GetRequiredService(type))!; + } + private static string GetJobId(object key) => (string)key.GetType().GetProperty("JobId")!.GetValue(key)!; + private static CancellationTokenRegistration GetRegistration(object entry) => + (CancellationTokenRegistration)entry.GetType().GetProperty("CancellationRegistration")!.GetValue(entry)!; + private static DurableJob CreateJob(IMessagingTestGrain receiver, string owner) => new() + { + Id = Guid.NewGuid().ToString("N"), + ShardId = "pump-result-lifetime", + Name = ReceiverTestServices.InboxJobName, + TargetGrainId = receiver.GetGrainId(), + DueTime = DateTimeOffset.UtcNow, + Metadata = new Dictionary { ["orleans.messaging.ownership-id"] = owner } + }; + private static IJobRunContext CreateRun(DurableJob job, int dequeueCount = 1) => + (IJobRunContext)Activator.CreateInstance(typeof(DurableJob).Assembly.GetType("Orleans.DurableJobs.JobRunContext", throwOnError: true)!, + job, Guid.NewGuid().ToString("N"), dequeueCount)!; + private static async Task InvokeAsync(IMessagingTestGrain receiver, IJobRunContext run) + { + var type = typeof(DurableJob).Assembly.GetType("Orleans.DurableJobs.IDurableJobReceiverExtension", throwOnError: true)!; + return await (ValueTask)type.GetMethod("HandleDurableJobAsync")! + .Invoke(receiver.AsReference(type), [run, TestContext.Current.CancellationToken])!; + } + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs new file mode 100644 index 00000000000..b197dde574a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs @@ -0,0 +1,332 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxQuiescenceTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task AlwaysInterleaveCallback_ObservesAdmittedClearOnSameActivationScheduler() + { + var receiver = NewGrain(); + var owner = CreateJob(receiver, ReceiverTestServices.InboxJobName, "owner:1"); + await receiver.SetInboxOwnershipAsync("owner:1", owner); + await RefreshSeededOwnerAsync(receiver); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + using var barrier = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeJobAsync(receiver, owner)).Status); + await barrier.WaitUntilEnteredAsync(); + var scheduler = barrier.EntryScheduler; + Assert.NotSame(TaskScheduler.Default, scheduler); + Assert.Same(context, barrier.EntryContext); + var operation = Assert.Single(GetPending(context).Cast()); + Assert.Equal("ClearOwnerWrite", operation.GetType().Name); + var finished = (TaskCompletionSource)operation.GetType().GetProperty("Finished")!.GetValue(operation)!; + + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeJobAsync(receiver, CreateJob(receiver, "test/probe-scheduler"))).Status); + Assert.Same(scheduler, grain.JobScheduler); + Assert.Same(context, grain.JobGrainContext); + for (var attempt = 2; attempt <= 4; attempt++) + { + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeJobAsync(receiver, owner, attempt)).Status); + Assert.Same(operation, Assert.Single(GetPending(context).Cast())); + } + Assert.Single(events.Events, item => item.Payload is GrainTimerEvents.Created created && ReferenceEquals(created.GrainContext, context)); + barrier.Release(); + await finished.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), static snapshot => snapshot.InboxJobId is null); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeJobAsync(receiver, CreateJob(receiver, "test/probe-scheduler"))).Status); + Assert.Same(scheduler, grain.JobScheduler); + Assert.Same(context, grain.JobGrainContext); + Assert.Empty(GetPending(context)); + Assert.Equal(DurableJobRunStatus.Completed, (await InvokeJobAsync(receiver, owner, 5)).Status); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OwnerDelete_AfterAcknowledgedWork_StopsComponentsAndUsesFreshActivation(bool pump) + { + var receiver = NewGrain(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/delete-ack"); + var first = CreateEnvelope(receiver, NewMessage(160, "first"), "messages/delete-ack"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var outbox = GetOutbox(context); + var second = CreateEnvelope(receiver, NewMessage(161, "second"), "messages/delete-ack"); + Task? secondDelivery = null; + + if (pump) + { + var owner = Assert.IsType(grain.GetSnapshotForTest().InboxJob); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeJobAsync(receiver, owner)).Status); + } + else + { + secondDelivery = DeliverAsync(receiver, second); + } + + handler.Release(); + if (secondDelivery is not null) Assert.Equal(DeliveryStatus.Accepted, (await secondDelivery).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, pump ? 1 : 2); + Assert.Equal(pump ? 1 : 2, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + using var storage = Fixture.Storage.BlockDelete(JournalId.FromGrainId(receiver.GetGrainId())); + var deletion = receiver.DeleteStateAndDeactivateAsync(); + await storage.WaitUntilEnteredAsync(); + AssertStopped(extension, outbox); + Assert.False(deletion.IsCompleted); + storage.Release(); + await deletion; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var deleted = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, deleted.ActivationId); + Assert.Empty(deleted.Effects); + Assert.Equal(0, deleted.ProcessedMessageCount); + Assert.Null(deleted.InboxJobId); + } + + + [Fact] + public async Task OwnerDelete_DrainsCanceledDeliveryAndGateWaiterBeforeReset() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = (IInboxExtension)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + using var blocked = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var first = CreateEnvelope(receiver, NewMessage(163, "owned-after-cancel")); + var second = CreateEnvelope(receiver, NewMessage(164, "gate-waiter")); + using var cancellation = new CancellationTokenSource(); + var delivery = DeliverWithCancellationAsync(receiver, first, cancellation.Token); + await blocked.WaitUntilEnteredAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery); + _ = await receiver.GetSnapshotAsync(); + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var token = TestContext.Current.CancellationToken; + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(second, token).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + var waiting = await started.Task; + Assert.False(waiting.IsCompleted); + var outbox = GetOutbox(context); + using var deletionStorage = Fixture.Storage.BlockDelete(JournalId.FromGrainId(receiver.GetGrainId())); + var deleteEntered = deletionStorage.WaitUntilEnteredAsync(); + var deletion = receiver.DeleteStateAndDeactivateAsync(); + await outbox.Stopping.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.False(deletion.IsCompleted); + Assert.False(deleteEntered.IsCompleted); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_gate").CurrentCount); + Assert.Single(CancellationCleanupProbe.Field>(extension, "_pendingOwnershipIds")); + blocked.Release(); + await Assert.ThrowsAnyAsync(() => waiting); + await deleteEntered; + AssertStopped(extension, outbox); + Assert.Equal(1, CancellationCleanupProbe.Field(extension, "_gate").CurrentCount); + Assert.Empty(CancellationCleanupProbe.Field>(extension, "_pendingOwnershipIds")); + Assert.Equal(1, grain.GetSnapshotForTest().InboxCount); + deletionStorage.Release(); + await deletion; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var fresh = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, fresh.ActivationId); + Assert.Equal(0, fresh.ProcessedMessageCount); + Assert.Equal(0, fresh.InboxCount); + Assert.Empty(fresh.Effects); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeliveryWhileOwnerClearIsAdmitted_WaitsAndAcceptsWithoutFencing(bool interleaved) + { + var receiver = NewGrain(); + var owner = CreateJob(receiver, ReceiverTestServices.InboxJobName, "clear-overlap:1"); + await receiver.SetInboxOwnershipAsync("clear-overlap:1", owner); + await RefreshSeededOwnerAsync(receiver); + var incoming = CreateEnvelope(receiver, NewMessage(181, "after-clear")); + await receiver.SetControlEnvelopeAsync(incoming); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + using var preparation = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + Assert.Equal(DurableJobRunStatus.InProgress, (await InvokeJobAsync(receiver, owner)).Status); + await preparation.WaitUntilEnteredAsync(); + Assert.Equal("ClearOwnerWrite", Assert.Single(GetPending(context).Cast()).GetType().Name); + Task delivery; + if (interleaved) + { + delivery = InvokeJobAsync(receiver, CreateJob(receiver, "test/deliver-envelope")); + await grain.ControlDeliveryEntered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + else + { + delivery = DeliverAsync(receiver, incoming); + } + Assert.False(delivery.IsCompleted); + preparation.Release(); + await delivery; + if (interleaved) + { + Assert.Equal(DurableJobRunStatus.Completed, (await (Task)delivery).Status); + } + else + { + Assert.Equal(DeliveryStatus.Accepted, (await (Task)delivery).Status); + } + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + Assert.Equal(1, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OwnerDeleteFailure_DeactivatesAndFreshOwnerObservesActualStorageOutcome(bool committed) + { + var receiver = NewGrain(); + var seed = new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 188, "preserved-before-delete"); + await receiver.StageEffectAsync(seed); + await receiver.RetryWriteStateAsync(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var outbox = GetOutbox(context); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journalId); + using var storage = Fixture.Storage.BlockDelete(journalId); + if (committed) Fixture.Storage.FailAfterDelete(journalId); + var deletion = receiver.DeleteStateAndDeactivateAsync(); + await storage.WaitUntilEnteredAsync(); + AssertStopped(extension, outbox); + if (committed) storage.Release(); + else storage.Fail(); + var failure = await Assert.ThrowsAsync(() => deletion); + var reason = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.IsType(reason); + Assert.Equal(failure.Message, reason.Message); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + Assert.Equal(seed, Assert.Single(grain.GetSnapshotForTest().Effects)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + if (committed) + { + Assert.Empty(recovered.Effects); + } + else + { + Assert.Equal(seed, Assert.Single(recovered.Effects)); + } + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(0, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task OwnerDelete_DiscardsStagedOutputAndLaterDeliveryUsesFreshOwner() + { + var receiver = NewGrain(); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 190, "delete-this")); + await receiver.RetryWriteStateAsync(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = GetOutbox(context); + var staged = TestApplicationProtocol.Create(Sessions, receiver.GetGrainId(), receiver.GetGrainId(), + "output/staged", NewMessage(193, "staged-output")); + { + await receiver.StageOutputAsync(staged); + Assert.Equal(staged.Envelope.MessageId, Assert.Single(outbox.Messages).Envelope.MessageId); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + await receiver.DeleteStateAndDeactivateAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(outbox.Messages); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.True(outbox.Stopping.IsCompleted); + Assert.Throws(() => outbox.Send(staged)); + } + Assert.Empty(outbox.Messages); + var fresh = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, fresh.ActivationId); + Assert.Empty(fresh.Effects); + Assert.Equal(0, fresh.OutboxCount); + var envelope = CreateEnvelope(receiver, NewMessage(191, "new-state")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var after = await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(fresh.ActivationId, after.ActivationId); + Assert.Equal("new-state", Assert.Single(after.Effects).Value); + Assert.Equal(1, after.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + private static JournaledTestOutbox GetOutbox(IGrainContext context) => + (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + + private static void AssertStopped(object extension, JournaledTestOutbox outbox) + { + Assert.True(outbox.Stopping.IsCompleted); + Assert.False(CancellationCleanupProbe.CoordinatorIsActive(extension)); + Assert.Empty(CancellationCleanupProbe.Field(extension, "_pendingWrites")); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_metricsActive")); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_reportedDepth")); + var results = CancellationCleanupProbe.Field(extension, "_pumpResults"); + var entries = (IDictionary)results.GetType().GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(results)!; + Assert.DoesNotContain(entries.Keys.Cast(), + key => (string)key.GetType().GetProperty("JobName")!.GetValue(key)! == ReceiverTestServices.InboxJobName); + } + + private static IList GetPending(IGrainContext context) + { + var type = ReceiverTestServices.GetImplementationType("InboxExtension"); + return (IList)type.GetField("_pendingWrites", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(context.ActivationServices.GetRequiredService(type))!; + } + + private static DurableJob CreateJob(IMessagingTestGrain receiver, string name, string? ownershipId = null) => new() + { + Id = Guid.NewGuid().ToString("N"), + ShardId = "quiescence-tests", + Name = name, + TargetGrainId = receiver.GetGrainId(), + DueTime = DateTimeOffset.UtcNow, + Metadata = ownershipId is null ? null : new Dictionary { ["orleans.messaging.ownership-id"] = ownershipId } + }; + + private static async Task InvokeJobAsync(IMessagingTestGrain receiver, DurableJob job, int dequeueCount = 1) + { + var assembly = typeof(DurableJob).Assembly; + var type = assembly.GetType("Orleans.DurableJobs.IDurableJobReceiverExtension", throwOnError: true)!; + var reference = receiver.AsReference(type); + var run = Activator.CreateInstance(assembly.GetType("Orleans.DurableJobs.JobRunContext", throwOnError: true)!, + job, Guid.NewGuid().ToString("N"), dequeueCount)!; + return await (ValueTask)type.GetMethod("HandleDurableJobAsync")! + .Invoke(reference, [run, TestContext.Current.CancellationToken])!; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs new file mode 100644 index 00000000000..2909d0cbaac --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs @@ -0,0 +1,92 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateEncodingTests : MessagingBehaviorTestBase +{ + public InboxStateEncodingTests() : base(new FaultingInboxCodecFixture()) { } + + [Fact] + public async Task CompleteCommandCodecFailure_PreservesOriginalCauseAndOwnedCleanup() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var extension = context.ActivationServices.GetRequiredService(CancellationCleanupProbe.ExtensionType); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/complete-codec-fault"); + var envelope = CreateEnvelope(receiver, NewMessage(204, "complete-codec-fault"), "messages/complete-codec-fault"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Completion remove command failed."); + ((FaultingInboxCodecFixture)Fixture).NextFailure = failure; + handler.Release(); + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(CancellationCleanupProbe.Field(extension, "_pendingWrites")); + Assert.Equal(1, CancellationCleanupProbe.Field(extension, "_gate").CurrentCount); + Assert.False(CancellationCleanupProbe.CoordinatorIsActive(extension)); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_metricsActive")); + Assert.Equal(0, CancellationCleanupProbe.Field(extension, "_reportedDepth")); + Assert.Empty(outbox.Messages); + _ = await receiver.GetSnapshotAsync(); + var fresh = Assert.IsType(Fixture.GetGrainContext(receiver).GrainInstance); + var replayed = Assert.IsType(fresh.ReplayedSnapshot); + Assert.Equal(1, replayed.InboxCount); + Assert.Equal(0, replayed.ProcessedMessageCount); + Assert.Empty(replayed.Effects); + Assert.Equal(0, replayed.OutboxCount); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InboxCodecFailure_DeactivatesBeforeStorageAndFreshScopeReplaysOnlyAcknowledgedState(bool snapshot) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Injected inbox command codec failure."); + var faulting = (FaultingInboxCodecFixture)Fixture; + faulting.NextFailure = failure; + faulting.FailOnSnapshot = snapshot; + if (snapshot) Fixture.Storage.RequestSnapshot(journal); + var envelope = CreateEnvelope(receiver, NewMessage(203, "codec")); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Same(failure, await grain.DeactivationFailure.Task); + Assert.Equal(snapshot, faulting.FailedSnapshot); + Assert.Empty(outbox.Messages); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + // Entry encoding precedes dictionary insertion; snapshot encoding observes staged values. + Assert.Equal(snapshot ? 1 : 0, grain.GetSnapshotForTest().InboxCount); + Assert.Empty(grain.GetSnapshotForTest().Effects); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs new file mode 100644 index 00000000000..94abb264c38 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs @@ -0,0 +1,228 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Messaging.Tests.Support; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Journaling.Json; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateManagerBoundaryTests +{ + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ActivationStandardState_UsesScopedCodecAndSingleOwnerEnrollment(bool keyedFirst) + { + const string selectedFormat = "orleans-binary"; + var builder = CreateBuilder(selectedFormat); + var codecType = GetDictionaryCodecType(builder.Services, selectedFormat); + builder.Services.AddKeyedScoped>(selectedFormat, (sp, _) => + new TrackingCodec((IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, codecType))); + builder.Services.AddScoped(); + var grainType = GrainType.Create("codec-scope"); + var manifest = new GrainManifest( + ImmutableDictionary.Empty.Add( + grainType, new GrainProperties( + ImmutableDictionary.Empty.WithComparers(StringComparer.Ordinal, StringComparer.Ordinal))), + ImmutableDictionary.Empty); + var manifestProvider = Substitute.For(); + manifestProvider.LocalGrainManifest.Returns(manifest); + manifestProvider.Current.Returns(new ClusterManifest( + new MajorMinorVersion(1, 0), + ImmutableDictionary.Empty, + [manifest])); + builder.Services.AddSingleton(manifestProvider); + builder.Services.AddSingleton(); + builder.Services.AddScoped(sp => + { + var context = Substitute.For(); + context.ActivationServices.Returns(sp); + context.GrainId.Returns(GrainId.Create(grainType, IdSpan.Create(Guid.NewGuid().ToString("N")))); + context.ObservableLifecycle.Returns(Substitute.For()); + return context; + }); + builder.Services.AddScoped(sp => new OwnedComponent( + sp.GetRequiredKeyedService>("state"), sp.GetRequiredService())); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + IDurableDictionary state; + OwnedComponent component; + Dependency dependency; + IJournaledStateManager journalOwner; + await using (var scope = services.CreateAsyncScope()) + { + var sp = scope.ServiceProvider; + Assert.Same(manifest.Grains[grainType], + sp.GetRequiredService().GetGrainProperties(sp.GetRequiredService().GrainId.Type)); + var app = sp.GetRequiredService(); + journalOwner = sp.GetRequiredService(); + Assert.Same(app, journalOwner); + var codec = Assert.IsType(sp.GetRequiredKeyedService>(selectedFormat)); + Assert.Throws(() => services.GetRequiredKeyedService>(selectedFormat)); + state = keyedFirst + ? sp.GetRequiredKeyedService>("state") + : app.GetOrAddState>("state"); + component = sp.GetRequiredService(); + dependency = sp.GetRequiredService(); + Assert.Same(state, component.State); + Assert.Same(dependency, component.Dependency); + Assert.Same(state, sp.GetRequiredKeyedService>("state")); + Assert.Same(state, app.GetOrAddState>("state")); + Assert.True(app.TryGetState>("state", out var found)); + Assert.Same(state, found); + Assert.True(journalOwner.TryGetStateMachine("state", out var machine)); + Assert.Same(state, machine); + sp.GetRequiredService().ObservableLifecycle.Received(1) + .Subscribe(Arg.Any(), GrainLifecycleStage.SetupState, Arg.Any()); + await journalOwner.InitializeAsync(Cancellation); + Assert.Equal(0, codec.Applies); + state.Add("committed", 17); + Assert.Equal(1, codec.SetsWritten); + await app.WriteStateAsync(Cancellation); + Assert.Equal(1, codec.SetsWritten); + Assert.Equal(17, state["committed"]); + Assert.Throws(() => app.GetOrAddState>("late")); + Assert.Throws(() => sp.GetRequiredKeyedService>("late")); + Assert.Same(state, app.GetOrAddState>("state")); + await journalOwner.DeleteStateAsync(Cancellation); + Assert.Empty(state); + Assert.False(dependency.Disposed); + Assert.Equal(0, component.Disposals); + await using var other = services.CreateAsyncScope(); + var otherOwner = other.ServiceProvider.GetRequiredService(); + Assert.NotSame(journalOwner, otherOwner); + Assert.NotSame(codec, other.ServiceProvider.GetRequiredKeyedService>(selectedFormat)); + } + Assert.True(dependency.Disposed); + Assert.Equal(1, component.Disposals); + await Assert.ThrowsAsync(() => journalOwner.WriteStateAsync(Cancellation).AsTask()); + } + + [Fact] + public async Task NamedStandaloneOwner_UsesSelectedFormatAndCallerOwnedComponentsAcrossReplay() + { + const string selectedFormat = "json"; + const string storageName = "caller-owned-store"; + var builder = CreateBuilder(selectedFormat); + builder.AddVolatileJournalStorage(storageName); + builder.UseJsonJournalFormat(StandardStateJsonContext.Default); + var codecType = GetDictionaryCodecType(builder.Services, selectedFormat); + builder.Services.AddKeyedSingleton>(selectedFormat, (sp, _) => + new TrackingCodec((IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, codecType))); + var dependenciesCreated = 0; + builder.Services.AddScoped(_ => { dependenciesCreated++; return new Dependency(); }); + var id = new JournalId("inbox-state-lifetime/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredKeyedService(storageName).CreateStandalone(id)); + builder.Services.AddScoped(sp => new OwnedComponent( + sp.GetRequiredKeyedService>("state"), sp.GetRequiredService())); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + Assert.Equal(0, dependenciesCreated); + var codec = Assert.IsType(services.GetRequiredKeyedService>(selectedFormat)); + Dependency dependency; + OwnedComponent component; + await using (var scope = services.CreateAsyncScope()) + { + var first = scope.ServiceProvider.GetRequiredService(); + Assert.False(first is IDurableStateManager); + dependency = scope.ServiceProvider.GetRequiredService(); + Assert.False(first.TryGetStateMachine("state", out _)); + component = scope.ServiceProvider.GetRequiredService(); + var state = component.State; + Assert.Same(scope.ServiceProvider.GetRequiredKeyedService>("state"), state); + Assert.True(first.TryGetStateMachine("state", out var registered)); + Assert.Same(state, registered); + await first.InitializeAsync(Cancellation); + Assert.Equal(0, codec.Applies); + state.Add("persisted", 23); + await first.WriteStateAsync(Cancellation); + Assert.Equal(1, codec.SetsWritten); + await first.DisposeAsync(); + Assert.Equal(0, component.Disposals); + Assert.False(dependency.Disposed); + Assert.Equal(1, dependenciesCreated); + await using var recoveryScope = services.CreateAsyncScope(); + var recovered = recoveryScope.ServiceProvider.GetRequiredService(); + Assert.NotSame(first, recovered); + Assert.False(recovered is IDurableStateManager); + var recoveredCodec = recoveryScope.ServiceProvider.GetRequiredKeyedService>(selectedFormat); + Assert.Same(codec, recoveredCodec); + var fresh = recoveryScope.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(Cancellation); + Assert.NotSame(state, fresh); + Assert.Equal(new KeyValuePair("persisted", 23), Assert.Single(fresh)); + Assert.Equal(1, codec.Applies); + await recovered.DeleteStateAsync(Cancellation); + Assert.Empty(fresh); + await recovered.DisposeAsync(); + Assert.Equal(0, component.Disposals); + Assert.False(dependency.Disposed); + Assert.Equal(1, dependenciesCreated); + } + + Assert.True(dependency.Disposed); + Assert.Equal(1, component.Disposals); + } + + private static Type GetDictionaryCodecType(IServiceCollection services, string selectedFormat) => services.Last(entry => + entry.ServiceType == typeof(IDurableDictionaryCommandCodec<,>) && Equals(entry.ServiceKey, selectedFormat)) + .KeyedImplementationType!.MakeGenericType(typeof(string), typeof(int)); + + internal static TestSiloBuilder CreateBuilder(string selectedFormat) + { + var builder = new TestSiloBuilder(); + builder.Services.AddSerializer(); + builder.Services.AddLogging(); + builder.Services.AddSingleton(TimeProvider.System); + builder.Services.AddKeyedSingleton(KeyedService.AnyKey, static (sp, _) => sp.GetRequiredService()); + builder.AddVolatileJournalStorage(); + builder.Services.Configure(options => options.JournalFormatKey = selectedFormat); + return builder; + } + + internal sealed class TestSiloBuilder : ISiloBuilder + { + public IServiceCollection Services { get; } = new ServiceCollection(); + public IConfiguration Configuration { get; } = new ConfigurationBuilder().Build(); + } + + private sealed class Dependency : IDisposable + { + public bool Disposed { get; private set; } + public void Dispose() => Disposed = true; + } + + private sealed class OwnedComponent(IDurableDictionary state, Dependency dependency) : IDisposable + { + public IDurableDictionary State { get; } = state; + public Dependency Dependency { get; } = dependency; + public int Disposals { get; private set; } + public void Dispose() => Disposals++; + } + + private sealed class TrackingCodec(IDurableDictionaryCommandCodec inner) : IDurableDictionaryCommandCodec + { + public int SetsWritten { get; private set; } + public int Applies { get; private set; } + public void WriteSet(string key, int value, JournalStreamWriter writer) { SetsWritten++; inner.WriteSet(key, value, writer); } + public void WriteRemove(string key, JournalStreamWriter writer) => inner.WriteRemove(key, writer); + public void WriteClear(JournalStreamWriter writer) => inner.WriteClear(writer); + public void WriteSnapshot(IReadOnlyCollection> items, JournalStreamWriter writer) => inner.WriteSnapshot(items, writer); + public void Apply(JournalBufferReader input, IDurableDictionaryCommandHandler consumer) + { + Applies++; + inner.Apply(input, consumer); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs new file mode 100644 index 00000000000..f94da664978 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs @@ -0,0 +1,146 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateProtocolTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task RawWriteDuringLocalPreparation_CapturesNoHandlerEffects() + { + var receiver = NewGrain(); + const string route = "messages/local-preparation"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(201, "local"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + await OnTurnAsync(context, () => context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "raw-write"); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.False(grain.ApplyAttempted.Task.IsCompleted); + var beforeApply = grain.GetSnapshotForTest(); + Assert.Empty(beforeApply.Effects); + Assert.Equal(1, beforeApply.InboxCount); + Assert.Equal(0, beforeApply.ProcessedMessageCount); + var captured = grain.Captures[^1]; + Assert.Empty(captured.Effects); + Assert.Equal(1, captured.InboxCount); + Assert.Equal(0, captured.ProcessedMessageCount); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + await receiver.RequestDeactivationAsync(); + var replayed = await receiver.GetSnapshotAsync(); + Assert.Equal(completed.Effects, replayed.Effects); + Assert.Equal(1, replayed.ProcessedMessageCount); + } + + [Fact] + public async Task HandlerFailureBeforeComplete_AccountsPreparationOutcomeWithoutMutationInference() + { + var receiver = NewGrain(); + const string route = "messages/no-earlier-write"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(203, "no-earlier-write"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Local preparation failed before Complete."); + await OnTurnAsync(context, () => + { + Assert.Equal(0, manager.PendingWriteByteCount); + grain.NextApplyFailure = failure; + }); + + handler.Release(); + var completed = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Contains(failure.Message, Assert.Single(completed.InboxDeadLetters).Reason, StringComparison.Ordinal); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(completed.Effects); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Equal(completed.InboxDeadLetters, recovered.InboxDeadLetters); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task CanceledOwnerDeleteWait_RetainsStoppedWorkflowThroughResetAndDeactivation() + { + var receiver = NewGrain(); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 204, "before-delete")); + await receiver.RetryWriteStateAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var inbox = (IInboxExtension)context.ActivationServices.GetRequiredService( + ReceiverTestServices.GetImplementationType("InboxExtension")); + var stoppedToken = CancellationCleanupProbe.Field(inbox, "_shutdownCts").Token; + using var storage = Fixture.Storage.BlockDelete(JournalId.FromGrainId(receiver.GetGrainId())); + using var cancellation = new CancellationTokenSource(); + var deleting = receiver.DeleteStateAndDeactivateAsync(); + var caller = deleting.WaitAsync(cancellation.Token); + await storage.WaitUntilEnteredAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => caller); + Assert.True(outbox.Stopping.IsCompleted); + Assert.True(stoppedToken.IsCancellationRequested); + Assert.False(deleting.IsCompleted); + Assert.False(context.Deactivated.IsCompleted); + var envelope = CreateEnvelope(receiver, NewMessage(205, "after-delete")); + Task rejected = null!; + await OnTurnAsync(context, () => rejected = inbox.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); + await Assert.ThrowsAnyAsync(() => rejected); + var outgoing = TestApplicationProtocol.Create(Sessions, receiver.GetGrainId(), receiver.GetGrainId(), + "messages/record", NewMessage(205, "after-delete")); + Assert.Throws(() => outbox.Send(outgoing)); + Assert.Single(grain.GetSnapshotForTest().Effects); + storage.Release(); + await deleting; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.True(outbox.Stopping.IsCompleted); + Assert.True(stoppedToken.IsCancellationRequested); + var fresh = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, fresh.ActivationId); + Assert.Empty(fresh.Effects); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(fresh.ActivationId, completed.ActivationId); + Assert.Equal("after-delete", Assert.Single(completed.Effects).Value); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completion.SetResult(); } + catch (Exception exception) { completion.SetException(exception); } + }); + return completion.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs new file mode 100644 index 00000000000..0a113ba6e50 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs @@ -0,0 +1,117 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class JournaledTestOutboxBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task EquivalentDuplicates_BeforeAndAfterCommit_PreserveOneOutput() + { + var owner = NewGrain(); + var original = CreateOutput(owner); + var copied = original; + var serializer = Fixture.Client.ServiceProvider.GetRequiredService>(); + var copy = serializer.Deserialize(serializer.SerializeToArray(original)); + Assert.Equal(original.Envelope.EncodedHeaders, copied.Envelope.EncodedHeaders); + Assert.NotEqual(original.Envelope.EncodedHeaders, copy.Envelope.EncodedHeaders); + + await owner.StageOutputAsync(original); + var retained = Assert.Single(Fixture.GetStagedOutput(owner)); + await owner.StageOutputAsync(copied); + Assert.Equal(retained.Envelope.EncodedHeaders, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.EncodedHeaders); + await owner.RetryWriteStateAsync(); + await owner.StageOutputAsync(copy); + Assert.Equal(retained.Envelope.EncodedHeaders, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.EncodedHeaders); + var before = await owner.GetSnapshotAsync(); + await owner.RequestDeactivationAsync(); + var recovered = await owner.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + await owner.StageOutputAsync(copy); + + var output = Assert.Single(Fixture.GetStagedOutput(owner)); + Assert.Equal(original.Envelope.MessageId, output.Envelope.MessageId); + Assert.Equal(0, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), output).Body); + Assert.Equal(1, recovered.OutboxCount); + } + + [Theory] + [InlineData("sender")] + [InlineData("receiver")] + [InlineData("body-bytes")] + public async Task ConflictingDuplicate_RejectsChangedEnvelopeAndPreservesCommittedOutput(string difference) + { + var owner = NewGrain(); + var original = CreateOutput(owner); + await owner.StageOutputAsync(original); + await owner.RetryWriteStateAsync(); + var retained = Assert.Single(Fixture.GetStagedOutput(owner)); + var conflicting = difference switch + { + "sender" => original with { Envelope = TestApplicationProtocol.With(original.ToInboxMessage(), Sessions, + sender: GrainId.Create("other-sender", "1")).Envelope }, + "receiver" => original with { ReceiverId = GrainId.Create("other-receiver", "1") }, + _ => original with { Envelope = TestApplicationProtocol.With(original.ToInboxMessage(), Sessions, + payload: CreateOutput(owner, changed: true).Envelope.Payload).Envelope } + }; + + var exception = await Assert.ThrowsAsync(() => owner.StageOutputAsync(conflicting)); + + Assert.Contains(original.Envelope.MessageId.ToString(), exception.Message, StringComparison.Ordinal); + Assert.Equal(retained.Envelope.EncodedHeaders, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.EncodedHeaders); + await owner.RetryWriteStateAsync(); + await owner.RequestDeactivationAsync(); + var recovered = await owner.GetSnapshotAsync(); + Assert.Equal(1, recovered.OutboxCount); + var output = Assert.Single(Fixture.GetStagedOutput(owner)); + Assert.Equal(original.Envelope.MessageId, output.Envelope.MessageId); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, output.Envelope)); + Assert.Equal(original.ReceiverId, output.ReceiverId); + Assert.Equal(0, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), output).Body); + } + + [Fact] + public async Task HandlerEquivalentDuplicateOutput_CommitsOneEffectAndOneOutput() + { + var receiver = NewGrain(); + var target = GrainId.Create("output-target", "handler"); + var envelope = CreateEnvelope( + receiver, + NewMessage(97, "duplicate-output") with { ForwardTo = target }, + "messages/duplicate-output"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && (snapshot.Effects.Count > 0 || snapshot.InboxDeadLetters.Count > 0)); + + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(1, completed.OutboxCount); + var output = Assert.Single(Fixture.GetStagedOutput(receiver)); + Assert.Equal(target, output.ReceiverId); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(output.Envelope.MessageId, Assert.Single(Fixture.GetStagedOutput(receiver)).Envelope.MessageId); + Assert.Empty(recovered.InboxDeadLetters); + } + + + + + + private OutboxMessage CreateOutput(IMessagingTestGrain owner, bool changed = false) => + TestApplicationProtocol.Create(Fixture.Client.ServiceProvider.GetRequiredService(), + owner.GetGrainId(), owner.GetGrainId(), "output", changed ? 1 : 0); +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs b/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs new file mode 100644 index 00000000000..39e60402481 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs @@ -0,0 +1,9 @@ +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class MessagingClusterCollection +{ + public const string Name = "Messaging cluster"; +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs new file mode 100644 index 00000000000..de8f35124ec --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs @@ -0,0 +1,380 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Placement; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingGrainTypeConfiguratorTests() : MessagingBehaviorTestBase(new BootstrapClusterFixture()) +{ + private BootstrapProbe Probe => ((BootstrapClusterFixture)Fixture).Probe; + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(typeof(PlainBootstrapGrain))] + [InlineData(typeof(ApplicationBootstrapGrain))] + [InlineData(typeof(InterfaceBootstrapGrain))] + [InlineData(typeof(GenericBootstrapGrain))] + [InlineData(typeof(DurableBootstrapGrain))] + [InlineData(typeof(MarkedDurableBootstrapGrain))] + public async Task SelectedComposition_BindsOnceAndReplaysFreshScopedState(Type grainClass) + { + var grain = CreateGrain(grainClass); + Assert.Equal(0, await grain.GetValueAsync()); + var first = Assert.Single(Probe.Get(grain.GetGrainId())); + var state = first.Context.ActivationServices.GetRequiredService(); + AssertComposition(first, state, grainClass, expectedActivationValue: 0); + var setup = GetSetup(first.Context); + Assert.Single(setup.GetInvocationList()); + await grain.SetValueAsync(41); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + using var handler = Fixture.HandlerProbe.Arm(grain.GetGrainId(), BootstrapState.Route); + var envelope = CreateEnvelope(grain); + Assert.Equal(DeliveryStatus.Accepted, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + await handler.WaitUntilEnteredAsync(); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Assert.Equal(41, first.Value!.Value); + Assert.Equal(1, first.Inbox!.Count); + Assert.Equal(0, first.Outbox!.Count); + Assert.Empty(GetProcessed(first.Context)); + Assert.Equal(0, state.HandlerCalls); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var storage = Fixture.Storage.BlockWrite(journal); + handler.Release(); + await storage.WaitUntilEnteredAsync(); + Assert.Equal(42, first.Value!.Value); + Assert.Equal(0, first.Inbox!.Count); + Assert.Equal(1, first.Outbox!.Count); + Assert.Single(GetProcessed(first.Context)); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + storage.Release(); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(0, first.Inbox.Count); + Assert.Single(GetProcessed(first.Context)); + var output = Assert.Single(first.Outbox.Messages); + Assert.Equal(1, state.HandlerCalls); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(1, state.HandlerCalls); + + var other = CreateGrain(grainClass); + Assert.Equal(0, await other.GetValueAsync()); + var isolated = Assert.Single(Probe.Get(other.GetGrainId())); + Assert.NotSame(first.Manager, isolated.Manager); + Assert.NotSame(first.Inbox, isolated.Inbox); + Assert.NotSame(first.Outbox, isolated.Outbox); + Assert.Empty(isolated.Outbox!.Messages); + Assert.Equal(42, await grain.GetValueAsync()); + await other.DeactivateAsync(); + await isolated.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + + await grain.DeactivateAsync(); + await first.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(1, first.Disposals); + Assert.Equal(1, first.GrainDisposals); + Assert.Equal(1, state.Disposals); + Assert.Equal(42, await grain.GetValueAsync()); + var observations = Probe.Get(grain.GetGrainId()); + Assert.Equal(2, observations.Length); + var recovered = observations[1]; + var recoveredState = recovered.Context.ActivationServices.GetRequiredService(); + AssertComposition(recovered, recoveredState, grainClass, expectedActivationValue: 42); + Assert.NotSame(first.Context, recovered.Context); + Assert.NotSame(first.Manager, recovered.Manager); + Assert.NotSame(first.Value, recovered.Value); + Assert.NotSame(first.Inbox, recovered.Inbox); + Assert.NotSame(first.Outbox, recovered.Outbox); + Assert.Same(setup, GetSetup(recovered.Context)); + Assert.Equal(output.Envelope.MessageId, Assert.Single(recovered.Outbox!.Messages).Envelope.MessageId); + Assert.Single(GetProcessed(recovered.Context)); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(0, recoveredState.HandlerCalls); + Assert.Equal(2, Fixture.Storage.GetReadCount(journal)); + } + + [Fact] + public async Task SelectedWithoutConstructorDependencies_MaterializesPrimaryStateBeforeRead() + { + var grain = Control(); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var read = Fixture.Storage.BlockRead(journal); + var activation = grain.PingAsync(); + try + { + await read.WaitUntilEnteredAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Same(observation.ConstructedGrain, observation.Context.GrainInstance); + var manager = observation.Context.ActivationServices.GetRequiredService(); + Assert.Equal(8, BootstrapState.ReadMessagingStates(manager).Count()); + Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox", out _)); + Assert.True(manager.TryGetStateMachine("test-handler-output", out _)); + Assert.Equal(0, observation.Activations); + Assert.Single(GetSetup(observation.Context).GetInvocationList()); + } + finally + { + read.Release(); + } + await activation; + Assert.Equal(1, Fixture.Storage.GetReadCount(journal)); + } + + [Theory] + [InlineData(typeof(UnselectedBootstrapGrain), false)] + [InlineData(typeof(JournalOnlyBootstrapGrain), true)] + public async Task UnselectedComposition_LeavesMessagingUnresolved(Type grainClass, bool journaled) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + await grain.PingAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Equal(1, observation.Activations); + Assert.Null(observation.Inbox); + Assert.Null(observation.Outbox); + Assert.Null(GetSetupOrDefault(observation.Context)); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + Assert.Equal(journaled ? 1 : 0, Fixture.Storage.GetCreationCount(journal)); + Assert.Equal(journaled ? 1 : 0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + if (journaled) + { + Assert.NotNull(observation.Manager); + Assert.True(observation.Manager.TryGetStateMachine("bootstrap-journal-only", out var state)); + Assert.Same(observation.Value, state); + Assert.False(observation.Manager.TryGetStateMachine("__orleans.messaging.inbox", out _)); + Assert.False(observation.Manager.TryGetStateMachine("test-handler-output", out _)); + Assert.Empty(BootstrapState.ReadMessagingStates(observation.Manager)); + } + else + { + Assert.Null(observation.Manager); + } + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(ReentrantBootstrapGrain), "non-reentrant")] + [InlineData(typeof(StatelessBootstrapGrain), "one activation")] + [InlineData(typeof(MayInterleaveBootstrapGrain), "non-reentrant")] + [InlineData(typeof(AlwaysInterleaveBootstrapGrain), "interleavable method")] + [InlineData(typeof(MetadataReentrantBootstrapGrain), "non-reentrant")] + [InlineData(typeof(MetadataMayInterleaveBootstrapGrain), "non-reentrant")] + public async Task UnsupportedMarkedModel_FailsBeforeStorageAndCleansScope(Type grainClass, string diagnostic) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains(diagnostic, exception.ToString(), StringComparison.Ordinal); + Assert.Contains(grainClass.Name, exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.NotNull(observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.Equal(0, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(MetadataStatelessPlacementBootstrapGrain), "StatelessWorkerPlacement")] + [InlineData(typeof(AliasedStatelessPlacementBootstrapGrain), BootstrapClusterFixture.StatelessPlacementAlias)] + public async Task ResolvedStatelessPlacement_FailsBeforeStorageAndCleansScope(Type grainClass, string placementKey) + { + Assert.False(grainClass.IsDefined(typeof(StatelessWorkerAttribute), inherit: true)); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var services = Fixture.Cluster.Silos[0].ServiceProvider; + var properties = services.GetRequiredService().GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(placementKey, properties.Properties[WellKnownGrainTypeProperties.PlacementStrategy]); + var placement = services.GetRequiredService().GetPlacementStrategy(grain.GetGrainId().Type); + Assert.Equal(new StatelessWorkerAttribute().PlacementStrategy.GetType(), placement.GetType()); + Assert.False(placement.IsUsingGrainDirectory); + if (placementKey == BootstrapClusterFixture.StatelessPlacementAlias) + { + Assert.Same(services.GetRequiredKeyedService(placementKey), placement); + } + + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("one activation", exception.ToString(), StringComparison.Ordinal); + Assert.Contains(grainClass.Name, exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.IsType(grainClass, observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.Equal(0, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(MetadataOrdinaryPlacementBootstrapGrain), nameof(RandomPlacement))] + [InlineData(typeof(AliasedOrdinaryPlacementBootstrapGrain), BootstrapClusterFixture.OrdinaryPlacementAlias)] + public async Task ResolvedOrdinaryPlacement_WithoutMessagingConstructorDependencies_InitializesNormally(Type grainClass, string placementKey) + { + Assert.False(grainClass.IsDefined(typeof(StatelessWorkerAttribute), inherit: true)); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var services = Fixture.Cluster.Silos[0].ServiceProvider; + var properties = services.GetRequiredService().GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(placementKey, properties.Properties[WellKnownGrainTypeProperties.PlacementStrategy]); + var placement = services.GetRequiredService().GetPlacementStrategy(grain.GetGrainId().Type); + Assert.IsType(placement); + Assert.True(placement.IsUsingGrainDirectory); + if (placementKey == BootstrapClusterFixture.OrdinaryPlacementAlias) + { + Assert.Same(services.GetRequiredKeyedService(placementKey), placement); + } + + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var read = Fixture.Storage.BlockRead(journal); + var activation = grain.PingAsync(); + BootstrapObservation observation; + try + { + await read.WaitUntilEnteredAsync(); + observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.IsType(grainClass, observation.ConstructedGrain); + Assert.Same(observation.ConstructedGrain, observation.Context.GrainInstance); + Assert.Equal(0, observation.Activations); + var manager = observation.Context.ActivationServices.GetRequiredService(); + Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox", out var inbox)); + Assert.Same(observation.Context.ActivationServices.GetRequiredKeyedService< + IDurableDictionary>("__orleans.messaging.inbox"), inbox); + Assert.True(manager.TryGetStateMachine("test-handler-output", out var outbox)); + Assert.Same(((JournaledTestOutbox)observation.Context.ActivationServices.GetRequiredService()).StoredMessages, outbox); + Assert.Single(GetSetup(observation.Context).GetInvocationList()); + } + finally + { + read.Release(); + } + + await activation; + Assert.Equal(1, observation.Activations); + Assert.Equal(1, Fixture.Storage.GetCreationCount(journal)); + Assert.Equal(1, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public async Task SetupFailure_DisposesScopeAndPreservesOriginalError() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.IsType(observation.ExpectedFailure); + Assert.Contains(observation.ExpectedFailure.Message, exception.ToString(), StringComparison.Ordinal); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.NotNull(observation.Inbox); + Assert.NotNull(observation.Outbox); + var extension = Assert.IsAssignableFrom(observation.Extension); + var shutdown = (CancellationTokenSource)extension.GetType().GetField("_shutdownCts", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(extension)!; + Assert.Throws(() => shutdown.Token); + extension.Dispose(); + Assert.Equal(1, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public async Task UnselectedEndpointDependency_FailsExplicitly() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("Inbox activation requires IMessagingGrain or DurableGrain", exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(1, observation.Disposals); + Assert.Equal(1, observation.GrainDisposals); + Assert.Null(GetSetupOrDefault(observation.Context)); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public void RepeatedRegistration_InstallsOneTypeConfigurator() + { + var services = new ServiceCollection(); + ReceiverTestServices.Add(services, static _ => { }); + ReceiverTestServices.Add(services, static _ => { }); + var descriptor = Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IConfigureGrainTypeComponents)); + Assert.Equal(ServiceLifetime.Singleton, descriptor.Lifetime); + Assert.Equal(ReceiverTestServices.GetImplementationType("MessagingGrainTypeConfigurator"), descriptor.ImplementationType); + Assert.Empty(typeof(IMessagingGrain).GetInterfaces()); + Assert.Empty(typeof(IMessagingGrain).GetMethods()); + Assert.False(typeof(IAddressable).IsAssignableFrom(typeof(IMessagingGrain))); + } + + private static void AssertComposition(BootstrapObservation observation, BootstrapState state, Type grainClass, int expectedActivationValue) + { + Assert.Equal(grainClass, observation.ConstructedGrain!.GetType()); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.Same(observation.ConstructedGrain, state.GrainAtActivation); + Assert.Equal(8, state.MessagingStateCountAtActivation); + Assert.Equal(1, observation.Activations); + Assert.Equal(expectedActivationValue, state.ActivationValue); + var services = observation.Context.ActivationServices; + Assert.Same(observation.Manager, services.GetRequiredService()); + var applicationManager = services.GetRequiredService(); + Assert.Same(observation.Manager, applicationManager); + Assert.Same(observation.Value, applicationManager.GetOrAddState>("bootstrap-value")); + Assert.True(applicationManager.TryGetState>("test-handler-output", out var applicationOutbox)); + Assert.Same(((JournaledTestOutbox)observation.Outbox!).StoredMessages, applicationOutbox); + Assert.Same(applicationOutbox, applicationManager.GetOrAddState>("test-handler-output")); + Assert.Same(observation.Value, services.GetRequiredKeyedService>("bootstrap-value")); + Assert.Same(observation.Inbox, services.GetRequiredService()); + Assert.Same(observation.Outbox, services.GetRequiredService()); + var tryGetHandler = observation.Inbox!.GetType().GetMethod("TryGetHandler", BindingFlags.Instance | BindingFlags.NonPublic)!; + object?[] arguments = [null]; + Assert.True((bool)tryGetHandler.Invoke(observation.Inbox, arguments)!); + var handler = arguments[0]; + Assert.Same(state, handler); + Assert.Equal(8, BootstrapState.ReadMessagingStates(observation.Manager!).Count()); + var primary = services.GetRequiredKeyedService>("__orleans.messaging.inbox"); + Assert.Same(primary, applicationManager.GetOrAddState>("__orleans.messaging.inbox")); + Assert.True(observation.Manager!.TryGetStateMachine("test-handler-output", out var output)); + Assert.Same(applicationOutbox, output); + } + private IBootstrapTestGrain CreateGrain(Type grainClass) => grainClass == typeof(GenericBootstrapGrain) + ? Fixture.Client.GetGrain>(Guid.NewGuid()) + : Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + private IBootstrapControlGrain Control() => Fixture.Client.GetGrain(Guid.NewGuid(), typeof(T).FullName!); + private InboxMessage CreateEnvelope(IBootstrapTestGrain grain) => + TestApplicationProtocol.CreateInbox(Fixture.Client.ServiceProvider.GetRequiredService(), + GrainId.Create("bootstrap-sender", "external"), BootstrapState.Route, 1); + private static IDurableDictionary GetProcessed(IGrainContext context) => + context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed"); + private static Delegate GetSetup(IGrainContext context) => Assert.IsAssignableFrom(GetSetupOrDefault(context)); + private static object? GetSetupOrDefault(IGrainContext context) + { + var shared = context.GetType().GetField("_shared", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(context)!; + return shared.GetType().GetField("_activationSetup", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(shared); + } + private void AssertNoStorageWork(GrainId grainId) + { + var journal = JournalId.FromGrainId(grainId); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + Assert.Equal(0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + } + private void AssertNoScheduledJobs(GrainId grainId) + { + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, grainId)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount("orleans.messaging.outbox-drain", grainId)); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs new file mode 100644 index 00000000000..9df6c9aedf0 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs @@ -0,0 +1,279 @@ +using System.Diagnostics.Metrics; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingMetricCardinalityTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task PendingAndProcessedDuplicates_RecordOneBoundedReceiptEach() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var services = context.ActivationServices; + var extension = (IInboxExtension)services.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + using var hold = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "hold-metric-pending"); + var turn = receiver.HoldPumpTurnAsync("hold-metric-pending", deactivate: false); + await hold.WaitUntilEnteredAsync(); + var envelope = CreateEnvelope(receiver, NewMessage(451, "duplicate-receipts")); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery()).Status); + var job = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + var pending = Fixture.GetSnapshot(receiver); + Assert.Equal(1, pending.InboxCount); + Assert.Equal(0, pending.ProcessedMessageCount); + Assert.Empty(pending.Effects); + Assert.Equal(job.Id, pending.InboxJob!.Id); + Assert.Equal(job.ShardId, pending.InboxJob.ShardId); + var instruments = services.GetRequiredService(ReceiverTestServices.GetImplementationType("MessagingInstruments")); + var received = (Instrument)instruments.GetType().GetField("_inboxMessagesReceived", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instruments)!; + using var probe = new MessageMetricListener(received.Meter); + var grainType = receiver.GetGrainId().Type.ToString(); + + Assert.Equal(DeliveryStatus.Duplicate, (await StartDelivery()).Status); + Assert.Single(probe.Read("inbox-messages-received")).AssertCounter(grainType, "duplicate"); + hold.Release(); + await turn; + var processed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(0, processed.InboxCount); + Assert.Equal(1, processed.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(processed.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + var receipts = probe.Read("inbox-messages-received"); + Assert.Equal(2, receipts.Length); + Assert.All(receipts, receipt => receipt.AssertCounter(grainType, "duplicate")); + Assert.Single(receipts.Select(receipt => receipt.Series).Distinct()); + + Task StartDelivery() + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + } + + [Theory] + [InlineData(1)] + [InlineData(64)] + public async Task DistinctApplicationRoutes_KeepFixedMetricSeriesAndMissingHandlerOutcomes(int routeCount) + { + var receiver = NewGrain(); + var missingHandler = NewGrain(); + await missingHandler.ConfigureHandlerAsync(false); + await RefreshSeededOwnerAsync(missingHandler); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var instruments = context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("MessagingInstruments")); + var received = (Instrument)instruments.GetType().GetField("_inboxMessagesReceived", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instruments)!; + using var probe = new MessageMetricListener(received.Meter); + var grainType = receiver.GetGrainId().Type.ToString(); + var routes = new HashSet(); + for (var index = 0; index < routeCount; index++) + { + var unknown = $"unknown/{Guid.NewGuid():N}"; + routes.Add(unknown); + var rejected = CreateEnvelope(missingHandler, NewMessage(index, "unknown"), unknown); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(missingHandler, rejected)).Status); + Assert.Equal(unknown, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), rejected).Route); + var valid = $"messages/cardinality/{Guid.NewGuid():N}"; + routes.Add(valid); + var accepted = CreateEnvelope(receiver, NewMessage(index, "accepted"), valid); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, accepted)).Status); + Assert.Equal(valid, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), accepted).Route); + await Fixture.WaitForEffectCountAsync(receiver, index + 1); + } + + await probe.WaitForCountAsync("inbox-processing-duration", routeCount); + var receivedRows = probe.Read("inbox-messages-received"); + Assert.Equal(routeCount * 2, routes.Count); + Assert.Equal(routeCount * 2, receivedRows.Length); + Assert.Equal(2, receivedRows.Select(row => row.Series).Distinct().Count()); + var acceptedRows = receivedRows.Where(row => row.Tags.Any(tag => tag.Key == "status" && Equals(tag.Value, "accepted"))).ToArray(); + var rejectedRows = receivedRows.Where(row => row.Tags.Any(tag => tag.Key == "status" && Equals(tag.Value, "handler_not_found"))).ToArray(); + Assert.Equal(routeCount, acceptedRows.Length); + Assert.Equal(routeCount, rejectedRows.Length); + Assert.All(acceptedRows, row => row.AssertCounter(grainType, "accepted")); + Assert.All(rejectedRows, row => row.AssertCounter(grainType, "handler_not_found")); + var processed = probe.Read("inbox-messages-processed"); + Assert.Equal(routeCount, processed.Length); + Assert.Single(processed.Select(row => row.Series).Distinct()); + Assert.All(processed, row => row.AssertCounter(grainType, "success")); + var durations = probe.Read("inbox-processing-duration"); + Assert.Equal(routeCount, durations.Length); + Assert.Single(durations.Select(row => row.Series).Distinct()); + Assert.All(durations, row => + { + Assert.Equal("ms", row.Unit); + Assert.True(row.Value >= 0); + Assert.Equal(new[] { new KeyValuePair("grain_type", grainType) }, row.Tags); + }); + var completed = await receiver.GetSnapshotAsync(); + Assert.Equal(routeCount, completed.Effects.Count); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + Assert.Equal(routeCount, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Empty(completed.InboxDeadLetters); + } +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingInstrumentTagTests +{ + [Theory] + [InlineData("OnInboxMessageReceived", "inbox-messages-received", "accepted", false)] + [InlineData("OnInboxMessageProcessed", "inbox-messages-processed", "success", false)] + [InlineData("OnOutboxMessageSent", "outbox-messages-sent", null, false)] + [InlineData("OnOutboxMessageDelivered", "outbox-messages-delivered", "duplicate", false)] + [InlineData("OnInboxProcessingDuration", "inbox-processing-duration", null, true)] + [InlineData("OnOutboxDeliveryDuration", "outbox-delivery-duration", null, true)] + public void MessageInstruments_EmitExactBoundedTagsAndUnchangedValues(string method, string instrumentName, string? status, bool duration) + { + using var factory = new TestMeterFactory(); + var instruments = Activator.CreateInstance(ReceiverTestServices.GetImplementationType("MessagingInstruments"), + new OrleansInstruments(factory))!; + using var probe = new MessageMetricListener(factory.Meter); + const string grainType = "instrument-test"; + object[] arguments = duration ? [TimeSpan.FromMilliseconds(12.5), grainType] + : status is not null ? [grainType, status] : [grainType]; + Invoke(instruments, method, arguments); + var measurement = Assert.Single(probe.Read(instrumentName)); + Assert.Equal(MessageMetricListener.Prefix + instrumentName, measurement.Name); + if (duration) + { + Assert.Equal("ms", measurement.Unit); + Assert.Equal(12.5, measurement.Value); + Assert.Equal(new[] { new KeyValuePair("grain_type", grainType) }, measurement.Tags); + Invoke(instruments, method, TimeSpan.FromMilliseconds(-10), grainType); + var values = probe.Read(instrumentName); + Assert.Equal(2, values.Length); + Assert.Equal(0, values[1].Value); + Assert.Equal(measurement.Tags, values[1].Tags); + } + else + { + measurement.AssertCounter(grainType, status); + } + } + + [Fact] + public void OrphanAndDepthInstruments_RetainExistingTagsAndValues() + { + using var factory = new TestMeterFactory(); + var instruments = Activator.CreateInstance(ReceiverTestServices.GetImplementationType("MessagingInstruments"), + new OrleansInstruments(factory))!; + using var probe = new MessageMetricListener(factory.Meter); + Invoke(instruments, "OnOrphanedJobReclaimed", "instrument-test", ReceiverTestServices.InboxJobName); + Invoke(instruments, "OnInboxDepthChanged", 3); + Invoke(instruments, "OnInboxDepthChanged", -1); + Invoke(instruments, "OnOutboxDepthChanged", 4); + Invoke(instruments, "OnOutboxDepthChanged", -1); + probe.ObserveGauges(); + var orphan = Assert.Single(probe.Read("orphaned-jobs-reclaimed")); + Assert.Equal(1, orphan.Value); + Assert.Null(orphan.Unit); + Assert.Equal(new[] { new KeyValuePair("grain_type", "instrument-test"), new("job_name", ReceiverTestServices.InboxJobName) }, orphan.Tags); + var inboxDepth = Assert.Single(probe.Read("inbox-depth")); + var outboxDepth = Assert.Single(probe.Read("outbox-depth")); + Assert.Equal(2, inboxDepth.Value); + Assert.Equal(3, outboxDepth.Value); + Assert.Empty(inboxDepth.Tags); + Assert.Empty(outboxDepth.Tags); + Assert.Null(inboxDepth.Unit); + Assert.Null(outboxDepth.Unit); + } + + private static void Invoke(object instruments, string method, params object[] arguments) => + instruments.GetType().GetMethod(method, BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(instruments, arguments); + + private sealed class TestMeterFactory : IMeterFactory + { + public Meter Meter { get; private set; } = null!; + public Meter Create(MeterOptions options) => Meter = new Meter(options); + public void Dispose() => Meter.Dispose(); + } +} + +internal sealed class MessageMetricListener : IDisposable +{ + internal const string Prefix = "orleans-messaging-"; + private readonly MeterListener _listener; + private readonly object _lock = new(); + private readonly List _measurements = []; + private readonly List<(string Name, int Count, TaskCompletionSource Completion)> _waiters = []; + + public MessageMetricListener(Meter meter) + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (ReferenceEquals(instrument.Meter, meter) && instrument.Name.StartsWith(Prefix, StringComparison.Ordinal)) + listener.EnableMeasurementEvents(instrument); + } + }; + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.Start(); + } + + private void Record(Instrument instrument, double value, ReadOnlySpan> tags) + { + lock (_lock) + { + _measurements.Add(new(instrument.Name, instrument.Unit, value, tags.ToArray())); + foreach (var waiter in _waiters.ToArray()) + { + if (_measurements.Count(row => row.Name == waiter.Name) >= waiter.Count) + { + _waiters.Remove(waiter); + waiter.Completion.TrySetResult(); + } + } + } + } + public Task WaitForCountAsync(string name, int count) + { + lock (_lock) + { + if (_measurements.Count(row => row.Name == Prefix + name) >= count) return Task.CompletedTask; + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add((Prefix + name, count, completion)); + return completion.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + } + public Measurement[] Read(string name) + { + lock (_lock) return _measurements.Where(row => row.Name == Prefix + name).ToArray(); + } + public void ObserveGauges() => _listener.RecordObservableInstruments(); + public void Dispose() => _listener.Dispose(); + internal sealed record Measurement(string Name, string? Unit, double Value, KeyValuePair[] Tags) + { + public string Series => string.Join(";", Tags.Select(tag => $"{tag.Key}={tag.Value}")); + public void AssertCounter(string grainType, string? status = null) + { + Assert.Equal(1, Value); + Assert.Null(Unit); + KeyValuePair[] expected = status is null + ? [new("grain_type", grainType)] + : [new("grain_type", grainType), new("status", status)]; + Assert.Equal(expected, Tags); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingOwnershipRecoveryTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingOwnershipRecoveryTests.cs new file mode 100644 index 00000000000..b2fec9c19af --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingOwnershipRecoveryTests.cs @@ -0,0 +1,299 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingOwnershipRecoveryTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task JournalFailureDuringLocalScheduling_RejectsLaterWriteAndFreshOwnerReplaysAcknowledgedState() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 77, "uncommitted")); + using var schedule = Fixture.JobManagerProbe.BlockNext("orleans.messaging.inbox-drain"); + var envelope = CreateEnvelope(receiver, NewMessage(77, "failed-during-schedule")); + var delivery = DeliverAsync(receiver, envelope); + await schedule.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var failure = await Assert.ThrowsAsync(() => oldManager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(failure, await oldGrain.DeactivationFailure.Task); + schedule.Continue(); + var rejected = await Assert.ThrowsAsync(() => delivery); + Assert.Contains("fenced", rejected.Message, StringComparison.Ordinal); + Assert.Equal(failure.Message, Assert.IsType(rejected.InnerException).Message); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var oldState = oldGrain.GetSnapshotForTest(); + Assert.Equal(1, oldState.InboxCount); + Assert.NotNull(oldState.InboxJobId); + Assert.NotNull(oldState.InboxJob); + Assert.Single(oldState.Effects); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(oldState.ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task FailedInboxAcceptance_FencesOldObjectsAndFreshReplayOmitsEnvelope() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var envelope = CreateEnvelope(receiver, NewMessage(2, "failed-acceptance")); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var failed = oldGrain.GetSnapshotForTest(); + Assert.Equal(1, failed.InboxCount); + Assert.NotNull(failed.InboxJob); + Assert.Empty(failed.Effects); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.NotSame(oldManager, Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService()); + Assert.Equal(0, recovered.InboxCount); + Assert.Null(recovered.InboxJob); + Assert.Empty(recovered.Effects); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } + + [Fact] + public async Task AmbiguousInboxAcceptanceCommit_PreservesAndProcessesRecoveredEnvelope() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + Fixture.Storage.FailAfterWrite(journalId); + var envelope = CreateEnvelope(receiver, NewMessage(76, "ambiguous-acceptance")); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, completed.ActivationId); + Assert.Equal("ambiguous-acceptance", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task Inbox_PrecommitCrash_ReclaimsScheduledOrphanAfterRecovery() + { + const string jobName = "orleans.messaging.inbox-drain"; + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var barrier = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var attemptBaseline = Fixture.Metrics.GetCount("orleans-durablejobs-job-attempts-started"); + var completionBaseline = Fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed"); + var orphanBaseline = Fixture.Metrics.GetCount( + "orleans-messaging-orphaned-jobs-reclaimed", + jobName); + Fixture.JobManagerProbe.DuplicateNext(jobName); + var envelope = CreateEnvelope(receiver, NewMessage(3, "inbox-orphan")); + + var delivery = DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + await Fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-job-attempts-started", + attemptBaseline + 2); + + Assert.Equal( + orphanBaseline, + Fixture.Metrics.GetCount("orleans-messaging-orphaned-jobs-reclaimed", jobName)); + barrier.Fail(); + await Assert.ThrowsAnyAsync(() => delivery); + + var recovered = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId); + await Fixture.Metrics.WaitForCountAsync( + "orleans-messaging-orphaned-jobs-reclaimed", + orphanBaseline + 2, + jobName); + await Fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-jobs-completed", + completionBaseline + 2); + + Assert.Equal(0, recovered.InboxCount); + Assert.Null(recovered.InboxJobId); + Assert.Null(recovered.InboxJob); + Assert.Empty(recovered.Effects); + Assert.Equal( + orphanBaseline + 2, + Fixture.Metrics.GetCount("orleans-messaging-orphaned-jobs-reclaimed", jobName)); + } + + [Fact] + public async Task InboxJobClearWriteFailure_FencesThenRecoversInFreshActivation() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/inbox-clear-retry"); + var envelope = CreateEnvelope(receiver, NewMessage(57, "inbox-clear-retry"), "messages/inbox-clear-retry"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId()), matchingWrite: 2); + handler.Release(); + + var recovered = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId); + var cleaned = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && snapshot.InboxJobId is null); + + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(cleaned.Effects).Count); + Assert.Empty(cleaned.InboxDeadLetters); + Assert.Null(cleaned.InboxJobId); + } + + [Fact] + public async Task DeliveryIntoEmptyInbox_ReplacesStalePersistedJobOwnership() + { + var receiver = NewGrain(); + var staleJobId = $"stale-{Guid.NewGuid():N}"; + await receiver.SetInboxOwnershipAsync( + staleJobId, + new DurableJob + { + Id = $"stale-job-{Guid.NewGuid():N}", + Name = "orleans.messaging.inbox-drain", + DueTime = DateTimeOffset.UtcNow, + TargetGrainId = receiver.GetGrainId(), + ShardId = $"stale-shard-{Guid.NewGuid():N}", + Metadata = new Dictionary + { + ["orleans.messaging.ownership-id"] = staleJobId + } + }); + await RefreshSeededOwnerAsync(receiver); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/stale-owner"); + var envelope = CreateEnvelope(receiver, NewMessage(58, "stale-owner"), "messages/stale-owner"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var accepted = Fixture.GetSnapshot(receiver); + + Assert.NotNull(accepted.InboxJobId); + Assert.NotEqual(staleJobId, accepted.InboxJobId); + + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("stale-owner", Assert.Single(completed.Effects).Value); + } + + [Fact] + public async Task Inbox_StaleGenerationCompletesWithoutClearingNewerOwner() + { + var receiver = NewGrain(); + const string route = "messages/stale-inbox-generation"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(60, "newer-inbox-owner"), route); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var owned = Fixture.GetSnapshot(receiver); + Assert.False(string.IsNullOrEmpty(owned.InboxJobId)); + var completionBaseline = Fixture.Metrics.GetCount("orleans-durablejobs-jobs-completed"); + var manager = Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService(); + + await manager.ScheduleJobAsync( + new ScheduleJobRequest + { + Target = receiver.GetGrainId(), + JobName = "orleans.messaging.inbox-drain", + DueTime = DateTimeOffset.UtcNow, + Metadata = new Dictionary + { + ["orleans.messaging.ownership-id"] = "0" + } + }, + TestContext.Current.CancellationToken); + await Fixture.Metrics.WaitForCountAsync( + "orleans-durablejobs-jobs-completed", + completionBaseline + 1); + + Assert.Equal(owned.InboxJobId, Fixture.GetSnapshot(receiver).InboxJobId); + handler.Release(); + Assert.Equal("newer-inbox-owner", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + } + + [Fact] + public async Task SchedulingFailureBeforeApply_PreservesStateAndRetrySchedulesOnce() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(61, "schedule-failure")); + Fixture.JobManagerProbe.FailNext(ReceiverTestServices.InboxJobName); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + var unchanged = await receiver.GetSnapshotAsync(); + Assert.Equal(0, unchanged.InboxCount); + Assert.Null(unchanged.InboxJobId); + Assert.Null(unchanged.InboxJob); + Assert.Empty(unchanged.Effects); + Assert.Empty(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(2, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(ReceiverTestServices.InboxJobName, receiver.GetGrainId())); + } + + [Fact] + public async Task AmbiguousSchedulingFailure_LeavesAcceptanceLocalAndRetryUsesNewToken() + { + const string jobName = "orleans.messaging.inbox-drain"; + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(59, "inbox-schedule-retry")); + var scheduledBaseline = Fixture.JobManagerProbe.GetScheduledJobs(jobName, receiver.GetGrainId()).Count; + Fixture.JobManagerProbe.FailAfterNext(jobName); + + await Assert.ThrowsAsync( + () => DeliverAsync(receiver, envelope)); + Assert.Equal(0, (await receiver.GetSnapshotAsync()).InboxCount); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + + var effect = Assert.Single(completed.Effects); + Assert.Equal("inbox-schedule-retry", effect.Value); + Assert.Equal(1, effect.Count); + Assert.Equal( + 2, + Fixture.JobManagerProbe.GetAttemptCount( + "orleans.messaging.inbox-drain", + receiver.GetGrainId())); + Assert.Equal( + 2, + Fixture.JobManagerProbe.GetSuccessCount(jobName, receiver.GetGrainId())); + var scheduled = Fixture.JobManagerProbe.GetScheduledJobs(jobName, receiver.GetGrainId()) + .Skip(scheduledBaseline) + .ToArray(); + Assert.Equal(2, scheduled.Length); + Assert.NotEqual(scheduled[0].Id, scheduled[1].Id); + Assert.NotEqual( + scheduled[0].Metadata!["orleans.messaging.ownership-id"], + scheduled[1].Metadata!["orleans.messaging.ownership-id"]); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs new file mode 100644 index 00000000000..0072eca521e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs @@ -0,0 +1,147 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingRoutingAndDeadLetterTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task InboxDeadLettersRetainNewestEntriesWithinConfiguredCapacity() + { + var receiver = NewGrain(); + var messageIds = new List(); + for (var sequence = 0; sequence < 3; sequence++) + { + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), + 20 + sequence, + $"dead-letter-{sequence}", + ThrowDuringPreparation: true)); + messageIds.Add(envelope.Envelope.MessageId); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.InboxDeadLetters.Any(entry => entry.MessageId == envelope.Envelope.MessageId)); + } + + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(2, state.InboxDeadLetters.Count); + Assert.DoesNotContain(state.InboxDeadLetters, entry => entry.MessageId == messageIds[0]); + Assert.Contains(state.InboxDeadLetters, entry => entry.MessageId == messageIds[1]); + Assert.Contains(state.InboxDeadLetters, entry => entry.MessageId == messageIds[2]); + } + + [Fact] + public async Task ActivationRemovesExpiredInboxDeadLetters() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), + 30, + "expired-dead-letter", + ThrowDuringPreparation: true)); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var before = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Fixture.Clock.Advance(TimeSpan.FromHours(2)); + await receiver.RequestDeactivationAsync(); + var after = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId + && snapshot.InboxDeadLetters.Count == 0); + + Assert.Empty(after.InboxDeadLetters); + } + + [Fact] + public async Task MalformedTypedBody_DeadLettersAndDoesNotBlockLaterValidMessage() + { + var receiver = NewGrain(); + var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed)).Status); + var poisoned = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Empty(poisoned.Effects); + var deadLetter = Assert.Single(poisoned.InboxDeadLetters); + Assert.Equal(malformed.Envelope.MessageId, deadLetter.MessageId); + Assert.Contains(nameof(TestMessage), deadLetter.Reason, StringComparison.Ordinal); + + var valid = CreateEnvelope(receiver, NewMessage(41, "valid-after-poison"), "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, valid)).Status); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("valid-after-poison", Assert.Single(recovered.Effects).Value); + Assert.Single(recovered.InboxDeadLetters); + } + + [Fact] + public async Task InboxDeadLetterRemoval_IsDurable() + { + var receiver = NewGrain(); + var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed)).Status); + _ = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.True(await receiver.RemoveInboxDeadLetterAsync( + malformed.Envelope.MessageId)); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + Assert.False(await receiver.RemoveInboxDeadLetterAsync( + malformed.Envelope.MessageId)); + + await receiver.RequestDeactivationAsync(); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + } + + [Fact] + public async Task HandlerNotFound_IsRejectedWithoutInboxPersistence() + { + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + await RefreshSeededOwnerAsync(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(71, "missing"), "unknown/route"); + + var result = await DeliverAsync(receiver, envelope); + + Assert.Equal(DeliveryStatus.HandlerNotFound, result.Status); + Assert.Equal("No inbox handler is registered.", result.Message); + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(0, state.InboxCount); + Assert.Empty(state.Effects); + } + + [Fact] + public async Task Deliver_ImplicitReceiverNamespacesTheSameIncomingCommand() + { + var receiver = NewGrain(); + var declaredReceiver = NewGrain(); + var envelope = CreateEnvelope( + declaredReceiver, + NewMessage(74, "wrong-receiver")); + + Assert.Null(typeof(InboxMessage).GetProperty("ReceiverId")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var first = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(first.Effects).LogicalId); + Assert.Equal(1, Assert.Single(first.Effects).Count); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(declaredReceiver, envelope)).Status); + var delivered = await Fixture.WaitForEffectCountAsync(declaredReceiver, 1); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(delivered.Effects).LogicalId); + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(declaredReceiver, envelope)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).ProcessedMessageCount); + Assert.Equal(1, (await declaredReceiver.GetSnapshotAsync()).ProcessedMessageCount); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs new file mode 100644 index 00000000000..2ac6559c263 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs @@ -0,0 +1,270 @@ +using System.Collections; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ProcessedRetentionMaintenanceTests : MessagingBehaviorTestBase +{ + public ProcessedRetentionMaintenanceTests() : base(new RetentionFixture()) { } + + [Fact] + public async Task SustainedNonemptyInbox_CompactsAtExpiryAndPreservesCurrentRecords() + { + var receiver = await StartBacklogAsync(); + var owner = (await receiver.GetSnapshotAsync()).InboxJobId; + var old = CreateEnvelope(receiver, NewMessage(120, "old")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Fixture.Clock.Advance(TimeSpan.FromMinutes(5)); + var current = CreateEnvelope(receiver, NewMessage(121, "current")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, current)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 2); + _ = await receiver.GetSnapshotAsync(); + Fixture.Clock.Advance(TimeSpan.FromMinutes(5) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, old)).Status); + var processed = GetProcessed(receiver); + Assert.Equal(2, processed.Count); + + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + var fresh = CreateEnvelope(receiver, NewMessage(122, "fresh")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, fresh)).Status); + var maintained = await Fixture.WaitForEffectCountAsync(receiver, 3); + _ = await receiver.GetSnapshotAsync(); + + Assert.Equal(1, maintained.InboxCount); + Assert.Equal(owner, maintained.InboxJobId); + Assert.Empty(maintained.InboxDeadLetters); + Assert.Equal(2, processed.Count); + Assert.False(processed.ContainsKey(old.Envelope.MessageId)); + Assert.True(processed.ContainsKey(current.Envelope.MessageId)); + Assert.True(processed.ContainsKey(fresh.Envelope.MessageId)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, current)).Status); + } + + [Fact] + public async Task FreshActivation_CompactsProcessedRecordsWithoutDeadLettersOrOwnerClear() + { + var receiver = await StartBacklogAsync(); + var old = CreateEnvelope(receiver, NewMessage(123, "initial-expiry")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + var before = await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + await receiver.RequestDeactivationAsync(); + var after = await receiver.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, after.ActivationId); + Assert.Equal(1, after.InboxCount); + Assert.Equal(before.InboxJobId, after.InboxJobId); + Assert.Empty(after.InboxDeadLetters); + Assert.Empty(GetProcessed(receiver)); + Assert.Equal(1, Assert.Single(after.Effects).Count); + } + + [Fact] + public async Task BusyInbox_AmortizesScansAcrossRetentionCadence() + { + var receiver = await StartBacklogAsync(); + var context = Fixture.GetGrainContext(receiver); + var extension = context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var counted = new CountingProcessedDictionary(GetProcessed(receiver)); + extension.GetType().GetField("_processed", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(extension, counted); + var oldest = CreateEnvelope(receiver, NewMessage(124, "oldest")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, oldest)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + var next = CreateEnvelope(receiver, NewMessage(125, "next")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, next)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 2); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(0, counted.Enumerations); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + var trigger = CreateEnvelope(receiver, NewMessage(126, "sweep")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, trigger)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 3); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(1, counted.Enumerations); + Assert.False(counted.ContainsKey(oldest.Envelope.MessageId)); + Assert.True(counted.ContainsKey(next.Envelope.MessageId)); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + for (var i = 0; i < 8; i++) + { + var message = CreateEnvelope(receiver, NewMessage(130 + i, "within-cadence")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 4 + i); + _ = await receiver.GetSnapshotAsync(); + } + Assert.Equal(1, counted.Enumerations); + Assert.True(counted.ContainsKey(next.Envelope.MessageId)); + Fixture.Clock.Advance(TimeSpan.FromMinutes(2.5) - TimeSpan.FromTicks(1)); + var following = CreateEnvelope(receiver, NewMessage(140, "following-sweep")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, following)).Status); + var after = await Fixture.WaitForEffectCountAsync(receiver, 12); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(2, counted.Enumerations); + Assert.False(counted.ContainsKey(next.Envelope.MessageId)); + Assert.True(counted.ContainsKey(trigger.Envelope.MessageId)); + Assert.Equal(1, after.InboxCount); + } + + [Fact] + public async Task PumpMaintenance_WritesOnlyWhenExpiredRecordsExist() + { + var receiver = await StartBacklogAsync(); + var old = CreateEnvelope(receiver, NewMessage(141, "pump-expiry")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + var owned = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await RunPumpAsync(receiver)).Status); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await RunPumpAsync(receiver)).Status); + + var maintained = await receiver.GetSnapshotAsync(); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(GetProcessed(receiver)); + Assert.Equal(1, maintained.InboxCount); + Assert.Equal(owned.InboxJobId, maintained.InboxJobId); + Assert.Empty(maintained.InboxDeadLetters); + Assert.Equal(1, Assert.Single(maintained.Effects).Count); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await RunPumpAsync(receiver)).Status); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + } + + [Fact] + public async Task MaintenanceWriteFailure_FencesOldStateAndFreshReplayRetriesCompaction() + { + var receiver = await StartBacklogAsync(); + var old = CreateEnvelope(receiver, NewMessage(142, "failed-maintenance")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + Fixture.Storage.FailWrite(journal); + + var failure = await Assert.ThrowsAsync(() => RunPumpAsync(receiver)); + + Assert.Same(failure, await oldGrain.DeactivationFailure.Task); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, oldGrain.GetSnapshotForTest().ProcessedMessageCount); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + var newGrain = Assert.IsType(Fixture.GetGrainContext(receiver).GrainInstance); + Assert.NotEqual(oldGrain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.IsType(newGrain.ReplayedSnapshot).ProcessedMessageCount); + Assert.Equal(0, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.InboxCount); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + } + + private async Task RunPumpAsync(IMessagingTestGrain receiver) + { + var snapshot = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var extension = (IDurableJobFeatureHandler)context.ActivationServices.GetRequiredService(ReceiverTestServices.GetImplementationType("InboxExtension")); + var run = new PumpContext(Assert.IsType(snapshot.InboxJob)); + using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); + var stopped = events.WaitForEventAsync(nameof(GrainTimerEvents.TickStop), + item => item.Payload is GrainTimerEvents.TickStop stop && ReferenceEquals(stop.GrainContext, context), + TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + DurableJobRunResult result = null!; + await OnTurnAsync(context, () => result = extension.ExecuteJobAsync(run, TestContext.Current.CancellationToken).GetAwaiter().GetResult()); + Assert.Equal(DurableJobRunStatus.InProgress, result.Status); + Assert.Null(Assert.IsType((await stopped).Payload).Exception); + await OnTurnAsync(context, () => result = extension.ExecuteJobAsync(run, TestContext.Current.CancellationToken).GetAwaiter().GetResult()); + return result; + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completed.SetResult(); } + catch (Exception exception) { completed.SetException(exception); } + }); + return completed.Task; + } + + private sealed class PumpContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + + private sealed class CountingProcessedDictionary(IDictionary inner) : IDictionary + { + public int Enumerations { get; private set; } + public DateTimeOffset this[HierarchicalKey key] { get => inner[key]; set => inner[key] = value; } + public ICollection Keys => inner.Keys; + public ICollection Values { get { Enumerations++; return inner.Values; } } + public int Count => inner.Count; + public bool IsReadOnly => inner.IsReadOnly; + public void Add(HierarchicalKey key, DateTimeOffset value) => inner.Add(key, value); + public void Add(KeyValuePair item) => inner.Add(item); + public void Clear() => inner.Clear(); + public bool Contains(KeyValuePair item) => inner.Contains(item); + public bool ContainsKey(HierarchicalKey key) => inner.ContainsKey(key); + public void CopyTo(KeyValuePair[] array, int index) => inner.CopyTo(array, index); + public bool Remove(HierarchicalKey key) => inner.Remove(key); + public bool Remove(KeyValuePair item) => inner.Remove(item); + public bool TryGetValue(HierarchicalKey key, out DateTimeOffset value) => inner.TryGetValue(key, out value); + public IEnumerator> GetEnumerator() { Enumerations++; return inner.GetEnumerator(); } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + private async Task StartBacklogAsync() + { + var receiver = NewGrain(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/retained-backlog"); + var backlog = CreateEnvelope(receiver, NewMessage(119, "backlog") with { ThrowDuringPreparation = true }, "messages/retained-backlog"); + var rescheduled = Fixture.Metrics.GetCount("orleans-durablejobs-jobs-rescheduled"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, backlog)).Status); + await handler.WaitUntilEnteredAsync(); + handler.Release(); + await Fixture.Metrics.WaitForCountAsync("orleans-durablejobs-jobs-rescheduled", rescheduled + 1); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.Effects); + Assert.Empty(pending.InboxDeadLetters); + return receiver; + } + + private IDurableDictionary GetProcessed(IMessagingTestGrain receiver) => + Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredKeyedService>( + "__orleans.messaging.inbox-processed"); + + private sealed class RetentionFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxProcessingAttempts = 10; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs b/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs new file mode 100644 index 00000000000..a8bc38600e9 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace Orleans.Messaging.Tests.Functional; + +[JsonSerializable(typeof(string))] +[JsonSerializable(typeof(int))] +[JsonSerializable(typeof(uint))] +[JsonSerializable(typeof(DateTime))] +internal partial class StandardStateJsonContext : JsonSerializerContext; diff --git a/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj index b6e3c992322..6eb416e53f3 100644 --- a/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj +++ b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj @@ -4,11 +4,15 @@ true enable enable + $(NoWarn);ORLEANSEXP005 + + + diff --git a/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs b/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs new file mode 100644 index 00000000000..47e0d937673 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs @@ -0,0 +1,137 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class ActivationValidationProbe +{ + private readonly ConcurrentDictionary> _observations = new(); + public Observation[] Get(GrainId grainId) => _observations.TryGetValue(grainId, out var values) ? values.ToArray() : []; + public Observation Track(IGrainContext context) + { + var observation = new Observation(context); + _observations.GetOrAdd(context.GrainId, static _ => new()).Enqueue(observation); + return observation; + } + + public sealed class Observation(IGrainContext context) + { + public IGrainContext Context { get; } = context; + public bool InstanceAvailableDuringConstruction { get; } = context.GrainInstance is not null; + public int Activated; + public int Calls; + } +} + +public interface IActivationValidationTestGrain : IGrainWithGuidKey +{ + Task IncrementAsync(); + Task DeactivateAsync(); +} + +public interface IAlwaysInterleaveValidationTestGrain : IActivationValidationTestGrain +{ + [AlwaysInterleave] + Task InterleaveAsync(); +} + +public abstract class ActivationValidationTestGrain : DurableGrain, IActivationValidationTestGrain +{ + private readonly IDurableValue _value; + private readonly ActivationValidationProbe.Observation _observation; + + protected ActivationValidationTestGrain(IGrainContext context, ActivationValidationProbe probe, + IDurableValue value) + { + _value = value; + _observation = probe.Track(context); + } + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _observation.Activated); + return base.OnActivateAsync(cancellationToken); + } + + public async Task IncrementAsync() + { + Interlocked.Increment(ref _observation.Calls); + _value.Value++; + await WriteStateAsync(); + return _value.Value; + } + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + +} + +public sealed class SupportedActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[Reentrant] +public sealed class ReentrantActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[StatelessWorker] +public sealed class StatelessActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[MayInterleave(nameof(Interleave))] +public sealed class MayInterleaveActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value) +{ + public static bool Interleave(IInvokable request) => true; +} + +public sealed class AlwaysInterleaveActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value), IAlwaysInterleaveValidationTestGrain +{ + public Task InterleaveAsync() => Task.CompletedTask; +} + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] +public sealed class ExecutionPropertyAttribute(string key, string value) : Attribute, IGrainPropertiesProviderAttribute +{ + public void Populate(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) => + properties[key] = value; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "TrUe")] +public sealed class MetadataReentrantActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[ExecutionProperty(WellKnownGrainTypeProperties.MayInterleavePredicate, nameof(Interleave))] +public sealed class MetadataMayInterleaveActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value) +{ + public static bool Interleave(IInvokable request) => true; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "false")] +public sealed class MetadataNonReentrantActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs new file mode 100644 index 00000000000..3884a741f81 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs @@ -0,0 +1,306 @@ +using System.Collections.Concurrent; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class BootstrapProbe +{ + private readonly ConcurrentDictionary> _activations = new(); + public void Add(BootstrapObservation observation) => + _activations.GetOrAdd(observation.Context.GrainId, static _ => new()).Enqueue(observation); + public BootstrapObservation[] Get(GrainId id) => _activations[id].ToArray(); +} + +public sealed class BootstrapObservation : IDisposable +{ + public BootstrapObservation(IGrainContext context, BootstrapProbe probe) + { + Context = context; + probe.Add(this); + } + + public IGrainContext Context { get; } + public object? ConstructedGrain { get; private set; } + public bool InstanceAvailableInConstructor { get; private set; } + public IJournaledStateManager? Manager { get; set; } + public IDurableValue? Value { get; set; } + public IInbox? Inbox { get; set; } + public IOutbox? Outbox { get; set; } + public object? Extension { get; set; } + public Exception? ExpectedFailure { get; set; } + public int Activations { get; set; } + public int GrainDisposals { get; set; } + public int Disposals { get; private set; } + + public void Constructed(object grain) + { + ConstructedGrain = grain; + InstanceAvailableInConstructor = Context.GrainInstance is not null; + } + + public void Dispose() => Disposals++; +} + +public sealed class BootstrapState : IInboxHandler, IDisposable +{ + public const string Route = "bootstrap"; + public static GrainId OutputTarget { get; } = GrainId.Create("bootstrap-output", "capture"); + private readonly HandlerProbe _handlers; + public BootstrapState(BootstrapObservation observation, IJournaledStateManager manager, HandlerProbe handlers, + [FromKeyedServices("bootstrap-value")] IDurableValue value, IInbox inbox, IOutbox outbox) + { + _handlers = handlers; + Observation = observation; + observation.Manager = manager; + observation.Value = value; + observation.Inbox = inbox; + observation.Outbox = outbox; + inbox.RegisterHandler(this); + } + + public BootstrapObservation Observation { get; } + public int MessagingStateCountAtActivation { get; private set; } + public object? GrainAtActivation { get; private set; } + public int ActivationValue { get; private set; } + public int HandlerCalls { get; private set; } + public IInboxHandler? HandlerOverride { get; set; } + public int Disposals { get; private set; } + public Task Read() => Task.FromResult(Observation.Value!.Value); + public async Task Set(int value) + { + Observation.Value!.Value = value; + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public async Task SendValue(int value) + { + var context = Observation.Context; + var envelope = TestApplicationProtocol.Create(context.ActivationServices.GetRequiredService(), context.GrainId, OutputTarget, "output", value); + Observation.Value!.Value = value; + Observation.Outbox!.Send(envelope); + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public async Task SendSynchronousValue(int value) + { + var context = Observation.Context; + var envelope = TestApplicationProtocol.Create(context.ActivationServices.GetRequiredService(), context.GrainId, OutputTarget, "output", value); + Observation.Value!.Value = value; + Observation.Outbox!.Send(envelope); + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public Task Persist() => Observation.Manager!.WriteStateAsync(CancellationToken.None).AsTask(); + public Task Activate() + { + Observation.Activations++; + GrainAtActivation = Observation.Context.GrainInstance; + MessagingStateCountAtActivation = ReadMessagingStates(Observation.Manager!).Count(); + ActivationValue = Observation.Value!.Value; + return Task.CompletedTask; + } + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (HandlerOverride is { } handler) + { + await handler.HandleAsync(context, cancellationToken); + return; + } + if (_handlers.TryGet(Observation.Context.GrainId, Route, out var barrier)) + { + barrier.Entered.TrySetResult(); + await barrier.Continue.Task.WaitAsync(cancellationToken); + } + var value = Observation.Value!.Value + 1; + var outgoing = TestApplicationProtocol.Create(Observation.Context.ActivationServices.GetRequiredService(), Observation.Context.GrainId, OutputTarget, "output", value); + Observation.Value.Value = value; + Observation.Outbox!.Send(outgoing); + HandlerCalls++; + context.Complete(); + } + public void Dispose() => Disposals++; + public static IEnumerable ReadMessagingStates(IJournaledStateManager manager) + { + foreach (var name in new[] { "inbox", "inbox-processed", "inbox-message-state", "inbox-dead-letters", + "inbox-job-id", "inbox-job-handle", "inbox-completed-job-id", "inbox-job-sequence" }) + { + if (manager.TryGetStateMachine("__orleans.messaging." + name, out var state)) yield return state; + } + } + +} + +public interface IBootstrapTestGrain : IGrainWithGuidKey +{ + Task GetValueAsync(); + Task SetValueAsync(int value); + Task SendValueAsync(int value); + Task SendSynchronousValueAsync(int value); + Task PersistAsync(); + Task DeactivateAsync(); +} +public interface IMarkedBootstrapTestGrain : IBootstrapTestGrain, IMessagingGrain; +public interface IGenericBootstrapTestGrain : IBootstrapTestGrain; + +public abstract class BootstrapGrainBase : Grain, IBootstrapTestGrain, IDisposable +{ + private readonly BootstrapState _state; + protected BootstrapGrainBase(BootstrapState state) + { + _state = state; + state.Observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) => _state.Activate(); + public Task GetValueAsync() => _state.Read(); + public Task SetValueAsync(int value) => _state.Set(value); + public Task SendValueAsync(int value) => _state.SendValue(value); + public Task SendSynchronousValueAsync(int value) => _state.SendSynchronousValue(value); + public Task PersistAsync() => _state.Persist(); + public Task DeactivateAsync() { DeactivateOnIdle(); return Task.CompletedTask; } + public void Dispose() => _state.Observation.GrainDisposals++; +} +public sealed class PlainBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IMessagingGrain; +public abstract class ApplicationBootstrapBase(BootstrapState state) : BootstrapGrainBase(state), IMessagingGrain; +public sealed class ApplicationBootstrapGrain(BootstrapState state) : ApplicationBootstrapBase(state); +public sealed class InterfaceBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IMarkedBootstrapTestGrain; +public sealed class GenericBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IGenericBootstrapTestGrain, IMessagingGrain; + +public class DurableBootstrapGrain : DurableGrain, IBootstrapTestGrain, IDisposable +{ + private readonly BootstrapState _state; + public DurableBootstrapGrain(BootstrapState state) + { + _state = state; + state.Observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) => _state.Activate(); + public Task GetValueAsync() => _state.Read(); + public Task SetValueAsync(int value) => _state.Set(value); + public Task SendValueAsync(int value) => _state.SendValue(value); + public Task SendSynchronousValueAsync(int value) => _state.SendSynchronousValue(value); + public Task PersistAsync() => _state.Persist(); + public Task DeactivateAsync() { DeactivateOnIdle(); return Task.CompletedTask; } + public void Dispose() => _state.Observation.GrainDisposals++; +} + +public sealed class MarkedDurableBootstrapGrain(BootstrapState state) : DurableBootstrapGrain(state), IMessagingGrain; + +public interface IBootstrapControlGrain : IGrainWithGuidKey { Task PingAsync(); } +public interface IInterleavingBootstrapControlGrain : IBootstrapControlGrain +{ + [AlwaysInterleave] Task InterleaveAsync(); +} +public abstract class BootstrapControlGrain : Grain, IBootstrapControlGrain, IDisposable +{ + protected BootstrapObservation Observation { get; } + protected BootstrapControlGrain(BootstrapObservation observation) + { + Observation = observation; + observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Observation.Activations++; + return Task.CompletedTask; + } + public Task PingAsync() => Task.CompletedTask; + public void Dispose() => Observation.GrainDisposals++; +} +public sealed class EagerBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +public sealed class UnselectedBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation); +public sealed class FailingBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +public sealed class UnselectedEndpointBootstrapGrain : BootstrapControlGrain +{ + public UnselectedEndpointBootstrapGrain(BootstrapObservation observation, + [FromKeyedServices(typeof(IInboxExtension))] IGrainExtension extension) : base(observation) => observation.Extension = extension; +} +public sealed class JournalOnlyBootstrapGrain : BootstrapControlGrain +{ + public JournalOnlyBootstrapGrain(BootstrapObservation observation, IJournaledStateManager manager, + [FromKeyedServices("bootstrap-journal-only")] IDurableValue value) : base(observation) + { + observation.Manager = manager; + observation.Value = value; + } +} +[Reentrant] +public sealed class ReentrantBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +[StatelessWorker] +public sealed class StatelessBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +[MayInterleave(nameof(Interleave))] +public sealed class MayInterleaveBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain +{ + public static bool Interleave(IInvokable request) => true; +} +public sealed class AlwaysInterleaveBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IInterleavingBootstrapControlGrain, IMessagingGrain +{ + public Task InterleaveAsync() => Task.CompletedTask; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "true")] +public sealed class MetadataReentrantBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.MayInterleavePredicate, nameof(Interleave))] +public sealed class MetadataMayInterleaveBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain +{ + public static bool Interleave(IInvokable request) => true; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, "StatelessWorkerPlacement")] +public sealed class MetadataStatelessPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, BootstrapClusterFixture.StatelessPlacementAlias)] +public sealed class AliasedStatelessPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, nameof(RandomPlacement))] +public sealed class MetadataOrdinaryPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, BootstrapClusterFixture.OrdinaryPlacementAlias)] +public sealed class AliasedOrdinaryPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +public sealed class BootstrapClusterFixture : MessagingClusterFixture +{ + public const string StatelessPlacementAlias = "bootstrap-worker-alias"; + public const string OrdinaryPlacementAlias = "bootstrap-directory-alias"; + + public BootstrapProbe Probe { get; } = new(); + protected override void ConfigureServices(IServiceCollection services) + { + services.AddKeyedSingleton(StatelessPlacementAlias, new StatelessWorkerAttribute(1).PlacementStrategy); + services.AddKeyedSingleton(OrdinaryPlacementAlias, new RandomPlacement()); + services.AddSingleton(Probe); + services.AddScoped(); + services.AddScoped(); + var extensionType = ReceiverTestServices.GetImplementationType("InboxExtension"); + var descriptor = services.Last(entry => entry.ServiceType == extensionType); + services.AddScoped(extensionType, provider => + { + var extension = descriptor.ImplementationFactory!(provider); + var context = provider.GetRequiredService(); + if (context.GrainInstance is FailingBootstrapGrain) + { + var observation = provider.GetRequiredService(); + observation.Manager = provider.GetRequiredService(); + observation.Inbox = provider.GetRequiredService(); + observation.Outbox = provider.GetRequiredService(); + observation.Extension = extension; + observation.ExpectedFailure = new IOException("Expected bootstrap runtime construction failure."); + ((IDisposable)extension).Dispose(); + throw observation.ExpectedFailure; + } + return extension; + }); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs new file mode 100644 index 00000000000..77edf5e3dd8 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs @@ -0,0 +1,151 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class DurableJobManagerProbe +{ + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _attempts = []; + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _successes = []; + private readonly ConcurrentDictionary _failures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _postScheduleFailures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _duplicates = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary<(string JobName, GrainId Target), ConcurrentQueue> _scheduledJobs = []; + private readonly ConcurrentDictionary _scheduleBarriers = new(StringComparer.Ordinal); + + public void FailNext(string jobName) => + _failures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public void FailAfterNext(string jobName) => + _postScheduleFailures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public void DuplicateNext(string jobName) => + _duplicates.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public int GetAttemptCount(string jobName, GrainId target) => + _attempts.TryGetValue((jobName, target), out var count) ? count : 0; + + public int GetSuccessCount(string jobName, GrainId target) => + _successes.TryGetValue((jobName, target), out var count) ? count : 0; + + public IReadOnlyList GetScheduledJobs(string jobName, GrainId target) => + _scheduledJobs.TryGetValue((jobName, target), out var jobs) ? jobs.ToArray() : []; + + public ScheduleBarrier BlockNext(string jobName) + { + var barrier = new ScheduleBarrier(); + if (!_scheduleBarriers.TryAdd(jobName, barrier)) + { + throw new InvalidOperationException($"A scheduling barrier is already armed for '{jobName}'."); + } + + return barrier; + } + + internal void OnAttempt(ScheduleJobRequest request) => + _attempts.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal void OnSuccess(ScheduleJobRequest request) => + _successes.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal void OnScheduled(ScheduleJobRequest request, DurableJob job) => + _scheduledJobs.GetOrAdd((request.JobName, request.Target), static _ => new()).Enqueue(job); + + internal bool ShouldFail(string jobName) + => TryConsumeFailure(_failures, jobName); + + internal bool ShouldFailAfterSchedule(string jobName) + => TryConsumeFailure(_postScheduleFailures, jobName); + + internal bool ShouldDuplicate(string jobName) + => TryConsumeFailure(_duplicates, jobName); + + internal async Task WaitIfBlockedAsync(string jobName, CancellationToken cancellationToken) + { + if (!_scheduleBarriers.TryRemove(jobName, out var barrier)) + { + return; + } + + barrier.Entered.TrySetResult(); + await barrier.Release.Task.WaitAsync(cancellationToken); + } + + private static bool TryConsumeFailure( + ConcurrentDictionary failures, + string jobName) + { + while (failures.TryGetValue(jobName, out var remaining) && remaining > 0) + { + if (failures.TryUpdate(jobName, remaining - 1, remaining)) + { + return true; + } + } + + return false; + } + + public sealed class ScheduleBarrier : IDisposable + { + internal TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + public void Continue() => Release.TrySetResult(); + + public void Dispose() => Continue(); + } +} + +internal sealed class ControlledDurableJobManager( + ILocalDurableJobManager inner, + DurableJobManagerProbe probe) : ILocalDurableJobManager +{ + public async Task ScheduleJobAsync( + ScheduleJobRequest request, + CancellationToken cancellationToken) + { + probe.OnAttempt(request); + if (probe.ShouldFail(request.JobName)) + { + throw new IOException($"Injected durable job scheduling failure for '{request.JobName}'."); + } + + await probe.WaitIfBlockedAsync(request.JobName, cancellationToken); + var result = await inner.ScheduleJobAsync(request, cancellationToken); + probe.OnScheduled(request, result); + if (probe.ShouldDuplicate(request.JobName)) + { + var duplicate = await inner.ScheduleJobAsync(request, cancellationToken); + probe.OnScheduled(request, duplicate); + } + + probe.OnSuccess(request); + if (probe.ShouldFailAfterSchedule(request.JobName)) + { + throw new IOException( + $"Injected durable job scheduling response failure for '{request.JobName}'."); + } + + return result; + } + + public Task CancelAsync(DurableJob job, CancellationToken cancellationToken) => + inner.CancelAsync(job, cancellationToken); + + public static void Decorate(IServiceCollection services, DurableJobManagerProbe probe) + { + var descriptor = services.Last(service => service.ServiceType == typeof(ILocalDurableJobManager)); + var factory = descriptor.ImplementationFactory + ?? throw new InvalidOperationException("The durable job manager registration must use an implementation factory."); + services.Remove(descriptor); + services.AddSingleton( + serviceProvider => new ControlledDurableJobManager( + (ILocalDurableJobManager)factory(serviceProvider), + probe)); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs b/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs new file mode 100644 index 00000000000..e9694f58927 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs @@ -0,0 +1,276 @@ +using System.Buffers; +using System.Collections.Concurrent; +using Microsoft.Extensions.Options; +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class ControlledJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog +{ + private VolatileJournalStorageProvider? _inner; + private readonly ConcurrentDictionary _readPlans = new(); + private readonly ConcurrentDictionary _deletePlans = new(); + private readonly ConcurrentDictionary _writePlans = new(); + private readonly ConcurrentDictionary _postWritePlans = new(); + private readonly ConcurrentDictionary _postDeleteFailures = new(); + private readonly ConcurrentDictionary _successfulWrites = new(); + private readonly ConcurrentDictionary _reads = new(); + private readonly ConcurrentDictionary _creations = new(); + private readonly ConcurrentDictionary _initializations = new(); + + private readonly ConcurrentDictionary _snapshots = new(); + public void RequestSnapshot(JournalId journalId) => _snapshots[journalId] = 0; + + public string? JournalFormatKey { get; private set; } + + public void Configure(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + JournalFormatKey = options.Value.JournalFormatKey; + _inner ??= new VolatileJournalStorageProvider(options); + } + + public IJournalStorage CreateStorage(JournalId journalId) + { + _creations.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + return new ControlledJournalStorage(this, journalId, Inner.CreateStorage(journalId)); + } + + public IAsyncEnumerable ListAsync( + JournalCatalogListOptions? options = null, + CancellationToken cancellationToken = default) => + Inner.ListAsync(options, cancellationToken); + + private VolatileJournalStorageProvider Inner => + _inner ?? throw new InvalidOperationException("The controlled journal storage provider has not been configured."); + + public WriteBarrier BlockWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + var plan = new WritePlan(matchingWrite, fail: false); + if (!_writePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + public WriteBarrier BlockDelete(JournalId journalId) + { + var plan = new WritePlan(1, fail: false); + if (!_deletePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A delete plan is already armed for journal '{journalId}'."); + } + return new WriteBarrier(plan); + } + + public WriteBarrier BlockRead(JournalId journalId, int matchingRead = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingRead); + var plan = new WritePlan(matchingRead, fail: false); + if (!_readPlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A read plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + public void FailWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_writePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + } + + public void FailAfterWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_postWritePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A post-write plan is already armed for journal '{journalId}'."); + } + } + + public WriteBarrier BlockAcknowledgement(JournalId journalId) + { + var plan = new WritePlan(1, fail: false); + if (!_postWritePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A post-write plan is already armed for journal '{journalId}'."); + } + return new WriteBarrier(plan); + } + + public void FailAfterDelete(JournalId journalId) + { + if (!_postDeleteFailures.TryAdd(journalId, 0)) + { + throw new InvalidOperationException($"A post-delete failure is already armed for journal '{journalId}'."); + } + } + + public int GetSuccessfulWriteCount(JournalId journalId) => + _successfulWrites.TryGetValue(journalId, out var count) ? count : 0; + + public int GetCreationCount(JournalId journalId) => _creations.TryGetValue(journalId, out var count) ? count : 0; + public int GetReadCount(JournalId journalId) => _reads.TryGetValue(journalId, out var count) ? count : 0; + public int GetInitializationCount(JournalId journalId) => _initializations.TryGetValue(journalId, out var count) ? count : 0; + + private async ValueTask BeforeWriteAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_writePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _writePlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.EntryScheduler = TaskScheduler.Current; + plan.EntryContext = ReceiverTestServices.CurrentGrainContext; + plan.Entered.TrySetResult(); + if (plan.Fail) + { + throw new IOException($"Injected journal write failure for '{journalId}'."); + } + + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private async ValueTask BeforeReadAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_readPlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _readPlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private void OnWriteSucceeded(JournalId journalId) => + _successfulWrites.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + + private async ValueTask AfterWriteAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_postWritePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _postWritePlans.TryRemove(new KeyValuePair(journalId, plan)); + if (plan.Fail) + { + throw new IOException($"Injected post-commit journal response failure for '{journalId}'."); + } + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + internal sealed class WritePlan(int target, bool fail) + { + public int Target { get; } = target; + public bool Fail { get; } = fail; + public int Seen; + public TaskScheduler? EntryScheduler; + public IGrainContext? EntryContext; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class WriteBarrier : IDisposable + { + private readonly WritePlan _plan; + + internal WriteBarrier(WritePlan plan) => _plan = plan; + + public TaskScheduler? EntryScheduler => _plan.EntryScheduler; + public IGrainContext? EntryContext => _plan.EntryContext; + public Task WaitUntilEnteredAsync() => _plan.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => _plan.Release.TrySetResult(); + public void Dispose() => Release(); + public void Fail() => Fail(new IOException("Injected blocked journal write failure.")); + public void Fail(Exception exception) => _plan.Release.TrySetException(exception); + } + + private sealed class ControlledJournalStorage( + ControlledJournalStorageProvider owner, + JournalId journalId, + IJournalStorage inner) : IJournalStorage + { + public bool IsCompactionRequested => owner._snapshots.ContainsKey(journalId) || inner.IsCompactionRequested; + + public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + var grain = ReceiverTestServices.CurrentGrainContext?.GrainInstance as MessagingTestGrain; + owner._reads.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + await owner.BeforeReadAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReadAsync(consumer, cancellationToken).ConfigureAwait(false); + grain?.CaptureStorageRead(); + } + + public ValueTask CreateIfNotExistsAsync( + IReadOnlyDictionary? metadata = null, + CancellationToken cancellationToken = default) + { + owner._initializations.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + return inner.CreateIfNotExistsAsync(metadata, cancellationToken); + } + + public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) => + inner.GetMetadataAsync(cancellationToken); + + public ValueTask UpdateMetadataAsync( + IReadOnlyDictionary? set = null, + IEnumerable? remove = null, + string? expectedETag = null, + CancellationToken cancellationToken = default) => + inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + + public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var instance = ReceiverTestServices.CurrentGrainContext?.GrainInstance; + var grain = instance as MessagingTestGrain; + var captured = grain?.CaptureStorageWrite(); + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReplaceAsync(value, cancellationToken).ConfigureAwait(false); + owner._snapshots.TryRemove(journalId, out _); + owner.OnWriteSucceeded(journalId); + await owner.AfterWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + if (captured is not null) grain!.PublishStoredSnapshot(captured); + } + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var instance = ReceiverTestServices.CurrentGrainContext?.GrainInstance; + var grain = instance as MessagingTestGrain; + var captured = grain?.CaptureStorageWrite(); + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.AppendAsync(value, cancellationToken).ConfigureAwait(false); + owner.OnWriteSucceeded(journalId); + await owner.AfterWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + if (captured is not null) grain!.PublishStoredSnapshot(captured); + } + + public async ValueTask DeleteAsync(CancellationToken cancellationToken) + { + if (owner._deletePlans.TryRemove(journalId, out var plan)) + { + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + await inner.DeleteAsync(cancellationToken).ConfigureAwait(false); + if (owner._postDeleteFailures.TryRemove(journalId, out _)) + { + throw new IOException($"Injected post-delete journal response failure for '{journalId}'."); + } + } + } +} diff --git a/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs b/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs new file mode 100644 index 00000000000..8cbd86e17c7 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs @@ -0,0 +1,48 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class FaultingInboxCodecFixture : MessagingClusterFixture +{ + public Exception? NextFailure { get; set; } + public bool FailOnSnapshot { get; set; } + public bool? FailedSnapshot { get; private set; } + protected override void ConfigureServices(IServiceCollection services) + { + var descriptor = services.Last(entry => entry.ServiceType == typeof(IDurableDictionaryCommandCodec<,>) + && Equals(entry.ServiceKey, "orleans-binary")); + var implementation = descriptor.KeyedImplementationType!.MakeGenericType(typeof(HierarchicalKey), typeof(InboxMessage)); + services.AddKeyedSingleton>("orleans-binary", (sp, _) => + new FaultingCodec(this, (IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, implementation))); + } + + private sealed class FaultingCodec(FaultingInboxCodecFixture owner, + IDurableDictionaryCommandCodec inner) + : IDurableDictionaryCommandCodec + { + private void Check(bool snapshot = false) + { + if (snapshot == owner.FailOnSnapshot && owner.NextFailure is { } failure) + { + owner.NextFailure = null; + owner.FailedSnapshot = snapshot; + throw failure; + } + } + public void WriteSet(HierarchicalKey key, InboxMessage value, JournalStreamWriter writer) + { + Check(); + inner.WriteSet(key, value, writer); + } + public void WriteRemove(HierarchicalKey key, JournalStreamWriter writer) { Check(); inner.WriteRemove(key, writer); } + public void WriteClear(JournalStreamWriter writer) { Check(); inner.WriteClear(writer); } + public void WriteSnapshot(IReadOnlyCollection> items, JournalStreamWriter writer) + { + Check(snapshot: true); + inner.WriteSnapshot(items, writer); + } + public void Apply(JournalBufferReader input, IDurableDictionaryCommandHandler consumer) => inner.Apply(input, consumer); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs new file mode 100644 index 00000000000..0ecb2480c55 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs @@ -0,0 +1,53 @@ +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class HandlerProbe +{ + private readonly ConcurrentDictionary<(GrainId GrainId, string Route), Barrier> _barriers = new(); + + public Barrier Arm(GrainId grainId, string route) + { + var barrier = new Barrier(this, (grainId, route)); + if (!_barriers.TryAdd((grainId, route), barrier)) + { + throw new InvalidOperationException($"A handler barrier is already armed for '{grainId}' and route '{route}'."); + } + + return barrier; + } + + public bool TryGet( + GrainId grainId, + string route, + [NotNullWhen(true)] out Barrier? barrier) => + _barriers.TryGetValue((grainId, route), out barrier); + + public sealed class Barrier : IDisposable + { + private readonly HandlerProbe _owner; + private readonly (GrainId GrainId, string Route) _key; + + internal Barrier(HandlerProbe owner, (GrainId GrainId, string Route) key) + { + _owner = owner; + _key = key; + } + + internal TaskCompletionSource Entered { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Continue { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => Continue.TrySetResult(); + + public void Dispose() + { + Release(); + _owner._barriers.TryRemove(new KeyValuePair<(GrainId GrainId, string Route), Barrier>(_key, this)); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs new file mode 100644 index 00000000000..75e558c8daf --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs @@ -0,0 +1,70 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +// Captures opaque handler output in the same journal as inbox effects. Dispatch belongs to the outbox layer. +internal sealed class JournaledTestOutbox(IDurableDictionary messages, IGrainContext context) + : IOutbox, ILifecycleObserver, IEnumerable> +{ + private bool _stopped; + private readonly TaskCompletionSource _stopping = new(TaskCreationOptions.RunContinuationsAsynchronously); + public GrainId SenderId => context.GrainId; + public Task Stopping => _stopping.Task; + public int SendCalls { get; private set; } + public Exception? NextSendFailure { get; set; } + public IDurableDictionary StoredMessages { get; } = messages; + public int Count => StoredMessages.Count; + public IEnumerable Messages => StoredMessages.Values; + public IEnumerator> GetEnumerator() => StoredMessages.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public void Send(OutboxMessage envelope) + { + SendCalls++; + if (_stopped) throw new InvalidOperationException("The test outbox owner is stopped."); + if (envelope.Envelope.MessageId.IsDefault || envelope.ReceiverId.IsDefault + || !envelope.Envelope.TryGetBytes(MessageHeaders.Payload, out _)) + { + throw new ArgumentException("An outgoing envelope requires identities and opaque payload bytes.", nameof(envelope)); + } + var sessions = context.ActivationServices.GetRequiredService(); + if (envelope.Envelope.TryGetValue(MessageHeaders.Sender, new Serializer(sessions), out var sender) + && sender != context.GrainId) + { + throw new InvalidOperationException($"The outgoing sender for '{envelope.Envelope.MessageId}' differs from the test outbox owner."); + } + if (TryGetMessage(envelope.Envelope.MessageId, out var existing)) + { + if (existing.ReceiverId != envelope.ReceiverId + || !string.Equals(TestApplicationProtocol.Subject(existing.Envelope), + TestApplicationProtocol.Subject(envelope.Envelope), StringComparison.Ordinal) + || !existing.Envelope.Payload.Span.SequenceEqual(envelope.Envelope.Payload.Span)) + { + throw new InvalidOperationException($"The outbox already contains a different envelope with message ID '{envelope.Envelope.MessageId}'."); + } + return; + } + if (NextSendFailure is { } failure) + { + NextSendFailure = null; + throw failure; + } + StoredMessages.Add(envelope.Envelope.MessageId, envelope); + } + + public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) => + StoredMessages.TryGetValue(messageId, out envelope); + public Task OnStart(CancellationToken cancellationToken) => Task.CompletedTask; + public Task OnStop(CancellationToken cancellationToken) + { + _stopped = true; + _stopping.TrySetResult(); + return Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs new file mode 100644 index 00000000000..15a2fd6af35 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs @@ -0,0 +1,83 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public abstract class MessagingBehaviorTestBase : IAsyncLifetime +{ + protected MessagingBehaviorTestBase() + : this(new MessagingClusterFixture()) + { + } + + protected MessagingBehaviorTestBase(MessagingClusterFixture fixture) => Fixture = fixture; + + protected MessagingClusterFixture Fixture { get; } + protected SerializerSessionPool Sessions => Fixture.Client.ServiceProvider.GetRequiredService(); + + public ValueTask InitializeAsync() => Fixture.InitializeAsync(); + public ValueTask DisposeAsync() => Fixture.DisposeAsync(); + + protected IMessagingTestGrain NewGrain() => + Fixture.Client.GetGrain(Guid.NewGuid()); + + protected static TestMessage NewMessage(int sequence, string value) => + new(TestApplicationProtocol.NewMessageId(), sequence, value); + + protected async Task RefreshSeededOwnerAsync(IMessagingTestGrain receiver) + { + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + Assert.NotSame(previous, Fixture.GetGrainContext(receiver)); + } + + protected static Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + DeliverWithCancellationAsync(receiver, envelope, TestContext.Current.CancellationToken); + + protected static async Task DeliverWithCancellationAsync( + IMessagingTestGrain receiver, + InboxMessage envelope, + CancellationToken cancellationToken) => + await receiver.AsReference().DeliverAsync(envelope, cancellationToken); + + protected static async Task WaitForBarrierAsync( + IMessagingTestGrain receiver, + HandlerProbe.Barrier barrier) + { + try + { + await barrier.WaitUntilEnteredAsync(); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Handler did not start. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, maxHandlers={snapshot.MaxConcurrentHandlers}, deadLetters={string.Join(" | ", snapshot.InboxDeadLetters.Select(static item => item.Reason))}.", + exception); + } + } + + protected InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + TestMessage message, + string route = "messages/record") => + CreateEnvelope(receiver, (object)message, route); + + protected InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + object body, + string route) + { + var sessions = Fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("external-test-sender", Guid.NewGuid().ToString("N")); + return TestApplicationProtocol.CreateInbox(sessions, sender, route, body); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs new file mode 100644 index 00000000000..a48111fbb18 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -0,0 +1,159 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public class MessagingClusterFixture : IAsyncLifetime +{ + public MessagingClusterFixture() : this(1) + { + } + + private MessagingClusterFixture(int initialSilos) + { + Clock = new FakeTimeProvider(DateTimeOffset.UtcNow); + Storage = new ControlledJournalStorageProvider(); + Metrics = new MessagingMetricProbe(); + JobManagerProbe = new DurableJobManagerProbe(); + HandlerProbe = new HandlerProbe(); + SnapshotProbe = new SnapshotProbe(); + ActivationProbe = new ActivationValidationProbe(); + var clusterId = $"messaging-{Guid.NewGuid():N}"; + var serviceId = $"messaging-service-{Guid.NewGuid():N}"; + var builder = new InProcessTestClusterBuilder((short)initialSilos); + builder.ConfigureClient(clientBuilder => + clientBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + })); + builder.ConfigureSilo((_, siloBuilder) => + { + siloBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + }); + siloBuilder.Services.AddSingleton(Clock); + siloBuilder.Services.UseTimeProviderForBackgroundAreas(TimeProvider.System); + siloBuilder.Services.AddSingleton(HandlerProbe); + siloBuilder.Services.AddSingleton(SnapshotProbe); + siloBuilder.Services.AddSingleton(ActivationProbe); + siloBuilder.Services.AddSingleton(); + siloBuilder.UseInMemoryDurableJobs(); + ReceiverTestServices.Add(siloBuilder.Services, ConfigureOptions); + siloBuilder.ConfigureServices(services => + ControlledDurableJobManager.Decorate(services, JobManagerProbe)); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.AddSingleton(Storage); + siloBuilder.Services.AddSingleton(serviceProvider => + { + Storage.Configure(serviceProvider.GetRequiredService>()); + return Storage; + }); + siloBuilder.Services.AddSingleton( + serviceProvider => (IJournalStorageCatalog)serviceProvider.GetRequiredService()); + ConfigureServices(siloBuilder.Services); + }); + Cluster = builder.Build(); + } + + public InProcessTestCluster Cluster { get; } + public IClusterClient Client => Cluster.Client!; + public FakeTimeProvider Clock { get; } + public ControlledJournalStorageProvider Storage { get; } + public MessagingMetricProbe Metrics { get; } + public DurableJobManagerProbe JobManagerProbe { get; } + public HandlerProbe HandlerProbe { get; } + public SnapshotProbe SnapshotProbe { get; } + public ActivationValidationProbe ActivationProbe { get; } + + public Task WaitForEffectCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.Effects.Sum(static effect => effect.Count) >= expected); + + public Task WaitForInboxCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.InboxCount == expected); + + public Task WaitForDeadLetterCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.InboxDeadLetters.Count + snapshot.OutboxDeadLetters.Count >= expected); + + public Task WaitForOutboxCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.OutboxCount == expected); + + public EndpointSnapshot GetSnapshot(IMessagingTestGrain grain) => + GetGrainInstance(grain).GetSnapshotForTest(); + + public OutboxMessage[] GetStagedOutput(IMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices.GetRequiredService().Messages.ToArray(); + + public ValueTask WriteStateAsync(IMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices + .GetRequiredService() + .WriteStateAsync(TestContext.Current.CancellationToken); + + internal IGrainContext GetGrainContext(IMessagingTestGrain grain) + { + if (!Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)) + { + throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' is not active."); + } + + return context; + } + + private MessagingTestGrain GetGrainInstance(IMessagingTestGrain grain) => + GetGrainContext(grain).GrainInstance as MessagingTestGrain + ?? throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' has an unexpected implementation."); + + protected virtual void ConfigureServices(IServiceCollection services) { } + + protected virtual void ConfigureOptions(InboxOptions options) + { + options.MaxCapacity = 2; + options.DeduplicationWindow = TimeSpan.FromMinutes(10); + options.MaxOutboxRetryAge = TimeSpan.FromMinutes(5); + options.MaxProcessingAttempts = 1; + options.MaxDeliveryAttempts = 3; + options.MaxRetainedDeadLetters = 2; + options.DeadLetterRetentionPeriod = TimeSpan.FromHours(1); + options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(25); + options.InboxBatchSize = 8; + options.OutboxBatchSize = 8; + } + + public ValueTask InitializeAsync() => new(Cluster.DeployAsync()); + public async ValueTask DisposeAsync() + { + await Cluster.DisposeAsync(); + Metrics.Dispose(); + } +} + +public sealed class DedupeExpiryClusterFixture : MessagingClusterFixture +{ +} + +public sealed class InboxCapacityClusterFixture : MessagingClusterFixture +{ + protected override void ConfigureOptions(InboxOptions options) + { + base.ConfigureOptions(options); + options.MaxCapacity = 1; + options.MaxProcessingAttempts = 2; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs new file mode 100644 index 00000000000..c07cf9cc4a4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs @@ -0,0 +1,121 @@ +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class MessagingMetricProbe : IDisposable +{ + private readonly ConcurrentDictionary<(string Instrument, string TagValue), long> _measurements = []; + private readonly ConcurrentDictionary _gauges = []; + private readonly object _lock = new(); + private TaskCompletionSource _changed = CreateSignal(); + private readonly MeterListener _listener; + + public MessagingMetricProbe() + { + _listener = new MeterListener + { + InstrumentPublished = static (instrument, listener) => + { + if (instrument.Meter.Name == "Microsoft.Orleans" + && instrument.Name is "orleans-messaging-orphaned-jobs-reclaimed" + or "orleans-messaging-inbox-messages-processed" + or "orleans-durablejobs-job-attempts-started" + or "orleans-durablejobs-handler-executions-started" + or "orleans-durablejobs-jobs-completed" + or "orleans-durablejobs-jobs-rescheduled" + or "orleans-messaging-inbox-depth" + or "orleans-messaging-outbox-depth") + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + _listener.SetMeasurementEventCallback(OnMeasurement); + _listener.Start(); + } + + public long GetCount(string instrument, string tagValue = "") => + _measurements.TryGetValue((instrument, tagValue), out var count) ? count : 0; + + public long GetDepth(string instrument) + { + _listener.RecordObservableInstruments(); + return _gauges.TryGetValue(instrument, out var value) ? value : 0; + } + + public Task WaitForCountAsync( + string instrument, + long expected, + string tagValue = "") => + WaitForCountWithCancellationAsync( + instrument, + expected, + tagValue, + TestContext.Current.CancellationToken); + + public async Task WaitForCountWithCancellationAsync( + string instrument, + long expected, + string tagValue, + CancellationToken cancellationToken) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + while (GetCount(instrument, tagValue) < expected) + { + Task changed; + lock (_lock) + { + if (GetCount(instrument, tagValue) >= expected) + { + return; + } + + changed = _changed.Task; + } + + await changed.WaitAsync(timeout.Token); + } + } + + public void Dispose() => _listener.Dispose(); + + private void OnMeasurement( + Instrument instrument, + long measurement, + ReadOnlySpan> tags, + object? state) + { + if (instrument.Name is "orleans-messaging-inbox-depth" or "orleans-messaging-outbox-depth") + { + _gauges[instrument.Name] = measurement; + return; + } + + var tagValue = ""; + var tagName = instrument.Name == "orleans-messaging-inbox-messages-processed" ? "status" : "job_name"; + foreach (var tag in tags) + { + if (tag.Key == tagName) + { + tagValue = tag.Value as string ?? ""; + break; + } + } + + _measurements.AddOrUpdate( + (instrument.Name, tagValue), + measurement, + (_, current) => current + measurement); + lock (_lock) + { + _changed.TrySetResult(); + _changed = CreateSignal(); + } + } + + private static TaskCompletionSource CreateSignal() => + new(TaskCreationOptions.RunContinuationsAsynchronously); +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs new file mode 100644 index 00000000000..4fa64fd0fc9 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -0,0 +1,502 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Messaging; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +public interface IMessagingTestGrain : IGrainWithGuidKey +{ + Task SendAsync(GrainId target, string route, TestMessage message); + Task SendDuplicateAsync(GrainId target, string route, TestMessage message); + Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message); + Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message); + Task RetryWriteStateAsync(); + Task StageEffectAsync(MessageEffect effect); + Task StageOutputAsync(OutboxMessage envelope); + Task AcceptAndDeactivateAsync(InboxMessage envelope); + Task SetInboxOwnershipAsync(string ownershipId, DurableJob job); + Task SeedInboxStateAsync(InboxMessage envelope, string? ownershipId, DurableJob? job); + Task ConfigureHandlerAsync(bool enabled); + Task RemoveInboxDeadLetterAsync(HierarchicalKey messageId); + Task RemoveOutboxDeadLetterAsync(HierarchicalKey messageId); + Task GetSnapshotAsync(); + Task RequestDeactivationAsync(); + Task SetControlEnvelopeAsync(InboxMessage envelope); + Task DeleteStateAndDeactivateAsync(); + Task HoldPumpTurnAsync(string barrierRoute, bool deactivate); + Task HoldPumpTurnAsync(string barrierRoute, InboxMessage replacement, bool deactivate); +} + +[GenerateSerializer, Immutable] +public sealed record TestMessage( + [property: Id(0)] HierarchicalKey LogicalId, + [property: Id(1)] int Sequence, + [property: Id(2)] string Value, + [property: Id(3)] GrainId? ForwardTo = null, + [property: Id(8)] bool ThrowDuringPreparation = false, + [property: Id(5)] bool CommitDuringHandling = false, + [property: Id(6)] bool DeleteDuringHandling = false, + [property: Id(9)] bool ThrowOnceDuringPreparation = false); + +[GenerateSerializer, Immutable] +public sealed record MessageEffect( + [property: Id(0)] HierarchicalKey LogicalId, + [property: Id(1)] int Count, + [property: Id(2)] int Sequence, + [property: Id(3)] string Value); + +[GenerateSerializer, Immutable] +public sealed record EndpointSnapshot( + [property: Id(0)] Guid ActivationId, + [property: Id(1)] string SiloAddress, + [property: Id(2)] int InboxCount, + [property: Id(3)] int OutboxCount, + [property: Id(4)] int MaxConcurrentHandlers, + [property: Id(5)] IReadOnlyList Effects, + [property: Id(6)] IReadOnlyList InboxDeadLetters, + [property: Id(7)] IReadOnlyList OutboxDeadLetters, + [property: Id(8)] string? InboxJobId, + [property: Id(9)] int ProcessedMessageCount, + [property: Id(12)] string? OutboxJobId, + [property: Id(16)] DurableJob? InboxJob, + [property: Id(17)] DurableJob? OutboxJob); + +[GenerateSerializer, Immutable] +public sealed record DeadLetterSnapshot( + [property: Id(0)] HierarchicalKey MessageId, + [property: Id(1)] string Route, + [property: Id(2)] string Reason, + [property: Id(3)] int AttemptCount, + [property: Id(4)] DateTimeOffset DeadLetteredAt); + +[GrainType("messaging-inbox-test")] +public sealed class MessagingTestGrain : DurableGrain, IMessagingTestGrain, IDurableJobHandler, + IObserver, IDisposable +{ + private readonly IGrainContext _grainContext; + private readonly IDisposable _lifecycleSubscription; + private readonly IJournaledStateManager _journalOwner; + private readonly IInbox _inbox; + private readonly IOutbox _outbox; + private readonly IMessagingDiagnostics _diagnostics; + private readonly IDurableDictionary _effects; + private readonly IDurableDictionary _processedMessages; + private readonly SerializerSessionPool _sessions; + private readonly TestHandlerConfiguration _handlerConfiguration; + internal IInboxHandler? HandlerOverride { get; set; } + private readonly IDurableValue _inboxJobId; + private readonly IDurableValue _inboxJob; + private readonly IDurableValue _outboxJobId; + private readonly IDurableValue _outboxJob; + private readonly ILocalSiloDetails _siloDetails; + private readonly HandlerProbe _handlerProbe; + private readonly SnapshotProbe _snapshotProbe; + private readonly Guid _activationId = Guid.NewGuid(); + private int _activeHandlers; + private int _maxConcurrentHandlers; + private readonly HashSet _failedOnce = []; + + public MessagingTestGrain( + IGrainContext grainContext, + IJournaledStateManager journalOwner, + IInbox inbox, + IOutbox outbox, + IMessagingDiagnostics diagnostics, + [FromKeyedServices("test-effects")] IDurableDictionary effects, + [FromKeyedServices("inbox")] IDurableValue applicationInboxState, + [FromKeyedServices("__orleans.messaging.inbox-processed")] IDurableDictionary processedMessages, + [FromKeyedServices("__orleans.messaging.inbox-job-id")] IDurableValue inboxJobId, + [FromKeyedServices("__orleans.messaging.inbox-job-handle")] IDurableValue inboxJob, + [FromKeyedServices("__orleans.messaging.outbox-job-id")] IDurableValue outboxJobId, + [FromKeyedServices("__orleans.messaging.outbox-job-handle")] IDurableValue outboxJob, + SerializerSessionPool sessions, + TestHandlerConfiguration handlerConfiguration, + ILocalSiloDetails siloDetails, + HandlerProbe handlerProbe, + SnapshotProbe snapshotProbe) + { + _grainContext = grainContext; + _journalOwner = journalOwner; + _inbox = inbox; + _outbox = outbox; + _diagnostics = diagnostics; + _effects = effects; + ArgumentNullException.ThrowIfNull(applicationInboxState); + _processedMessages = processedMessages; + _inboxJobId = inboxJobId; + _inboxJob = inboxJob; + _outboxJobId = outboxJobId; + _outboxJob = outboxJob; + _sessions = sessions; + _handlerConfiguration = handlerConfiguration; + _siloDetails = siloDetails; + _handlerProbe = handlerProbe; + _snapshotProbe = snapshotProbe; + _lifecycleSubscription = GrainLifecycleEvents.AllEvents.Subscribe(this); + } + + public override async Task OnActivateAsync(CancellationToken cancellationToken) + { + if (_handlerConfiguration.IsEnabled(this.GetGrainId())) + { + _inbox.RegisterHandler(new ApplicationDispatcher(this)); + } + await base.OnActivateAsync(cancellationToken); + _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); + } + + public async Task SendAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.Envelope.MessageId; + } + + public async Task SendDuplicateAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.Envelope.MessageId; + } + + public async Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message) + { + var messageId = await SendAsync(target, route, message); + DeactivateOnIdle(); + return messageId; + } + + public Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + return Task.FromResult(envelope.Envelope.MessageId); + } + + public async Task RetryWriteStateAsync() => await WriteStateAsync(); + + public async Task AcceptAndDeactivateAsync(InboxMessage envelope) + { + var extension = (IInboxExtension)ServiceProvider.GetRequiredKeyedService(typeof(IInboxExtension)); + var result = await extension.DeliverAsync(envelope); + AcceptedSnapshot = CreateSnapshot(); + DeactivateOnIdle(); + return result; + } + + public Task StageOutputAsync(OutboxMessage envelope) + { + _outbox.Send(envelope); + return Task.CompletedTask; + } + + public Task StageEffectAsync(MessageEffect effect) + { + _effects[effect.LogicalId] = effect; + return Task.CompletedTask; + } + + public async Task SetInboxOwnershipAsync(string ownershipId, DurableJob job) + { + _inboxJobId.Value = ownershipId; + _inboxJob.Value = job; + await WriteStateAsync(); + } + + public async Task SeedInboxStateAsync(InboxMessage envelope, string? ownershipId, DurableJob? job) + { + var messages = ServiceProvider.GetRequiredKeyedService>( + "__orleans.messaging.inbox"); + messages.Add(envelope.Envelope.MessageId, envelope); + _inboxJobId.Value = ownershipId; + _inboxJob.Value = job; + await WriteStateAsync(); + } + + public Task ConfigureHandlerAsync(bool enabled) + { + _handlerConfiguration.Set(this.GetGrainId(), enabled); + return Task.CompletedTask; + } + + public async Task RemoveInboxDeadLetterAsync(HierarchicalKey messageId) + { + if (!_diagnostics.RemoveInboxDeadLetter(messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + + public async Task RemoveOutboxDeadLetterAsync(HierarchicalKey messageId) + { + if (!_diagnostics.RemoveOutboxDeadLetter(messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + + public Task GetSnapshotAsync() => Task.FromResult(CreateSnapshot()); + + internal EndpointSnapshot GetSnapshotForTest() => CreateSnapshot(); + + public Task RequestDeactivationAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + public async Task DeleteStateAndDeactivateAsync() + { + try + { + var inbox = (ILifecycleObserver)ServiceProvider.GetRequiredKeyedService(typeof(IInboxExtension)); + var outbox = (ILifecycleObserver)_outbox; + await Task.WhenAll(inbox.OnStop(CancellationToken.None), outbox.OnStop(CancellationToken.None)); + await _journalOwner.DeleteStateAsync(CancellationToken.None); + } + finally + { + DeactivateOnIdle(); + } + } + + internal EndpointSnapshot? AcceptedSnapshot { get; private set; } + + private InboxMessage? _controlEnvelope; + internal TaskCompletionSource ControlDeliveryEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task SetControlEnvelopeAsync(InboxMessage envelope) + { + _controlEnvelope = envelope; + return Task.CompletedTask; + } + + internal TaskScheduler? JobScheduler { get; private set; } + internal IGrainContext? JobGrainContext { get; private set; } + + public async Task ExecuteJobAsync(IJobRunContext context, CancellationToken attemptCancellationToken) + { + JobScheduler = TaskScheduler.Current; + JobGrainContext = ReceiverTestServices.CurrentGrainContext; + switch (context.Job.Name) + { + case "test/deliver-envelope": + ControlDeliveryEntered.TrySetResult(); + var extension = (IInboxExtension)ServiceProvider.GetRequiredKeyedService(typeof(IInboxExtension)); + await extension.DeliverAsync(_controlEnvelope ?? throw new InvalidOperationException("A control envelope must be configured."), attemptCancellationToken); + break; + case "test/write-journal": + await StateManager.WriteStateAsync(attemptCancellationToken); + break; + case "test/probe-scheduler": + break; + default: + throw new NotSupportedException($"Unknown test job '{context.Job.Name}'."); + } + } + + public Task HoldPumpTurnAsync(string barrierRoute, bool deactivate) => + HoldPumpTurnCoreAsync(barrierRoute, null, deactivate); + + public Task HoldPumpTurnAsync(string barrierRoute, InboxMessage replacement, bool deactivate) => + HoldPumpTurnCoreAsync(barrierRoute, replacement, deactivate); + + private async Task HoldPumpTurnCoreAsync(string barrierRoute, InboxMessage? replacement, bool deactivate) + { + if (!_handlerProbe.TryGet(this.GetGrainId(), barrierRoute, out var barrier)) + { + throw new InvalidOperationException("The pump-turn barrier must be armed."); + } + barrier.Entered.TrySetResult(); + await barrier.Continue.Task; + if (replacement is { } envelope) + { + var extension = (IInboxExtension)ServiceProvider.GetRequiredKeyedService(typeof(IInboxExtension)); + await extension.DeliverAsync(envelope); + } + if (deactivate) + { + DeactivateOnIdle(); + } + } + + internal TaskCompletionSource DeactivationFailure { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + void IObserver.OnNext(GrainLifecycleEvents.LifecycleEvent value) + { + if (value is GrainLifecycleEvents.Deactivating { Reason.Exception: { } exception } deactivating + && ReferenceEquals(deactivating.GrainContext, _grainContext)) + { + DeactivationFailure.TrySetResult(exception); + } + } + + void IObserver.OnError(Exception error) => DeactivationFailure.TrySetException(error); + void IObserver.OnCompleted() { } + public void Dispose() + { + _controlEnvelope = null; + _lifecycleSubscription.Dispose(); + } + + private readonly ConcurrentQueue _captures = new(); + private readonly ConcurrentQueue _outputCaptures = new(); + internal IReadOnlyList Captures => _captures.ToArray(); + internal IReadOnlyList OutputCaptures => _outputCaptures.ToArray(); + internal Exception? NextApplyFailure { get; set; } + internal TaskCompletionSource ApplyAttempted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal void ClearCaptures() => _captures.Clear(); + + internal EndpointSnapshot CaptureStorageWrite() + { + var snapshot = CreateSnapshot(); + _captures.Enqueue(snapshot); + _outputCaptures.Enqueue(_outbox.Messages.Select(static envelope => envelope.Envelope.MessageId).ToArray()); + return snapshot; + } + + internal void PublishStoredSnapshot(EndpointSnapshot snapshot) => _snapshotProbe.Publish(this.GetGrainId(), snapshot); + internal void CaptureStorageRead() => ReplayedSnapshot = CreateSnapshot(); + internal EndpointSnapshot? ReplayedSnapshot { get; private set; } + + private OutboxMessage CreateEnvelope(GrainId target, string route, TestMessage message) => + TestApplicationProtocol.Create(_sessions, this.GetGrainId(), target, route, message); + + private async ValueTask HandleAsync( + TestMessage message, + IInboxHandlerContext context, + CancellationToken cancellationToken) + { + var active = Interlocked.Increment(ref _activeHandlers); + _maxConcurrentHandlers = Math.Max(_maxConcurrentHandlers, active); + try + { + if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route, out var gate)) + { + gate.Entered.TrySetResult(); + await gate.Continue.Task.WaitAsync(cancellationToken); + } + + if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route + "/application-preparation", out var preparation)) + { + preparation.Entered.TrySetResult(); + await preparation.Continue.Task.WaitAsync(cancellationToken); + } + cancellationToken.ThrowIfCancellationRequested(); + if (message.CommitDuringHandling) + { + await WriteStateAsync(cancellationToken); + } + if (message.ThrowDuringPreparation || (message.ThrowOnceDuringPreparation && _failedOnce.Add(message.LogicalId))) + { + throw new InvalidOperationException($"Injected handler preparation failure for {message.LogicalId}."); + } + + var outgoing = message.ForwardTo is { } destination + ? TestApplicationProtocol.Create(_sessions, this.GetGrainId(), destination, "messages/forwarded", + message with { ForwardTo = null, ThrowDuringPreparation = false }, + context.Message.Envelope.MessageId.CreateChildKey("forwarded")) + : (OutboxMessage?)null; + if (NextApplyFailure is { } failure) + { + NextApplyFailure = null; + throw failure; + } + _effects.TryGetValue(message.LogicalId, out var prior); + _effects[message.LogicalId] = new MessageEffect(message.LogicalId, (prior?.Count ?? 0) + 1, message.Sequence, message.Value); + ApplyAttempted.TrySetResult(); + if (outgoing is { } output) + { + _outbox.Send(output); + if (TestApplicationProtocol.Read(_sessions, context.Message).Route == "messages/duplicate-output") + { + _outbox.Send(output); + } + } + context.Complete(); + } + finally + { + Interlocked.Decrement(ref _activeHandlers); + } + } + + private void PublishSnapshot() => _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); + + private EndpointSnapshot CreateSnapshot() => + new( + _activationId, + _siloDetails.SiloAddress.ToParsableString(), + _inbox.Count, + _outbox.Count, + _maxConcurrentHandlers, + _effects.Values.OrderBy(static effect => effect.Sequence).ToArray(), + _diagnostics.InboxDeadLetters.Select(ToSnapshot).ToArray(), + _diagnostics.OutboxDeadLetters.Select(ToSnapshot).ToArray(), + _inboxJobId.Value, + _processedMessages.Count, + _outboxJobId.Value, + _inboxJob.Value, + _outboxJob.Value); + + private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => + new( + deadLetter.Message.Envelope.MessageId, + TestApplicationProtocol.Read(_sessions, deadLetter.Message).Route, + deadLetter.Reason, + deadLetter.AttemptCount, + deadLetter.DeadLetteredAt); + + private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => + new( + deadLetter.Message.Envelope.MessageId, + TestApplicationProtocol.Read(_sessions, deadLetter.Message).Route, + deadLetter.Reason, + deadLetter.AttemptCount, + deadLetter.DeadLetteredAt); + + private sealed class ApplicationDispatcher(MessagingTestGrain owner) : IInboxHandler + { + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (owner.HandlerOverride is { } handler) + { + return handler.HandleAsync(context, cancellationToken); + } + + var application = TestApplicationProtocol.Read(owner._sessions, context.Message); + if (!application.Route.StartsWith("messages/", StringComparison.Ordinal) && application.Route != "typed") + { + throw new InvalidOperationException($"Unknown application route '{application.Route}'."); + } + return owner.HandleAsync(application.Body as TestMessage + ?? throw new InvalidOperationException($"Expected {nameof(TestMessage)} application payload."), + context, cancellationToken); + } + } +} + +// Tests can change registration between real activations without adding a production registry API. +public sealed class TestHandlerConfiguration +{ + private readonly ConcurrentDictionary _enabled = new(); + public bool IsEnabled(GrainId id) => !_enabled.TryGetValue(id, out var enabled) || enabled; + public void Set(GrainId id, bool enabled) => _enabled[id] = enabled; +} diff --git a/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs new file mode 100644 index 00000000000..43baddcd16a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs @@ -0,0 +1,116 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Orleans.Timers; + +namespace Orleans.Messaging.Tests.Support; + +internal static class ReceiverTestServices +{ + public const string InboxJobName = "orleans.messaging.inbox-drain"; + + private static readonly Func ReadCurrentContext = typeof(IGrainContext).Assembly + .GetType("Orleans.Runtime.RuntimeContext", throwOnError: true)! + .GetProperty("Current")!.GetMethod!.CreateDelegate>(); + + public static IGrainContext? CurrentGrainContext => ReadCurrentContext(); + + public static Type GetImplementationType(string name) => + typeof(IInbox).Assembly.GetType($"Orleans.Messaging.{name}", throwOnError: true)!; + + public static void Add(IServiceCollection services, Action configure) + { + var inboxType = GetImplementationType("Inbox"); + var extensionType = GetImplementationType("InboxExtension"); + var instrumentsType = GetImplementationType("MessagingInstruments"); + var pumpResultsType = GetImplementationType("MessagingPumpResults"); + var configuratorType = GetImplementationType("MessagingGrainTypeConfigurator"); + services.Configure(options => options.JournalFormatKey = "orleans-binary"); + services.AddOptions().Configure(configure); + services.TryAddSingleton(instrumentsType); + services.TryAddScoped(extensionType, sp => CreateInstance( + extensionType, + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(typeof(ILogger<>).MakeGenericType(extensionType)), + sp.GetRequiredService(instrumentsType), + sp.GetRequiredService(inboxType), + GetDictionary(sp, "inbox"), + GetDictionary(sp, "inbox-processed"), + GetInternalDictionary(sp, "InboxMessageState", "inbox-message-state"), + GetInternalDictionary(sp, "InboxDeadLetter", "inbox-dead-letters"), + GetValue(sp, "inbox-job-id"), + GetValue(sp, "inbox-job-handle"), + GetValue(sp, "inbox-completed-job-id"), + GetValue(sp, "inbox-job-sequence"), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(pumpResultsType), + sp.GetRequiredService(), + sp.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs), + sp.GetRequiredService>().Value)); + + services.TryAddKeyedScoped( + typeof(IInboxExtension), + (sp, _) => (IGrainExtension)sp.GetRequiredService(extensionType)); + services.TryAddScoped(inboxType, sp => + { + var options = sp.GetRequiredService>().Value; + _ = GetDictionary(sp, "inbox-processed"); + _ = GetInternalDictionary(sp, "InboxMessageState", "inbox-message-state"); + _ = GetInternalDictionary(sp, "InboxDeadLetter", "inbox-dead-letters"); + _ = GetValue(sp, "inbox-job-id"); + _ = GetValue(sp, "inbox-job-handle"); + _ = GetValue(sp, "inbox-completed-job-id"); + _ = GetValue(sp, "inbox-job-sequence"); + return CreateInstance(inboxType, + GetDictionary(sp, "inbox"), + sp.GetServices(), options.MaxCapacity); + }); + services.TryAddScoped(sp => (IInbox)sp.GetRequiredService(inboxType)); + + services.AddScoped(static sp => + new JournaledTestOutbox( + sp.GetRequiredKeyedService>("test-handler-output"), + sp.GetRequiredService())); + services.TryAddScoped(typeof(IMessagingDiagnostics), GetImplementationType("MessagingDiagnostics")); + services.TryAddScoped(pumpResultsType, sp => + { + var options = sp.GetRequiredService>().Value; + var completedRetentionPeriod = TimeSpan.FromMinutes(10); + var abandonedRetentionPeriod = options.JobStatusPollInterval <= TimeSpan.MaxValue / 4 + ? options.JobStatusPollInterval * 4 + : TimeSpan.MaxValue; + return CreateInstance(pumpResultsType, + sp.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs), + completedRetentionPeriod, + TimeSpan.FromTicks(Math.Max(completedRetentionPeriod.Ticks, abandonedRetentionPeriod.Ticks)), + 65_536); + }); + services.TryAddEnumerable(ServiceDescriptor.Singleton(typeof(IConfigureGrainTypeComponents), configuratorType)); + } + + private static IDurableDictionary GetDictionary(IServiceProvider services, string stateName) + where TKey : notnull => + services.GetRequiredKeyedService>($"__orleans.messaging.{stateName}"); + + private static object GetInternalDictionary(IServiceProvider services, string valueType, string stateName) => + services.GetRequiredKeyedService(typeof(IDurableDictionary<,>).MakeGenericType(typeof(TKey), GetImplementationType(valueType)), + $"__orleans.messaging.{stateName}"); + + private static IDurableValue GetValue(IServiceProvider services, string stateName) => + services.GetRequiredKeyedService>($"__orleans.messaging.{stateName}"); + + private static object CreateInstance(Type type, params object?[] arguments) => + Activator.CreateInstance(type, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DoNotWrapExceptions, + binder: null, arguments, culture: null)!; +} diff --git a/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs new file mode 100644 index 00000000000..81be9cbe4f3 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs @@ -0,0 +1,98 @@ +using System.Collections.Concurrent; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class SnapshotProbe +{ + private readonly ConcurrentDictionary _latest = new(); + private readonly ConcurrentDictionary> _waiters = new(); + + internal int WaiterListCount => _waiters.Count; + + public async Task WaitAsync( + GrainId grainId, + Func predicate, + TimeSpan? timeout = null) + { + if (_latest.TryGetValue(grainId, out var current) && predicate(current)) + { + return current; + } + + var waiter = new Waiter(predicate); + List waiters; + while (true) + { + waiters = _waiters.GetOrAdd(grainId, static _ => []); + lock (waiters) + { + if (!_waiters.TryGetValue(grainId, out var currentWaiters) + || !ReferenceEquals(waiters, currentWaiters)) + { + continue; + } + + if (_latest.TryGetValue(grainId, out current) && predicate(current)) + { + RemoveWaiterListIfEmpty(grainId, waiters); + return current; + } + + waiters.Add(waiter); + break; + } + } + + try + { + return await waiter.Completion.Task.WaitAsync(timeout ?? TimeSpan.FromSeconds(30)); + } + finally + { + lock (waiters) + { + waiters.Remove(waiter); + RemoveWaiterListIfEmpty(grainId, waiters); + } + } + } + + public void Publish(GrainId grainId, EndpointSnapshot snapshot) + { + _latest[grainId] = snapshot; + if (!_waiters.TryGetValue(grainId, out var waiters)) + { + return; + } + + lock (waiters) + { + foreach (var waiter in waiters.ToArray()) + { + if (waiter.Predicate(snapshot)) + { + waiters.Remove(waiter); + waiter.Completion.TrySetResult(snapshot); + } + } + + RemoveWaiterListIfEmpty(grainId, waiters); + } + } + + private void RemoveWaiterListIfEmpty(GrainId grainId, List waiters) + { + if (waiters.Count == 0) + { + _waiters.TryRemove(new KeyValuePair>(grainId, waiters)); + } + } + + private sealed class Waiter(Func predicate) + { + public Func Predicate { get; } = predicate; + public TaskCompletionSource Completion { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs b/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs new file mode 100644 index 00000000000..03c412d3113 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs @@ -0,0 +1,86 @@ +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class SnapshotProbeTests +{ + [Fact] + public async Task WaitAsync_TimeoutRemovesWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "timeout"); + var predicateCalls = 0; + + await Assert.ThrowsAsync( + () => probe.WaitAsync( + grainId, + _ => + { + predicateCalls++; + return false; + }, + TimeSpan.Zero)); + + Assert.Equal(0, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 0)); + + Assert.Equal(0, predicateCalls); + } + + [Fact] + public async Task WaitAsync_SuccessRemovesWaiterList() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "success"); + var wait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + var snapshot = CreateSnapshot(inboxCount: 1); + + probe.Publish(grainId, snapshot); + + Assert.Equal(0, probe.WaiterListCount); + Assert.Same(snapshot, await wait); + } + + [Fact] + public async Task WaitAsync_TimeoutRetainsListWithActiveWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "shared-list"); + var activeWait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + + await Assert.ThrowsAsync( + () => probe.WaitAsync(grainId, static _ => false, TimeSpan.Zero)); + + Assert.Equal(1, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 1)); + + await activeWait; + Assert.Equal(0, probe.WaiterListCount); + } + + private static EndpointSnapshot CreateSnapshot(int inboxCount) => + new( + Guid.Empty, + string.Empty, + inboxCount, + 0, + 0, + [], + [], + [], + null, + 0, + null, + null, + null); +} diff --git a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs new file mode 100644 index 00000000000..6d7a98a15a4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs @@ -0,0 +1,101 @@ +using System.Buffers; +using System.Text; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging.Tests.Support; + +// Routes and typed values belong to this test application, not to the messaging transport. +[GenerateSerializer] +public sealed record TestApplicationMessage( + [property: Id(0)] string Route, + [property: Id(1)] object? Body); + +internal static class TestApplicationProtocol +{ + public static HierarchicalKey NewMessageId() => HierarchicalKey.Create("tests", Guid.NewGuid().ToString("N")); + + public static OutboxMessage Create(SerializerSessionPool sessions, GrainId sender, GrainId receiver, + string route, object? body, HierarchicalKey messageId = default) => + new() + { + ReceiverId = receiver, + Envelope = CreateInbox(sessions, sender, route, body, messageId).Envelope + }; + + public static InboxMessage CreateInbox(SerializerSessionPool sessions, GrainId sender, + string route, object? body, HierarchicalKey messageId = default) + { + var key = messageId.IsDefault + ? body is TestMessage message ? message.LogicalId : NewMessageId() + : messageId; + using var builder = new EnvelopeBuilder(key); + builder.AddBytes(MessageHeaders.Payload, Encode(sessions, new TestApplicationMessage(route, body))); + builder.AddBytes(MessageHeaders.Subject, Encoding.UTF8.GetBytes(route)); + builder.AddValue(MessageHeaders.Sender, sender, new Serializer(sessions)); + return new InboxMessage { Envelope = builder.Build() }; + } + + public static byte[] Encode(SerializerSessionPool sessions, T value) + { + var output = new ArrayBufferWriter(); + using var session = sessions.GetSession(); + var writer = Writer.Create(output, session); + sessions.CodecProvider.GetCodec().WriteField(ref writer, 0, typeof(T), value); + writer.Commit(); + return output.WrittenSpan.ToArray(); + } + + public static T Decode(SerializerSessionPool sessions, ReadOnlyMemory payload) + { + using var session = sessions.GetSession(); + var reader = Reader.Create(payload.Span, session); + var field = reader.ReadFieldHeader(); + return sessions.CodecProvider.GetCodec().ReadValue(ref reader, field)!; + } + + public static TestApplicationMessage Read(SerializerSessionPool sessions, InboxMessage message) => + Decode(sessions, message.Envelope.Payload); + + public static TestApplicationMessage Read(SerializerSessionPool sessions, OutboxMessage message) => + Decode(sessions, message.Envelope.Payload); + + public static GrainId Sender(SerializerSessionPool sessions, Envelope envelope) => + envelope.TryGetValue(MessageHeaders.Sender, new Serializer(sessions), out var sender) + ? sender : default; + + public static string? Subject(Envelope envelope) => + envelope.TryGetSubject(out var subject) ? subject : null; + + public static InboxMessage With(InboxMessage message, SerializerSessionPool sessions, + HierarchicalKey? messageId = null, GrainId? sender = null, string? subject = null, + ReadOnlyMemory? payload = null, bool removeSender = false, bool removeSubject = false) + { + using var builder = new EnvelopeBuilder(messageId ?? message.Envelope.MessageId); + foreach (var key in message.Envelope.Keys) + { + if (key == MessageHeaders.Sender && (sender.HasValue || removeSender) + || key == MessageHeaders.Subject && (subject is not null || removeSubject) + || key == MessageHeaders.Payload && payload.HasValue) + { + continue; + } + if (!message.Envelope.TryGetBytes(key, out var bytes)) + { + throw new InvalidOperationException($"Missing enumerated header '{key}'."); + } + builder.AddBytes(key, bytes.Span); + } + if (sender is { } value) + { + builder.AddValue(MessageHeaders.Sender, value, new Serializer(sessions)); + } + if (subject is not null) builder.AddBytes(MessageHeaders.Subject, Encoding.UTF8.GetBytes(subject)); + if (payload is { } body) builder.AddBytes(MessageHeaders.Payload, body.Span); + return new() { Envelope = builder.Build() }; + } +} From f17bcf4e5bf75bea00f0f442c54cd1c1b09bf79c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 09:53:28 -0700 Subject: [PATCH 22/31] chore(messaging): generate renamed receiver API --- .../Orleans.Messaging/Orleans.Messaging.cs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 746510e9f65..e38c47dfdc7 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -8,6 +8,17 @@ //------------------------------------------------------------------------------ namespace Orleans.Messaging { + public sealed partial class DeadLetter + { + public int AttemptCount { get { throw null; } init { } } + + public System.DateTimeOffset DeadLetteredAt { get { throw null; } init { } } + + public required TMessage Message { get { throw null; } init { } } + + public required string Reason { get { throw null; } init { } } + } + [GenerateSerializer] [Alias("Orleans.Messaging.DeliveryResult")] public readonly partial struct DeliveryResult @@ -156,6 +167,20 @@ public partial interface IInboxHandlerContext void Complete(); } + public partial interface IMessagingDiagnostics + { + System.Collections.Generic.IReadOnlyList> InboxDeadLetters { get; } + + System.Collections.Generic.IReadOnlyList> OutboxDeadLetters { get; } + + bool RemoveInboxDeadLetter(HierarchicalKey messageId); + bool RemoveOutboxDeadLetter(HierarchicalKey messageId); + } + + public partial interface IMessagingGrain + { + } + [GenerateSerializer] [Alias("Orleans.Messaging.InboxMessage")] public readonly partial struct InboxMessage From 47966fab97838bdc7ba485c284320f1a9c26e0ee Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 15:47:49 -0700 Subject: [PATCH 23/31] test(messaging): register outcome hooks during grain setup --- .../Functional/InboxHandlerCompletionTests.cs | 23 +++++-------------- .../Support/MessagingTestGrains.cs | 18 +++++++++++++++ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs index 39119f2a21f..e780958df69 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -439,7 +439,11 @@ public async Task CompletionWrite_HookOutcomePreservesActualCommit(bool postComm using var events = new DiagnosticEventCollector(GrainTimerEvents.ListenerName); var input = await DeliverAsync(rig); var error = new IOException("Journal hook failed."); - await OnTurnAsync(rig.Context, () => rig.Manager.Hooks.Add(new FailingHook(postCommit, error))); + await OnTurnAsync(rig.Context, () => + { + if (postCommit) rig.Grain.AfterJournalOperationFailure = error; + else rig.Grain.BeforeJournalOperationFailure = error; + }); var timer = GetTimer(events, rig); var finished = await FinishedAsync(rig); handler.Release.TrySetResult(); @@ -449,7 +453,7 @@ public async Task CompletionWrite_HookOutcomePreservesActualCommit(bool postComm Assert.IsType( ((GrainTimerEvents.TickStop)(await TimerStoppedAsync(events, timer)).Payload!).Exception); AssertSuccess(rig, input, outputCount: 0); - await OnTurnAsync(rig.Context, rig.Manager.Hooks.Clear); + await OnTurnAsync(rig.Context, () => rig.Grain.AfterJournalOperationFailure = null); await AssertHealthyAsync(rig); } else @@ -650,21 +654,6 @@ private sealed class JobContext(DurableJob job) : IJobRunContext public int DequeueCount => 1; } - private sealed class FailingHook(bool postCommit, Exception error) : IJournaledStateHook - { - public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - if (!postCommit) throw error; - return default; - } - - public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - if (postCommit) throw error; - return default; - } - } - private sealed class StartAtTimerStop(Rig rig, IGrainTimer preceding, IDurableJobFeatureHandler feature, JobContext run, CancellationToken cancellationToken, DiagnosticEventCollector events) : IObserver diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index 4fa64fd0fc9..be0f2952108 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -91,6 +91,8 @@ public sealed class MessagingTestGrain : DurableGrain, IMessagingTestGrain, IDur private readonly SerializerSessionPool _sessions; private readonly TestHandlerConfiguration _handlerConfiguration; internal IInboxHandler? HandlerOverride { get; set; } + internal Exception? BeforeJournalOperationFailure { get; set; } + internal Exception? AfterJournalOperationFailure { get; set; } private readonly IDurableValue _inboxJobId; private readonly IDurableValue _inboxJob; private readonly IDurableValue _outboxJobId; @@ -140,6 +142,7 @@ public MessagingTestGrain( _handlerProbe = handlerProbe; _snapshotProbe = snapshotProbe; _lifecycleSubscription = GrainLifecycleEvents.AllEvents.Subscribe(this); + _journalOwner.Hooks.Add(new ControlledJournalHook(this)); } public override async Task OnActivateAsync(CancellationToken cancellationToken) @@ -490,6 +493,21 @@ public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken can ?? throw new InvalidOperationException($"Expected {nameof(TestMessage)} application payload."), context, cancellationToken); } + + private sealed class ControlledJournalHook(MessagingTestGrain owner) : IJournaledStateHook + { + public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (owner.BeforeJournalOperationFailure is { } failure) throw failure; + return default; + } + + public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (owner.AfterJournalOperationFailure is { } failure) throw failure; + return default; + } + } } } From e3c35ebf0a3e9ee0511725f785abb37df1cd7507 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 15:52:45 -0700 Subject: [PATCH 24/31] test(messaging): adapt receiver fixtures to latest published envelope --- .../Functional/ByteMessagingPayloadTests.cs | 15 ++-- .../Functional/InboxHeaderBehaviorTests.cs | 85 ++++++++++--------- .../Functional/InboxIdentityTests.cs | 15 ++-- .../JournaledTestOutboxBehaviorTests.cs | 15 ++-- .../Support/JournaledTestOutbox.cs | 8 +- .../Support/TestApplicationProtocol.cs | 64 +++++++------- 6 files changed, 106 insertions(+), 96 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs index e9151a1a184..60a1d78971a 100644 --- a/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs @@ -1,5 +1,4 @@ using System.Reflection; -using System.Runtime.InteropServices; using Microsoft.Extensions.DependencyInjection; using Orleans.Messaging.Tests.Support; using Orleans.Runtime; @@ -41,7 +40,7 @@ public async Task DirectAdmission_CallerCancellationLeavesActualAcceptanceAndHan scheduling.Continue(); await actual.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); - Assert.Equal(publishedPayload, handler.Payload); + Assert.Same(publishedPayload, handler.Payload); Assert.Equal(expected, handler.Payload.ToArray()); handler.Release.TrySetResult(); await handler.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); @@ -66,12 +65,10 @@ public async Task RpcAcceptanceAndDuplicate_CopyPayloadAndPreserveCallerBytesAcr await OnTurnAsync(context, () => grain.HandlerOverride = handler); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); - Assert.True(MemoryMarshal.TryGetArray(envelope.Envelope.Payload, out var caller)); - Assert.True(MemoryMarshal.TryGetArray(handler.Payload, out var received)); - Assert.NotSame(caller.Array, received.Array); - caller.Array![caller.Offset] ^= 0xff; + Assert.NotSame(envelope.Envelope.Payload, handler.Payload); + envelope.Envelope.Payload[0] ^= 0xff; Assert.Equal(expected, handler.Payload.ToArray()); - caller.Array[caller.Offset] ^= 0xff; + envelope.Envelope.Payload[0] ^= 0xff; Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); handler.Release.TrySetResult(); await handler.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); @@ -120,7 +117,7 @@ private static Task OnTurnAsync(IGrainContext context, Action action) private sealed class PayloadProbe(byte[] expected) : IInboxHandler, IDisposable { - public ReadOnlyMemory Payload { get; private set; } + public byte[] Payload { get; private set; } = null!; public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -134,7 +131,7 @@ public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationTok await Release.Task.WaitAsync(cancellationToken); Assert.Equal(expected, Payload.ToArray()); context.Complete(); - Assert.Equal(Payload, context.Message.Envelope.Payload); + Assert.Same(Payload, context.Message.Envelope.Payload); Assert.Equal(expected, context.Message.Envelope.Payload.ToArray()); Completed.TrySetResult(); } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs index 7056ae69a10..88045275fa9 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -1,5 +1,4 @@ using System.Text; -using System.Runtime.InteropServices; using Microsoft.Extensions.DependencyInjection; using Orleans.Messaging.Tests.Support; using Orleans.Journaling; @@ -29,8 +28,8 @@ public async Task RawMessage_AbsentOptionalMetadataCommitsAndDeduplicates(bool e var grain = Assert.IsType(context.GrainInstance); byte[] payload = empty ? [] : [0, 255, 128, 42]; var message = Raw(HierarchicalKey.Create("raw", Guid.NewGuid().ToString("N")), payload); - Assert.False(message.Envelope.TryGetSubject(out _)); - Assert.False(message.Envelope.TryGetBytes(MessageHeaders.Sender, out _)); + Assert.Null(message.Envelope.Subject); + Assert.Null(message.Envelope.Headers); using var handler = new RawHandler(context); await OnTurnAsync(context, () => grain.HandlerOverride = handler); using var storage = Fixture.Storage.BlockAcknowledgement(JournalId.FromGrainId(context.GrainId)); @@ -80,9 +79,10 @@ await OnTurnAsync(context, async () => ReceiverTestServices.GetImplementationType("InboxExtension")); Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(changed)).Status); var retained = Assert.Single(pending.Messages).Envelope; - Assert.Equal(first.EncodedHeaders, retained.EncodedHeaders); - Assert.True(retained.TryGetBytes("custom.trace", out var trace)); - Assert.Equal("original", Encoding.UTF8.GetString(trace.Span)); + Assert.Same(first.Payload, retained.Payload); + Assert.Same(first.Headers, retained.Headers); + Assert.True(Assert.IsType>(retained.Headers).TryGetValue("custom.trace", out var trace)); + Assert.Equal("original", Encoding.UTF8.GetString(trace)); Assert.Equal(GrainId.Create("producer", "first"), TestApplicationProtocol.Sender(Sessions, retained)); }); Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); @@ -100,16 +100,14 @@ public async Task UnknownHeaders_RecoverAndRemainAvailableAfterRetryAndDeadLette var key = HierarchicalKey.Create("headers", Guid.NewGuid().ToString("N")); var body = NewMessage(810, "poison") with { LogicalId = key, ThrowDuringPreparation = true }; var application = CreateEnvelope(receiver, body); - using var builder = new EnvelopeBuilder(key); - foreach (var header in application.Envelope.Keys) - { - Assert.True(application.Envelope.TryGetBytes(header, out var bytes)); - builder.AddBytes(header, bytes.Span); - } byte[] opaque = [0xff, 0, 0x80, 0x7f]; - builder.AddBytes("future.opaque", opaque); - builder.AddValue("future.number", 8675309, new Serializer(Sessions)); - var message = new InboxMessage { Envelope = builder.Build() }; + var headers = new Dictionary(Assert.IsType>(application.Envelope.Headers), + StringComparer.Ordinal) + { + ["future.opaque"] = opaque, + ["future.number"] = new Serializer(Sessions).SerializeToArray(8675309) + }; + var message = application with { Envelope = application.Envelope with { Headers = headers } }; var retry = Fixture.Metrics.WaitForCountAsync("orleans-messaging-inbox-messages-processed", 1, "retry"); await receiver.SeedInboxStateAsync(message, null, null); var previous = Fixture.GetGrainContext(receiver); @@ -125,10 +123,11 @@ public async Task UnknownHeaders_RecoverAndRemainAvailableAfterRetryAndDeadLette await OnTurnAsync(current, () => { var stored = Assert.Single(current.ActivationServices.GetRequiredService().Messages).Envelope; - Assert.True(stored.TryGetBytes("future.opaque", out var bytes)); - Assert.Equal(opaque, bytes.ToArray()); - Assert.True(stored.TryGetValue("future.number", new Serializer(Sessions), out var number)); - Assert.Equal(8675309, number); + var metadata = Assert.IsType>(stored.Headers); + Assert.True(metadata.TryGetValue("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes); + Assert.True(metadata.TryGetValue("future.number", out var number)); + Assert.Equal(8675309, new Serializer(Sessions).Deserialize(number)); }); Fixture.Clock.Advance(TimeSpan.FromHours(1)); await receiver.RequestDeactivationAsync(); @@ -143,10 +142,11 @@ await OnTurnAsync(current, () => await OnTurnAsync(current, () => { var retained = Assert.Single(diagnostics.InboxDeadLetters).Message.Envelope; - Assert.True(retained.TryGetBytes("future.opaque", out var bytes)); - Assert.Equal(opaque, bytes.ToArray()); - Assert.True(retained.TryGetValue("future.number", new Serializer(Sessions), out var number)); - Assert.Equal(8675309, number); + var metadata = Assert.IsType>(retained.Headers); + Assert.True(metadata.TryGetValue("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes); + Assert.True(metadata.TryGetValue("future.number", out var number)); + Assert.Equal(8675309, new Serializer(Sessions).Deserialize(number)); Assert.Equal(application.Envelope.Payload.ToArray(), retained.Payload.ToArray()); }); await receiver.RequestDeactivationAsync(); @@ -157,10 +157,11 @@ await OnTurnAsync(recovered, () => { var retained = Assert.Single(recovered.ActivationServices.GetRequiredService() .InboxDeadLetters).Message.Envelope; - Assert.True(retained.TryGetBytes("future.opaque", out var bytes)); - Assert.Equal(opaque, bytes.ToArray()); - Assert.True(retained.TryGetValue("future.number", new Serializer(Sessions), out var number)); - Assert.Equal(8675309, number); + var metadata = Assert.IsType>(retained.Headers); + Assert.True(metadata.TryGetValue("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes); + Assert.True(metadata.TryGetValue("future.number", out var number)); + Assert.Equal(8675309, new Serializer(Sessions).Deserialize(number)); }); Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, Raw(key, []))).Status); } @@ -168,7 +169,7 @@ await OnTurnAsync(recovered, () => [Theory] [InlineData(false)] [InlineData(true)] - public async Task MalformedMessage_RejectsBeforeSchedulingOrMutation(bool completed) + public async Task InvalidMessageHeader_RejectsBeforeSchedulingOrMutation(bool completed) { var receiver = NewGrain(); _ = await receiver.GetSnapshotAsync(); @@ -184,14 +185,16 @@ await OnTurnAsync(context, async () => await context.ActivationServices.GetRequiredService().WriteStateAsync(CancellationToken.None); }); } - Assert.True(MemoryMarshal.TryGetArray(message.Envelope.EncodedHeaders, out var encoded)); - encoded.Array![encoded.Offset] = 255; // Represents malformed external/generated input. + message = message with { Envelope = message.Envelope with + { + Headers = new(StringComparer.Ordinal) { ["invalid"] = null! } + }}; var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); await OnTurnAsync(context, async () => { var extension = (IInboxExtension)context.ActivationServices.GetRequiredService( ReceiverTestServices.GetImplementationType("InboxExtension")); - await Assert.ThrowsAsync(() => extension.DeliverAsync(message, TestContext.Current.CancellationToken).AsTask()); + await Assert.ThrowsAsync(() => extension.DeliverAsync(message, TestContext.Current.CancellationToken).AsTask()); }); Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, context.GrainId)); @@ -202,13 +205,15 @@ await OnTurnAsync(context, async () => } [Fact] - public async Task MalformedRecoveredHeaders_FailBeforeSchedulingOrBusinessMutation() + public async Task InvalidRecoveredHeaders_FailBeforeSchedulingOrBusinessMutation() { var receiver = NewGrain(); var key = HierarchicalKey.Create("recovery", Guid.NewGuid().ToString("N")); var message = Raw(key, []); - Assert.True(MemoryMarshal.TryGetArray(message.Envelope.EncodedHeaders, out var encoded)); - encoded.Array![encoded.Offset] = 255; + message = message with { Envelope = message.Envelope with + { + Headers = new(StringComparer.Ordinal) { ["invalid"] = null! } + }}; await receiver.SeedInboxStateAsync(message, null, null); var previous = Fixture.GetGrainContext(receiver); var journal = JournalId.FromGrainId(previous.GrainId); @@ -216,18 +221,18 @@ public async Task MalformedRecoveredHeaders_FailBeforeSchedulingOrBusinessMutati await receiver.RequestDeactivationAsync(); await WaitAsync(previous.Deactivated); var failure = await Assert.ThrowsAnyAsync(() => receiver.GetSnapshotAsync()); - Assert.Contains("envelope format", failure.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.Contains("Header values must not be null", failure.ToString(), StringComparison.Ordinal); Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, previous.GrainId)); } private InboxMessage Raw(HierarchicalKey key, byte[] payload, string? trace = null, GrainId? sender = null) { - using var builder = new EnvelopeBuilder(key); - builder.AddBytes(MessageHeaders.Payload, payload); - if (trace is not null) builder.AddBytes("custom.trace", Encoding.UTF8.GetBytes(trace)); - if (sender is { } value) builder.AddValue(MessageHeaders.Sender, value, new Serializer(Sessions)); - return new() { Envelope = builder.Build() }; + Dictionary? headers = null; + if (trace is not null) (headers ??= new(StringComparer.Ordinal))["custom.trace"] = Encoding.UTF8.GetBytes(trace); + if (sender is { } value) (headers ??= new(StringComparer.Ordinal))["sender"] = + new Serializer(Sessions).SerializeToArray(value); + return new() { Envelope = new() { MessageId = key, Payload = payload, Headers = headers } }; } private static Task OnTurnAsync(IGrainContext context, Action action) => diff --git a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs index 6ae704bf821..2243e584595 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs @@ -28,7 +28,7 @@ public async Task PendingCommand_NewSenderCoalescesAndPreservesOriginalEnvelope( await WaitAsync(handler.Entered.Task); var writes = Writes(rig); var scheduled = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId); - ReadOnlyMemory storedPayload = default; + byte[] storedPayload = null!; await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Envelope.Payload); Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, repeated)).Status); await OnTurnAsync(rig.Context, () => @@ -37,7 +37,7 @@ await OnTurnAsync(rig.Context, () => Assert.True(inbox.TryGetMessage(key, out var stored)); Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, stored.Envelope)); Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(stored.Envelope)); - Assert.Equal(storedPayload, stored.Envelope.Payload); + Assert.Same(storedPayload, stored.Envelope.Payload); Assert.Equal(original.Envelope.Payload.ToArray(), stored.Envelope.Payload.ToArray()); Assert.Single(rig.Pending); Assert.Empty(rig.Processed); @@ -76,7 +76,7 @@ public async Task PendingCommand_ConflictingContentRejectsBeforeSchedulingOrMuta var writes = Writes(rig); var job = rig.Grain.GetSnapshotForTest().InboxJob; var scheduled = Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId); - ReadOnlyMemory storedPayload = default; + byte[] storedPayload = null!; await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Envelope.Payload); var failure = await Assert.ThrowsAsync(() => DeliverOnTurnAsync(rig, conflict)); Assert.Contains(key.ToString(), failure.Message, StringComparison.Ordinal); @@ -85,7 +85,7 @@ await OnTurnAsync(rig.Context, () => { Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, Assert.Single(rig.Pending).Value.Envelope)); Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(Assert.Single(rig.Pending).Value.Envelope)); - Assert.Equal(storedPayload, Assert.Single(rig.Pending).Value.Envelope.Payload); + Assert.Same(storedPayload, Assert.Single(rig.Pending).Value.Envelope.Payload); Assert.Equal(original.Envelope.Payload.ToArray(), Assert.Single(rig.Pending).Value.Envelope.Payload.ToArray()); Assert.Empty(rig.Processed); Assert.Empty(rig.Grain.GetSnapshotForTest().Effects); @@ -283,8 +283,8 @@ public async Task MissingOptionalMetadata_RawHandlerCompletesWithoutSubject(bool var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); Assert.Equal(1, Assert.Single(completed.Effects).Count); Assert.Equal(1, completed.ProcessedMessageCount); - Assert.False(raw.Envelope.TryGetSubject(out _)); - if (removeSender) Assert.False(raw.Envelope.TryGetBytes(MessageHeaders.Sender, out _)); + Assert.Null(raw.Envelope.Subject); + if (removeSender) Assert.DoesNotContain("sender", Assert.IsType>(raw.Envelope.Headers)); Assert.Equal(bytes, raw.Envelope.Payload.ToArray()); Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, raw)).Status); Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); @@ -314,7 +314,8 @@ public async Task OversizedMetadata_RejectsBeforeSchedulingOrPersistence(string _ => "inventory.reserve.v1" }; var writes = Writes(rig); - Assert.ThrowsAny(() => Create(rig, key, subject, "oversized")); + var message = Create(rig, key, subject, "oversized"); + await Assert.ThrowsAnyAsync(() => DeliverOnTurnAsync(rig, message)); Assert.Equal(writes, Writes(rig)); Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, rig.Context.GrainId)); Assert.Empty(rig.Pending); diff --git a/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs index 0a113ba6e50..6798c2b83c9 100644 --- a/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs @@ -22,16 +22,20 @@ public async Task EquivalentDuplicates_BeforeAndAfterCommit_PreserveOneOutput() var copied = original; var serializer = Fixture.Client.ServiceProvider.GetRequiredService>(); var copy = serializer.Deserialize(serializer.SerializeToArray(original)); - Assert.Equal(original.Envelope.EncodedHeaders, copied.Envelope.EncodedHeaders); - Assert.NotEqual(original.Envelope.EncodedHeaders, copy.Envelope.EncodedHeaders); + Assert.Same(original.Envelope.Payload, copied.Envelope.Payload); + Assert.Same(original.Envelope.Headers, copied.Envelope.Headers); + Assert.NotSame(original.Envelope.Payload, copy.Envelope.Payload); + Assert.NotSame(original.Envelope.Headers, copy.Envelope.Headers); await owner.StageOutputAsync(original); var retained = Assert.Single(Fixture.GetStagedOutput(owner)); await owner.StageOutputAsync(copied); - Assert.Equal(retained.Envelope.EncodedHeaders, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.EncodedHeaders); + Assert.Same(retained.Envelope.Headers, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Headers); + Assert.Same(retained.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Payload); await owner.RetryWriteStateAsync(); await owner.StageOutputAsync(copy); - Assert.Equal(retained.Envelope.EncodedHeaders, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.EncodedHeaders); + Assert.Same(retained.Envelope.Headers, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Headers); + Assert.Same(retained.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Payload); var before = await owner.GetSnapshotAsync(); await owner.RequestDeactivationAsync(); var recovered = await owner.GetSnapshotAsync(); @@ -67,7 +71,8 @@ public async Task ConflictingDuplicate_RejectsChangedEnvelopeAndPreservesCommitt var exception = await Assert.ThrowsAsync(() => owner.StageOutputAsync(conflicting)); Assert.Contains(original.Envelope.MessageId.ToString(), exception.Message, StringComparison.Ordinal); - Assert.Equal(retained.Envelope.EncodedHeaders, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.EncodedHeaders); + Assert.Same(retained.Envelope.Headers, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Headers); + Assert.Same(retained.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Payload); await owner.RetryWriteStateAsync(); await owner.RequestDeactivationAsync(); var recovered = await owner.GetSnapshotAsync(); diff --git a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs index 75e558c8daf..964da2e6934 100644 --- a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs +++ b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs @@ -29,13 +29,13 @@ public void Send(OutboxMessage envelope) SendCalls++; if (_stopped) throw new InvalidOperationException("The test outbox owner is stopped."); if (envelope.Envelope.MessageId.IsDefault || envelope.ReceiverId.IsDefault - || !envelope.Envelope.TryGetBytes(MessageHeaders.Payload, out _)) + || envelope.Envelope.Payload is null) { throw new ArgumentException("An outgoing envelope requires identities and opaque payload bytes.", nameof(envelope)); } var sessions = context.ActivationServices.GetRequiredService(); - if (envelope.Envelope.TryGetValue(MessageHeaders.Sender, new Serializer(sessions), out var sender) - && sender != context.GrainId) + if (envelope.Envelope.Headers is { } headers && headers.TryGetValue("sender", out var encodedSender) + && new Serializer(sessions).Deserialize(encodedSender) != context.GrainId) { throw new InvalidOperationException($"The outgoing sender for '{envelope.Envelope.MessageId}' differs from the test outbox owner."); } @@ -44,7 +44,7 @@ public void Send(OutboxMessage envelope) if (existing.ReceiverId != envelope.ReceiverId || !string.Equals(TestApplicationProtocol.Subject(existing.Envelope), TestApplicationProtocol.Subject(envelope.Envelope), StringComparison.Ordinal) - || !existing.Envelope.Payload.Span.SequenceEqual(envelope.Envelope.Payload.Span)) + || !existing.Envelope.Payload.AsSpan().SequenceEqual(envelope.Envelope.Payload)) { throw new InvalidOperationException($"The outbox already contains a different envelope with message ID '{envelope.Envelope.MessageId}'."); } diff --git a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs index 6d7a98a15a4..c0f4cb4d52c 100644 --- a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs +++ b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.Text; using Orleans.Runtime; using Orleans.Serialization; using Orleans.Serialization.Buffers; @@ -33,11 +32,19 @@ public static InboxMessage CreateInbox(SerializerSessionPool sessions, GrainId s var key = messageId.IsDefault ? body is TestMessage message ? message.LogicalId : NewMessageId() : messageId; - using var builder = new EnvelopeBuilder(key); - builder.AddBytes(MessageHeaders.Payload, Encode(sessions, new TestApplicationMessage(route, body))); - builder.AddBytes(MessageHeaders.Subject, Encoding.UTF8.GetBytes(route)); - builder.AddValue(MessageHeaders.Sender, sender, new Serializer(sessions)); - return new InboxMessage { Envelope = builder.Build() }; + return new() + { + Envelope = new() + { + MessageId = key, + Payload = Encode(sessions, new TestApplicationMessage(route, body)), + Subject = route, + Headers = new(StringComparer.Ordinal) + { + ["sender"] = new Serializer(sessions).SerializeToArray(sender) + } + } + }; } public static byte[] Encode(SerializerSessionPool sessions, T value) @@ -50,10 +57,10 @@ public static byte[] Encode(SerializerSessionPool sessions, T value) return output.WrittenSpan.ToArray(); } - public static T Decode(SerializerSessionPool sessions, ReadOnlyMemory payload) + public static T Decode(SerializerSessionPool sessions, byte[] payload) { using var session = sessions.GetSession(); - var reader = Reader.Create(payload.Span, session); + var reader = Reader.Create(payload, session); var field = reader.ReadFieldHeader(); return sessions.CodecProvider.GetCodec().ReadValue(ref reader, field)!; } @@ -65,37 +72,32 @@ public static TestApplicationMessage Read(SerializerSessionPool sessions, Outbox Decode(sessions, message.Envelope.Payload); public static GrainId Sender(SerializerSessionPool sessions, Envelope envelope) => - envelope.TryGetValue(MessageHeaders.Sender, new Serializer(sessions), out var sender) - ? sender : default; + envelope.Headers is { } headers && headers.TryGetValue("sender", out var sender) + ? new Serializer(sessions).Deserialize(sender) : default; public static string? Subject(Envelope envelope) => - envelope.TryGetSubject(out var subject) ? subject : null; + envelope.Subject; public static InboxMessage With(InboxMessage message, SerializerSessionPool sessions, HierarchicalKey? messageId = null, GrainId? sender = null, string? subject = null, - ReadOnlyMemory? payload = null, bool removeSender = false, bool removeSubject = false) + byte[]? payload = null, bool removeSender = false, bool removeSubject = false) { - using var builder = new EnvelopeBuilder(messageId ?? message.Envelope.MessageId); - foreach (var key in message.Envelope.Keys) - { - if (key == MessageHeaders.Sender && (sender.HasValue || removeSender) - || key == MessageHeaders.Subject && (subject is not null || removeSubject) - || key == MessageHeaders.Payload && payload.HasValue) - { - continue; - } - if (!message.Envelope.TryGetBytes(key, out var bytes)) - { - throw new InvalidOperationException($"Missing enumerated header '{key}'."); - } - builder.AddBytes(key, bytes.Span); - } + var headers = message.Envelope.Headers is { } existing + ? new Dictionary(existing, StringComparer.Ordinal) : null; + if (removeSender) headers?.Remove("sender"); if (sender is { } value) { - builder.AddValue(MessageHeaders.Sender, value, new Serializer(sessions)); + (headers ??= new(StringComparer.Ordinal))["sender"] = new Serializer(sessions).SerializeToArray(value); } - if (subject is not null) builder.AddBytes(MessageHeaders.Subject, Encoding.UTF8.GetBytes(subject)); - if (payload is { } body) builder.AddBytes(MessageHeaders.Payload, body.Span); - return new() { Envelope = builder.Build() }; + return new() + { + Envelope = new() + { + MessageId = messageId ?? message.Envelope.MessageId, + Payload = payload ?? message.Envelope.Payload, + Subject = removeSubject ? null : subject ?? message.Envelope.Subject, + Headers = headers + } + }; } } From aee1a13617d4af02f93d3ed614ecb93ff78427c7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:22:43 -0700 Subject: [PATCH 25/31] test(messaging): scope setup hook to owning test grain --- .../Support/MessagingTestGrains.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index be0f2952108..76309bce633 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -493,20 +493,20 @@ public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken can ?? throw new InvalidOperationException($"Expected {nameof(TestMessage)} application payload."), context, cancellationToken); } + } - private sealed class ControlledJournalHook(MessagingTestGrain owner) : IJournaledStateHook + private sealed class ControlledJournalHook(MessagingTestGrain owner) : IJournaledStateHook + { + public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { - public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - if (owner.BeforeJournalOperationFailure is { } failure) throw failure; - return default; - } + if (owner.BeforeJournalOperationFailure is { } failure) throw failure; + return default; + } - public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - if (owner.AfterJournalOperationFailure is { } failure) throw failure; - return default; - } + public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (owner.AfterJournalOperationFailure is { } failure) throw failure; + return default; } } } From ee2fde28978e194b6792cc043120463f22c8de7b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:33:00 -0700 Subject: [PATCH 26/31] docs(messaging): preserve receiver guarantees for current envelope --- src/Orleans.Messaging/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 71c4a2bb8d2..b3d75b18012 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -1,7 +1,8 @@ # Microsoft Orleans Messaging This intermediate project supplies application command identities, envelopes, distinct incoming and outgoing -messages, and handler contracts. It remains non-packable while runtime and hosting layers assemble +messages, handler contracts, and journaled inbox processing. It remains non-packable while outgoing runtime +and hosting layers assemble `Microsoft.Orleans.Messaging`. `Envelope` has a required `HierarchicalKey MessageId` and non-null `byte[] Payload`, which can be empty. @@ -31,3 +32,14 @@ The default key is unset. Serialization stores the canonical path and reconstruc Decode local results, perform asynchronous preparation, build outgoing messages, and check cancellation before the first shared mutation. From that mutation through method return, execute synchronously: apply safe-to-commit changes, send through `IOutbox`, complete, and return. The runtime owns persistence and acknowledgement. + +The receiver acknowledges acceptance after its wakeup is scheduled and the journal commits the incoming +message with its logical generation and exact returned physical job handle. Equivalent pending submissions +preserve the first payload and header dictionary. Conflicts fail before scheduling or shared mutation. +Caller cancellation ends the wait while the admitted operation retains its gate through the actual outcome. + +Recovery validates incoming messages and dead letters before marking the inbox ready or scheduling work. +`DeadLetter` and `DeadLetter` diagnostics retain the corresponding message shape. +Preparation failures follow bounded retry and dead-letter policy. Errors after Complete preserve the logical +outcome through acknowledgement. Stop closes admission and drains actual operations before deletion or scope +disposal. Storage failures retain their first cause; a fresh activation restores the actual journal outcome. From 1257fb8626572c7dee16eaaa5f3776d9f04266d2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:53:48 -0700 Subject: [PATCH 27/31] test(messaging): adopt dynamic journal hook registration --- .../Functional/InboxEagerManagerTests.cs | 16 +++++++++++++++- .../Support/MessagingTestGrains.cs | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs index dbbdbb57238..a506c64ccb7 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs @@ -135,14 +135,28 @@ private sealed class EagerManager(IJournalFormat format, bool reverseStateOrder) private readonly Dictionary _states = []; private readonly JournalBufferWriter _writer = format.CreateWriter(); private readonly Dictionary _ids = []; + private readonly List _hooks = []; private IEnumerable States => reverseStateOrder ? _states.Values.Reverse() : _states.Values; private System.Runtime.ExceptionServices.ExceptionDispatchInfo? _failure; - public IList Hooks { get; } = []; + public IReadOnlyList Hooks => _hooks; public Action? BeforeCapture { get; set; } public int Writes { get; private set; } public int Requests { get; private set; } public bool IsFenced => _failure is not null; public List Batches { get; } = []; + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } public void RegisterStateMachine(string name, IStateMachine state) { _states.Add(name, state); diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index 76309bce633..fa04ad5b312 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -142,7 +142,7 @@ public MessagingTestGrain( _handlerProbe = handlerProbe; _snapshotProbe = snapshotProbe; _lifecycleSubscription = GrainLifecycleEvents.AllEvents.Subscribe(this); - _journalOwner.Hooks.Add(new ControlledJournalHook(this)); + _journalOwner.AddHookAsync(new ControlledJournalHook(this)).GetAwaiter().GetResult(); } public override async Task OnActivateAsync(CancellationToken cancellationToken) From a28d87771affae6ef1413545a69685267f926e15 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 18:36:22 -0700 Subject: [PATCH 28/31] test(messaging): keep journal hook registrations private --- .../Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs index a506c64ccb7..7dd0a9b2dcc 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxEagerManagerTests.cs @@ -138,7 +138,6 @@ private sealed class EagerManager(IJournalFormat format, bool reverseStateOrder) private readonly List _hooks = []; private IEnumerable States => reverseStateOrder ? _states.Values.Reverse() : _states.Values; private System.Runtime.ExceptionServices.ExceptionDispatchInfo? _failure; - public IReadOnlyList Hooks => _hooks; public Action? BeforeCapture { get; set; } public int Writes { get; private set; } public int Requests { get; private set; } From a6d66047f92114a9c857acba1b85b0d85af33c7c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:38:17 -0700 Subject: [PATCH 29/31] feat(messaging): deliver journal-acknowledged outbox commands --- .../Orleans.Messaging.csproj | 2 +- src/Orleans.Messaging/Outbox.State.cs | 57 + src/Orleans.Messaging/Outbox.cs | 1809 +++++++++ .../Contracts/OutboxCodecBoundaryTests.cs | 1711 +++++++++ .../Contracts/OutboxDeliveryBatchTests.cs | 3285 +++++++++++++++++ .../MessagingGrainTypeConfiguratorTests.cs | 430 ++- .../Support/BootstrapOutboxServices.cs | 176 + .../Support/BootstrapTestGrains.cs | 31 +- .../Support/ControlledTimerRegistry.cs | 137 + .../Support/OutboxTestMessages.cs | 31 + .../Support/TestJournaledStateHook.cs | 23 + 11 files changed, 7662 insertions(+), 30 deletions(-) create mode 100644 src/Orleans.Messaging/Outbox.State.cs create mode 100644 src/Orleans.Messaging/Outbox.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs create mode 100644 test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ControlledTimerRegistry.cs create mode 100644 test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs create mode 100644 test/Orleans.Messaging.Tests/Support/TestJournaledStateHook.cs diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj index 2069ef3a10b..e64fe64cc3d 100644 --- a/src/Orleans.Messaging/Orleans.Messaging.csproj +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -2,7 +2,7 @@ Microsoft.Orleans.Messaging Microsoft Orleans Messaging - Opaque messaging contracts and journaled inbox processing for Microsoft Orleans. + Opaque messaging contracts and journaled inbox and outbox processing for Microsoft Orleans. $(PackageTags) Messaging Inbox Outbox false diff --git a/src/Orleans.Messaging/Outbox.State.cs b/src/Orleans.Messaging/Outbox.State.cs new file mode 100644 index 00000000000..eb20fbaf8c0 --- /dev/null +++ b/src/Orleans.Messaging/Outbox.State.cs @@ -0,0 +1,57 @@ +using Orleans.Journaling; + +namespace Orleans.Messaging; + +internal sealed partial class Outbox +{ + // The existing sequence stream supplies the feature's capture/ACK boundary for every journal writer. + private sealed class SequenceState(Outbox owner, IDurableValueCommandCodec codec) + : IDurableValue, IStateMachine, IDurableValueCommandHandler + { + private long _value; + private bool _dirty; + + public long Value + { + get => _value; + set + { + _value = value; + _dirty = true; + } + } + + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, codec).Apply(entry.Reader, this); + + public void Reset(JournalStreamWriter writer) + { + _value = 0; + _dirty = false; + owner.ResetState(); + } + + public void OnRecoveryCompleted() => owner.OnRecoveryCompleted(); + + public void WritePendingEntries(JournalStreamWriter writer) + { + owner.CaptureWrite(); + if (_dirty) + { + codec.WriteSet(_value, writer); + _dirty = false; + } + } + + public void WriteSnapshot(JournalStreamWriter writer) + { + owner.CaptureWrite(); + codec.WriteSet(_value, writer); + _dirty = false; + } + + public void OnWriteCompleted() => owner.CompleteCapture(); + + void IDurableValueCommandHandler.ApplySet(long value) => _value = value; + } +} diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs new file mode 100644 index 00000000000..fa3b83026fc --- /dev/null +++ b/src/Orleans.Messaging/Outbox.cs @@ -0,0 +1,1809 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.TypeSystem; +using Orleans.Timers; + +namespace Orleans.Messaging; + +/// +/// Stages outbound messages using owner-bound durable collections. +/// The sequence state associates message cohorts with journal acknowledgement. +/// +internal sealed partial class Outbox : IOutbox, IDurableJobFeatureHandler, ILifecycleObserver, IJournaledStateHook +{ + internal const string JobName = "orleans.messaging.outbox-flush"; + public bool CanHandle(string jobName) => string.Equals(jobName, JobName, StringComparison.Ordinal); + + private readonly IJournaledStateManager _stateManager; + private readonly IDurableDictionary _messages; + private readonly IGrainFactory _grainFactory; + private readonly IGrainContext _grainContext; + private readonly Serializer _senderSerializer; + private readonly string _grainType; + private readonly ITimerRegistry _timerRegistry; + private readonly ILogger _logger; + private readonly MessagingInstruments _instruments; + private readonly TimeSpan _backpressureRetryDelay; + private readonly TimeSpan _maxRetryAge; + private readonly int _maxDeliveryAttempts; + private readonly int _batchSize; + private readonly TimeSpan _idleRetirementGracePeriod; + private DateTimeOffset? _idleRetirementAt; + private PumpOwner? _pendingRetirementOwner; + private string? _previousCompletedOwner; + private readonly TimeSpan _deadLetterRetentionPeriod; + private readonly int _maxRetainedDeadLetters; + private readonly IDurableDictionary _messageStates; + private readonly IDurableDictionary _deadLetters; + private readonly IDurableValue _jobId; + private readonly IDurableValue _job; + private readonly IDurableValue _completedJobId; + private readonly IDurableValue _jobSequence; + private readonly ILocalDurableJobManager _jobManager; + private readonly TimeProvider _jobTimeProvider; + private readonly MessagingPumpResults _pumpResults; + private readonly MessagingPumpCoordinator _pumpCoordinator = new(); + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly SemaphoreSlim _deliveryGate = new(1, 1); + private readonly CancellationTokenSource _shutdown = new(); + private readonly Dictionary _pendingMessages = []; + private int _unacknowledgedMessageCount; + private PendingMessage[] _capturedMessages = []; + private bool _captureStarted; + private OwnershipProposal? _preparedOwnership; + private string? _preparingOwnershipId; + private Task? _ownershipPreparation; + private int _activeOwnershipPreparations; + private TaskCompletionSource? _ownershipPreparationsDrained; + private int _activeWrites; + private TaskCompletionSource? _writesDrained; + private string? _committingOwnershipId; + private DurableJob? _committingJob; + private string? _committingCompletedJobId; + private string? _durableOwnershipId; + private DurableJob? _durableJob; + private string? _durableCompletedJobId; + private PendingDeliveryBatch? _pendingDeliveryBatch; + private bool _recoveryCompleted; + private int _ensureJobScheduledQueued; + private int _activePumpTurns; + private bool _localPumpRequested; + private PumpTimerState? _pumpTimer; + private LocalPumpTimerState? _localPumpTimer; + private EnsureJobTimerState? _ensureJobTimer; + private string _ownershipEpoch = Guid.NewGuid().ToString("N"); + private long _reservedSequence; + private long _stateGeneration; + private string? _ownershipStateError; + private ExceptionDispatchInfo? _failure; + private int _metricsActive; + private int _reportedDepth; + + public Outbox( + IJournaledStateManager manager, + IGrainFactory grainFactory, + IGrainContext grainContext, + Serializer senderSerializer, + ITimerRegistry timerRegistry, + ILogger logger, + MessagingInstruments instruments, + ILocalDurableJobManager jobManager, + IDurableJobHandlerRegistry jobHandlers, + MessagingPumpResults pumpResults, + [FromKeyedServices(DurableJobTimeProviderNames.DurableJobs)] TimeProvider jobTimeProvider, + IOptions options, + [FromKeyedServices(MessagingStateNames.Outbox)] IDurableDictionary messages, + [FromKeyedServices(MessagingStateNames.OutboxMessageState)] IDurableDictionary messageStates, + [FromKeyedServices(MessagingStateNames.OutboxDeadLetters)] IDurableDictionary deadLetters, + [FromKeyedServices(MessagingStateNames.OutboxJobId)] IDurableValue jobId, + [FromKeyedServices(MessagingStateNames.OutboxJobHandle)] IDurableValue job, + [FromKeyedServices(MessagingStateNames.OutboxCompletedJobId)] IDurableValue completedJobId, + IDurableValueCommandCodec jobSequenceCodec) + { + ArgumentNullException.ThrowIfNull(manager); + ArgumentNullException.ThrowIfNull(grainFactory); + ArgumentNullException.ThrowIfNull(grainContext); + ArgumentNullException.ThrowIfNull(senderSerializer); + ArgumentNullException.ThrowIfNull(timerRegistry); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(instruments); + ArgumentNullException.ThrowIfNull(jobManager); + ArgumentNullException.ThrowIfNull(jobHandlers); + ArgumentNullException.ThrowIfNull(pumpResults); + ArgumentNullException.ThrowIfNull(jobTimeProvider); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(messages); + ArgumentNullException.ThrowIfNull(messageStates); + ArgumentNullException.ThrowIfNull(deadLetters); + ArgumentNullException.ThrowIfNull(jobId); + ArgumentNullException.ThrowIfNull(job); + ArgumentNullException.ThrowIfNull(completedJobId); + ArgumentNullException.ThrowIfNull(jobSequenceCodec); + _stateManager = manager; + _grainFactory = grainFactory; + _grainContext = grainContext; + _senderSerializer = senderSerializer; + _grainType = grainContext.GrainId.Type.ToString(); + _timerRegistry = timerRegistry; + _logger = logger; + _instruments = instruments; + _jobManager = jobManager; + _pumpResults = pumpResults; + _jobTimeProvider = jobTimeProvider; + _backpressureRetryDelay = options.Value.BackpressureRetryDelay; + _maxRetryAge = options.Value.MaxOutboxRetryAge; + _maxDeliveryAttempts = options.Value.MaxDeliveryAttempts; + _batchSize = options.Value.OutboxBatchSize; + _idleRetirementGracePeriod = options.Value.OutboxIdleRetirementGracePeriod; + _deadLetterRetentionPeriod = options.Value.DeadLetterRetentionPeriod; + _maxRetainedDeadLetters = options.Value.MaxRetainedDeadLetters; + _messages = messages; + _messageStates = messageStates; + _deadLetters = deadLetters; + _jobId = jobId; + _job = job; + _completedJobId = completedJobId; + var sequence = new SequenceState(this, jobSequenceCodec); + _jobSequence = sequence; + manager.RegisterStateMachine(MessagingStateNames.OutboxJobSequence, sequence); + jobHandlers.Register(this); + // Before hooks run in reverse registration order, after all asynchronous preparation. + manager.Hooks.Insert(0, this); + + var lifecycle = grainContext.ObservableLifecycle; + lifecycle.Subscribe(RuntimeTypeNameFormatter.Format(GetType()), GrainLifecycleStage.Activate, this); + } + + internal IDurableDictionary MessageState => _messages; + internal IDurableDictionary AttemptState => _messageStates; + internal IDurableDictionary DeadLetterState => _deadLetters; + internal IDurableValue JobIdState => _jobId; + internal IDurableValue JobState => _job; + internal IDurableValue CompletedJobIdState => _completedJobId; + internal IDurableValue JobSequenceState => _jobSequence; + + public GrainId SenderId => _grainContext.GrainId; + + public int Count => _messages.Count; + + public IEnumerable Messages => _messages.Values; + + public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) => + _messages.TryGetValue(messageId, out envelope); + + private async ValueTask PrepareOwnershipAsync(bool replaceExisting, CancellationToken cancellationToken = default) + { + if (HasPreparedOwnership() || HasCommittedOwnership() && !replaceExisting) + { + return; + } + var preparation = _ownershipPreparation ??= ScheduleOwnershipAsync(cancellationToken); + try + { + await preparation.ConfigureAwait(true); + } + finally + { + if (ReferenceEquals(_ownershipPreparation, preparation)) + { + _ownershipPreparation = null; + } + } + } + + private async Task ScheduleOwnershipAsync(CancellationToken cancellationToken) + { + using var cancellation = MessagingCancellation.Combine(cancellationToken, _shutdown.Token, out var combinedToken); + var previousOwner = CurrentOwner; + var sequence = checked(++_reservedSequence); + var id = MessagingJobOwnership.CreateId(_ownershipEpoch, sequence); + _preparingOwnershipId = id; + try + { + var job = await _jobManager.ScheduleJobAsync(new ScheduleJobRequest + { + Target = _grainContext.GrainId, + JobName = JobName, + DueTime = _jobTimeProvider.GetUtcNow(), + Metadata = MessagingJobOwnership.CreateMetadata(id) + }, combinedToken).ConfigureAwait(true); + ValidateOwner(previousOwner); + _preparedOwnership = new(id, sequence, job, previousOwner); + } + catch + { + _failure?.Throw(); + throw; + } + finally + { + _preparingOwnershipId = null; + } + } + + async ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (operation == JournaledStateOperation.Delete || _unacknowledgedMessageCount == 0 || HasCommittedOwnership()) + { + return; + } + + ValidateReady(); + // Owned writes can retain _gate through ACK. Share scheduling, not that gate, with the capture hook. + StartOwnershipPreparation(); + try + { + await PrepareOwnershipAsync(replaceExisting: false, cancellationToken).ConfigureAwait(true); + ValidateReady(); + ApplyPreparedOwnership(); + } + finally + { + CompleteOwnershipPreparation(); + } + } + + public void Send(OutboxMessage envelope) + { + ValidateReady(); + ValidateEnvelope(envelope); + if (_messages.TryGetValue(envelope.Envelope.MessageId, out var existing)) + { + ValidateEquivalent(existing, envelope); + return; + } + _pendingMessages.Add(envelope.Envelope.MessageId, new(envelope.Envelope.MessageId)); + + try + { + StageMessage(envelope); + } + catch (Exception exception) + { + FailStaging(exception); + } + } + + private void ValidateEnvelope(OutboxMessage message) + { + EnvelopeValidation.Validate(message); + if (GetSender(message) is { } sender && sender != _grainContext.GrainId) + { + throw new InvalidOperationException( + $"Outbox sender '{sender}' does not match the owning grain '{_grainContext.GrainId}'."); + } + } + + private GrainId? GetSender(OutboxMessage message) => + message.Envelope.Headers is { } headers && headers.TryGetValue("sender", out var value) + ? _senderSerializer.Deserialize(value) : null; + + private void StageMessage(OutboxMessage envelope) + { + RestorePendingRetirement(); + var state = new OutboxMessageState { EnqueuedAt = _jobTimeProvider.GetUtcNow() }; + _idleRetirementAt = null; + _messages.Add(envelope.Envelope.MessageId, envelope); + _messageStates.Add(envelope.Envelope.MessageId, state); + _unacknowledgedMessageCount++; + EnsureMetricsActive(); + ReconcileOutboxDepth(); + _instruments.OnOutboxMessageSent(_grainType); + } + + private static void ValidateEquivalent(OutboxMessage existing, OutboxMessage envelope) + { + if (!EnvelopeEquivalence.AreEquivalent(existing, envelope)) + { + throw new InvalidOperationException( + $"The outbox already contains a different envelope with message ID '{envelope.Envelope.MessageId}'."); + } + } + + private void StartOwnershipPreparation() + { + _activeOwnershipPreparations++; + } + + private void CompleteOwnershipPreparation() + { + if (--_activeOwnershipPreparations == 0) + { + var waiter = _ownershipPreparationsDrained; + _ownershipPreparationsDrained = null; + waiter?.TrySetResult(); + } + ReleaseUnusedOwnership(); + } + + private void ReleaseUnusedOwnership() + { + if (_activeOwnershipPreparations == 0 && _unacknowledgedMessageCount == 0 && !_captureStarted && _gate.CurrentCount != 0) + { + _preparedOwnership = null; + } + } + + private bool HasPreparedOwnership() => _preparedOwnership is { } proposal + && proposal.PreviousOwner.Generation == _stateGeneration + && ((string.Equals(proposal.PreviousOwner.Id, _jobId.Value, StringComparison.Ordinal) + && (proposal.PreviousOwner.Job is null && _job.Value is null + || MessagingJobOwnership.IsSamePhysicalJob(proposal.PreviousOwner.Job, _job.Value))) + || (string.Equals(proposal.Id, _jobId.Value, StringComparison.Ordinal) + && MessagingJobOwnership.IsSamePhysicalJob(proposal.Job, _job.Value))); + + private void ApplyPreparedOwnership() + { + var owner = _preparedOwnership!; + if (string.Equals(owner.Id, _jobId.Value, StringComparison.Ordinal) + && MessagingJobOwnership.IsSamePhysicalJob(owner.Job, _job.Value)) + { + return; + } + ValidateOwner(owner.PreviousOwner); + _jobId.Value = owner.Id; + _job.Value = owner.Job; + _jobSequence.Value = owner.Sequence; + } + + private void StageWrite(OutboxWrite operation) + { + ValidateGeneration(operation.Generation); + Items deliveries = default; + Dictionary? deadLetters = null; + switch (operation) + { + case DeliveryWrite delivery: + ValidateOwner(delivery.Owner); + for (var index = 0; index < delivery.Outcomes.Count; index++) + { + ValidateCandidate(delivery.Outcomes[index].Candidate); + } + if (delivery.Outcomes.Count == 1) + { + deliveries = new(PrepareDelivery(delivery.Outcomes[0], ref deadLetters)); + } + else + { + var prepared = new PreparedDelivery[delivery.Outcomes.Count]; + for (var index = 0; index < prepared.Length; index++) + { + prepared[index] = PrepareDelivery(delivery.Outcomes[index], ref deadLetters); + } + deliveries = new(prepared); + } + delivery.Prepared = deliveries; + break; + case ClearOwnerWrite clear: + ValidateOwner(clear.Owner); + break; + case OwnershipWrite: + if (!HasPreparedOwnership()) + { + throw new InvalidOperationException("Outbox ownership must be prepared before staging."); + } + break; + case CompactWrite: + deadLetters = _deadLetters.ToDictionary(static pair => pair.Key, static pair => pair.Value); + DeadLetterRetention.Compact(deadLetters, _jobTimeProvider.GetUtcNow(), + _deadLetterRetentionPeriod, _maxRetainedDeadLetters, static entry => entry.DeadLetteredAt); + break; + } + if (operation is OwnershipWrite) + { + ApplyPreparedOwnership(); + } + for (var index = 0; index < deliveries.Count; index++) + { + var result = deliveries[index]; + var envelope = result.Outcome.Candidate.Envelope; + if (result.Remove) + { + _messages.Remove(envelope.Envelope.MessageId); + _messageStates.Remove(envelope.Envelope.MessageId); + } + else + { + _messageStates[envelope.Envelope.MessageId] = result.Retry!; + } + RecordDeliveryMetrics(result); + } + if (deadLetters is not null) + { + foreach (var key in _deadLetters.Keys.Where(key => !deadLetters.ContainsKey(key)).ToArray()) + { + _deadLetters.Remove(key); + } + foreach (var entry in deadLetters) + { + if (!_deadLetters.TryGetValue(entry.Key, out var existing) || !ReferenceEquals(existing, entry.Value)) + { + _deadLetters[entry.Key] = entry.Value; + } + } + } + if (Count == 0) + { + var idleAt = GetIdleRetirementAt(); + if (CanRetireOwner() && _jobTimeProvider.GetUtcNow() >= idleAt) + { + if (operation is ClearOwnerWrite completed) + { + ClearOwner(completed.Owner); + } + else if (operation is DeliveryWrite { ClearOwnershipWhenEmpty: true } delivered) + { + ClearOwner(delivered.Owner); + } + } + } + ReconcileOutboxDepth(); + } + + private DateTimeOffset GetIdleRetirementAt() => _idleRetirementAt ??= + MessagingTime.AddClamped(_jobTimeProvider.GetUtcNow(), _idleRetirementGracePeriod); + + private bool CanRetireOwner() => Count == 0 && _activeOwnershipPreparations == 0 + && _unacknowledgedMessageCount == 0 && !_captureStarted && _activeWrites == 0; + + private void ClearOwner(PumpOwner owner) + { + // Admission and staging run synchronously. Hooks/capture may subsequently admit a new + // cohort, which must prepare a new durable wakeup before its own acknowledgement. + ValidateOwner(owner); + _pendingRetirementOwner = owner; + _previousCompletedOwner = _completedJobId.Value; + _completedJobId.Value = owner.Id; + _jobId.Value = null; + _job.Value = null; + _idleRetirementAt = null; + } + + private void RestorePendingRetirement() + { + if (_pendingRetirementOwner is not { } owner || _captureStarted) + { + return; + } + // A hook may admit work after staging clear but before capture. Reuse only the + // still-acknowledged physical handle, never overwrite an independently changed owner. + if (owner.Generation == _stateGeneration && _jobId.Value is null && _job.Value is null + && string.Equals(_durableOwnershipId, owner.Id, StringComparison.Ordinal) + && MessagingJobOwnership.IsSamePhysicalJob(_durableJob, owner.Job) + && string.Equals(_completedJobId.Value, owner.Id, StringComparison.Ordinal)) + { + _jobId.Value = owner.Id; + _job.Value = owner.Job; + _completedJobId.Value = _previousCompletedOwner; + } + _pendingRetirementOwner = null; + _previousCompletedOwner = null; + } + + private void CaptureWrite() + { + if (_captureStarted) + { + return; + } + var messages = _unacknowledgedMessageCount == 0 ? [] + : _pendingMessages.Values.Where(static entry => !entry.Captured).ToArray(); + var ownerChanged = !string.Equals(_durableOwnershipId, _jobId.Value, StringComparison.Ordinal) + || !(_durableJob is null && _job.Value is null || MessagingJobOwnership.IsSamePhysicalJob(_durableJob, _job.Value)) + || !string.Equals(_durableCompletedJobId, _completedJobId.Value, StringComparison.Ordinal); + if (messages.Length == 0 && !ownerChanged) + { + return; + } + _captureStarted = true; + _pendingRetirementOwner = null; + _previousCompletedOwner = null; + _capturedMessages = messages; + foreach (var message in messages) + { + message.Captured = true; + } + _committingOwnershipId = _jobId.Value; + _committingJob = _job.Value; + _committingCompletedJobId = _completedJobId.Value; + } + + private void CompleteCapture() + { + if (!_captureStarted) + { + return; + } + _durableOwnershipId = _committingOwnershipId; + _durableJob = _committingJob; + _durableCompletedJobId = _committingCompletedJobId; + foreach (var message in _capturedMessages) + { + _unacknowledgedMessageCount--; + _pendingMessages.Remove(message.MessageId); + } + _capturedMessages = []; + _captureStarted = false; + ReleaseUnusedOwnership(); + ScheduleLocalPump(); + } + + internal void FailPersistence(Exception exception) + { + if (_failure is not null) + { + return; + } + _failure = ExceptionDispatchInfo.Capture(exception); + Stop(); + } + + [DoesNotReturn] + private void FailStaging(Exception exception) + { + FailPersistence(exception); + LogStagingFailed(_logger, _failure!.SourceException); + try + { + _grainContext.Deactivate(new DeactivationReason( + DeactivationReasonCode.ApplicationError, _failure.SourceException, "Outbox staging failed."), CancellationToken.None); + } + catch (Exception deactivationException) + { + LogDeactivationFailed(_logger, deactivationException); + } + _failure.Throw(); + } + + private void OnRecoveryCompleted() + { + foreach (var (key, message) in _messages) + { + ValidateEnvelope(message); + if (key != message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered outbox key does not match its command identity."); + } + } + foreach (var letter in _deadLetters.Values) ValidateEnvelope(letter.Message); + _reservedSequence = _jobSequence.Value; + _durableOwnershipId = _jobId.Value; + _durableJob = _job.Value; + _durableCompletedJobId = _completedJobId.Value; + _ownershipStateError = MessagingJobOwnership.GetPairError(_jobId.Value, _job.Value); + _recoveryCompleted = true; + ReconcileOutboxDepth(); + } + + private void ResetState() + { + _recoveryCompleted = false; + _pumpCoordinator.Reset(); + _pumpResults.Clear(JobName); + _localPumpRequested = false; + DisposePumpTimers(); + _stateGeneration++; + _ownershipEpoch = Guid.NewGuid().ToString("N"); + _reservedSequence = 0; + _idleRetirementAt = null; + _pendingRetirementOwner = null; + _previousCompletedOwner = null; + _pendingMessages.Clear(); + _unacknowledgedMessageCount = 0; + _preparedOwnership = null; + _capturedMessages = []; + _captureStarted = false; + _durableOwnershipId = null; + _durableJob = null; + _durableCompletedJobId = null; + _ownershipStateError = null; + } + + private async ValueTask SubmitAsync(OutboxWrite operation) + { + ValidateReady(); + try + { + StageWrite(operation); + } + catch (Exception exception) + { + FailStaging(exception); + } + _activeWrites++; + try + { + await _stateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + _failure?.Throw(); + if (operation is DeliveryWrite delivery) + { + var delivered = 0; + var backpressured = 0; + for (var index = 0; index < delivery.Prepared.Count; index++) + { + var status = delivery.Prepared[index].Outcome.Result?.Status; + if (status is DeliveryStatus.Accepted or DeliveryStatus.Duplicate or DeliveryStatus.DeadLettered) + { + delivered++; + } + else if (status == DeliveryStatus.Backpressured) + { + backpressured++; + } + } + LogDeliveryComplete(_logger, delivered, backpressured, delivery.Prepared.Count - delivered - backpressured, Count); + } + } + catch (JournaledStatePreCommitException exception) + { + // This owned operation has already staged its effects. Retire it and replay a fresh owner. + FailPersistence(exception); + LogPrerequisiteFailed(_logger, exception); + try + { + _grainContext.Deactivate(new DeactivationReason( + DeactivationReasonCode.ApplicationError, exception, "Outbox persistence prerequisite failed."), CancellationToken.None); + } + catch (Exception deactivationException) + { + LogDeactivationFailed(_logger, deactivationException); + } + throw; + } + catch (JournaledStatePostCommitException) + { + throw; + } + catch (Exception exception) + { + FailPersistence(exception); + _failure!.Throw(); + throw; + } + finally + { + if (--_activeWrites == 0) + { + var waiter = _writesDrained; + _writesDrained = null; + waiter?.TrySetResult(); + } + } + } + + private PumpOwner CurrentOwner => new(_jobId.Value, _job.Value, _stateGeneration); + + private void ValidateReady() + { + _failure?.Throw(); + _shutdown.Token.ThrowIfCancellationRequested(); + ThrowIfOwnershipStateInvalid(); + if (!_recoveryCompleted) + { + throw new InvalidOperationException("Outbox initialization has not completed."); + } + } + + private void ValidateGeneration(long generation) + { + ValidateReady(); + if (generation != _stateGeneration) + { + throw new InvalidOperationException("The admitted outbox operation belongs to an obsolete activation or deletion generation."); + } + } + + private void ValidateOwner(PumpOwner owner) + { + ValidateGeneration(owner.Generation); + if (!string.Equals(owner.Id, _jobId.Value, StringComparison.Ordinal) + || !(owner.Job is null && _job.Value is null || MessagingJobOwnership.IsSamePhysicalJob(owner.Job, _job.Value))) + { + throw new InvalidOperationException("The admitted outbox operation no longer owns the acknowledged physical job."); + } + } + + private void ThrowIfOwnershipStateInvalid() + { + var error = _ownershipStateError ?? MessagingJobOwnership.GetPairError(_jobId.Value, _job.Value); + if (error is not null) + { + throw new InvalidOperationException(error); + } + } + + private bool HasCommittedOwnership() => MessagingJobOwnership.HasOwner(_jobId.Value, _job.Value) + && string.Equals(_durableOwnershipId, _jobId.Value, StringComparison.Ordinal) + && MessagingJobOwnership.IsSamePhysicalJob(_durableJob, _job.Value); + + private bool IsCurrentPump(DurableJob job, long stateGeneration) => _failure is null && !_shutdown.IsCancellationRequested + && _recoveryCompleted && stateGeneration == _stateGeneration + && MessagingJobOwnership.IsSamePhysicalJob(_job.Value, job) && HasCommittedOwnership(); + + private bool IsOwnershipTransitionPending(string ownershipId) => + string.Equals(ownershipId, _preparingOwnershipId, StringComparison.Ordinal) + || (_preparedOwnership is { } proposal && string.Equals(ownershipId, proposal.Id, StringComparison.Ordinal) + && (!HasCommittedOwnership() || !MessagingJobOwnership.IsSamePhysicalJob(proposal.Job, _job.Value))) + || !string.Equals(_durableOwnershipId, _jobId.Value, StringComparison.Ordinal) + || (_job.Value is not null && !MessagingJobOwnership.IsSamePhysicalJob(_durableJob, _job.Value)); + + private bool IsPendingAcknowledgement(HierarchicalKey id) => + _pendingMessages.ContainsKey(id); + + private Items SelectMessages() + { + var now = _jobTimeProvider.GetUtcNow(); + DeliveryCandidate first = default; + List? candidates = null; + var count = 0; + foreach (var envelope in _messages.Values) + { + if (!IsPendingAcknowledgement(envelope.Envelope.MessageId) && IsReadyForAttempt(envelope, now)) + { + var candidate = new DeliveryCandidate(envelope, + _messageStates.TryGetValue(envelope.Envelope.MessageId, out var state) ? CopyState(state) : null); + if (count++ == 0) + { + first = candidate; + } + else + { + (candidates ??= [first]).Add(candidate); + } + if (count == _batchSize) + { + break; + } + } + } + if (count == 0) + { + LogNoMessages(_logger, Count); + return default; + } + LogDeliveringMessages(_logger, count); + return candidates is null ? new(first) : new(candidates.ToArray()); + } + + private static OutboxMessageState CopyState(OutboxMessageState state) => new() + { + AttemptCount = state.AttemptCount, + LastError = state.LastError, + NextAttemptAt = state.NextAttemptAt, + EnqueuedAt = state.EnqueuedAt + }; + + private void ValidateCandidate(DeliveryCandidate candidate) + { + if (IsPendingAcknowledgement(candidate.Envelope.Envelope.MessageId) + || !_messages.TryGetValue(candidate.Envelope.Envelope.MessageId, out var current) + || !EnvelopeEquivalence.AreEquivalent(candidate.Envelope, current)) + { + throw new InvalidOperationException("The admitted outbox delivery no longer refers to a durable pending message."); + } + _messageStates.TryGetValue(current.Envelope.MessageId, out var state); + var expected = candidate.State; + if (state?.AttemptCount != expected?.AttemptCount || state?.NextAttemptAt != expected?.NextAttemptAt + || state?.EnqueuedAt != expected?.EnqueuedAt || state?.LastError != expected?.LastError) + { + throw new InvalidOperationException("The admitted outbox delivery attempt state has changed."); + } + } + + private async Task DeliverAsync(DeliveryCandidate candidate, CancellationToken cancellationToken) + { + if (candidate.State?.EnqueuedAt is { } enqueuedAt + && MessagingTime.IsExpired(_jobTimeProvider.GetUtcNow(), enqueuedAt, _maxRetryAge)) + { + return new(candidate, null, $"The message exceeded the maximum retry age of {_maxRetryAge}.", TimeSpan.Zero, Expired: true); + } + var start = Stopwatch.GetTimestamp(); + try + { + var envelope = candidate.Envelope; + var target = envelope.ReceiverId == _grainContext.GrainId + ? _grainContext.GetGrainExtension() + : _grainFactory.GetGrain(envelope.ReceiverId); + var result = await target.DeliverAsync(envelope.ToInboxMessage(), cancellationToken).ConfigureAwait(true); + return new(candidate, result, null, Stopwatch.GetElapsedTime(start), Expired: false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + if (_logger.IsEnabled(LogLevel.Error)) + { + LogDeliveryError(_logger, exception, candidate.Envelope.Envelope.MessageId, GetSender(candidate.Envelope), + candidate.Envelope.ReceiverId); + } + return new(candidate, null, exception.ToString(), Stopwatch.GetElapsedTime(start), Expired: false); + } + } + + public async Task DeliverPendingMessagesAsync(CancellationToken cancellationToken = default) + { + await _deliveryGate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + ValidateReady(); + var owner = CurrentOwner; + var candidates = SelectMessages(); + Items outcomes = default; + if (candidates.Count == 1) + { + outcomes = new(await DeliverAsync(candidates[0], cancellationToken).ConfigureAwait(true)); + } + else if (candidates.Count > 1) + { + var results = new DeliveryOutcome[candidates.Count]; + for (var index = 0; index < candidates.Count; index++) + { + results[index] = await DeliverAsync(candidates[index], cancellationToken).ConfigureAwait(true); + } + outcomes = new(results); + } + cancellationToken.ThrowIfCancellationRequested(); + ValidateOwner(owner); + if (outcomes.Count > 0) + { + await SubmitAsync(new DeliveryWrite(owner, outcomes)).ConfigureAwait(true); + } + } + finally + { + _deliveryGate.Release(); + } + } + + private async Task AdvancePendingDeliveriesAsync(DurableJob job, long generation, + CancellationToken cancellationToken, CancellationToken attemptCancellationToken, bool clearOwnershipWhenEmpty) + { + await _deliveryGate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + ValidateReady(); + if (!IsCurrentPump(job, generation)) + { + return; + } + if (_pendingDeliveryBatch is { } batch) + { + if (batch.Owner.Generation != generation || !MessagingJobOwnership.IsSamePhysicalJob(batch.Owner.Job, job) + || batch.Cancellation.IsCancellationRequested) + { + CancelPendingDeliveryBatchAsync().Ignore(); + return; + } + if (!batch.Completion.IsCompleted) + { + return; + } + _pendingDeliveryBatch = null; + using (batch) + { + var outcomes = await batch.GetOutcomesAsync().ConfigureAwait(true); + cancellationToken.ThrowIfCancellationRequested(); + ValidateOwner(batch.Owner); + await SubmitAsync(new DeliveryWrite(batch.Owner, outcomes, clearOwnershipWhenEmpty)).ConfigureAwait(true); + } + return; + } + + var candidates = SelectMessages(); + if (candidates.Count == 0) + { + return; + } + var owner = new PumpOwner(_jobId.Value, job, generation); + // This source owns explicit batch cancellation as well as activation/attempt cancellation. + var cancellation = attemptCancellationToken == _shutdown.Token || !attemptCancellationToken.CanBeCanceled + ? CancellationTokenSource.CreateLinkedTokenSource(_shutdown.Token) + : CancellationTokenSource.CreateLinkedTokenSource(attemptCancellationToken, _shutdown.Token); + var pending = new PendingDeliveryBatch(owner, cancellation); + try + { + for (var index = 0; index < candidates.Count; index++) + { + var candidate = candidates[index]; + if (candidate.Envelope.ReceiverId == _grainContext.GrainId) + { + var attempt = DeliverAsync(candidate, cancellationToken); + pending.Add(attempt); + await attempt.ConfigureAwait(true); + ValidateReady(); + if (!IsCurrentPump(job, generation)) + { + await pending.CancelAsync(_logger).ConfigureAwait(true); + return; + } + } + else + { + pending.Add(DeliverAsync(candidate, cancellation.Token)); + } + } + pending.Seal(); + if (pending.Completion.IsCompleted) + { + using (pending) + { + var outcomes = await pending.GetOutcomesAsync().ConfigureAwait(true); + cancellationToken.ThrowIfCancellationRequested(); + ValidateOwner(owner); + await SubmitAsync(new DeliveryWrite(owner, outcomes, clearOwnershipWhenEmpty)).ConfigureAwait(true); + } + } + else + { + _pendingDeliveryBatch = pending; + WakeOnDeliveryCompletionAsync(pending).Ignore(); + } + } + catch + { + await pending.CancelAsync(_logger).ConfigureAwait(true); + throw; + } + } + finally + { + _deliveryGate.Release(); + } + } + + private async Task WakeOnDeliveryCompletionAsync(PendingDeliveryBatch batch) + { + // Completion wakes the pump; the retained batch remains authoritative for success, cancellation, and failure. + await ((Task)batch.Completion).ConfigureAwait( + ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + if (ReferenceEquals(_pendingDeliveryBatch, batch) && IsCurrentPump(batch.Owner.Job!, batch.Owner.Generation)) + { + ScheduleLocalPump(); + } + } + + private PreparedDelivery PrepareDelivery(DeliveryOutcome outcome, ref Dictionary? deadLetters) + { + var status = outcome.Result?.Status; + if (status is DeliveryStatus.Accepted or DeliveryStatus.Duplicate or DeliveryStatus.DeadLettered) + { + return new(outcome, true, null); + } + var state = outcome.Candidate.State is { } existing ? CopyState(existing) : new OutboxMessageState(); + var now = _jobTimeProvider.GetUtcNow(); + if (!outcome.Expired) + { + state.AttemptCount = checked(state.AttemptCount + 1); + } + state.EnqueuedAt ??= now; + state.LastError = outcome.Error ?? status switch + { + DeliveryStatus.Backpressured => "The receiver is backpressured.", + DeliveryStatus.HandlerNotFound => outcome.Result?.Message ?? "The receiver has no registered inbox handler.", + _ => $"Unexpected delivery status '{status}'." + }; + if (outcome.Expired || state.AttemptCount >= _maxDeliveryAttempts + || MessagingTime.IsExpired(now, state.EnqueuedAt.Value, _maxRetryAge)) + { + var letter = new OutboxDeadLetter + { + Message = outcome.Candidate.Envelope, + DeadLetteredAt = now, + Reason = state.LastError, + AttemptCount = state.AttemptCount + }; + deadLetters ??= _deadLetters.ToDictionary(static pair => pair.Key, static pair => pair.Value); + DeadLetterRetention.Compact(deadLetters, now, _deadLetterRetentionPeriod, + _maxRetainedDeadLetters, static entry => entry.DeadLetteredAt, + reservedCapacity: deadLetters.ContainsKey(letter.Message.Envelope.MessageId) ? 0 : 1); + deadLetters[letter.Message.Envelope.MessageId] = letter; + return new(outcome, true, null); + } + var exponent = Math.Min(state.AttemptCount - 1, InboxOptions.MaximumBackoffExponent); + state.NextAttemptAt = MessagingTime.AddClamped(now, TimeSpan.FromTicks(_backpressureRetryDelay.Ticks * (1L << exponent))); + return new(outcome, false, state); + } + + private void RecordDeliveryMetrics(PreparedDelivery delivery) + { + var outcome = delivery.Outcome; + var envelope = outcome.Candidate.Envelope; + var status = outcome.Result?.Status; + var metricStatus = status switch + { + DeliveryStatus.Accepted => "accepted", + DeliveryStatus.Duplicate => "duplicate", + DeliveryStatus.DeadLettered => "deadlettered", + DeliveryStatus.Backpressured => "backpressured", + DeliveryStatus.HandlerNotFound => "handler_not_found", + null => "error", + _ => status.Value.ToString().ToLowerInvariant() + }; + _instruments.OnOutboxMessageDelivered(_grainType, metricStatus); + _instruments.OnOutboxDeliveryDuration(outcome.Duration, _grainType); + switch (status) + { + case DeliveryStatus.Accepted: + case DeliveryStatus.Duplicate: + case DeliveryStatus.DeadLettered: + if (_logger.IsEnabled(LogLevel.Debug)) + { + LogMessageDelivered(_logger, envelope.Envelope.MessageId, GetSender(envelope), envelope.ReceiverId, status.Value); + } + break; + case DeliveryStatus.Backpressured: + LogDeliveryBackpressured(_logger, envelope.Envelope.MessageId, envelope.ReceiverId); + break; + case DeliveryStatus.HandlerNotFound: + if (_logger.IsEnabled(LogLevel.Warning)) + { + LogDeliveryHandlerNotFound(_logger, envelope.Envelope.MessageId, GetSender(envelope), envelope.ReceiverId, outcome.Result?.Message); + } + break; + case { } unexpected: + if (_logger.IsEnabled(LogLevel.Warning)) + { + LogUnexpectedDeliveryStatus(_logger, unexpected, envelope.Envelope.MessageId, GetSender(envelope), envelope.ReceiverId); + } + break; + } + } + + private bool IsReadyForAttempt(OutboxMessage envelope, DateTimeOffset now) => + !_messageStates.TryGetValue(envelope.Envelope.MessageId, out var state) + || state.EnqueuedAt is { } enqueuedAt && MessagingTime.IsExpired(now, enqueuedAt, _maxRetryAge) + || state.NextAttemptAt is null || state.NextAttemptAt <= now; + + private DateTimeOffset? GetNextAttemptAt(OutboxMessage envelope, DateTimeOffset now) + { + if (!_messageStates.TryGetValue(envelope.Envelope.MessageId, out var state)) + { + return null; + } + var retryAt = state.NextAttemptAt ?? now; + var expiresAt = state.EnqueuedAt is { } enqueuedAt ? MessagingTime.AddClamped(enqueuedAt, _maxRetryAge) : retryAt; + return retryAt <= expiresAt ? retryAt : expiresAt; + } + + private async Task CancelPendingDeliveryBatchAsync() + { + if (_pendingDeliveryBatch is { } batch) + { + try + { + await batch.CancelAsync(_logger).ConfigureAwait(true); + } + finally + { + if (ReferenceEquals(_pendingDeliveryBatch, batch)) + { + _pendingDeliveryBatch = null; + } + } + } + } + + public async Task OnStart(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + ValidateReady(); + EnsureMetricsActive(); + if (_messages.Count > 0) + { + LogPumpStartingOnActivation(_logger, _messages.Count); + } + if (_messages.Count > 0 && !HasCommittedOwnership()) + { + QueueEnsureJobScheduled(); + } + ScheduleLocalPump(); + if (_deadLetters.Count > _maxRetainedDeadLetters || _deadLetters.Values.Any(entry => + MessagingTime.IsExpired(_jobTimeProvider.GetUtcNow(), entry.DeadLetteredAt, _deadLetterRetentionPeriod))) + { + await SubmitAsync(new CompactWrite(_stateGeneration)).ConfigureAwait(true); + } + } + + public async Task OnStop(CancellationToken cancellationToken = default) + { + try + { + Stop(); + } + finally + { + try + { + await _deliveryGate.WaitAsync(CancellationToken.None).ConfigureAwait(true); + try + { + await CancelPendingDeliveryBatchAsync().ConfigureAwait(true); + } + finally + { + _deliveryGate.Release(); + } + } + finally + { + try + { + if (_activeOwnershipPreparations != 0) + { + await (_ownershipPreparationsDrained ??= new(TaskCreationOptions.RunContinuationsAsynchronously)).Task.ConfigureAwait(true); + } + } + finally + { + if (_activeWrites != 0) + { + await (_writesDrained ??= new(TaskCreationOptions.RunContinuationsAsynchronously)).Task.ConfigureAwait(true); + } + } + } + } + } + + private void Stop() + { + _stateGeneration++; + _preparedOwnership = null; + _pumpCoordinator.Reset(); + _pumpResults.Clear(JobName); + try + { + _shutdown.Cancel(); + } + catch (AggregateException exception) + { + LogCancellationFailed(_logger, exception); + } + finally + { + DisposePumpTimers(); + CancelPendingDeliveryBatchAsync().Ignore(); + if (Interlocked.Exchange(ref _metricsActive, 0) != 0) + { + _instruments.OnOutboxDepthChanged(-Interlocked.Exchange(ref _reportedDepth, 0)); + } + } + } + + private void DisposePumpTimers() + { + _pumpTimer?.Dispose(); + _localPumpTimer?.Dispose(); + _ensureJobTimer?.Dispose(); + _pumpTimer = null; + _localPumpTimer = null; + _ensureJobTimer = null; + } + + private void EnsureMetricsActive() + { + if (Interlocked.Exchange(ref _metricsActive, 1) == 0) + { + _reportedDepth = Count; + _instruments.OnOutboxDepthChanged(Count); + } + } + + private void ReconcileOutboxDepth() + { + if (Volatile.Read(ref _metricsActive) != 0) + { + var count = Count; + _instruments.OnOutboxDepthChanged(count - Interlocked.Exchange(ref _reportedDepth, count)); + } + } + + private void QueueEnsureJobScheduled() + { + if (Interlocked.Exchange(ref _ensureJobScheduledQueued, 1) != 0) + { + return; + } + try + { + (_ensureJobTimer ??= new(this)).Queue(new(_stateGeneration)); + } + catch + { + Volatile.Write(ref _ensureJobScheduledQueued, 0); + throw; + } + } + + private void ScheduleLocalPump() + { + if (_shutdown.IsCancellationRequested || _failure is not null || Count == 0 || !HasCommittedOwnership()) + { + return; + } + + _localPumpRequested = true; + if (_pumpCoordinator.IsActive) + { + return; + } + + var job = _job.Value!; + if (!_pumpCoordinator.TryAcquire(_jobId.Value!, _shutdown.Token, out var lease)) + { + return; + } + + _localPumpRequested = false; + _activePumpTurns++; + try + { + (_localPumpTimer ??= new(this)).Queue(new(lease, job, _stateGeneration)); + } + catch + { + _activePumpTurns--; + _pumpCoordinator.Release(lease); + throw; + } + } + + internal async Task EnsureJobScheduledAsync(bool replaceExisting, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + StartOwnershipPreparation(); + try + { + ValidateReady(); + if (_messages.Count > 0 && (replaceExisting || !HasCommittedOwnership())) + { + await PrepareOwnershipAsync(replaceExisting, CancellationToken.None).ConfigureAwait(true); + await SubmitAsync(new OwnershipWrite(_stateGeneration)).ConfigureAwait(true); + } + } + finally + { + _gate.Release(); + CompleteOwnershipPreparation(); + } + } + + public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) + { + _failure?.Throw(); + _shutdown.Token.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); + ThrowIfOwnershipStateInvalid(); + if (!MessagingJobOwnership.TryGetOwnershipId(context.Job, out var ownershipId)) + { + return DurableJobRunResult.Completed; + } + + if (IsOwnershipTransitionPending(ownershipId)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + var key = new MessagingPumpExecutionKey( + JobName, + context.Job.Id, + context.RunId, + Volatile.Read(ref _stateGeneration)); + if (!string.Equals(_jobId.Value, ownershipId, StringComparison.Ordinal)) + { + var disposition = MessagingJobOwnership.ResolveMismatch( + _recoveryCompleted, + HasCommittedOwnership(), + MessagingJobOwnership.IsCompleted(_durableCompletedJobId, ownershipId), + Count > 0); + if (disposition == OwnershipMismatchDisposition.ReclaimOrphan) + { + LogOrphanedJobReclaimed(_logger, ownershipId, _grainContext.GrainId); + _instruments.OnOrphanedJobReclaimed(_grainContext.GrainId.Type.ToString(), JobName); + return CompleteObsoleteExecution(key); + } + + if (disposition == OwnershipMismatchDisposition.CompleteStale) + { + return CompleteObsoleteExecution(key); + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!_recoveryCompleted || IsOwnershipTransitionPending(ownershipId)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!MessagingJobOwnership.IsSamePhysicalJob(_job.Value, context.Job)) + { + return CompleteObsoleteExecution(key); + } + + if (_pumpResults.TryTake(key, out var result, out var exception)) + { + if (exception is not null) + { + throw exception; + } + + return result!; + } + + if (!_pumpCoordinator.TryAcquire(ownershipId, cancellationToken, out var lease)) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + if (!_pumpResults.TryStart(key, cancellationToken, out var execution)) + { + _pumpCoordinator.Release(lease); + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + _activePumpTurns++; + try + { + (_pumpTimer ??= new(this)).Queue(new(execution, lease, context.Job, cancellationToken)); + } + catch (Exception registrationException) + { + _activePumpTurns--; + _pumpCoordinator.Release(lease); + _pumpResults.Fail(execution, registrationException); + throw; + } + + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + + private DurableJobRunResult CompleteObsoleteExecution(MessagingPumpExecutionKey key) + { + // Stable ownership retires this run; its retained result will no longer be polled. + _pumpResults.TryTake(key, out _, out _); + return DurableJobRunResult.Completed; + } + + private async Task RunPumpTimerAsync( + MessagingPumpExecution execution, + MessagingPumpLease lease, + DurableJob job, + CancellationToken jobCancellation, + CancellationToken timerCancellation) + { + if (!_pumpCoordinator.IsCurrent(lease) || !IsCurrentPump(job, execution.Key.StateGeneration)) + { + _pumpResults.Discard(execution); + return; + } + + if (!_pumpResults.TryBegin(execution)) + { + return; + } + + DurableJobRunResult? result = null; + Exception? failure = null; + try + { + using var linkedCancellation = MessagingCancellation.Combine( + jobCancellation, timerCancellation, _shutdown.Token, out var combinedToken); + result = await ExecuteJobCoreAsync( + lease.OwnershipId, + job, + execution.Key.StateGeneration, + combinedToken, + jobCancellation); + } + catch (Exception exception) + { + failure = exception; + } + finally + { + if (failure is null) + { + _pumpResults.Complete(execution, result!); + } + else + { + _pumpResults.Fail(execution, failure); + } + } + } + + internal async ValueTask ExecuteJobCoreAsync(string jobId, DurableJob job, long stateGeneration, + CancellationToken cancellationToken, CancellationToken attemptCancellationToken, bool clearOwnershipWhenEmpty = true) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + ValidateReady(); + if (!IsCurrentPump(job, stateGeneration) || IsOwnershipTransitionPending(jobId)) + { + return MessagingJobOwnership.IsCompleted(_durableCompletedJobId, jobId) + ? DurableJobRunResult.Completed : DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + } + finally + { + _gate.Release(); + } + await AdvancePendingDeliveriesAsync(job, stateGeneration, cancellationToken, attemptCancellationToken, clearOwnershipWhenEmpty).ConfigureAwait(true); + if (_pendingDeliveryBatch is not null) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + await _gate.WaitAsync(cancellationToken).ConfigureAwait(true); + try + { + ValidateReady(); + if (!IsCurrentPump(job, stateGeneration) || IsOwnershipTransitionPending(jobId)) + { + return MessagingJobOwnership.IsCompleted(_durableCompletedJobId, jobId) + ? DurableJobRunResult.Completed : DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + if (Count == 0 && _activeOwnershipPreparations == 0) + { + if (!clearOwnershipWhenEmpty) + { + return DurableJobRunResult.Completed; + } + var idleAt = GetIdleRetirementAt(); + if (_jobTimeProvider.GetUtcNow() < idleAt) + { + return DurableJobRunResult.RescheduleAt(idleAt); + } + if (!CanRetireOwner()) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + await SubmitAsync(new ClearOwnerWrite(new(jobId, job, stateGeneration))).ConfigureAwait(true); + if (MessagingJobOwnership.IsCompleted(_durableCompletedJobId, jobId)) + { + return DurableJobRunResult.Completed; + } + } + if (_unacknowledgedMessageCount > 0) + { + return DurableJobRunResult.InProgress(TimeSpan.FromMilliseconds(10)); + } + var now = _jobTimeProvider.GetUtcNow(); + var next = Count == 0 ? now : DateTimeOffset.MaxValue; + foreach (var envelope in _messages.Values) + { + if (GetNextAttemptAt(envelope, now) is not { } at || at <= now) + { + next = now; + break; + } + if (at < next) + { + next = at; + } + } + return DurableJobRunResult.RescheduleAt(next); + } + finally + { + _gate.Release(); + } + } + + [LoggerMessage(Level = LogLevel.Error, Message = "Outbox persistence prerequisite failed; retiring the staged operation.")] + private static partial void LogPrerequisiteFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "Outbox command staging failed.")] + private static partial void LogStagingFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "Requesting deactivation after an outbox staging failure failed.")] + private static partial void LogDeactivationFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "An outbox cancellation callback failed during cleanup.")] + private static partial void LogCancellationFailed(ILogger logger, Exception exception); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "No durable messages to deliver (all {Count} messages are still pending)")] + private static partial void LogNoMessages(ILogger logger, int count); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Delivering {Count} durable messages from outbox")] + private static partial void LogDeliveringMessages(ILogger logger, int count); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Delivered message {MessageId} from {SenderId} to {ReceiverId} (Status: {Status})")] + private static partial void LogMessageDelivered(ILogger logger, HierarchicalKey messageId, GrainId? senderId, GrainId receiverId, DeliveryStatus status); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "No inbox handler for message {MessageId} from {SenderId} to {ReceiverId}: {Message}")] + private static partial void LogDeliveryHandlerNotFound(ILogger logger, HierarchicalKey messageId, GrainId? senderId, GrainId receiverId, string? message); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Backpressured delivering message {MessageId} to {ReceiverId}, will retry later")] + private static partial void LogDeliveryBackpressured(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Unexpected delivery status {Status} for message {MessageId} from {SenderId} to {ReceiverId}")] + private static partial void LogUnexpectedDeliveryStatus(ILogger logger, DeliveryStatus status, HierarchicalKey messageId, GrainId? senderId, GrainId receiverId); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Error delivering message {MessageId} from {SenderId} to {ReceiverId}")] + private static partial void LogDeliveryError(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId? senderId, GrainId receiverId); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Outbox delivery complete: {DeliveredCount} delivered, {BackpressuredCount} backpressured, {FailedCount} failed, {RemainingCount} remaining")] + private static partial void LogDeliveryComplete(ILogger logger, int deliveredCount, int backpressuredCount, int failedCount, int remainingCount); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Grain activated with {Count} pending outbox messages, starting pump")] + private static partial void LogPumpStartingOnActivation(ILogger logger, int count); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Error in outbox pump loop")] + private static partial void LogPumpLoopError(ILogger logger, Exception exception); + + [LoggerMessage( + Level = LogLevel.Information, + Message = "Reclaimed orphaned outbox job ownership {OwnershipId} for grain {GrainId}")] + private static partial void LogOrphanedJobReclaimed(ILogger logger, string ownershipId, GrainId grainId); + + private readonly record struct PumpTurn(MessagingPumpExecution Execution, + MessagingPumpLease Lease, DurableJob Job, CancellationToken JobCancellation); + private readonly record struct LocalPumpTurn(MessagingPumpLease Lease, DurableJob Job, long Generation); + private readonly record struct EnsureJobTurn(long Generation); + + private sealed class PumpTimerState(Outbox owner) : MessagingTurn + { + protected override IGrainTimer RegisterTimer(long registrationGeneration) => owner._timerRegistry.RegisterGrainTimer( + owner._grainContext, (state, token) => state.RunAsync(registrationGeneration, token), this, + new GrainTimerCreationOptions(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan) { Interleave = false, KeepAlive = true }); + + protected override async Task ExecuteAsync(PumpTurn turn, CancellationToken timerCancellation) + { + try + { + await owner.RunPumpTimerAsync(turn.Execution, turn.Lease, turn.Job, turn.JobCancellation, timerCancellation); + } + finally + { + Release(turn); + if (owner._localPumpRequested) owner.ScheduleLocalPump(); + } + } + + protected override void Discard(PumpTurn turn) + { + owner._pumpResults.Discard(turn.Execution); + Release(turn); + } + + private void Release(PumpTurn turn) + { + owner._pumpCoordinator.Release(turn.Lease); + owner._activePumpTurns--; + } + } + + private sealed class LocalPumpTimerState(Outbox owner) : MessagingTurn + { + protected override IGrainTimer RegisterTimer(long registrationGeneration) => owner._timerRegistry.RegisterGrainTimer( + owner._grainContext, (state, token) => state.RunAsync(registrationGeneration, token), this, + new GrainTimerCreationOptions(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan) { Interleave = false, KeepAlive = true }); + + protected override async Task ExecuteAsync(LocalPumpTurn turn, CancellationToken timerCancellation) + { + try + { + if (owner._pumpCoordinator.IsCurrent(turn.Lease) && owner.IsCurrentPump(turn.Job, turn.Generation)) + { + using var cancellation = MessagingCancellation.Combine(timerCancellation, owner._shutdown.Token, out var combinedToken); + _ = await owner.ExecuteJobCoreAsync(turn.Lease.OwnershipId, turn.Job, turn.Generation, + combinedToken, owner._shutdown.Token, clearOwnershipWhenEmpty: false); + } + } + finally + { + Discard(turn); + if (owner._localPumpRequested) owner.ScheduleLocalPump(); + } + } + + protected override void Discard(LocalPumpTurn turn) + { + owner._pumpCoordinator.Release(turn.Lease); + owner._activePumpTurns--; + } + } + + private readonly record struct PumpOwner(string? Id, DurableJob? Job, long Generation); + private sealed class PendingMessage(HierarchicalKey messageId) + { + public HierarchicalKey MessageId { get; } = messageId; + public bool Captured { get; set; } + } + + private sealed record OwnershipProposal(string Id, long Sequence, DurableJob Job, PumpOwner PreviousOwner); + private readonly record struct DeliveryCandidate(OutboxMessage Envelope, OutboxMessageState? State); + private readonly record struct DeliveryOutcome(DeliveryCandidate Candidate, DeliveryResult? Result, string? Error, TimeSpan Duration, bool Expired); + private readonly record struct PreparedDelivery(DeliveryOutcome Outcome, bool Remove, OutboxMessageState? Retry); + + private abstract class OutboxWrite(long generation) + { + public long Generation { get; } = generation; + } + // The scalar form carries no collection; arrays are reserved for true fan-out. + private readonly struct Items + { + private readonly T _single; + private readonly T[]? _many; + public Items(T single) { _single = single; _many = null; Count = 1; } + public Items(T[] many) { _single = default!; _many = many; Count = many.Length; } + public int Count { get; } + public T this[int index] => _many is null ? _single : _many[index]; + } + + private sealed class DeliveryWrite(PumpOwner owner, Items outcomes, bool clearOwnershipWhenEmpty = false) : OutboxWrite(owner.Generation) + { + public PumpOwner Owner { get; } = owner; + public Items Outcomes { get; } = outcomes; + public bool ClearOwnershipWhenEmpty { get; } = clearOwnershipWhenEmpty; + public Items Prepared { get; set; } + } + private sealed class ClearOwnerWrite(PumpOwner owner) : OutboxWrite(owner.Generation) + { + public PumpOwner Owner { get; } = owner; + } + private sealed class OwnershipWrite(long generation) : OutboxWrite(generation); + private sealed class CompactWrite(long generation) : OutboxWrite(generation); + + private sealed class PendingDeliveryBatch(PumpOwner owner, CancellationTokenSource cancellation) : IDisposable + { + private bool _disposed; + private Task? _drain; + private bool _sealed; + public PumpOwner Owner { get; } = owner; + public Task Completion { get; private set; } = Task.CompletedTask; + public CancellationTokenSource Cancellation { get; } = cancellation; + public Task? SingleAttempt { get; private set; } + public List>? Attempts { get; private set; } + + public void Add(Task attempt) + { + if (SingleAttempt is null) + { + SingleAttempt = attempt; + } + else + { + (Attempts ??= [SingleAttempt]).Add(attempt); + } + } + + public void Seal() + { + if (!_sealed) + { + _sealed = true; + Completion = Attempts is { } attempts ? Task.WhenAll(attempts) + : SingleAttempt ?? (Task)Task.CompletedTask; + } + } + + public async ValueTask> GetOutcomesAsync() + { + if (Attempts is null) + { + return new(await SingleAttempt!.ConfigureAwait(true)); + } + return new(await ((Task)Completion).ConfigureAwait(true)); + } + public Task CancelAsync(ILogger logger) + { + if (_drain is { } drain) + { + return drain; + } + if (_disposed) + { + return Task.CompletedTask; + } + // Seal partial fan-out too: an exception while admitting self-delivery must still drain remote attempts. + if (!_sealed) + { + Seal(); + } + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _drain = completion.Task; + try + { + Cancellation.Cancel(); + } + catch (AggregateException exception) + { + LogCancellationFailed(logger, exception); + } + DrainAsync(completion).Ignore(); + return _drain; + } + + private async Task DrainAsync(TaskCompletionSource completion) + { + // Delivery already logs transport failures. Retirement observes every outcome before releasing the token source. + await Completion.ConfigureAwait( + ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + Dispose(); + completion.TrySetResult(); + } + + public void Dispose() + { + if (!_disposed) + { + _disposed = true; + Cancellation.Dispose(); + } + } + } + + private sealed class EnsureJobTimerState(Outbox owner) : MessagingTurn + { + protected override IGrainTimer RegisterTimer(long registrationGeneration) => owner._timerRegistry.RegisterGrainTimer( + owner._grainContext, (state, token) => state.RunAsync(registrationGeneration, token), this, + new GrainTimerCreationOptions(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan) { Interleave = false, KeepAlive = true }); + + protected override async Task ExecuteAsync(EnsureJobTurn turn, CancellationToken cancellationToken) + { + try + { + if (turn.Generation == owner._stateGeneration) + { + await owner.EnsureJobScheduledAsync(false, cancellationToken).ConfigureAwait(true); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested || owner._shutdown.IsCancellationRequested) + { + } + catch (Exception exception) + { + LogPumpLoopError(owner._logger, exception); + if (owner._failure is null && !owner._shutdown.IsCancellationRequested) + { + owner._grainContext.Deactivate(new DeactivationReason( + DeactivationReasonCode.ApplicationError, exception, "Outbox ownership repair failed."), CancellationToken.None); + } + } + finally + { + Discard(turn); + } + } + + protected override void Discard(EnsureJobTurn turn) => Volatile.Write(ref owner._ensureJobScheduledQueued, 0); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs b/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs new file mode 100644 index 00000000000..d007ddd545a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs @@ -0,0 +1,1711 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using NSubstitute; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Orleans.Timers; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class OutboxCodecBoundaryTests +{ + [Fact] + public async Task Send_ReusesBodyAndEncodesStandardDictionaryCommandsDuringStaging() + { + await using var fixture = await CodecFixture.CreateAsync(); + var envelope = fixture.CreateEnvelope(); + Assert.Equal(1, fixture.Probe.Count(nameof(Payload))); + + await fixture.SendAsync(envelope); + await fixture.SendAsync(envelope); + + Assert.Equal(1, fixture.Probe.Count(nameof(OutboxMessage))); + Assert.Equal(1, fixture.Probe.Count("OutboxMessageState")); + Assert.Equal(0, fixture.Probe.Count(nameof(DurableJob))); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + Assert.Equal(1, fixture.Outbox.Count); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, fixture.Probe.Count(nameof(Payload))); + Assert.Equal(new[] { "staging" }, fixture.Probe.Phases(nameof(OutboxMessage))); + Assert.Equal(new[] { "staging" }, fixture.Probe.Phases("OutboxMessageState")); + Assert.Equal(new[] { "capture" }, fixture.Probe.Phases(nameof(DurableJob))); + Assert.Equal(1, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + var restored = Assert.Single(recovered.Messages).Value; + Assert.Equal(4096, Assert.IsType(recovered.ReadApplication(restored).Body).Bytes.Length); + } + + [Fact] + public async Task DeadLetter_EncodesAtJournalApplicationWithoutPayloadPreflight() + { + await using var fixture = await CodecFixture.CreateAsync(delivery: DeliveryResult.HandlerNotFound()); + await fixture.SendAsync(fixture.CreateEnvelope()); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + + Assert.Equal(1, fixture.Probe.Count(nameof(Payload))); + Assert.Equal(1, fixture.Probe.Count(nameof(OutboxMessage))); + Assert.Equal(new[] { "staging" }, fixture.Probe.Phases("OutboxDeadLetter")); + Assert.Equal(1, fixture.Probe.Count("OutboxMessageState")); + Assert.Equal(1, fixture.Probe.Count(nameof(DurableJob))); + Assert.Empty(fixture.Messages); + Assert.Equal(1, fixture.DeadLetterCount); + Assert.Equal(2, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + } + + [Fact] + public async Task RetryState_EncodesOnceWhenApplied() + { + await using var fixture = await CodecFixture.CreateAsync(delivery: DeliveryResult.Backpressured(), maxAttempts: 3); + await fixture.SendAsync(fixture.CreateEnvelope()); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + + Assert.Equal(new[] { "staging", "staging" }, fixture.Probe.Phases("OutboxMessageState")); + Assert.Equal(1, fixture.Probe.Count(nameof(Payload))); + Assert.Equal(1, fixture.Probe.Count(nameof(OutboxMessage))); + Assert.Equal(1, fixture.Probe.Count(nameof(DurableJob))); + Assert.Single(fixture.Messages); + Assert.Equal(0, fixture.DeadLetterCount); + } + + [Fact] + public async Task BodyCodecFailure_IsReportedByApplicationBeforeIntentAdmission() + { + await using var fixture = await CodecFixture.CreateAsync(); + fixture.Probe.FailureType = nameof(Payload); + Assert.Same(fixture.Probe.Failure, Assert.Throws(() => fixture.CreateEnvelope())); + Assert.Equal(0, fixture.Outbox.Count); + Assert.Equal(0, fixture.Probe.Count(nameof(OutboxMessage))); + Assert.Equal(0, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Null(fixture.States.Failure); + } + + [Theory] + [InlineData(nameof(OutboxMessage), "staging", 0)] + [InlineData("OutboxMessageState", "staging", 1)] + [InlineData(nameof(DurableJob), "capture", 1)] + public async Task CodecFailure_ReportsAtStandardEncodingBoundaryAndPreservesReplay(string failedType, string phase, int stagedMessages) + { + await using var fixture = await CodecFixture.CreateAsync(); + var envelope = fixture.CreateEnvelope(); + fixture.Probe.FailureType = failedType; + InvalidOperationException error; + if (phase == "staging") + { + error = await Assert.ThrowsAsync(() => fixture.SendAsync(envelope)); + Assert.Null(fixture.States.Failure); + var deactivation = Assert.Single(fixture.Context.ReceivedCalls(), call => call.GetMethodInfo().Name == "Deactivate"); + Assert.Same(error, Assert.IsType(deactivation.GetArguments()[0]).Exception); + } + else + { + await fixture.SendAsync(envelope); + Assert.Equal(0, fixture.Probe.Count(failedType)); + error = await Assert.ThrowsAsync(() => fixture.WriteAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(error, fixture.States.Failure); + var rejected = await Assert.ThrowsAsync(() => fixture.WriteAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(error, rejected.InnerException); + } + Assert.Same(fixture.Probe.Failure, error); + Assert.Equal(new[] { phase }, fixture.Probe.Phases(failedType)); + Assert.Equal(stagedMessages, fixture.Messages.Count); + Assert.Equal(0, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Same(error, Assert.Throws(() => fixture.Outbox.Send(envelope))); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.Empty(recovered.Messages); + Assert.Null(recovered.Job.Value); + } + + [Fact] + public async Task DeadLetterCodecFailure_PreservesPreviouslyCommittedMessageOnFreshReplay() + { + await using var fixture = await CodecFixture.CreateAsync(delivery: DeliveryResult.HandlerNotFound()); + var envelope = fixture.CreateEnvelope(); + await fixture.SendAsync(envelope); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + fixture.Probe.FailureType = "OutboxDeadLetter"; + + var error = await Assert.ThrowsAsync(() => fixture.DeliverAsync()); + Assert.Same(fixture.Probe.Failure, error); + Assert.Null(fixture.States.Failure); + Assert.Single(fixture.Context.ReceivedCalls(), call => call.GetMethodInfo().Name == "Deactivate"); + Assert.Empty(fixture.Messages); + Assert.Equal(new[] { "staging" }, fixture.Probe.Phases("OutboxDeadLetter")); + Assert.Equal(1, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(recovered.Messages).Key); + Assert.Equal(0, recovered.DeadLetterCount); + } + + [Fact] + public async Task FeatureRepairSchedulingFailure_RequestsDeactivationWithoutJournalMutation() + { + await using var seed = await CodecFixture.CreateAsync(); + var journal = new JournalId($"repair-fault/{Guid.NewGuid():N}"); + { var seedEnvelope = seed.CreateEnvelope(); await seed.SeedOwnerlessJournalAsync(journal, seedEnvelope); } + await using var fixture = await CodecFixture.CreateAsync(seed.Storage, journal); + var failure = new IOException("Admitted scheduling failed."); + fixture.Jobs.ScheduleJobAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromException(failure)); + var writes = fixture.Storage.GetSuccessfulWriteCount(journal); + + await fixture.RunRepairTimerAsync(); + + Assert.Null(fixture.States.Failure); + Assert.Equal(writes, fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Single(fixture.Jobs.ReceivedCalls(), call => call.GetMethodInfo().Name == "ScheduleJobAsync"); + var deactivation = Assert.Single(fixture.Context.ReceivedCalls(), call => call.GetMethodInfo().Name == "Deactivate"); + Assert.Same(failure, Assert.IsType(deactivation.GetArguments()[0]).Exception); + Assert.Null(fixture.Job.Value); + await using var recovered = await CodecFixture.CreateAsync(seed.Storage, journal); + await recovered.RunRepairTimerAsync(); + Assert.NotNull(recovered.Job.Value); + Assert.Single(recovered.Messages); + } + + [Theory] + [InlineData("message", false)] + [InlineData("sender", false)] + [InlineData("receiver", false)] + [InlineData("message", true)] + [InlineData("sender", true)] + [InlineData("receiver", true)] + public async Task DirectSendMalformedStructure_FailsBeforeIntentOrJournalMutation(string field, bool existingIntent) + { + await using var fixture = await CodecFixture.CreateAsync(); + var valid = fixture.CreateEnvelope(); + if (existingIntent) { await fixture.SendAsync(valid); } + var invalid = field switch + { + "message" => valid with { Envelope = default }, + "sender" => fixture.CreateRaw(valid.Envelope.MessageId, valid.Envelope.Payload, default(GrainId)), + "receiver" => valid with { ReceiverId = default }, + _ => throw new ArgumentOutOfRangeException(nameof(field)) + }; + + if (field == "sender") + { + var ownership = Assert.Throws(() => fixture.Outbox.Send(invalid)); + Assert.Contains("does not match the owning grain", ownership.Message, StringComparison.Ordinal); + } + else + { + Assert.ThrowsAny(() => fixture.Outbox.Send(invalid)); + } + + Assert.Equal(existingIntent ? 1 : 0, fixture.Outbox.Count); + Assert.Equal(existingIntent ? 1 : 0, fixture.Messages.Count); + Assert.Empty(fixture.Jobs.ReceivedCalls()); + Assert.Equal(0, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Equal(existingIntent ? 1 : 0, fixture.Probe.Count(nameof(OutboxMessage))); + Assert.Null(fixture.States.Failure); + if (existingIntent) { Assert.Equal(valid, Assert.Single(fixture.Outbox.Messages)); } + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task DirectSendApplicationRoute_PreservesExplicitIdentityNullBodyAndMissingHandlerOutcome(string? route) + { + await using var fixture = await CodecFixture.CreateAsync(delivery: DeliveryResult.HandlerNotFound()); + var message = fixture.CreateRaw(HierarchicalKey.Create("test", "codec-boundary", "nullable-body", "0"), + fixture.EncodeApplication(route!, null), fixture.Context.GrainId, "codec"); + Assert.Null(fixture.ReadApplication(message).Body); + + await fixture.SendAsync(message); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + Assert.Equal(message.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + await fixture.DeliverAsync(); + + Assert.Empty(fixture.Messages); + Assert.Equal(1, fixture.DeadLetterCount); + Assert.Equal(2, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Null(fixture.States.Failure); + } + + [Theory] + [InlineData(0, false)] + [InlineData(1, false)] + [InlineData(2, false)] + [InlineData(3, false)] + [InlineData(4, false)] + [InlineData(5, false)] + [InlineData(6, false)] + [InlineData(0, true)] + [InlineData(1, true)] + [InlineData(2, true)] + [InlineData(3, true)] + [InlineData(4, true)] + [InlineData(5, true)] + [InlineData(6, true)] + public async Task RealFacets_CaptureAckAndReplayAreIndependentOfRegistrationOrder(int rotation, bool snapshot) + { + await using var fixture = await CodecFixture.CreateAsync(stateOrder: rotation, snapshot: snapshot); + Assert.Equal(7, fixture.States.StateCount); + Assert.IsAssignableFrom>( + fixture.States.GetState("__orleans.messaging.outbox")); + var first = fixture.CreateEnvelope(); + await fixture.SendAsync(first); + var storage = fixture.Storage.BlockWrite(fixture.JournalId); + var write = fixture.WriteAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + var later = fixture.CreateEnvelope(); + await fixture.SendAsync(later); + Assert.Equal(2, fixture.Outbox.Count); + Assert.Equal(2, fixture.Messages.Count); + storage.Release(); + await write; + + Assert.True(fixture.Messages.ContainsKey(later.Envelope.MessageId)); + await using (var captured = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId)) + { + Assert.Equal(first.Envelope.MessageId, Assert.Single(captured.Messages).Key); + Assert.False(captured.Messages.ContainsKey(later.Envelope.MessageId)); + } + Assert.True(fixture.Outbox.TryGetMessage(later.Envelope.MessageId, out _)); + var owner = Assert.IsType(fixture.Job.Value); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, fixture.Messages.Count); + Assert.Same(owner, fixture.Job.Value); + Assert.Single(fixture.Jobs.ReceivedCalls(), call => call.GetMethodInfo().Name == nameof(ILocalDurableJobManager.ScheduleJobAsync)); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId, stateOrder: 6 - rotation, snapshot: snapshot); + Assert.Equal(2, recovered.Outbox.Count); + Assert.True(recovered.Outbox.TryGetMessage(first.Envelope.MessageId, out _)); + Assert.True(recovered.Outbox.TryGetMessage(later.Envelope.MessageId, out _)); + Assert.Equal(owner.Id, recovered.Job.Value!.Id); + Assert.Equal(owner.ShardId, recovered.Job.Value.ShardId); + Assert.Empty(recovered.Jobs.ReceivedCalls()); + await recovered.DeleteAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, recovered.Outbox.Count); + Assert.Empty(recovered.Messages); + Assert.Null(recovered.Job.Value); + Assert.ThrowsAny(() => recovered.Outbox.Send(first)); + await using var fresh = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + await fresh.SendAsync(fresh.CreateEnvelope()); + await fresh.WriteAsync(TestContext.Current.CancellationToken); + Assert.Single(fresh.Messages); + Assert.NotNull(fresh.Job.Value); + } + + [Theory] + [InlineData(0, false)] + [InlineData(6, false)] + [InlineData(0, true)] + [InlineData(6, true)] + public async Task OpaqueSend_StagingDuringStorageAwaitBelongsToNextCohort(int rotation, bool alreadyPending) + { + await using var fixture = await CodecFixture.CreateAsync(stateOrder: rotation); + var prepared = fixture.CreateEnvelope(); + if (alreadyPending) + { + var existingEnvelope = fixture.CreateEnvelope(); + await fixture.SendAsync(existingEnvelope); + } + var storage = fixture.Storage.BlockWrite(fixture.JournalId); + var write = fixture.WriteAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + Assert.Equal(alreadyPending ? 1 : 0, fixture.Messages.Count); + fixture.Outbox.Send(prepared); + Assert.Equal(alreadyPending ? 2 : 1, fixture.Outbox.Count); + Assert.Equal(0, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + storage.Release(); + await write; + Assert.Equal(alreadyPending ? 2 : 1, fixture.Messages.Count); + await using (var captured = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId)) + { + Assert.Equal(alreadyPending ? 1 : 0, captured.Messages.Count); + } + await fixture.WriteAsync(TestContext.Current.CancellationToken); + Assert.Equal(alreadyPending ? 2 : 1, fixture.Messages.Count); + Assert.Equal(2, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Single(fixture.Jobs.ReceivedCalls(), call => call.GetMethodInfo().Name == nameof(ILocalDurableJobManager.ScheduleJobAsync)); + var job = Assert.IsType(fixture.Job.Value); + Assert.Equal("opaque-shard", job.ShardId); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.Equal(fixture.Outbox.Count, recovered.Outbox.Count); + Assert.Equal(job.Id, recovered.Job.Value!.Id); + } + + [Fact] + public async Task EmptyJournal_UsesCanonicalOwnerBoundCollectionsAndOneSequenceState() + { + await using var fixture = await CodecFixture.CreateAsync(); + Assert.Equal(7, fixture.States.StateCount); + foreach (var name in BootstrapOutboxServices.StateNames.Take(6)) + { + var state = fixture.States.GetState(name); + Assert.Same(typeof(IStateMachine).Assembly, state.GetType().Assembly); + Assert.Same(state, fixture.ResolveStandardState(name)); + } + var sequence = fixture.States.GetState(BootstrapOutboxServices.StateNames[6]); + Assert.Same(typeof(IOutbox).Assembly, sequence.GetType().Assembly); + Assert.IsAssignableFrom>(sequence); + Assert.False(fixture.Manager is IDurableStateManager); + Assert.Equal(0, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + var batch = fixture.CreateEnvelope(); + Assert.Equal(0, fixture.Outbox.Count); + fixture.Outbox.Send(batch); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + Assert.Single(fixture.Messages); + Assert.Equal(1, ((IDurableValue)sequence).Value); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ActualManager_ExplicitInitialRecoveryRetryPrecedesOutboxStartup(bool hasCommittedOwner) + { + await using var seed = await CodecFixture.CreateAsync(); + var envelope = seed.CreateEnvelope(); + var journal = hasCommittedOwner ? seed.JournalId : new JournalId($"outbox-retry/{Guid.NewGuid():N}"); + if (hasCommittedOwner) + { + await seed.SendAsync(envelope); + await seed.WriteAsync(TestContext.Current.CancellationToken); + } + else + { + await seed.SeedOwnerlessJournalAsync(journal, envelope); + } + var reads = seed.Storage.GetReadCount(journal); + var writes = seed.Storage.GetSuccessfulWriteCount(journal); + await using var fixture = await CodecFixture.CreateAsync(seed.Storage, journal, initialize: false); + fixture.Probe.ReadFailureType = nameof(OutboxMessage); + var failure = await Assert.ThrowsAsync(() => fixture.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(fixture.Probe.Failure, failure.GetBaseException()); + Assert.Contains("Failed to recover journaling state", failure.Message, StringComparison.Ordinal); + Assert.Equal(1, fixture.Probe.ReadFailures); + Assert.Equal(reads + 1, fixture.Storage.GetReadCount(journal)); + Assert.Empty(fixture.Jobs.ReceivedCalls()); + Assert.DoesNotContain(fixture.TimerRegistry.ReceivedCalls(), call => call.GetMethodInfo().Name == "RegisterGrainTimer"); + Assert.Equal(writes, fixture.Storage.GetSuccessfulWriteCount(journal)); + + fixture.Probe.ReadFailureType = null; + await fixture.Manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(reads + 2, fixture.Storage.GetReadCount(journal)); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + Assert.Equal(1, fixture.Outbox.Count); + Assert.Empty(fixture.Jobs.ReceivedCalls()); + Assert.DoesNotContain(fixture.TimerRegistry.ReceivedCalls(), call => call.GetMethodInfo().Name == "RegisterGrainTimer"); + await ((ILifecycleObserver)fixture.Outbox).OnStart(TestContext.Current.CancellationToken); + if (hasCommittedOwner) + { + Assert.Equal(seed.Job.Value!.Id, fixture.Job.Value!.Id); + Assert.Equal(seed.Job.Value.ShardId, fixture.Job.Value.ShardId); + Assert.Empty(fixture.Jobs.ReceivedCalls()); + } + else + { + Assert.Null(fixture.Job.Value); + await fixture.RunRepairTimerAsync(); + Assert.Single(fixture.Jobs.ReceivedCalls(), call => call.GetMethodInfo().Name == nameof(ILocalDurableJobManager.ScheduleJobAsync)); + Assert.NotNull(fixture.Job.Value); + } + Assert.Equal(writes + (hasCommittedOwner ? 0 : 1), fixture.Storage.GetSuccessfulWriteCount(journal)); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + Assert.Equal(0, fixture.Outbox.Count); + Assert.Null(fixture.States.Failure); + } + + [Fact] + public async Task CoalescedOwnerRepair_CompletesAfterPriorAckWithoutAnotherStorageWrite() + { + await using var seed = await CodecFixture.CreateAsync(); + var journal = new JournalId($"standard-cohort/{Guid.NewGuid():N}"); + var recoveredCommand = HierarchicalKey.Create("test", "owner-repair", "command", "recovered"); + var arrivingCommand = HierarchicalKey.Create("test", "owner-repair", "command", "arriving"); + { var seedEnvelope = seed.CreateEnvelope(recoveredCommand); await seed.SeedOwnerlessJournalAsync(journal, seedEnvelope); } + await using var fixture = await CodecFixture.CreateAsync(seed.Storage, journal); + await fixture.SendAsync(fixture.CreateEnvelope(arrivingCommand)); + Assert.NotEqual(recoveredCommand, arrivingCommand); + Assert.Equal(2, fixture.Outbox.Count); + Assert.True(fixture.Outbox.TryGetMessage(recoveredCommand, out _)); + Assert.True(fixture.Outbox.TryGetMessage(arrivingCommand, out _)); + var writes = fixture.Storage.GetSuccessfulWriteCount(journal); + using var storage = fixture.Storage.BlockWrite(journal); + var first = fixture.WriteAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + var repair = fixture.RunRepairTimerAsync(); + Assert.False(repair.IsCompleted); + Assert.Equal(writes, fixture.Storage.GetSuccessfulWriteCount(journal)); + storage.Release(); + await Task.WhenAll(first, repair).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(writes + 1, fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(2, fixture.Messages.Count); + Assert.NotNull(fixture.Job.Value); + Assert.Single(fixture.Jobs.ReceivedCalls(), call => call.GetMethodInfo().Name == nameof(ILocalDurableJobManager.ScheduleJobAsync)); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + } + + [Fact] + public async Task NoOpWrites_LeaveAllFacetsReadyForNextIntent() + { + await using var fixture = await CodecFixture.CreateAsync(); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + var writes = fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + Assert.Equal(writes, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Equal(0, fixture.Outbox.Count); + Assert.Empty(fixture.Jobs.ReceivedCalls()); + await fixture.SendAsync(fixture.CreateEnvelope()); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + Assert.Single(fixture.Messages); + Assert.NotNull(fixture.Job.Value); + Assert.Null(fixture.States.Failure); + } + + [Theory] + [InlineData(0)] + [InlineData(6)] + public async Task CanceledCallerDuringStorageAck_PreservesOwnedCaptureAndAck(int rotation) + { + await using var fixture = await CodecFixture.CreateAsync(stateOrder: rotation); + await fixture.SendAsync(fixture.CreateEnvelope()); + var storage = fixture.Storage.BlockWrite(fixture.JournalId); + using var cancellation = new CancellationTokenSource(); + var write = fixture.WriteAsync(cancellation.Token).AsTask(); + await storage.WaitUntilEnteredAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => write); + Assert.Equal(0, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + storage.Release(); + await fixture.WriteAsync(TestContext.Current.CancellationToken); + Assert.Single(fixture.Messages); + Assert.Equal(1, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Null(fixture.States.Failure); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + Assert.Equal(0, fixture.Outbox.Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public Task EquivalentDuplicate_PreservesOriginalPayloadThroughCaptureAck(bool duringCapture) => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var command = HierarchicalKey.Create("test", "payload", "command", "0"); + var original = fixture.CreateEnvelope(command, "payload.v1"); + var expected = original.Envelope.Payload.ToArray(); + fixture.Outbox.Send(original); + var duplicate = fixture.CreateEnvelope(HierarchicalKey.Parse(command.ToString(), provider: null), "payload.v1"); + Assert.NotSame(original.Envelope.Payload, duplicate.Envelope.Payload); + using var storage = fixture.Storage.BlockWrite(fixture.JournalId); + Task? write = null; + if (duringCapture) + { + write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + } + fixture.Outbox.Send(duplicate); + Assert.Equal(1, fixture.Outbox.Count); + Assert.Equal(1, fixture.Probe.Count(nameof(OutboxMessage))); + Assert.True(fixture.Outbox.TryGetMessage(command, out var stored)); + Assert.Same(original.Envelope.Payload, stored.Envelope.Payload); + Assert.Equal(expected, stored.Envelope.Payload.ToArray()); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Receiver.ReceivedCalls()); + storage.Release(); + if (write is not null) await write; + else await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(fixture.Jobs.ReceivedCalls()); + Assert.Equal(1, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + Assert.Equal(command, ((InboxMessage)Assert.Single(fixture.Receiver.ReceivedCalls()).GetArguments()[0]!).Envelope.MessageId); + Assert.Equal(expected, original.Envelope.Payload.ToArray()); + Assert.Equal(expected, duplicate.Envelope.Payload.ToArray()); + }); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public Task CustomHeadersAndSender_DuplicatePreservesFirstIntentThroughReplay(bool committed) => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var id = HierarchicalKey.Create("test", "headers", "original"); + var receiver = GrainId.Create("receiver", "headers"); + var first = new OutboxMessage + { + ReceiverId = receiver, + Envelope = new Envelope + { + MessageId = id, Payload = [1, 2, 3], Subject = "headers.v1", + Headers = new Dictionary(StringComparer.Ordinal) + { + ["sender"] = fixture.Serializer().SerializeToArray(fixture.Context.GrainId), + ["trace"] = [4, 5], + ["label"] = fixture.Serializer().SerializeToArray("original") + } + } + }; + fixture.Outbox.Send(first); + if (committed) await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var writes = fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId); + fixture.Outbox.Send(new OutboxMessage + { + ReceiverId = receiver, + Envelope = new Envelope + { + MessageId = id, Payload = [1, 2, 3], Subject = "headers.v1", + Headers = new Dictionary(StringComparer.Ordinal) + { + ["trace"] = [9], + ["label"] = fixture.Serializer().SerializeToArray("changed") + } + } + }); + Assert.Equal(1, fixture.Outbox.Count); + Assert.True(fixture.Outbox.TryGetMessage(id, out var retained)); + Assert.Same(first.Envelope.Payload, retained.Envelope.Payload); + Assert.Same(first.Envelope.Headers, retained.Envelope.Headers); + Assert.Equal(writes, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Single(fixture.Jobs.ReceivedCalls()); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.True(recovered.Outbox.TryGetMessage(id, out var restored)); + Assert.Equal(receiver, restored.ReceiverId); + var restoredHeaders = Assert.IsType>(restored.Envelope.Headers); + Assert.Equal(new byte[] { 4, 5 }, restoredHeaders["trace"]); + Assert.Equal("original", recovered.Serializer().Deserialize(restoredHeaders["label"])); + Assert.Equal(fixture.Context.GrainId, recovered.Serializer().Deserialize(restoredHeaders["sender"])); + Assert.Equal(first.Envelope.Payload, restored.Envelope.Payload); + Assert.NotSame(first.Envelope.Headers, restoredHeaders); + await recovered.DeliverAsync(); + Assert.Empty(recovered.Messages); + var incoming = (InboxMessage)Assert.Single(recovered.Receiver.ReceivedCalls()).GetArguments()[0]!; + Assert.Same(restored.Envelope.Payload, incoming.Envelope.Payload); + Assert.Same(restored.Envelope.Headers, incoming.Envelope.Headers); + }); + + [Fact] + public Task MissingReceiver_RejectsBeforeAnyIntentOrJournalMutation() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var message = fixture.CreateRaw(HierarchicalKey.Create("test", "receiver", "missing"), ReadOnlyMemory.Empty) + with + { ReceiverId = default }; + Assert.ThrowsAny(() => fixture.Outbox.Send(message)); + Assert.Empty(fixture.Messages); + Assert.Equal(0, fixture.Outbox.Count); + Assert.Equal(0, fixture.Probe.Count(nameof(OutboxMessage))); + Assert.Empty(fixture.Jobs.ReceivedCalls()); + Assert.Equal(0, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Null(fixture.States.Failure); + }); + + [Fact] + public Task Recovery_MismatchedOutgoingKeyFailsBeforeOwnerRepair() => OnOwnerAsync(async () => + { + await using var seed = await CodecFixture.CreateAsync(); + var message = seed.CreateEnvelope(); + seed.Messages.Add(HierarchicalKey.Create("test", "different", "key"), message); + await seed.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await using var recovered = await CodecFixture.CreateAsync(seed.Storage, seed.JournalId, initialize: false); + var error = await Assert.ThrowsAsync(() => + recovered.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Contains("recovered outbox key", error.Message, StringComparison.Ordinal); + Assert.Empty(recovered.Jobs.ReceivedCalls()); + Assert.Empty(recovered.Receiver.ReceivedCalls()); + Assert.False((bool)recovered.Outbox.GetType().GetField("_recoveryCompleted", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(recovered.Outbox)!); + }); + + [Fact] + public Task MinimalOutgoing_DeliversZeroCopyIncomingWithoutReceiverEncoding() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var outgoing = fixture.CreateRaw(HierarchicalKey.Create("test", "incoming", "minimal"), ReadOnlyMemory.Empty); + Assert.Null(outgoing.Envelope.Headers); + Assert.Null(outgoing.Envelope.Subject); + Assert.Empty(outgoing.Envelope.Payload); + fixture.Outbox.Send(outgoing); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Receiver.ReceivedCalls()); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + var incoming = (InboxMessage)Assert.Single(fixture.Receiver.ReceivedCalls()).GetArguments()[0]!; + Assert.Equal(outgoing.Envelope.MessageId, incoming.Envelope.MessageId); + Assert.Same(outgoing.Envelope.Payload, incoming.Envelope.Payload); + Assert.Same(outgoing.Envelope.Headers, incoming.Envelope.Headers); + Assert.Null(typeof(InboxMessage).GetProperty(nameof(OutboxMessage.ReceiverId))); + var otherDestination = outgoing with { ReceiverId = GrainId.Create("receiver", "different") }; + Assert.Equal(fixture.Serializer().SerializeToArray(incoming), + fixture.Serializer().SerializeToArray(otherDestination.ToInboxMessage())); + Assert.Empty(fixture.Messages); + Assert.Single(fixture.Jobs.ReceivedCalls()); + }); + + [Fact] + public Task ApplicationIdentity_EquivalentReconstructionPreservesOriginalRetainedIntent() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var command = HierarchicalKey.Create("tenant/acme", "orders", "1042", "reserve-stock"); + var reconstructed = HierarchicalKey.Parse(command.ToString(), provider: null); + Assert.Equal(command, reconstructed); + var envelope = fixture.CreateEnvelope(command, "inventory.reserve.v1"); + var repeat = fixture.CreateEnvelope(reconstructed, "inventory.reserve.v1"); + fixture.Outbox.Send(envelope); + fixture.Outbox.Send(repeat); + Assert.Equal(1, fixture.Outbox.Count); + Assert.Equal(1, fixture.Probe.Count(nameof(OutboxMessage))); + Assert.True(fixture.Outbox.TryGetMessage(reconstructed, out var stored)); + Assert.Same(envelope.Envelope.Payload, stored.Envelope.Payload); + Assert.NotSame(repeat.Envelope.Payload, stored.Envelope.Payload); + Assert.Equal("inventory.reserve.v1", OutboxTestMessages.Subject(stored.Envelope)); + Assert.Equal(envelope.Envelope.Payload.ToArray(), stored.Envelope.Payload.ToArray()); + using var storage = fixture.Storage.BlockWrite(fixture.JournalId); + var writing = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + fixture.Outbox.Send(repeat); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Receiver.ReceivedCalls()); + storage.Release(); + await writing; + await using (var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId)) + { + Assert.True(recovered.Outbox.TryGetMessage(reconstructed, out var restored)); + Assert.Equal(command, restored.Envelope.MessageId); + Assert.Equal("inventory.reserve.v1", OutboxTestMessages.Subject(restored.Envelope)); + Assert.Equal(envelope.Envelope.Payload.ToArray(), restored.Envelope.Payload.ToArray()); + Assert.Equal(fixture.Job.Value!.Id, recovered.Job.Value!.Id); + Assert.Equal(fixture.Job.Value.ShardId, recovered.Job.Value.ShardId); + } + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + var delivery = Assert.Single(fixture.Receiver.ReceivedCalls()); + Assert.Equal(command, ((InboxMessage)delivery.GetArguments()[0]!).Envelope.MessageId); + Assert.Single(fixture.Jobs.ReceivedCalls()); + Assert.Equal(envelope.Envelope.Payload.ToArray(), repeat.Envelope.Payload.ToArray()); + }); + + [Theory] + [InlineData("destination", false)] + [InlineData("subject", false)] + [InlineData("body", false)] + [InlineData("destination", true)] + [InlineData("subject", true)] + [InlineData("body", true)] + public Task ApplicationIdentity_ConflictingIntentPreservesOwnerPayloadAndAttempts(string conflict, bool committed) => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var command = HierarchicalKey.Create("tenant", "orders", "1042", "reserve-stock"); + var original = fixture.CreateEnvelope(command, "inventory.reserve.v1"); + var page = original.Envelope.Payload; + + fixture.Outbox.Send(original); + if (committed) await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var owner = fixture.Job.Value; + var writes = fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId); + var commandEncodes = fixture.Probe.Count(nameof(OutboxMessage)); + + var conflicting = fixture.CreateRaw(command, + conflict == "body" ? fixture.EncodeApplication("changed-body", 99) : original.Envelope.Payload, + fixture.Context.GrainId, conflict == "subject" ? "inventory.reserve.V1" : "inventory.reserve.v1", + conflict == "destination" ? GrainId.Create("receiver", "other") : original.ReceiverId); + var error = Assert.Throws(() => fixture.Outbox.Send(conflicting)); + Assert.Contains(command.ToString(), error.Message, StringComparison.Ordinal); + + Assert.Equal(1, fixture.Outbox.Count); + Assert.Equal(commandEncodes, fixture.Probe.Count(nameof(OutboxMessage))); + Assert.Equal(writes, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Same(owner, fixture.Job.Value); + Assert.True(fixture.Outbox.TryGetMessage(HierarchicalKey.Parse(command.ToString(), provider: null), out var stored)); + Assert.Equal(page, stored.Envelope.Payload); + Assert.Equal(original.ReceiverId, stored.ReceiverId); + Assert.Equal(OutboxTestMessages.Subject(original.Envelope), OutboxTestMessages.Subject(stored.Envelope)); + Assert.Equal(original.Envelope.Payload.ToArray(), stored.Envelope.Payload.ToArray()); + Assert.Null(fixture.States.Failure); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + + Assert.Single(fixture.Jobs.ReceivedCalls()); + Assert.Single(fixture.Receiver.ReceivedCalls()); + }); + + [Fact] + public Task ApplicationIdentity_ParentAndRecipientChildrenRemainIndependentAcrossReplay() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var command = HierarchicalKey.Create("tenant", "campaign", "1042", "notify"); + var firstId = command.CreateChildKey("recipient/a"); + var secondId = command.CreateChildKey("recipient/b"); + Assert.NotEqual(command, firstId); + Assert.NotEqual(firstId, secondId); + Assert.Equal(firstId, HierarchicalKey.Parse(command.ToString(), provider: null).CreateChildKey("recipient/a")); + var parent = fixture.CreateEnvelope(command, "campaign.notify.v1"); + var first = fixture.CreateEnvelope(firstId, "campaign.notify.v1"); + var second = fixture.CreateEnvelope(secondId, "campaign.notify.v1") with { ReceiverId = GrainId.Create("receiver", "other") }; + var retry = fixture.CreateEnvelope(HierarchicalKey.Parse(firstId.ToString(), provider: null), "campaign.notify.v1"); + fixture.Outbox.Send(parent); + fixture.Outbox.Send(first); + fixture.Outbox.Send(second); + fixture.Outbox.Send(retry); + Assert.Equal(3, fixture.Outbox.Count); + Assert.Equal(3, fixture.Probe.Count(nameof(OutboxMessage))); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await using (var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId)) + { + Assert.Equal(3, recovered.Outbox.Count); + foreach (var id in new[] { command, firstId, secondId }) Assert.True(recovered.Outbox.TryGetMessage(id, out _)); + Assert.Equal(fixture.Job.Value!.Id, recovered.Job.Value!.Id); + Assert.Equal(fixture.Job.Value.ShardId, recovered.Job.Value.ShardId); + } + using var storage = fixture.Storage.BlockWrite(fixture.JournalId); + var delivering = fixture.DeliverAsync(); + await storage.WaitUntilEnteredAsync(); + Assert.Empty(fixture.Messages); + Assert.Equal(3, fixture.Receiver.ReceivedCalls().Count()); + storage.Release(); + await delivering; + await using var drained = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.Empty(drained.Messages); + Assert.Single(fixture.Jobs.ReceivedCalls()); + }); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public Task ApplicationIdentity_AttemptsDeadLettersAndRemovalRetainExactKeyAcrossReplay(bool deadLetter) => OnOwnerAsync(async () => + { + var outcome = deadLetter ? DeliveryResult.HandlerNotFound() : DeliveryResult.Backpressured(); + await using var fixture = await CodecFixture.CreateAsync(delivery: outcome, maxAttempts: deadLetter ? 1 : 3); + var command = HierarchicalKey.Create("tenant", "orders", "1042", "reserve-stock"); + var message = fixture.CreateEnvelope(command, "inventory.reserve.v1"); + fixture.Outbox.Send(message); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + var recoveredKey = HierarchicalKey.Parse(command.ToString(), provider: null); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId, delivery: outcome, maxAttempts: deadLetter ? 1 : 3); + Assert.Empty(recovered.Jobs.ReceivedCalls()); + if (deadLetter) + { + Assert.Equal(command, Assert.Single(recovered.DeadLetterKeys)); + Assert.Empty(recovered.AttemptKeys); + Assert.False(recovered.Outbox.TryGetMessage(recoveredKey, out _)); + var dead = recovered.ReadDeadLetterEnvelope(recoveredKey); + Assert.Equal(command, dead.Envelope.MessageId); + Assert.Equal(OutboxTestMessages.Subject(message.Envelope), OutboxTestMessages.Subject(dead.Envelope)); + Assert.Equal(message.Envelope.Payload.ToArray(), dead.Envelope.Payload.ToArray()); + Assert.True(recovered.RemoveDeadLetter(recoveredKey)); + await recovered.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await using var removed = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.Empty(removed.DeadLetterKeys); + Assert.Empty(removed.Messages); + } + else + { + Assert.Equal(command, Assert.Single(recovered.AttemptKeys)); + Assert.Empty(recovered.DeadLetterKeys); + Assert.True(recovered.Outbox.TryGetMessage(recoveredKey, out var pending)); + Assert.Equal(command, pending.Envelope.MessageId); + Assert.Equal(OutboxTestMessages.Subject(message.Envelope), OutboxTestMessages.Subject(pending.Envelope)); + Assert.Equal(message.Envelope.Payload.ToArray(), pending.Envelope.Payload.ToArray()); + Assert.Equal(fixture.Job.Value!.Id, recovered.Job.Value!.Id); + Assert.Equal(fixture.Job.Value.ShardId, recovered.Job.Value.ShardId); + } + var delivered = Assert.Single(fixture.Receiver.ReceivedCalls()); + Assert.Equal(command, ((InboxMessage)delivered.GetArguments()[0]!).Envelope.MessageId); + Assert.Null(fixture.States.Failure); + }); + + [Fact] + public Task ApplicationIdentity_RetryAfterOwnerRetirementPreservesTheOriginalCommandKey() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var command = HierarchicalKey.Create("tenant", "orders", "1042", "reserve-stock"); + var first = fixture.CreateEnvelope(command, "inventory.reserve.v1"); + fixture.Outbox.Send(first); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + var retiredJob = fixture.Job.Value!; + await fixture.PumpAsync(retiredJob); + Assert.Null(fixture.Job.Value); + var retry = fixture.CreateEnvelope(HierarchicalKey.Parse(command.ToString(), provider: null), "inventory.reserve.v1"); + fixture.Outbox.Send(retry); + Assert.Equal(command, Assert.Single(fixture.Messages).Key); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var replacement = fixture.Job.Value!; + Assert.NotEqual(retiredJob.Id, replacement.Id); + Assert.Equal(2, fixture.Jobs.ReceivedCalls().Count()); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.True(recovered.Outbox.TryGetMessage(command, out var stored)); + Assert.Equal(OutboxTestMessages.Subject(retry.Envelope), OutboxTestMessages.Subject(stored.Envelope)); + Assert.Equal(retry.Envelope.Payload.ToArray(), stored.Envelope.Payload.ToArray()); + Assert.Equal(replacement.Id, recovered.Job.Value!.Id); + Assert.Equal(replacement.ShardId, recovered.Job.Value.ShardId); + await recovered.DeliverAsync(); + Assert.Empty(recovered.Messages); + }); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public Task SynchronousSend_RespectsOpaqueIdentityAndOriginalEnvelope(bool duplicateFirst) => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var envelope = fixture.CreateEnvelope(); + if (!duplicateFirst) fixture.Outbox.Send(envelope); + fixture.Outbox.Send(envelope); + var conflict = fixture.CreateRaw(envelope.Envelope.MessageId, fixture.EncodeApplication("conflicting-route", null), fixture.Context.GrainId, "codec"); + Assert.Throws(() => fixture.Outbox.Send(conflict)); + fixture.Outbox.Send(envelope); + fixture.Outbox.Send(envelope); + Assert.Equal(envelope, Assert.Single(fixture.Messages).Value); + Assert.Equal(1, fixture.Outbox.Count); + Assert.Equal(1, fixture.Probe.Count(nameof(OutboxMessage))); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + // There is no prepared-handle reservation after ACK. Sending again is a new intent; + // receiver-side durable deduplication owns suppression of repeated business effects. + fixture.Outbox.Send(envelope); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + Assert.Equal(4, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Single(fixture.Jobs.ReceivedCalls()); + }); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public Task SynchronousSend_SchedulesBeforeCaptureAndIncludesArrivalsDuringScheduling(bool snapshot) => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(snapshot: snapshot, businessState: true); + using var scheduling = fixture.BlockSchedule(); + using var storage = fixture.Storage.BlockWrite(fixture.JournalId); + var first = fixture.CreateEnvelope(); + fixture.Business!.Value = 42; + fixture.Outbox.Send(first); + var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await scheduling.WaitAsync(); + Assert.Equal(0, fixture.Probe.Captures); + Assert.Equal(0, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.True((await fixture.CallbackAsync(scheduling.Job!)).IsInProgress); + var second = fixture.CreateEnvelope(); + fixture.Outbox.Send(second); + fixture.Outbox.Send(first); + Assert.Equal(2, fixture.Outbox.Count); + scheduling.Release(); + await storage.WaitUntilEnteredAsync(); + Assert.Same(scheduling.Job, fixture.Job.Value); + Assert.True((await fixture.CallbackAsync(scheduling.Job!)).IsInProgress); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Receiver.ReceivedCalls()); + storage.Release(); + await write; + await using (var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId, businessState: true)) + { + Assert.Equal(42, recovered.Business!.Value); + Assert.Equal(new[] { first.Envelope.MessageId, second.Envelope.MessageId }.OrderBy(static key => key.ToString(), StringComparer.Ordinal), recovered.Messages.Keys.OrderBy(static key => key.ToString(), StringComparer.Ordinal)); + Assert.Equal(scheduling.Job!.Id, recovered.Job.Value!.Id); + Assert.Equal(scheduling.Job.ShardId, recovered.Job.Value.ShardId); + } + await fixture.DeliverAsync(); + Assert.Equal(2, fixture.Receiver.ReceivedCalls().Count()); + Assert.Empty(fixture.Messages); + Assert.Single(fixture.Jobs.ReceivedCalls()); + }); + + [Fact] + public Task SynchronousSend_ArrivalDuringLaterHookHasWakeupBeforeCapture() => OnOwnerAsync(async () => + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hook = new TestJournaledStateHook + { + BeforeOperationAsync = async (_, _) => + { + entered.TrySetResult(); + await release.Task; + } + }; + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + Assert.Same(fixture.Outbox, fixture.Manager.Hooks[0]); + var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } + } + finally { release.TrySetResult(); } + await write; + Assert.Single(fixture.Jobs.ReceivedCalls()); + Assert.NotNull(fixture.Job.Value); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.Single(recovered.Messages); + Assert.Equal(fixture.Job.Value.Id, recovered.Job.Value!.Id); + }); + + [Fact] + public Task SynchronousSend_EmptyFinalHookAndCaptureShareOneTurn() => OnOwnerAsync(async () => + { + Task? staged = null; + OutboxMessage envelope = default; + await using var fixture = await CodecFixture.CreateAsync(configure: f => + { + envelope = f.CreateEnvelope(); + var hook = Assert.IsAssignableFrom(Assert.Single(f.Manager.Hooks)); + f.Manager.Hooks[0] = new CaptureBoundaryHook(hook, () => + { + staged ??= Task.Factory.StartNew(() => f.Outbox.Send(envelope), TestContext.Current.CancellationToken, + TaskCreationOptions.None, TaskScheduler.Current); + }); + f.Manager.Hooks.Add(new TestJournaledStateHook { BeforeOperationAsync = async (_, _) => await Task.Yield() }); + }); + using var storage = fixture.Storage.BlockWrite(fixture.JournalId); + var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + await Assert.IsAssignableFrom(staged); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + Assert.Empty(fixture.Jobs.ReceivedCalls()); + storage.Release(); + await write; + await using (var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId)) + { + Assert.Empty(recovered.Messages); + Assert.Null(recovered.Job.Value); + } + await fixture.DeliverAsync(); + Assert.Empty(fixture.Receiver.ReceivedCalls()); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(fixture.Jobs.ReceivedCalls()); + await fixture.DeliverAsync(); + Assert.Equal(envelope.Envelope.MessageId, ((InboxMessage)Assert.Single(fixture.Receiver.ReceivedCalls()).GetArguments()[0]!).Envelope.MessageId); + Assert.Empty(fixture.Messages); + }, preventInlining: true); + + [Fact] + public Task SynchronousSend_StorageAwaitArrivalRemainsPendingUntilItsOwnAck() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + var first = fixture.CreateEnvelope(); + fixture.Outbox.Send(first); + using var storage = fixture.Storage.BlockWrite(fixture.JournalId); + var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + var later = fixture.CreateEnvelope(); + fixture.Outbox.Send(later); + storage.Release(); + await write; + using var removal = fixture.Storage.BlockWrite(fixture.JournalId); + var deliver = fixture.DeliverAsync(); + await removal.WaitUntilEnteredAsync(); + var delivery = Assert.Single(fixture.Receiver.ReceivedCalls()); + Assert.Equal(first.Envelope.MessageId, ((InboxMessage)delivery.GetArguments()[0]!).Envelope.MessageId); + Assert.Equal(later.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + await using (var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId)) + { + Assert.Equal(first.Envelope.MessageId, Assert.Single(recovered.Messages).Key); + } + removal.Release(); + await deliver; + await fixture.DeliverAsync(); + Assert.Equal(2, fixture.Receiver.ReceivedCalls().Count()); + Assert.Empty(fixture.Messages); + Assert.Single(fixture.Jobs.ReceivedCalls()); + }); + + [Fact] + public Task SynchronousSend_SchedulingFailureRetainsBusinessAndIntentForExplicitRetry() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(businessState: true); + var failure = new IOException("Scheduling failed before journal capture."); + using var scheduling = fixture.BlockSchedule(); + fixture.Business!.Value = 42; + var envelope = fixture.CreateEnvelope(); + fixture.Outbox.Send(envelope); + var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await scheduling.WaitAsync(); + scheduling.Fail(failure); + var error = await Assert.ThrowsAsync(() => write); + Assert.Same(failure, error.InnerException); + Assert.Equal(0, fixture.Probe.Captures); + Assert.Equal(0, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Equal(42, fixture.Business.Value); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + Assert.Null(fixture.Job.Value); + await using (var uncommitted = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId, businessState: true)) + { + Assert.Equal(0, uncommitted.Business!.Value); + Assert.Empty(uncommitted.Messages); + } + using var retry = fixture.BlockSchedule(); + var retried = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await retry.WaitAsync(); + Assert.Equal(0, fixture.Probe.Captures); + retry.Release(); + await retried; + Assert.Equal(2, fixture.Jobs.ReceivedCalls().Count()); + Assert.Equal(1, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId, businessState: true); + Assert.Equal(42, recovered.Business!.Value); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(recovered.Messages).Key); + Assert.Equal(retry.Job!.Id, recovered.Job.Value!.Id); + }); + + [Fact] + public Task SynchronousSend_CanceledCallerLeavesSchedulingAndAckOwnedByJournal() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + using var scheduling = fixture.BlockSchedule(); + using var cancellation = new CancellationTokenSource(); + { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } + var write = fixture.Manager.WriteStateAsync(cancellation.Token).AsTask(); + await scheduling.WaitAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => write); + Assert.False(scheduling.Token.IsCancellationRequested); + Assert.Equal(0, fixture.Probe.Captures); + scheduling.Release(); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(fixture.Jobs.ReceivedCalls()); + Assert.Equal(1, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + }); + + [Fact] + public Task SynchronousSend_OverlappingWritesShareOnePhysicalWakeup() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + using var scheduling = fixture.BlockSchedule(); + var envelope = fixture.CreateEnvelope(); + fixture.Outbox.Send(envelope); + var prepared = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await scheduling.WaitAsync(); + fixture.Outbox.Send(envelope); + var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + scheduling.Release(); + await prepared; + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(fixture.Jobs.ReceivedCalls()); + Assert.Equal(1, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + Assert.Same(scheduling.Job, fixture.Job.Value); + }); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public Task SynchronousSend_OwnerRetirementArrivalRetainsPreCaptureOwnerOrPreparesPostCaptureOwner(bool beforeCapture) => OnOwnerAsync(async () => + { + var hook = new TestJournaledStateHook(); + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + var old = fixture.Job.Value!; + var next = fixture.CreateEnvelope(); + if (beforeCapture) + { + hook.BeforeOperation = (_, _) => fixture.Outbox.Send(next); + } + using var storage = fixture.Storage.BlockWrite(fixture.JournalId); + var retirement = fixture.PumpAsync(old).AsTask(); + await storage.WaitUntilEnteredAsync(); + if (!beforeCapture) fixture.Outbox.Send(next); + Assert.Empty(fixture.Receiver.ReceivedCalls().Skip(1)); + storage.Release(); + await retirement.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + hook.BeforeOperation = null; + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(next.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + var replacement = fixture.Job.Value!; + if (beforeCapture) + { + Assert.Same(old, replacement); + Assert.Single(fixture.Jobs.ReceivedCalls()); + } + else + { + Assert.NotEqual(old.Id, replacement.Id); + Assert.Equal(2, fixture.Jobs.ReceivedCalls().Count()); + } + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.Equal(next.Envelope.MessageId, Assert.Single(recovered.Messages).Key); + Assert.Equal(replacement.Id, recovered.Job.Value!.Id); + Assert.Equal(replacement.ShardId, recovered.Job.Value.ShardId); + await recovered.DeliverAsync(); + Assert.Empty(recovered.Messages); + }); + + [Fact] + public Task SynchronousSend_PostCommitFailurePreservesAcknowledgedOwnedDelivery() => OnOwnerAsync(async () => + { + var hook = new TestJournaledStateHook(); + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var failure = new IOException("After hook failed after acknowledged delivery removal."); + hook.AfterOperation = (_, _) => throw failure; + var error = await Assert.ThrowsAsync(() => fixture.DeliverAsync()); + Assert.Same(failure, error.InnerException); + Assert.Empty(fixture.Messages); + Assert.Equal(2, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + hook.AfterOperation = null; + { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + Assert.Equal(2, fixture.Receiver.ReceivedCalls().Count()); + }); + + [Fact] + public Task SynchronousSend_OwnedPreCommitFailureRetiresStagedOperationForFreshReplay() => OnOwnerAsync(async () => + { + var hook = new TestJournaledStateHook(); + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + var envelope = fixture.CreateEnvelope(); + fixture.Outbox.Send(envelope); + await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var failure = new IOException("Blocked prerequisite on an owned delivery write."); + hook.BeforeOperation = (_, _) => throw failure; + var error = await Assert.ThrowsAsync(() => fixture.DeliverAsync()); + Assert.Same(failure, error.InnerException); + Assert.Empty(fixture.Messages); + Assert.Equal(1, fixture.Storage.GetSuccessfulWriteCount(fixture.JournalId)); + Assert.Same(error, Assert.Throws(() => { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); })); + var deactivate = Assert.Single(fixture.Context.ReceivedCalls(), call => call.GetMethodInfo().Name == "Deactivate"); + Assert.Same(error, Assert.IsType(deactivate.GetArguments()[0]).Exception); + await using var recovered = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(recovered.Messages).Key); + await recovered.DeliverAsync(); + Assert.Empty(recovered.Messages); + }); + + [Fact] + public Task SynchronousSend_StopBeforeQueuedHookPreventsNewScheduling() => OnOwnerAsync(async () => + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hook = new TestJournaledStateHook + { + BeforeOperationAsync = async (_, _) => + { + entered.TrySetResult(); + await release.Task; + } + }; + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } + var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await ((ILifecycleObserver)fixture.Outbox).OnStop(CancellationToken.None); + } + finally { release.TrySetResult(); } + var error = await Assert.ThrowsAsync(() => write); + Assert.IsAssignableFrom(error.InnerException); + Assert.Empty(fixture.Jobs.ReceivedCalls()); + Assert.Equal(0, fixture.Probe.Captures); + await fixture.Manager.DeleteStateAsync(TestContext.Current.CancellationToken); + Assert.Empty(fixture.Messages); + }); + + [Fact] + public Task SynchronousSend_StopDrainsSchedulingBeforeTerminalDeletion() => OnOwnerAsync(async () => + { + await using var fixture = await CodecFixture.CreateAsync(); + using var scheduling = fixture.BlockSchedule(); + { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } + var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await scheduling.WaitAsync(); + var stop = ((ILifecycleObserver)fixture.Outbox).OnStop(CancellationToken.None); + Assert.True(scheduling.Token.IsCancellationRequested); + Assert.False(stop.IsCompleted); + scheduling.Release(); + var error = await Assert.ThrowsAsync(() => write); + Assert.IsAssignableFrom(error.InnerException); + await stop; + Assert.Equal(0, fixture.Probe.Captures); + await fixture.Manager.DeleteStateAsync(TestContext.Current.CancellationToken); + Assert.Empty(fixture.Messages); + Assert.ThrowsAny(() => { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); }); + await using var fresh = await CodecFixture.CreateAsync(fixture.Storage, fixture.JournalId); + { var newEnvelope = fresh.CreateEnvelope(); fresh.Outbox.Send(newEnvelope); } + await fresh.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(fresh.Messages); + }); + + private static async Task OnOwnerAsync(Func action, bool preventInlining = false) + { + var scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, maxConcurrencyLevel: 1); + try + { + await Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.None, + preventInlining ? new NonInliningScheduler(scheduler.ExclusiveScheduler) : scheduler.ExclusiveScheduler) + .Unwrap().WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + finally + { + scheduler.Complete(); + await scheduler.Completion; + } + } + + private sealed class CaptureBoundaryHook(IJournaledStateHook inner, Action afterPrerequisite) : IJournaledStateHook + { + public async ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + await inner.BeforeOperationAsync(operation, cancellationToken); + afterPrerequisite(); + } + + public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => + inner.AfterOperationAsync(operation, cancellationToken); + } + + private sealed class NonInliningScheduler(TaskScheduler scheduler) : TaskScheduler + { + protected override void QueueTask(Task task) => Task.Factory.StartNew(() => TryExecuteTask(task), + CancellationToken.None, TaskCreationOptions.None, scheduler); + protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) => false; + protected override IEnumerable GetScheduledTasks() => []; + } + + private sealed class ScheduleBarrier(ILocalDurableJobManager jobs) : IDisposable + { + private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _continue = new(TaskCreationOptions.RunContinuationsAsynchronously); + public DurableJob? Job { get; private set; } + public CancellationToken Token { get; private set; } + public void Arm() => jobs.ScheduleJobAsync(Arg.Any(), Arg.Any()) + .Returns(call => ScheduleAsync(call.ArgAt(0), call.ArgAt(1))); + private async Task ScheduleAsync(ScheduleJobRequest request, CancellationToken token) + { + Token = token; + Job = new DurableJob + { + Id = Guid.NewGuid().ToString("N"), + ShardId = "opaque-shard", + Name = request.JobName, + DueTime = request.DueTime, + TargetGrainId = request.Target, + Metadata = request.Metadata + }; + _entered.TrySetResult(); + await _continue.Task; + return Job; + } + public Task WaitAsync() => _entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + public void Release() => _continue.TrySetResult(); + public void Fail(Exception failure) => _continue.TrySetException(failure); + public void Dispose() => Release(); + } + + [GenerateSerializer] + public sealed record Payload([property: Id(0)] byte[] Bytes); + + public sealed class CodecProbe + { + private readonly ConcurrentQueue<(string Type, string Phase)> _calls = new(); + public required SerializerSessionPool InnerSessions { get; init; } + public string Phase { get; set; } = "before-admission"; + public int Captures { get; set; } + public string? FailureType { get; set; } + public string? ReadFailureType { get; set; } + public int ReadFailures { get; private set; } + public void Reading(Type type) + { + if (type.Name == ReadFailureType) + { + ReadFailures++; + throw Failure; + } + } + public InvalidOperationException Failure { get; } = new("Injected journal codec failure."); + public void Writing(Type type) + { + _calls.Enqueue((type.Name, Phase)); + if (type.Name == FailureType) + { + throw Failure; + } + } + public int Count(string type) => _calls.Count(call => call.Type == type); + public string[] Phases(string type) => _calls.Where(call => call.Type == type).Select(static call => call.Phase).ToArray(); + } + + public sealed class CountingCodec(CodecProbe probe) : IFieldCodec + { + private readonly IFieldCodec _inner = probe.InnerSessions.CodecProvider.GetCodec(); + public void WriteField(ref Writer writer, uint fieldIdDelta, Type? expectedType, [AllowNull] T value) + where TBufferWriter : IBufferWriter + { + probe.Writing(typeof(T)); + _inner.WriteField(ref writer, fieldIdDelta, expectedType, value); + } + [return: MaybeNull] + public T ReadValue(ref Reader reader, Field field) + { + probe.Reading(typeof(T)); + return _inner.ReadValue(ref reader, field); + } + } + + private sealed class CodecFixture : IAsyncDisposable + { + private readonly ServiceProvider _innerServices; + private readonly ServiceProvider _services; + private readonly AsyncServiceScope _scope; + private readonly Func _deliver; + public CodecProbe Probe { get; } + public JournalId JournalId { get; } + public ControlledJournalStorageProvider Storage { get; } + public IJournaledStateManager Manager { get; } + public IOutbox Outbox { get; } + public IDurableValue? Business { get; } + public IInboxExtension Receiver { get; } + public IDurableDictionary Messages { get; } + public IDurableValue Job { get; } + public StateTrackingManager States { get; } + public IGrainContext Context { get; } + public ITimerRegistry TimerRegistry { get; } + public ILocalDurableJobManager Jobs { get; } + public int DeadLetterCount + { + get + { + var entries = States.GetState("__orleans.messaging.outbox-dead-letters"); + return (int)entries.GetType().GetProperty("Count")!.GetValue(entries)!; + } + } + + public IEnumerable AttemptKeys => (IEnumerable)States + .GetState("__orleans.messaging.outbox-message-state") + .GetType().GetProperty("Keys")!.GetValue(States.GetState("__orleans.messaging.outbox-message-state"))!; + public IEnumerable DeadLetterKeys => (IEnumerable)DeadLetterState.GetType() + .GetProperty("Keys")!.GetValue(DeadLetterState)!; + private IStateMachine DeadLetterState => States.GetState("__orleans.messaging.outbox-dead-letters"); + public OutboxMessage ReadDeadLetterEnvelope(HierarchicalKey messageId) + { + var value = DeadLetterState.GetType().GetProperty("Item")!.GetValue(DeadLetterState, [messageId])!; + return (OutboxMessage)value.GetType().GetProperty("Message")!.GetValue(value)!; + } + public bool RemoveDeadLetter(HierarchicalKey messageId) => + (bool)DeadLetterState.GetType().GetMethod("Remove", [typeof(HierarchicalKey)])!.Invoke(DeadLetterState, [messageId])!; + + private CodecFixture(ControlledJournalStorageProvider? storage, JournalId? journalId, DeliveryResult delivery, int maxAttempts, int stateOrder, bool snapshot, bool businessState) + { + JournalId = journalId ?? new JournalId($"codec-boundary/{Guid.NewGuid():N}"); + Storage = storage ?? new ControlledJournalStorageProvider(); + Storage.Configure(Options.Create(new JournaledStateManagerOptions { JournalFormatKey = "orleans-binary" })); + _innerServices = new ServiceCollection().AddSerializer().BuildServiceProvider(); + Probe = new CodecProbe { InnerSessions = _innerServices.GetRequiredService() }; + var services = new ServiceCollection(); + services.AddSerializer(builder => builder.Configure(options => + { + options.AddFieldCodec(typeof(CountingCodec)); + options.AddFieldCodec(typeof(CountingCodec)); + options.AddFieldCodec(typeof(CountingCodec)); + options.AddFieldCodec(typeof(CountingCodec<>).MakeGenericType(Implementation("OutboxMessageState"))); + options.AddFieldCodec(typeof(CountingCodec<>).MakeGenericType(Implementation("OutboxDeadLetter"))); + })); + services.AddSingleton(Probe); + services.AddLogging(); + services.AddKeyedSingleton(JournalingTimeProviderNames.Journaling, TimeProvider.System); + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, TimeProvider.System); + services.Configure(options => options.JournalFormatKey = "orleans-binary"); + var silo = Substitute.For(); + silo.Services.Returns(services); + silo.AddJournaling(); + services.AddSingleton(snapshot ? new SnapshotStorageProvider(Storage) : Storage); + services.AddScoped(sp => new StateTrackingManager(sp.GetRequiredService().CreateStandalone(JournalId), Probe)); + services.AddScoped(sp => sp.GetRequiredService()); + var context = Context = Substitute.For(); + context.GrainId.Returns(GrainId.Create("sender", "codec-boundary")); + context.GrainInstance.Returns(new object()); + context.ObservableLifecycle.Returns(Substitute.For()); + services.AddSingleton(context); + TimerRegistry = Substitute.For(); + services.AddSingleton(TimerRegistry); + services.AddSingleton(Substitute.For()); + var jobs = Jobs = Substitute.For(); + jobs.ScheduleJobAsync(Arg.Any(), Arg.Any()).Returns(call => + { + var request = call.ArgAt(0); + return Task.FromResult(new DurableJob + { + Id = Guid.NewGuid().ToString("N"), + ShardId = "opaque-shard", + Name = request.JobName, + DueTime = request.DueTime, + TargetGrainId = request.Target, + Metadata = request.Metadata + }); + }); + services.AddSingleton(jobs); + var inbox = Receiver = Substitute.For(); + inbox.DeliverAsync(Arg.Any(), Arg.Any()).Returns(ValueTask.FromResult(delivery)); + var grains = Substitute.For(); + grains.GetGrain(Arg.Any()).Returns(inbox); + services.AddSingleton(grains); + services.AddSingleton(Implementation("MessagingInstruments"), Implementation("MessagingInstruments") + .GetMethod("CreateForDirectConstruction", BindingFlags.Static | BindingFlags.NonPublic)!.Invoke(null, null)!); + services.AddScoped(Implementation("MessagingPumpResults"), _ => Activator.CreateInstance(Implementation("MessagingPumpResults"), nonPublic: true)!); + services.Configure(options => + { + options.MaxDeliveryAttempts = maxAttempts; + // These capture/retirement boundaries deliberately exercise the zero-grace policy. + options.OutboxIdleRetirementGracePeriod = TimeSpan.Zero; + }); + _services = services.BuildServiceProvider(); + _scope = _services.CreateAsyncScope(); + Manager = _scope.ServiceProvider.GetRequiredService(); + var dependencies = _scope.ServiceProvider; + Outbox = (IOutbox)ActivatorUtilities.CreateInstance(dependencies, Implementation("Outbox"), + dependencies.GetRequiredKeyedService>("orleans-binary")); + States = _scope.ServiceProvider.GetRequiredService(); + if (businessState) Business = dependencies.GetRequiredKeyedService>("business"); + States.RegisterStates(stateOrder); + Messages = States.GetState>("__orleans.messaging.outbox"); + Job = States.GetState>("__orleans.messaging.outbox-job-handle"); + _deliver = Outbox.GetType().GetMethod("DeliverPendingMessagesAsync")!.CreateDelegate>(Outbox); + } + + public static async Task CreateAsync(ControlledJournalStorageProvider? storage = null, JournalId? journalId = null, + DeliveryResult? delivery = null, int maxAttempts = 1, int stateOrder = 0, bool snapshot = false, bool initialize = true, bool businessState = false, Action? configure = null) + { + var result = new CodecFixture(storage, journalId, delivery ?? DeliveryResult.Accepted(), maxAttempts, stateOrder, snapshot, businessState); + configure?.Invoke(result); + if (initialize) + { + await result.Manager.InitializeAsync(TestContext.Current.CancellationToken); + await ((ILifecycleObserver)result.Outbox).OnStart(TestContext.Current.CancellationToken); + } + return result; + } + private int _messageSequence; + private HierarchicalKey NextMessageId() => HierarchicalKey.Create( + "test", "codec-boundary", "command", (++_messageSequence).ToString(CultureInfo.InvariantCulture)); + public OutboxMessage CreateEnvelope() => CreateEnvelope(NextMessageId()); + public OutboxMessage CreateEnvelope(HierarchicalKey messageId, string subject = "codec") => + CreateRaw(messageId, EncodeApplication(subject, new Payload(new byte[4096])), Context.GrainId, subject); + public OutboxMessage CreateNullBodyEnvelope() => + CreateRaw(NextMessageId(), EncodeApplication("codec", null), Context.GrainId, "codec"); + public OutboxMessage CreateRaw(HierarchicalKey messageId, ReadOnlyMemory payload, + GrainId? sender = null, string? subject = null, GrainId? receiver = null) => OutboxTestMessages.Create( + messageId, receiver ?? GrainId.Create("receiver", "codec-boundary"), payload, sender, subject, + _scope.ServiceProvider.GetRequiredService>()); + public Serializer Serializer() => _scope.ServiceProvider.GetRequiredService>(); + public byte[] EncodeApplication(string route, object? body) => + TestApplicationProtocol.Encode(_scope.ServiceProvider.GetRequiredService(), new TestApplicationMessage(route, body)); + public TestApplicationMessage ReadApplication(OutboxMessage message) => + TestApplicationProtocol.Read(_scope.ServiceProvider.GetRequiredService(), message); + + public object ResolveStandardState(string name) + { + Type contract = name switch + { + "__orleans.messaging.outbox" => typeof(IDurableDictionary), + "__orleans.messaging.outbox-message-state" => typeof(IDurableDictionary<,>).MakeGenericType(typeof(HierarchicalKey), Implementation("OutboxMessageState")), + "__orleans.messaging.outbox-dead-letters" => typeof(IDurableDictionary<,>).MakeGenericType(typeof(HierarchicalKey), Implementation("OutboxDeadLetter")), + "__orleans.messaging.outbox-job-id" or "__orleans.messaging.outbox-completed-job-id" => typeof(IDurableValue), + "__orleans.messaging.outbox-job-handle" => typeof(IDurableValue), + _ => throw new ArgumentOutOfRangeException(nameof(name)) + }; + return _scope.ServiceProvider.GetRequiredKeyedService(contract, name); + } + + public ScheduleBarrier BlockSchedule() + { + var result = new ScheduleBarrier(Jobs); + result.Arm(); + return result; + } + + public ValueTask CallbackAsync(DurableJob job) + { + var context = Substitute.For(); + context.Job.Returns(job); + context.RunId.Returns("early-run"); + context.DequeueCount.Returns(1); + return ((IDurableJobFeatureHandler)Outbox).ExecuteJobAsync(context, TestContext.Current.CancellationToken); + } + + public ValueTask PumpAsync(DurableJob job) + { + var generation = Outbox.GetType().GetField("_stateGeneration", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(Outbox); + return (ValueTask)Outbox.GetType().GetMethod("ExecuteJobCoreAsync", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(Outbox, [job.Metadata!["orleans.messaging.ownership-id"], job, generation, TestContext.Current.CancellationToken, TestContext.Current.CancellationToken, true])!; + } + + public async Task SeedOwnerlessJournalAsync(JournalId journal, OutboxMessage envelope) + { + await using var manager = _services.GetRequiredService().CreateStandalone(journal); + var services = new ServiceCollection(); + services.AddSerializer(); + services.AddLogging(); + services.AddKeyedSingleton(JournalingTimeProviderNames.Journaling, TimeProvider.System); + services.Configure(options => options.JournalFormatKey = "orleans-binary"); + var silo = Substitute.For(); + silo.Services.Returns(services); + silo.AddJournaling(); + services.AddSingleton(manager); + await using var dependencies = services.BuildServiceProvider(); + var messages = dependencies.GetRequiredKeyedService>("__orleans.messaging.outbox"); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + messages.Add(envelope.Envelope.MessageId, envelope); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + public async Task RunRepairTimerAsync() + { + var call = Assert.Single(TimerRegistry.ReceivedCalls(), value => value.GetMethodInfo().Name == "RegisterGrainTimer"); + var arguments = call.GetArguments(); + var callback = (Delegate)arguments[1]!; + await ((Task)callback.DynamicInvoke(arguments[2], TestContext.Current.CancellationToken)!) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } + + public Task SendAsync(OutboxMessage envelope) + { + Probe.Phase = "staging"; + Outbox.Send(envelope); + return Task.CompletedTask; + } + + public async ValueTask WriteAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var operation = WriteOwnedAsync(); + await operation.WaitAsync(cancellationToken); + + async Task WriteOwnedAsync() + { + try + { + await Manager.WriteStateAsync(CancellationToken.None); + } + catch (Exception exception) when (exception is not JournaledStatePreCommitException and not JournaledStatePostCommitException) + { + Outbox.GetType().GetMethod("FailPersistence", BindingFlags.Instance | BindingFlags.NonPublic)! + .CreateDelegate>(Outbox)(exception); + throw; + } + } + } + + public async ValueTask DeleteAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await DeleteOwnedAsync().WaitAsync(cancellationToken); + + async Task DeleteOwnedAsync() + { + try + { + await ((ILifecycleObserver)Outbox).OnStop(CancellationToken.None); + await Manager.DeleteStateAsync(CancellationToken.None); + } + finally + { + await DisposeAsync(); + } + } + } + + public Task DeliverAsync() + { + Probe.Phase = "staging"; + return _deliver(TestContext.Current.CancellationToken); + } + public async ValueTask DisposeAsync() + { + await ((ILifecycleObserver)Outbox).OnStop(CancellationToken.None); + await _scope.DisposeAsync(); + await _services.DisposeAsync(); + await _innerServices.DisposeAsync(); + } + private static Type Implementation(string name) => ReceiverTestServices.GetImplementationType(name); + } + + private sealed class StateTrackingManager(IJournaledStateManager inner, CodecProbe probe) : IJournaledStateManager + { + private readonly Dictionary _states = new(StringComparer.Ordinal); + public IList Hooks => inner.Hooks; + public Exception? Failure { get; private set; } + public int StateCount => _states.Count; + public void RegisterStateMachine(string name, IStateMachine state) => _states.Add(name, state); + public void RegisterStates(int rotation) + { + var entries = _states.ToArray(); + foreach (var entry in entries.Skip(rotation).Concat(entries.Take(rotation))) + { + inner.RegisterStateMachine(entry.Key, new TrackedState(entry.Value, probe)); + } + } + public T GetState(string name) => (T)_states[name]; + public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? state) => _states.TryGetValue(name, out state); + public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); + public async ValueTask WriteStateAsync(CancellationToken cancellationToken) + { + try + { + await inner.WriteStateAsync(cancellationToken); + } + catch (Exception exception) + { + Failure ??= exception; + throw; + } + } + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => inner.DeleteStateAsync(cancellationToken); + public ValueTask DisposeAsync() => inner.DisposeAsync(); + + private sealed class TrackedState(IStateMachine state, CodecProbe probe) : IStateMachine + { + public void WritePendingEntries(JournalStreamWriter writer) { probe.Phase = "capture"; probe.Captures++; state.WritePendingEntries(writer); } + public void WriteSnapshot(JournalStreamWriter writer) { probe.Phase = "capture"; probe.Captures++; state.WriteSnapshot(writer); } + public void OnWriteCompleted() => state.OnWriteCompleted(); + public void Reset(JournalStreamWriter writer) => state.Reset(writer); + public void OnRecoveryCompleted() => state.OnRecoveryCompleted(); + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => state.ReplayEntry(entry, context); + } + } + + private sealed class SnapshotStorageProvider(IJournalStorageProvider inner) : IJournalStorageProvider + { + public IJournalStorage CreateStorage(JournalId journalId) => new SnapshotStorage(inner.CreateStorage(journalId)); + private sealed class SnapshotStorage(IJournalStorage inner) : IJournalStorage + { + public bool IsCompactionRequested => true; + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) => inner.ReadAsync(consumer, cancellationToken); + public ValueTask CreateIfNotExistsAsync(IReadOnlyDictionary? metadata = null, CancellationToken cancellationToken = default) => inner.CreateIfNotExistsAsync(metadata, cancellationToken); + public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) => inner.GetMetadataAsync(cancellationToken); + public ValueTask UpdateMetadataAsync(IReadOnlyDictionary? set = null, IEnumerable? remove = null, string? expectedETag = null, CancellationToken cancellationToken = default) => inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) => inner.ReplaceAsync(value, cancellationToken); + public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) => inner.AppendAsync(value, cancellationToken); + public ValueTask DeleteAsync(CancellationToken cancellationToken) => inner.DeleteAsync(cancellationToken); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs b/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs new file mode 100644 index 00000000000..152b708c7d7 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs @@ -0,0 +1,3285 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Globalization; +using System.Reflection; +using System.Runtime.ExceptionServices; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Orleans.Serialization; +using Orleans.Timers; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class OutboxDeliveryBatchTests +{ + [Theory] + [InlineData(DeliveryStatus.Accepted)] + [InlineData(DeliveryStatus.HandlerNotFound)] + [InlineData(DeliveryStatus.Backpressured)] + public async Task RawPayloads_DeliveryRetryDeadLetterRemovalRecoveryAndShutdownPreserveBytes(DeliveryStatus status) + { + using var fixture = new OutboxFixture(deliver: _ => ValueTask.FromResult(new DeliveryResult { Status = status }), + maxDeliveryAttempts: status == DeliveryStatus.HandlerNotFound ? 1 : 3); + var observer = Assert.Single(fixture.Messages).Value; + var expected = observer.Envelope.Payload.ToArray(); + await fixture.DeliverAsync(); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Equal(status == DeliveryStatus.Backpressured ? 1 : 0, fixture.Messages.Count); + if (status == DeliveryStatus.HandlerNotFound) + { + Assert.True(fixture.DeadLetters.Remove(fixture.MessageId)); + Assert.Equal(0, fixture.DeadLetters.Count); + } + if (status == DeliveryStatus.Backpressured) + { + using var recovered = fixture.Recreate(); + var restored = Assert.Single(recovered.Messages).Value; + Assert.Equal(expected, restored.Envelope.Payload.ToArray()); + Assert.NotSame(observer.Envelope.Payload, restored.Envelope.Payload); + } + await fixture.StopAsync(); + fixture.Dispose(); + Assert.Equal(expected, observer.Envelope.Payload.ToArray()); + } + + [Fact] + public async Task ReusableLocalTurn_TwoAcknowledgedBurstsReuseExactTimerAndState() + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + await fixture.CommitAsync(); + await fixture.RunRegisteredTimerAtAsync(0); + await fixture.SendAsync(fixture.CreateEnvelope(fixture.NextMessageId())); + await fixture.CommitAsync(); + Assert.Same(fixture.ArmState(0), fixture.ArmState(1)); + Assert.Equal(["LocalPumpTimerState"], fixture.TimerRegistrationNames); + Assert.Equal(["LocalPumpTimerState", "LocalPumpTimerState"], fixture.ArmedTimerNames); + await fixture.RunRegisteredTimerAtAsync(1); + Assert.Equal(2, fixture.DeliveryCount); + Assert.Empty(fixture.Messages); + Assert.Equal(0, fixture.LocalField("_activePumpTurns")); + } + + [Fact] + public async Task ReusableActiveTurn_OriginalAckRetiresOnlyOriginalPayloadWhileNewArmWaits() + { + using var cancellation = new CancellationTokenSource(); + using var fixture = new OutboxFixture(_ => ValueTask.FromResult(DeliveryResult.Backpressured()), durableJobId: "active:1", jobTimeProvider: new FakeTimeProvider()); + var captured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.Manager.AfterCapture = () => { captured.TrySetResult(); return release.Task; }; + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "old", cancellation.Token)).IsInProgress); + var active = fixture.RunRegisteredTimerAtAsync(0); + await captured.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, fixture.LocalField("_activePumpTurns")); + cancellation.Cancel(); + Assert.True((await fixture.ExecuteJobAsync(job, "new", TestContext.Current.CancellationToken)).IsInProgress); + var newest = fixture.PumpEntries.Keys.Cast().Single(key => GetPumpRunId(key) == "new"); + Assert.Equal(2, fixture.LocalField("_activePumpTurns")); + release.SetResult(); + await active; + Assert.True(fixture.PumpEntries.Contains(newest)); + Assert.Equal(1, fixture.LocalField("_activePumpTurns")); + Assert.Equal(1, fixture.Manager.WriteCompletedCount); + fixture.Manager.AfterCapture = null; + await fixture.RunRegisteredTimerAtAsync(1); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, + (await fixture.ExecuteJobAsync(job, "new", TestContext.Current.CancellationToken)).Status); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Equal(0, fixture.LocalField("_activePumpTurns")); + Assert.Equal(["PumpTimerState"], fixture.TimerRegistrationNames); + Assert.Same(fixture.ArmState(0), fixture.ArmState(1)); + } + + [Fact] + public async Task ReusableQueuedTurn_StopRetiresQueuedPayloadBeforeSavedCallbackCanRun() + { + using var fixture = new OutboxFixture(durableJobId: "queued:1"); + Assert.True((await fixture.ExecuteJobAsync("queued:1")).IsInProgress); + var entry = Assert.Single(fixture.PumpEntries.Values.Cast()); + await fixture.StopAsync(); + Assert.Empty(fixture.PumpEntries); + Assert.Equal(default, GetPumpRegistration(entry)); + Assert.Equal(0, fixture.LocalField("_activePumpTurns")); + await fixture.RunRegisteredTimerAtAsync(0); + Assert.Equal(0, fixture.DeliveryCount); + Assert.Single(fixture.Messages); + Assert.Equal(0, fixture.Manager.WriteCount); + } + + [Fact] + public async Task DisposedPhysicalTurn_OldRegistrationCannotConsumeNewQueuedPayload() + { + using var jobCancellation = new CancellationTokenSource(); + using var timerCancellation = new CancellationTokenSource(); + using var fixture = new OutboxFixture(_ => ValueTask.FromResult(DeliveryResult.Backpressured()), + durableJobId: "physical:1", jobTimeProvider: new FakeTimeProvider()); + var captured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.Manager.AfterCapture = () => { captured.TrySetResult(); return release.Task; }; + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "old", jobCancellation.Token)).IsInProgress); + var active = fixture.RunRegisteredTimerAsync(timerCancellation.Token); + await captured.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + jobCancellation.Cancel(); + timerCancellation.Cancel(); + Assert.True((await fixture.ExecuteJobAsync(job, "new", TestContext.Current.CancellationToken)).IsInProgress); + var newest = fixture.PumpEntries.Keys.Cast().Single(key => GetPumpRunId(key) == "new"); + release.SetResult(); + await active; + fixture.Manager.AfterCapture = null; + await fixture.RunRegisteredTimerAtAsync(0); // Retained old physical callback, not the latest arm. + Assert.True(fixture.PumpEntries.Contains(newest)); + Assert.Equal(1, fixture.LocalField("_activePumpTurns")); + Assert.Equal(1, fixture.DeliveryCount); + await fixture.RunRegisteredTimerAtAsync(1); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, + (await fixture.ExecuteJobAsync(job, "new", TestContext.Current.CancellationToken)).Status); + Assert.Equal(["PumpTimerState", "PumpTimerState"], fixture.TimerRegistrationNames); + Assert.Equal(0, fixture.LocalField("_activePumpTurns")); + } + + [Theory] + [InlineData(DeliveryStatus.Accepted)] + [InlineData(DeliveryStatus.Duplicate)] + [InlineData(DeliveryStatus.Backpressured)] + [InlineData(DeliveryStatus.HandlerNotFound)] + [InlineData(DeliveryStatus.DeadLettered)] + public async Task ScalarDelivery_RetainsExactTaskAndUsesNoFanoutStorage(DeliveryStatus status) + { + var response = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var fixture = new OutboxFixture(_ => new(response.Task), durableJobId: "scalar:1", maxDeliveryAttempts: 1); + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "scalar", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAtAsync(0); + var pending = fixture.LocalField("_pendingDeliveryBatch")!; + var attempt = Assert.IsAssignableFrom(pending.GetType().GetProperty("SingleAttempt")!.GetValue(pending)); + Assert.Same(attempt, pending.GetType().GetProperty("Completion")!.GetValue(pending)); + Assert.Null(pending.GetType().GetProperty("Attempts")!.GetValue(pending)); + var wake = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.ArmObserved = () => wake.TrySetResult(); + response.SetResult(new DeliveryResult { Status = status }); + await wake.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Same(attempt, pending.GetType().GetProperty("Completion")!.GetValue(pending)); + await fixture.RunRegisteredTimerAtAsync(1); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Null(fixture.LocalField("_pendingDeliveryBatch")); + Assert.Empty(fixture.Messages); + Assert.Equal(status is DeliveryStatus.Backpressured or DeliveryStatus.HandlerNotFound ? 1 : 0, fixture.DeadLetters.Count); + } + + [Fact] + public async Task OwnedResources_AreLazyUntilActualDrainWaiterExists() + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + await fixture.CommitAsync(); + await fixture.DeliverAsync(); + Assert.Null(fixture.LocalField("_writesDrained")); + Assert.Equal(0, fixture.LocalField("_activeWrites")); + await fixture.StopAsync(); + Assert.Null(fixture.LocalField("_writesDrained")); + } + + [Fact] + public async Task ZeroIdleGrace_DeliveryAndOwnerRetirementShareOneAcknowledgedWrite() + { + using var fixture = new OutboxFixture(durableJobId: "combined:1"); + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "combined", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAtAsync(0); + Assert.Equal(DurableJobRunStatus.Completed, + (await fixture.ExecuteJobAsync(job, "combined", TestContext.Current.CancellationToken)).Status); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(1, fixture.Manager.WriteCompletedCount); + Assert.Null(fixture.Job.Value); + Assert.Equal("combined:1", fixture.CompletedJobId.Value); + Assert.Null(fixture.LocalField("_writesDrained")); + } + + [Fact] + public async Task IdleGrace_RetainsPhysicalOwnerForBurstsThenRetiresAtExactDeadline() + { + var clock = new FakeTimeProvider(); + using var fixture = new OutboxFixture(durableJobId: "grace:1", jobTimeProvider: clock, + idleRetirementGracePeriod: TimeSpan.FromMilliseconds(100)); + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "first", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAtAsync(0); + Assert.Equal(clock.GetUtcNow() + TimeSpan.FromMilliseconds(100), + (await fixture.ExecuteJobAsync(job, "first", TestContext.Current.CancellationToken)).RescheduleTime); + Assert.Same(job, fixture.Job.Value); + clock.Advance(TimeSpan.FromMilliseconds(99)); + await fixture.SendAsync(fixture.CreateEnvelope(fixture.NextMessageId())); + await fixture.CommitAsync(); + await fixture.RunRegisteredTimerAtAsync(1); + Assert.Equal(2, fixture.DeliveryCount); + Assert.Same(job, fixture.Job.Value); + Assert.Equal(0, ((RecordingJobManager)fixture.JobManager).AttemptCount); + var deadline = clock.GetUtcNow() + TimeSpan.FromMilliseconds(100); + Assert.True((await fixture.ExecuteJobAsync(job, "second", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAtAsync(2); + Assert.Equal(deadline, (await fixture.ExecuteJobAsync(job, "second", TestContext.Current.CancellationToken)).RescheduleTime); + clock.Advance(TimeSpan.FromMilliseconds(100)); + Assert.True((await fixture.ExecuteJobAsync(job, "idle", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAtAsync(3); + Assert.Equal(DurableJobRunStatus.Completed, + (await fixture.ExecuteJobAsync(job, "idle", TestContext.Current.CancellationToken)).Status); + Assert.Null(fixture.Job.Value); + Assert.Equal("grace:1", fixture.CompletedJobId.Value); + Assert.Equal(4, fixture.Manager.WriteCount); + } + + [Fact] + public async Task CombinedRetirement_HookLateSendRestoresExactAcknowledgedPhysicalOwner() + { + var hook = new TestJournaledStateHook(); + using var fixture = new OutboxFixture(durableJobId: "hook:1", hook: hook); + var job = fixture.Job.Value!; + var late = fixture.CreateEnvelope(fixture.NextMessageId()); + hook.BeforeOperation = (_, _) => fixture.Outbox.Send(late); + Assert.True((await fixture.ExecuteJobAsync(job, "hook", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAtAsync(0); + Assert.Equal(late.Envelope.MessageId, Assert.Single(fixture.Messages).Key); + Assert.False(fixture.IsPending(late.Envelope.MessageId)); + Assert.Same(job, fixture.Job.Value); + Assert.Null(fixture.CompletedJobId.Value); + Assert.Equal(0, ((RecordingJobManager)fixture.JobManager).AttemptCount); + Assert.Equal(1, fixture.Manager.WriteCount); + } + + [Fact] + public async Task RemoteDelivery_ResolvesEachDestinationThroughGrainFactory() + { + using var fixture = new OutboxFixture(hasMessage: false, batchSize: 128); + await fixture.SendAsync(fixture.Envelope); + await fixture.CommitAsync(); + await fixture.DeliverAsync(); + await fixture.SendAsync(fixture.CreateEnvelope(fixture.NextMessageId())); + await fixture.CommitAsync(); + await fixture.DeliverAsync(); + Assert.Equal(2, fixture.ProxyAcquisitionCount); + for (var index = 0; index < 65; index++) + { + var newEnvelope = fixture.CreateEnvelope(fixture.NextMessageId()) with + { + ReceiverId = GrainId.Create("destination", index.ToString()) + }; + await fixture.SendAsync(newEnvelope); + } + await fixture.CommitAsync(); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + Assert.Equal(67, fixture.ProxyAcquisitionCount); + Assert.Equal(67, fixture.DeliveryCount); + } + + [Fact] + public async Task LocalPump_AcknowledgementWakesOnceAndRetainsPhysicalRecoveryOwner() + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + Assert.Empty(fixture.ArmedTimerNames); + Assert.Equal(0, fixture.DeliveryCount); + await fixture.CommitAsync(); + var job = fixture.Job.Value; + var id = fixture.JobId.Value; + Assert.Equal(["LocalPumpTimerState"], fixture.ArmedTimerNames); + await fixture.CommitAsync(); + Assert.Equal(["LocalPumpTimerState"], fixture.ArmedTimerNames); + await fixture.RunRegisteredTimerAtAsync(0); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Empty(fixture.Messages); + Assert.Same(job, fixture.Job.Value); + Assert.Equal(id, fixture.JobId.Value); + + Assert.True((await fixture.ExecuteJobAsync(job!, "recovery-retirement", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAtAsync(1); + Assert.Null(fixture.Job.Value); + Assert.Null(fixture.JobId.Value); + Assert.Equal(id, fixture.CompletedJobId.Value); + } + + [Theory] + [InlineData(DeliveryStatus.Accepted)] + [InlineData(DeliveryStatus.Backpressured)] + public async Task LocalPump_RemoteCompletionWakesWithoutJobPollingAndKeepsActualOutcome(DeliveryStatus status) + { + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var fixture = new OutboxFixture(_ => new(release.Task), hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + await fixture.CommitAsync(); + await fixture.RunRegisteredTimerAtAsync(0); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Single(fixture.Messages); + Assert.Equal(["LocalPumpTimerState"], fixture.ArmedTimerNames); + var queued = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.ArmObserved = () => queued.TrySetResult(); + release.SetResult(status == DeliveryStatus.Accepted ? DeliveryResult.Accepted() : DeliveryResult.Backpressured()); + await queued.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(["LocalPumpTimerState", "LocalPumpTimerState"], fixture.ArmedTimerNames); + await fixture.RunRegisteredTimerAtAsync(1); + Assert.Equal(1, fixture.DeliveryCount); + if (status == DeliveryStatus.Accepted) + { + Assert.Empty(fixture.Messages); + } + else + { + Assert.Single(fixture.Messages); + Assert.Equal(1, fixture.MessageStates.GetProperty(fixture.MessageId, "AttemptCount")); + Assert.Equal(2, fixture.ArmedTimerNames.Length); + } + } + + [Fact] + public async Task RecoveryAndLifecycleStart_CoalesceMissingOwnerRepair() + { + var timerRegistry = Substitute.For(); + var jobManager = new RecordingJobManager(); + using var fixture = new OutboxFixture( + hasMessage: true, + timerRegistry: timerRegistry, + jobManager: jobManager); + + Assert.DoesNotContain(timerRegistry.ReceivedCalls(), static call => call.GetMethodInfo().Name == "RegisterGrainTimer"); + Assert.Equal(0, jobManager.AttemptCount); + Assert.Equal(0, fixture.Manager.WriteCount); + await fixture.StartAsync(); + await fixture.StartAsync(); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + + Assert.NotNull(fixture.JobId.Value); + Assert.Equal(1, jobManager.AttemptCount); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal( + ["EnsureJobTimerState", "LocalPumpTimerState"], + fixture.ArmedTimerNames); + } + + [Fact] + public async Task HealthyRecoveredOwnership_IsRetainedWithoutSchedulingReplacement() + { + const string ownershipId = "recovered:1"; + var timerRegistry = Substitute.For(); + var jobManager = new RecordingJobManager(); + using var fixture = new OutboxFixture( + hasMessage: true, + timerRegistry: timerRegistry, + jobManager: jobManager, + durableJobId: ownershipId); + + var handle = fixture.Job.Value; + await fixture.StartAsync(); + + Assert.Same(handle, fixture.Job.Value); + Assert.Equal(ownershipId, fixture.JobId.Value); + Assert.Equal(0, jobManager.AttemptCount); + Assert.Equal(["LocalPumpTimerState"], fixture.ArmedTimerNames); + await fixture.RunRegisteredTimerAtAsync(0); + Assert.Empty(fixture.Messages); + Assert.Same(handle, fixture.Job.Value); + Assert.Equal(1, fixture.DeliveryCount); + } + + [Fact] + public async Task RecoveredLogicalOwnerWithoutHandle_FailsLifecycleAndCallbackBoundaries() + { + const string incompleteOwnershipId = "incomplete:1"; + var timerRegistry = Substitute.For(); + var jobManager = new RecordingJobManager(); + using var fixture = new OutboxFixture( + hasMessage: true, + timerRegistry: timerRegistry, + jobManager: jobManager, + durableJobId: incompleteOwnershipId, + hasDurableJobHandle: false); + + + var lifecycleException = await Assert.ThrowsAsync( + () => fixture.StartAsync()); + var callbackException = await Assert.ThrowsAsync( + async () => await fixture.ExecuteJobAsync( + incompleteOwnershipId, + "orphaned-physical-job", + "run-after-recovery", + TestContext.Current.CancellationToken)); + Assert.Contains("both be present or both be absent", lifecycleException.Message, StringComparison.Ordinal); + Assert.Equal(lifecycleException.Message, callbackException.Message); + Assert.Equal(incompleteOwnershipId, fixture.JobId.Value); + Assert.Null(fixture.Job.Value); + Assert.Equal(0, jobManager.AttemptCount); + Assert.Equal(0, fixture.Manager.WriteCount); + } + + [Fact] + public async Task RecoveredHandleWithMismatchedOwnershipMetadata_FailsLifecycleAndCallbackBoundaries() + { + const string ownershipId = "owner:1"; + using var fixture = new OutboxFixture( + hasMessage: true, + durableJobId: ownershipId); + fixture.Job.Value = fixture.CreateJobForTest("physical-job", "different-owner:2"); + + + var lifecycleException = await Assert.ThrowsAsync( + () => fixture.StartAsync()); + var callbackException = await Assert.ThrowsAsync( + async () => await fixture.ExecuteJobAsync( + ownershipId, + "physical-job", + "run", + TestContext.Current.CancellationToken)); + Assert.Contains("metadata does not match", lifecycleException.Message, StringComparison.Ordinal); + Assert.Equal(lifecycleException.Message, callbackException.Message); + } + + [Fact] + public async Task DuplicatePhysicalCallbacks_CoalesceByLogicalOwnership() + { + const string ownershipId = "owner:1"; + var timerRegistry = Substitute.For(); + using var fixture = new OutboxFixture( + hasMessage: true, + timerRegistry: timerRegistry, + durableJobId: ownershipId); + + var first = await fixture.ExecuteJobAsync(ownershipId, $"job-{ownershipId}", "run-1", TestContext.Current.CancellationToken); + var second = await fixture.ExecuteJobAsync(ownershipId, $"job-{ownershipId}", "run-2", TestContext.Current.CancellationToken); + + Assert.True(first.IsInProgress); + Assert.True(second.IsInProgress); + Assert.Single( + timerRegistry.ReceivedCalls(), + static call => call.GetMethodInfo().Name == "RegisterGrainTimer"); + } + + [Fact] + public async Task TerminalFailureInvalidatesQueuedPumpAndFreshCallbackTakesOwnership() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + Assert.True((await fixture.ExecuteJobAsync("owner:1")).IsInProgress); + var failure = new IOException("Terminal failure."); + fixture.FailPersistence(failure); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, fixture.DeliveryCount); + Assert.Single(fixture.Messages); + using var recovered = fixture.Recreate(); + Assert.True((await recovered.ExecuteJobAsync("owner:1")).IsInProgress); + await recovered.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, recovered.DeliveryCount); + Assert.Empty(recovered.Messages); + } + + [Theory] + [InlineData("_gate")] + [InlineData("_deliveryGate")] + public async Task TerminalFailureWhilePumpWaitsForGate_PreservesDurableState(string gateName) + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + var gate = fixture.GetGate(gateName); + await gate.WaitAsync(TestContext.Current.CancellationToken); + Assert.True((await fixture.ExecuteJobAsync("owner:1")).IsInProgress); + var turn = fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + try + { + Assert.False(turn.IsCompleted); + fixture.FailPersistence(new IOException("Terminal failure while queued.")); + } + finally + { + gate.Release(); + } + await turn.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(0, fixture.DeliveryCount); + Assert.Equal(0, fixture.Manager.CaptureCount); + Assert.Single(fixture.Messages); + Assert.Equal("owner:1", fixture.JobId.Value); + Assert.Null(fixture.CompletedJobId.Value); + } + + [Theory] + [InlineData("_gate")] + [InlineData("_deliveryGate")] + public async Task CommittedPhysicalOwnerChangeWhilePumpWaits_PreservesReplacement(string gateName) + { + const string ownershipId = "owner:1"; + using var fixture = new OutboxFixture(durableJobId: ownershipId); + var replacement = fixture.CreateJobForTest("replacement-physical-job", ownershipId); + var gate = fixture.GetGate(gateName); + await gate.WaitAsync(TestContext.Current.CancellationToken); + Assert.True((await fixture.ExecuteJobAsync(ownershipId)).IsInProgress); + var turn = fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + try + { + Assert.False(turn.IsCompleted); + fixture.Job.Value = replacement; + fixture.Manager.CommitExternalOwner(); + } + finally + { + gate.Release(); + } + + await turn.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.Equal(0, fixture.DeliveryCount); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Single(fixture.Messages); + Assert.Equal(ownershipId, fixture.JobId.Value); + Assert.Same(replacement, fixture.Job.Value); + Assert.Null(fixture.CompletedJobId.Value); + } + + [Fact] + public async Task RemoteBatchFromPreviousPhysicalOwner_IsDiscarded() + { + const string ownershipId = "owner:1"; + var outcome = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var fixture = new OutboxFixture( + _ => new ValueTask(outcome.Task), + durableJobId: ownershipId); + Assert.True((await fixture.ExecuteJobAsync(ownershipId)).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, fixture.DeliveryCount); + var replacement = fixture.CreateJobForTest("replacement-physical-job", ownershipId); + fixture.Job.Value = replacement; + fixture.Manager.CommitExternalOwner(); + outcome.SetResult(DeliveryResult.Accepted()); + fixture.TimerRegistry.ClearReceivedCalls(); + + Assert.True((await fixture.ExecuteJobAsync( + replacement, "replacement-run", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, fixture.DeliveryCount); + Assert.Single(fixture.Messages); + Assert.True(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(ownershipId, fixture.JobId.Value); + Assert.Same(replacement, fixture.Job.Value); + Assert.Null(fixture.CompletedJobId.Value); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task CanceledRemoteBatch_CapturedTokenSupportsLateRegistration(bool stopActivation, bool failDelivery) + { + const string ownershipId = "owner:1"; + var captured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Task? remoteAttempt = null; + var callbacks = 0; + var lateFailure = new IOException("Late canceled remote failure."); + using var fixture = new OutboxFixture( + token => new ValueTask(remoteAttempt = RunRemoteAttemptAsync(token)), + durableJobId: ownershipId); + Assert.True((await fixture.ExecuteJobAsync(ownershipId)).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + var token = await captured.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var (cancellation, batchAttempts) = fixture.GetPendingBatchState(); + var attempt = Assert.IsAssignableFrom>(remoteAttempt); + Assert.False(attempt.IsCompleted); + + var replacement = fixture.CreateJobForTest("replacement-physical-job", ownershipId); + Task stopping = Task.CompletedTask; + if (stopActivation) + { + stopping = fixture.StopAsync(); + Assert.False(stopping.IsCompleted); + } + else + { + fixture.Job.Value = replacement; + fixture.Manager.CommitExternalOwner(); + fixture.TimerRegistry.ClearReceivedCalls(); + Assert.True((await fixture.ExecuteJobAsync( + replacement, "replacement-run", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + } + + await cancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.True(token.IsCancellationRequested); + Assert.False(attempt.IsCompleted); + Assert.Equal(token, cancellation.Token); + Assert.True(token.WaitHandle.WaitOne(0)); + var drained = fixture.GetPendingBatchDrainTask(); + release.SetResult(); + if (failDelivery) + { + Assert.Same(lateFailure, await Assert.ThrowsAsync( + () => attempt.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken))); + } + else + { + Assert.Equal(DeliveryStatus.Accepted, + (await attempt.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)).Status); + } + + await Task.WhenAll(batchAttempts).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await drained.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await stopping.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Throws(() => cancellation.Token); + Assert.Throws(() => token.WaitHandle); + Assert.Equal(2, callbacks); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Single(fixture.Messages); + Assert.Equal(0, fixture.MessageStates.GetProperty(fixture.MessageId, "AttemptCount")); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(stopActivation ? 0 : 1, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.FaultCount); + Assert.Equal(ownershipId, fixture.JobId.Value); + Assert.Equal(stopActivation ? $"job-{ownershipId}" : replacement.Id, fixture.Job.Value?.Id); + Assert.Null(fixture.CompletedJobId.Value); + + async Task RunRemoteAttemptAsync(CancellationToken cancellationToken) + { + using var initialRegistration = cancellationToken.Register(cancellationObserved.SetResult); + captured.SetResult(cancellationToken); + await release.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.True(cancellationToken.CanBeCanceled); + Assert.True(cancellationToken.IsCancellationRequested); + using var lateRegistration = cancellationToken.Register(() => callbacks++); + using var lateUnsafeRegistration = cancellationToken.UnsafeRegister(_ => callbacks++, null); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + Assert.True(linked.IsCancellationRequested); + var cancellationError = Assert.Throws(cancellationToken.ThrowIfCancellationRequested); + Assert.Equal(cancellationToken, cancellationError.CancellationToken); + var delayError = await Assert.ThrowsAnyAsync( + () => Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken)); + Assert.Equal(cancellationToken, delayError.CancellationToken); + if (failDelivery) + { + throw lateFailure; + } + return DeliveryResult.Accepted(); + } + } + + [Theory] + [InlineData("replacement", false)] + [InlineData("replacement", true)] + [InlineData("stop", false)] + [InlineData("stop", true)] + [InlineData("fault", false)] + [InlineData("fault", true)] + public async Task PendingBatchCancellation_DrainsBeforeDisposalAndPreservesCleanup(string trigger, bool callbackThrows) + { + const string ownershipId = "owner:1"; + var canceled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var callbacks = 0; + var lateCallbacks = 0; + var callbackFailure = new InvalidOperationException("Remote cancellation callback failed."); + var terminalFailure = new IOException("Original terminal journal failure."); + using var fixture = new OutboxFixture(token => new(RemoteAsync(token)), durableJobId: ownershipId); + await fixture.StartAsync(); + await fixture.ExecuteJobAsync(ownershipId); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + var (source, attempts) = fixture.GetPendingBatchState(); + var token = source.Token; + var waitHandle = token.WaitHandle; + Task stopping = Task.CompletedTask; + try + { + if (trigger == "replacement") + { + var replacement = fixture.CreateJobForTest("replacement", ownershipId); + fixture.Job.Value = replacement; + fixture.Manager.CommitExternalOwner(); + fixture.TimerRegistry.ClearReceivedCalls(); + await fixture.ExecuteJobAsync(replacement, "replacement-run", TestContext.Current.CancellationToken); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + Assert.True((await fixture.ExecuteJobAsync(replacement, "replacement-run", TestContext.Current.CancellationToken)).IsInProgress); + Assert.Equal(1, fixture.GetOutboxDepth()); + } + else + { + if (trigger == "fault") fixture.FailPersistence(terminalFailure); + stopping = fixture.StopAsync(); + Assert.Equal(0, fixture.GetOutboxDepth()); + Assert.Empty(fixture.PumpEntries); + Assert.False(stopping.IsCompleted); + } + await canceled.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(2, callbacks); + Assert.All(attempts, attempt => Assert.False(attempt.IsCompleted)); + Assert.Equal(token, source.Token); + Assert.Same(waitHandle, token.WaitHandle); + Assert.True(waitHandle.WaitOne(0)); + using var lateRegistration = token.Register(() => lateCallbacks++); + Assert.Equal(1, lateCallbacks); + Assert.Single(fixture.Messages); + Assert.Equal(0, fixture.MessageStates.GetProperty(fixture.MessageId, "AttemptCount")); + } + finally + { + release.TrySetResult(); + await Task.WhenAll(attempts).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await stopping.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } + await fixture.StopAsync(); + await fixture.StopAsync(); + Assert.Throws(() => source.Token); + Assert.Throws(() => token.WaitHandle); + Assert.Equal(2, callbacks); + Assert.Equal(2, lateCallbacks); + var cancellationErrors = fixture.LoggedExceptions.OfType().ToArray(); + if (callbackThrows) Assert.Same(callbackFailure, Assert.Single(Assert.Single(cancellationErrors).Flatten().InnerExceptions)); + else Assert.Empty(cancellationErrors); + Assert.Equal(0, fixture.GetOutboxDepth()); + Assert.Empty(fixture.PumpEntries); + Assert.Single(fixture.Messages); + Assert.Equal(trigger == "replacement" ? 1 : 0, fixture.Manager.WriteCount); + Assert.Equal(trigger == "fault" ? 1 : 0, fixture.Manager.FaultCount); + if (trigger == "fault") Assert.Same(terminalFailure, await Assert.ThrowsAsync(() => fixture.CommitAsync().AsTask())); + + async Task RemoteAsync(CancellationToken cancellationToken) + { + using var observe = cancellationToken.Register(() => { callbacks++; canceled.TrySetResult(); }); + using var throwing = cancellationToken.Register(() => + { + callbacks++; + if (callbackThrows) throw callbackFailure; + }); + await release.Task; + using var late = cancellationToken.Register(() => lateCallbacks++); + return DeliveryResult.Accepted(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task LoopbackTurnCancellation_DrainsRemoteAttemptWithoutMaskingOriginalCause(bool callbackThrows) + { + var remoteStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var loopbackStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var remoteCanceled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseRemote = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseLoopback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var calls = 0; + var callbacks = 0; + var callbackFailure = new InvalidOperationException("Remote callback during loopback retirement."); + using var fixture = new OutboxFixture(token => new(DeliverControlledAsync(token)), durableJobId: "owner:1"); + var newEnvelope = fixture.CreateEnvelope(fixture.NextMessageId()) with { ReceiverId = fixture.SenderId }; + await fixture.SendAsync(newEnvelope); + await fixture.CommitAsync(); + using var timer = new CancellationTokenSource(); + await fixture.ExecuteJobAsync("owner:1"); + var turn = fixture.RunRegisteredTimerAsync(timer.Token); + var remoteToken = await remoteStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var loopbackToken = await loopbackStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var waitHandle = remoteToken.WaitHandle; + try + { + timer.Cancel(); + Assert.False(remoteToken.IsCancellationRequested); + Assert.True(loopbackToken.IsCancellationRequested); + releaseLoopback.TrySetResult(); + await remoteCanceled.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.False(turn.IsCompleted); + Assert.Same(waitHandle, remoteToken.WaitHandle); + Assert.True(waitHandle.WaitOne(0)); + Assert.Equal(1, callbacks); + Assert.Equal(2, fixture.Outbox.Count); + } + finally + { + releaseLoopback.TrySetResult(); + releaseRemote.TrySetResult(); + } + var failure = await Assert.ThrowsAnyAsync( + () => turn.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Equal(loopbackToken, failure.CancellationToken); + var cancellationErrors = fixture.LoggedExceptions.OfType().ToArray(); + if (callbackThrows) Assert.Same(callbackFailure, Assert.Single(Assert.Single(cancellationErrors).Flatten().InnerExceptions)); + else Assert.Empty(cancellationErrors); + Assert.Throws(() => remoteToken.WaitHandle); + Assert.Equal(1, callbacks); + Assert.Equal(2, fixture.Messages.Count); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.FaultCount); + + async Task DeliverControlledAsync(CancellationToken token) + { + if (++calls == 1) + { + using var registration = token.Register(() => + { + callbacks++; + remoteCanceled.TrySetResult(); + if (callbackThrows) throw callbackFailure; + }); + remoteStarted.TrySetResult(token); + await releaseRemote.Task; + return DeliveryResult.Accepted(); + } + loopbackStarted.TrySetResult(token); + await releaseLoopback.Task; + token.ThrowIfCancellationRequested(); + return DeliveryResult.Accepted(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StaleQueuedPump_ReleasesWaitingEntryAndRegistration(bool replaceLogicalOwner) + { + const string owner = "owner:1"; + using var fixture = new OutboxFixture(durableJobId: owner); + using var cancellation = new CancellationTokenSource(); + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "old-run", cancellation.Token)).IsInProgress); + var oldEntry = Assert.Single(fixture.PumpEntries.Values.Cast()); + var oldKey = Assert.Single(fixture.PumpEntries.Keys.Cast()); + Assert.NotEqual(default, GetPumpRegistration(oldEntry)); + var nextOwner = replaceLogicalOwner ? "owner:2" : owner; + var replacement = fixture.CreateJobForTest("replacement", nextOwner); + fixture.JobId.Value = nextOwner; + fixture.Job.Value = replacement; + fixture.Manager.CommitExternalOwner(); + + if (replaceLogicalOwner) + { + Assert.True((await fixture.ExecuteJobAsync(replacement, "new-run", TestContext.Current.CancellationToken)).IsInProgress); + } + await fixture.RunRegisteredTimerAtAsync(0); + + Assert.False(fixture.PumpEntries.Contains(oldKey)); + Assert.Equal(replaceLogicalOwner ? 1 : 0, fixture.PumpEntries.Count); + Assert.Equal(default, GetPumpRegistration(oldEntry)); + Assert.Equal(replaceLogicalOwner ? 1 : 0, fixture.DeliveryCount); + if (replaceLogicalOwner) Assert.Empty(fixture.Messages); + else Assert.Single(fixture.Messages); + cancellation.Cancel(); + if (!replaceLogicalOwner) + { + Assert.True((await fixture.ExecuteJobAsync(replacement, "new-run", TestContext.Current.CancellationToken)).IsInProgress); + } + await fixture.RunRegisteredTimerAtAsync(1); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Equal(DurableJobRunStatus.Completed, + (await fixture.ExecuteJobAsync(replacement, "new-run", TestContext.Current.CancellationToken)).Status); + Assert.Empty(fixture.PumpEntries); + Assert.Equal(["PumpTimerState"], fixture.TimerRegistrationNames); + Assert.Equal(0, fixture.LocalField("_activePumpTurns")); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StaleCompletedPoll_RetiresOnlyExactRun(bool replaceLogicalOwner) + { + const string owner = "owner:1"; + using var fixture = new OutboxFixture(_ => ValueTask.FromResult(DeliveryResult.Backpressured()), durableJobId: owner); + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "completed-run", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + var completedEntry = Assert.Single(fixture.PumpEntries.Values.Cast()); + Assert.Equal("Completed", completedEntry.GetType().GetProperty("State")!.GetValue(completedEntry)!.ToString()); + fixture.TimerRegistry.ClearReceivedCalls(); + Assert.True((await fixture.ExecuteJobAsync(job, "waiting-run", TestContext.Current.CancellationToken)).IsInProgress); + var waitingKey = fixture.PumpEntries.Keys.Cast().Single(key => GetPumpRunId(key) == "waiting-run"); + var waitingEntry = fixture.PumpEntries[waitingKey]; + var replacementOwner = replaceLogicalOwner ? "owner:2" : owner; + fixture.JobId.Value = replacementOwner; + fixture.Job.Value = fixture.CreateJobForTest("replacement", replacementOwner); + fixture.Manager.CommitExternalOwner(); + + Assert.Equal(DurableJobRunStatus.Completed, + (await fixture.ExecuteJobAsync(job, "completed-run", TestContext.Current.CancellationToken)).Status); + + Assert.Equal(waitingKey, Assert.Single(fixture.PumpEntries.Keys.Cast())); + Assert.Same(waitingEntry, fixture.PumpEntries[waitingKey]); + Assert.NotEqual(default, GetPumpRegistration(waitingEntry!)); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + Assert.Empty(fixture.PumpEntries); + Assert.Equal(1, fixture.MessageStates.GetProperty(fixture.MessageId, "AttemptCount")); + } + + [Fact] + public async Task OwnerClearPoll_ConsumesRetainedCompletionExactlyOnce() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "run", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + Assert.Null(fixture.JobId.Value); + Assert.Single(fixture.PumpEntries); + var writes = fixture.Manager.WriteCount; + + Assert.Equal(DurableJobRunStatus.Completed, (await fixture.ExecuteJobAsync(job, "run", TestContext.Current.CancellationToken)).Status); + Assert.Empty(fixture.PumpEntries); + Assert.Equal(DurableJobRunStatus.Completed, (await fixture.ExecuteJobAsync(job, "run", TestContext.Current.CancellationToken)).Status); + Assert.Equal(writes, fixture.Manager.WriteCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StopOrFault_ClearsOutboxEntriesAndPreservesInboxRetry(bool fault) + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + using var cancellation = new CancellationTokenSource(); + Assert.True((await fixture.ExecuteJobAsync(fixture.Job.Value!, "run", cancellation.Token)).IsInProgress); + var outboxEntry = Assert.Single(fixture.PumpEntries.Values.Cast()); + var inboxKey = fixture.AddInboxPumpEntry(cancellation.Token); + var inboxEntry = fixture.PumpEntries[inboxKey]; + + if (fault) { fixture.FailPersistence(new IOException("Terminal activation fault.")); } + else { await fixture.StopAsync(); } + + Assert.Equal(inboxKey, Assert.Single(fixture.PumpEntries.Keys.Cast())); + Assert.Same(inboxEntry, fixture.PumpEntries[inboxKey]); + Assert.NotEqual(default, GetPumpRegistration(inboxEntry!)); + Assert.Equal(default, GetPumpRegistration(outboxEntry)); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, fixture.DeliveryCount); + Assert.Single(fixture.Messages); + Assert.Single(fixture.PumpEntries); + using var recovered = fixture.Recreate(); + Assert.True((await recovered.ExecuteJobAsync("owner:1")).IsInProgress); + await recovered.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + Assert.Empty(recovered.Messages); + Assert.Equal(1, recovered.DeliveryCount); + } + + [Fact] + public async Task QuiescentDelete_ClearsOnlyRetainedOutboxResults() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + Assert.True((await fixture.ExecuteJobAsync("owner:1")).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + using var cancellation = new CancellationTokenSource(); + var inboxKey = fixture.AddInboxPumpEntry(cancellation.Token); + var inboxEntry = fixture.PumpEntries[inboxKey]; + + await fixture.DeleteAsync(TestContext.Current.CancellationToken); + + Assert.Equal(inboxKey, Assert.Single(fixture.PumpEntries.Keys.Cast())); + Assert.Same(inboxEntry, fixture.PumpEntries[inboxKey]); + Assert.NotEqual(default, GetPumpRegistration(inboxEntry!)); + } + + [Fact] + public async Task StalePoll_PreservesRunningExecutionUntilItSettles() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var fixture = new OutboxFixture(async _ => { entered.SetResult(); return await release.Task; }, durableJobId: "owner:1", loopback: true); + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "run", TestContext.Current.CancellationToken)).IsInProgress); + var timer = fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var entry = Assert.Single(fixture.PumpEntries.Values.Cast()); + Assert.Equal("Running", entry.GetType().GetProperty("State")!.GetValue(entry)!.ToString()); + fixture.Job.Value = fixture.CreateJobForTest("replacement", "owner:1"); + fixture.Manager.CommitExternalOwner(); + + Assert.Equal(DurableJobRunStatus.Completed, (await fixture.ExecuteJobAsync(job, "run", TestContext.Current.CancellationToken)).Status); + Assert.Same(entry, Assert.Single(fixture.PumpEntries.Values.Cast())); + release.SetResult(DeliveryResult.Accepted()); + await timer.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(DurableJobRunStatus.Completed, (await fixture.ExecuteJobAsync(job, "run", TestContext.Current.CancellationToken)).Status); + Assert.Empty(fixture.PumpEntries); + Assert.Single(fixture.Messages); + Assert.Equal(1, fixture.Manager.WriteCount); + } + + [Fact] + public async Task ReusedTimer_SupersededQueuedGenerationCannotDiscardNewestOutcome() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + var job = fixture.Job.Value!; + using var oldCancellation = new CancellationTokenSource(); + Assert.True((await fixture.ExecuteJobAsync(job, "same-run", oldCancellation.Token)).IsInProgress); + var oldEntry = Assert.Single(fixture.PumpEntries.Values.Cast()); + oldCancellation.Cancel(); + await Assert.ThrowsAnyAsync(async () => + await fixture.ExecuteJobAsync(job, "same-run", TestContext.Current.CancellationToken)); + Assert.True((await fixture.ExecuteJobAsync(job, "same-run", TestContext.Current.CancellationToken)).IsInProgress); + var newEntry = Assert.Single(fixture.PumpEntries.Values.Cast()); + Assert.NotEqual(oldEntry.GetType().GetProperty("Generation")!.GetValue(oldEntry), newEntry.GetType().GetProperty("Generation")!.GetValue(newEntry)); + + await fixture.RunRegisteredTimerAtAsync(0); + + Assert.Same(newEntry, Assert.Single(fixture.PumpEntries.Values.Cast())); + Assert.Equal(default, GetPumpRegistration(oldEntry)); + Assert.Equal(default, GetPumpRegistration(newEntry)); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Equal(["PumpTimerState"], fixture.TimerRegistrationNames); + Assert.Same(fixture.ArmState(0), fixture.ArmState(1)); + await fixture.RunRegisteredTimerAtAsync(1); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Equal(DurableJobRunStatus.Completed, (await fixture.ExecuteJobAsync(job, "same-run", TestContext.Current.CancellationToken)).Status); + Assert.Empty(fixture.PumpEntries); + } + + [Fact] + public async Task CanceledPoll_PreservesCompletionForLiveRetry() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + var job = fixture.Job.Value!; + Assert.True((await fixture.ExecuteJobAsync(job, "run", TestContext.Current.CancellationToken)).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + var entry = Assert.Single(fixture.PumpEntries.Values.Cast()); + using var canceled = new CancellationTokenSource(); + canceled.Cancel(); + + await Assert.ThrowsAnyAsync(async () => await fixture.ExecuteJobAsync(job, "run", canceled.Token)); + Assert.Same(entry, Assert.Single(fixture.PumpEntries.Values.Cast())); + Assert.Equal(DurableJobRunStatus.Completed, (await fixture.ExecuteJobAsync(job, "run", TestContext.Current.CancellationToken)).Status); + Assert.Empty(fixture.PumpEntries); + } + + private static CancellationTokenRegistration GetPumpRegistration(object entry) => + (CancellationTokenRegistration)entry.GetType().GetProperty("CancellationRegistration")!.GetValue(entry)!; + + private static string GetPumpRunId(object key) => (string)key.GetType().GetProperty("RunId")!.GetValue(key)!; + + [Fact] + public async Task FailureAfterCombinedDeliveryCommit_ReplayObservesAcknowledgedRetirement() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + fixture.Manager.AfterNextWrite(() => fixture.FailPersistence(new IOException("Activation failed after commit."))); + Assert.True((await fixture.ExecuteJobAsync("owner:1")).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + Assert.Empty(fixture.Messages); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Null(fixture.JobId.Value); + Assert.Null(fixture.Job.Value); + Assert.Equal("owner:1", fixture.CompletedJobId.Value); + using var recovered = fixture.Recreate(); + Assert.Equal(DurableJobRunStatus.Completed, (await recovered.ExecuteJobAsync("owner:1")).Status); + Assert.Empty(recovered.ArmedTimerNames); + Assert.Null(recovered.JobId.Value); + Assert.Null(recovered.Job.Value); + Assert.Equal("owner:1", recovered.CompletedJobId.Value); + Assert.Equal(0, recovered.Manager.WriteCount); + Assert.Equal(DurableJobRunStatus.Completed, (await recovered.ExecuteJobAsync("owner:1")).Status); + Assert.Equal(0, recovered.Manager.WriteCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task TerminalFailureDuringLoopbackDelivery_DiscardsStaleOutcome(bool failDelivery) + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var outcome = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var fixture = new OutboxFixture(async _ => { entered.SetResult(); return await outcome.Task; }, + maxDeliveryAttempts: 1, durableJobId: "owner:1", loopback: true); + Assert.True((await fixture.ExecuteJobAsync("owner:1")).IsInProgress); + var turn = fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + fixture.FailPersistence(new IOException("Terminal journal failure.")); + if (failDelivery) { outcome.SetException(new IOException("Stale result.")); } + else { outcome.SetResult(DeliveryResult.Accepted()); } + await turn.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Single(fixture.Messages); + Assert.Equal(0, fixture.MessageStates.GetProperty(fixture.MessageId, "AttemptCount")); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(0, fixture.Manager.CaptureCount); + Assert.Equal("owner:1", fixture.JobId.Value); + Assert.NotNull(fixture.Job.Value); + Assert.Null(fixture.CompletedJobId.Value); + } + + [Fact] + public async Task NonAuthoritativePhysicalCallbackForCurrentGeneration_CompletesWithoutPumping() + { + const string ownershipId = "owner:1"; + var timerRegistry = Substitute.For(); + using var fixture = new OutboxFixture( + hasMessage: true, + timerRegistry: timerRegistry, + durableJobId: ownershipId); + + var result = await fixture.ExecuteJobAsync( + ownershipId, + "duplicate-physical-job", + "duplicate-run", + TestContext.Current.CancellationToken); + + Assert.Equal(DurableJobRunStatus.Completed, result.Status); + Assert.Equal(ownershipId, fixture.JobId.Value); + Assert.DoesNotContain( + timerRegistry.ReceivedCalls(), + static call => call.GetMethodInfo().Name == "RegisterGrainTimer"); + } + + [Fact] + public async Task CanceledQueuedPump_AllowsDuplicateCallbackToTakeOwnership() + { + const string ownershipId = "owner:1"; + var timerRegistry = Substitute.For(); + using var fixture = new OutboxFixture( + hasMessage: true, + timerRegistry: timerRegistry, + durableJobId: ownershipId); + using var cancellation = new CancellationTokenSource(); + + Assert.True((await fixture.ExecuteJobAsync( + ownershipId, + $"job-{ownershipId}", + "run-1", + cancellation.Token)).IsInProgress); + cancellation.Cancel(); + Assert.True((await fixture.ExecuteJobAsync( + ownershipId, + $"job-{ownershipId}", + "run-2", + TestContext.Current.CancellationToken)).IsInProgress); + + Assert.Equal( + 1, + timerRegistry.ReceivedCalls().Count(static call => call.GetMethodInfo().Name == "RegisterGrainTimer")); + } + + [Fact] + public async Task CanceledRunningPump_ReleasesLogicalOwnershipForLocalRetryAndDuplicateCallback() + { + const string ownershipId = "owner:1"; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var timerRegistry = Substitute.For(); + using var fixture = new OutboxFixture( + async cancellationToken => + { + entered.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return DeliveryResult.Accepted(); + }, + hasMessage: true, + timerRegistry: timerRegistry, + durableJobId: ownershipId); + using var cancellation = new CancellationTokenSource(); + + Assert.True((await fixture.ExecuteJobAsync( + ownershipId, + $"job-{ownershipId}", + "run-1", + cancellation.Token)).IsInProgress); + var running = fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var localRetryArmed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.ArmObserved = () => + { + if (fixture.ArmedTimerNames[^1] == "LocalPumpTimerState") + { + localRetryArmed.TrySetResult(); + } + }; + cancellation.Cancel(); + await running; + await localRetryArmed.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.True((await fixture.ExecuteJobAsync( + ownershipId, + $"job-{ownershipId}", + "run-2", + TestContext.Current.CancellationToken)).IsInProgress); + Assert.Equal( + 2, + timerRegistry.ReceivedCalls().Count(static call => call.GetMethodInfo().Name == "RegisterGrainTimer")); + Assert.Equal(["PumpTimerState", "LocalPumpTimerState"], fixture.ArmedTimerNames); + } + + [Fact] + public async Task OwnerDeletion_DrainsRemoteBatchBeforeDeletingStorage() + { + var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var fixture = new OutboxFixture(_ => new(delivered.Task), durableJobId: "owner:1"); + Assert.True((await fixture.ExecuteJobAsync("owner:1")).IsInProgress); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + var deleting = fixture.DeleteAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(deleting.IsCompleted); + Assert.Single(fixture.Messages); + Assert.Equal(0, fixture.Manager.DeleteCount); + delivered.SetResult(DeliveryResult.Accepted()); + await deleting.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, fixture.Manager.DeleteCount); + Assert.Empty(fixture.Messages); + Assert.Equal(0, fixture.Manager.FaultCount); + await Assert.ThrowsAnyAsync(() => fixture.StartAsync()); + } + + [Fact] + public async Task CallbackWithoutOwnershipMetadata_CannotClaimJournaledOwnership() + { + const string ownershipId = "owner:1"; + var timerRegistry = Substitute.For(); + using var fixture = new OutboxFixture( + hasMessage: true, + timerRegistry: timerRegistry, + durableJobId: ownershipId); + + var result = await fixture.ExecuteJobWithoutOwnershipMetadataAsync( + ownershipId, + TestContext.Current.CancellationToken); + + Assert.Equal(DurableJobRunStatus.Completed, result.Status); + Assert.Equal(ownershipId, fixture.JobId.Value); + Assert.DoesNotContain( + timerRegistry.ReceivedCalls(), + static call => call.GetMethodInfo().Name == "RegisterGrainTimer"); + } + + [Fact] + public async Task QuiescentDelete_DiscardsLocalIntentAndRotatesOwnershipEpoch() + { + using var fixture = new OutboxFixture(hasMessage: false); + var before = fixture.GetOwnershipEpoch(); + await fixture.SendAsync(fixture.Envelope); + await fixture.DeleteAsync(TestContext.Current.CancellationToken); + Assert.NotEqual(before, fixture.GetOwnershipEpoch()); + Assert.Equal(0, fixture.Outbox.Count); + Assert.Empty(fixture.Messages); + } + + [Fact] + public void FreshActivation_RotatesOwnershipEpoch() + { + using var fixture = new OutboxFixture(hasMessage: false); + using var recovered = fixture.Recreate(); + Assert.NotEqual(fixture.GetOwnershipEpoch(), recovered.GetOwnershipEpoch()); + } + + [Fact] + public async Task EnsureJobTimerCompletion_RetainsHealthyOwnershipWithoutQueueingReplacement() + { + var timerRegistry = Substitute.For(); + using var fixture = new OutboxFixture( + hasMessage: true, + timerRegistry: timerRegistry, + jobManager: new RecordingJobManager()); + + await fixture.StartAsync(); + await fixture.RunRegisteredTimerAsync(TestContext.Current.CancellationToken); + + Assert.Equal(["EnsureJobTimerState", "LocalPumpTimerState"], fixture.ArmedTimerNames); + } + + [Fact] + public async Task OwnershipPersistenceFailure_FencesBeforeAnotherSchedulingAttempt() + { + var jobs = new RecordingJobManager(); + using var fixture = new OutboxFixture(jobManager: jobs); + var expected = new IOException("Ownership write failure."); + fixture.Manager.FailNextWrite(expected); + var actual = await Assert.ThrowsAsync(() => fixture.EnsureJobScheduledAsync(true, TestContext.Current.CancellationToken)); + Assert.Same(expected, actual); + Assert.Equal(1, jobs.AttemptCount); + Assert.Equal(1, fixture.Manager.FaultCount); + Assert.Equal(1, fixture.Manager.WriteCount); + await Assert.ThrowsAsync(() => fixture.EnsureJobScheduledAsync(true, TestContext.Current.CancellationToken)); + Assert.Equal(1, jobs.AttemptCount); + using var recovered = fixture.Recreate(); + Assert.Null(recovered.Job.Value); + Assert.Null(recovered.JobId.Value); + } + + [Fact] + public async Task FreshReplayAfterFailedReplacement_RetainsPrecedingOwner() + { + var jobs = new RecordingJobManager(); + using var fixture = new OutboxFixture(jobManager: jobs, durableJobId: "owner:1"); + fixture.Manager.FailNextWrite(new IOException("Failed before append.")); + await Assert.ThrowsAsync(() => fixture.EnsureJobScheduledAsync(true, TestContext.Current.CancellationToken)); + var candidate = Assert.IsType(jobs.LastJob); + Assert.Same(candidate, fixture.Job.Value); + using var recovered = fixture.Recreate(); + Assert.Equal("owner:1", recovered.JobId.Value); + Assert.Equal("job-owner:1", recovered.Job.Value?.Id); + Assert.Equal(DurableJobRunStatus.Completed, (await recovered.ExecuteJobAsync(candidate, "candidate", TestContext.Current.CancellationToken)).Status); + } + + [Fact] + public async Task FreshReplayAfterAmbiguousReplacement_RestoresCommittedCandidate() + { + var jobs = new RecordingJobManager(); + using var fixture = new OutboxFixture(jobManager: jobs, durableJobId: "owner:1"); + fixture.Manager.FailAfterNextWrite(new IOException("Acknowledgement failure.")); + await Assert.ThrowsAsync(() => fixture.EnsureJobScheduledAsync(true, TestContext.Current.CancellationToken)); + var candidate = Assert.IsType(jobs.LastJob); + using var recovered = fixture.Recreate(); + Assert.Equal(candidate.Metadata!["orleans.messaging.ownership-id"], recovered.JobId.Value); + Assert.Equal(candidate.Id, recovered.Job.Value?.Id); + Assert.Equal(candidate.ShardId, recovered.Job.Value?.ShardId); + Assert.Equal(DurableJobRunStatus.Completed, (await recovered.ExecuteJobAsync("owner:1")).Status); + Assert.Equal(0, ((RecordingJobManager)recovered.JobManager).AttemptCount); + } + + [Fact] + public async Task TerminalFault_PreservesOldInstanceStateAndRejectsCallbacks() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + var failure = new IOException("Terminal owner write."); + fixture.Manager.FailNextWrite(failure); + await Assert.ThrowsAsync(() => fixture.EnsureJobScheduledAsync(true, TestContext.Current.CancellationToken)); + var failedOwner = fixture.Job.Value; + var callback = await Assert.ThrowsAsync(async () => await fixture.ExecuteJobAsync("owner:1")); + Assert.Same(failure, callback); + Assert.Same(failedOwner, fixture.Job.Value); + Assert.Equal(1, fixture.Manager.FaultCount); + Assert.Equal(1, fixture.Manager.WriteCount); + } + + [Fact] + public async Task QueuedWriteDuringFeaturePreparation_CapturesOnlyPreviouslyStagedState() + { + var jobs = new BlockingJobManager(); + using var fixture = new OutboxFixture(jobManager: jobs, durableJobId: "owner:1"); + var scheduling = fixture.EnsureJobScheduledAsync(true, TestContext.Current.CancellationToken); + await jobs.WaitUntilScheduledAsync(); + await fixture.CommitAsync(); + Assert.Equal("owner:1", fixture.JobId.Value); + Assert.Equal(1, fixture.Manager.CaptureCount); + Assert.True((await fixture.ExecuteJobAsync(jobs.OwnershipId!)).IsInProgress); + jobs.Release(); + await scheduling.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(jobs.OwnershipId, fixture.JobId.Value); + Assert.Equal(2, fixture.Manager.WriteCompletedCount); + } + + [Fact] + public async Task FeatureSchedulingFailure_LeavesJournalHealthyAndUnchanged() + { + var jobs = new RecordingJobManager(alwaysFail: true); + using var fixture = new OutboxFixture(jobManager: jobs, jobTimeProvider: new FakeTimeProvider()); + await Assert.ThrowsAsync(() => fixture.EnsureJobScheduledAsync(false, TestContext.Current.CancellationToken)); + Assert.Equal(1, jobs.AttemptCount); + Assert.Equal(0, fixture.Manager.CaptureCount); + Assert.Equal(0, fixture.Manager.FaultCount); + Assert.Equal(0, fixture.PendingMessageCount); + Assert.Null(fixture.JobId.Value); + Assert.Null(fixture.Job.Value); + Assert.Equal(0, fixture.JobSequence.Value); + // With no unacknowledged intent, an unrelated no-op write remains healthy. + await fixture.CommitAsync(); + Assert.Null(fixture.Manager.Failure); + Assert.Equal(1, jobs.AttemptCount); + Assert.Null(fixture.Job.Value); + Assert.Equal(1, fixture.Manager.WriteCompletedCount); + Assert.Single(fixture.Messages); + } + + [Fact] + public async Task RemoteBatch_OutlivesTimerTurnButCancelsWithDurableAttempt() + { + const string ownershipId = "owner:1"; + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var tokenCaptured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var timerRegistry = Substitute.For(); + using var fixture = new OutboxFixture( + async cancellationToken => + { + tokenCaptured.TrySetResult(cancellationToken); + using var registration = cancellationToken.Register(cancellationObserved.SetResult); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return DeliveryResult.Accepted(); + }, + hasMessage: true, + timerRegistry: timerRegistry, + durableJobId: ownershipId); + using var attemptCancellation = new CancellationTokenSource(); + using var timerCancellation = new CancellationTokenSource(); + + Assert.True((await fixture.ExecuteJobAsync( + ownershipId, + $"job-{ownershipId}", + "run", + attemptCancellation.Token)).IsInProgress); + await fixture.RunRegisteredTimerAsync(timerCancellation.Token); + _ = await tokenCaptured.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + timerCancellation.Cancel(); + Assert.False(cancellationObserved.Task.IsCompleted); + + attemptCancellation.Cancel(); + await cancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } + + [Fact] + public async Task TerminalFailureDuringDelivery_PreservesJournaledMessage() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var result = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var fixture = new OutboxFixture(async _ => { entered.SetResult(); return await result.Task; }); + var delivery = fixture.DeliverAsync(); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var failure = new IOException("Terminal failure."); + fixture.FailPersistence(failure); + result.SetException(new IOException("Transport failure.")); + Assert.Same(failure, await Assert.ThrowsAsync(() => delivery)); + Assert.Single(fixture.Messages); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(0, fixture.Manager.CaptureCount); + } + + [Fact] + public async Task DuplicateAfterCommitRemainsDeliverableAndUnfenced() + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + await fixture.CommitAsync(); + var duplicate = fixture.CreateEquivalentEnvelope(); + + await fixture.SendAsync(duplicate); + + Assert.NotSame(fixture.Envelope.Envelope.Payload, duplicate.Envelope.Payload); + Assert.Equal(0, fixture.PendingMessageCount); + await fixture.DeliverAsync(); + Assert.Equal(1, fixture.DeliveryCount); + Assert.False(fixture.Messages.ContainsKey(fixture.MessageId)); + } + + [Fact] + public void SenderMustMatchOwningGrain() + { + using var fixture = new OutboxFixture(hasMessage: false); + Assert.Equal(fixture.SenderId, fixture.Outbox.SenderId); + var envelope = fixture.CreateEnvelope( + fixture.NextMessageId(), + senderId: GrainId.Create("sender", "spoofed")); + + var exception = Assert.Throws(() => fixture.Outbox.Send(envelope)); + + Assert.Contains("does not match the owning grain", exception.Message, StringComparison.Ordinal); + Assert.Empty(fixture.Messages); + } + + [Fact] + public async Task DurableDuplicateFollowedByNoOpWriteDoesNotFenceDelivery() + { + using var fixture = new OutboxFixture(); + + await fixture.SendAsync(fixture.CreateEquivalentEnvelope()); + await fixture.CommitAsync(); + + Assert.Equal(1, fixture.Manager.WriteCompletedCount); + Assert.Equal(0, fixture.PendingMessageCount); + await fixture.DeliverAsync(); + Assert.Equal(1, fixture.DeliveryCount); + } + + [Fact] + public async Task DuplicateBeforeFirstCommitRemainsPendingUntilOwningCommit() + { + using var fixture = new OutboxFixture(hasMessage: false); + + await fixture.SendAsync(fixture.Envelope); + await fixture.SendAsync(fixture.CreateEquivalentEnvelope()); + + Assert.Equal(1, fixture.Outbox.Count); + Assert.Single(fixture.Messages); + Assert.Equal(1, fixture.PendingMessageCount); + await fixture.DeliverAsync(); + Assert.Equal(0, fixture.DeliveryCount); + + await fixture.CommitAsync(); + + Assert.Equal(0, fixture.PendingMessageCount); + await fixture.DeliverAsync(); + Assert.Equal(1, fixture.DeliveryCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ConflictingDuplicateFailsWithoutMutatingDurableOrProvisionalMessage(bool commitFirst) + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + if (commitFirst) + { + await fixture.CommitAsync(); + } + + var exception = await Assert.ThrowsAsync( + () => { var newEnvelope = fixture.CreateConflictingEnvelope(); return fixture.SendAsync(newEnvelope); }); + + Assert.Contains(fixture.MessageId.ToString(), exception.Message, StringComparison.Ordinal); + Assert.True(fixture.Outbox.TryGetMessage(fixture.MessageId, out var stored)); + Assert.Equal(fixture.Envelope.Envelope.Payload.ToArray(), stored.Envelope.Payload.ToArray()); + Assert.Equal(commitFirst ? 0 : 1, fixture.PendingMessageCount); + } + + [Fact] + public async Task FreshActivationDiscardsUncommittedIntentAndCanSendAgain() + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + await fixture.StopAsync(); + using var recovered = fixture.Recreate(); + Assert.Equal(0, recovered.Outbox.Count); + await recovered.SendAsync(fixture.Envelope); + await recovered.CommitAsync(); + Assert.Single(recovered.Messages); + Assert.Equal(0, recovered.PendingMessageCount); + } + + [Fact] + public async Task MessageAddedAfterWriteCaptureRemainsFencedForNextCommit() + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + var late = fixture.CreateEnvelope(fixture.NextMessageId()); + fixture.Manager.AfterCapture = () => fixture.SendAsync(late); + await fixture.CommitAsync(); + Assert.False(fixture.IsPending(fixture.MessageId)); + Assert.True(fixture.IsPending(late.Envelope.MessageId)); + Assert.True(fixture.Messages.ContainsKey(late.Envelope.MessageId)); + using (var recovered = fixture.Recreate()) + { + Assert.False(recovered.Messages.ContainsKey(late.Envelope.MessageId)); + Assert.Single(recovered.Messages); + } + Assert.Equal(2, fixture.Outbox.Count); + fixture.Manager.AfterCapture = null; + await fixture.CommitAsync(); + Assert.Equal(0, fixture.PendingMessageCount); + Assert.Equal(2, fixture.Messages.Count); + } + + [Fact] + public async Task CancellationAfterSuccessfulDelivery_PreservesStateBeforeApply() + { + CancellationTokenSource? cancellation = null; + using var fixture = new OutboxFixture( + _ => + { + cancellation!.Cancel(); + return ValueTask.FromResult(DeliveryResult.Accepted()); + }); + fixture.ActivateMetrics(); + Assert.Equal(1, fixture.GetOutboxDepth()); + + for (var attempt = 0; attempt < 2; attempt++) + { + using var currentCancellation = new CancellationTokenSource(); + cancellation = currentCancellation; + var exception = await Assert.ThrowsAnyAsync( + () => fixture.DeliverWithCancellationAsync(currentCancellation.Token)); + + Assert.Equal(currentCancellation.Token, exception.CancellationToken); + Assert.True(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.True(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(1, fixture.GetOutboxDepth()); + } + + Assert.Equal(0, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.FaultCount); + } + + [Fact] + public async Task CancellationBeforeDeadLetterApply_PreservesFailureState() + { + using var cancellation = new CancellationTokenSource(); + using var fixture = new OutboxFixture( + _ => + { + cancellation.Cancel(); + return ValueTask.FromResult(DeliveryResult.HandlerNotFound()); + }, + maxDeliveryAttempts: 1); + + await Assert.ThrowsAnyAsync( + () => fixture.DeliverWithCancellationAsync(cancellation.Token)); + + Assert.True(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.MessageStates.GetProperty(fixture.MessageId, "AttemptCount")); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(0, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.FaultCount); + } + + [Fact] + public async Task LaterWritePreservesMessageAfterCanceledLocalDeliveryOutcome() + { + using var cancellation = new CancellationTokenSource(); + using var fixture = new OutboxFixture(_ => { cancellation.Cancel(); return ValueTask.FromResult(DeliveryResult.Accepted()); }); + await Assert.ThrowsAnyAsync(() => fixture.DeliverWithCancellationAsync(cancellation.Token)); + await fixture.CommitAsync(); + using var recovered = fixture.Recreate(); + Assert.Single(recovered.Messages); + Assert.True(recovered.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(0, recovered.DeadLetters.Count); + } + + [Fact] + public async Task TerminalFailurePreservesOriginalExceptionForLaterWrites() + { + var failure = new IOException("Write failure."); + using var fixture = new OutboxFixture(writeException: failure); + Assert.Same(failure, await Assert.ThrowsAsync(() => fixture.DeliverAsync())); + Assert.Same(failure, await Assert.ThrowsAsync(() => fixture.CommitAsync().AsTask())); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(1, fixture.Manager.FaultCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeliveryWriteFailure_FreshReplayResolvesAcknowledgement(bool committed) + { + using var fixture = new OutboxFixture(); + var failure = new IOException("Delivery append failed."); + if (committed) { fixture.Manager.FailAfterNextWrite(failure); } + else { fixture.Manager.FailNextWrite(failure); } + Assert.Same(failure, await Assert.ThrowsAsync(() => fixture.DeliverAsync())); + Assert.Empty(fixture.Messages); + Assert.Equal(1, fixture.Manager.FaultCount); + using var recovered = fixture.Recreate(); + Assert.Equal(committed ? 0 : 1, recovered.Messages.Count); + Assert.Equal(!committed, recovered.MessageStates.ContainsKey(fixture.MessageId)); + } + + [Theory] + [InlineData(0, "messages")] + [InlineData(1, "messageStates")] + [InlineData(2, "deadLetters")] + [InlineData(3, "jobId")] + [InlineData(4, "job")] + [InlineData(5, "completedJobId")] + [InlineData(6, "jobSequenceCodec")] + public void StateRegistration_RequiresStandardStatesAndSequenceCodec(int missingDependency, string parameter) + { + var error = Assert.Throws(() => new OutboxFixture(missingDependency: missingDependency)); + Assert.Equal(parameter, error.ParamName); + } + + [Fact] + public async Task NormalDeliveryCommitsBatchOnce() + { + using var fixture = new OutboxFixture( + _ => ValueTask.FromResult(DeliveryResult.Accepted())); + + await fixture.DeliverAsync(); + + Assert.False(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.False(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.FaultCount); + } + + [Fact] + public async Task ReceiverDeadLetterRemovesMessageWithoutCreatingSenderDeadLetter() + { + using var fixture = new OutboxFixture( + _ => ValueTask.FromResult(DeliveryResult.DeadLettered("Receiver rejected the payload."))); + + await fixture.DeliverAsync(); + await fixture.DeliverAsync(); + + Assert.False(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.False(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.DeadLetters.Count); + Assert.Equal(1, fixture.DeliveryCount); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.FaultCount); + } + + [Fact] + public async Task CancellationBeforeMutation_QueuesNoWrite() + { + using var cancellation = new CancellationTokenSource(); + using var fixture = new OutboxFixture( + token => + { + cancellation.Cancel(); + return ValueTask.FromException(new OperationCanceledException(token)); + }); + + await Assert.ThrowsAnyAsync( + () => fixture.DeliverWithCancellationAsync(cancellation.Token)); + + Assert.True(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.Equal(0, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.FaultCount); + } + + [Fact] + public async Task UnrelatedDeliveryCancellation_IsPersistedAsTerminalFailure() + { + using var fixture = new OutboxFixture( + _ => ValueTask.FromException( + new OperationCanceledException("Receiver canceled its operation.")), + maxDeliveryAttempts: 1); + + await fixture.DeliverAsync(); + + Assert.False(fixture.Messages.ContainsKey(fixture.MessageId)); + Assert.False(fixture.MessageStates.ContainsKey(fixture.MessageId)); + Assert.Equal(1, fixture.DeadLetters.Count); + Assert.Equal(1, fixture.Manager.WriteCount); + Assert.Equal(0, fixture.Manager.FaultCount); + } + + [Fact] + public async Task RecoveredNonemptyOutboxAndNewIntent_AcquireOwnerBeforeCapture() + { + var jobs = new BlockingJobManager(); + using var fixture = new OutboxFixture(jobManager: jobs); + await fixture.StartAsync(); + var outgoing = fixture.CreateEnvelope(fixture.NextMessageId()); + fixture.Outbox.Send(outgoing); + var preparation = fixture.CommitAsync().AsTask(); + await jobs.WaitUntilScheduledAsync(); + Assert.Equal(2, fixture.Outbox.Count); + Assert.Equal(2, fixture.Outbox.Messages.Count()); + Assert.True(fixture.Outbox.TryGetMessage(outgoing.Envelope.MessageId, out _)); + Assert.Equal(2, fixture.Messages.Count); + Assert.Null(fixture.JobId.Value); + Assert.Null(fixture.Job.Value); + Assert.Equal(0, fixture.JobSequence.Value); + Assert.Equal(0, fixture.Manager.CaptureCount); + Assert.False(preparation.IsCompleted); + jobs.Release(); + await preparation; + Assert.Equal(2, fixture.Messages.Count); + Assert.Equal(jobs.OwnershipId, fixture.JobId.Value); + Assert.Equal("replacement", fixture.Job.Value?.Id); + Assert.Equal("test", fixture.Job.Value?.ShardId); + Assert.Equal(1, fixture.JobSequence.Value); + Assert.Equal(1, fixture.Manager.CaptureCount); + Assert.Equal(0, fixture.PendingMessageCount); + await fixture.RunRegisteredTimerAtAsync(0); + Assert.Equal(1, fixture.Manager.CaptureCount); + using var recovered = fixture.Recreate(); + Assert.Equal(2, recovered.Messages.Count); + Assert.Equal(fixture.Job.Value?.Id, recovered.Job.Value?.Id); + Assert.Equal(fixture.Job.Value?.ShardId, recovered.Job.Value?.ShardId); + } + + [Fact] + public async Task RecoveredNonemptyOutboxWithoutNewIntent_PreparesRepairBeforeWrite() + { + var jobs = new RecordingJobManager(); + using var fixture = new OutboxFixture(jobManager: jobs); + await fixture.EnsureJobScheduledAsync(false, TestContext.Current.CancellationToken); + Assert.Single(fixture.Messages); + Assert.Equal(1, jobs.AttemptCount); + Assert.Same(jobs.LastJob, fixture.Job.Value); + Assert.Equal(jobs.LastJob!.Metadata!["orleans.messaging.ownership-id"], fixture.JobId.Value); + Assert.Equal(1, fixture.Manager.CaptureCount); + } + + [Fact] + public async Task ConcurrentWrites_ShareOwnerForSynchronousStaging() + { + var jobs = new BlockingJobManager(); + using var fixture = new OutboxFixture(hasMessage: false, jobManager: jobs); + fixture.Outbox.Send(fixture.Envelope); + var first = fixture.CommitAsync().AsTask(); + await jobs.WaitUntilScheduledAsync(); + var late = fixture.CreateEnvelope(fixture.NextMessageId()); + fixture.Outbox.Send(late); + { var newEnvelope = fixture.CreateEquivalentEnvelope(); fixture.Outbox.Send(newEnvelope); } + var second = fixture.CommitAsync().AsTask(); + Assert.Equal(2, fixture.Outbox.Count); + Assert.Equal(2, fixture.Outbox.Messages.Count()); + Assert.Equal(2, fixture.Messages.Count); + jobs.Release(); + await first; + await second; + Assert.Equal(2, fixture.Outbox.Count); + await fixture.CommitAsync(); + Assert.Equal(2, fixture.Messages.Count); + Assert.Equal(0, fixture.PendingMessageCount); + using var captured = fixture.Recreate(); + Assert.Equal(2, captured.Outbox.Count); + Assert.True(captured.Outbox.TryGetMessage(late.Envelope.MessageId, out _)); + await fixture.CommitAsync(); + Assert.Equal(2, fixture.Messages.Count); + } + + [Fact] + public async Task CancelledCallerWait_PreservesQueuedCaptureAndAcknowledgement() + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.Manager.AfterCapture = () => { entered.TrySetResult(); return release.Task; }; + using var cancellation = new CancellationTokenSource(); + var write = fixture.Manager.WriteStateAsync(cancellation.Token).AsTask(); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => write); + Assert.Equal(1, fixture.PendingMessageCount); + Assert.Equal(1, fixture.Outbox.Count); + Assert.Equal(1, fixture.GetOutboxDepth()); + Assert.Equal(0, fixture.Manager.WriteCompletedCount); + Assert.Equal(0, fixture.Manager.FaultCount); + Assert.True((await fixture.ExecuteJobAsync(fixture.Job.Value!, "before-ack", TestContext.Current.CancellationToken)).IsInProgress); + release.SetResult(); + await fixture.Manager.WaitForIdleAsync(); + Assert.Equal(0, fixture.PendingMessageCount); + Assert.Equal(1, fixture.Outbox.Count); + Assert.Equal(1, fixture.GetOutboxDepth()); + Assert.Equal(1, fixture.Manager.WriteCompletedCount); + using var recovered = fixture.Recreate(); + Assert.Single(recovered.Messages); + Assert.Equal(fixture.Job.Value?.Id, recovered.Job.Value?.Id); + } + + [Fact] + public async Task TerminalFaultDuringScheduling_StopsCaptureWithoutDurableMutation() + { + var jobs = new BlockingJobManager(); + using var fixture = new OutboxFixture(hasMessage: false, jobManager: jobs); + fixture.Outbox.Send(fixture.Envelope); + var preparation = fixture.CommitAsync().AsTask(); + await jobs.WaitUntilScheduledAsync(); + var failure = new IOException("Terminal operation fault."); + fixture.FailPersistence(failure); + var prerequisite = await Assert.ThrowsAsync(() => preparation); + Assert.Same(failure, prerequisite.InnerException); + Assert.Single(fixture.Messages); + Assert.Null(fixture.JobId.Value); + Assert.Null(fixture.Job.Value); + Assert.Equal(0, fixture.JobSequence.Value); + Assert.Equal(0, fixture.Manager.CaptureCount); + Assert.Equal(1, fixture.Manager.FaultCount); + } + + [Fact] + public async Task HealthyOwner_AdditionalSendPreservesExactHandle() + { + var jobs = new RecordingJobManager(); + using var fixture = new OutboxFixture(jobManager: jobs, durableJobId: "owner:1"); + var handle = fixture.Job.Value; + await fixture.SendAsync(fixture.CreateEnvelope(fixture.NextMessageId())); + await fixture.CommitAsync(); + Assert.Equal(2, fixture.Messages.Count); + Assert.Same(handle, fixture.Job.Value); + Assert.Equal("owner:1", fixture.JobId.Value); + Assert.Equal(0, jobs.AttemptCount); + } + + [Fact] + public async Task CompletedTransportThenCanceledBatch_LeavesEarlierOutcomesLocal() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var count = 0; + using var cancellation = new CancellationTokenSource(); + using var fixture = new OutboxFixture(async _ => + { + if (++count == 1) { return DeliveryResult.Accepted(); } + entered.TrySetResult(); + return await release.Task; + }, durableJobId: "owner:1"); + var second = fixture.CreateEnvelope(fixture.NextMessageId()); + await fixture.SendAsync(second); + await fixture.CommitAsync(); + var delivery = fixture.DeliverWithCancellationAsync(cancellation.Token); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await fixture.CommitAsync(); + Assert.Equal(2, fixture.Messages.Count); + cancellation.Cancel(); + release.SetResult(DeliveryResult.Accepted()); + await Assert.ThrowsAnyAsync(() => delivery); + Assert.Equal(2, fixture.Messages.Count); + Assert.Equal(2, fixture.Manager.CaptureCount); + using var recovered = fixture.Recreate(); + Assert.Equal(2, recovered.Messages.Count); + } + + [Fact] + public void StandardCollectionsAndSequenceState_PreserveSevenStreams() + { + using var fixture = new OutboxFixture(hasMessage: false); + Assert.Equal(7, fixture.Manager.RegistrationCount); + Assert.Null(fixture.Outbox.GetType().GetMethod("OnWriteFinalizingAsync")); + Assert.Null(fixture.Outbox.GetType().GetMethod("OnWritePreparingAsync")); + Assert.IsAssignableFrom>(fixture.PrimaryState); + } + + [Theory] + [InlineData(1)] + [InlineData(64)] + [InlineData(256)] + public async Task Depth_SendAndCountUseConstantDictionaryWorkPerIntent(int batchSize) + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + await fixture.StartAsync(); + var containsCalls = fixture.Messages.ContainsKeyCalls; + var lookupCalls = fixture.Messages.TryGetValueCalls; + for (var i = 0; i < batchSize; i++) + { + await fixture.SendAsync(fixture.CreateEnvelope(fixture.NextMessageId())); + Assert.Equal(i + 2, fixture.Outbox.Count); + } + + Assert.Equal(0, fixture.Messages.ContainsKeyCalls - containsCalls); + Assert.Equal(batchSize, fixture.Messages.TryGetValueCalls - lookupCalls); + Assert.Equal(batchSize + 1, fixture.GetOutboxDepth()); + Assert.Equal(batchSize, fixture.PendingMessageCount); + Assert.Equal(batchSize + 1, fixture.Messages.Count); + await fixture.CommitAsync(); + Assert.Equal(batchSize + 1, fixture.Outbox.Count); + Assert.Equal(batchSize + 1, fixture.GetOutboxDepth()); + Assert.Equal(0, fixture.PendingMessageCount); + } + + [Fact] + public async Task Depth_FinalizedOverlapAndLateIntentsRemainDistinctThroughAcknowledgement() + { + using var fixture = new OutboxFixture(hasMessage: false); + fixture.Outbox.Send(fixture.Envelope); + var duringPreparation = fixture.CreateEnvelope(fixture.NextMessageId()); + var afterCapture = fixture.CreateEnvelope(fixture.NextMessageId()); + fixture.Manager.AfterCapture = async () => + { + Assert.Equal(2, fixture.Outbox.Count); + Assert.Equal(2, fixture.Outbox.Messages.Count()); + Assert.Equal(2, fixture.GetOutboxDepth()); + Assert.Equal(2, fixture.PendingMessageCount); + Assert.Equal(2, fixture.Messages.Count); + await fixture.SendAsync(fixture.CreateEquivalentEnvelope()); + await Assert.ThrowsAsync(() => { var newEnvelope = fixture.CreateConflictingEnvelope(); return fixture.SendAsync(newEnvelope); }); + await fixture.SendAsync(afterCapture); + Assert.Equal(3, fixture.Outbox.Count); + Assert.Equal(3, fixture.GetOutboxDepth()); + }; + await fixture.SendAsync(duringPreparation); + Assert.Equal(2, fixture.Outbox.Count); + Assert.Equal(2, fixture.GetOutboxDepth()); + Assert.Equal(2, fixture.Messages.Count); + await fixture.CommitAsync(); + + Assert.Equal(3, fixture.Outbox.Count); + Assert.Equal(3, fixture.Outbox.Messages.Count()); + Assert.Equal(3, fixture.GetOutboxDepth()); + Assert.False(fixture.IsPending(fixture.MessageId)); + Assert.False(fixture.IsPending(duringPreparation.Envelope.MessageId)); + Assert.True(fixture.IsPending(afterCapture.Envelope.MessageId)); + using var captured = fixture.Recreate(); + Assert.Equal(2, captured.Outbox.Count); + fixture.Manager.AfterCapture = null; + await fixture.CommitAsync(); + Assert.Equal(3, fixture.Outbox.Count); + Assert.Equal(3, fixture.GetOutboxDepth()); + Assert.Equal(0, fixture.PendingMessageCount); + using var recovered = fixture.Recreate(); + await recovered.StartAsync(); + Assert.Equal(3, recovered.Outbox.Count); + Assert.Equal(3, recovered.GetOutboxDepth()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Depth_FailedCaptureRemovesMetricContributionAndFreshReplayUsesDurableOutcome(bool committed) + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + var late = fixture.CreateEnvelope(fixture.NextMessageId()); + var failure = new IOException("Outbox depth append failure."); + if (committed) + { + fixture.Manager.FailAfterNextWrite(failure); + } + else + { + fixture.Manager.FailNextWrite(failure); + } + fixture.Manager.AfterCapture = async () => + { + await fixture.SendAsync(late); + Assert.Equal(2, fixture.Outbox.Count); + Assert.Equal(2, fixture.GetOutboxDepth()); + }; + + Assert.Same(failure, await Assert.ThrowsAsync(() => fixture.CommitAsync().AsTask())); + Assert.Equal(2, fixture.Outbox.Count); + Assert.Equal(2, fixture.Outbox.Messages.Count()); + Assert.Equal(0, fixture.GetOutboxDepth()); + Assert.Equal(2, fixture.PendingMessageCount); + Assert.Equal(0, fixture.Manager.WriteCompletedCount); + await fixture.StopAsync(); + Assert.Equal(0, fixture.GetOutboxDepth()); + using var recovered = fixture.Recreate(); + await recovered.StartAsync(); + Assert.Equal(committed ? 1 : 0, recovered.Outbox.Count); + Assert.Equal(committed ? 1 : 0, recovered.GetOutboxDepth()); + Assert.False(recovered.Outbox.TryGetMessage(late.Envelope.MessageId, out _)); + } + + [Theory] + [InlineData(DeliveryStatus.Accepted)] + [InlineData(DeliveryStatus.Duplicate)] + [InlineData(DeliveryStatus.DeadLettered)] + [InlineData(DeliveryStatus.Backpressured)] + [InlineData(DeliveryStatus.HandlerNotFound)] + public async Task Depth_DeliveryOutcomesAdjustOnlyRemovedMessages(DeliveryStatus status) + { + var result = status switch + { + DeliveryStatus.Accepted => DeliveryResult.Accepted(), + DeliveryStatus.Duplicate => DeliveryResult.Duplicate(), + DeliveryStatus.DeadLettered => DeliveryResult.DeadLettered("rejected"), + DeliveryStatus.Backpressured => DeliveryResult.Backpressured(), + DeliveryStatus.HandlerNotFound => DeliveryResult.HandlerNotFound(), + _ => throw new ArgumentOutOfRangeException(nameof(status)) + }; + using var fixture = new OutboxFixture(_ => ValueTask.FromResult(result), + maxDeliveryAttempts: status == DeliveryStatus.HandlerNotFound ? 1 : 3, durableJobId: "owner:1"); + var outgoing = fixture.CreateEnvelope(fixture.NextMessageId()); + await fixture.SendAsync(outgoing); + Assert.Equal(2, fixture.GetOutboxDepth()); + var expected = status == DeliveryStatus.Backpressured ? 2 : 1; + fixture.Manager.AfterCapture = () => + { + Assert.Equal(expected, fixture.Outbox.Count); + Assert.Equal(expected, fixture.Outbox.Messages.Count()); + Assert.Equal(expected, fixture.GetOutboxDepth()); + Assert.True(fixture.IsPending(outgoing.Envelope.MessageId)); + return Task.CompletedTask; + }; + await fixture.DeliverAsync(); + + Assert.Equal(1, fixture.DeliveryCount); + Assert.Equal(expected, fixture.Outbox.Count); + Assert.Equal(expected, fixture.GetOutboxDepth()); + Assert.Equal(0, fixture.PendingMessageCount); + Assert.Equal(status == DeliveryStatus.HandlerNotFound ? 1 : 0, fixture.DeadLetters.Count); + using var recovered = fixture.Recreate(); + await recovered.StartAsync(); + Assert.Equal(expected, recovered.Outbox.Count); + Assert.Equal(expected, recovered.GetOutboxDepth()); + } + + [Fact] + public async Task Depth_QuiescentDeleteDiscardsLocalIntentsAndStartsNewEpoch() + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + await fixture.SendAsync(fixture.CreateEnvelope(fixture.NextMessageId())); + var epoch = fixture.GetOwnershipEpoch(); + Assert.Equal(2, fixture.GetOutboxDepth()); + await fixture.DeleteAsync(TestContext.Current.CancellationToken); + + Assert.Equal(0, fixture.Outbox.Count); + Assert.Empty(fixture.Outbox.Messages); + Assert.Equal(0, fixture.GetOutboxDepth()); + Assert.NotEqual(epoch, fixture.GetOwnershipEpoch()); + using var fresh = fixture.Recreate(); + await fresh.SendAsync(fresh.Envelope); + await fresh.CommitAsync(); + Assert.Equal(1, fresh.Outbox.Count); + Assert.Equal(1, fresh.GetOutboxDepth()); + Assert.Equal(0, fresh.PendingMessageCount); + } + + [Fact] + public async Task Depth_StopIsIdempotentAndFreshRecoveryReactivatesOnlyPersistedMessages() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + await fixture.StartAsync(); + Assert.Equal(1, fixture.Outbox.Count); + Assert.Equal(1, fixture.GetOutboxDepth()); + await fixture.SendAsync(fixture.CreateEnvelope(fixture.NextMessageId())); + Assert.Equal(2, fixture.GetOutboxDepth()); + await fixture.StopAsync(); + Assert.Equal(2, fixture.Outbox.Count); + Assert.Equal(0, fixture.GetOutboxDepth()); + await fixture.StopAsync(); + Assert.Equal(0, fixture.GetOutboxDepth()); + using var recovered = fixture.Recreate(); + await recovered.StartAsync(); + Assert.Equal(1, recovered.Outbox.Count); + Assert.Equal(1, recovered.GetOutboxDepth()); + await recovered.StopAsync(); + Assert.Equal(0, recovered.GetOutboxDepth()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StateWriteAdmission_SeesStagedOutboxCommands(bool eagerCapture) + { + using var fixture = new OutboxFixture(hasMessage: false, eagerCapture: eagerCapture); + await fixture.SendAsync(fixture.Envelope); + await fixture.CommitAsync(); + Assert.Single(fixture.Messages); + Assert.Equal(0, fixture.PendingMessageCount); + Assert.Equal(1, fixture.Outbox.Count); + await fixture.DeliverAsync(); + Assert.Empty(fixture.Messages); + Assert.Equal(0, fixture.Outbox.Count); + Assert.Equal(2, fixture.Manager.CaptureCount); + Assert.Equal(2, fixture.Manager.WriteCompletedCount); + Assert.Null(fixture.Manager.Failure); + } + + + + + + + + + + + + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OwnedWriteOutcome_CompletesCapturedAndQueuedOperationsWithoutStateFaultHooks(bool failWrite) + { + var scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, maxConcurrencyLevel: 1); + try + { + await Task.Factory.StartNew(RunAsync, TestContext.Current.CancellationToken, + TaskCreationOptions.None, scheduler.ExclusiveScheduler).Unwrap(); + } + finally + { + scheduler.Complete(); + await scheduler.Completion; + } + + async Task RunAsync() + { + using var fixture = new OutboxFixture(_ => ValueTask.FromResult(DeliveryResult.Backpressured()), durableJobId: "owner:1"); + await fixture.StartAsync(); + var failure = new IOException("Owned storage append failed."); + if (failWrite) fixture.Manager.FailNextWrite(failure); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.Manager.AfterCapture = () => { entered.SetResult(); return release.Task; }; + var delivery = fixture.DeliverAsync(); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Same(scheduler.ExclusiveScheduler, TaskScheduler.Current); + fixture.Manager.AfterCapture = null; + var repair = fixture.EnsureJobScheduledAsync(true, TestContext.Current.CancellationToken); + Task[] completions = [delivery, repair]; + Assert.Equal(2, completions.Length); + Assert.All(completions, completion => Assert.False(completion.IsCompleted)); + Assert.Equal(0, fixture.GetGate("_deliveryGate").CurrentCount); + Assert.Equal(0, fixture.GetGate("_gate").CurrentCount); + release.SetResult(); + if (failWrite) + { + Assert.Same(failure, await Assert.ThrowsAsync(() => delivery.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken))); + Assert.Same(failure, await Assert.ThrowsAsync(() => repair.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken))); + foreach (var completion in completions) + { + Assert.Same(failure, await Assert.ThrowsAsync(() => completion)); + } + Assert.Same(failure, await Assert.ThrowsAsync(() => fixture.ExecuteJobAsync("owner:1").AsTask())); + Assert.Equal(0, fixture.GetOutboxDepth()); + Assert.Equal(0, fixture.Manager.WriteCompletedCount); + using var recovered = fixture.Recreate(); + Assert.Equal("owner:1", recovered.JobId.Value); + Assert.Single(recovered.Messages); + Assert.Equal(0, recovered.MessageStates.GetProperty(recovered.MessageId, "AttemptCount")); + } + else + { + await Task.WhenAll(delivery, repair).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.All(completions, completion => Assert.True(completion.IsCompletedSuccessfully)); + Assert.Equal(1, fixture.GetOutboxDepth()); + Assert.Equal(2, fixture.Manager.WriteCompletedCount); + using var recovered = fixture.Recreate(); + Assert.Equal(fixture.JobId.Value, recovered.JobId.Value); + Assert.Single(recovered.Messages); + Assert.Equal(1, recovered.MessageStates.GetProperty(recovered.MessageId, "AttemptCount")); + } + Assert.Same(scheduler.ExclusiveScheduler, TaskScheduler.Current); + Assert.Equal(1, fixture.GetGate("_deliveryGate").CurrentCount); + Assert.Equal(1, fixture.GetGate("_gate").CurrentCount); + await fixture.StopAsync().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(0, fixture.GetOutboxDepth()); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StoppedOwnerDelete_DiscardsIntentsAndRejectsStoppedOwnerSend(bool stage) + { + using var fixture = new OutboxFixture(hasMessage: false); + if (stage) fixture.Outbox.Send(fixture.Envelope); + var epoch = fixture.GetOwnershipEpoch(); + Assert.Equal(stage ? 1 : 0, fixture.Outbox.Count); + await fixture.DeleteAsync(TestContext.Current.CancellationToken); + Assert.NotEqual(epoch, fixture.GetOwnershipEpoch()); + Assert.Equal(0, fixture.Outbox.Count); + Assert.Equal(0, fixture.PendingMessageCount); + Assert.Null(fixture.Job.Value); + Assert.ThrowsAny(() => fixture.Outbox.Send(fixture.Envelope)); + Assert.ThrowsAny(() => fixture.Outbox.Send(fixture.Envelope)); + await Assert.ThrowsAnyAsync(() => fixture.StartAsync()); + using var fresh = fixture.Recreate(); + fresh.Outbox.Send(fresh.Envelope); + await fresh.CommitAsync(); + Assert.Single(fresh.Messages); + Assert.Equal(0, fixture.Outbox.Count); + Assert.Equal(0, fixture.GetOutboxDepth()); + } + + [Fact] + public async Task StoppedOwnerDelete_CanceledWaitRetainsCleanupAndNeverResumesAdmission() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + var owner = fixture.Job.Value!; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.Manager.BeforeDelete = () => { entered.SetResult(); return release.Task; }; + using var cancellation = new CancellationTokenSource(); + var deleting = fixture.DeleteAsync(cancellation.Token).AsTask(); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => deleting); + Assert.False(fixture.DeleteCompletion.IsCompleted); + Assert.ThrowsAny(() => fixture.Outbox.Send(fixture.Envelope)); + await Assert.ThrowsAnyAsync(() => fixture.ExecuteJobAsync(owner, "during-delete", TestContext.Current.CancellationToken).AsTask()); + Assert.Single(fixture.Messages); + release.SetResult(); + await fixture.DeleteCompletion.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Empty(fixture.Messages); + await Assert.ThrowsAnyAsync(() => fixture.StartAsync()); + using var fresh = fixture.Recreate(); + fresh.Outbox.Send(fresh.Envelope); + await fresh.CommitAsync(); + Assert.Single(fresh.Messages); + } + + [Fact] + public async Task StoppedOwnerDelete_WaitsForActualCallerWriteBeforeDeleting() + { + using var fixture = new OutboxFixture(hasMessage: false); + await fixture.SendAsync(fixture.Envelope); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.Manager.AfterCapture = () => { entered.SetResult(); return release.Task; }; + var writing = fixture.CommitAsync().AsTask(); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var deleting = fixture.DeleteAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(deleting.IsCompleted); + Assert.Equal(0, fixture.Manager.DeleteCount); + Assert.Equal(1, fixture.PendingMessageCount); + Assert.Equal(1, fixture.Outbox.Count); + release.SetResult(); + await writing; + await deleting.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, fixture.Manager.WriteCompletedCount); + Assert.Equal(1, fixture.Manager.DeleteCount); + Assert.Empty(fixture.Messages); + Assert.Equal(0, fixture.PendingMessageCount); + } + + [Fact] + public async Task StoppedOwnerDelete_StorageFailurePreservesCauseWithoutRevivingFeature() + { + using var fixture = new OutboxFixture(durableJobId: "owner:1"); + await fixture.StartAsync(); + var failure = new IOException("Deletion storage failed."); + fixture.Manager.BeforeDelete = () => Task.FromException(failure); + Assert.Same(failure, await Assert.ThrowsAsync(() => fixture.DeleteAsync(TestContext.Current.CancellationToken).AsTask())); + Assert.Same(failure, fixture.Manager.Failure); + Assert.ThrowsAny(() => fixture.Outbox.Send(fixture.Envelope)); + await Assert.ThrowsAnyAsync(() => fixture.ExecuteJobAsync("owner:1").AsTask()); + Assert.Single(fixture.Messages); + Assert.Equal(0, fixture.GetOutboxDepth()); + using var recovered = fixture.Recreate(); + Assert.Single(recovered.Messages); + Assert.Equal("owner:1", recovered.JobId.Value); + await fixture.StopAsync(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Stop_DrainsOwnedMaintenanceWriteThroughActualOutcome(bool failWrite) + { + var clock = new FakeTimeProvider(); + using var fixture = new OutboxFixture(_ => ValueTask.FromResult(DeliveryResult.HandlerNotFound()), + maxDeliveryAttempts: 1, durableJobId: "owner:1", jobTimeProvider: clock); + await fixture.DeliverAsync(); + Assert.Equal(1, fixture.DeadLetters.Count); + clock.Advance(TimeSpan.FromDays(31)); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.Manager.AfterCapture = () => { entered.SetResult(); return release.Task; }; + var failure = new IOException("Maintenance write failed."); + if (failWrite) fixture.Manager.FailNextWrite(failure); + var maintenance = fixture.StartAsync(); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var stop = fixture.StopAsync(); + try + { + Assert.False(stop.IsCompleted); + Assert.False(maintenance.IsCompleted); + Assert.ThrowsAny(() => fixture.Outbox.Send(fixture.Envelope)); + Assert.Equal(0, fixture.GetOutboxDepth()); + } + finally + { + release.TrySetResult(); + } + if (failWrite) + { + Assert.Same(failure, await Assert.ThrowsAsync(() => maintenance.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken))); + } + else + { + await maintenance.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + } + await stop.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + using var recovered = fixture.Recreate(); + Assert.Equal(failWrite ? 1 : 0, recovered.DeadLetters.Count); + } + + private sealed class ExceptionLogger(List exceptions) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (exception is not null) exceptions.Add(exception); + } + } + + private sealed class OutboxFixture : IDisposable + { + private static readonly Assembly MessagingAssembly = typeof(IOutbox).Assembly; + private readonly IOutbox _outbox; + private readonly MethodInfo _deliverMethod; + private readonly Instrument _outboxDepthInstrument; + private readonly FieldInfo _pendingMessageIdsField; + private readonly ServiceProvider _services; + private readonly List _timerArms = []; + public Action? ArmObserved { get; set; } + + public OutboxFixture( + Func>? deliver = null, + int maxDeliveryAttempts = 3, + Exception? writeException = null, + bool hasMessage = true, + int missingDependency = -1, + ILocalDurableJobManager? jobManager = null, + ITimerRegistry? timerRegistry = null, + TimeProvider? jobTimeProvider = null, + TimeSpan? backpressureRetryDelay = null, + string? durableJobId = null, + bool hasDurableJobHandle = true, + bool loopback = false, + TestStorage? storage = null, + OutboxMessage? envelope = null, + bool eagerCapture = false, + TimeSpan? idleRetirementGracePeriod = null, + int batchSize = 8, + IJournaledStateHook? hook = null) + { + _services = CreateServices(writeException, eagerCapture); + Manager = (TestStateManager)_services.GetRequiredService(); + if (hook is not null) Manager.Hooks.Add(hook); + MessageId = envelope?.Envelope.MessageId ?? HierarchicalKey.Create("test", "outbox", "command", "0"); + SenderId = GrainId.Create("sender", "1"); + ReceiverId = loopback ? SenderId : GrainId.Create("receiver", "1"); + Envelope = envelope ?? CreateEnvelope(MessageId); + + var delivery = deliver ?? (_ => ValueTask.FromResult(DeliveryResult.Accepted())); + var inbox = Substitute.For(); + inbox.DeliverAsync(Arg.Any(), Arg.Any()) + .Returns(call => + { + DeliveryCount++; + return delivery(call.ArgAt(1)); + }); + var grainFactory = Substitute.For(); + grainFactory.GetGrain(Arg.Any()).Returns(_ => { ProxyAcquisitionCount++; return inbox; }); + var grainContext = Substitute.For(); + grainContext.GrainId.Returns(SenderId); + var binder = Substitute.For(); + binder.GetExtension().Returns(inbox); + grainContext.GetComponent(typeof(IGrainExtensionBinder)).Returns(binder); + grainContext.GrainInstance.Returns(new object()); + grainContext.ObservableLifecycle.Returns(Substitute.For()); + + TimerRegistry = timerRegistry ?? Substitute.For(); + TimerRegistry.RegisterGrainTimer(default!, default(Func)!, default!, default) + .ReturnsForAnyArgs(call => + { + var arguments = call.Args(); + var timer = Substitute.For(); + timer.When(value => value.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan)).Do(_ => + { + _timerArms.Add(arguments); + ArmObserved?.Invoke(); + }); + return timer; + }); + JobManager = jobManager ?? new RecordingJobManager(); + var outboxType = GetInternalType("Orleans.Messaging.Outbox"); + var instrumentsType = GetInternalType("Orleans.Messaging.MessagingInstruments"); + var instruments = instrumentsType + .GetMethod("CreateForDirectConstruction", BindingFlags.Static | BindingFlags.NonPublic)! + .Invoke(null, null)!; + var depthTracker = instrumentsType + .GetField("_outboxDepth", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(instruments)!; + _outboxDepthInstrument = (Instrument)depthTracker.GetType() + .GetField("_gauge", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(depthTracker)!; + var pumpResults = Activator.CreateInstance( + GetInternalType("Orleans.Messaging.MessagingPumpResults"), + nonPublic: true)!; + var logger = Activator.CreateInstance(typeof(ExceptionLogger<>).MakeGenericType(outboxType), LoggedExceptions)!; + + object?[] states = + [ + _services.GetRequiredKeyedService>(BootstrapOutboxServices.StateNames[0]), + _services.GetRequiredKeyedService(typeof(IDurableDictionary<,>).MakeGenericType(typeof(HierarchicalKey), GetInternalType("Orleans.Messaging.OutboxMessageState")), BootstrapOutboxServices.StateNames[1]), + _services.GetRequiredKeyedService(typeof(IDurableDictionary<,>).MakeGenericType(typeof(HierarchicalKey), GetInternalType("Orleans.Messaging.OutboxDeadLetter")), BootstrapOutboxServices.StateNames[2]), + _services.GetRequiredKeyedService>(BootstrapOutboxServices.StateNames[3]), + _services.GetRequiredKeyedService>(BootstrapOutboxServices.StateNames[4]), + _services.GetRequiredKeyedService>(BootstrapOutboxServices.StateNames[5]), + _services.GetRequiredKeyedService>("orleans-binary") + ]; + if (missingDependency >= 0) states[missingDependency] = null; + try + { + _outbox = (IOutbox)Activator.CreateInstance( + outboxType, + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, + binder: null, + [ + Manager, + grainFactory, + grainContext, + _services.GetRequiredService>(), + TimerRegistry, + logger, + instruments, + JobManager, + Substitute.For(), + pumpResults, + jobTimeProvider ?? TimeProvider.System, + Options.Create( + new InboxOptions + { + BackpressureRetryDelay = backpressureRetryDelay ?? TimeSpan.FromMilliseconds(1), + MaxOutboxRetryAge = TimeSpan.FromMinutes(5), + MaxDeliveryAttempts = maxDeliveryAttempts, + OutboxBatchSize = batchSize, + OutboxIdleRetirementGracePeriod = idleRetirementGracePeriod ?? TimeSpan.Zero + }), + .. states + ], + culture: null)!; + } + catch (TargetInvocationException exception) when (exception.InnerException is not null) + { + _services.Dispose(); + ExceptionDispatchInfo.Capture(exception.InnerException).Throw(); + throw; + } + Messages = new TestDurableDictionary(Manager.GetState>("__orleans.messaging.outbox"), _services.GetRequiredService>>()); + MessageStates = WrapInternalDictionary("__orleans.messaging.outbox-message-state", "Orleans.Messaging.OutboxMessageState"); + DeadLetters = WrapInternalDictionary("__orleans.messaging.outbox-dead-letters", "Orleans.Messaging.OutboxDeadLetter"); + JobId = new(Manager.GetState>("__orleans.messaging.outbox-job-id")); + Job = new(Manager.GetState>("__orleans.messaging.outbox-job-handle")); + CompletedJobId = new(Manager.GetState>("__orleans.messaging.outbox-completed-job-id")); + JobSequence = new(Manager.GetState>("__orleans.messaging.outbox-job-sequence")); + outboxType.GetField("_messages", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(_outbox, Messages); + outboxType.GetField("_messageStates", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(_outbox, MessageStates.Instance); + Manager.PrepareBindings(); + if (durableJobId is not null) + { + JobId.Value = durableJobId; + if (hasDurableJobHandle) Job.Value = CreateJob($"job-{durableJobId}", durableJobId); + } + if (hasMessage) + { + Messages.Add(MessageId, Envelope); + var messageState = CreateInternal("Orleans.Messaging.OutboxMessageState"); + messageState.GetType().GetProperty("EnqueuedAt")!.SetValue(messageState, TimeProvider.System.GetUtcNow()); + MessageStates.Add(MessageId, messageState); + } + Manager.BindAdapters([Messages, MessageStates, DeadLetters, JobId, Job, CompletedJobId, JobSequence], storage); + Manager.ExternalOwnerCommitted = () => + { + outboxType.GetField("_durableOwnershipId", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(_outbox, JobId.Value); + outboxType.GetField("_durableJob", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(_outbox, Job.Value); + outboxType.GetField("_durableCompletedJobId", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(_outbox, CompletedJobId.Value); + }; + Manager.InitializeAsync(TestContext.Current.CancellationToken).GetAwaiter().GetResult(); + + _deliverMethod = outboxType.GetMethod("DeliverPendingMessagesAsync")!; + _pendingMessageIdsField = outboxType.GetField("_pendingMessages", BindingFlags.Instance | BindingFlags.NonPublic)!; + } + + private bool _disposed; + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _services.Dispose(); + } + + private static ServiceProvider CreateServices(Exception? writeException, bool eagerCapture) + { + var services = new ServiceCollection().AddSerializer().AddLogging(); + services.AddKeyedSingleton(JournalingTimeProviderNames.Journaling, TimeProvider.System); + services.Configure(options => options.JournalFormatKey = "orleans-binary"); + var silo = Substitute.For(); + silo.Services.Returns(services); + silo.AddJournaling(); + var storage = new ControlledJournalStorageProvider(); + storage.Configure(Options.Create(new JournaledStateManagerOptions { JournalFormatKey = "orleans-binary" })); + services.AddSingleton(storage); + services.AddSingleton(provider => + new TestStateManager(provider.GetRequiredKeyedService("orleans-binary"), writeException, eagerCapture)); + return services.BuildServiceProvider(); + } + + private UntypedDurableDictionary WrapInternalDictionary(string name, string valueType) + { + var type = typeof(TestDurableDictionary<,>).MakeGenericType(typeof(HierarchicalKey), GetInternalType(valueType)); + var values = typeof(Dictionary<,>).MakeGenericType(typeof(HierarchicalKey), GetInternalType(valueType)); + return new(Activator.CreateInstance(type, Manager.GetState(name), _services.GetRequiredService(typeof(Serializer<>).MakeGenericType(values)))!); + } + public IOutbox Outbox => _outbox; + public IStateMachine PrimaryState => Manager.GetState("__orleans.messaging.outbox"); + private object PumpResults => _outbox.GetType().GetField("_pumpResults", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(_outbox)!; + public IDictionary PumpEntries => (IDictionary)PumpResults.GetType().GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(PumpResults)!; + + public object AddInboxPumpEntry(CancellationToken cancellationToken) + { + var generation = (long)_outbox.GetType().GetField("_stateGeneration", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(_outbox)!; + var key = Activator.CreateInstance(GetInternalType("Orleans.Messaging.MessagingPumpExecutionKey"), + "orleans.messaging.inbox-drain", "inbox-job", "inbox-run", generation)!; + object?[] arguments = [key, cancellationToken, null]; + Assert.True((bool)PumpResults.GetType().GetMethod("TryStart")!.Invoke(PumpResults, arguments)!); + return key; + } + + public Task RunRegisteredTimerAtAsync(int index) => + RunTimerAsync(_timerArms[index], TestContext.Current.CancellationToken); + + public object ArmState(int index) => _timerArms[index][2]!; + public string[] ArmedTimerNames => _timerArms.Select(static arguments => arguments[2]!.GetType().Name).ToArray(); + public string[] TimerRegistrationNames => TimerRegistry.ReceivedCalls() + .Where(static call => call.GetMethodInfo().Name == "RegisterGrainTimer") + .Select(static call => call.GetArguments()[2]!.GetType().Name).ToArray(); + + public Task StopAsync() => ((ILifecycleObserver)_outbox).OnStop(TestContext.Current.CancellationToken); + public OutboxFixture Recreate() => new(storage: Manager.Storage, envelope: Envelope); + + public HierarchicalKey MessageId { get; } + public GrainId SenderId { get; } + public GrainId ReceiverId { get; } + public OutboxMessage Envelope { get; } + public int DeliveryCount { get; private set; } + public int ProxyAcquisitionCount { get; private set; } + public object? LocalField(string name) => _outbox.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(_outbox); + public List LoggedExceptions { get; } = []; + public TestDurableDictionary Messages { get; } + public UntypedDurableDictionary MessageStates { get; } + public UntypedDurableDictionary DeadLetters { get; } + public TestStateManager Manager { get; } + public ILocalDurableJobManager JobManager { get; } + public ITimerRegistry TimerRegistry { get; } + public TestDurableValue JobId { get; } + public TestDurableValue Job { get; } + public TestDurableValue CompletedJobId { get; } + public TestDurableValue JobSequence { get; } + public int PendingMessageCount => GetPendingMessageIds().Count; + + public Task DeliverAsync() => + DeliverWithCancellationAsync(TestContext.Current.CancellationToken); + + public Task DeliverWithCancellationAsync(CancellationToken cancellationToken) => + (Task)_deliverMethod.Invoke(_outbox, [cancellationToken])!; + + public Task EnsureJobScheduledAsync(bool replaceExisting, CancellationToken cancellationToken) => + (Task)_outbox.GetType() + .GetMethod("EnsureJobScheduledAsync", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(_outbox, [replaceExisting, cancellationToken])!; + + public ValueTask ExecuteJobAsync(string ownershipId) => + ExecuteJobAsync(ownershipId, $"job-{ownershipId}", $"run-{ownershipId}", TestContext.Current.CancellationToken); + + public ValueTask ExecuteJobAsync( + DurableJob job, + string runId, + CancellationToken cancellationToken) + { + var context = Substitute.For(); + context.Job.Returns(job); + context.RunId.Returns(runId); + context.DequeueCount.Returns(1); + return (ValueTask)_outbox.GetType() + .GetMethod("ExecuteJobAsync", BindingFlags.Instance | BindingFlags.Public)! + .Invoke(_outbox, [context, cancellationToken])!; + } + + public ValueTask ExecuteJobAsync( + string ownershipId, + string physicalJobId, + string runId, + CancellationToken cancellationToken) + { + var context = Substitute.For(); + context.Job.Returns(CreateJob(physicalJobId, ownershipId)); + context.RunId.Returns(runId); + context.DequeueCount.Returns(1); + return (ValueTask)_outbox.GetType() + .GetMethod("ExecuteJobAsync", BindingFlags.Instance | BindingFlags.Public)! + .Invoke(_outbox, [context, cancellationToken])!; + } + + public DurableJob CreateJobForTest(string physicalJobId, string ownershipId) => + CreateJob(physicalJobId, ownershipId); + + private DurableJob CreateJob(string physicalJobId, string ownershipId) => + new() + { + Id = physicalJobId, + Name = "orleans.messaging.outbox-flush", + DueTime = DateTimeOffset.UnixEpoch, + TargetGrainId = SenderId, + ShardId = $"shard-{physicalJobId}", + Metadata = new Dictionary(StringComparer.Ordinal) + { + ["orleans.messaging.ownership-id"] = ownershipId + } + }; + + public ValueTask ExecuteJobWithoutOwnershipMetadataAsync( + string physicalJobId, + CancellationToken cancellationToken) + { + var context = Substitute.For(); + context.Job.Returns(new DurableJob + { + Id = physicalJobId, + Name = "orleans.messaging.outbox-flush", + DueTime = DateTimeOffset.UnixEpoch, + TargetGrainId = SenderId, + ShardId = "test" + }); + context.RunId.Returns($"run-{physicalJobId}"); + context.DequeueCount.Returns(1); + return (ValueTask)_outbox.GetType() + .GetMethod("ExecuteJobAsync", BindingFlags.Instance | BindingFlags.Public)! + .Invoke(_outbox, [context, cancellationToken])!; + } + + public Task GetPendingBatchDrainTask() + { + var batch = _outbox.GetType().GetField("_pendingDeliveryBatch", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(_outbox)!; + return (Task)batch.GetType().GetField("_drain", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(batch)!; + } + + public (CancellationTokenSource Source, Task[] Attempts) GetPendingBatchState() + { + var batch = _outbox.GetType().GetField("_pendingDeliveryBatch", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_outbox)!; + var source = (CancellationTokenSource)batch.GetType().GetProperty("Cancellation")!.GetValue(batch)!; + var collection = (IEnumerable?)batch.GetType().GetProperty("Attempts")!.GetValue(batch); + var single = (Task?)batch.GetType().GetProperty("SingleAttempt")!.GetValue(batch); + var attempts = collection is null ? new[] { Assert.IsAssignableFrom(single) } : collection.Cast().ToArray(); + return (source, attempts); + } + + public SemaphoreSlim GetGate(string fieldName) => + (SemaphoreSlim)_outbox.GetType() + .GetField(fieldName, BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_outbox)!; + + public Task StartAsync() => + StartWithCancellationAsync(TestContext.Current.CancellationToken); + + public Task StartWithCancellationAsync(CancellationToken cancellationToken) => + ((ILifecycleObserver)_outbox).OnStart(cancellationToken); + + public string GetOwnershipEpoch() => + (string)_outbox.GetType() + .GetField("_ownershipEpoch", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(_outbox)!; + + public Task RunRegisteredTimerAsync(CancellationToken timerCancellationToken) => + RunTimerAsync(_timerArms[^1], timerCancellationToken); + + public async Task RunRegisteredTimersAsync() + { + foreach (var arguments in _timerArms.ToArray()) + { + await RunTimerAsync(arguments, TestContext.Current.CancellationToken); + } + } + + private static async Task RunTimerAsync(object?[] arguments, CancellationToken timerCancellationToken) + { + var callback = (Delegate)arguments[1]!; + await (Task)callback.DynamicInvoke(arguments[2], timerCancellationToken)!; + } + + public void ActivateMetrics() => + _outbox.GetType() + .GetMethod("EnsureMetricsActive", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(_outbox, null); + + public long GetOutboxDepth() + { + long? result = null; + using var listener = new MeterListener + { + InstrumentPublished = (instrument, meterListener) => + { + if (ReferenceEquals(instrument, _outboxDepthInstrument)) + { + meterListener.EnableMeasurementEvents(instrument); + } + } + }; + listener.SetMeasurementEventCallback( + (instrument, measurement, tags, state) => result = measurement); + listener.Start(); + listener.RecordObservableInstruments(); + return result ?? throw new InvalidOperationException("The outbox depth gauge did not report a value."); + } + + public Task SendAsync(OutboxMessage envelope) + { + _outbox.Send(envelope); + return Task.CompletedTask; + } + + public Task DeleteCompletion { get; private set; } = Task.CompletedTask; + + public async ValueTask DeleteAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + DeleteCompletion = DeleteOwnedAsync(); + await DeleteCompletion.WaitAsync(cancellationToken); + + async Task DeleteOwnedAsync() + { + try + { + await StopAsync(); + await Manager.WaitForIdleAsync(); + await Manager.DeleteStateAsync(CancellationToken.None); + } + finally + { + Dispose(); + } + } + } + + public async ValueTask CommitAsync() + { + try + { + await Manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + catch (Exception exception) when (exception is not JournaledStatePreCommitException and not JournaledStatePostCommitException) + { + FailPersistence(exception); + throw; + } + } + + public void FailPersistence(Exception exception) + { + Manager.Fail(exception); + _outbox.GetType().GetMethod("FailPersistence", BindingFlags.Instance | BindingFlags.NonPublic)! + .CreateDelegate>(_outbox)(exception); + } + + private int _nextMessageId; + public HierarchicalKey NextMessageId() => HierarchicalKey.Create( + "test", "outbox", "command", (++_nextMessageId).ToString(CultureInfo.InvariantCulture)); + + public bool IsPending(HierarchicalKey messageId) => GetPendingMessageIds().Contains(messageId); + + public OutboxMessage CreateEquivalentEnvelope() => CreateEnvelope(MessageId); + + public OutboxMessage CreateConflictingEnvelope() => CreateEnvelope(MessageId, routeKey: "conflict"); + + public OutboxMessage CreateEnvelope( + HierarchicalKey messageId, + string routeKey = "test", + GrainId? senderId = null) => OutboxTestMessages.Create(messageId, ReceiverId, + TestApplicationProtocol.Encode(_services.GetRequiredService(), new TestApplicationMessage(routeKey, 1)), + senderId ?? SenderId, routeKey, _services.GetRequiredService>()); + + private IDictionary GetPendingMessageIds() => + (IDictionary)_pendingMessageIdsField.GetValue(_outbox)!; + + private static Type GetInternalType(string typeName) => + MessagingAssembly.GetType(typeName, throwOnError: true)!; + + private static object CreateInternal(string typeName) => + Activator.CreateInstance(GetInternalType(typeName), nonPublic: true)!; + } + + private interface ITestDurableState + { + long Version { get; } + object Capture(); + void Restore(object snapshot); + } + + private sealed class UntypedDurableDictionary(object instance) : ITestDurableState + { + private readonly Type _type = instance.GetType(); + + public object Instance { get; } = instance; + public int Count => (int)_type.GetProperty("Count")!.GetValue(Instance)!; + public long Version => ((ITestDurableState)Instance).Version; + + + public void Add(HierarchicalKey key, object value) => + _type.GetMethod("Add", [typeof(HierarchicalKey), value.GetType()])!.Invoke(Instance, [key, value]); + + public bool ContainsKey(HierarchicalKey key) => + (bool)_type.GetMethod("ContainsKey")!.Invoke(Instance, [key])!; + + public bool Remove(HierarchicalKey key) => + (bool)_type.GetMethod("Remove", [typeof(HierarchicalKey)])!.Invoke(Instance, [key])!; + + public T GetProperty(HierarchicalKey key, string propertyName) + { + var value = _type.GetProperty("Item")!.GetValue(Instance, [key])!; + return (T)value.GetType().GetProperty(propertyName)!.GetValue(value)!; + } + + public object Capture() => ((ITestDurableState)Instance).Capture(); + public void Restore(object snapshot) => ((ITestDurableState)Instance).Restore(snapshot); + } + + private sealed class TestStorage(object[] snapshots) + { + public object[] Snapshots { get; set; } = snapshots; + } + + private sealed class TestStateManager : IJournaledStateManager, IDisposable + { + private readonly Dictionary _states = new(StringComparer.Ordinal); + private readonly JournalBufferWriter _writer; + private readonly bool _eagerCapture; + private ITestDurableState[] _adapters = []; + private Task _tail = Task.CompletedTask; + private ExceptionDispatchInfo? _failure; + private Exception? _nextWriteException; + private Exception? _nextPostWriteException; + private Action? _afterNextWrite; + private bool _initialized; + + public TestStateManager(IJournalFormat format, Exception? writeException, bool eagerCapture) + { + _writer = format.CreateWriter(); + _nextWriteException = writeException; + _eagerCapture = eagerCapture; + } + + public IList Hooks { get; private set; } = []; + public TestStorage Storage { get; private set; } = null!; + public int RegistrationCount => _states.Count; + public int WriteCount { get; private set; } + public int CaptureCount { get; private set; } + public int WriteCompletedCount { get; private set; } + public int FaultCount { get; private set; } + public int DeleteCount { get; private set; } + public Exception? Failure => _failure?.SourceException; + public Func? AfterCapture { get; set; } + public Action? ExternalOwnerCommitted { get; set; } + public Func? BeforeDelete { get; set; } + + public void BindAdapters(ITestDurableState[] adapters, TestStorage? storage) + { + _adapters = adapters; + Storage = storage ?? new TestStorage(adapters.Select(static state => state.Capture()).ToArray()); + } + + public T GetState(string name) => (T)_states[name]; + public void RegisterStateMachine(string name, IStateMachine state) => _states.Add(name, state); + public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? state) => _states.TryGetValue(name, out state); + + public ValueTask InitializeAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _failure?.Throw(); + if (!_initialized) + { + _initialized = true; + Hooks = Array.AsReadOnly(Hooks.ToArray()); + ResetStates(); + for (var i = 0; i < _adapters.Length; i++) _adapters[i].Restore(Storage.Snapshots[i]); + foreach (var state in _states.Values) state.OnRecoveryCompleted(); + } + return default; + } + + public ValueTask WriteStateAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _failure?.Throw(); + var work = RunWriteAsync(_tail); + _tail = work; + return new(work.WaitAsync(cancellationToken)); + } + + private async Task RunWriteAsync(Task previous) + { + if (!_eagerCapture) await Task.Yield(); + await previous.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + _failure?.Throw(); + try + { + for (var index = Hooks.Count - 1; index >= 0; index--) + { + await Hooks[index].BeforeOperationAsync(JournaledStateOperation.Write, CancellationToken.None); + } + } + catch (Exception exception) + { + throw new JournaledStatePreCommitException(JournaledStateOperation.Write, exception); + } + try + { + WriteCount++; + uint id = 1; + foreach (var state in _states.Values) state.WritePendingEntries(_writer.CreateJournalStreamWriter(new(id++))); + using var buffer = _writer.GetBuffer(); + _writer.Reset(); + var snapshot = _adapters.Select(static state => state.Capture()).ToArray(); + CaptureCount++; + if (AfterCapture is { } afterCapture) await afterCapture(); + if (_nextWriteException is { } exception) + { + _nextWriteException = null; + throw exception; + } + Storage.Snapshots = snapshot; + if (_nextPostWriteException is { } postException) + { + _nextPostWriteException = null; + throw postException; + } + if (buffer.Length > 0) + { + foreach (var state in _states.Values) state.OnWriteCompleted(); + } + WriteCompletedCount++; + var afterWrite = _afterNextWrite; + _afterNextWrite = null; + afterWrite?.Invoke(); + } + catch (Exception exception) + { + Fail(exception); + throw; + } + List? failures = null; + foreach (var hook in Hooks) + { + try { await hook.AfterOperationAsync(JournaledStateOperation.Write, CancellationToken.None); } + catch (Exception exception) { (failures ??= []).Add(exception); } + } + if (failures is not null) throw new JournaledStatePostCommitException(JournaledStateOperation.Write, new AggregateException(failures)); + } + + public void Fail(Exception exception) + { + if (_failure is null) + { + _failure = ExceptionDispatchInfo.Capture(exception); + FaultCount++; + } + } + public Task WaitForIdleAsync() => _tail.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + public void FailNextWrite(Exception exception) => _nextWriteException = exception; + public void FailAfterNextWrite(Exception exception) => _nextPostWriteException = exception; + public void AfterNextWrite(Action action) => _afterNextWrite = action; + public void CommitExternalOwner() + { + WriteCount++; + Storage.Snapshots = _adapters.Select(static state => state.Capture()).ToArray(); + ExternalOwnerCommitted!(); + } + public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _failure?.Throw(); + try + { + if (BeforeDelete is { } beforeDelete) await beforeDelete(); + ResetStates(); + Storage.Snapshots = _adapters.Select(static state => state.Capture()).ToArray(); + DeleteCount++; + } + catch (Exception exception) + { + Fail(exception); + throw; + } + } + public void PrepareBindings() => ResetStates(); + + private void ResetStates() + { + _writer.Reset(); + uint id = 1; + foreach (var state in _states.Values) state.Reset(_writer.CreateJournalStreamWriter(new(id++))); + } + + public void Dispose() => _writer.Dispose(); + } + + private sealed class RecordingJobManager(bool alwaysFail = false) : ILocalDurableJobManager + { + private readonly SemaphoreSlim _attempted = new(0); + private int _attemptCount; + + public int AttemptCount => Volatile.Read(ref _attemptCount); + public DurableJob? LastJob { get; private set; } + + public Task ScheduleJobAsync(ScheduleJobRequest request, CancellationToken cancellationToken) + { + var count = Interlocked.Increment(ref _attemptCount); + _attempted.Release(); + if (alwaysFail) + { + return Task.FromException(new IOException($"Injected scheduling failure {count}.")); + } + + LastJob = new DurableJob + { + Id = $"job-{count}", + Name = request.JobName, + DueTime = request.DueTime, + TargetGrainId = request.Target, + ShardId = "test", + Metadata = request.Metadata + }; + return Task.FromResult(LastJob); + } + + public Task CancelAsync(DurableJob job, CancellationToken cancellationToken) => + Task.FromResult(true); + + public async Task WaitForAttemptCountAsync(int expected) + { + while (AttemptCount < expected) + { + await _attempted.WaitAsync(TimeSpan.FromSeconds(10)); + } + } + } + + private sealed class BlockingJobManager : ILocalDurableJobManager + { + private readonly TaskCompletionSource _attempted = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public string? OwnershipId { get; private set; } + public DurableJob? LastJob { get; private set; } + public bool IgnoreCancellation { get; init; } + public bool ThrowOnCancellation { get; init; } + public Exception? Failure { get; init; } + public int CancellationCallbacks { get; private set; } + + public async Task ScheduleJobAsync( + ScheduleJobRequest request, + CancellationToken cancellationToken) + { + OwnershipId = request.Metadata!["orleans.messaging.ownership-id"]; + _attempted.TrySetResult(); + using var registration = ThrowOnCancellation ? cancellationToken.Register(() => + { + CancellationCallbacks++; + throw new InvalidOperationException("Application cancellation callback failure."); + }) : default; + await _release.Task.WaitAsync(IgnoreCancellation ? CancellationToken.None : cancellationToken); + LastJob = new DurableJob + { + Id = "replacement", + Name = request.JobName, + DueTime = request.DueTime, + TargetGrainId = request.Target, + ShardId = "test", + Metadata = request.Metadata + }; + if (Failure is { } failure) throw failure; + return LastJob; + } + + public Task CancelAsync(DurableJob job, CancellationToken cancellationToken) => + Task.FromResult(true); + + public Task WaitUntilScheduledAsync() => + _attempted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + public void Release() => _release.TrySetResult(); + } + + private sealed class TestDurableValue(IDurableValue inner) : IDurableValue, ITestDurableState + { + + public T? Value + { + get => inner.Value; + set + { + inner.Value = value; + Version++; + } + } + + public long Version { get; private set; } + + public object Capture() => new Snapshot(inner.Value); + + public void Restore(object snapshot) + { + ((IDurableValueCommandHandler)inner).ApplySet(((Snapshot)snapshot).Value!); + Version++; + } + + private sealed record Snapshot(T? Value); + } + + private sealed class TestDurableDictionary + : IDurableDictionary, ITestDurableState + where TKey : notnull + { + private readonly IDurableDictionary _items; + private long _version; + + private readonly Serializer> _serializer; + public TestDurableDictionary(IDurableDictionary inner, Serializer> serializer) + { + _items = inner; + _serializer = serializer; + } + + public TValue this[TKey key] + { + get => _items[key]; + set + { + _items[key] = value; + _version++; + } + } + + public ICollection Keys => _items.Keys; + public ICollection Values => _items.Values; + public int Count => _items.Count; + public bool IsReadOnly => false; + public long Version => _version; + public int ContainsKeyCalls { get; private set; } + public int TryGetValueCalls { get; private set; } + + public void Add(TKey key, TValue value) + { + _items.Add(key, value); + _version++; + } + + public void Add(KeyValuePair item) + { + ((ICollection>)_items).Add(item); + _version++; + } + + public void Clear() + { + if (_items.Count > 0) + { + _items.Clear(); + _version++; + } + } + + public bool Contains(KeyValuePair item) => ((ICollection>)_items).Contains(item); + public bool ContainsKey(TKey key) + { + ContainsKeyCalls++; + return _items.ContainsKey(key); + } + public void CopyTo(KeyValuePair[] array, int arrayIndex) => + ((ICollection>)_items).CopyTo(array, arrayIndex); + public IEnumerator> GetEnumerator() => _items.GetEnumerator(); + public bool Remove(TKey key) + { + if (!_items.Remove(key)) + { + return false; + } + + _version++; + return true; + } + + public bool Remove(KeyValuePair item) + { + if (!((ICollection>)_items).Remove(item)) + { + return false; + } + + _version++; + return true; + } + + public bool TryGetValue(TKey key, out TValue value) + { + TryGetValueCalls++; + return _items.TryGetValue(key, out value!); + } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + object ITestDurableState.Capture() => + _serializer.SerializeToArray(_items.ToDictionary(static pair => pair.Key, static pair => pair.Value)); + + void ITestDurableState.Restore(object snapshot) + { + var handler = (IDurableDictionaryCommandHandler)_items; + var values = _serializer.Deserialize((byte[])snapshot)!; + handler.Reset(values.Count); + foreach (var (key, value) in values) + { + handler.ApplySet(key, value); + } + + _version++; + } + + + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs index de8f35124ec..8e5a1ba38fd 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs @@ -1,6 +1,9 @@ using System.Reflection; +using System.Globalization; using Microsoft.Extensions.DependencyInjection; +using NSubstitute; using Orleans.Concurrency; +using Orleans.DurableJobs; using Orleans.Messaging.Tests.Support; using Orleans.Journaling; using Orleans.Metadata; @@ -17,6 +20,8 @@ namespace Orleans.Messaging.Tests.Functional; [TestArea("Messaging")] public sealed class MessagingGrainTypeConfiguratorTests() : MessagingBehaviorTestBase(new BootstrapClusterFixture()) { + + private BootstrapDeliveryProbe Delivery => ((BootstrapClusterFixture)Fixture).Delivery; private BootstrapProbe Probe => ((BootstrapClusterFixture)Fixture).Probe; private static CancellationToken Cancellation => TestContext.Current.CancellationToken; @@ -42,15 +47,24 @@ public async Task SelectedComposition_BindsOnceAndReplaysFreshScopedState(Type g var envelope = CreateEnvelope(grain); Assert.Equal(DeliveryStatus.Accepted, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); await handler.WaitUntilEnteredAsync(); - var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); Assert.Equal(41, first.Value!.Value); Assert.Equal(1, first.Inbox!.Count); Assert.Equal(0, first.Outbox!.Count); Assert.Empty(GetProcessed(first.Context)); Assert.Equal(0, state.HandlerCalls); + using var preparation = Fixture.JobManagerProbe.BlockNext(BootstrapOutboxServices.JobName); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + handler.Release(); + await preparation.WaitUntilEnteredAsync(); + Assert.Equal(42, first.Value!.Value); + Assert.Equal(0, first.Inbox!.Count); + Assert.Equal(1, first.Outbox!.Count); + Assert.Single(GetProcessed(first.Context)); + Assert.Equal(1, state.HandlerCalls); Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetSuccessCount(BootstrapOutboxServices.JobName, grain.GetGrainId())); var storage = Fixture.Storage.BlockWrite(journal); - handler.Release(); + preparation.Continue(); await storage.WaitUntilEnteredAsync(); Assert.Equal(42, first.Value!.Value); Assert.Equal(0, first.Inbox!.Count); @@ -62,6 +76,10 @@ public async Task SelectedComposition_BindsOnceAndReplaysFreshScopedState(Type g Assert.Equal(0, first.Inbox.Count); Assert.Single(GetProcessed(first.Context)); var output = Assert.Single(first.Outbox.Messages); + var scheduled = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(BootstrapOutboxServices.JobName, grain.GetGrainId())); + var owner = first.Context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.outbox-job-handle").Value; + Assert.Equal(scheduled.Id, owner!.Id); + Assert.Equal(scheduled.ShardId, owner.ShardId); Assert.Equal(1, state.HandlerCalls); Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); Assert.Equal(1, state.HandlerCalls); @@ -99,6 +117,356 @@ public async Task SelectedComposition_BindsOnceAndReplaysFreshScopedState(Type g Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); Assert.Equal(0, recoveredState.HandlerCalls); Assert.Equal(2, Fixture.Storage.GetReadCount(journal)); + + var delivered = Delivery.WaitForOutputAsync(grain.GetGrainId()); + var drained = Delivery.WaitForDrainAsync(grain.GetGrainId()); + Delivery.Release(); + var receipt = await delivered; + await drained; + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(output.Envelope.MessageId, receipt.MessageId); + Assert.Equal(42, receipt.Value); + Assert.Empty(recovered.Outbox.Messages); + var sink = Fixture.Client.GetGrain("capture"); + Assert.Equal(1, await sink.GetMessageCountAsync()); + Assert.Equal(DeliveryStatus.Duplicate, (await sink.AsReference().DeliverAsync(output.ToInboxMessage(), Cancellation)).Status); + Assert.Equal(1, await sink.GetMessageCountAsync()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SynchronousEnvelopeSend_SchedulesBeforeCaptureAndDispatchesAfterAck(bool handlerSend) + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + using var scheduling = Fixture.JobManagerProbe.BlockNext(BootstrapOutboxServices.JobName); + var handler = new SynchronousHandler(observation); + Task operation; + if (handlerSend) + { + await OnOwnerTurnAsync(observation.Context, () => + { + observation.Context.ActivationServices.GetRequiredService().HandlerOverride = handler; + return Task.CompletedTask; + }); + var input = CreateEnvelope(grain); + operation = grain.AsReference().DeliverAsync(input, Cancellation).AsTask(); + } + else operation = grain.SendSynchronousValueAsync(42); + await scheduling.WaitUntilEnteredAsync(); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Assert.Equal(42, observation.Value!.Value); + Assert.Single(observation.Outbox!.Messages); + Assert.Equal(handlerSend ? 1 : 0, handler.Applied); + if (handlerSend) + { + Assert.Equal(0, observation.Inbox!.Count); + Assert.Single(GetProcessed(observation.Context)); + } + using var storage = Fixture.Storage.BlockWrite(journal); + var delivered = Delivery.WaitForOutputAsync(grain.GetGrainId()); + Delivery.Release(); + scheduling.Continue(); + await storage.WaitUntilEnteredAsync(); + Assert.False(delivered.IsCompleted); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + storage.Release(); + await operation; + var receipt = await delivered; + Assert.Equal(42, receipt.Value); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(1, Fixture.JobManagerProbe.GetSuccessCount(BootstrapOutboxServices.JobName, grain.GetGrainId())); + if (handlerSend) + { + await OnOwnerTurnAsync(observation.Context, () => + { + Assert.Throws(() => handler.Context!.Complete()); + return Task.CompletedTask; + }); + Assert.Equal(1, handler.Applied); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SynchronousEnvelopeSend_SchedulingFailureExplicitlyRetriesPendingBusinessState(bool ambiguous) + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (ambiguous) Fixture.JobManagerProbe.FailAfterNext(BootstrapOutboxServices.JobName); + else Fixture.JobManagerProbe.FailNext(BootstrapOutboxServices.JobName); + var error = await Assert.ThrowsAsync(() => grain.SendSynchronousValueAsync(42)); + Assert.IsType(error.InnerException); + Assert.Equal(42, await grain.GetValueAsync()); + var pending = Assert.Single(observation.Outbox!.Messages); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.False(observation.Context.Deactivated.IsCompleted); + await grain.PersistAsync(); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(pending.Envelope.MessageId, Assert.Single(observation.Outbox.Messages).Envelope.MessageId); + Assert.Equal(2, Fixture.JobManagerProbe.GetAttemptCount(BootstrapOutboxServices.JobName, grain.GetGrainId())); + await grain.DeactivateAsync(); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(42, await grain.GetValueAsync()); + var recovered = Probe.Get(grain.GetGrainId())[^1]; + Assert.Equal(pending.Envelope.MessageId, Assert.Single(recovered.Outbox!.Messages).Envelope.MessageId); + } + + [Fact] + public async Task SynchronousHandler_PreCommitFailureRetiresPendingEffectsAndFreshReplayAppliesOnce() + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var handler = new SynchronousHandler(observation); + await OnOwnerTurnAsync(observation.Context, () => + { + observation.Context.ActivationServices.GetRequiredService().HandlerOverride = handler; + return Task.CompletedTask; + }); + Fixture.JobManagerProbe.FailNext(BootstrapOutboxServices.JobName); + var input = CreateEnvelope(grain); + Assert.Equal(DeliveryStatus.Accepted, (await grain.AsReference().DeliverAsync(input, Cancellation)).Status); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(1, handler.Applied); + Assert.Equal(42, observation.Value!.Value); + Assert.Equal(handler.Output.Envelope.MessageId, Assert.Single(observation.Outbox!.Messages).Envelope.MessageId); + using var recovery = Fixture.HandlerProbe.Arm(grain.GetGrainId(), BootstrapState.Route); + var activation = grain.GetValueAsync(); + await recovery.WaitUntilEnteredAsync(); + var recovered = Probe.Get(grain.GetGrainId())[^1]; + Assert.NotSame(observation.Context, recovered.Context); + Assert.Equal(11, recovered.Value!.Value); + Assert.Empty(recovered.Outbox!.Messages); + using var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(grain.GetGrainId())); + recovery.Release(); + await storage.WaitUntilEnteredAsync(); + Assert.Equal(12, recovered.Value!.Value); + Assert.Single(recovered.Outbox.Messages); + Assert.Single(GetProcessed(recovered.Context)); + storage.Release(); + await activation; + Assert.Equal(12, await grain.GetValueAsync()); + Assert.Equal(1, recovered.Context.ActivationServices.GetRequiredService().HandlerCalls); + Assert.Equal(1, handler.Applied); + } + + [Fact] + public async Task SynchronousHandler_PreCommitAcceptanceFailureRetiresOwnerBeforeFreshAcceptance() + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + await OnOwnerTurnAsync(observation.Context, () => + { + observation.JournalHook.BeforeOperation = (_, _) => throw new IOException("Acceptance prerequisite failed."); + return Task.CompletedTask; + }); + var envelope = CreateEnvelope(grain); + var error = await Assert.ThrowsAsync(() => + grain.AsReference().DeliverAsync(envelope, Cancellation).AsTask()); + Assert.IsType(error.InnerException); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(observation.Inbox!.Messages).Envelope.MessageId); + Assert.Equal(11, await grain.GetValueAsync()); + var recovered = Probe.Get(grain.GetGrainId())[^1]; + Assert.NotSame(observation.Context, recovered.Context); + Assert.Empty(recovered.Outbox!.Messages); + Assert.Equal(0, recovered.Inbox!.Count); + using var handler = Fixture.HandlerProbe.Arm(grain.GetGrainId(), BootstrapState.Route); + Assert.Equal(DeliveryStatus.Accepted, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + await handler.WaitUntilEnteredAsync(); + using var storage = Fixture.Storage.BlockWrite(journal); + handler.Release(); + await storage.WaitUntilEnteredAsync(); + Assert.Equal(12, recovered.Value!.Value); + Assert.Single(GetProcessed(recovered.Context)); + storage.Release(); + Assert.Equal(12, await grain.GetValueAsync()); + Assert.Equal(1, recovered.Context.ActivationServices.GetRequiredService().HandlerCalls); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SynchronousHandler_PostCommitFailureKeepsAcknowledgedAccounting(bool acceptance) + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var handler = new SynchronousHandler(observation); + var reached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var handled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var failHook = true; + var failure = new IOException("Post-persistence hook failed."); + var hook = observation.JournalHook; + hook.AfterOperation = (_, _) => + { + if (observation.Value!.Value == 42) handled.TrySetResult(); + if (failHook && (acceptance || observation.Value.Value == 42)) + { + reached.TrySetResult(); + throw failure; + } + }; + await OnOwnerTurnAsync(observation.Context, () => + { + observation.Context.ActivationServices.GetRequiredService().HandlerOverride = handler; + return Task.CompletedTask; + }); + var envelope = CreateEnvelope(grain); + var delivery = grain.AsReference().DeliverAsync(envelope, Cancellation).AsTask(); + if (acceptance) + { + var error = await Assert.ThrowsAsync(() => delivery); + Assert.IsType(error.InnerException); + } + else Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await reached.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + await OnOwnerTurnAsync(observation.Context, () => + { + failHook = false; + return Task.CompletedTask; + }); + Assert.False(observation.Context.Deactivated.IsCompleted); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + await handled.Task.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(1, handler.Applied); + Assert.Single(GetProcessed(observation.Context)); + Assert.Single(observation.Outbox!.Messages); + await OnOwnerTurnAsync(observation.Context, () => + { + hook.AfterOperation = null; + return Task.CompletedTask; + }); + await grain.PersistAsync(); + Assert.False(observation.Context.Deactivated.IsCompleted); + } + + private sealed class SynchronousHandler(BootstrapObservation observation) : IInboxHandler + { + public IInboxHandlerContext? Context { get; private set; } + public OutboxMessage Output { get; private set; } + public int Applied { get; private set; } + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Context = context; + Output = OutboxTestMessages.CreateApplication(observation.Context.ActivationServices, + context.Message.Envelope.MessageId.CreateChildKey("output"), observation.Context.GrainId, BootstrapState.OutputTarget, "output", 42); + Applied++; + observation.Value!.Value = 42; + observation.Outbox!.Send(Output); + context.Complete(); + return ValueTask.CompletedTask; + } + } + + private static Task OnOwnerTurnAsync(IGrainContext context, Func action) + { + var task = new Task(action); + context.Scheduler.QueueTask(task); + return task.Unwrap().WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + } + + [Fact] + public async Task OrdinaryOpaqueSend_StagesBusinessBeforeCaptureAndDispatchesAfterAck() + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + using var scheduling = Fixture.JobManagerProbe.BlockNext(BootstrapOutboxServices.JobName); + var operation = grain.SendValueAsync(42); + await scheduling.WaitUntilEnteredAsync(); + Assert.Equal(42, observation.Value!.Value); + Assert.Equal(1, observation.Outbox!.Count); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var storage = Fixture.Storage.BlockWrite(journal); + var delivered = Delivery.WaitForOutputAsync(grain.GetGrainId()); + Delivery.Release(); + scheduling.Continue(); + await storage.WaitUntilEnteredAsync(); + Assert.Equal(42, observation.Value.Value); + Assert.Single(observation.Outbox.Messages); + Assert.False(delivered.IsCompleted); + Assert.Equal(1, Fixture.JobManagerProbe.GetSuccessCount(BootstrapOutboxServices.JobName, grain.GetGrainId())); + storage.Release(); + await operation; + var receipt = await delivered; + Assert.Equal(42, receipt.Value); + Assert.Equal(42, await grain.GetValueAsync()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OrdinaryOpaqueSend_SchedulingFailurePreservesPendingBusinessForExplicitRetry(bool ambiguous) + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (ambiguous) Fixture.JobManagerProbe.FailAfterNext(BootstrapOutboxServices.JobName); + else Fixture.JobManagerProbe.FailNext(BootstrapOutboxServices.JobName); + var prerequisite = await Assert.ThrowsAsync(() => grain.SendValueAsync(42)); + Assert.IsType(prerequisite.InnerException); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(1, observation.Outbox!.Count); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(ambiguous ? 1 : 0, Fixture.JobManagerProbe.GetScheduledJobs(BootstrapOutboxServices.JobName, grain.GetGrainId()).Count); + await grain.PersistAsync(); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(42, TestApplicationProtocol.Read(observation.Context.ActivationServices.GetRequiredService(), Assert.Single(observation.Outbox.Messages)).Body); + Assert.Equal(2, Fixture.JobManagerProbe.GetAttemptCount(BootstrapOutboxServices.JobName, grain.GetGrainId())); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OrdinaryOpaqueSend_FreshReplayResolvesActualWriteOutcome(bool committed) + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (committed) Fixture.Storage.FailAfterWrite(journal); + else Fixture.Storage.FailWrite(journal); + await Assert.ThrowsAsync(() => grain.SendValueAsync(42)); + Assert.Equal(writes + (committed ? 1 : 0), Fixture.Storage.GetSuccessfulWriteCount(journal)); + var scheduled = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(BootstrapOutboxServices.JobName, grain.GetGrainId())); + observation.Context.Deactivate(new DeactivationReason( + DeactivationReasonCode.ApplicationRequested, "Replay the actual failed-write outcome."), Cancellation); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(committed ? 42 : 11, await grain.GetValueAsync()); + var recovered = Probe.Get(grain.GetGrainId())[^1]; + Assert.NotSame(observation.Context, recovered.Context); + Assert.Equal(committed ? 1 : 0, recovered.Outbox!.Count); + var owner = recovered.Context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.outbox-job-handle").Value; + if (committed) + { + Assert.Equal(scheduled.Id, owner!.Id); + Assert.Equal(scheduled.ShardId, owner.ShardId); + Assert.Equal(42, TestApplicationProtocol.Read(recovered.Context.ActivationServices.GetRequiredService(), Assert.Single(recovered.Outbox.Messages)).Body); + } + else Assert.Null(owner); + Assert.Equal(1, Fixture.JobManagerProbe.GetAttemptCount(BootstrapOutboxServices.JobName, grain.GetGrainId())); } [Fact] @@ -116,7 +484,7 @@ public async Task SelectedWithoutConstructorDependencies_MaterializesPrimaryStat var manager = observation.Context.ActivationServices.GetRequiredService(); Assert.Equal(8, BootstrapState.ReadMessagingStates(manager).Count()); Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox", out _)); - Assert.True(manager.TryGetStateMachine("test-handler-output", out _)); + Assert.True(manager.TryGetStateMachine(BootstrapOutboxServices.StateName, out _)); Assert.Equal(0, observation.Activations); Assert.Single(GetSetup(observation.Context).GetInvocationList()); } @@ -150,7 +518,7 @@ public async Task UnselectedComposition_LeavesMessagingUnresolved(Type grainClas Assert.True(observation.Manager.TryGetStateMachine("bootstrap-journal-only", out var state)); Assert.Same(observation.Value, state); Assert.False(observation.Manager.TryGetStateMachine("__orleans.messaging.inbox", out _)); - Assert.False(observation.Manager.TryGetStateMachine("test-handler-output", out _)); + Assert.False(observation.Manager.TryGetStateMachine(BootstrapOutboxServices.StateName, out _)); Assert.Empty(BootstrapState.ReadMessagingStates(observation.Manager)); } else @@ -252,8 +620,9 @@ public async Task ResolvedOrdinaryPlacement_WithoutMessagingConstructorDependenc Assert.True(manager.TryGetStateMachine("__orleans.messaging.inbox", out var inbox)); Assert.Same(observation.Context.ActivationServices.GetRequiredKeyedService< IDurableDictionary>("__orleans.messaging.inbox"), inbox); - Assert.True(manager.TryGetStateMachine("test-handler-output", out var outbox)); - Assert.Same(((JournaledTestOutbox)observation.Context.ActivationServices.GetRequiredService()).StoredMessages, outbox); + Assert.True(manager.TryGetStateMachine(BootstrapOutboxServices.StateName, out var outbox)); + Assert.Same(observation.Context.ActivationServices.GetRequiredKeyedService>( + BootstrapOutboxServices.StateName), outbox); Assert.Single(GetSetup(observation.Context).GetInvocationList()); } finally @@ -310,11 +679,28 @@ public async Task UnselectedEndpointDependency_FailsExplicitly() public void RepeatedRegistration_InstallsOneTypeConfigurator() { var services = new ServiceCollection(); + var silo = Substitute.For(); + silo.Services.Returns(services); + silo.AddJournaling(); ReceiverTestServices.Add(services, static _ => { }); + BootstrapOutboxServices.Add(services); ReceiverTestServices.Add(services, static _ => { }); - var descriptor = Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IConfigureGrainTypeComponents)); + BootstrapOutboxServices.Add(services); + var descriptor = Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IConfigureGrainTypeComponents) + && descriptor.ImplementationType == ReceiverTestServices.GetImplementationType("MessagingGrainTypeConfigurator")); Assert.Equal(ServiceLifetime.Singleton, descriptor.Lifetime); Assert.Equal(ReceiverTestServices.GetImplementationType("MessagingGrainTypeConfigurator"), descriptor.ImplementationType); + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IOutbox)); + foreach (var stateName in BootstrapOutboxServices.StateNames.Take(6)) + { + Assert.DoesNotContain(services, descriptor => descriptor.IsKeyedService && Equals(descriptor.ServiceKey, stateName)); + } + Assert.Single(services, descriptor => descriptor.IsKeyedService + && Equals(descriptor.ServiceKey, BootstrapOutboxServices.StateNames[6])); + Assert.Single(services, descriptor => descriptor.IsKeyedService && Equals(descriptor.ServiceKey, KeyedService.AnyKey) + && descriptor.ServiceType == typeof(IDurableDictionary<,>)); + Assert.Single(services, descriptor => descriptor.IsKeyedService && Equals(descriptor.ServiceKey, KeyedService.AnyKey) + && descriptor.ServiceType == typeof(IDurableValue<>)); Assert.Empty(typeof(IMessagingGrain).GetInterfaces()); Assert.Empty(typeof(IMessagingGrain).GetMethods()); Assert.False(typeof(IAddressable).IsAssignableFrom(typeof(IMessagingGrain))); @@ -333,30 +719,34 @@ private static void AssertComposition(BootstrapObservation observation, Bootstra var applicationManager = services.GetRequiredService(); Assert.Same(observation.Manager, applicationManager); Assert.Same(observation.Value, applicationManager.GetOrAddState>("bootstrap-value")); - Assert.True(applicationManager.TryGetState>("test-handler-output", out var applicationOutbox)); - Assert.Same(((JournaledTestOutbox)observation.Outbox!).StoredMessages, applicationOutbox); - Assert.Same(applicationOutbox, applicationManager.GetOrAddState>("test-handler-output")); + Assert.True(applicationManager.TryGetState>(BootstrapOutboxServices.StateName, out var applicationOutbox)); + Assert.Same(services.GetRequiredKeyedService>(BootstrapOutboxServices.StateName), applicationOutbox); + Assert.Same(applicationOutbox, applicationManager.GetOrAddState>(BootstrapOutboxServices.StateName)); Assert.Same(observation.Value, services.GetRequiredKeyedService>("bootstrap-value")); Assert.Same(observation.Inbox, services.GetRequiredService()); Assert.Same(observation.Outbox, services.GetRequiredService()); - var tryGetHandler = observation.Inbox!.GetType().GetMethod("TryGetHandler", BindingFlags.Instance | BindingFlags.NonPublic)!; - object?[] arguments = [null]; - Assert.True((bool)tryGetHandler.Invoke(observation.Inbox, arguments)!); - var handler = arguments[0]; - Assert.Same(state, handler); + Assert.Equal(ReceiverTestServices.GetImplementationType("Outbox"), observation.Outbox!.GetType()); + Assert.Same(state, observation.Context.ActivationServices.GetRequiredService()); Assert.Equal(8, BootstrapState.ReadMessagingStates(observation.Manager!).Count()); var primary = services.GetRequiredKeyedService>("__orleans.messaging.inbox"); Assert.Same(primary, applicationManager.GetOrAddState>("__orleans.messaging.inbox")); - Assert.True(observation.Manager!.TryGetStateMachine("test-handler-output", out var output)); - Assert.Same(applicationOutbox, output); + Assert.Same(observation.Outbox, services.GetRequiredService(ReceiverTestServices.GetImplementationType("Outbox"))); + foreach (var name in BootstrapOutboxServices.StateNames) + { + Assert.True(observation.Manager!.TryGetStateMachine(name, out _)); + } + Assert.True(observation.Manager!.TryGetStateMachine("__orleans.messaging.outbox-job-handle", out var jobState)); + Assert.Same(jobState, services.GetRequiredKeyedService>("__orleans.messaging.outbox-job-handle")); } private IBootstrapTestGrain CreateGrain(Type grainClass) => grainClass == typeof(GenericBootstrapGrain) ? Fixture.Client.GetGrain>(Guid.NewGuid()) : Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); private IBootstrapControlGrain Control() => Fixture.Client.GetGrain(Guid.NewGuid(), typeof(T).FullName!); + private int _commandSequence; private InboxMessage CreateEnvelope(IBootstrapTestGrain grain) => - TestApplicationProtocol.CreateInbox(Fixture.Client.ServiceProvider.GetRequiredService(), - GrainId.Create("bootstrap-sender", "external"), BootstrapState.Route, 1); + OutboxTestMessages.CreateApplication(Fixture.Client.ServiceProvider, + HierarchicalKey.Create("test", grain.GetGrainId().ToString(), "command", (++_commandSequence).ToString(CultureInfo.InvariantCulture)), + GrainId.Create("bootstrap-sender", "external"), grain.GetGrainId(), BootstrapState.Route, 1).ToInboxMessage(); private static IDurableDictionary GetProcessed(IGrainContext context) => context.ActivationServices.GetRequiredKeyedService>("__orleans.messaging.inbox-processed"); private static Delegate GetSetup(IGrainContext context) => Assert.IsAssignableFrom(GetSetupOrDefault(context)); @@ -375,6 +765,6 @@ private void AssertNoStorageWork(GrainId grainId) private void AssertNoScheduledJobs(GrainId grainId) { Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(ReceiverTestServices.InboxJobName, grainId)); - Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount("orleans.messaging.outbox-drain", grainId)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(BootstrapOutboxServices.JobName, grainId)); } } diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs new file mode 100644 index 00000000000..d0ebb883daa --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs @@ -0,0 +1,176 @@ +using System.Collections.Concurrent; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +internal static class BootstrapOutboxServices +{ + public const string JobName = "orleans.messaging.outbox-flush"; + public const string StateName = "__orleans.messaging.outbox"; + public static readonly string[] StateNames = + [ + StateName, + "__orleans.messaging.outbox-message-state", + "__orleans.messaging.outbox-dead-letters", + "__orleans.messaging.outbox-job-id", + "__orleans.messaging.outbox-job-handle", + "__orleans.messaging.outbox-completed-job-id", + "__orleans.messaging.outbox-job-sequence" + ]; + + public static void Add(IServiceCollection services) + { + var outboxType = ReceiverTestServices.GetImplementationType("Outbox"); + services.RemoveAll(); + services.RemoveAllKeyed(KeyedService.AnyKey); + services.RemoveAllKeyed>("test-handler-output"); + foreach (var descriptor in services.Where(descriptor => descriptor.IsKeyedService + && StateNames.Take(6).Any(name => Equals(descriptor.ServiceKey, name))).ToArray()) + { + services.Remove(descriptor); + } + services.TryAddScoped(outboxType, provider => ActivatorUtilities.CreateInstance(provider, outboxType, + provider.GetRequiredKeyedService>("orleans-binary"))); + services.TryAddScoped(provider => provider.GetRequiredKeyedService>("orleans-binary")); + services.AddScoped(provider => (IOutbox)provider.GetRequiredService(outboxType)); + AddAlias(typeof(IDurableValue), StateNames[6], "JobSequenceState"); + + void AddAlias(Type serviceType, string key, string property) + { + var getter = outboxType.GetProperty(property, BindingFlags.Instance | BindingFlags.NonPublic)!; + services.RemoveAllKeyed(serviceType, key); + services.AddKeyedScoped(serviceType, key, (provider, _) => getter.GetValue(provider.GetRequiredService(outboxType))!); + } + } + + +} + +public sealed class BootstrapDeliveryProbe : IOutgoingGrainCallFilter, ILoggerProvider +{ + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly ConcurrentDictionary> _outputs = new(); + + private readonly ConcurrentDictionary _drains = new(); + + public Task WaitForDrainAsync(GrainId sender) => + _drains.GetOrAdd(sender, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)).Task.WaitAsync(TimeSpan.FromSeconds(30)); + + public ILogger CreateLogger(string categoryName) => new CompletionLogger(this, categoryName); + public void Dispose() { } + + private sealed class CompletionLogger(BootstrapDeliveryProbe owner, string category) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => category == "Orleans.Messaging.Outbox"; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (IsEnabled(logLevel) && eventId.Name == "LogDeliveryComplete" + && state is IEnumerable> fields + && fields.Any(field => field.Key == "RemainingCount" && field.Value is 0) + && ReceiverTestServices.CurrentGrainContext is { } context) + { + owner._drains.GetOrAdd(context.GrainId, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)).TrySetResult(); + } + } + } + + public async Task Invoke(IOutgoingGrainCallContext context) + { + if (context.TargetId == BootstrapState.OutputTarget && context.MethodName == nameof(IInboxExtension.DeliverAsync)) + { + await _release.Task.WaitAsync(context.Request.GetCancellationToken()); + } + await context.Invoke(); + } + + public void Release() => _release.TrySetResult(); + public Task<(HierarchicalKey MessageId, int Value)> WaitForOutputAsync(GrainId sender) => + GetOutput(sender).Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void OnOutput(GrainId sender, HierarchicalKey messageId, int value) => GetOutput(sender).TrySetResult((messageId, value)); + private TaskCompletionSource<(HierarchicalKey MessageId, int Value)> GetOutput(GrainId sender) => + _outputs.GetOrAdd(sender, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)); +} + +public interface IBootstrapOutputGrain : IGrainWithStringKey +{ + Task GetMessageCountAsync(); +} + +[GrainType("bootstrap-output")] +public sealed class BootstrapOutputGrain : Grain, IBootstrapOutputGrain, IMessagingGrain, IInboxHandler, + IStateMachine, IDurableDictionaryCommandHandler +{ + private readonly Dictionary _values = []; + private readonly List<(GrainId Sender, HierarchicalKey MessageId, int Value)> _pending = []; + private (GrainId Sender, HierarchicalKey MessageId, int Value)[] _captured = []; + private readonly BootstrapDeliveryProbe _probe; + private readonly Orleans.Serialization.Session.SerializerSessionPool _sessions; + private readonly Orleans.Serialization.Serializer _senderSerializer; + private readonly IDurableDictionaryCommandCodec _codec; + + public BootstrapOutputGrain(IJournaledStateManager manager, IInbox inbox, BootstrapDeliveryProbe probe, + IDurableDictionaryCommandCodec codec, Orleans.Serialization.Session.SerializerSessionPool sessions, Orleans.Serialization.Serializer senderSerializer) + { + _sessions = sessions; + _senderSerializer = senderSerializer; + _probe = probe; + _codec = codec; + manager.RegisterStateMachine("bootstrap-output-values", this); + inbox.RegisterHandler(this); + } + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var value = (int)(TestApplicationProtocol.Read(_sessions, context.Message).Body + ?? throw new InvalidOperationException("The bootstrap output must contain an integer.")); + if (context.Message.Envelope.Headers is not { } headers || !headers.TryGetValue("sender", out var senderBytes)) + { + throw new InvalidOperationException("The bootstrap output must supply its sender."); + } + var sender = _senderSerializer.Deserialize(senderBytes); + _values.Add(context.Message.Envelope.MessageId, value); + _pending.Add((sender, context.Message.Envelope.MessageId, value)); + context.Complete(); + return ValueTask.CompletedTask; + } + + public Task GetMessageCountAsync() => Task.FromResult(_values.Count); + public void WritePendingEntries(JournalStreamWriter writer) + { + _captured = _pending.ToArray(); + _pending.Clear(); + foreach (var item in _captured) _codec.WriteSet(item.MessageId, item.Value, writer); + } + public void WriteSnapshot(JournalStreamWriter writer) + { + _captured = _pending.ToArray(); + _pending.Clear(); + _codec.WriteSnapshot(_values, writer); + } + public void OnWriteCompleted() + { + foreach (var output in _captured) _probe.OnOutput(output.Sender, output.MessageId, output.Value); + _captured = []; + } + public void Reset(JournalStreamWriter writer) + { + _values.Clear(); + _pending.Clear(); + _captured = []; + } + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); + + public void ApplySet(HierarchicalKey key, int value) => _values[key] = value; + public void ApplyRemove(HierarchicalKey key) => _values.Remove(key); + public void ApplyClear() => _values.Clear(); + public void Reset(int capacityHint) => _values.Clear(); +} diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs index 3884a741f81..cc751e6dd6e 100644 --- a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Globalization; using System.Reflection; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -30,6 +31,7 @@ public BootstrapObservation(IGrainContext context, BootstrapProbe probe) public object? ConstructedGrain { get; private set; } public bool InstanceAvailableInConstructor { get; private set; } public IJournaledStateManager? Manager { get; set; } + internal TestJournaledStateHook JournalHook { get; } = new(); public IDurableValue? Value { get; set; } public IInbox? Inbox { get; set; } public IOutbox? Outbox { get; set; } @@ -62,6 +64,7 @@ public BootstrapState(BootstrapObservation observation, IJournaledStateManager m observation.Value = value; observation.Inbox = inbox; observation.Outbox = outbox; + manager.Hooks.Add(observation.JournalHook); inbox.RegisterHandler(this); } @@ -81,7 +84,9 @@ public async Task Set(int value) public async Task SendValue(int value) { var context = Observation.Context; - var envelope = TestApplicationProtocol.Create(context.ActivationServices.GetRequiredService(), context.GrainId, OutputTarget, "output", value); + var envelope = OutboxTestMessages.CreateApplication(context.ActivationServices, + HierarchicalKey.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), + context.GrainId, OutputTarget, "output", value); Observation.Value!.Value = value; Observation.Outbox!.Send(envelope); await Observation.Manager!.WriteStateAsync(CancellationToken.None); @@ -89,7 +94,9 @@ public async Task SendValue(int value) public async Task SendSynchronousValue(int value) { var context = Observation.Context; - var envelope = TestApplicationProtocol.Create(context.ActivationServices.GetRequiredService(), context.GrainId, OutputTarget, "output", value); + var envelope = OutboxTestMessages.CreateApplication(context.ActivationServices, + HierarchicalKey.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), + context.GrainId, OutputTarget, "output", value); Observation.Value!.Value = value; Observation.Outbox!.Send(envelope); await Observation.Manager!.WriteStateAsync(CancellationToken.None); @@ -104,20 +111,21 @@ public Task Activate() return Task.CompletedTask; } - public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) => + HandlerOverride is { } handler + ? handler.HandleAsync(context, cancellationToken) + : HandleCoreAsync(context, cancellationToken); + + private async ValueTask HandleCoreAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { - if (HandlerOverride is { } handler) - { - await handler.HandleAsync(context, cancellationToken); - return; - } if (_handlers.TryGet(Observation.Context.GrainId, Route, out var barrier)) { barrier.Entered.TrySetResult(); await barrier.Continue.Task.WaitAsync(cancellationToken); } var value = Observation.Value!.Value + 1; - var outgoing = TestApplicationProtocol.Create(Observation.Context.ActivationServices.GetRequiredService(), Observation.Context.GrainId, OutputTarget, "output", value); + var outgoing = OutboxTestMessages.CreateApplication(Observation.Context.ActivationServices, + context.Message.Envelope.MessageId.CreateChildKey("output"), Observation.Context.GrainId, OutputTarget, "output", value); Observation.Value.Value = value; Observation.Outbox!.Send(outgoing); HandlerCalls++; @@ -276,11 +284,16 @@ public sealed class BootstrapClusterFixture : MessagingClusterFixture public const string OrdinaryPlacementAlias = "bootstrap-directory-alias"; public BootstrapProbe Probe { get; } = new(); + public BootstrapDeliveryProbe Delivery { get; } = new(); protected override void ConfigureServices(IServiceCollection services) { + BootstrapOutboxServices.Add(services); services.AddKeyedSingleton(StatelessPlacementAlias, new StatelessWorkerAttribute(1).PlacementStrategy); services.AddKeyedSingleton(OrdinaryPlacementAlias, new RandomPlacement()); services.AddSingleton(Probe); + services.AddSingleton(Delivery); + services.AddSingleton(Delivery); + services.AddSingleton(Delivery); services.AddScoped(); services.AddScoped(); var extensionType = ReceiverTestServices.GetImplementationType("InboxExtension"); diff --git a/test/Orleans.Messaging.Tests/Support/ControlledTimerRegistry.cs b/test/Orleans.Messaging.Tests/Support/ControlledTimerRegistry.cs new file mode 100644 index 00000000000..24ba656a47e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ControlledTimerRegistry.cs @@ -0,0 +1,137 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Timers; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class OutboxPumpTimerProbe +{ + private Barrier? _barrier; + + public Barrier BlockNext(int participantCount = 2) + { + var barrier = new Barrier(this, participantCount); + if (Interlocked.CompareExchange(ref _barrier, barrier, null) is not null) + { + throw new InvalidOperationException("An outbox pump timer barrier is already armed."); + } + + return barrier; + } + + internal Task WaitIfBlockedAsync(object? state, CancellationToken cancellationToken) + { + if (state?.GetType().FullName is not ("Orleans.Messaging.Outbox+PumpTimerState" + or "Orleans.Messaging.Outbox+LocalPumpTimerState")) + { + return Task.CompletedTask; + } + + return Volatile.Read(ref _barrier)?.WaitAsync(cancellationToken) ?? Task.CompletedTask; + } + + private void Remove(Barrier barrier) => + Interlocked.CompareExchange(ref _barrier, null, barrier); + + public sealed class Barrier : IDisposable + { + private readonly OutboxPumpTimerProbe _owner; + private readonly TaskCompletionSource _entered = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _remaining; + + internal Barrier(OutboxPumpTimerProbe owner, int participantCount) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(participantCount); + _owner = owner; + _remaining = participantCount; + } + + internal async Task WaitAsync(CancellationToken cancellationToken) + { + var remaining = Interlocked.Decrement(ref _remaining); + if (remaining < 0) + { + return; + } + + if (remaining == 0) + { + _entered.TrySetResult(); + } + + await _release.Task.WaitAsync(cancellationToken); + } + + public Task WaitUntilEnteredAsync() => _entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + public void Release() => _release.TrySetResult(); + + public void Dispose() + { + Release(); + _owner.Remove(this); + } + } +} + +internal sealed class ControlledTimerRegistry( + ITimerRegistry inner, + OutboxPumpTimerProbe probe) : ITimerRegistry +{ + [Obsolete] + public IDisposable RegisterTimer( + IGrainContext grainContext, + Func callback, + object? state, + TimeSpan dueTime, + TimeSpan period) => + inner.RegisterTimer(grainContext, callback, state, dueTime, period); + + public IGrainTimer RegisterGrainTimer( + IGrainContext grainContext, + Func callback, + TState state, + GrainTimerCreationOptions options) => + inner.RegisterGrainTimer( + grainContext, + async (callbackState, cancellationToken) => + { + await probe.WaitIfBlockedAsync(callbackState, cancellationToken); + await callback(callbackState, cancellationToken); + }, + state, + options); + + public static void Decorate(IServiceCollection services, OutboxPumpTimerProbe probe) + { + var descriptor = services.Last(service => service.ServiceType == typeof(ITimerRegistry)); + services.Remove(descriptor); + services.Add(new ServiceDescriptor( + typeof(ITimerRegistry), + serviceProvider => new ControlledTimerRegistry(CreateInner(serviceProvider, descriptor), probe), + descriptor.Lifetime)); + } + + private static ITimerRegistry CreateInner( + IServiceProvider serviceProvider, + ServiceDescriptor descriptor) + { + if (descriptor.ImplementationInstance is ITimerRegistry instance) + { + return instance; + } + + if (descriptor.ImplementationFactory is { } factory) + { + return (ITimerRegistry)factory(serviceProvider); + } + + return (ITimerRegistry)ActivatorUtilities.CreateInstance( + serviceProvider, + descriptor.ImplementationType + ?? throw new InvalidOperationException("The timer registry registration has no implementation.")); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs new file mode 100644 index 00000000000..d5846e85113 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs @@ -0,0 +1,31 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +internal static class OutboxTestMessages +{ + public static OutboxMessage Create(HierarchicalKey messageId, GrainId receiver, ReadOnlyMemory payload, + GrainId? sender, string? subject, Serializer senderSerializer) => new() + { + ReceiverId = receiver, + Envelope = new Envelope + { + MessageId = messageId, + Payload = payload.ToArray(), + Subject = subject, + Headers = sender is { } senderId + ? new Dictionary(StringComparer.Ordinal) { ["sender"] = senderSerializer.SerializeToArray(senderId) } + : null + } + }; + + public static OutboxMessage CreateApplication(IServiceProvider services, HierarchicalKey messageId, + GrainId sender, GrainId receiver, string subject, object? body) => Create(messageId, receiver, + TestApplicationProtocol.Encode(services.GetRequiredService(), new TestApplicationMessage(subject, body)), + sender, subject, services.GetRequiredService>()); + + public static string? Subject(Envelope envelope) => envelope.Subject; +} diff --git a/test/Orleans.Messaging.Tests/Support/TestJournaledStateHook.cs b/test/Orleans.Messaging.Tests/Support/TestJournaledStateHook.cs new file mode 100644 index 00000000000..5bd7e42dc7f --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/TestJournaledStateHook.cs @@ -0,0 +1,23 @@ +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +internal sealed class TestJournaledStateHook : IJournaledStateHook +{ + public Action? BeforeOperation { get; set; } + public Func? BeforeOperationAsync { get; set; } + public Action? AfterOperation { get; set; } + public Func? AfterOperationAsync { get; set; } + + ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + BeforeOperation?.Invoke(operation, cancellationToken); + return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + + ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + AfterOperation?.Invoke(operation, cancellationToken); + return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } +} From 0c14f80262c10469d1e707039567219b42d9da70 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:47:56 -0700 Subject: [PATCH 30/31] refactor(messaging): register setup hooks through journal owner --- src/Orleans.Messaging/Outbox.cs | 6 +++-- .../Contracts/OutboxCodecBoundaryTests.cs | 24 ++++++++++++------- .../Contracts/OutboxDeliveryBatchTests.cs | 18 +++++++++++--- .../Support/BootstrapTestGrains.cs | 2 +- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index fa3b83026fc..15450d82381 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -161,8 +161,10 @@ public Outbox( _jobSequence = sequence; manager.RegisterStateMachine(MessagingStateNames.OutboxJobSequence, sequence); jobHandlers.Register(this); - // Before hooks run in reverse registration order, after all asynchronous preparation. - manager.Hooks.Insert(0, this); + // Setup registration completes synchronously, before initialization or persistence. + var registration = manager.AddHookAsync(this, index: 0); + Debug.Assert(registration.IsCompleted); + registration.GetAwaiter().GetResult(); var lifecycle = grainContext.ObservableLifecycle; lifecycle.Subscribe(RuntimeTypeNameFormatter.Format(GetType()), GrainLifecycleStage.Activate, this); diff --git a/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs b/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs index d007ddd545a..871345e8cb5 100644 --- a/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs @@ -946,7 +946,7 @@ public Task SynchronousSend_ArrivalDuringLaterHookHasWakeupBeforeCapture() => On await release.Task; } }; - await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.AddHookAsync(hook).GetAwaiter().GetResult()); Assert.Same(fixture.Outbox, fixture.Manager.Hooks[0]); var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -972,12 +972,14 @@ public Task SynchronousSend_EmptyFinalHookAndCaptureShareOneTurn() => OnOwnerAsy { envelope = f.CreateEnvelope(); var hook = Assert.IsAssignableFrom(Assert.Single(f.Manager.Hooks)); - f.Manager.Hooks[0] = new CaptureBoundaryHook(hook, () => + f.Manager.RemoveHookAsync(hook).GetAwaiter().GetResult(); + f.Manager.AddHookAsync(new CaptureBoundaryHook(hook, () => { staged ??= Task.Factory.StartNew(() => f.Outbox.Send(envelope), TestContext.Current.CancellationToken, TaskCreationOptions.None, TaskScheduler.Current); - }); - f.Manager.Hooks.Add(new TestJournaledStateHook { BeforeOperationAsync = async (_, _) => await Task.Yield() }); + }), index: 0).GetAwaiter().GetResult(); + f.Manager.AddHookAsync(new TestJournaledStateHook { BeforeOperationAsync = async (_, _) => await Task.Yield() }) + .GetAwaiter().GetResult(); }); using var storage = fixture.Storage.BlockWrite(fixture.JournalId); var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -1118,7 +1120,7 @@ public Task SynchronousSend_OverlappingWritesShareOnePhysicalWakeup() => OnOwner public Task SynchronousSend_OwnerRetirementArrivalRetainsPreCaptureOwnerOrPreparesPostCaptureOwner(bool beforeCapture) => OnOwnerAsync(async () => { var hook = new TestJournaledStateHook(); - await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.AddHookAsync(hook).GetAwaiter().GetResult()); { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); await fixture.DeliverAsync(); @@ -1161,7 +1163,7 @@ public Task SynchronousSend_OwnerRetirementArrivalRetainsPreCaptureOwnerOrPrepar public Task SynchronousSend_PostCommitFailurePreservesAcknowledgedOwnedDelivery() => OnOwnerAsync(async () => { var hook = new TestJournaledStateHook(); - await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.AddHookAsync(hook).GetAwaiter().GetResult()); { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); var failure = new IOException("After hook failed after acknowledged delivery removal."); @@ -1182,7 +1184,7 @@ public Task SynchronousSend_PostCommitFailurePreservesAcknowledgedOwnedDelivery( public Task SynchronousSend_OwnedPreCommitFailureRetiresStagedOperationForFreshReplay() => OnOwnerAsync(async () => { var hook = new TestJournaledStateHook(); - await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.AddHookAsync(hook).GetAwaiter().GetResult()); var envelope = fixture.CreateEnvelope(); fixture.Outbox.Send(envelope); await fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken); @@ -1214,7 +1216,7 @@ public Task SynchronousSend_StopBeforeQueuedHookPreventsNewScheduling() => OnOwn await release.Task; } }; - await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.Hooks.Add(hook)); + await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.AddHookAsync(hook).GetAwaiter().GetResult()); { var newEnvelope = fixture.CreateEnvelope(); fixture.Outbox.Send(newEnvelope); } var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -1652,7 +1654,11 @@ public async ValueTask DisposeAsync() private sealed class StateTrackingManager(IJournaledStateManager inner, CodecProbe probe) : IJournaledStateManager { private readonly Dictionary _states = new(StringComparer.Ordinal); - public IList Hooks => inner.Hooks; + public IReadOnlyList Hooks => inner.Hooks; + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + inner.AddHookAsync(hook, index, cancellationToken); + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + inner.RemoveHookAsync(hook, cancellationToken); public Exception? Failure { get; private set; } public int StateCount => _states.Count; public void RegisterStateMachine(string name, IStateMachine state) => _states.Add(name, state); diff --git a/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs b/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs index 152b708c7d7..a7caef8b383 100644 --- a/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs @@ -2401,7 +2401,7 @@ public OutboxFixture( { _services = CreateServices(writeException, eagerCapture); Manager = (TestStateManager)_services.GetRequiredService(); - if (hook is not null) Manager.Hooks.Add(hook); + if (hook is not null) Manager.AddHookAsync(hook).GetAwaiter().GetResult(); MessageId = envelope?.Envelope.MessageId ?? HierarchicalKey.Create("test", "outbox", "command", "0"); SenderId = GrainId.Create("sender", "1"); ReceiverId = loopback ? SenderId : GrainId.Create("receiver", "1"); @@ -2906,7 +2906,20 @@ public TestStateManager(IJournalFormat format, Exception? writeException, bool e _eagerCapture = eagerCapture; } - public IList Hooks { get; private set; } = []; + private readonly List _hooks = []; + public IReadOnlyList Hooks => _hooks; + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!_hooks.Remove(hook)) throw new InvalidOperationException("The test journal hook is not registered."); + return default; + } public TestStorage Storage { get; private set; } = null!; public int RegistrationCount => _states.Count; public int WriteCount { get; private set; } @@ -2936,7 +2949,6 @@ public ValueTask InitializeAsync(CancellationToken cancellationToken) if (!_initialized) { _initialized = true; - Hooks = Array.AsReadOnly(Hooks.ToArray()); ResetStates(); for (var i = 0; i < _adapters.Length; i++) _adapters[i].Restore(Storage.Snapshots[i]); foreach (var state in _states.Values) state.OnRecoveryCompleted(); diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs index cc751e6dd6e..0ea5679ec09 100644 --- a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs @@ -64,7 +64,7 @@ public BootstrapState(BootstrapObservation observation, IJournaledStateManager m observation.Value = value; observation.Inbox = inbox; observation.Outbox = outbox; - manager.Hooks.Add(observation.JournalHook); + manager.AddHookAsync(observation.JournalHook).GetAwaiter().GetResult(); inbox.RegisterHandler(this); } From 3677a083caeee7aac92183cfc47ddc53185263e3 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 17:29:55 -0700 Subject: [PATCH 31/31] refactor(tests): retain outbox hooks without manager inspection --- .../Contracts/OutboxCodecBoundaryTests.cs | 12 +++++----- .../Contracts/OutboxDeliveryBatchTests.cs | 7 +++--- .../Support/OutboxTestMessages.cs | 22 +++++++++---------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs b/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs index 871345e8cb5..6ff013783b0 100644 --- a/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/OutboxCodecBoundaryTests.cs @@ -553,7 +553,9 @@ public Task CustomHeadersAndSender_DuplicatePreservesFirstIntentThroughReplay(bo ReceiverId = receiver, Envelope = new Envelope { - MessageId = id, Payload = [1, 2, 3], Subject = "headers.v1", + MessageId = id, + Payload = [1, 2, 3], + Subject = "headers.v1", Headers = new Dictionary(StringComparer.Ordinal) { ["sender"] = fixture.Serializer().SerializeToArray(fixture.Context.GrainId), @@ -570,7 +572,9 @@ public Task CustomHeadersAndSender_DuplicatePreservesFirstIntentThroughReplay(bo ReceiverId = receiver, Envelope = new Envelope { - MessageId = id, Payload = [1, 2, 3], Subject = "headers.v1", + MessageId = id, + Payload = [1, 2, 3], + Subject = "headers.v1", Headers = new Dictionary(StringComparer.Ordinal) { ["trace"] = [9], @@ -947,7 +951,6 @@ public Task SynchronousSend_ArrivalDuringLaterHookHasWakeupBeforeCapture() => On } }; await using var fixture = await CodecFixture.CreateAsync(configure: f => f.Manager.AddHookAsync(hook).GetAwaiter().GetResult()); - Assert.Same(fixture.Outbox, fixture.Manager.Hooks[0]); var write = fixture.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -971,7 +974,7 @@ public Task SynchronousSend_EmptyFinalHookAndCaptureShareOneTurn() => OnOwnerAsy await using var fixture = await CodecFixture.CreateAsync(configure: f => { envelope = f.CreateEnvelope(); - var hook = Assert.IsAssignableFrom(Assert.Single(f.Manager.Hooks)); + var hook = Assert.IsAssignableFrom(f.Outbox); f.Manager.RemoveHookAsync(hook).GetAwaiter().GetResult(); f.Manager.AddHookAsync(new CaptureBoundaryHook(hook, () => { @@ -1654,7 +1657,6 @@ public async ValueTask DisposeAsync() private sealed class StateTrackingManager(IJournaledStateManager inner, CodecProbe probe) : IJournaledStateManager { private readonly Dictionary _states = new(StringComparer.Ordinal); - public IReadOnlyList Hooks => inner.Hooks; public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => inner.AddHookAsync(hook, index, cancellationToken); public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => diff --git a/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs b/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs index a7caef8b383..95f329fe6be 100644 --- a/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/OutboxDeliveryBatchTests.cs @@ -2907,7 +2907,6 @@ public TestStateManager(IJournalFormat format, Exception? writeException, bool e } private readonly List _hooks = []; - public IReadOnlyList Hooks => _hooks; public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -2972,9 +2971,9 @@ private async Task RunWriteAsync(Task previous) _failure?.Throw(); try { - for (var index = Hooks.Count - 1; index >= 0; index--) + for (var index = _hooks.Count - 1; index >= 0; index--) { - await Hooks[index].BeforeOperationAsync(JournaledStateOperation.Write, CancellationToken.None); + await _hooks[index].BeforeOperationAsync(JournaledStateOperation.Write, CancellationToken.None); } } catch (Exception exception) @@ -3017,7 +3016,7 @@ private async Task RunWriteAsync(Task previous) throw; } List? failures = null; - foreach (var hook in Hooks) + foreach (var hook in _hooks) { try { await hook.AfterOperationAsync(JournaledStateOperation.Write, CancellationToken.None); } catch (Exception exception) { (failures ??= []).Add(exception); } diff --git a/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs index d5846e85113..a2188e7b0ed 100644 --- a/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs +++ b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs @@ -9,18 +9,18 @@ internal static class OutboxTestMessages { public static OutboxMessage Create(HierarchicalKey messageId, GrainId receiver, ReadOnlyMemory payload, GrainId? sender, string? subject, Serializer senderSerializer) => new() - { - ReceiverId = receiver, - Envelope = new Envelope { - MessageId = messageId, - Payload = payload.ToArray(), - Subject = subject, - Headers = sender is { } senderId - ? new Dictionary(StringComparer.Ordinal) { ["sender"] = senderSerializer.SerializeToArray(senderId) } - : null - } - }; + ReceiverId = receiver, + Envelope = new Envelope + { + MessageId = messageId, + Payload = payload.ToArray(), + Subject = subject, + Headers = sender is { } senderId + ? new Dictionary(StringComparer.Ordinal) { ["sender"] = senderSerializer.SerializeToArray(senderId) } + : null + } + }; public static OutboxMessage CreateApplication(IServiceProvider services, HierarchicalKey messageId, GrainId sender, GrainId receiver, string subject, object? body) => Create(messageId, receiver,