diff --git a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/MessageBusOutboxWorker.cs b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/MessageBusOutboxWorker.cs index a68f1c90deb..9e8ae4a6ce9 100644 --- a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/MessageBusOutboxWorker.cs +++ b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/MessageBusOutboxWorker.cs @@ -12,6 +12,7 @@ internal sealed class PostgresMessageBusOutboxWorker(PostgresOutboxProcessor pro { private readonly object _lock = new(); private ContinuousTask? _task; + private Task _processing = Task.CompletedTask; /// /// Starts the outbox processing background task. This call is idempotent: invoking it again @@ -28,14 +29,15 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - _task = new ContinuousTask(processor.ProcessAsync); + _task = new ContinuousTask(ProcessAsync); } return Task.CompletedTask; } /// - /// Stops the outbox processing background task and waits for it to complete gracefully. + /// Stops the outbox processing background task and waits until the messages that were being dispatched + /// are either removed from the outbox or released. /// /// A token that signals when shutdown should be forced. public async Task StopAsync(CancellationToken cancellationToken) @@ -54,5 +56,18 @@ public async Task StopAsync(CancellationToken cancellationToken) } await task.DisposeAsync(); + + // The continuous task stops waiting once it is cancelled, while the processor still removes the + // messages a transport has already accepted. The wait ends early when the shutdown is forced. + await Volatile.Read(ref _processing) + .WaitAsync(cancellationToken) + .ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + } + + private Task ProcessAsync(CancellationToken cancellationToken) + { + var processing = processor.ProcessAsync(cancellationToken); + Volatile.Write(ref _processing, processing); + return processing; } } diff --git a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/OutboxProcessor.cs b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/OutboxProcessor.cs index 40a7e4314c9..900b5fccf31 100644 --- a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/OutboxProcessor.cs +++ b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/OutboxProcessor.cs @@ -1,8 +1,10 @@ using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.ObjectPool; using Mocha.Middlewares; +using Mocha.Threading; using Npgsql; namespace Mocha.Outbox; @@ -13,6 +15,12 @@ namespace Mocha.Outbox; /// public sealed class PostgresOutboxProcessor { + private static readonly TimeSpan s_pauseDelay = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan s_maxPauseDelay = TimeSpan.FromSeconds(10); + + // Stays below the longest delay a timer accepts, which is about 24.8 days. + private static readonly TimeSpan s_maxPollingInterval = TimeSpan.FromDays(24); + private readonly ILogger _logger; private readonly IServiceProvider _services; private readonly IMessagingRuntime _runtime; @@ -66,59 +74,103 @@ internal PostgresOutboxProcessor( /// A token that signals when the processor should stop. public async Task ProcessAsync(CancellationToken cancellationToken) { - while (!cancellationToken.IsCancellationRequested) + var workers = new Task[_options.MaxConcurrentMessages]; + for (var i = 0; i < workers.Length; i++) { - using var joinedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + workers[i] = ProcessWorkerAsync(cancellationToken); + } + + await Task.WhenAll(workers); + } + + private async Task ProcessWorkerAsync(CancellationToken cancellationToken) + { + using var worker = new WorkerState(_options.DispatchTimeout, cancellationToken); + Task? signaled = null; + while (!cancellationToken.IsCancellationRequested) + { try { - var signaled = _signal.WaitAsync(joinedCts.Token); + signaled ??= _signal.WaitAsync(cancellationToken); - var nextPollingInterval = await ProcessPendingAsync(cancellationToken); + var pass = await ProcessPendingAsync(worker, cancellationToken); - if (nextPollingInterval is not null) + if (pass.IsPaused) { - _logger.OutboxProcessorSleeping(nextPollingInterval.Value); + // The released message is claimed again after the pause, with or without a signal. + await worker.Backoff.WaitAsync(cancellationToken); + continue; + } + + if (pass.NextPollingInterval is { } nextPollingInterval) + { + _logger.OutboxProcessorSleeping(nextPollingInterval); - await Task.WhenAny(Task.Delay(nextPollingInterval.Value, cancellationToken), signaled); + try + { + await signaled.WaitAsync(nextPollingInterval, cancellationToken); + } + catch (TimeoutException) + { + continue; + } } else { await signaled; } + + signaled = null; } - catch (OperationCanceledException) + catch (Exception) when (cancellationToken.IsCancellationRequested) { - joinedCts.TryCancel(); + // The host is stopping. } - catch + catch (OperationCanceledException) when (signaled is { IsCanceled: true }) { - joinedCts.TryCancel(); - throw; + signaled = null; + } + catch (Exception ex) + { + // A failure outside of a dispatch, such as a lost database connection, pauses this worker only. + if (signaled is { IsFaulted: true }) + { + signaled = null; + } + + _logger.UnexpectedErrorWhileProcessingOutboxEvent(ex); + await worker.Backoff.WaitAsync(cancellationToken); } } } - private async ValueTask ProcessPendingAsync(CancellationToken cancellationToken) + private async ValueTask ProcessPendingAsync(WorkerState worker, CancellationToken cancellationToken) { await using var scope = _services.CreateAsyncScope(); await using var connection = _options.CreateConnection(scope.ServiceProvider); await connection.OpenAsync(cancellationToken); - while (true) + while (!cancellationToken.IsCancellationRequested) { using var activity = OpenTelemetry.Source.StartActivity( "Process Message Outbox", ActivityKind.Consumer, new ActivityContext()); - if (!await ProcessEventAsync(connection, cancellationToken)) + switch (await ProcessEventAsync(connection, worker, cancellationToken)) { - return await GetNextPollingIntervalAsync(connection, cancellationToken); - } + case ClaimResult.Empty: + return new PassResult( + IsPaused: false, + await GetNextPollingIntervalAsync(connection, cancellationToken)); - cancellationToken.ThrowIfCancellationRequested(); + case ClaimResult.Released: + return new PassResult(IsPaused: true, NextPollingInterval: null); + } } + + return default; } private async Task GetNextPollingIntervalAsync( @@ -131,123 +183,214 @@ public async Task ProcessAsync(CancellationToken cancellationToken) var result = await command.ExecuteScalarAsync(cancellationToken); - return result is not null and not DBNull ? ((DateTime)result - DateTimeOffset.UtcNow).Duration() : null; + if (result is null or DBNull) + { + return null; + } + + var interval = ((DateTime)result - DateTimeOffset.UtcNow).Duration(); + + return interval < s_maxPollingInterval ? interval : s_maxPollingInterval; } - private async Task ProcessEventAsync(NpgsqlConnection connection, CancellationToken cancellationToken) + private async Task ProcessEventAsync( + NpgsqlConnection connection, + WorkerState worker, + CancellationToken cancellationToken) { using var activity = OpenTelemetry.Source.StartActivity( "Process Message Outbox Event", ActivityKind.Producer, new ActivityContext()); + // An exception before the commit rolls the claim back when the transaction is disposed. await using var transaction = await connection.BeginTransactionAsync(cancellationToken); - try + Guid id; + int timesSent; + DispatchOutcome outcome; + + // Lock an event for processing and increment TimesSent in case of failure + await using (var command = connection.CreateCommand()) { - // Lock an event for processing and increment TimesSent in case of failure - await using var command = connection.CreateCommand(); command.Transaction = transaction; command.CommandText = _options.Queries.ProcessEvent; await command.PrepareAsync(cancellationToken); - try + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) { - await using var reader = await command.ExecuteReaderAsync(cancellationToken); + return ClaimResult.Empty; + } - if (await reader.ReadAsync(cancellationToken)) - { - var id = reader.GetGuid(0); - var envelope = Serializer.ReadMessageEnvelopeSafe(reader, 1, _logger); - var timesSent = reader.GetInt32(2); + id = reader.GetGuid(0); + var envelope = Serializer.ReadMessageEnvelopeSafe(reader, 1, _logger); + timesSent = reader.GetInt32(2); - // A message this instance cannot read or route stays in the outbox and is - // retried after its backoff. - if (envelope is null) - { - _logger.CouldNotDeserializeMessageBody(id); - LogIfExhausted(id, timesSent); + if (TryResolve(id, envelope, out var messageType, out var endpoint, out outcome)) + { + var dispatchToken = worker.StartDispatch(); - return true; - } + try + { + await SendAsync(envelope, endpoint, messageType, dispatchToken); - if (envelope.DeliverBy is { } deadline && deadline <= _timeProvider.GetUtcNow()) - { - await reader.CloseAsync(); - await DeleteEventAsync(connection, id, transaction, cancellationToken); - _logger.OutboxMessageExpired(id); - return true; - } + worker.Backoff.Reset(); + outcome = DispatchOutcome.Sent; + } + catch (Exception ex) + { + outcome = OnDispatchFailed(ex, id, envelope, timesSent, dispatchToken, cancellationToken); + } + finally + { + worker.EndDispatch(); + } + } + } - var messageType = GetMessageType(envelope.MessageType); + switch (outcome) + { + case DispatchOutcome.Sent: + // The transport has the message, so its row is removed whatever the token says. + await DeleteEventAsync(connection, id, transaction, CancellationToken.None); + await transaction.CommitAsync(CancellationToken.None); + return ClaimResult.Processed; + + case DispatchOutcome.Expired: + await DeleteEventAsync(connection, id, transaction, cancellationToken); + await transaction.CommitAsync(CancellationToken.None); + _logger.OutboxMessageExpired(id); + return ClaimResult.Processed; + + case DispatchOutcome.Released: + // The rollback restores the attempt count and the position of the row. + await transaction.RollbackAsync(CancellationToken.None); + return ClaimResult.Released; + + default: + // The commit stores the incremented attempt count and the backoff of the row. + await transaction.CommitAsync(CancellationToken.None); + LogIfExhausted(id, timesSent); + return ClaimResult.Processed; + } + } - if (messageType is null) - { - _logger.CouldNotResolveMessageType(id, envelope.MessageType); - LogIfExhausted(id, timesSent); + /// + /// Resolves what a dispatch of the message needs. When the message cannot be dispatched, + /// says how its claim ends. + /// + private bool TryResolve( + Guid id, + [NotNullWhen(true)] MessageEnvelope? envelope, + [NotNullWhen(true)] out MessageType? messageType, + [NotNullWhen(true)] out DispatchEndpoint? endpoint, + out DispatchOutcome outcome) + { + messageType = null; + endpoint = null; + outcome = DispatchOutcome.Failed; - return true; - } + // A message this instance cannot read or route stays in the outbox and is + // retried after its backoff. + if (envelope is null) + { + _logger.CouldNotDeserializeMessageBody(id); + return false; + } - var endpoint = ResolveDispatchEndpoint(envelope); + if (envelope.DeliverBy is { } deadline && deadline <= _timeProvider.GetUtcNow()) + { + outcome = DispatchOutcome.Expired; + return false; + } - if (endpoint is null) - { - _logger.CouldNotDetermineDestination(id, envelope.DestinationAddress); - LogIfExhausted(id, timesSent); + messageType = GetMessageType(envelope.MessageType); - return true; - } + if (messageType is null) + { + _logger.CouldNotResolveMessageType(id, envelope.MessageType); + return false; + } - await SendAsync(envelope, endpoint, messageType, cancellationToken); + endpoint = ResolveDispatchEndpoint(envelope); - await reader.CloseAsync(); + if (endpoint is null) + { + _logger.CouldNotDetermineDestination(id, envelope.DestinationAddress); + return false; + } - await DeleteEventAsync(connection, id, transaction, cancellationToken); + return true; + } - return true; - } - else - { - return false; - } - } - finally - { - try - { - await transaction.CommitAsync(cancellationToken); - } - catch - { - // Commit failed (e.g., connection lost). Attempt rollback. - // If commit actually succeeded server-side, the message stays - // with times_sent incremented - safe, just causes a retry. - try - { - await transaction.RollbackAsync(CancellationToken.None); - } - catch - { - /* swallow */ - } - } - } + /// + /// Logs why the dispatch did not complete and returns whether the attempt counts. + /// + private DispatchOutcome OnDispatchFailed( + Exception exception, + Guid id, + MessageEnvelope envelope, + int timesSent, + CancellationToken dispatchToken, + CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + _logger.OutboxDispatchCancelled( + exception, + id, + envelope.MessageId, + envelope.MessageType, + envelope.DestinationAddress); + + return DispatchOutcome.Released; } - catch (Exception ex) + + if (dispatchToken.IsCancellationRequested) + { + _logger.OutboxDispatchTimedOut( + exception, + id, + envelope.MessageId, + envelope.MessageType, + envelope.DestinationAddress, + _options.DispatchTimeout); + + return DispatchOutcome.Released; + } + + if (exception is TransportUnavailableException) { - // Log only - no RollbackAsync here (commit already handled in finally) - _logger.UnexpectedErrorWhileProcessingOutboxEvent(ex); - throw; + _logger.OutboxTransportUnavailable( + exception, + id, + envelope.MessageId, + envelope.MessageType, + envelope.DestinationAddress); + + return DispatchOutcome.Released; } + + _logger.OutboxDispatchFailed( + exception, + id, + envelope.MessageId, + envelope.MessageType, + envelope.DestinationAddress, + timesSent, + _options.MaxAttempts); + + return DispatchOutcome.Failed; } private void LogIfExhausted(Guid id, int timesSent) { - if (timesSent >= PostgresMessageOutboxQueries.MaxAttempts) + if (timesSent >= _options.MaxAttempts) { - _logger.OutboxMessageExhausted(id, PostgresMessageOutboxQueries.MaxAttempts); + _logger.OutboxMessageExhausted(id, _options.MaxAttempts); } } @@ -371,6 +514,70 @@ private async Task DeleteEventAsync( await command.ExecuteNonQueryAsync(cancellationToken); } + + private enum ClaimResult + { + // No message was eligible. + Empty, + + // A message was claimed and its outcome was stored. + Processed, + + // A message was claimed and handed back unchanged. + Released + } + + private enum DispatchOutcome + { + // The transport accepted the message. + Sent, + + // The delivery deadline of the message has passed. + Expired, + + // The attempt failed and counts. + Failed, + + // The attempt was cut short and does not count. + Released + } + + private readonly record struct PassResult(bool IsPaused, TimeSpan? NextPollingInterval); + + // The state one worker keeps across the messages it dispatches. + private sealed class WorkerState(TimeSpan dispatchTimeout, CancellationToken stopToken) : IDisposable + { + private CancellationTokenSource? _dispatchCancellation; + + // The pause between two passes after a released claim or a failure of the worker. + public ExponentialBackoff Backoff { get; } = new(int.MaxValue, s_pauseDelay, s_maxPauseDelay); + + // Returns the token of the next dispatch, which the dispatch timeout and a host stop cancel. + public CancellationToken StartDispatch() + { + if (dispatchTimeout == Timeout.InfiniteTimeSpan) + { + return stopToken; + } + + _dispatchCancellation ??= CancellationTokenSource.CreateLinkedTokenSource(stopToken); + _dispatchCancellation.CancelAfter(dispatchTimeout); + + return _dispatchCancellation.Token; + } + + public void EndDispatch() + { + // A source that was cancelled cannot be reset, so the next dispatch gets a new one. + if (_dispatchCancellation is { } cancellation && !cancellation.TryReset()) + { + cancellation.Dispose(); + _dispatchCancellation = null; + } + } + + public void Dispose() => _dispatchCancellation?.Dispose(); + } } internal static partial class Logs @@ -390,7 +597,7 @@ internal static partial class Logs "Could not resolve a dispatch endpoint for the destination {DestinationAddress} of message with ID {Id}. The message will be retried.")] public static partial void CouldNotDetermineDestination(this ILogger logger, Guid id, string? destinationAddress); - [LoggerMessage(4, LogLevel.Information, "Outbox processor is sleeping for {NextPollingInterval}.")] + [LoggerMessage(4, LogLevel.Debug, "Outbox processor is sleeping for {NextPollingInterval}.")] public static partial void OutboxProcessorSleeping(this ILogger logger, TimeSpan nextPollingInterval); [LoggerMessage(5, LogLevel.Error, "An unexpected error occurred while processing outbox event")] @@ -410,6 +617,57 @@ internal static partial class Logs [LoggerMessage(8, LogLevel.Information, "Expired outbox message with ID {Id} was discarded.")] public static partial void OutboxMessageExpired(this ILogger logger, Guid id); + + [LoggerMessage( + 9, + LogLevel.Warning, + "The dispatch of outbox message {Id} (message ID {MessageId}, type {MessageType}) to {DestinationAddress} failed because the transport is unavailable. The attempt is not counted and the message is dispatched again.")] + public static partial void OutboxTransportUnavailable( + this ILogger logger, + Exception exception, + Guid id, + string? messageId, + string? messageType, + string? destinationAddress); + + [LoggerMessage( + 10, + LogLevel.Warning, + "The dispatch of outbox message {Id} (message ID {MessageId}, type {MessageType}) to {DestinationAddress} did not complete within the dispatch timeout of {DispatchTimeout}. The attempt is not counted and the message is dispatched again.")] + public static partial void OutboxDispatchTimedOut( + this ILogger logger, + Exception exception, + Guid id, + string? messageId, + string? messageType, + string? destinationAddress, + TimeSpan dispatchTimeout); + + [LoggerMessage( + 11, + LogLevel.Debug, + "The dispatch of outbox message {Id} (message ID {MessageId}, type {MessageType}) to {DestinationAddress} was cancelled because the host is stopping. The attempt is not counted.")] + public static partial void OutboxDispatchCancelled( + this ILogger logger, + Exception exception, + Guid id, + string? messageId, + string? messageType, + string? destinationAddress); + + [LoggerMessage( + 12, + LogLevel.Error, + "The dispatch of outbox message {Id} (message ID {MessageId}, type {MessageType}) to {DestinationAddress} failed on attempt {Attempt} of {MaxAttempts}.")] + public static partial void OutboxDispatchFailed( + this ILogger logger, + Exception exception, + Guid id, + string? messageId, + string? messageType, + string? destinationAddress, + int attempt, + int maxAttempts); } file static class Serializer @@ -427,16 +685,4 @@ file static class Serializer return null; } } - - public static void TryCancel(this CancellationTokenSource cts) - { - try - { - cts.Cancel(); - } - catch - { - // ignore - } - } } diff --git a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/OutboxServiceCollectionExtensions.cs b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/OutboxServiceCollectionExtensions.cs index 1e16b12ac83..5d2da5b2bde 100644 --- a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/OutboxServiceCollectionExtensions.cs +++ b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/OutboxServiceCollectionExtensions.cs @@ -16,6 +16,9 @@ namespace Mocha.Outbox; /// public static class OutboxServiceCollectionExtensions { + // Stays below the longest delay a timer accepts, which is about 24.8 days. + private static readonly TimeSpan s_maxDispatchTimeout = TimeSpan.FromDays(24); + /// /// Registers the full Postgres outbox pipeline: table info discovery from the EF Core model, /// the , a hosted background worker, and a scoped @@ -43,7 +46,15 @@ public static IEntityFrameworkCoreBuilder UsePostgresOutbox( ArgumentNullException.ThrowIfNull(configure); var contextType = builder.ContextType; - builder.Services.AddOptions(builder.Name).Configure(configure); + builder.Services.AddOptions(builder.Name) + .Configure(configure) + .Validate(o => o.MaxConcurrentMessages > 0, "MaxConcurrentMessages must be greater than zero.") + .Validate( + o => o.DispatchTimeout == Timeout.InfiniteTimeSpan + || (o.DispatchTimeout > TimeSpan.Zero && o.DispatchTimeout <= s_maxDispatchTimeout), + "DispatchTimeout must be greater than zero and at most 24 days, or Timeout.InfiniteTimeSpan.") + .Validate(o => o.MaxAttempts > 0, "MaxAttempts must be greater than zero.") + .Validate(o => o.MaxRetryDelay > TimeSpan.Zero, "MaxRetryDelay must be greater than zero."); builder .Services.AddOptions(builder.Name) @@ -58,7 +69,8 @@ public static IEntityFrameworkCoreBuilder UsePostgresOutbox( builder .Services.AddOptions(builder.Name) - .Configure>((options, tableInfoMonitor) => + .Configure, IOptionsMonitor>( + (options, tableInfoMonitor, outboxOptionsMonitor) => { options.CreateConnection = services => { @@ -67,7 +79,11 @@ public static IEntityFrameworkCoreBuilder UsePostgresOutbox( }; var tableInfo = tableInfoMonitor.Get(builder.Name); - options.Queries = PostgresMessageOutboxQueries.From(tableInfo.Outbox); + var outboxOptions = outboxOptionsMonitor.Get(builder.Name); + options.Queries = PostgresMessageOutboxQueries.From(tableInfo.Outbox, outboxOptions); + options.MaxConcurrentMessages = outboxOptions.MaxConcurrentMessages; + options.DispatchTimeout = outboxOptions.DispatchTimeout; + options.MaxAttempts = outboxOptions.MaxAttempts; }); builder.Services.AddSingleton(sp => diff --git a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresMessageOutboxOptions.cs b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresMessageOutboxOptions.cs index a444c38cf27..e982753dc27 100644 --- a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresMessageOutboxOptions.cs +++ b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresMessageOutboxOptions.cs @@ -17,4 +17,20 @@ internal sealed class PostgresMessageOutboxOptions /// Gets or sets a factory that creates a closed connection using services from the current scope. /// public Func CreateConnection { get; set; } = null!; + + /// + /// Gets or sets the maximum number of messages dispatched concurrently. + /// + public int MaxConcurrentMessages { get; set; } = 1; + + /// + /// Gets or sets the longest time a single dispatch may take, or + /// for no limit. + /// + public TimeSpan DispatchTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets the number of failed dispatch attempts after which a message is no longer retried. + /// + public int MaxAttempts { get; set; } = 10; } diff --git a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresMessageOutboxQueries.cs b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresMessageOutboxQueries.cs index d6126db7e7a..c6e30e2d99b 100644 --- a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresMessageOutboxQueries.cs +++ b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresMessageOutboxQueries.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Mocha.EntityFrameworkCore.Postgres; namespace Mocha.Outbox; @@ -8,10 +9,9 @@ namespace Mocha.Outbox; /// internal sealed class PostgresMessageOutboxQueries { - /// - /// The number of dispatch attempts after which a message is no longer retried. - /// - public const int MaxAttempts = 10; + // Two to the power of 30 seconds is about 34 years, and a larger exponent would overflow for a high + // attempt count. + private const int MaxBackoffExponent = 30; /// /// Gets or sets the SQL statement to insert a new message envelope into the outbox table. @@ -38,9 +38,17 @@ internal sealed class PostgresMessageOutboxQueries /// Creates a new instance with SQL queries built from the provided table metadata. /// /// The outbox table info containing column and table names. + /// The outbox options that supply the attempt limit and the retry delay cap. /// A fully initialized instance. - public static PostgresMessageOutboxQueries From(OutboxTableInfo t) + public static PostgresMessageOutboxQueries From(OutboxTableInfo t, PostgresOutboxOptions options) { + var maxAttempts = options.MaxAttempts.ToString(CultureInfo.InvariantCulture); + var maxRetryDelay = options.MaxRetryDelay.TotalSeconds.ToString("R", CultureInfo.InvariantCulture); + var retryDelay = + $""" + INTERVAL '1 second' * LEAST(POWER(2, LEAST("{t.TimesSent}", {MaxBackoffExponent})), {maxRetryDelay}) + """; + return new PostgresMessageOutboxQueries { InsertEnvelope = $""" @@ -50,19 +58,19 @@ public static PostgresMessageOutboxQueries From(OutboxTableInfo t) NextPollingInterval = $""" SELECT MIN( - "{t.CreatedAt}" + INTERVAL '1 seconds' * POWER(2, "{t.TimesSent}") + "{t.CreatedAt}" + {retryDelay} ) AS "NextWakeUpTime" FROM {t.QualifiedTableName} - WHERE "{t.TimesSent}" < {MaxAttempts}; + WHERE "{t.TimesSent}" < {maxAttempts}; """, ProcessEvent = $""" UPDATE {t.QualifiedTableName} SET "{t.TimesSent}" = "{t.TimesSent}" + 1, - "{t.CreatedAt}" = NOW() + INTERVAL '1 second' * POWER(2, "{t.TimesSent}") + "{t.CreatedAt}" = NOW() + {retryDelay} WHERE "{t.Id}" = ( SELECT "{t.Id}" FROM {t.QualifiedTableName} - WHERE "{t.TimesSent}" < {MaxAttempts} AND "{t.CreatedAt}" <= NOW() + WHERE "{t.TimesSent}" < {maxAttempts} AND "{t.CreatedAt}" <= NOW() ORDER BY "{t.CreatedAt}", "{t.Id}" FOR UPDATE SKIP LOCKED LIMIT 1 diff --git a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresOutboxOptions.cs b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresOutboxOptions.cs index cc70588790a..4817c5d880d 100644 --- a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresOutboxOptions.cs +++ b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Outbox/PostgresOutboxOptions.cs @@ -10,4 +10,29 @@ public sealed class PostgresOutboxOptions /// transaction and throws an without one. The default is true. /// public bool RequireTransaction { get; set; } = true; + + /// + /// Gets or sets the maximum number of messages dispatched concurrently, which must be positive and defaults to one. + /// Values greater than one allow dispatches to complete out of order. + /// + public int MaxConcurrentMessages { get; set; } = 1; + + /// + /// Gets or sets the longest time a single dispatch may take before it is cancelled and the message is + /// released for a later attempt that does not count towards . The default is + /// 30 seconds, and switches the timeout off. + /// + public TimeSpan DispatchTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets the number of failed dispatch attempts after which a message is no longer retried and + /// stays in the outbox table. The value must be at least one and defaults to 10. + /// + public int MaxAttempts { get; set; } = 10; + + /// + /// Gets or sets the upper bound of the delay before a failed message is dispatched again, which doubles + /// with every failed attempt starting at one second. The value must be positive and defaults to 10 minutes. + /// + public TimeSpan MaxRetryDelay { get; set; } = TimeSpan.FromMinutes(10); } diff --git a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Scheduling/ScheduledMessageDispatcher.cs b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Scheduling/ScheduledMessageDispatcher.cs index 78df7cd4304..7572a22abc4 100644 --- a/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Scheduling/ScheduledMessageDispatcher.cs +++ b/src/Mocha/src/Mocha.EntityFrameworkCore.Postgres/Scheduling/ScheduledMessageDispatcher.cs @@ -128,6 +128,7 @@ private async Task ProcessMessageAsync(NpgsqlConnection connection, Cancel new ActivityContext()); await using var transaction = await connection.BeginTransactionAsync(cancellationToken); + var sent = false; try { @@ -182,9 +183,12 @@ private async Task ProcessMessageAsync(NpgsqlConnection connection, Cancel { await SendAsync(envelope, endpoint, messageType, cancellationToken); + // The transport has the message, so its row is removed whatever the token says. + sent = true; + await reader.CloseAsync(); - await DeleteMessageAsync(connection, id, transaction, cancellationToken); + await DeleteMessageAsync(connection, id, transaction, CancellationToken.None); } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -208,7 +212,7 @@ private async Task ProcessMessageAsync(NpgsqlConnection connection, Cancel { try { - await transaction.CommitAsync(cancellationToken); + await transaction.CommitAsync(sent ? CancellationToken.None : cancellationToken); } catch { diff --git a/src/Mocha/src/Mocha.Hosting/Configuration/MessageBusHealthCheckOptions.cs b/src/Mocha/src/Mocha.Hosting/Configuration/MessageBusHealthCheckOptions.cs index fe866abcae0..6f19d486223 100644 --- a/src/Mocha/src/Mocha.Hosting/Configuration/MessageBusHealthCheckOptions.cs +++ b/src/Mocha/src/Mocha.Hosting/Configuration/MessageBusHealthCheckOptions.cs @@ -2,5 +2,7 @@ namespace Mocha.Hosting; internal sealed class MessageBusHealthCheckOptions { - public Uri? Endpoint { get; set; } + public Uri? Endpoint { get; init; } + + public required TimeSpan Timeout { get; init; } } diff --git a/src/Mocha/src/Mocha.Hosting/Health/MessageBusHealthCheck.cs b/src/Mocha/src/Mocha.Hosting/Health/MessageBusHealthCheck.cs index a2e8987e08c..1c2d3043cfd 100644 --- a/src/Mocha/src/Mocha.Hosting/Health/MessageBusHealthCheck.cs +++ b/src/Mocha/src/Mocha.Hosting/Health/MessageBusHealthCheck.cs @@ -1,24 +1,84 @@ using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Options; +using Mocha.Events; namespace Mocha.Hosting; -internal sealed class MessageBusHealthCheck(IMessageBus dispatcher, IOptions options) +internal sealed class MessageBusHealthCheck( + IMessageBus dispatcher, + IMessagingRuntime runtime, + TimeProvider timeProvider, + MessageBusHealthCheckOptions options) : IHealthCheck { public async Task CheckHealthAsync( HealthCheckContext context, CancellationToken cancellationToken = default) { - var sendOptions = options.Value.Endpoint is { } endpoint - ? new SendOptions { Endpoint = endpoint } - : SendOptions.Default; + var failureStatus = context.Registration.FailureStatus; - var request = new HealthRequest("Health Check"); - var response = await dispatcher.RequestAsync(request, sendOptions, cancellationToken); + if (GetTransportHealthFailure(failureStatus) is { } unhealthy) + { + return unhealthy; + } + + var timeout = options.Timeout; + var sendOptions = new SendOptions + { + Endpoint = options.Endpoint, + ExpirationTime = timeProvider.GetUtcNow().Add(timeout) + }; + + HealthResponse response; + using var timeoutSource = new CancellationTokenSource(timeout, timeProvider); + using var requestSource = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutSource.Token); + + try + { + var request = new HealthRequest("Health Check"); + response = await dispatcher.RequestAsync(request, sendOptions, requestSource.Token); + } + catch (OperationCanceledException) + when (timeoutSource.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + return new HealthCheckResult(failureStatus, CreateTimeoutDescription(timeout)); + } + catch (ResponseTimeoutException ex) + { + return new HealthCheckResult(failureStatus, CreateTimeoutDescription(ex.Timeout), ex); + } + + if (GetTransportHealthFailure(failureStatus) is { } failedDuringRequest) + { + return failedDuringRequest; + } return response.Message == "OK" ? HealthCheckResult.Healthy("Message Bus is healthy.") - : HealthCheckResult.Unhealthy("Message Bus is unhealthy."); + : new HealthCheckResult(failureStatus, "Message Bus is unhealthy."); + } + + private HealthCheckResult? GetTransportHealthFailure(HealthStatus failureStatus) + { + string? description = null; + Dictionary? reasons = null; + + foreach (var transport in runtime.Transports) + { + if (transport.HealthFailureReason is { } reason) + { + description ??= $"Transport '{transport.Name}' is unhealthy. {reason}"; + reasons ??= []; + reasons[transport.Name] = reason; + } + } + + return description is null + ? null + : new HealthCheckResult(failureStatus, description, exception: null, reasons); } + + private static string CreateTimeoutDescription(TimeSpan timeout) + => $"Message Bus did not answer the health request within {timeout}."; } diff --git a/src/Mocha/src/Mocha.Hosting/Health/MessageBusHealthCheckExtensions.cs b/src/Mocha/src/Mocha.Hosting/Health/MessageBusHealthCheckExtensions.cs index 8d0832aa5eb..8225d264f4c 100644 --- a/src/Mocha/src/Mocha.Hosting/Health/MessageBusHealthCheckExtensions.cs +++ b/src/Mocha/src/Mocha.Hosting/Health/MessageBusHealthCheckExtensions.cs @@ -8,6 +8,9 @@ namespace Mocha.Hosting; /// public static class MessageBusHealthCheckExtensions { + private static readonly string[] s_defaultTags = ["ready"]; + private static readonly TimeSpan s_defaultTimeout = TimeSpan.FromSeconds(10); + /// /// Registers the built-in health request handler so the message bus can respond to health check requests. /// @@ -20,25 +23,52 @@ public static IMessageBusHostBuilder AddHealthCheck(this IMessageBusHostBuilder } /// - /// Adds a message bus health check to the ASP.NET Core health checks system, tagged with "ready" and "live". + /// Adds a message bus check to the ASP.NET Core health checks system. /// /// - /// The check sends a through the message bus and verifies the response. - /// When an is provided, the request is routed to that specific endpoint - /// rather than using the default routing. + /// The check requires every local transport to be healthy and a to be answered + /// within . Each unhealthy transport is listed with its reason in the result data. /// /// The health checks builder to extend. - /// An optional endpoint URI to target with the health check request. + /// + /// The endpoint the health request is sent to, or to use default routing. + /// + /// The name of the health check. + /// + /// The status reported when the check fails, or for . + /// + /// The tags of the health check, or for the tag "ready". + /// + /// The time within which the health request must be answered, or for 10 seconds. + /// The health request expires after this time. + /// /// The same instance for chaining. - public static IHealthChecksBuilder AddMessageBus(this IHealthChecksBuilder builder, Uri? endpoint = null) + /// is zero or negative. + public static IHealthChecksBuilder AddMessageBus( + this IHealthChecksBuilder builder, + Uri? endpoint = null, + string name = "MessageBus", + HealthStatus? failureStatus = null, + IEnumerable? tags = null, + TimeSpan? timeout = null) { - if (endpoint is not null) + if (timeout is { } value) { - builder.Services.Configure(o => o.Endpoint = endpoint); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(value, TimeSpan.Zero, nameof(timeout)); } - builder.AddCheck("MessageBus", HealthStatus.Unhealthy, ["ready", "live"]); + var options = new MessageBusHealthCheckOptions { Endpoint = endpoint, Timeout = timeout ?? s_defaultTimeout }; - return builder; + // The check applies the timeout itself, so the registration carries none. + return builder.Add( + new HealthCheckRegistration( + name, + services => new MessageBusHealthCheck( + services.GetRequiredService(), + services.GetRequiredService(), + services.GetService() ?? TimeProvider.System, + options), + failureStatus, + tags ?? s_defaultTags)); } } diff --git a/src/Mocha/src/Mocha.Threading/ExponentialBackoff.cs b/src/Mocha/src/Mocha.Threading/ExponentialBackoff.cs index 2f370208139..0fb300109d4 100644 --- a/src/Mocha/src/Mocha.Threading/ExponentialBackoff.cs +++ b/src/Mocha/src/Mocha.Threading/ExponentialBackoff.cs @@ -4,10 +4,9 @@ namespace Mocha.Threading; /// Implements an exponential backoff strategy with jitter to manage retry delays. /// /// -/// Uses a randomized multiplier derived from the base delay to avoid the thundering herd problem -/// when multiple consumers retry simultaneously. The delay doubles with each attempt, capped at -/// a configurable maximum. Call after a successful operation to restart the -/// backoff sequence. +/// The delay doubles with each attempt up to a configurable maximum, and each wait is randomized +/// between half of that value and the full value. Call after a successful +/// operation to restart the backoff sequence. /// public sealed class ExponentialBackoff { @@ -21,7 +20,7 @@ public sealed class ExponentialBackoff /// Creates a new exponential backoff instance with the specified retry limits and delay bounds. /// /// The maximum number of retry attempts before returns false. - /// The base delay used to compute the randomized multiplier for each backoff interval. + /// The base delay, which doubles with each retry. /// The upper bound on the computed delay; no single wait will exceed this duration. public ExponentialBackoff(int maxRetries, TimeSpan delay, TimeSpan maxDelay) { @@ -64,24 +63,43 @@ public async Task WaitAsync(CancellationToken cancellationToken) } /// - /// Computes the next backoff delay using the current power level and a randomized jitter multiplier. + /// Computes the randomized delay for the next retry. /// /// - /// A representing the delay, clamped to the configured maximum delay. + /// A representing the delay, which never exceeds the configured maximum delay. /// public TimeSpan CalculateDelay() + => CalculateDelay(_power, TimeSpan.FromMilliseconds(_delayInMs), TimeSpan.FromMilliseconds(_maxDelayInMs)); + + /// + /// Computes a randomized exponential delay for a failed attempt. The result lies between half of + /// the exponential delay and the exponential delay, which is capped at . + /// + /// The number of the failed attempt, starting at 1. + /// + /// The base delay, which doubles with each attempt. The first attempt waits between one and two base delays. + /// + /// The upper bound on the computed delay. + /// + /// or is negative. + /// + public static TimeSpan CalculateDelay(int attempt, TimeSpan delay, TimeSpan maxDelay) { - // we use a random multiplier because we don't want to have a thundering herd problem - var multiplier = Random.Shared.Next(_delayInMs / 2, _delayInMs); - if (multiplier == 0) + if (delay < TimeSpan.Zero) + { + throw ThrowHelper.DelayMustNotBeNegative(nameof(delay), delay); + } + + if (maxDelay < TimeSpan.Zero) { - multiplier = 1; + throw ThrowHelper.DelayMustNotBeNegative(nameof(maxDelay), maxDelay); } - var waitTimeInMs = Math.Pow(2, _power) * multiplier; + // delay * 2^attempt + var exponentialDelayInMs = Math.ScaleB(delay.TotalMilliseconds, Math.Max(attempt, 1)); + var ceilingInMs = Math.Min(maxDelay.TotalMilliseconds, exponentialDelayInMs); - // we make sure that we don't exceed the max delay - return TimeSpan.FromMilliseconds(Math.Min(_maxDelayInMs, waitTimeInMs)); + return TimeSpan.FromMilliseconds(ceilingInMs * (0.5 + Random.Shared.NextDouble() / 2)); } /// diff --git a/src/Mocha/src/Mocha.Threading/ThrowHelper.cs b/src/Mocha/src/Mocha.Threading/ThrowHelper.cs new file mode 100644 index 00000000000..458d19ea76f --- /dev/null +++ b/src/Mocha/src/Mocha.Threading/ThrowHelper.cs @@ -0,0 +1,7 @@ +namespace Mocha.Threading; + +internal static class ThrowHelper +{ + public static ArgumentOutOfRangeException DelayMustNotBeNegative(string parameterName, TimeSpan delay) + => new(parameterName, delay, "The delay must not be negative."); +} diff --git a/src/Mocha/src/Mocha.Transport.AzureServiceBus/AzureServiceBusReceiveEndpoint.cs b/src/Mocha/src/Mocha.Transport.AzureServiceBus/AzureServiceBusReceiveEndpoint.cs index 4478be16a3b..1f5ab3d7736 100644 --- a/src/Mocha/src/Mocha.Transport.AzureServiceBus/AzureServiceBusReceiveEndpoint.cs +++ b/src/Mocha/src/Mocha.Transport.AzureServiceBus/AzureServiceBusReceiveEndpoint.cs @@ -15,6 +15,8 @@ namespace Mocha.Transport.AzureServiceBus; public sealed class AzureServiceBusReceiveEndpoint(AzureServiceBusMessagingTransport transport) : ReceiveEndpoint(transport) { + private static readonly TimeSpan s_temporaryResourceCleanupTimeout = TimeSpan.FromSeconds(30); + private MessageProcessor? _processor; private QueueHeartbeat? _heartbeat; private ILogger _logger = null!; @@ -228,7 +230,8 @@ private Task OnProcessorError(ProcessErrorEventArgs args) /// Stops when the messaging runtime deactivates this endpoint, then disposes the heartbeat and /// processor resources. Disposal is best-effort: a failure while releasing either resource is /// logged as a warning and does not fail the stop. For a temporary endpoint, also removes the - /// Mocha-owned, auto-provisioned forwarding subscriptions that route to this endpoint's queue. + /// Mocha-owned, auto-provisioned forwarding subscriptions that route to this endpoint's queue, + /// whether or not is canceled. /// protected override async ValueTask OnStopAsync( IMessagingRuntimeContext context, @@ -240,11 +243,10 @@ protected override async ValueTask OnStopAsync( { await _processor.StopProcessingAsync(cancellationToken); } - - if (Configuration.IsTemporary) - { - await CleanupTemporaryResourcesAsync(cancellationToken); - } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The stop no longer waits for the processor. Disposing the processor below ends the processing. } finally { @@ -276,6 +278,29 @@ protected override async ValueTask OnStopAsync( _processor = null; } } + + if (Configuration.IsTemporary) + { + await CleanupTemporaryResourcesAsync(); + } + } + + /// + /// Removes the temporary resources of this endpoint within a fixed time. A cleanup that + /// does not complete in time is logged as a warning and does not fail the stop. + /// + private async Task CleanupTemporaryResourcesAsync() + { + using var timeout = new CancellationTokenSource(s_temporaryResourceCleanupTimeout); + + try + { + await CleanupTemporaryResourcesAsync(timeout.Token); + } + catch (OperationCanceledException ex) when (timeout.IsCancellationRequested) + { + _logger.TemporaryResourceCleanupTimedOut(ex, Queue.Name, s_temporaryResourceCleanupTimeout); + } } /// @@ -402,6 +427,15 @@ public static partial void TemporaryQueueCleanupFailed( Exception exception, string queueName); + [LoggerMessage( + LogLevel.Warning, + "Deleting the temporary resources of Azure Service Bus queue '{QueueName}' did not complete within {Timeout}")] + public static partial void TemporaryResourceCleanupTimedOut( + this ILogger logger, + Exception exception, + string queueName, + TimeSpan timeout); + [LoggerMessage( LogLevel.Warning, "Failed to dispose the keep-alive heartbeat for Azure Service Bus queue '{QueueName}'")] diff --git a/src/Mocha/src/Mocha.Transport.Postgres/PostgresMessagingTransport.cs b/src/Mocha/src/Mocha.Transport.Postgres/PostgresMessagingTransport.cs index 0b424018883..4bd82caaa9a 100644 --- a/src/Mocha/src/Mocha.Transport.Postgres/PostgresMessagingTransport.cs +++ b/src/Mocha/src/Mocha.Transport.Postgres/PostgresMessagingTransport.cs @@ -11,6 +11,8 @@ namespace Mocha.Transport.Postgres; /// public sealed class PostgresMessagingTransport : MessagingTransport { + private static readonly TimeSpan s_unregisterTimeout = TimeSpan.FromSeconds(5); + private readonly Action _configure; private readonly PostgresBackgroundTaskScheduler _backgroundTasks = new(); private IReadOnlyPostgresSchemaOptions _schemaOptions = null!; @@ -428,7 +430,9 @@ protected override async ValueTask OnAfterStopAsync(CancellationToken cancellati { try { - await ConsumerManager.UnregisterAsync(cancellationToken); + // Unregistering is cleanup, so it does not observe the stop token. + using var timeout = new CancellationTokenSource(s_unregisterTimeout); + await ConsumerManager.UnregisterAsync(timeout.Token); } catch { diff --git a/src/Mocha/src/Mocha.Transport.Postgres/PostgresReceiveEndpoint.cs b/src/Mocha/src/Mocha.Transport.Postgres/PostgresReceiveEndpoint.cs index 066b5eccb81..f62e0f71224 100644 --- a/src/Mocha/src/Mocha.Transport.Postgres/PostgresReceiveEndpoint.cs +++ b/src/Mocha/src/Mocha.Transport.Postgres/PostgresReceiveEndpoint.cs @@ -310,7 +310,9 @@ protected override async ValueTask OnStopAsync( { try { - await _pollingTask.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + // The loop is already cancelled. The wait covers its unwinding and the release of its + // leased messages, so it does not observe the stop token. + await _pollingTask.WaitAsync(TimeSpan.FromSeconds(5)); } catch (TimeoutException) { diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Configurations/RabbitMQDefaultQueueOptions.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Configurations/RabbitMQDefaultQueueOptions.cs index cb044fb4e6f..5171886ea41 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Configurations/RabbitMQDefaultQueueOptions.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Configurations/RabbitMQDefaultQueueOptions.cs @@ -7,7 +7,7 @@ public sealed class RabbitMQDefaultQueueOptions { /// /// Gets or sets the default queue type (Classic, Quorum, or Stream). - /// When set, all auto-provisioned queues will use this queue type unless explicitly overridden. + /// When no queue type is configured, RabbitMQ selects the queue type. /// public string? QueueType { get; set; } @@ -29,22 +29,22 @@ public sealed class RabbitMQDefaultQueueOptions public Dictionary Arguments { get; set; } = new(); /// - /// Applies these defaults to a queue configuration, without overriding explicitly set values. - /// Quorum and stream queue types are not applied to queues that have auto-delete or exclusive - /// set, since those properties are incompatible with these queue types. Default arguments are - /// also skipped for incompatible queues because they may contain queue-type-specific settings - /// (e.g. x-delivery-limit is only valid for quorum queues). + /// Applies queue defaults without overriding explicitly configured values. + /// Auto-delete and exclusive queues skip queue type and argument defaults; non-durable queues skip them + /// when the defaults specify quorum or stream. /// internal void ApplyTo(RabbitMQQueueConfiguration configuration) { configuration.Durable ??= Durable; configuration.AutoDelete ??= AutoDelete; - // Quorum and stream queues do not support auto-delete or exclusive properties. - // Skip applying queue type and default arguments for incompatible configurations - // (e.g. reply queues) since default arguments are often queue-type-specific. + var queueType = GetQueueType(); + var isIncompatibleWithQueueType = - configuration.AutoDelete is true || configuration.Exclusive is true; + configuration.AutoDelete is true + || configuration.Exclusive is true + || (configuration.Durable is false + && queueType is RabbitMQQueueType.Quorum or RabbitMQQueueType.Stream); if (isIncompatibleWithQueueType) { @@ -66,4 +66,26 @@ internal void ApplyTo(RabbitMQQueueConfiguration configuration) } } } + + /// + /// Applies quorum replication settings to a generated reply queue. + /// Message limits and queue lifecycle defaults are not applied. + /// + internal void ApplyToReplyQueue(RabbitMQQueueConfiguration configuration) + { + if (GetQueueType() is not RabbitMQQueueType.Quorum) + { + return; + } + + configuration.Arguments ??= new Dictionary(); + configuration.Arguments.TryAdd("x-queue-type", RabbitMQQueueType.Quorum); + if (Arguments.TryGetValue("x-quorum-initial-group-size", out var groupSize)) + { + configuration.Arguments.TryAdd("x-quorum-initial-group-size", groupSize); + } + } + + private string? GetQueueType() + => QueueType ?? (Arguments.TryGetValue("x-queue-type", out var value) ? value as string : null); } diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Configurations/RabbitMQTransportConfiguration.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Configurations/RabbitMQTransportConfiguration.cs index bf2313345ac..9252804c589 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Configurations/RabbitMQTransportConfiguration.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Configurations/RabbitMQTransportConfiguration.cs @@ -63,6 +63,19 @@ public RabbitMQTransportConfiguration() /// Gets or sets the bus-level defaults applied to all auto-provisioned queues and exchanges. /// public RabbitMQBusDefaults Defaults { get; set; } = new(); + + /// + /// Gets or sets how long the transport keeps retrying while the broker cannot be reached before its start fails, + /// or null for two minutes. + /// + public TimeSpan? ConnectTimeout { get; set; } + + /// + /// Gets or sets how long a publish waits for the broker's confirmation before it fails with + /// , or null for two minutes. + /// waits until the caller cancels. + /// + public TimeSpan? PublishTimeout { get; set; } } /// @@ -115,4 +128,15 @@ public async ValueTask CreateAsync(CancellationToken cancellationTo var connection = await factory.CreateConnectionAsync(cancellationToken); return connection; } + + /// + /// Creates a new connection, named unless the factory sets a name. + /// + internal async ValueTask CreateAsync(string clientProvidedName, CancellationToken cancellationToken) + { + var connection = await factory.CreateConnectionAsync( + factory.ClientProvidedName ?? clientProvidedName, + cancellationToken); + return connection; + } } diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/ChannelOpenBlockedException.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/ChannelOpenBlockedException.cs new file mode 100644 index 00000000000..2a8e1b5ab85 --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/ChannelOpenBlockedException.cs @@ -0,0 +1,8 @@ +namespace Mocha.Transport.RabbitMQ; + +/// +/// Thrown when the broker did not answer a request to open a channel because it blocked the connection. +/// The client keeps the channel number of the request until the connection is replaced. +/// +internal sealed class ChannelOpenBlockedException(string message, Exception innerException) + : TimeoutException(message, innerException); diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/PublishDeadline.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/PublishDeadline.cs new file mode 100644 index 00000000000..f07d3748049 --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/PublishDeadline.cs @@ -0,0 +1,51 @@ +namespace Mocha.Transport.RabbitMQ; + +/// +/// The cancellation of one publish: its token is cancelled when the publish timeout elapses or when the +/// caller cancels. Instances are reused, so a publish allocates none. +/// +internal sealed class PublishDeadline : IDisposable +{ + private readonly CancellationTokenSource _source = new(); + private CancellationTokenRegistration _caller; + + /// + /// Gets the token that the operations of the publish observe. + /// + public CancellationToken Token => _source.Token; + + /// + /// Gets whether the token was cancelled, by the timeout or by the caller. + /// + public bool IsCancellationRequested => _source.IsCancellationRequested; + + /// + /// Starts the timeout and follows the caller's token. + /// + public void Start(TimeSpan timeout, CancellationToken callerToken) + { + _source.CancelAfter(timeout); + if (callerToken.CanBeCanceled) + { + _caller = callerToken.UnsafeRegister(static state => ((CancellationTokenSource)state!).Cancel(), _source); + } + } + + /// + /// Stops the timeout and stops following the caller's token. Returns false when the token was + /// cancelled, after which the instance cannot be used again. + /// + public bool TryReset() + { + _caller.Dispose(); + _caller = default; + return _source.TryReset(); + } + + /// + public void Dispose() + { + _caller.Dispose(); + _source.Dispose(); + } +} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConnectionManager.Logging.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConnectionManager.Logging.cs new file mode 100644 index 00000000000..d0a0772f7ae --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConnectionManager.Logging.cs @@ -0,0 +1,95 @@ +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ; + +internal sealed partial class RabbitMQConnectionManager +{ + /// + /// Gets the properties that identify the connection in structured logs: service, instance, transport, + /// connection name and role, broker host, port and virtual host. + /// + public IReadOnlyDictionary LogContext { get; init; } = new Dictionary(); + + /// + /// Begins a logger scope that carries . + /// + public IDisposable? BeginLogScope() => _logger.BeginScope(LogContext); +} + +internal static partial class Logs +{ + [LoggerMessage(LogLevel.Debug, "Connecting RabbitMQ connection {ConnectionName} (attempt {Attempt})")] + public static partial void ConnectionAttemptStarted(this ILogger logger, string connectionName, int attempt); + + [LoggerMessage( + LogLevel.Information, + "RabbitMQ connection {ConnectionName} is ready after {Elapsed} (attempt {Attempt})")] + public static partial void ConnectionReady( + this ILogger logger, + string connectionName, + int attempt, + TimeSpan elapsed); + + [LoggerMessage( + Message = "RabbitMQ connection {ConnectionName} is unavailable for {Elapsed}: attempt {Attempt} failed. " + + "Retrying in {Delay}")] + public static partial void ConnectionAttemptFailed( + this ILogger logger, + Exception exception, + LogLevel level, + string connectionName, + int attempt, + TimeSpan elapsed, + TimeSpan delay); + + [LoggerMessage( + LogLevel.Error, + "RabbitMQ connection {ConnectionName} could not be established after {Elapsed} and {Attempt} attempts. " + + "The start fails")] + public static partial void ConnectionFailed( + this ILogger logger, + Exception exception, + string connectionName, + int attempt, + TimeSpan elapsed); + + [LoggerMessage( + LogLevel.Warning, + "RabbitMQ connection {ConnectionName} was lost: {ReplyCode} {ReplyText} (initiator: {Initiator}, " + + "cause: {Cause}). Reconnecting in the background")] + public static partial void ConnectionLost( + this ILogger logger, + string connectionName, + ushort replyCode, + string? replyText, + ShutdownInitiator initiator, + string? cause); + + [LoggerMessage(LogLevel.Warning, "Closing RabbitMQ connection {ConnectionName}: {Reason}")] + public static partial void ConnectionAborted(this ILogger logger, string connectionName, string reason); + + [LoggerMessage(LogLevel.Information, "RabbitMQ connection {ConnectionName} is closed")] + public static partial void ConnectionStopped(this ILogger logger, string connectionName); + + [LoggerMessage( + LogLevel.Warning, + "RabbitMQ connection {ConnectionName} is blocked by the broker: {Reason}. Publishes wait until the " + + "broker unblocks it")] + public static partial void ConnectionBlocked(this ILogger logger, string connectionName, string reason); + + [LoggerMessage( + LogLevel.Information, + "RabbitMQ connection {ConnectionName} was unblocked by the broker after {Elapsed}")] + public static partial void ConnectionUnblocked(this ILogger logger, string connectionName, TimeSpan elapsed); + + [LoggerMessage(LogLevel.Debug, "Closing a discarded RabbitMQ connection of {ConnectionName} failed")] + public static partial void ConnectionCloseFailed(this ILogger logger, Exception exception, string connectionName); + + [LoggerMessage(LogLevel.Error, "A callback of RabbitMQ connection {ConnectionName} failed. Detail: {Detail}")] + public static partial void ConnectionCallbackFailed( + this ILogger logger, + Exception exception, + string connectionName, + string detail); +} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConnectionManager.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConnectionManager.cs new file mode 100644 index 00000000000..90395bc6672 --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConnectionManager.cs @@ -0,0 +1,767 @@ +using System.Text; +using Microsoft.Extensions.Logging; +using Mocha.Threading; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; + +namespace Mocha.Transport.RabbitMQ; + +/// +/// Owns one RabbitMQ connection: connects it at start, hands it out while it is ready, and replaces it in +/// the background after it was lost. +/// +internal sealed partial class RabbitMQConnectionManager : IAsyncDisposable +{ + private static readonly TimeSpan s_initialRetryDelay = TimeSpan.FromSeconds(1); + private static readonly TimeSpan s_maxRetryDelay = TimeSpan.FromSeconds(10); + private static readonly TimeSpan s_closeTimeout = TimeSpan.FromSeconds(5); + private static readonly CancellationToken s_cancelled = new(canceled: true); + + private readonly ILogger _logger; + private readonly Func> _connectionFactory; + private readonly Func _setUpConnectionAsync; + private readonly CancellationTokenSource _disposeCts = new(); + private readonly object _sync = new(); + + // The ready connection, or null while there is none. + private IConnection? _connection; + + // Completes with the next ready connection, and is replaced when that connection is lost. + private TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); + + // The connect routine that runs or ran last. At most one runs at a time. + private Task _connecting = Task.CompletedTask; + // Completed while the ready connection is not blocked by the broker, and replaced while it is. + private TaskCompletionSource _unblocked = CreateCompletedSource(); + + private Task? _disposeTask; + private long _unavailableSince; + private long _blockedSince; + private bool _isDisposed; + + /// + /// Creates a connection manager. + /// + /// The logger for connection lifecycle events. + /// The name that identifies the connection in logs. + /// The clock used for elapsed times and retry delays. + /// Creates a new RabbitMQ connection. + /// + /// Prepares a new connection before it is handed out. A failure discards the connection and counts as a + /// failed attempt. + /// + public RabbitMQConnectionManager( + ILogger logger, + string connectionName, + TimeProvider timeProvider, + Func> connectionFactory, + Func setUpConnectionAsync) + { + _logger = logger; + _connectionFactory = connectionFactory; + _setUpConnectionAsync = setUpConnectionAsync; + ConnectionName = connectionName; + TimeProvider = timeProvider; + } + + /// + /// Gets the name that identifies the connection in logs. + /// + public string ConnectionName { get; } + + /// + /// Gets the clock used for elapsed times and retry delays. + /// + public TimeProvider TimeProvider { get; } + + /// + /// Gets or sets how long keeps retrying while the broker cannot be reached. + /// + public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromMinutes(2); + + /// + /// Gets or sets how long one connection attempt may take before it is abandoned and retried. + /// + public TimeSpan AttemptTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets the ready connection, or null while there is none. + /// + public IConnection? Connection => Volatile.Read(ref _connection) is { IsOpen: true } connection ? connection : null; + + /// + /// Gets whether a ready connection exists. + /// + public bool IsConnected => Connection is not null; + + /// + /// Gets whether the broker blocked the ready connection. The broker does not read from a blocked + /// connection, so publishes and requests on it stay unanswered until the broker unblocks it. + /// + public bool IsBlocked => Volatile.Read(ref _blockedSince) != 0 && IsConnected; + + /// + /// Waits while the broker blocks the ready connection. The wait also ends when the connection is lost. + /// + public Task WaitWhileBlockedAsync(CancellationToken cancellationToken) + { + var unblocked = Volatile.Read(ref _unblocked).Task; + return unblocked.IsCompleted ? Task.CompletedTask : unblocked.WaitAsync(cancellationToken); + } + + /// + /// Gets whether the manager was disposed. + /// + public bool IsDisposed => Volatile.Read(ref _isDisposed); + + /// + /// Gets the ready connection, waiting while there is none. The wait ends when a connection becomes + /// ready, when is cancelled, or when the manager is disposed. + /// + public ValueTask GetConnectionAsync(CancellationToken cancellationToken) + => Volatile.Read(ref _connection) is { IsOpen: true } connection + ? new ValueTask(connection) + : WaitForConnectionAsync(cancellationToken); + + private async ValueTask WaitForConnectionAsync(CancellationToken cancellationToken) + { + while (true) + { + Task? ready = null; + Task? connecting = null; + IConnection? closed = null; + lock (_sync) + { + ObjectDisposedException.ThrowIf(_isDisposed, this); + if (_connection is { } current) + { + if (current.IsOpen) + { + return current; + } + + closed = current; + } + else + { + if (_connecting.IsCompleted) + { + // The manager was not started, or its start failed. + _unavailableSince = TimeProvider.GetTimestamp(); + _connecting = Task.Run(() => ConnectWithRetryAsync(deadline: null, CancellationToken.None)); + } + + ready = _ready.Task; + connecting = _connecting; + } + } + + if (closed is not null) + { + // The connection closed and its shutdown event has not arrived yet. + OnConnectionLost(closed, GetCloseReason(closed)); + continue; + } + + // A start that fails ends its routine without a connection. The wait then starts the next one. + await Task.WhenAny(ready!, connecting!).WaitAsync(cancellationToken); + if (ready!.IsCompleted) + { + var connection = await ready; + if (connection.IsOpen) + { + return connection; + } + } + } + } + + /// + /// Connects, retrying with exponential backoff until elapsed. A failure that + /// repeats on every attempt, such as refused credentials, fails at once. + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + while (true) + { + Task? connecting = null; + Task? ready = null; + IConnection? closed = null; + lock (_sync) + { + ObjectDisposedException.ThrowIf(_isDisposed, this); + if (_connection is { } current) + { + if (current.IsOpen) + { + return; + } + + closed = current; + } + else if (_connecting.IsCompleted) + { + var timeout = ConnectTimeout; + _unavailableSince = TimeProvider.GetTimestamp(); + _connecting = connecting = Task.Run(() => ConnectWithRetryAsync(timeout, cancellationToken)); + } + else + { + ready = _ready.Task; + } + } + + if (closed is not null) + { + OnConnectionLost(closed, GetCloseReason(closed)); + continue; + } + + if (connecting is not null) + { + try + { + await connecting; + return; + } + catch (OperationCanceledException) + when (_disposeCts.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + throw ThrowHelper.ConnectionManagerDisposed(); + } + } + + // A recovery already runs in the background. The start waits for it as long as it would have + // kept trying itself. + try + { + await ready!.WaitAsync(ConnectTimeout, TimeProvider, cancellationToken); + return; + } + catch (TimeoutException ex) + { + throw ThrowHelper.ConnectionTimedOut(ConnectTimeout, ex); + } + } + } + + private async Task ConnectWithRetryAsync(TimeSpan? deadline, CancellationToken cancellationToken) + { + using var loopCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _disposeCts.Token); + var loopToken = loopCts.Token; + var started = TimeProvider.GetTimestamp(); + + for (var attempt = 1; ; attempt++) + { + loopToken.ThrowIfCancellationRequested(); + + // The last attempt before the deadline ends with the deadline. + var attemptTimeout = AttemptTimeout; + if (deadline is { } limit && limit > TimeSpan.Zero) + { + var remaining = limit - TimeProvider.GetElapsedTime(started); + if (remaining < attemptTimeout) + { + attemptTimeout = remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero; + } + } + + using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(loopToken); + attemptCts.CancelAfter(attemptTimeout); + + try + { + using (BeginLogScope()) + { + _logger.ConnectionAttemptStarted(ConnectionName, attempt); + } + + var connection = await ConnectOnceAsync(attemptCts.Token); + if (!TryHandOut(connection, attempt)) + { + // The manager was disposed while the attempt ran. Disposal cancelled the routine's token + // before it set the flag, so this ends the routine. + await CloseConnectionAsync(connection); + loopToken.ThrowIfCancellationRequested(); + } + + return; + } + catch (Exception ex) when (!loopToken.IsCancellationRequested) + { + // Nobody cancelled the routine, so a cancellation is a timeout: of the attempt, or of the + // client, which reports a request the broker did not answer in time as a cancellation. + var timedOut = ex is OperationCanceledException; + var failure = !timedOut ? ex + : attemptCts.IsCancellationRequested ? ThrowHelper.ConnectionAttemptTimedOut(attemptTimeout, ex) + : ThrowHelper.ConnectionRequestTimedOut(ex); + var delay = ExponentialBackoff.CalculateDelay(attempt, s_initialRetryDelay, s_maxRetryDelay); + var elapsed = TimeProvider.GetElapsedTime(Volatile.Read(ref _unavailableSince)); + + if (deadline is { } connectTimeout + && (RabbitMQFailure.IsPermanent(failure) + || TimeProvider.GetElapsedTime(started) + delay > connectTimeout)) + { + using (BeginLogScope()) + { + _logger.ConnectionFailed(failure, ConnectionName, attempt, elapsed); + } + + if (timedOut) + { + throw ThrowHelper.ConnectionTimedOut(connectTimeout, ex); + } + + throw; + } + + using (BeginLogScope()) + { + _logger.ConnectionAttemptFailed( + failure, + RabbitMQFailure.GetLogLevel(failure), + ConnectionName, + attempt, + elapsed, + delay); + } + + await Task.Delay(delay, TimeProvider, loopToken); + } + } + } + + private async Task ConnectOnceAsync(CancellationToken cancellationToken) + { + var connection = await _connectionFactory(cancellationToken); + try + { + connection.ConnectionShutdownAsync += (_, reason) => + { + OnConnectionLost(connection, reason); + return Task.CompletedTask; + }; + connection.CallbackExceptionAsync += OnCallbackExceptionAsync; + connection.ConnectionBlockedAsync += (_, e) => + { + OnConnectionBlocked(connection, e.Reason); + return Task.CompletedTask; + }; + connection.ConnectionUnblockedAsync += (_, _) => + { + OnConnectionUnblocked(connection); + return Task.CompletedTask; + }; + + // The client does not end a request with the token once the request is sent. + await _setUpConnectionAsync(connection, cancellationToken).WaitOrAbandonAsync(cancellationToken); + + if (!connection.IsOpen) + { + throw ThrowHelper.ConnectionClosedDuringSetup(connection.CloseReason); + } + + return connection; + } + catch + { + await CloseConnectionAsync(connection); + throw; + } + } + + private bool TryHandOut(IConnection connection, int attempt) + { + lock (_sync) + { + if (_isDisposed) + { + return false; + } + + Volatile.Write(ref _blockedSince, 0); + _unblocked.TrySetResult(); + Volatile.Write(ref _connection, connection); + _ready.TrySetResult(connection); + } + + using (BeginLogScope()) + { + _logger.ConnectionReady( + ConnectionName, + attempt, + TimeProvider.GetElapsedTime(Volatile.Read(ref _unavailableSince))); + } + + // A connection that closed before it was handed out raised its shutdown event to nobody. + if (!connection.IsOpen) + { + OnConnectionLost(connection, GetCloseReason(connection)); + } + + return true; + } + + private void OnConnectionLost(IConnection connection, ShutdownEventArgs? reason) + { + if (!TryBeginRecovery(connection)) + { + return; + } + + using (BeginLogScope()) + { + _logger.ConnectionLost( + ConnectionName, + reason?.ReplyCode ?? 0, + reason?.ReplyText, + reason?.Initiator ?? ShutdownInitiator.Library, + reason?.Exception?.Message); + } + } + + /// + /// Starts replacing the ready connection in the background. Returns false when + /// is not the ready connection, so only the first report for a connection + /// has an effect. + /// + private bool TryBeginRecovery(IConnection connection) + { + lock (_sync) + { + if (_isDisposed || !ReferenceEquals(_connection, connection)) + { + return false; + } + + // Callers that arrive from here on wait for the next connection. + _ready = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Volatile.Write(ref _connection, null); + Volatile.Write(ref _unavailableSince, TimeProvider.GetTimestamp()); + Volatile.Write(ref _blockedSince, 0); + _unblocked.TrySetResult(); + + // The client raises the shutdown event on its reader loop, which must not wait for the cleanup. + _connecting = Task.Run(() => RecoverAsync(connection)); + return true; + } + } + + private async Task RecoverAsync(IConnection lostConnection) + { + // The lost connection is closed first, so the broker has released what belongs to it, such as an + // exclusive queue. Disposing it also stops the client's own automatic recovery. + await CloseConnectionAsync(lostConnection); + await ConnectWithRetryAsync(deadline: null, CancellationToken.None); + } + + private Task OnCallbackExceptionAsync(object? sender, CallbackExceptionEventArgs e) + { + using (BeginLogScope()) + { + _logger.ConnectionCallbackFailed(e.Exception, ConnectionName, Print(e.Detail)); + } + + return Task.CompletedTask; + } + + private void OnConnectionBlocked(IConnection connection, string reason) + { + lock (_sync) + { + if (!ReferenceEquals(_connection, connection)) + { + return; + } + + if (_blockedSince == 0) + { + Volatile.Write(ref _blockedSince, TimeProvider.GetTimestamp()); + Volatile.Write(ref _unblocked, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + } + } + + using (BeginLogScope()) + { + _logger.ConnectionBlocked(ConnectionName, reason); + } + } + + private void OnConnectionUnblocked(IConnection connection) + { + long blockedSince; + lock (_sync) + { + if (!ReferenceEquals(_connection, connection)) + { + return; + } + + blockedSince = _blockedSince; + Volatile.Write(ref _blockedSince, 0); + _unblocked.TrySetResult(); + } + + using (BeginLogScope()) + { + _logger.ConnectionUnblocked( + ConnectionName, + blockedSince == 0 ? TimeSpan.Zero : TimeProvider.GetElapsedTime(blockedSince)); + } + } + + // A disposed connection of the client throws instead of giving its close reason. + private static ShutdownEventArgs? GetCloseReason(IConnection connection) + { + try + { + return connection.CloseReason; + } + catch (ObjectDisposedException) + { + return null; + } + } + + private static TaskCompletionSource CreateCompletedSource() + { + var source = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + source.SetResult(); + return source; + } + + /// + /// Opens a channel on a connection. The open does not end with a caller's token, because the client keeps + /// the channel number of a cancelled open until the connection closes. When the broker does not answer + /// within , the connection is closed, which starts a recovery. + /// + /// + /// The broker did not answer because it blocked the connection. The connection stays, and the channel + /// number of the open is lost until the connection is replaced. + /// + public async Task OpenChannelAsync( + IConnection connection, + CreateChannelOptions options, + TimeSpan timeout) + { + using var openCts = CancellationTokenSource.CreateLinkedTokenSource(_disposeCts.Token); + openCts.CancelAfter(timeout); + + try + { + // The client does not end the request with the token once the request is sent. + return await connection.CreateChannelAsync(options, openCts.Token).WaitOrAbandonAsync(openCts.Token); + } + catch (Exception) when (_disposeCts.IsCancellationRequested) + { + throw ThrowHelper.ConnectionManagerDisposed(); + } + catch (OperationCanceledException ex) + { + // The timeout elapsed, or the client's own timeout for a request did. + lock (_sync) + { + if (_blockedSince != 0 && ReferenceEquals(_connection, connection)) + { + // The broker does not read from a blocked connection, so the silence says nothing about + // the connection. Replacing it would send every pending publish again. + throw ThrowHelper.ChannelOpenBlocked(ex); + } + } + + // Closing the connection is the only way to give the channel number back, and it ends a request + // that is still in flight. + ReportUnresponsive(connection, "the broker did not answer the request to open a channel"); + throw ThrowHelper.ChannelOpenTimedOut(ex); + } + } + + /// + /// Replaces the ready connection because the broker stopped answering on it. The connection is closed in + /// the background before the next one is created. + /// + /// The connection the broker does not answer on. + /// Why the connection is replaced, for the log. + public void ReportUnresponsive(IConnection connection, string reason) + { + if (!TryBeginRecovery(connection)) + { + return; + } + + using (BeginLogScope()) + { + _logger.ConnectionAborted(ConnectionName, reason); + } + } + + /// + /// Closes and disposes a connection, waiting no longer than the close timeout for either step. + /// + private async Task CloseConnectionAsync(IConnection connection) + { + try + { + if (connection.IsOpen) + { + await connection.AbortAsync(s_closeTimeout); + } + } + catch (Exception ex) + { + LogCloseFailed(ex); + } + + try + { + // Disposing a connection whose broker stopped answering can take as long as the client's own timeouts. + await connection.DisposeAsync().AsTask().WaitAsync(s_closeTimeout); + } + catch (Exception ex) + { + LogCloseFailed(ex); + } + } + + private void LogCloseFailed(Exception exception) + { + using (BeginLogScope()) + { + _logger.ConnectionCloseFailed(exception, ConnectionName); + } + } + + /// + /// Closes and disposes a channel, ignoring failures of either step and waiting no longer than the close + /// timeout. A close that takes longer continues in the background. + /// + public static async Task CloseChannelAsync(IChannel channel) + { + var closing = CloseChannelCompletelyAsync(channel); + if (closing.IsCompleted) + { + return; + } + + try + { + await closing.WaitAsync(s_closeTimeout); + } + catch (TimeoutException) + { + // The broker does not answer. + } + } + + /// + /// Closes and disposes a channel, ignoring failures of either step. Completes when the client has given + /// the channel number back, which takes as long as the client's timeout for a request when the broker + /// does not answer. + /// + public static async Task CloseChannelCompletelyAsync(IChannel channel) + { + try + { + if (channel.IsOpen) + { + await channel.CloseAsync(); + } + else + { + // A connection with automatic recovery forgets a channel the broker closed only when the + // channel is closed here as well. The cancelled token keeps the client from waiting for a reply. + await channel.CloseAsync(Constants.ReplySuccess, "Goodbye", abort: true, s_cancelled); + } + } + catch + { + // Ignore close errors. + } + finally + { + try + { + await channel.DisposeAsync(); + } + catch + { + // Ignore disposal errors. + } + } + } + + /// + /// Stops connecting and closes the connection. Pending and later calls of + /// fail with . + /// + public ValueTask DisposeAsync() + { + lock (_sync) + { + return new ValueTask(_disposeTask ??= DisposeCoreAsync()); + } + } + + private async Task DisposeCoreAsync() + { + // Yields first, so the caller's lock is not held while the cancellation callbacks run. + await Task.Yield(); + await _disposeCts.CancelAsync(); + + IConnection? connection; + Task connecting; + lock (_sync) + { + _isDisposed = true; + connection = _connection; + _connection = null; + connecting = _connecting; + + _unblocked.TrySetResult(); + _ready.TrySetException(ThrowHelper.ConnectionManagerDisposed()); + + // Reading the exception marks it as observed when nobody waits for a connection. + _ = _ready.Task.Exception; + } + + try + { + await connecting; + } + catch + { + // The connect routine ended with the cancellation above, or failed the start it belonged to. + } + + if (connection is not null) + { + await CloseConnectionAsync(connection); + } + + using (BeginLogScope()) + { + _logger.ConnectionStopped(ConnectionName); + } + } + + private static string Print(IDictionary? detail) + { + if (detail is null || detail.Count == 0) + { + return string.Empty; + } + + var builder = new StringBuilder(); + foreach (var item in detail) + { + if (builder.Length > 0) + { + builder.Append(", "); + } + + builder.Append(item.Key).Append(": ").Append(item.Value); + } + + return builder.ToString(); + } +} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConnectionManagerBase.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConnectionManagerBase.cs deleted file mode 100644 index 6c460a0d7eb..00000000000 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConnectionManagerBase.cs +++ /dev/null @@ -1,351 +0,0 @@ -using System.Text; -using Microsoft.Extensions.Logging; -using RabbitMQ.Client; -using RabbitMQ.Client.Events; - -namespace Mocha.Transport.RabbitMQ; - -/// -/// Base class for RabbitMQ connection management with automatic reconnection, -/// exponential backoff, and connection event handling. -/// -public abstract class RabbitMQConnectionManagerBase : IAsyncDisposable -{ - private const int InitialConnectionRetryDelaySeconds = 1; - private const int MaxConnectionAttempts = 5; - private static readonly TimeSpan s_maxBackoffDelay = TimeSpan.FromSeconds(60); - - private readonly Func> _connectionFactory; - private readonly SemaphoreSlim _connectionLock = new(1, 1); - - private IConnection? _currentConnection; - private bool _isDisposed; - - /// - /// Creates a new connection manager base with the specified logger and connection factory. - /// - /// The logger for connection lifecycle events. - /// A factory delegate that creates new RabbitMQ connections on demand. - protected RabbitMQConnectionManagerBase( - ILogger logger, - Func> connectionFactory) - { - Logger = logger; - _connectionFactory = connectionFactory; - } - - /// - /// Gets the logger used for connection lifecycle events. - /// - protected ILogger Logger { get; } - - /// - /// Gets a value indicating whether the current connection is open and usable. - /// - public bool IsConnected => _currentConnection?.IsOpen ?? false; - - /// - /// Gets the current underlying RabbitMQ connection, or null if no connection has been established. - /// - protected IConnection? CurrentConnection => _currentConnection; - - /// - /// Gets a value indicating whether this manager has been disposed. - /// - protected bool IsDisposed => _isDisposed; - - /// - /// Gets the current connection, creating one if necessary. - /// - public async ValueTask GetConnectionAsync(CancellationToken cancellationToken) - { - ObjectDisposedException.ThrowIf(_isDisposed, this); - - var connection = Volatile.Read(ref _currentConnection); - if (connection is { IsOpen: true }) - { - return connection; - } - - await _connectionLock.WaitAsync(cancellationToken); - try - { - // Double-check after acquiring lock - connection = _currentConnection; - if (connection is { IsOpen: true }) - { - return connection; - } - - await CreateConnectionWithRetryAsync(cancellationToken); - return _currentConnection!; - } - finally - { - _connectionLock.Release(); - } - } - - /// - /// Ensures connection is established, creating it if necessary. - /// - public async Task EnsureConnectedAsync(CancellationToken cancellationToken = default) - { - await GetConnectionAsync(cancellationToken); - } - - /// - /// Called before a new connection is created. - /// Override to perform cleanup (e.g., clearing channel pools). - /// - protected virtual Task OnBeforeConnectionCreatedAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } - - /// - /// Called after a new connection is successfully created and setup actions have run. - /// Override to perform post-connection setup (e.g., reconnecting consumers). - /// - protected virtual Task OnAfterConnectionCreatedAsync(IConnection connection, CancellationToken cancellationToken) - { - return Task.CompletedTask; - } - - /// - /// Called when the connection is shut down (not by application). - /// Override to perform cleanup. - /// - protected virtual Task OnConnectionLostAsync() - { - return Task.CompletedTask; - } - - /// - /// Called after automatic recovery succeeds. - /// Override to perform post-recovery actions. - /// - protected virtual Task OnConnectionRecoveredAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } - - /// - /// Creates connection using the factory delegate with exponential backoff retry. - /// Must be called while holding _connectionLock. - /// - private async Task CreateConnectionWithRetryAsync(CancellationToken cancellationToken) - { - var attempt = 0; - var baseDelay = TimeSpan.FromSeconds(InitialConnectionRetryDelaySeconds); - - Logger.CreatingConnectionUsingFactoryDelegate(); - - while (attempt < MaxConnectionAttempts) - { - try - { - // Clean up existing connection if any - await DisposeConnectionInternalAsync(); - - // Allow derived classes to perform cleanup - await OnBeforeConnectionCreatedAsync(cancellationToken); - - // Create new connection - var connection = await _connectionFactory(cancellationToken); - - // Wire up connection events - WireConnectionEvents(connection); - - _currentConnection = connection; - - Logger.SuccessfullyCreatedConnection(connection.ClientProvidedName ?? "Unknown"); - - // Run setup actions (declare exchanges, queues, etc.) - await OnConnectionEstablished(connection, cancellationToken); - - // Allow derived classes to perform post-connection setup - await OnAfterConnectionCreatedAsync(connection, cancellationToken); - - return; - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - attempt++; - - if (attempt >= MaxConnectionAttempts) - { - Logger.FailedToCreateConnectionAfterAttempts(ex, MaxConnectionAttempts); - throw; - } - - var delay = CalculateBackoffDelay(baseDelay, attempt); - Logger.FailedToCreateConnectionRetrying(ex, attempt, MaxConnectionAttempts, delay); - - await Task.Delay(delay, cancellationToken); - } - } - } - - private void WireConnectionEvents(IConnection connection) - { - connection.CallbackExceptionAsync += OnCallbackExceptionAsync; - connection.ConnectionShutdownAsync += OnConnectionShutdownAsync; - connection.RecoverySucceededAsync += OnRecoverySucceededAsync; - connection.ConnectionRecoveryErrorAsync += OnConnectionRecoveryErrorAsync; - connection.ConnectionBlockedAsync += OnConnectionBlockedAsync; - connection.ConnectionUnblockedAsync += OnConnectionUnblockedAsync; - } - - private void UnwireConnectionEvents(IConnection connection) - { - connection.CallbackExceptionAsync -= OnCallbackExceptionAsync; - connection.ConnectionShutdownAsync -= OnConnectionShutdownAsync; - connection.RecoverySucceededAsync -= OnRecoverySucceededAsync; - connection.ConnectionRecoveryErrorAsync -= OnConnectionRecoveryErrorAsync; - connection.ConnectionBlockedAsync -= OnConnectionBlockedAsync; - connection.ConnectionUnblockedAsync -= OnConnectionUnblockedAsync; - } - - protected virtual Task OnConnectionEstablished(IConnection connection, CancellationToken cancellationToken) - => Task.CompletedTask; - - private Task OnCallbackExceptionAsync(object? sender, CallbackExceptionEventArgs e) - { - Logger.ExceptionInConnectionCallback(e.Exception, e.Detail.Print()); - - return Task.CompletedTask; - } - - private async Task OnConnectionShutdownAsync(object? sender, ShutdownEventArgs e) - { - if (e.Initiator == ShutdownInitiator.Application) - { - Logger.ConnectionClosedByApplication(); - return; - } - - Logger.ConnectionShutdownDetected(e.Initiator, e.ReplyText); - - await OnConnectionLostAsync(); - } - - private async Task OnRecoverySucceededAsync(object? sender, AsyncEventArgs e) - { - Logger.ConnectionRecoverySucceeded(); - - try - { - if (_currentConnection is not null) - { - await OnConnectionEstablished(_currentConnection, CancellationToken.None); - } - - await OnConnectionRecoveredAsync(CancellationToken.None); - - Logger.SuccessfullyRecoveredConnectionTopologyAndConsumers(); - } - catch (Exception ex) - { - Logger.ErrorDuringPostRecoveryOperations(ex); - } - } - - private Task OnConnectionRecoveryErrorAsync(object? sender, ConnectionRecoveryErrorEventArgs e) - { - Logger.ConnectionRecoveryFailedWillRetry(e.Exception, e.Exception?.Message); - return Task.CompletedTask; - } - - private Task OnConnectionBlockedAsync(object? sender, ConnectionBlockedEventArgs e) - { - Logger.ConnectionBlocked(e.Reason); - return Task.CompletedTask; - } - - private Task OnConnectionUnblockedAsync(object? sender, AsyncEventArgs e) - { - Logger.ConnectionUnblocked(); - return Task.CompletedTask; - } - - private static TimeSpan CalculateBackoffDelay(TimeSpan baseDelay, int attempt) - { - var delay = baseDelay * Math.Pow(2, attempt - 1); - return delay > s_maxBackoffDelay ? s_maxBackoffDelay : delay; - } - - private async Task DisposeConnectionInternalAsync() - { - if (_currentConnection is null) - { - return; - } - - try - { - UnwireConnectionEvents(_currentConnection); - - if (_currentConnection.IsOpen) - { - await _currentConnection.CloseAsync(); - } - - await _currentConnection.DisposeAsync(); - } - catch (Exception ex) - { - Logger.ErrorDisposingConnection(ex); - } - finally - { - _currentConnection = null; - } - } - - /// - /// Override to perform cleanup before base disposal. - /// - protected virtual ValueTask DisposeAsyncCore() - { - return ValueTask.CompletedTask; - } - - /// - /// Disposes the connection manager, calling for derived cleanup then closing the connection. - /// - public async ValueTask DisposeAsync() - { - if (_isDisposed) - { - return; - } - - _isDisposed = true; - - // Allow derived classes to clean up first - await DisposeAsyncCore(); - - // Dispose connection - await DisposeConnectionInternalAsync(); - - _connectionLock.Dispose(); - } -} - -file static class Extensions -{ - public static string Print(this IDictionary? obj) - { - var sb = new StringBuilder(); - foreach (var item in obj ?? Enumerable.Empty>()) - { - sb.AppendLine($"{item.Key}: {item.Value}"); - } - return sb.ToString(); - } -} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumer.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumer.cs new file mode 100644 index 00000000000..96a8232762b --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumer.cs @@ -0,0 +1,794 @@ +using System.Buffers; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Mocha.Threading; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; + +namespace Mocha.Transport.RabbitMQ; + +/// +/// Consumes one queue. A single task attaches the consumer to a channel, waits until the channel or the +/// broker-side consumer is lost, and attaches again, until the consumer is stopped. +/// +internal sealed class RabbitMQConsumer : IAsyncDisposable +{ + private static readonly CancellationToken s_noDrain = new(canceled: true); + private static readonly TimeSpan s_initialRetryDelay = TimeSpan.FromSeconds(1); + private static readonly TimeSpan s_maxRetryDelay = TimeSpan.FromSeconds(30); + + private static readonly CreateChannelOptions s_channelOptions = new( + publisherConfirmationsEnabled: false, + publisherConfirmationTrackingEnabled: false, + consumerDispatchConcurrency: 1); + + private readonly RabbitMQConsumerManager _manager; + private readonly Func? _prepareAsync; + private readonly Func _messageHandler; + + // Ends the lifecycle task. + private readonly CancellationTokenSource _stopRequested = new(); + + // Ends the wait for running handlers during a stop. + private readonly CancellationTokenSource _drainDeadline = new(); + + // One queue for the lifetime of the consumer. Every delivery names the attachment it arrived on. + private readonly Channel _deliveries = + System.Threading.Channels.Channel.CreateUnbounded(); + + // One permit per concurrent handler. A handler holds its permit until it returns, also after its + // channel was lost. + private readonly SemaphoreSlim _slots; + + private readonly TaskCompletionSource _firstAttempt = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly object _stopSync = new(); + private readonly Task _lifecycle; + private readonly Task _workers; + + // Written by the lifecycle task, and by the stop after the lifecycle task ended. + private volatile Attachment? _attachment; + private volatile ConsumerState _state; + + // Completed and replaced on every change of the state. + private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously); + private Task? _stopTask; + private bool _wasAttached; + + public RabbitMQConsumer( + RabbitMQConsumerManager manager, + string queueName, + Func? prepareAsync, + Func messageHandler, + ushort maxPrefetch, + ushort maxConcurrency) + { + _manager = manager; + _prepareAsync = prepareAsync; + _messageHandler = messageHandler; + QueueName = queueName; + MaxPrefetch = maxPrefetch; + MaxConcurrency = maxConcurrency; + _slots = new SemaphoreSlim(maxConcurrency, maxConcurrency); + + var workers = new Task[maxConcurrency]; + for (var i = 0; i < workers.Length; i++) + { + workers[i] = Task.Run(WorkAsync); + } + + _workers = Task.WhenAll(workers); + _lifecycle = Task.Run(RunAsync); + } + + /// + /// Gets the name of the queue the consumer consumes. + /// + public string QueueName { get; } + + /// + /// Gets the largest number of unacknowledged messages the broker delivers to the consumer. + /// + public ushort MaxPrefetch { get; } + + /// + /// Gets the largest number of messages that are handled at the same time. + /// + public ushort MaxConcurrency { get; } + + /// + /// Gets the consumer tag the broker assigned, or null while the consumer is not attached. + /// + public string? ConsumerTag => _attachment?.Tag; + + /// + /// Gets the channel the consumer receives deliveries on, or null while it is not attached. + /// + public IChannel? Channel => _attachment?.Channel; + + /// + /// Gets whether the consumer is attached to an open channel and the broker delivers to it. + /// + public bool IsAttached => HealthFailureReason is null; + + /// + /// Gets the number of handlers that have not returned, including cancelled handlers of a lost channel. + /// + public int RunningHandlers => Math.Max(0, MaxConcurrency - _slots.CurrentCount); + + /// + /// Gets the reason the consumer does not consume, or null when it is attached. + /// + public string? HealthFailureReason + => _state switch + { + ConsumerState.Attached when _attachment is { IsLost: false, Channel.IsOpen: true } => null, + ConsumerState.WaitingForHandlers => "The consumer waits for a handler of a lost channel to return.", + ConsumerState.Stopped => "The consumer is stopped.", + _ => "The consumer is not attached." + }; + + /// + /// Gets a task that completes after the first attempt to attach: with null when the consumer + /// attached or no connection was ready, and with the failure when the attempt failed. + /// + public Task FirstAttempt => _firstAttempt.Task; + + /// + /// Waits until the consumer is attached. Fails when the consumer is stopped. + /// + public ValueTask WaitUntilAttachedAsync(CancellationToken cancellationToken) + { + return IsAttached ? ValueTask.CompletedTask : WaitAsync(cancellationToken); + + async ValueTask WaitAsync(CancellationToken cancellationToken) + { + while (true) + { + // Read before the state, so a change after the check below completes this task. + var changed = Volatile.Read(ref _stateChanged).Task; + if (IsAttached) + { + return; + } + + if (_state == ConsumerState.Stopped) + { + throw ThrowHelper.ConsumerStopped(QueueName); + } + + await changed.WaitAsync(cancellationToken); + } + } + } + + private void SetState(ConsumerState state) + { + _state = state; + Interlocked + .Exchange(ref _stateChanged, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)) + .TrySetResult(); + } + + private async Task RunAsync() + { + var stop = _stopRequested.Token; + var connections = _manager.Connection; + var timeProvider = connections.TimeProvider; + var logger = _manager.Logger; + var detachedSince = timeProvider.GetTimestamp(); + var attempt = 0; + + try + { + while (true) + { + SetState(ConsumerState.WaitingForConnection); + if (!connections.IsConnected) + { + // A registration does not wait for an outage to end. + _firstAttempt.TrySetResult(null); + } + + IConnection connection; + Attachment attachment; + try + { + connection = await connections.GetConnectionAsync(stop); + SetState(ConsumerState.Attaching); + attachment = await AttachAsync(connection, stop); + } + catch (Exception ex) when (!stop.IsCancellationRequested) + { + attempt++; + var delay = ExponentialBackoff.CalculateDelay(attempt, s_initialRetryDelay, s_maxRetryDelay); + + // A first attempt the broker refuses fails the registration, which reports it. + if (!_firstAttempt.TrySetResult(ex) || !RabbitMQFailure.IsPermanent(ex)) + { + using (connections.BeginLogScope()) + { + logger.ConsumerAttachFailed( + ex, + RabbitMQFailure.GetLogLevel(ex), + QueueName, + connections.ConnectionName, + attempt, + timeProvider.GetElapsedTime(detachedSince), + delay); + } + } + + await Task.Delay(delay, timeProvider, stop); + continue; + } + + _attachment = attachment; + _wasAttached = true; + SetState(ConsumerState.Attached); + _firstAttempt.TrySetResult(null); + using (connections.BeginLogScope()) + { + logger.ConsumerAttached( + attachment.Tag, + QueueName, + connections.ConnectionName, + timeProvider.GetElapsedTime(detachedSince), + MaxPrefetch, + MaxConcurrency); + } + + // Completes on a channel shutdown and on a cancel by the broker, also when either came first. + var reason = await attachment.Lost.WaitAsync(stop); + + detachedSince = timeProvider.GetTimestamp(); + _attachment = null; + SetState(ConsumerState.WaitingForConnection); + var cancelledHandlers = RunningHandlers; + attachment.CancelHandlers(); + LogLoss(reason, cancelledHandlers, isConnectionLost: !connection.IsOpen); + await RabbitMQConnectionManager.CloseChannelAsync(attachment.Channel); + + // An attachment that did not last counts as a failed attempt, so a consumer that the broker + // keeps cancelling backs off. + attempt = timeProvider.GetElapsedTime(attachment.AttachedAt, detachedSince) < s_initialRetryDelay + ? attempt + 1 + : 0; + if (attempt > 0) + { + var delay = ExponentialBackoff.CalculateDelay(attempt, s_initialRetryDelay, s_maxRetryDelay); + await Task.Delay(delay, timeProvider, stop); + } + } + } + catch (Exception) when (stop.IsCancellationRequested) + { + // The stop takes over. A registration that still waits for the first attempt ends with it. + _firstAttempt.TrySetResult(null); + } + catch (Exception ex) + { + _firstAttempt.TrySetResult(ex); + SetState(ConsumerState.Stopped); + if (!connections.IsDisposed) + { + using (connections.BeginLogScope()) + { + logger.ConsumerFailed(ex, QueueName, connections.ConnectionName); + } + } + } + } + + /// + /// Opens a channel, prepares it, waits for a free handler slot and starts consuming. + /// + private async Task AttachAsync(IConnection connection, CancellationToken stop) + { + var connections = _manager.Connection; + var timeout = _manager.AttachTimeout; + + // The broker does not answer on a connection it blocked. + await connections.WaitWhileBlockedAsync(stop); + + var opening = connections.OpenChannelAsync(connection, s_channelOptions, timeout); + IChannel channel; + try + { + channel = await opening.WaitAsync(stop); + } + catch (OperationCanceledException) when (stop.IsCancellationRequested) + { + // The consumer was stopped. The channel is closed once it is open. + _ = CloseWhenOpenedAsync(opening); + throw; + } + + var attachment = new Attachment(this, channel); + using var attachCts = CancellationTokenSource.CreateLinkedTokenSource(stop); + var attachToken = attachCts.Token; + try + { + channel.ChannelShutdownAsync += attachment.OnChannelShutdownAsync; + + attachCts.CancelAfter(timeout); + if (_prepareAsync is not null) + { + await _prepareAsync(connection, channel, attachToken).WaitOrAbandonAsync(attachToken); + } + + // The queue and its bindings exist from here on, also while the consumer waits for a handler. + attachCts.CancelAfter(Timeout.InfiniteTimeSpan); + await WaitForFreeSlotAsync(stop); + + attachCts.CancelAfter(timeout); + await channel + .BasicQosAsync(prefetchSize: 0, prefetchCount: MaxPrefetch, global: false, attachToken) + .WaitOrAbandonAsync(attachToken); + + attachment.AttachedAt = connections.TimeProvider.GetTimestamp(); + attachment.Tag = await channel + .BasicConsumeAsync(QueueName, autoAck: false, consumer: attachment, attachToken) + .WaitOrAbandonAsync(attachToken); + + return attachment; + } + catch (Exception ex) + { + // A request may still be in flight, so the channel is closed without waiting for it. + attachment.StopAccepting(); + _ = RabbitMQConnectionManager.CloseChannelAsync(channel); + + if (ex is not OperationCanceledException || stop.IsCancellationRequested) + { + throw; + } + + // Without the timeout the client's own timeout for a request ended the attempt. + throw attachCts.IsCancellationRequested + ? ThrowHelper.ConsumerAttachTimedOut(QueueName, timeout) + : ThrowHelper.ConsumerAttachRequestTimedOut(QueueName, ex); + } + } + + private static async Task CloseWhenOpenedAsync(Task opening) + { + try + { + await RabbitMQConnectionManager.CloseChannelAsync(await opening); + } + catch + { + // The channel could not be opened. + } + } + + /// + /// Waits until at least one handler slot is free. Handlers of a lost channel keep their slot until they + /// return, and a new attachment needs one free slot. + /// + private async ValueTask WaitForFreeSlotAsync(CancellationToken stop) + { + if (_slots.CurrentCount > 0) + { + return; + } + + SetState(ConsumerState.WaitingForHandlers); + + // The handlers were cancelled when their channel was lost. Only a handler that ignores the + // cancellation is still running after the grace period. + var gracePeriod = _manager.BlockedGracePeriod; + if (!await _slots.WaitAsync(gracePeriod, stop)) + { + var connections = _manager.Connection; + var blockedSince = connections.TimeProvider.GetTimestamp(); + using (connections.BeginLogScope()) + { + _manager.Logger.ConsumerBlockedByHandlers( + QueueName, + connections.ConnectionName, + MaxConcurrency, + gracePeriod); + } + + await _slots.WaitAsync(stop); + + using (connections.BeginLogScope()) + { + _manager.Logger.ConsumerUnblocked( + QueueName, + connections.ConnectionName, + connections.TimeProvider.GetElapsedTime(blockedSince) + gracePeriod); + } + } + + _slots.Release(); + SetState(ConsumerState.Attaching); + } + + private void LogLoss(ShutdownEventArgs? reason, int cancelledHandlers, bool isConnectionLost) + { + var connections = _manager.Connection; + using (connections.BeginLogScope()) + { + if (reason is null) + { + _manager.Logger.ConsumerCancelled(QueueName, connections.ConnectionName, cancelledHandlers); + return; + } + + // The loss of the connection has its own line. This one adds what happened to running handlers. + var level = isConnectionLost && cancelledHandlers == 0 ? LogLevel.Debug : LogLevel.Warning; + _manager.Logger.ConsumerChannelLost( + level, + QueueName, + connections.ConnectionName, + reason.ReplyCode, + reason.ReplyText, + reason.Initiator, + cancelledHandlers); + } + } + + private async Task WorkAsync() + { + var reader = _deliveries.Reader; + while (await reader.WaitToReadAsync()) + { + while (reader.TryRead(out var delivery)) + { + try + { + // What a lost or stopping attachment left in the queue is skipped. The broker delivers + // it again once the channel is closed. + var attachment = delivery.Attachment; + if (!attachment.IsAccepting) + { + continue; + } + + await _slots.WaitAsync(); + try + { + await _messageHandler(attachment.Channel, delivery.EventArgs, attachment.HandlerToken); + } + catch (OperationCanceledException) when (attachment.HandlerToken.IsCancellationRequested) + { + // The channel was lost or the stop stopped waiting. + } + catch (Exception ex) + { + LogDeliveryFailed(ex, delivery.EventArgs); + } + finally + { + _slots.Release(); + } + } + finally + { + ArrayPool.Shared.Return(delivery.Buffer); + } + } + } + } + + private void LogDeliveryFailed(Exception exception, BasicDeliverEventArgs delivery) + { + var connections = _manager.Connection; + using (connections.BeginLogScope()) + { + _manager.Logger.DeliveryFailed( + exception, + delivery.BasicProperties.MessageId, + QueueName, + connections.ConnectionName, + delivery.ConsumerTag, + delivery.DeliveryTag, + delivery.BasicProperties.CorrelationId); + } + } + + /// + /// Stops consuming and waits for running handlers until is cancelled, then + /// cancels the handlers that still run and closes the channel. Closing the channel does not depend on + /// the token, and the broker delivers unacknowledged messages again. + /// + public Task StopAsync(CancellationToken drainToken) + { + Task stopTask; + lock (_stopSync) + { + stopTask = _stopTask ??= StopCoreAsync(); + } + + return stopTask.IsCompleted ? stopTask : WaitForStopAsync(stopTask, drainToken); + } + + private async Task WaitForStopAsync(Task stopTask, CancellationToken drainToken) + { + // The token of any caller ends the wait for running handlers. + await using var registration = drainToken.UnsafeRegister( + static state => ((CancellationTokenSource)state!).Cancel(), + _drainDeadline); + + await stopTask; + } + + private async Task StopCoreAsync() + { + // Leaves the caller's lock. + await Task.Yield(); + + var connections = _manager.Connection; + var logger = _manager.Logger; + var started = connections.TimeProvider.GetTimestamp(); + var drainToken = _drainDeadline.Token; + + await _stopRequested.CancelAsync(); + await _lifecycle; + + // From here on this method owns the attachment. + var attachment = _attachment; + _attachment = null; + + if (RunningHandlers > 0) + { + using (connections.BeginLogScope()) + { + logger.ConsumerDrainStarted(QueueName, connections.ConnectionName, RunningHandlers); + } + } + + if (attachment is not null) + { + if (attachment is { IsLost: false, Channel.IsOpen: true }) + { + try + { + // The broker stops delivering. Handlers that run settle their messages on the open channel. + await attachment.Channel + .BasicCancelAsync(attachment.Tag, noWait: false, drainToken) + .WaitOrAbandonAsync(drainToken); + } + catch + { + // The channel is closed below. + } + + attachment.StopAccepting(); + } + else + { + // The channel is gone, so its handlers cannot settle their messages any more. + attachment.CancelHandlers(); + } + } + + // The workers finish the handlers they started, skip what is left in the queue, and end. + _deliveries.Writer.TryComplete(); + try + { + await _workers.WaitAsync(drainToken); + } + catch (OperationCanceledException) + { + var runningHandlers = RunningHandlers; + if (runningHandlers > 0) + { + using (connections.BeginLogScope()) + { + logger.ConsumerDrainTimedOut( + QueueName, + connections.ConnectionName, + runningHandlers, + connections.TimeProvider.GetElapsedTime(started)); + } + } + + attachment?.CancelHandlers(); + } + + if (attachment is not null) + { + await RabbitMQConnectionManager.CloseChannelAsync(attachment.Channel); + } + + SetState(ConsumerState.Stopped); + _manager.Remove(this); + + if (_wasAttached) + { + using (connections.BeginLogScope()) + { + logger.ConsumerStopped( + QueueName, + connections.ConnectionName, + connections.TimeProvider.GetElapsedTime(started), + RunningHandlers); + } + } + } + + /// + /// Stops consuming, cancels running handlers without waiting for them, and closes the channel. + /// + public async ValueTask DisposeAsync() => await StopAsync(s_noDrain); + + private readonly record struct Delivery(Attachment Attachment, BasicDeliverEventArgs EventArgs, byte[] Buffer); + + /// + /// The broker-side consumer on one channel. A lost attachment is never used again. + /// + private sealed class Attachment : AsyncDefaultBasicConsumer + { + // Never disposed: it has no timer and is not linked, so it holds nothing that needs disposing. + private readonly CancellationTokenSource _handlers = new(); + private readonly TaskCompletionSource _lost = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly RabbitMQConsumer _owner; + private volatile bool _isAccepting = true; + + public Attachment(RabbitMQConsumer owner, IChannel channel) + : base(channel) + { + _owner = owner; + HandlerToken = _handlers.Token; + } + + /// + /// Gets or sets the consumer tag the broker assigned. + /// + public string Tag { get; set; } = string.Empty; + + /// + /// Gets or sets the timestamp at which consuming was requested. + /// + public long AttachedAt { get; set; } + + /// + /// Gets a task that completes when the channel shut down, with the reason, or when the broker + /// cancelled the consumer, with null. + /// + public Task Lost => _lost.Task; + + /// + /// Gets whether the channel shut down or the broker cancelled the consumer. + /// + public bool IsLost => _lost.Task.IsCompleted; + + /// + /// Gets the token that is cancelled for the handlers of this attachment. + /// + public CancellationToken HandlerToken { get; } + + /// + /// Gets whether handlers may still start for deliveries of this attachment. + /// + public bool IsAccepting => _isAccepting; + + public override Task HandleBasicDeliverAsync( + string consumerTag, + ulong deliveryTag, + bool redelivered, + string exchange, + string routingKey, + IReadOnlyBasicProperties properties, + ReadOnlyMemory body, + CancellationToken cancellationToken = default) + { + if (!_isAccepting) + { + // The broker delivers the message again once the channel is closed. + return Task.CompletedTask; + } + + // The client returns the body's buffer to its pool once this callback returns. + var buffer = ArrayPool.Shared.Rent(body.Length); + body.CopyTo(buffer); + + var eventArgs = new BasicDeliverEventArgs( + consumerTag, + deliveryTag, + redelivered, + exchange, + routingKey, + properties, + buffer.AsMemory(0, body.Length), + HandlerToken); + + if (!_owner._deliveries.Writer.TryWrite(new Delivery(this, eventArgs, buffer))) + { + ArrayPool.Shared.Return(buffer); + } + + return Task.CompletedTask; + } + + public override async Task HandleBasicCancelAsync( + string consumerTag, + CancellationToken cancellationToken = default) + { + await base.HandleBasicCancelAsync(consumerTag, cancellationToken); + OnLost(reason: null); + } + + public Task OnChannelShutdownAsync(object? _, ShutdownEventArgs reason) + { + OnLost(reason); + return Task.CompletedTask; + } + + private void OnLost(ShutdownEventArgs? reason) + { + _isAccepting = false; + _lost.TrySetResult(reason); + } + + /// + /// Stops starting handlers for deliveries of this attachment. + /// + public void StopAccepting() => _isAccepting = false; + + /// + /// Stops starting handlers and cancels the handlers that run. + /// + public void CancelHandlers() + { + _isAccepting = false; + _ = CancelHandlersAsync(); + } + + private async Task CancelHandlersAsync() + { + try + { + // The callbacks of handlers run on the thread pool, not on the caller. + await _handlers.CancelAsync(); + } + catch (Exception ex) + { + var connections = _owner._manager.Connection; + using (connections.BeginLogScope()) + { + _owner._manager.Logger.HandlerCancellationFailed( + ex, + _owner.QueueName, + connections.ConnectionName); + } + } + } + } +} + +/// +/// Identifies what a consumer is doing. +/// +internal enum ConsumerState +{ + /// + /// The consumer waits for a ready connection. + /// + WaitingForConnection, + + /// + /// The consumer opens a channel, declares its queue and starts consuming. + /// + Attaching, + + /// + /// The consumer declared its queue and waits until a handler of a lost channel returns. + /// + WaitingForHandlers, + + /// + /// The broker delivers messages to the consumer. + /// + Attached, + + /// + /// The consumer was stopped. + /// + Stopped +} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumerManager.Logging.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumerManager.Logging.cs new file mode 100644 index 00000000000..caa0da21623 --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumerManager.Logging.cs @@ -0,0 +1,182 @@ +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ; + +internal static partial class Logs +{ + [LoggerMessage( + LogLevel.Information, + "Consumer {ConsumerTag} is attached to queue {Queue} on {ConnectionName} after {Elapsed} " + + "(prefetch {MaxPrefetch}, concurrency {MaxConcurrency})")] + public static partial void ConsumerAttached( + this ILogger logger, + string consumerTag, + string queue, + string connectionName, + TimeSpan elapsed, + ushort maxPrefetch, + ushort maxConcurrency); + + [LoggerMessage( + Message = "The consumer for queue {Queue} on {ConnectionName} is detached for {Elapsed}: attempt {Attempt} " + + "failed. Retrying in {Delay}")] + public static partial void ConsumerAttachFailed( + this ILogger logger, + Exception exception, + LogLevel level, + string queue, + string connectionName, + int attempt, + TimeSpan elapsed, + TimeSpan delay); + + [LoggerMessage( + LogLevel.Error, + "The broker refused the consumer for queue {Queue} on {ConnectionName}. Its endpoint does not start")] + public static partial void ConsumerRegistrationFailed( + this ILogger logger, + Exception exception, + string queue, + string connectionName); + + [LoggerMessage( + Message = "The consumer for queue {Queue} on {ConnectionName} lost its channel: {ReplyCode} {ReplyText} " + + "(initiator: {Initiator}). {CancelledHandlers} running handlers were cancelled, and the broker " + + "delivers their messages again")] + public static partial void ConsumerChannelLost( + this ILogger logger, + LogLevel level, + string queue, + string connectionName, + ushort replyCode, + string replyText, + ShutdownInitiator initiator, + int cancelledHandlers); + + [LoggerMessage( + LogLevel.Warning, + "The broker cancelled the consumer for queue {Queue} on {ConnectionName}, which it does when the queue " + + "is deleted. {CancelledHandlers} running handlers were cancelled. Attaching again")] + public static partial void ConsumerCancelled( + this ILogger logger, + string queue, + string connectionName, + int cancelledHandlers); + + [LoggerMessage( + LogLevel.Error, + "The consumer for queue {Queue} on {ConnectionName} cannot attach: all {MaxConcurrency} handlers of a " + + "lost channel are still running {GracePeriod} after they were cancelled. The endpoint consumes " + + "again when one of them returns")] + public static partial void ConsumerBlockedByHandlers( + this ILogger logger, + string queue, + string connectionName, + ushort maxConcurrency, + TimeSpan gracePeriod); + + [LoggerMessage( + LogLevel.Information, + "A handler of the consumer for queue {Queue} on {ConnectionName} returned after {Elapsed}. " + + "The consumer attaches again")] + public static partial void ConsumerUnblocked( + this ILogger logger, + string queue, + string connectionName, + TimeSpan elapsed); + + [LoggerMessage( + LogLevel.Information, + "Stopping the consumer for queue {Queue} on {ConnectionName}: waiting for {RunningHandlers} running " + + "handlers")] + public static partial void ConsumerDrainStarted( + this ILogger logger, + string queue, + string connectionName, + int runningHandlers); + + [LoggerMessage( + LogLevel.Warning, + "The consumer for queue {Queue} on {ConnectionName} stopped waiting for its handlers after {Elapsed}. " + + "{RunningHandlers} handlers were cancelled, and the broker returns their unacknowledged messages " + + "to the queue")] + public static partial void ConsumerDrainTimedOut( + this ILogger logger, + string queue, + string connectionName, + int runningHandlers, + TimeSpan elapsed); + + [LoggerMessage( + LogLevel.Information, + "Stopped the consumer for queue {Queue} on {ConnectionName} after {Elapsed}. {RunningHandlers} " + + "handlers are still running")] + public static partial void ConsumerStopped( + this ILogger logger, + string queue, + string connectionName, + TimeSpan elapsed, + int runningHandlers); + + [LoggerMessage( + LogLevel.Critical, + "The consumer for queue {Queue} on {ConnectionName} failed unexpectedly and no longer consumes")] + public static partial void ConsumerFailed( + this ILogger logger, + Exception exception, + string queue, + string connectionName); + + [LoggerMessage(LogLevel.Error, "Stopping the consumer for queue {Queue} on {ConnectionName} failed")] + public static partial void ConsumerStopFailed( + this ILogger logger, + Exception exception, + string queue, + string connectionName); + + [LoggerMessage( + LogLevel.Error, + "Handling message {MessageId} from queue {Queue} on {ConnectionName} failed (consumer {ConsumerTag}, " + + "delivery {DeliveryTag}, correlation {CorrelationId})")] + public static partial void DeliveryFailed( + this ILogger logger, + Exception exception, + string? messageId, + string queue, + string connectionName, + string consumerTag, + ulong deliveryTag, + string? correlationId); + + [LoggerMessage( + LogLevel.Error, + "Cancelling the handlers of the consumer for queue {Queue} on {ConnectionName} failed")] + public static partial void HandlerCancellationFailed( + this ILogger logger, + Exception exception, + string queue, + string connectionName); + + [LoggerMessage(LogLevel.Warning, "Deleting queue {Queue} on {ConnectionName} failed")] + public static partial void QueueDeleteFailed( + this ILogger logger, + Exception exception, + string queue, + string connectionName); + + [LoggerMessage( + LogLevel.Information, + "Skipped the {Settlement} of message {MessageId} because its channel closed (consumer {ConsumerTag}, " + + "delivery {DeliveryTag}, exchange {Exchange}, routing key {RoutingKey}, redelivered: {Redelivered}). " + + "The broker delivers the message again")] + public static partial void SettlementSkipped( + this ILogger logger, + string settlement, + string? messageId, + string consumerTag, + ulong deliveryTag, + string exchange, + string routingKey, + bool redelivered); +} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumerManager.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumerManager.cs index f99ee4edc6f..0f6fdfd56f8 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumerManager.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQConsumerManager.cs @@ -1,4 +1,5 @@ using System.Collections.Immutable; +using System.Runtime.ExceptionServices; using Microsoft.Extensions.Logging; using RabbitMQ.Client; using RabbitMQ.Client.Events; @@ -6,520 +7,259 @@ namespace Mocha.Transport.RabbitMQ; /// -/// Manages RabbitMQ connections and consumers with automatic reconnection. +/// Owns the RabbitMQ connection that consumes messages, and the consumers on it. /// -public sealed class RabbitMQConsumerManager : RabbitMQConnectionManagerBase +internal sealed class RabbitMQConsumerManager : IAsyncDisposable { - private ImmutableArray _registeredConsumers = []; + private static readonly CreateChannelOptions s_probeChannelOptions = new( + publisherConfirmationsEnabled: false, + publisherConfirmationTrackingEnabled: false); - private CancellationTokenSource? _monitoringCts; - private Task? _monitoringTask; + private readonly RabbitMQConnectionManager _connection; + private readonly object _sync = new(); + private ImmutableArray _consumers = []; + private Task? _disposeTask; + private volatile bool _isDisposed; /// - /// Creates a new consumer manager that will use the specified connection factory to establish RabbitMQ connections. + /// Creates a consumer manager. /// - /// The logger for connection and consumer lifecycle events. - /// A factory delegate that creates new RabbitMQ connections on demand. - public RabbitMQConsumerManager( - ILogger logger, - Func> connectionFactory) : base(logger, connectionFactory) { } + /// The logger for consumer events. + /// The manager of the connection that consumes messages. + public RabbitMQConsumerManager(ILogger logger, RabbitMQConnectionManager connection) + { + Logger = logger; + _connection = connection; + } /// - /// Registers a new consumer for the specified queue, creating a dedicated channel with the given prefetch count. + /// Gets the logger for consumer events. /// - /// - /// If a connection is already established, the consumer is connected immediately. - /// On reconnection, all registered consumers are automatically re-attached. - /// Disposing the returned handle unregisters the consumer and closes its channel. - /// - /// The name of the RabbitMQ queue to consume from. - /// The callback invoked for each delivered message, receiving the channel, delivery event args, and a cancellation token. - /// The maximum number of unacknowledged messages the broker will deliver to this consumer. - /// The maximum number of messages dispatched concurrently to the consumer callback. When greater than 1, multiple messages are processed in parallel. - /// A token to cancel the registration operation. - /// An handle that, when disposed, unregisters the consumer and releases its channel. - public async Task RegisterConsumerAsync( - string queueName, - Func messageHandler, - ushort prefetchCount, - ushort consumerDispatchConcurrency, - CancellationToken cancellationToken) - { - var registration = new RegisteredConsumer - { - Manager = this, - QueueName = queueName, - MessageHandler = messageHandler, - PrefetchCount = prefetchCount, - ConsumerDispatchConcurrency = consumerDispatchConcurrency - }; + public ILogger Logger { get; } - AddConsumer(registration); + /// + /// Gets the manager of the connection that consumes messages. + /// + public RabbitMQConnectionManager Connection => _connection; - if (IsConnected) - { - await registration.ConnectAsync(CurrentConnection!, cancellationToken); - } + /// + /// Gets or sets how long one attempt to attach a consumer may take before it is abandoned and retried. + /// + public TimeSpan AttachTimeout { get; set; } = TimeSpan.FromSeconds(60); - Logger.RegisteredConsumerForQueue(queueName); + /// + /// Gets or sets how long a consumer waits for a handler of a lost channel to return before it reports + /// that it is blocked. + /// + public TimeSpan BlockedGracePeriod { get; set; } = TimeSpan.FromSeconds(30); - return registration; - } + /// + /// Gets whether a ready connection exists. + /// + public bool IsConnected => _connection.IsConnected; /// - /// Called after a new RabbitMQ connection is established, either on first connect or after a - /// connection loss that required creating an entirely new connection. + /// Gets whether the connection is ready and every consumer is attached. /// - /// - /// This method re-attaches every previously registered consumer to the new connection by - /// creating fresh channels and restarting consumption. It also starts a background health - /// monitor that polls every 30 seconds and triggers reconnection when the connection drops. - /// Because consumers are automatically re-attached, callers of - /// do not need to handle reconnection themselves. - /// - /// The newly created . - /// A token to cancel the reconnection of consumers. - /// A task that completes when all consumers have been re-attached and health monitoring has started. - protected override async Task OnAfterConnectionCreatedAsync( - IConnection connection, - CancellationToken cancellationToken) + public bool IsHealthy => HealthFailureReason is null; + + /// + /// Gets the reason the connection or a consumer is unhealthy, or null when all are healthy. + /// + public string? HealthFailureReason { - // Reconnect existing consumers - await ReconnectConsumersAsync(connection, cancellationToken); + get + { + if (!_connection.IsConnected) + { + return "The consumer connection is unavailable."; + } - // Start health monitoring if not already running - StartHealthMonitoring(); + foreach (var consumer in _consumers) + { + if (consumer.HealthFailureReason is { } reason) + { + return $"Queue '{consumer.QueueName}': {reason}"; + } + } + + return null; + } } /// - /// Called when the RabbitMQ client library's built-in automatic recovery succeeds and the - /// existing connection object is restored rather than replaced. + /// Connects, as describes. /// - /// - /// Even though the client library recovers the connection, channels and consumers may be in - /// an inconsistent state. This method iterates all registered consumers and reconnects each - /// one with a fresh channel, ensuring that message delivery resumes correctly after recovery. - /// - /// A token to cancel the consumer reconnection. - /// A task that completes when all consumers have been re-attached to the recovered connection. - protected override async Task OnConnectionRecoveredAsync(CancellationToken cancellationToken) + public Task StartAsync(CancellationToken cancellationToken) => _connection.StartAsync(cancellationToken); + + /// + /// Makes one round trip on a new connection, so a connection the broker refuses after the handshake + /// does not count as ready. + /// + public static async Task SetUpConnectionAsync(IConnection connection, CancellationToken cancellationToken) { - if (CurrentConnection is not null) - { - await ReconnectConsumersAsync(CurrentConnection, cancellationToken); - } + await using var channel = await connection.CreateChannelAsync(s_probeChannelOptions, cancellationToken); + await channel.CloseAsync(cancellationToken); } - private async Task ReconnectConsumersAsync(IConnection connection, CancellationToken cancellationToken) + /// + /// Registers a consumer for a queue and waits for its first attempt to attach. The consumer attaches + /// again by itself whenever its channel or its connection is lost, until it is stopped. + /// + /// The name of the queue to consume. + /// + /// Runs on every new channel of the consumer before it consumes, for example to declare the queue and + /// its bindings, or null to consume at once. + /// + /// Handles one delivered message. + /// The largest number of unacknowledged messages the broker delivers. + /// The largest number of messages that are handled at the same time. + /// Ends the wait for the first attempt and stops the consumer. + /// + /// The consumer. It is not attached when no connection was ready or the first attempt failed for a + /// reason that a later attempt can overcome. + /// + /// The broker refused the first attempt. The consumer is stopped. + public async Task RegisterConsumerAsync( + string queueName, + Func? prepareAsync, + Func messageHandler, + ushort maxPrefetch, + ushort maxConcurrency, + CancellationToken cancellationToken) { - var consumers = _registeredConsumers; + ObjectDisposedException.ThrowIf(_isDisposed, this); + cancellationToken.ThrowIfCancellationRequested(); + ArgumentOutOfRangeException.ThrowIfZero(maxConcurrency); - if (consumers.IsEmpty) + var consumer = new RabbitMQConsumer(this, queueName, prepareAsync, messageHandler, maxPrefetch, maxConcurrency); + ImmutableInterlocked.Update(ref _consumers, static (consumers, added) => consumers.Add(added), consumer); + + if (_isDisposed) { - return; + await consumer.DisposeAsync(); + ObjectDisposedException.ThrowIf(_isDisposed, this); } - Logger.ReconnectConsumersAsync(consumers.Length); + Exception? failure; + try + { + failure = await consumer.FirstAttempt.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) + { + await consumer.DisposeAsync(); + throw; + } + + // The disposal of the manager stopped the consumer while it attached. + ObjectDisposedException.ThrowIf(_isDisposed, this); - foreach (var consumer in consumers) + if (failure is not null && RabbitMQFailure.IsPermanent(failure)) { - try + using (_connection.BeginLogScope()) { - await consumer.ConnectAsync(connection, cancellationToken); - Logger.ReconnectConsumerForQueue(consumer.QueueName); - } - catch (Exception ex) - { - Logger.FailedToReconnectConsumerForQueue(ex, consumer.QueueName); + Logger.ConsumerRegistrationFailed(failure, queueName, _connection.ConnectionName); } + + // A consumer the broker refuses does not keep attaching in the background. + await consumer.DisposeAsync(); + ExceptionDispatchInfo.Throw(failure); } + + return consumer; } - private void StartHealthMonitoring() + /// + /// Deletes a queue on the ready connection. Does nothing when there is no ready connection, and logs + /// a failure instead of throwing it. + /// + public async Task DeleteQueueAsync(string queueName, CancellationToken cancellationToken) { - if (_monitoringCts is { IsCancellationRequested: false }) + if (_connection.Connection is not { } connection || cancellationToken.IsCancellationRequested) { return; } - _monitoringCts = new CancellationTokenSource(); - _monitoringTask = MonitorConnectionHealthAsync(_monitoringCts.Token); - } - - private async Task MonitorConnectionHealthAsync(CancellationToken cancellationToken) - { - while (!cancellationToken.IsCancellationRequested) + // A cancelled open would cost the connection a channel number, so the open runs to its end. + var opening = _connection.OpenChannelAsync(connection, s_probeChannelOptions, AttachTimeout); + try { + var channel = await opening.WaitOrAbandonAsync(cancellationToken); try { - await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken); - - if (!IsConnected) - { - Logger.ConnectionHealthCheckFailedAttemptingReconnection(); - - try - { - await EnsureConnectedAsync(cancellationToken); - } - catch (Exception ex) - { - Logger.ReconnectionAttemptFailed(ex); - } - } - else - { - Logger.ConnectionHealthCheckOk(); - } + await channel + .QueueDeleteAsync(queueName, cancellationToken: cancellationToken) + .WaitOrAbandonAsync(cancellationToken); } - catch (OperationCanceledException) + finally { - break; + _ = RabbitMQConnectionManager.CloseChannelAsync(channel); } - catch (Exception ex) + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The caller cancelled the cleanup. A channel that still opens is closed. + _ = CloseWhenOpenedAsync(opening); + } + catch (Exception ex) + { + using (_connection.BeginLogScope()) { - Logger.ErrorDuringConnectionHealthCheck(ex); + Logger.QueueDeleteFailed(ex, queueName, _connection.ConnectionName); } } } - private void AddConsumer(RegisteredConsumer consumer) + private static async Task CloseWhenOpenedAsync(Task opening) { - ImmutableInterlocked.Update(ref _registeredConsumers, consumers => consumers.Add(consumer)); - } - - private void RemoveConsumer(RegisteredConsumer consumer) - { - ImmutableInterlocked.Update(ref _registeredConsumers, consumers => consumers.Remove(consumer)); - } - - protected override async ValueTask DisposeAsyncCore() - { - // Stop monitoring - if (_monitoringCts is not null) + try { - await _monitoringCts.CancelAsync(); - _monitoringCts.Dispose(); - _monitoringCts = null; + await RabbitMQConnectionManager.CloseChannelAsync(await opening); } - - if (_monitoringTask is not null) + catch { - try - { - await _monitoringTask; - } - catch - { - // Ignore - } + // The channel could not be opened. } + } - // Dispose all consumers - var consumers = ImmutableInterlocked.InterlockedExchange(ref _registeredConsumers, []); + internal void Remove(RabbitMQConsumer consumer) + => ImmutableInterlocked.Update(ref _consumers, static (consumers, removed) => consumers.Remove(removed), consumer); - foreach (var consumer in consumers) + /// + /// Stops every consumer without waiting for running handlers, and closes the connection. + /// + public ValueTask DisposeAsync() + { + lock (_sync) { - try - { - await consumer.DisposeAsync(); - } - catch (Exception ex) - { - Logger.ErrorDisposingConsumer(ex, consumer.QueueName); - } + return new ValueTask(_disposeTask ??= DisposeCoreAsync()); } } - /// - /// Represents a consumer registration that maintains a dedicated channel to a specific queue, - /// supporting reconnection and graceful disposal. - /// - internal sealed class RegisteredConsumer : IAsyncDisposable + private async Task DisposeCoreAsync() { - /// - /// Gets the owning consumer manager responsible for connection lifecycle. - /// - public required RabbitMQConsumerManager Manager { get; init; } - - /// - /// Gets the name of the RabbitMQ queue this consumer is bound to. - /// - public required string QueueName { get; init; } - - /// - /// Gets the callback invoked for each message delivered from the queue. - /// - public required Func< - IChannel, - BasicDeliverEventArgs, - CancellationToken, - ValueTask - > MessageHandler - { get; init; } - - /// - /// Gets the maximum number of unacknowledged messages the broker delivers to this consumer. - /// - public required ushort PrefetchCount { get; init; } - - /// - /// Gets the maximum number of messages dispatched concurrently to the consumer callback. - /// - public required ushort ConsumerDispatchConcurrency { get; init; } - - /// - /// Gets or sets the server-assigned consumer tag identifying this consumer on the channel. - /// - public string? ConsumerTag { get; set; } + // Leaves the caller's lock. + await Task.Yield(); + _isDisposed = true; - /// - /// Gets or sets the dedicated channel used by this consumer for message delivery. - /// - public IChannel? Channel { get; set; } - - private readonly CancellationTokenSource _consumerCts = new(); + // The consumers stop side by side, so a broker that does not answer delays the disposal only once. + await Task.WhenAll(_consumers.Select(StopConsumerAsync)); + await _connection.DisposeAsync(); + } - /// - /// Establishes a new channel on the given connection, configures QoS prefetch, and starts consuming messages from the queue. - /// - /// - /// Any previously active channel is disconnected before a new one is created. - /// On failure, the partially created channel is disposed and the consumer state is reset. - /// - /// The RabbitMQ connection to create a channel on. - /// A token to cancel the connect operation. - public async Task ConnectAsync(IConnection connection, CancellationToken cancellationToken) + private async Task StopConsumerAsync(RabbitMQConsumer consumer) + { + try { - await DisconnectAsync(cancellationToken); - - IChannel? channel = null; - try - { - var channelOptions = new CreateChannelOptions( - publisherConfirmationsEnabled: false, - publisherConfirmationTrackingEnabled: false, - consumerDispatchConcurrency: ConsumerDispatchConcurrency); - channel = await connection.CreateChannelAsync(channelOptions, cancellationToken); - var consumer = new AsyncEventingBasicConsumer(channel); - consumer.ReceivedAsync += async (_, eventArgs) => - await MessageHandler(channel, eventArgs, _consumerCts.Token); - - await channel.BasicQosAsync( - prefetchSize: 0, - prefetchCount: PrefetchCount, //100 - global: false, - cancellationToken: cancellationToken); - - var tag = await channel.BasicConsumeAsync( - queue: QueueName, - autoAck: false, - consumer: consumer, - cancellationToken: cancellationToken); - - ConsumerTag = tag; - Channel = channel; - } - catch - { - if (channel is not null) - { - await channel.DisposeAsync(); - } - - ConsumerTag = null; - Channel = null; - throw; - } + await consumer.DisposeAsync(); } - - private async Task DisconnectAsync(CancellationToken cancellationToken) + catch (Exception ex) { - if (ConsumerTag is not null && Channel is { IsOpen: true }) + using (_connection.BeginLogScope()) { - try - { - await Channel.BasicCancelAsync(ConsumerTag, cancellationToken: cancellationToken); - } - catch - { - // Ignore - } - } - - ConsumerTag = null; - - if (Channel is not null) - { - try - { - if (Channel.IsOpen) - { - await Channel.CloseAsync(cancellationToken: cancellationToken); - } - - await Channel.DisposeAsync(); - } - catch - { - // Ignore - } - - Channel = null; + Logger.ConsumerStopFailed(ex, consumer.QueueName, _connection.ConnectionName); } } - - /// - /// Disconnects the consumer from its channel, unregisters it from the manager, and releases all resources. - /// - public async ValueTask DisposeAsync() - { - Manager.RemoveConsumer(this); - await _consumerCts.CancelAsync(); - await DisconnectAsync(CancellationToken.None); - _consumerCts.Dispose(); - } } } - -internal static partial class Logs -{ - [LoggerMessage(LogLevel.Information, "Creating RabbitMQ connection using factory delegate")] - public static partial void CreatingConnectionUsingFactoryDelegate(this ILogger logger); - - [LoggerMessage(LogLevel.Information, "Successfully created connection: {ConnectionName}")] - public static partial void SuccessfullyCreatedConnection(this ILogger logger, string? connectionName); - - [LoggerMessage(LogLevel.Error, "Failed to create connection after {Attempts} attempts")] - public static partial void FailedToCreateConnectionAfterAttempts(this ILogger logger, Exception ex, int attempts); - - [LoggerMessage( - LogLevel.Warning, - "Failed to create connection (attempt {Attempt}/{MaxAttempts}). Retrying in {Delay}")] - public static partial void FailedToCreateConnectionRetrying( - this ILogger logger, - Exception ex, - int attempt, - int maxAttempts, - TimeSpan delay); - - [LoggerMessage(LogLevel.Error, "Error processing message from queue {Queue}")] - public static partial void ErrorProcessingMessageFromQueue(this ILogger logger, Exception ex, string queue); - - [LoggerMessage(LogLevel.Information, "Registered consumer for queue {Queue}")] - public static partial void RegisteredConsumerForQueue(this ILogger logger, string queue); - - [LoggerMessage(LogLevel.Warning, "Cannot provision topology - connection not available")] - public static partial void CannotProvisionTopologyConnectionNotAvailable(this ILogger logger); - - [LoggerMessage(LogLevel.Information, "Provisioning RabbitMQ topology")] - public static partial void ProvisioningRabbitMQTopology(this ILogger logger); - - [LoggerMessage(LogLevel.Debug, "Declared exchange: {Exchange}")] - public static partial void DeclaredExchange(this ILogger logger, string exchange); - - [LoggerMessage(LogLevel.Debug, "Declared queue: {Queue}")] - public static partial void DeclaredQueue(this ILogger logger, string queue); - - [LoggerMessage(LogLevel.Debug, "Created binding: {Queue} -> {Exchange}/{RoutingKey}")] - public static partial void CreatedBinding(this ILogger logger, string queue, string exchange, string routingKey); - - [LoggerMessage(LogLevel.Information, "Successfully provisioned topology")] - public static partial void SuccessfullyProvisionedTopology(this ILogger logger); - - [LoggerMessage(LogLevel.Error, "Failed to provision topology")] - public static partial void FailedToProvisionTopology(this ILogger logger, Exception ex); - - [LoggerMessage(LogLevel.Information, "Reconnecting {Count} registered consumers")] - public static partial void ReconnectConsumersAsync(this ILogger logger, int count); - - [LoggerMessage(LogLevel.Warning, "Cannot recreate consumer for queue {Queue} - no factory or handler")] - public static partial void CannotRecreateConsumerNoFactoryOrHandler(this ILogger logger, string queue); - - [LoggerMessage(LogLevel.Information, "Reconnected consumer for queue {Queue}")] - public static partial void ReconnectConsumerForQueue(this ILogger logger, string queue); - - [LoggerMessage(LogLevel.Error, "Failed to reconnect consumer for queue {Queue}")] - public static partial void FailedToReconnectConsumerForQueue(this ILogger logger, Exception ex, string queue); - - [LoggerMessage(LogLevel.Information, "Connection closed by application")] - public static partial void ConnectionClosedByApplication(this ILogger logger); - - [LoggerMessage(LogLevel.Warning, "Connection shutdown detected. Initiator: {Initiator}, Reason: {Reason}")] - public static partial void ConnectionShutdownDetected( - this ILogger logger, - ShutdownInitiator initiator, - string reason); - - [LoggerMessage(LogLevel.Error, "Failed to reconnect after shutdown")] - public static partial void FailedToReconnectAfterShutdown(this ILogger logger, Exception ex); - - [LoggerMessage(LogLevel.Information, "Connection recovery succeeded")] - public static partial void ConnectionRecoverySucceeded(this ILogger logger); - - [LoggerMessage(LogLevel.Information, "Successfully recovered connection, topology, and consumers")] - public static partial void SuccessfullyRecoveredConnectionTopologyAndConsumers(this ILogger logger); - - [LoggerMessage(LogLevel.Error, "Error during post-recovery operations")] - public static partial void ErrorDuringPostRecoveryOperations(this ILogger logger, Exception ex); - - [LoggerMessage(LogLevel.Information, "Connection recovery started")] - public static partial void ConnectionRecoveryStarted(this ILogger logger); - - [LoggerMessage(LogLevel.Warning, "Connection blocked: {Reason}")] - public static partial void ConnectionBlocked(this ILogger logger, string reason); - - [LoggerMessage(LogLevel.Information, "Connection unblocked")] - public static partial void ConnectionUnblocked(this ILogger logger); - - [LoggerMessage(LogLevel.Warning, "Connection health check failed - attempting reconnection")] - public static partial void ConnectionHealthCheckFailedAttemptingReconnection(this ILogger logger); - - [LoggerMessage(LogLevel.Error, "Reconnection attempt failed")] - public static partial void ReconnectionAttemptFailed(this ILogger logger, Exception ex); - - [LoggerMessage(LogLevel.Trace, "Connection health check: OK")] - public static partial void ConnectionHealthCheckOk(this ILogger logger); - - [LoggerMessage(LogLevel.Error, "Error during connection health check")] - public static partial void ErrorDuringConnectionHealthCheck(this ILogger logger, Exception ex); - - [LoggerMessage(LogLevel.Error, "Error disposing connection")] - public static partial void ErrorDisposingConnection(this ILogger logger, Exception ex); - - [LoggerMessage(LogLevel.Information, "DelegateConnectionManager disposed")] - public static partial void DelegateConnectionManagerDisposed(this ILogger logger); - - [LoggerMessage(LogLevel.Error, "Exception in connection callback. Detail: {Detail}")] - public static partial void ExceptionInConnectionCallback(this ILogger logger, Exception ex, string detail); - - [LoggerMessage(LogLevel.Error, "Connection recovery failed. Will retry. Error: {ErrorMessage}")] - public static partial void ConnectionRecoveryFailedWillRetry( - this ILogger logger, - Exception? exception, - string? errorMessage); - - [LoggerMessage(LogLevel.Information, "Consumer tag changed after recovery. Old: {OldTag}, New: {NewTag}")] - public static partial void ConsumerTagChangedAfterRecovery(this ILogger logger, string oldTag, string newTag); - - [LoggerMessage(LogLevel.Debug, "Updated consumer tag for queue {Queue}: {OldTag} -> {NewTag}")] - public static partial void UpdatedConsumerTagForQueue( - this ILogger logger, - string queue, - string oldTag, - string newTag); - - [LoggerMessage( - LogLevel.Warning, - "Server-named queue changed after recovery. Old: {OldName}, New: {NewName}. This may affect consumers expecting the old queue name.")] - public static partial void ServerNamedQueueChangedAfterRecovery( - this ILogger logger, - string oldName, - string newName); - - [LoggerMessage(LogLevel.Information, "Recovering consumer {ConsumerTag}")] - public static partial void RecoveringConsumer(this ILogger logger, string consumerTag); - - [LoggerMessage(LogLevel.Error, "Error disposing consumer {ConsumerTag}")] - public static partial void ErrorDisposingConsumer(this ILogger logger, Exception ex, string consumerTag); -} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.Logging.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.Logging.cs new file mode 100644 index 00000000000..4c03579806d --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.Logging.cs @@ -0,0 +1,103 @@ +using Microsoft.Extensions.Logging; + +namespace Mocha.Transport.RabbitMQ; + +internal static partial class Logs +{ + [LoggerMessage( + LogLevel.Debug, + "The broker confirmed message {MessageId} for endpoint {Endpoint} on {ConnectionName} after {Elapsed} " + + "(attempt {Attempt})")] + public static partial void PublishConfirmed( + this ILogger logger, + string? messageId, + string endpoint, + string connectionName, + int attempt, + TimeSpan elapsed); + + [LoggerMessage( + LogLevel.Warning, + "The broker confirmed message {MessageId} for endpoint {Endpoint} on {ConnectionName} only after " + + "{Elapsed} (attempt {Attempt}), which is longer than {Threshold}")] + public static partial void PublishConfirmedSlowly( + this ILogger logger, + string? messageId, + string endpoint, + string connectionName, + int attempt, + TimeSpan elapsed, + TimeSpan threshold); + + [LoggerMessage( + LogLevel.Error, + "Publishing message {MessageId} to endpoint {Endpoint} on {ConnectionName} failed after {Elapsed} " + + "(attempt {Attempt})")] + public static partial void PublishFailed( + this ILogger logger, + Exception exception, + string? messageId, + string endpoint, + string connectionName, + int attempt, + TimeSpan elapsed); + + [LoggerMessage( + LogLevel.Debug, + "Publishing message {MessageId} to endpoint {Endpoint} on {ConnectionName} was cancelled by the caller " + + "after {Elapsed} (attempt {Attempt})")] + public static partial void PublishCancelled( + this ILogger logger, + string? messageId, + string endpoint, + string connectionName, + int attempt, + TimeSpan elapsed); + + [LoggerMessage( + LogLevel.Information, + "Publishing message {MessageId} to endpoint {Endpoint} on {ConnectionName} failed (attempt {Attempt}) " + + "and is attempted again in {Delay}. The broker delivers the message twice if it already has it. " + + "Reason: {Reason}")] + public static partial void PublishRetryScheduled( + this ILogger logger, + string? messageId, + string endpoint, + string connectionName, + int attempt, + TimeSpan delay, + string reason); + + [LoggerMessage( + LogLevel.Warning, + "{Count} publishes on RabbitMQ connection {ConnectionName} have waited for the broker for at least " + + "{Interval}. The connection is {ConnectionStatus}")] + public static partial void PublishesStalled( + this ILogger logger, + string connectionName, + long count, + TimeSpan interval, + string connectionStatus); + + [LoggerMessage(LogLevel.Debug, "Opened publisher channel {ChannelNumber} on {ConnectionName}")] + public static partial void PublisherChannelOpened(this ILogger logger, string connectionName, int channelNumber); + + [LoggerMessage( + LogLevel.Warning, + "The broker returned message {MessageId} because queue {Queue} does not exist on {ConnectionName}. " + + "Declaring the queue and its bindings again")] + public static partial void MessageReturned( + this ILogger logger, + string? messageId, + string queue, + string connectionName); + + [LoggerMessage( + LogLevel.Information, + "Declared queue {Queue} and its bindings again on {ConnectionName}. Publishing message {MessageId} again")] + public static partial void QueueRedeclared( + this ILogger logger, + string queue, + string connectionName, + string? messageId); +} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.Publishing.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.Publishing.cs new file mode 100644 index 00000000000..e8824f10ca2 --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.Publishing.cs @@ -0,0 +1,275 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using Mocha.Middlewares; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ; + +internal sealed partial class RabbitMQDispatcher +{ + private static readonly TimeSpan s_slowPublishThreshold = TimeSpan.FromSeconds(10); + + private readonly Stack _deadlines = new(); + private ITimer? _stalledPublishCheck; + private TimeSpan _stalledPublishInterval = TimeSpan.FromSeconds(30); + private long _publishesStarted; + private long _publishesFinished; + private long _startedAtLastCheck; + private long _finishedAtLastCheck; + + /// + /// Gets or sets how often the dispatcher checks for publishes that wait for the broker. A publish that + /// waits for a whole interval while no other publish finishes is reported. + /// + public TimeSpan StalledPublishInterval + { + get => _stalledPublishInterval; + set + { + lock (_sync) + { + _stalledPublishInterval = value; + _stalledPublishCheck?.Change(value, value); + } + } + } + + /// + /// Counts a publish as in flight and returns the deadline its operations observe, or null when + /// the publish timeout is infinite and the operations observe the caller's token. + /// + public PublishDeadline? StartPublish(CancellationToken callerToken) + { + Interlocked.Increment(ref _publishesStarted); + + var timeout = PublishTimeout; + if (timeout == Timeout.InfiniteTimeSpan) + { + return null; + } + + PublishDeadline? deadline; + lock (_deadlines) + { + _deadlines.TryPop(out deadline); + } + + deadline ??= new PublishDeadline(); + deadline.Start(timeout, callerToken); + return deadline; + } + + /// + /// Counts a publish as finished and takes its deadline back. + /// + public void FinishPublish(PublishDeadline? deadline) + { + Interlocked.Increment(ref _publishesFinished); + + if (deadline is null) + { + return; + } + + if (!deadline.TryReset()) + { + deadline.Dispose(); + return; + } + + lock (_deadlines) + { + if (_deadlines.Count < MaxPublisherChannels) + { + _deadlines.Push(deadline); + return; + } + } + + deadline.Dispose(); + } + + /// + /// Gets whether a failed publish is attempted again: the failure is transient, the publish was not + /// cancelled, and the dispatcher is not disposed. + /// + public bool CanRetry(Exception exception, CancellationToken cancellationToken) + => !cancellationToken.IsCancellationRequested + && !IsDisposed + && RabbitMQFailure.Classify(exception) == RabbitMQFailureKind.Transient; + + private void StartStalledPublishCheck() + { + lock (_sync) + { + if (_isDisposed || _stalledPublishCheck is not null) + { + return; + } + + var interval = _stalledPublishInterval; + _stalledPublishCheck = _connection.TimeProvider.CreateTimer( + static state => ((RabbitMQDispatcher)state!).CheckForStalledPublishes(), + this, + interval, + interval); + } + } + + private void StopStalledPublishCheck() + { + ITimer? timer; + lock (_sync) + { + timer = _stalledPublishCheck; + _stalledPublishCheck = null; + } + + timer?.Dispose(); + } + + private void CheckForStalledPublishes() + { + // Read in this order, the number in flight is never negative. + var finished = Volatile.Read(ref _publishesFinished); + var started = Volatile.Read(ref _publishesStarted); + var stalled = _startedAtLastCheck - _finishedAtLastCheck; + + // No publish finished since the last check, so every publish that was in flight then still is. + if (stalled > 0 && finished == _finishedAtLastCheck && _logger.IsEnabled(LogLevel.Warning)) + { + var status = !_connection.IsConnected ? "unavailable" + : _connection.IsBlocked ? "blocked by the broker" + : "ready"; + + using (_connection.BeginLogScope()) + { + _logger.PublishesStalled(_connection.ConnectionName, stalled, StalledPublishInterval, status); + } + } + + _startedAtLastCheck = started; + _finishedAtLastCheck = finished; + } + + public void LogPublishConfirmed(IDispatchContext context, int attempt, long started) + { + var elapsed = Stopwatch.GetElapsedTime(started); + var isSlow = elapsed >= s_slowPublishThreshold; + if (!_logger.IsEnabled(isSlow ? LogLevel.Warning : LogLevel.Debug)) + { + return; + } + + using (_connection.BeginLogScope()) + using (_logger.BeginMessageScope(context)) + { + var messageId = context.Envelope?.MessageId; + var endpoint = context.Endpoint.Name; + if (isSlow) + { + _logger.PublishConfirmedSlowly( + messageId, + endpoint, + _connection.ConnectionName, + attempt, + elapsed, + s_slowPublishThreshold); + } + else + { + _logger.PublishConfirmed(messageId, endpoint, _connection.ConnectionName, attempt, elapsed); + } + } + } + + public void LogPublishFailed(IDispatchContext context, Exception exception, int attempt, long started) + { + if (!_logger.IsEnabled(LogLevel.Error)) + { + return; + } + + using (_connection.BeginLogScope()) + using (_logger.BeginMessageScope(context)) + { + _logger.PublishFailed( + exception, + context.Envelope?.MessageId, + context.Endpoint.Name, + _connection.ConnectionName, + attempt, + Stopwatch.GetElapsedTime(started)); + } + } + + public void LogPublishCancelled(IDispatchContext context, int attempt, long started) + { + if (!_logger.IsEnabled(LogLevel.Debug)) + { + return; + } + + using (_connection.BeginLogScope()) + using (_logger.BeginMessageScope(context)) + { + _logger.PublishCancelled( + context.Envelope?.MessageId, + context.Endpoint.Name, + _connection.ConnectionName, + attempt, + Stopwatch.GetElapsedTime(started)); + } + } + + public void LogPublishRetry(IDispatchContext context, Exception exception, int attempt, TimeSpan delay) + { + if (!_logger.IsEnabled(LogLevel.Information)) + { + return; + } + + using (_connection.BeginLogScope()) + using (_logger.BeginMessageScope(context)) + { + _logger.PublishRetryScheduled( + context.Envelope?.MessageId, + context.Endpoint.Name, + _connection.ConnectionName, + attempt, + delay, + $"{exception.GetType().Name}: {exception.Message}"); + } + } + + public void LogMessageReturned(IDispatchContext context, string queueName) + { + using (_connection.BeginLogScope()) + using (_logger.BeginMessageScope(context)) + { + _logger.MessageReturned(context.Envelope?.MessageId, queueName, _connection.ConnectionName); + } + } + + public void LogQueueRedeclared(IDispatchContext context, string queueName) + { + using (_connection.BeginLogScope()) + using (_logger.BeginMessageScope(context)) + { + _logger.QueueRedeclared(queueName, _connection.ConnectionName, context.Envelope?.MessageId); + } + } + + private void LogPublisherChannelOpened(IChannel channel) + { + if (!_logger.IsEnabled(LogLevel.Debug)) + { + return; + } + + using (_connection.BeginLogScope()) + { + _logger.PublisherChannelOpened(_connection.ConnectionName, channel.ChannelNumber); + } + } +} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.cs index c0a943f774f..4e769779cc1 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQDispatcher.cs @@ -1,159 +1,407 @@ -using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using RabbitMQ.Client; namespace Mocha.Transport.RabbitMQ; /// -/// Manages RabbitMQ connections and provides channel pooling for publishing messages. +/// Owns the RabbitMQ connection that publishes messages, and leases its channels to one publish at a time. /// -public sealed class RabbitMQDispatcher( - ILogger logger, - Func> connectionFactory, - Func onConnectionEstablished) - : RabbitMQConnectionManagerBase(logger, connectionFactory) +internal sealed partial class RabbitMQDispatcher : IAsyncDisposable { - private const int MaxPooledChannels = 10; + /// + /// The largest number of publisher channels per connection, which is also the largest number of + /// publishes in flight. + /// + internal const int MaxPublisherChannels = 100; - private readonly ConcurrentQueue _channelPool = new(); - private int _pooledChannelCount; + private static readonly CreateChannelOptions s_publishChannelOptions = new( + publisherConfirmationsEnabled: true, + publisherConfirmationTrackingEnabled: true, + outstandingPublisherConfirmationsRateLimiter: null); + + private readonly ILogger _logger; + private readonly RabbitMQConnectionManager _connection; + private readonly object _sync = new(); + private PublisherChannelPool? _pool; + private bool _isDisposed; /// - /// Rents a channel from the pool or creates a new one. - /// The caller is responsible for returning the channel via ReturnChannelAsync. + /// Creates a dispatcher. /// - public async ValueTask RentChannelAsync(CancellationToken cancellationToken = default) + /// The logger for publish events. + /// The manager of the connection that publishes messages. + public RabbitMQDispatcher(ILogger logger, RabbitMQConnectionManager connection) { - ObjectDisposedException.ThrowIf(IsDisposed, this); + _logger = logger; + _connection = connection; + } - // Try to get a channel from the pool - while (_channelPool.TryDequeue(out var pooledChannel)) - { - Interlocked.Decrement(ref _pooledChannelCount); + /// + /// Gets the manager of the connection that publishes messages. + /// + public RabbitMQConnectionManager Connection => _connection; + + /// + /// Gets whether a ready connection exists. + /// + public bool IsConnected => _connection.IsConnected; + + /// + /// Gets whether the dispatcher was disposed, after which no publish can succeed. + /// + public bool IsDisposed => Volatile.Read(ref _isDisposed); + + /// + /// Gets or sets how long opening a publisher channel may take before its connection is closed and replaced. + /// + public TimeSpan ChannelOpenTimeout { get; set; } = TimeSpan.FromSeconds(60); - if (pooledChannel.IsOpen) + /// + /// Gets or sets how long a publish waits for the broker's confirmation, including the wait for a + /// connection and all retries. waits until the caller cancels. + /// + public TimeSpan PublishTimeout { get; set; } = TimeSpan.FromMinutes(2); + + /// + /// Connects and starts the stalled publish check. + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + await _connection.StartAsync(cancellationToken); + StartStalledPublishCheck(); + } + + /// + /// Leases a channel of the ready connection to one publish, waiting while there is no connection or all + /// channels are leased. The caller gives the lease back with . + /// + public async ValueTask RentChannelAsync(CancellationToken cancellationToken) + { + while (true) + { + var connection = await _connection.GetConnectionAsync(cancellationToken); + if (GetPool(connection) is not { } pool) { - Logger.RentedChannelFromPool(); + // The connection was replaced in the meantime. + continue; + } + + await pool.AcquireAsync(cancellationToken); - return pooledChannel; + if (pool.IsRetired) + { + // The connection was replaced while this publish waited. The other waiters of the pool follow. + pool.Release(channel: null, reusable: false); + continue; } - // Channel is closed, dispose it - await DisposeChannelSafelyAsync(pooledChannel); + var ownsPermit = true; + try + { + while (true) + { + while (pool.TryTakeIdle(out var idle)) + { + if (idle.IsOpen) + { + return new PublisherLease(pool, idle); + } + + _ = RabbitMQConnectionManager.CloseChannelAsync(idle); + } + + if (!_connection.IsBlocked) + { + break; + } + + // The broker does not read from a blocked connection, so it would not answer a request to + // open a channel. The publish waits here and takes a channel that became idle meanwhile. + await _connection.WaitWhileBlockedAsync(cancellationToken); + if (pool.IsRetired || !ReferenceEquals(_connection.Connection, connection)) + { + break; + } + } + + if (pool.IsRetired || !ReferenceEquals(_connection.Connection, connection)) + { + // The connection was lost while this publish waited. + ownsPermit = false; + pool.Release(channel: null, reusable: false); + continue; + } + + var opening = _connection.OpenChannelAsync(connection, s_publishChannelOptions, ChannelOpenTimeout); + try + { + var channel = await opening.WaitAsync(cancellationToken); + LogPublisherChannelOpened(channel); + return new PublisherLease(pool, channel); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The caller gave up. The channel keeps its permit until it is open and goes to the pool. + ownsPermit = false; + _ = ReturnWhenOpenedAsync(pool, opening); + throw; + } + catch (ChannelOpenBlockedException) + { + ownsPermit = false; + OnChannelNumberLost(pool); + throw; + } + } + catch when (ownsPermit) + { + pool.Release(channel: null, reusable: false); + throw; + } } + } - // No valid pooled channel, create a new one - var connection = await GetConnectionAsync(cancellationToken); - var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken); + /// + /// Gives a lease back. The channel is leased again only when is true, + /// which requires that the broker answered the last publish on it. Otherwise the channel is closed. + /// + public void Return(PublisherLease lease, bool reusable) => lease.Pool.Release(lease.Channel, reusable); - Logger.CreatedNewChannel(); + private async Task ReturnWhenOpenedAsync(PublisherChannelPool pool, Task opening) + { + IChannel? channel = null; + try + { + channel = await opening; + } + catch (ChannelOpenBlockedException) + { + OnChannelNumberLost(pool); + return; + } + catch + { + // The channel could not be opened. + } - return channel; + pool.Release(channel, reusable: channel is not null); } /// - /// Returns a channel to the pool for reuse. - /// If the channel is closed or the pool is full, the channel is disposed. + /// Keeps the permit of a channel number the client lost to a request the blocked connection did not + /// answer, so the connection never has more channels than permits. A connection that lost half of its + /// channels this way is replaced. /// - public async ValueTask ReturnChannelAsync(IChannel channel) + private void OnChannelNumberLost(PublisherChannelPool pool) { - if (IsDisposed || !channel.IsOpen) + if (pool.LoseChannelNumber()) { - await DisposeChannelSafelyAsync(channel); - - return; + _connection.ReportUnresponsive( + pool.Connection, + "half of its publisher channels were lost to requests the broker did not answer while it blocked the connection"); } + } - // Check if pool has room - if (Interlocked.Increment(ref _pooledChannelCount) <= MaxPooledChannels) + private PublisherChannelPool? GetPool(IConnection connection) + { + var pool = Volatile.Read(ref _pool); + if (pool is not null && ReferenceEquals(pool.Connection, connection)) { - _channelPool.Enqueue(channel); - - Logger.ReturnedChannelToPool(_pooledChannelCount); + return pool; } - else + + lock (_sync) { - // Pool is full, dispose the channel - Interlocked.Decrement(ref _pooledChannelCount); + ObjectDisposedException.ThrowIf(_isDisposed, this); - await DisposeChannelSafelyAsync(channel); + pool = _pool; + if (pool is not null && ReferenceEquals(pool.Connection, connection)) + { + return pool; + } - Logger.ChannelPoolFullDisposedChannel(); + // Only the ready connection gets a pool, so a caller that still holds a lost connection asks again. + if (!ReferenceEquals(_connection.Connection, connection)) + { + return null; + } + + // The pool of a lost connection retired itself when the connection shut down. + pool?.Retire(); + pool = new PublisherChannelPool(connection, MaxPublisherChannels); + Volatile.Write(ref _pool, pool); + return pool; } } - protected override async Task OnConnectionEstablished(IConnection connection, CancellationToken cancellationToken) + /// + /// Stops the stalled publish check, closes the publisher channels and closes the connection. + /// + public async ValueTask DisposeAsync() { - await onConnectionEstablished(connection, cancellationToken); - } + PublisherChannelPool? pool; + lock (_sync) + { + _isDisposed = true; + pool = _pool; + _pool = null; + } - protected override async Task OnBeforeConnectionCreatedAsync(CancellationToken cancellationToken) - { - await ClearChannelPoolAsync(); + StopStalledPublishCheck(); + pool?.Retire(); + await _connection.DisposeAsync(); } +} - protected override async Task OnConnectionLostAsync() - { - await ClearChannelPoolAsync(); - } +/// +/// A publisher channel that is leased to one publish. +/// +/// The pool the channel belongs to. +/// The leased channel, which has publisher confirms enabled. +internal readonly record struct PublisherLease(PublisherChannelPool Pool, IChannel Channel) +{ + /// + /// Gets the connection the channel belongs to. + /// + public IConnection Connection => Pool.Connection; +} + +/// +/// The publisher channels of one connection. A permit bounds the channels and the publishes in flight, and +/// channels the broker answered on last wait on a stack for the next publish. +/// +internal sealed class PublisherChannelPool +{ + // Larger than any number of waiters, and far from an overflow of the permit count. + private const int AllWaiters = 1 << 30; - protected override async Task OnConnectionRecoveredAsync(CancellationToken cancellationToken) + private readonly SemaphoreSlim _permits; + private readonly Stack _idle; + private readonly int _capacity; + private int _lostChannelNumbers; + private bool _isRetired; + + public PublisherChannelPool(IConnection connection, int maxChannels) { - await ClearChannelPoolAsync(); + Connection = connection; + + // A channel maximum of zero means that the broker sets no limit. + _capacity = connection.ChannelMax == 0 ? maxChannels : Math.Min(maxChannels, connection.ChannelMax); + _permits = new SemaphoreSlim(_capacity); + _idle = new Stack(_capacity); + + connection.ConnectionShutdownAsync += (_, _) => + { + Retire(); + return Task.CompletedTask; + }; + + // A connection that closed before the subscription raised its shutdown event to nobody. + if (!connection.IsOpen) + { + Retire(); + } } - private async Task ClearChannelPoolAsync() + /// + /// Gets the connection the channels belong to. + /// + public IConnection Connection { get; } + + /// + /// Gets whether the connection was lost or the dispatcher was disposed, after which the pool leases + /// no channel. + /// + public bool IsRetired => Volatile.Read(ref _isRetired); + + /// + /// Takes a permit, waiting while all are taken. The wait also ends when the pool is retired. + /// + public ValueTask AcquireAsync(CancellationToken cancellationToken) + => _permits.Wait(0, CancellationToken.None) + ? ValueTask.CompletedTask + : new ValueTask(_permits.WaitAsync(cancellationToken)); + + /// + /// Takes the channel that was returned last, if there is one. + /// + public bool TryTakeIdle(out IChannel channel) { - while (_channelPool.TryDequeue(out var channel)) + lock (_idle) { - Interlocked.Decrement(ref _pooledChannelCount); - await DisposeChannelSafelyAsync(channel); + return _idle.TryPop(out channel!); } - - Logger.ClearedChannelPool(); } - private static async ValueTask DisposeChannelSafelyAsync(IChannel channel) + /// + /// Gives a permit back, and keeps its channel for the next publish when it is reusable and still open. + /// A channel that is not kept is closed first, so the connection never has more channels than permits. + /// + public void Release(IChannel? channel, bool reusable) { - try + if (channel is not null) { - if (channel.IsOpen) + var isKept = false; + if (reusable && channel.IsOpen) { - await channel.CloseAsync(); + lock (_idle) + { + if (!_isRetired) + { + _idle.Push(channel); + isKept = true; + } + } } - await channel.DisposeAsync(); - } - catch - { - // Ignore disposal errors + if (!isKept) + { + _ = CloseAndReleaseAsync(channel); + return; + } } + + _permits.Release(); } - protected override async ValueTask DisposeAsyncCore() + private async Task CloseAndReleaseAsync(IChannel channel) { - await ClearChannelPoolAsync(); + // The permit follows the channel number, which the client gives back when the close has completed. + await RabbitMQConnectionManager.CloseChannelCompletelyAsync(channel); + _permits.Release(); } -} - -// Add these log messages to your Logs class -internal static partial class Logs -{ - [LoggerMessage(LogLevel.Debug, "Rented channel from pool")] - public static partial void RentedChannelFromPool(this ILogger logger); - [LoggerMessage(LogLevel.Debug, "Created new channel")] - public static partial void CreatedNewChannel(this ILogger logger); + /// + /// Keeps a permit for good, because the client lost its channel number. Returns true when half of + /// the permits are gone. + /// + public bool LoseChannelNumber() => Interlocked.Increment(ref _lostChannelNumbers) * 2 >= _capacity; - [LoggerMessage(LogLevel.Debug, "Returned channel to pool. Pool size: {PoolSize}")] - public static partial void ReturnedChannelToPool(this ILogger logger, int poolSize); + /// + /// Closes the idle channels and ends every wait for a permit. Channels that are returned later are + /// closed as well. + /// + public void Retire() + { + IChannel[] idle; + lock (_idle) + { + if (_isRetired) + { + return; + } - [LoggerMessage(LogLevel.Debug, "Channel pool full, disposed channel")] - public static partial void ChannelPoolFullDisposedChannel(this ILogger logger); + _isRetired = true; + idle = _idle.ToArray(); + _idle.Clear(); + } - [LoggerMessage(LogLevel.Debug, "Cleared channel pool")] - public static partial void ClearedChannelPool(this ILogger logger); + // Whoever takes a permit of a retired pool gives it back and asks for the pool of the next connection. + _permits.Release(AllWaiters); - [LoggerMessage(LogLevel.Information, "RabbitMQ Dispatcher disposed")] - public static partial void DispatcherDisposed(this ILogger logger); + foreach (var channel in idle) + { + _ = RabbitMQConnectionManager.CloseChannelAsync(channel); + } + } } diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQFailure.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQFailure.cs new file mode 100644 index 00000000000..bc3d9ff062a --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Connection/RabbitMQFailure.cs @@ -0,0 +1,88 @@ +using System.Net.Sockets; +using System.Security.Authentication; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ; + +/// +/// Identifies whether a failed RabbitMQ operation can succeed on a later attempt. +/// +internal enum RabbitMQFailureKind +{ + /// + /// The connection or the broker is unavailable, and a later attempt can succeed. + /// + Transient, + + /// + /// The broker or the configuration refuses the operation, and it fails the same way on every attempt. + /// + Permanent +} + +/// +/// Classifies failures of RabbitMQ operations for connection startup, recovery logging and publish retries. +/// +internal static class RabbitMQFailure +{ + /// + /// Gets whether a later attempt of the failed operation can succeed. Callers handle their own + /// cancellation before they ask. + /// + public static RabbitMQFailureKind Classify(Exception exception) + { + for (var current = exception; current is not null; current = current.InnerException) + { + switch (current) + { + // One entry per broker node. A node that is only unreachable keeps the failure transient. + case AggregateException { InnerExceptions.Count: > 0 } nodes: + foreach (var node in nodes.InnerExceptions) + { + if (Classify(node) == RabbitMQFailureKind.Transient) + { + return RabbitMQFailureKind.Transient; + } + } + + return RabbitMQFailureKind.Permanent; + + case AuthenticationFailureException or AuthenticationException or PublishException: + return RabbitMQFailureKind.Permanent; + + // The broker closed the channel or the connection and said why. + case OperationInterruptedException { ShutdownReason: { Initiator: ShutdownInitiator.Peer } reason }: + return reason.ReplyCode is Constants.ConnectionForced or Constants.InternalError + ? RabbitMQFailureKind.Transient + : RabbitMQFailureKind.Permanent; + } + } + + return exception + is IOException + or SocketException + or TimeoutException + or OperationCanceledException + or ObjectDisposedException + or ChannelClosedException + or OperationInterruptedException + or ConnectFailureException + ? RabbitMQFailureKind.Transient + : RabbitMQFailureKind.Permanent; + } + + /// + /// Gets whether the failed operation fails the same way on every attempt. + /// + public static bool IsPermanent(Exception exception) => Classify(exception) == RabbitMQFailureKind.Permanent; + + /// + /// Gets the level a failed attempt that is retried is logged at: for a + /// transient failure, and for a permanent one. + /// + public static LogLevel GetLogLevel(Exception exception) + => IsPermanent(exception) ? LogLevel.Error : LogLevel.Warning; +} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/IRabbitMQMessagingTransportDescriptor.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/IRabbitMQMessagingTransportDescriptor.cs index 1a48a670abe..de10ecb0930 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/IRabbitMQMessagingTransportDescriptor.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/IRabbitMQMessagingTransportDescriptor.cs @@ -27,6 +27,26 @@ public interface IRabbitMQMessagingTransportDescriptor IRabbitMQMessagingTransportDescriptor ConnectionProvider( Func connectionFactory); + /// + /// Sets how long the transport keeps retrying while the broker cannot be reached before its start fails. + /// The default is two minutes, and fails the start after the first failed attempt. + /// + /// How long to keep retrying, between zero and 4,294,967,294 milliseconds. + /// The descriptor for method chaining. + IRabbitMQMessagingTransportDescriptor ConnectTimeout(TimeSpan timeout); + + /// + /// Sets how long a publish waits for the broker's confirmation, including the wait for a connection and + /// every retry, before it fails with . The default is two minutes, + /// and waits until the caller cancels. + /// + /// + /// How long to wait, more than zero and at most 4,294,967,294 milliseconds, or + /// . + /// + /// The descriptor for method chaining. + IRabbitMQMessagingTransportDescriptor PublishTimeout(TimeSpan timeout); + /// /// Configures bus-level defaults that are applied to all auto-provisioned queues and exchanges. /// diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/IRabbitMQQueueDescriptor.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/IRabbitMQQueueDescriptor.cs index 3a7076b5b87..855bb4d93fe 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/IRabbitMQQueueDescriptor.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/IRabbitMQQueueDescriptor.cs @@ -16,7 +16,8 @@ public interface IRabbitMQQueueDescriptor IRabbitMQQueueDescriptor Durable(bool durable = true); /// - /// Configures the backing queue as a quorum queue for high availability and data safety. + /// Configures a quorum queue. + /// Delivery limits are determined by RabbitMQ unless explicitly configured. /// /// The descriptor for method chaining. IRabbitMQQueueDescriptor Quorum(); diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/RabbitMQMessagingTransportDescriptor.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/RabbitMQMessagingTransportDescriptor.cs index 4cb99dcd694..4e4def48685 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/RabbitMQMessagingTransportDescriptor.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/RabbitMQMessagingTransportDescriptor.cs @@ -149,6 +149,31 @@ public IRabbitMQMessagingTransportDescriptor ConnectionProvider( return this; } + /// + public IRabbitMQMessagingTransportDescriptor ConnectTimeout(TimeSpan timeout) + { + ArgumentOutOfRangeException.ThrowIfLessThan(timeout, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfGreaterThan(timeout, TimeSpan.FromMilliseconds(uint.MaxValue - 1)); + + Configuration.ConnectTimeout = timeout; + + return this; + } + + /// + public IRabbitMQMessagingTransportDescriptor PublishTimeout(TimeSpan timeout) + { + if (timeout != Timeout.InfiniteTimeSpan) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(timeout, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfGreaterThan(timeout, TimeSpan.FromMilliseconds(uint.MaxValue - 1)); + } + + Configuration.PublishTimeout = timeout; + + return this; + } + /// public IRabbitMQMessagingTransportDescriptor ConfigureDefaults(Action configure) { diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/RabbitMQQueueDescriptor.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/RabbitMQQueueDescriptor.cs index 262ff930e6e..111280eb939 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/RabbitMQQueueDescriptor.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Descriptors/RabbitMQQueueDescriptor.cs @@ -156,7 +156,7 @@ public IRabbitMQQueueDescriptor UseReceive( { if (before is not null && after is not null) { - throw ThrowHelper.BeforeAndAfterConflict(); + throw global::Mocha.ThrowHelper.BeforeAndAfterConflict(); } if (before is null && after is null) diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Extensions/RequestTaskExtensions.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Extensions/RequestTaskExtensions.cs new file mode 100644 index 00000000000..4141f23720e --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Extensions/RequestTaskExtensions.cs @@ -0,0 +1,58 @@ +namespace Mocha.Transport.RabbitMQ; + +internal static class RequestTaskExtensions +{ + /// + /// Waits for a request to the broker, and stops waiting when is + /// cancelled. RabbitMQ.Client does not end a request with a token once the request is sent, so the + /// abandoned request stays in flight, and its channel must not be used again. + /// + public static Task WaitOrAbandonAsync(this Task request, CancellationToken cancellationToken) + { + return request.IsCompleted || !cancellationToken.CanBeCanceled + ? request + : WaitAsync(request, cancellationToken); + + static async Task WaitAsync(Task request, CancellationToken cancellationToken) + { + try + { + await request.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + ObserveFailure(request); + throw; + } + } + } + + /// + public static Task WaitOrAbandonAsync(this Task request, CancellationToken cancellationToken) + { + return request.IsCompleted || !cancellationToken.CanBeCanceled + ? request + : WaitAsync(request, cancellationToken); + + static async Task WaitAsync(Task request, CancellationToken cancellationToken) + { + try + { + return await request.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + ObserveFailure(request); + throw; + } + } + } + + // A failure of the abandoned request is read, so it does not surface as an unobserved task exception. + private static void ObserveFailure(Task request) + => request.ContinueWith( + static task => _ = task.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); +} diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Middlewares/Receive/RabbitMQAcknowledgementMiddleware.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Middlewares/Receive/RabbitMQAcknowledgementMiddleware.cs index 6072c2be2ca..ac672fc6b57 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Middlewares/Receive/RabbitMQAcknowledgementMiddleware.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Middlewares/Receive/RabbitMQAcknowledgementMiddleware.cs @@ -1,5 +1,10 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Mocha.Middlewares; using Mocha.Transport.RabbitMQ.Features; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Exceptions; namespace Mocha.Transport.RabbitMQ.Middlewares; @@ -7,7 +12,7 @@ namespace Mocha.Transport.RabbitMQ.Middlewares; /// Receive middleware that sends a BasicAck on successful processing and a BasicNack (with requeue) on failure, /// ensuring messages are properly acknowledged or returned to the broker. /// -internal sealed class RabbitMQAcknowledgementMiddleware +internal sealed class RabbitMQAcknowledgementMiddleware(ILogger logger) { /// /// Invokes the next middleware in the pipeline and acknowledges or negatively acknowledges the message based on the outcome. @@ -19,37 +24,81 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next var feature = context.Features.GetOrSet(); var channel = feature.Channel; var eventArgs = feature.EventArgs; - try { await next(context); - - // Once next returns the handler's work is committed, so the ack is sent without the - // shutdown token. A cancelled handler throws and is nacked with requeue instead. - if (channel.IsOpen) - { - await channel.BasicAckAsync(eventArgs.DeliveryTag, false, CancellationToken.None); - } } catch { - if (channel.IsOpen) + await TrySettleAsync(context, channel, eventArgs, ack: false); + throw; + } + + // Once next returns the handler's work is committed, so the ack is sent without the shutdown token. + await TrySettleAsync(context, channel, eventArgs, ack: true); + } + + private async ValueTask TrySettleAsync( + IReceiveContext context, + IChannel channel, + BasicDeliverEventArgs eventArgs, + bool ack) + { + if (channel.IsOpen) + { + try { - await channel.BasicNackAsync(eventArgs.DeliveryTag, false, true, CancellationToken.None); - } + if (ack) + { + await channel.BasicAckAsync(eventArgs.DeliveryTag, false, CancellationToken.None); + } + else + { + await channel.BasicNackAsync(eventArgs.DeliveryTag, false, true, CancellationToken.None); + } - throw; + return; + } + catch (Exception ex) when (ex is AlreadyClosedException or ObjectDisposedException) + { + // The channel closed after the check above. + } } + + LogSkippedSettlement(context, eventArgs, ack ? "ack" : "nack"); } - private static readonly RabbitMQAcknowledgementMiddleware s_instance = new(); + private void LogSkippedSettlement(IReceiveContext context, BasicDeliverEventArgs delivery, string settlement) + { + if (!logger.IsEnabled(LogLevel.Information)) + { + return; + } + + using ((context.Transport as RabbitMQMessagingTransport)?.ConsumerManager.Connection.BeginLogScope()) + using (logger.BeginMessageScope(context)) + { + logger.SettlementSkipped( + settlement, + delivery.BasicProperties.MessageId ?? context.MessageId, + delivery.ConsumerTag, + delivery.DeliveryTag, + delivery.Exchange, + delivery.RoutingKey, + delivery.Redelivered); + } + } /// - /// Creates a that wraps the acknowledgement middleware singleton. + /// Creates middleware that acknowledges handled deliveries and requeues failed deliveries. /// - /// A middleware configuration keyed as "RabbitMQAcknowledgement". public static ReceiveMiddlewareConfiguration Create() => new( - static (_, next) => ctx => s_instance.InvokeAsync(ctx, next), + static (context, next) => + { + var middleware = new RabbitMQAcknowledgementMiddleware( + context.Services.GetRequiredService>()); + return ctx => middleware.InvokeAsync(ctx, next); + }, "RabbitMQAcknowledgement"); } diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Mocha.Transport.RabbitMQ.csproj b/src/Mocha/src/Mocha.Transport.RabbitMQ/Mocha.Transport.RabbitMQ.csproj index 27fddde27c1..61d1f295bfb 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Mocha.Transport.RabbitMQ.csproj +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Mocha.Transport.RabbitMQ.csproj @@ -9,7 +9,12 @@ - + + diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQDispatchEndpoint.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQDispatchEndpoint.cs index 6918ff6f0c1..14ddc0ecce9 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQDispatchEndpoint.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQDispatchEndpoint.cs @@ -1,17 +1,25 @@ +using System.Diagnostics; using Mocha.Middlewares; +using Mocha.Threading; using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; using static System.StringSplitOptions; namespace Mocha.Transport.RabbitMQ; /// /// RabbitMQ dispatch endpoint that publishes outbound messages to a target queue or exchange -/// using pooled channels from the transport's dispatcher. +/// and waits for the broker to confirm each message. /// /// The owning RabbitMQ transport instance. public sealed class RabbitMQDispatchEndpoint(RabbitMQMessagingTransport transport) : DispatchEndpoint(transport) { + private static readonly TimeSpan s_initialRetryDelay = TimeSpan.FromMilliseconds(250); + private static readonly TimeSpan s_maxRetryDelay = TimeSpan.FromSeconds(5); + + private volatile bool _isProvisioned; + /// /// Gets the target queue for this endpoint, or null if the endpoint targets an exchange. /// @@ -24,27 +32,183 @@ public sealed class RabbitMQDispatchEndpoint(RabbitMQMessagingTransport transpor protected override async ValueTask DispatchAsync(IDispatchContext context) { - if (context.Envelope is not { } envelope) + var envelope = context.Envelope ?? throw ThrowHelper.EnvelopeRequired(); + var dispatcher = transport.Dispatcher; + var callerToken = context.CancellationToken; + var started = Stopwatch.GetTimestamp(); + var attempt = 1; + Exception? lastFailure = null; + + var deadline = dispatcher.StartPublish(callerToken); + var cancellationToken = deadline?.Token ?? callerToken; + + try + { + while (true) + { + try + { + await PublishOnceAsync(context, envelope, cancellationToken); + break; + } + catch (Exception ex) when (dispatcher.CanRetry(ex, cancellationToken)) + { + // The message may have reached the broker, so the next attempt can deliver it twice. + lastFailure = ex; + var delay = ExponentialBackoff.CalculateDelay(attempt, s_initialRetryDelay, s_maxRetryDelay); + dispatcher.LogPublishRetry(context, ex, attempt, delay); + attempt++; + + // The next attempt waits for a connection by itself. The delay keeps a publish that fails + // on a ready connection from spinning. + await Task.Delay(delay, cancellationToken); + } + } + + dispatcher.LogPublishConfirmed(context, attempt, started); + } + catch (Exception ex) + { + if (callerToken.IsCancellationRequested) + { + dispatcher.LogPublishCancelled(context, attempt, started); + if (ex is OperationCanceledException) + { + throw ThrowHelper.PublishCancelled(ex, callerToken); + } + + throw; + } + + if (deadline is { IsCancellationRequested: true } && !dispatcher.IsDisposed) + { + var timedOut = ThrowHelper.PublishTimedOut(dispatcher.PublishTimeout, lastFailure ?? ex); + dispatcher.LogPublishFailed(context, timedOut, attempt, started); + throw timedOut; + } + + dispatcher.LogPublishFailed(context, ex, attempt, started); + throw; + } + finally { - throw new InvalidOperationException("Envelope is not set"); + dispatcher.FinishPublish(deadline); } + } + /// + /// Publishes the message on a leased channel and waits for the broker's answer. The channel goes back + /// to the pool only when the broker answered: with a confirmation, a rejection or a return. + /// + private async ValueTask PublishOnceAsync( + IDispatchContext context, + MessageEnvelope envelope, + CancellationToken cancellationToken) + { var dispatcher = transport.Dispatcher; - var cancellationToken = context.CancellationToken; var timeProvider = context.Services.GetTimeProvider(); - var channel = await dispatcher.RentChannelAsync(cancellationToken); + + // A request is published once its reply queue exists and is consumed, so the reply cannot be dropped. + if (transport.ReplyReceiveEndpoint is RabbitMQReceiveEndpoint replyEndpoint + && envelope.ResponseAddress == replyEndpoint.Source.Address.ToString()) + { + await replyEndpoint.WaitUntilAttachedAsync(cancellationToken); + } + + var lease = await dispatcher.RentChannelAsync(cancellationToken); + var isChannelReusable = false; try { - await EnsureProvisionedAsync(channel, cancellationToken); - await DispatchAsync(channel, envelope, timeProvider, cancellationToken); + await EnsureProvisionedAsync(lease, cancellationToken); + + PublishException returned; + try + { + await PublishAsync(lease.Channel, envelope, timeProvider, cancellationToken); + isChannelReusable = true; + return; + } + catch (PublishException ex) + { + isChannelReusable = true; + if (!ex.IsReturn || Queue is null) + { + throw; + } + + returned = ex; + } + + // The queue of the endpoint does not exist. + if (Queue.Exclusive || !transport.Topology.ShouldProvision(Queue)) + { + throw ThrowHelper.MessageReturned(Queue.Name, returned); + } + + isChannelReusable = false; + dispatcher.LogMessageReturned(context, Queue.Name); + await ProvisionQueueAsync(lease, Queue, cancellationToken); + dispatcher.LogQueueRedeclared(context, Queue.Name); + + try + { + await PublishAsync(lease.Channel, envelope, timeProvider, cancellationToken); + isChannelReusable = true; + } + catch (PublishException ex) + { + isChannelReusable = true; + if (ex.IsReturn) + { + throw ThrowHelper.MessageReturned(Queue.Name, ex); + } + + throw; + } } finally { - await dispatcher.ReturnChannelAsync(channel); + dispatcher.Return(lease, isChannelReusable); } } - private async ValueTask DispatchAsync( + private async ValueTask EnsureProvisionedAsync(PublisherLease lease, CancellationToken cancellationToken) + { + if (_isProvisioned) + { + return; + } + + var topology = transport.Topology; + + // An exclusive queue belongs to the connection that declares it, which must be the consumer's. + if (Queue is { Exclusive: false } queue && topology.ShouldProvision(queue)) + { + await ProvisionQueueAsync(lease, queue, cancellationToken); + } + + if (Exchange is { } exchange && topology.ShouldProvision(exchange)) + { + await exchange + .ProvisionAsync(lease.Channel, lease.Connection, cancellationToken) + .WaitOrAbandonAsync(cancellationToken); + } + + _isProvisioned = true; + } + + /// + /// Declares a queue, its bindings and the exchanges they start at. + /// + private Task ProvisionQueueAsync(PublisherLease lease, RabbitMQQueue queue, CancellationToken cancellationToken) + { + var provisioner = new RabbitMQTopologyProvisioner(transport.Topology, lease.Connection); + return provisioner + .ProvisionQueueAsync(queue, lease.Channel, cancellationToken) + .WaitOrAbandonAsync(cancellationToken); + } + + private ValueTask PublishAsync( IChannel channel, MessageEnvelope envelope, TimeProvider timeProvider, @@ -125,31 +289,17 @@ private async ValueTask DispatchAsync( var properties = RabbitMQMessageEnvelopeFormatter.Format(envelope, timeProvider); - await channel.BasicPublishAsync(exchangeName, routingKey, false, properties, envelope.Body, cancellationToken); - } - - private bool _isProvisioned; + // A message sent to a missing queue is confirmed and dropped unless the broker is asked to return it. + // A reply whose requester is gone has no queue, and is dropped. + var mandatory = Queue is not null && Kind != DispatchEndpointKind.Reply; - private async ValueTask EnsureProvisionedAsync(IChannel channel, CancellationToken cancellationToken) - { - if (_isProvisioned) - { - return; - } - - var autoProvision = ((RabbitMQMessagingTopology)transport.Topology).AutoProvision; - - if (Queue is not null && (Queue.AutoProvision ?? autoProvision)) - { - await Queue.ProvisionAsync(channel, cancellationToken); - } - - if (Exchange is not null && (Exchange.AutoProvision ?? autoProvision)) - { - await Exchange.ProvisionAsync(channel, cancellationToken); - } - - _isProvisioned = true; + return channel.BasicPublishAsync( + exchangeName, + routingKey, + mandatory, + properties, + envelope.Body, + cancellationToken); } protected override void OnInitialize( diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQMessagingTransport.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQMessagingTransport.cs index 8834858d668..dfcabfc55b7 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQMessagingTransport.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQMessagingTransport.cs @@ -25,27 +25,35 @@ public RabbitMQMessagingTransport(Action private RabbitMQMessagingTopology _topology = null!; /// - public override MessagingTopology Topology => _topology; + public override RabbitMQMessagingTopology Topology => _topology; /// - /// Gets the consumer manager responsible for registering and maintaining queue consumers with automatic reconnection. + /// Gets the owner of the connection that consumes messages, and of the consumers on it. /// - public RabbitMQConsumerManager ConsumerManager { get; private set; } = null!; + internal RabbitMQConsumerManager ConsumerManager { get; private set; } = null!; /// - /// Gets the dispatcher that provides pooled channels for publishing messages to RabbitMQ. + /// Gets the owner of the connection that publishes messages, and of its channels. /// - public RabbitMQDispatcher Dispatcher { get; private set; } = null!; + internal RabbitMQDispatcher Dispatcher { get; private set; } = null!; /// /// Gets the connection provider used to create RabbitMQ connections. /// public IRabbitMQConnectionProvider Connection { get; private set; } = null!; + /// + /// Gets the reason the transport is unhealthy, or null when it is started, both broker connections + /// are ready, and every receive endpoint is healthy. + /// + public override string? HealthFailureReason + => !IsStarted ? "The transport is not started." + : !Dispatcher.IsConnected ? "The dispatcher connection is unavailable." + : ConsumerManager.HealthFailureReason ?? base.HealthFailureReason; + /// /// Resolves the RabbitMQ connection provider, builds the transport topology URI, and creates - /// the and instances used for the - /// lifetime of this transport. + /// the owners of the two connections used for the lifetime of this transport. /// /// /// Called once during the messaging host initialization phase, after the base transport has @@ -92,52 +100,74 @@ protected override void OnAfterInitialized(IMessagingSetupContext context) _topology.AddBinding(binding); } - Dispatcher = CreateDispatcher(context); - ConsumerManager = CreateConsumerManager(context); + // Reconnect delays and timeouts follow the system clock, also when the application replaces the clock. + var timeProvider = TimeProvider.System; + + Dispatcher = new RabbitMQDispatcher( + context.Services.GetRequiredService>(), + CreateConnectionManager( + context, + context.Services.GetRequiredService>(), + timeProvider, + "dispatcher", + ProvisionTopologyAsync)); + + ConsumerManager = new RabbitMQConsumerManager( + context.Services.GetRequiredService>(), + CreateConnectionManager( + context, + context.Services.GetRequiredService>(), + timeProvider, + "consumer", + RabbitMQConsumerManager.SetUpConnectionAsync)); + + if (configuration.ConnectTimeout is { } connectTimeout) + { + Dispatcher.Connection.ConnectTimeout = connectTimeout; + ConsumerManager.Connection.ConnectTimeout = connectTimeout; + } + + if (configuration.PublishTimeout is { } publishTimeout) + { + Dispatcher.PublishTimeout = publishTimeout; + } } - private RabbitMQConsumerManager CreateConsumerManager(IMessagingSetupContext context) + private async Task ProvisionTopologyAsync(IConnection connection, CancellationToken cancellationToken) { - var logger = context.Services.GetRequiredService>(); - - return new RabbitMQConsumerManager(logger, Connection.CreateAsync); + await using var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken); + await new RabbitMQTopologyProvisioner(_topology, connection).ProvisionAsync(channel, cancellationToken); } - private RabbitMQDispatcher CreateDispatcher(IMessagingSetupContext context) + private RabbitMQConnectionManager CreateConnectionManager( + IMessagingSetupContext context, + ILogger logger, + TimeProvider timeProvider, + string role, + Func setUpConnectionAsync) { - var logger = context.Services.GetRequiredService>(); + var serviceName = context.Host.ServiceName ?? context.Host.ProcessName; + var connectionName = $"{serviceName} {Name} {role}"; - return new RabbitMQDispatcher(logger, Connection.CreateAsync, ProvisionTopologyAsync); + Func> connectionFactory = + Connection is ConnectionFactoryRabbitMQConnectionProvider provider + ? cancellationToken => provider.CreateAsync(connectionName, cancellationToken) + : Connection.CreateAsync; - async Task ProvisionTopologyAsync(IConnection connection, CancellationToken ct) + return new RabbitMQConnectionManager(logger, connectionName, timeProvider, connectionFactory, setUpConnectionAsync) { - await using var channel = await connection.CreateChannelAsync(cancellationToken: ct); - var autoProvision = _topology.AutoProvision; - - foreach (var exchange in _topology.Exchanges) - { - if (exchange.AutoProvision ?? autoProvision) - { - await exchange.ProvisionAsync(channel, ct); - } - } - - foreach (var queue in _topology.Queues) - { - if (queue.AutoProvision ?? autoProvision) - { - await queue.ProvisionAsync(channel, ct); - } - } - - foreach (var binding in _topology.Bindings) + LogContext = new Dictionary { - if (binding.AutoProvision ?? autoProvision) - { - await binding.ProvisionAsync(channel, ct); - } + ["ServiceName"] = serviceName, + ["InstanceId"] = context.Host.InstanceId, + ["Transport"] = Name, + ["ConnectionName"] = connectionName, + ["ConnectionRole"] = role, + ["BrokerHost"] = Connection.Host, + ["BrokerPort"] = Connection.Port, + ["VirtualHost"] = Connection.VirtualHost } - } + }; } /// @@ -149,7 +179,6 @@ public override TransportDescription Describe() var entities = new List(); var links = new List(); - var autoProvision = _topology.AutoProvision; foreach (var exchange in _topology.Exchanges) { @@ -165,7 +194,7 @@ public override TransportDescription Describe() ["type"] = exchange.Type, ["durable"] = exchange.Durable, ["autoDelete"] = exchange.AutoDelete, - ["autoProvision"] = exchange.AutoProvision ?? autoProvision, + ["autoProvision"] = _topology.ShouldProvision(exchange), ["origin"] = exchange.Origin })); } @@ -184,7 +213,7 @@ public override TransportDescription Describe() ["durable"] = queue.Durable, ["exclusive"] = queue.Exclusive, ["autoDelete"] = queue.AutoDelete, - ["autoProvision"] = queue.AutoProvision ?? autoProvision, + ["autoProvision"] = _topology.ShouldProvision(queue), ["origin"] = queue.Origin })); } @@ -210,7 +239,7 @@ public override TransportDescription Describe() new Dictionary { ["routingKey"] = string.IsNullOrEmpty(binding.RoutingKey) ? null : binding.RoutingKey, - ["autoProvision"] = binding.AutoProvision ?? autoProvision, + ["autoProvision"] = _topology.ShouldProvision(binding), ["origin"] = binding.Origin })); } @@ -310,14 +339,12 @@ public override bool TryGetDispatchEndpoint(Uri address, [NotNullWhen(true)] out } /// - /// Ensures that both the consumer and dispatcher RabbitMQ connections are established before - /// the transport's endpoints begin processing messages. + /// Connects the consumer and dispatcher RabbitMQ connections before the transport's endpoints start. /// /// - /// Both and connect concurrently via - /// Task.WhenAll. If either connection fails, the start-up will fail and the host will - /// not begin consuming or dispatching messages. This is the last opportunity to guarantee - /// network connectivity before the messaging pipeline is active. + /// While the broker cannot be reached, both connections keep retrying for up to the connect timeout, two minutes + /// by default, before the start fails. Refused credentials, refused virtual host access, and topology the broker + /// rejects fail the start at once. /// /// The configuration context for the current startup phase. /// A token to cancel the connection establishment. @@ -325,10 +352,9 @@ protected override async ValueTask OnBeforeStartAsync( IMessagingConfigurationContext context, CancellationToken cancellationToken) { - // TODO we probably should make this resilient! await Task.WhenAll( - ConsumerManager.EnsureConnectedAsync(cancellationToken), - Dispatcher.EnsureConnectedAsync(cancellationToken)); + ConsumerManager.StartAsync(cancellationToken), + Dispatcher.StartAsync(cancellationToken)); } /// @@ -353,7 +379,7 @@ protected override MessagingTransportConfiguration CreateConfiguration(IMessagin /// /// Creates a new bound to this transport, which will - /// consume messages from a RabbitMQ queue via the . + /// consume messages from a RabbitMQ queue on the transport's consumer connection. /// /// A new, uninitialized for this transport. protected override ReceiveEndpoint CreateReceiveEndpoint() @@ -363,7 +389,7 @@ protected override ReceiveEndpoint CreateReceiveEndpoint() /// /// Creates a new bound to this transport, which will - /// publish messages to RabbitMQ exchanges or queues via the . + /// publish messages to RabbitMQ exchanges or queues on the transport's dispatcher connection. /// /// A new, uninitialized for this transport. protected override DispatchEndpoint CreateDispatchEndpoint() diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQReceiveEndpoint.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQReceiveEndpoint.cs index 18afec90c3a..c950a3c0968 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQReceiveEndpoint.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/RabbitMQReceiveEndpoint.cs @@ -19,6 +19,11 @@ public sealed class RabbitMQReceiveEndpoint(RabbitMQMessagingTransport transport /// public RabbitMQQueue Queue { get; private set; } = null!; + /// + /// Gets whether the endpoint's consumer is attached and the broker delivers to it. + /// + internal bool IsHealthy => _consumer is { IsAttached: true }; + protected override void OnInitialize( IMessagingConfigurationContext context, RabbitMQReceiveEndpointConfiguration configuration) @@ -45,7 +50,7 @@ protected override void OnComplete( throw new InvalidOperationException("Queue name is required"); } - var topology = (RabbitMQMessagingTopology)Transport.Topology; + var topology = transport.Topology; Queue = topology.Queues.FirstOrDefault(q => q.Name == configuration.QueueName) @@ -54,23 +59,35 @@ protected override void OnComplete( Source = Queue; } - private IAsyncDisposable? _consumer; + private RabbitMQConsumer? _consumer; - protected override async ValueTask OnStartAsync( - IMessagingRuntimeContext context, - CancellationToken cancellationToken) + /// + /// Waits until the endpoint's consumer is attached. Fails when the endpoint is not started. + /// + internal ValueTask WaitUntilAttachedAsync(CancellationToken cancellationToken) { - if (Transport is not RabbitMQMessagingTransport rabbitMQMessagingTransport) + if (_consumer is not { } consumer) { - throw new InvalidOperationException("Transport is not a RabbitMQMessagingTransport"); + throw ThrowHelper.ReplyEndpointNotStarted(); } - _consumer = await rabbitMQMessagingTransport.ConsumerManager.RegisterConsumerAsync( + return consumer.WaitUntilAttachedAsync(cancellationToken); + } + + protected override async ValueTask OnStartAsync( + IMessagingRuntimeContext context, + CancellationToken cancellationToken) + { + _consumer = await transport.ConsumerManager.RegisterConsumerAsync( Queue.Name, + PrepareChannelAsync, ProcessMessageAsync, _maxPrefetch, _consumerDispatchConcurrency, cancellationToken); + + Task PrepareChannelAsync(IConnection connection, IChannel channel, CancellationToken ct) + => new RabbitMQTopologyProvisioner(transport.Topology, connection).ProvisionQueueAsync(Queue, channel, ct); } private async ValueTask ProcessMessageAsync( @@ -102,9 +119,18 @@ protected override async ValueTask OnStopAsync( { if (_consumer is not null) { - await _consumer.DisposeAsync(); + await _consumer.StopAsync(cancellationToken); } _consumer = null; + + // A generated temporary reply queue is deleted on graceful stop when auto-provisioning is enabled. + if (Kind == ReceiveEndpointKind.Reply + && Configuration.IsTemporary + && Queue.Origin is not TopologyOrigin.Declared + && transport.Topology.ShouldProvision(Queue)) + { + await transport.ConsumerManager.DeleteQueueAsync(Queue.Name, cancellationToken); + } } } diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/ThrowHelper.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/ThrowHelper.cs index 05076b43d18..0273d2ebcfa 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/ThrowHelper.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/ThrowHelper.cs @@ -1,7 +1,42 @@ +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Exceptions; + namespace Mocha.Transport.RabbitMQ; internal static class ThrowHelper { + public static InvalidOperationException EnvelopeRequired() + => new("Envelope is not set"); + + public static TimeoutException ConnectionAttemptTimedOut(TimeSpan timeout, Exception innerException) + => new($"The connection attempt did not complete within {timeout}.", innerException); + + public static TimeoutException ConnectionRequestTimedOut(Exception innerException) + => new("The broker did not answer a request of the connection attempt in time.", innerException); + + public static ChannelOpenBlockedException ChannelOpenBlocked(Exception innerException) + => new( + "The broker did not answer the request to open a channel, because it blocked the connection.", + innerException); + + public static TimeoutException ChannelOpenTimedOut(Exception innerException) + => new("The broker did not answer the request to open a channel.", innerException); + + public static ObjectDisposedException ConnectionManagerDisposed() + => new(typeof(RabbitMQConnectionManager).FullName); + + public static OperationCanceledException PublishCancelled(Exception innerException, CancellationToken callerToken) + => new(innerException.Message, innerException, callerToken); + + public static TransportUnavailableException PublishTimedOut(TimeSpan timeout, Exception? innerException) + => new( + $"The broker did not confirm the message within the publish timeout of {timeout}. " + + "The broker may or may not have received the message.", + innerException); + + public static InvalidOperationException ReplyEndpointNotStarted() + => new("The RabbitMQ reply endpoint is not started."); + public static Exception BeforeAndAfterConflict() => Mocha.ThrowHelper.BeforeAndAfterConflict(); @@ -10,4 +45,26 @@ public static Exception TemporaryEndpointQueueConflict(string queueName, string? $"Queue '{queueName}' is explicitly declared without auto-delete, which conflicts with " + $"receive endpoint '{endpointName}' being marked Temporary(). Declare the queue with " + "AutoDelete(), or remove Temporary() from the endpoint."); + + public static OperationInterruptedException ConnectionClosedDuringSetup(ShutdownEventArgs? reason) + => reason is null ? new OperationInterruptedException() : new AlreadyClosedException(reason); + + public static TimeoutException ConsumerAttachTimedOut(string queueName, TimeSpan timeout) + => new($"Attaching the consumer for queue '{queueName}' did not complete within {timeout}."); + + public static TimeoutException ConsumerAttachRequestTimedOut(string queueName, Exception innerException) + => new( + $"The broker did not answer a request while the consumer for queue '{queueName}' attached.", + innerException); + + public static InvalidOperationException ConsumerStopped(string queueName) + => new($"The consumer for queue '{queueName}' is stopped."); + + public static TimeoutException ConnectionTimedOut(TimeSpan timeout, Exception innerException) + => new($"Connecting to RabbitMQ did not complete within {timeout}.", innerException); + + public static InvalidOperationException MessageReturned(string queueName, Exception innerException) + => new( + $"The broker returned the message because no queue named '{queueName}' exists.", + innerException); } diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/Configurations/RabbitMQQueueConfiguration.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/Configurations/RabbitMQQueueConfiguration.cs index b18b37d6d2f..3ca5c2fbacb 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/Configurations/RabbitMQQueueConfiguration.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/Configurations/RabbitMQQueueConfiguration.cs @@ -43,4 +43,9 @@ public sealed class RabbitMQQueueConfiguration : TopologyConfiguration public bool? AutoProvision { get; set; } + + /// + /// Gets or sets a value indicating whether this is a generated temporary reply queue. + /// + internal bool IsTemporaryReplyQueue { get; set; } } diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/Extensions/RabbitMQQueueTopologyDescriptorExtensions.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/Extensions/RabbitMQQueueTopologyDescriptorExtensions.cs index b5e5bb640f5..29a659d9a83 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/Extensions/RabbitMQQueueTopologyDescriptorExtensions.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/Extensions/RabbitMQQueueTopologyDescriptorExtensions.cs @@ -145,11 +145,11 @@ public static IRabbitMQQueueTopologyDescriptor QuorumInitialGroupSize(this IRabb } /// - /// Sets the maximum delivery limit for messages in the queue. + /// Sets the redelivery limit for a quorum queue. /// Messages that exceed this limit will be dead-lettered or discarded. /// /// The queue descriptor. - /// The maximum number of delivery attempts. + /// The broker's redelivery limit, or -1 for unlimited redeliveries. /// The descriptor for method chaining. public static IRabbitMQQueueTopologyDescriptor MaxDeliveryLimit(this IRabbitMQQueueTopologyDescriptor descriptor, int limit) { diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/IRabbitMQResource.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/IRabbitMQResource.cs index 5a1f06fa48c..6d0ab1faca1 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/IRabbitMQResource.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/IRabbitMQResource.cs @@ -11,6 +11,7 @@ public interface IRabbitMQResource /// Declares this resource on the broker using the specified channel. /// /// The RabbitMQ channel to use for declaring the resource. + /// The connection that owns the channel. /// A token to cancel the provisioning operation. - Task ProvisionAsync(IChannel channel, CancellationToken cancellationToken); + Task ProvisionAsync(IChannel channel, IConnection connection, CancellationToken cancellationToken); } diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQBinding.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQBinding.cs index 661b0861e0a..12c3af10d05 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQBinding.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQBinding.cs @@ -53,8 +53,9 @@ private protected static string BuildQuery(string routingKey) /// Declares this binding on the broker using the specified channel. /// /// The RabbitMQ channel to use for declaring the binding. + /// The connection that owns the channel. /// A token to cancel the provisioning operation. - public abstract Task ProvisionAsync(IChannel channel, CancellationToken cancellationToken); + public abstract Task ProvisionAsync(IChannel channel, IConnection connection, CancellationToken cancellationToken); } /// @@ -94,7 +95,7 @@ internal void SetDestination(RabbitMQExchange destination) } /// - public override async Task ProvisionAsync(IChannel channel, CancellationToken cancellationToken) + public override async Task ProvisionAsync(IChannel channel, IConnection connection, CancellationToken cancellationToken) { await channel.ExchangeBindAsync( Destination.Name, @@ -142,7 +143,7 @@ internal void SetDestination(RabbitMQQueue destination) } /// - public override async Task ProvisionAsync(IChannel channel, CancellationToken cancellationToken) + public override async Task ProvisionAsync(IChannel channel, IConnection connection, CancellationToken cancellationToken) { await channel.QueueBindAsync( Destination.Name, diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQExchange.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQExchange.cs index a7cccc1ae89..bed5ff51fbf 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQExchange.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQExchange.cs @@ -88,8 +88,9 @@ internal void AddBinding(RabbitMQBinding binding) /// Declares this exchange on the broker using the specified channel. /// /// The RabbitMQ channel to use for declaring the exchange. + /// The connection that owns the channel. /// A token to cancel the provisioning operation. - public async Task ProvisionAsync(IChannel channel, CancellationToken cancellationToken) + public async Task ProvisionAsync(IChannel channel, IConnection connection, CancellationToken cancellationToken) { await channel.ExchangeDeclareAsync( CachedName.Value, diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQMessagingTopology.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQMessagingTopology.cs index 78a7fca2ca4..8b4d3096336 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQMessagingTopology.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQMessagingTopology.cs @@ -32,6 +32,21 @@ public sealed class RabbitMQMessagingTopology( /// public bool AutoProvision => autoProvision; + /// + /// Gets whether the exchange is auto-provisioned, by its own setting or else the topology default. + /// + internal bool ShouldProvision(RabbitMQExchange exchange) => exchange.AutoProvision ?? autoProvision; + + /// + /// Gets whether the queue is auto-provisioned, by its own setting or else the topology default. + /// + internal bool ShouldProvision(RabbitMQQueue queue) => queue.AutoProvision ?? autoProvision; + + /// + /// Gets whether the binding is auto-provisioned, by its own setting or else the topology default. + /// + internal bool ShouldProvision(RabbitMQBinding binding) => binding.AutoProvision ?? autoProvision; + /// /// Gets the list of exchanges registered in this topology. /// @@ -149,7 +164,14 @@ private RabbitMQQueue CreateQueue(RabbitMQQueueConfiguration configuration) { configuration.Topology = this; - defaults.Queue.ApplyTo(configuration); + if (configuration.IsTemporaryReplyQueue) + { + defaults.Queue.ApplyToReplyQueue(configuration); + } + else + { + defaults.Queue.ApplyTo(configuration); + } var queue = new RabbitMQQueue(); queue.Initialize(configuration); diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQQueue.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQQueue.cs index 23da6ab16a9..4eef7bc3324 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQQueue.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQQueue.cs @@ -94,19 +94,45 @@ internal void MarkTemporary(TimeSpan expiry) Arguments = Arguments.SetItem("x-expires", (int)expiry.TotalMilliseconds); } + /// + /// Marks this queue as not auto-delete, and makes the broker delete it once it has been unused for + /// . + /// + /// How long the queue may stay without consumers before the broker deletes it. + internal void MarkExpiring(TimeSpan expiration) + { + AutoDelete = false; + Arguments = Arguments.SetItem("x-expires", (int)expiration.TotalMilliseconds); + } + /// /// Declares this queue on the broker using the specified channel. /// /// The RabbitMQ channel to use for declaring the queue. + /// The connection that owns the channel. /// A token to cancel the provisioning operation. - public async Task ProvisionAsync(IChannel channel, CancellationToken cancellationToken) + public async Task ProvisionAsync(IChannel channel, IConnection connection, CancellationToken cancellationToken) { + var arguments = Arguments; + var isLegacyBroker = connection.ServerProperties is { } properties + && properties.TryGetValue("version", out var version) + && version switch + { + byte[] bytes => bytes.AsSpan().StartsWith("3."u8), + string text => text.StartsWith("3.", StringComparison.Ordinal), + _ => false + }; + if (isLegacyBroker && arguments.TryGetValue("x-delivery-limit", out var limit) && limit is -1) + { + arguments = arguments.Remove("x-delivery-limit"); + } + await channel.QueueDeclareAsync( CachedName.Value, Durable, Exclusive, AutoDelete, - Arguments, + arguments, cancellationToken: cancellationToken); } } diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQRoutingStrategy.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQRoutingStrategy.cs index bdd103f152d..b21ec5533fa 100644 --- a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQRoutingStrategy.cs +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQRoutingStrategy.cs @@ -8,8 +8,10 @@ namespace Mocha.Transport.RabbitMQ; /// public sealed class RabbitMQRoutingStrategy : RoutingStrategy { + private static readonly TimeSpan s_replyQueueExpiration = TimeSpan.FromMinutes(1); + private RabbitMQMessagingTopology _topology => - field ??= (RabbitMQMessagingTopology)Transport.Topology; + field ??= Transport.Topology; /// public override DispatchEndpointConfiguration? CreateEndpointConfiguration( @@ -213,16 +215,24 @@ public override void DiscoverTopology( EnsureNoTemporaryQueueConflict(rabbitConfiguration); } + // Generated temporary reply queues expire without consumers and inherit quorum replication settings. + var isReplyQueue = rabbitConfiguration.IsTemporary && rabbitEndpoint.Kind == ReceiveEndpointKind.Reply; + var queue = _topology.GetOrAddQueue( rabbitConfiguration.QueueName, _ => new RabbitMQQueueConfiguration { - AutoDelete = rabbitConfiguration.IsTemporary, + AutoDelete = rabbitConfiguration.IsTemporary && !isReplyQueue, AutoProvision = rabbitConfiguration.AutoProvision, - Origin = TopologyOrigin.Endpoint + Origin = TopologyOrigin.Endpoint, + IsTemporaryReplyQueue = isReplyQueue }); - if (rabbitConfiguration.IsTemporary && queue.Origin is not TopologyOrigin.Declared) + if (isReplyQueue && queue.Origin is not TopologyOrigin.Declared) + { + queue.MarkExpiring(rabbitConfiguration.TemporaryExpiry ?? s_replyQueueExpiration); + } + else if (rabbitConfiguration.IsTemporary && queue.Origin is not TopologyOrigin.Declared) { queue.MarkTemporary(rabbitConfiguration.TemporaryExpiry ?? RabbitMQReceiveEndpointConfiguration.TemporaryDefaults.Expiry); diff --git a/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQTopologyProvisioner.cs b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQTopologyProvisioner.cs new file mode 100644 index 00000000000..6082391d795 --- /dev/null +++ b/src/Mocha/src/Mocha.Transport.RabbitMQ/Topology/RabbitMQTopologyProvisioner.cs @@ -0,0 +1,87 @@ +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ; + +internal sealed class RabbitMQTopologyProvisioner(RabbitMQMessagingTopology topology, IConnection connection) +{ + public async Task ProvisionAsync(IChannel channel, CancellationToken cancellationToken) + { + foreach (var exchange in topology.Exchanges) + { + if (topology.ShouldProvision(exchange)) + { + await exchange.ProvisionAsync(channel, connection, cancellationToken); + } + } + + // An exclusive queue belongs to the connection that declares it, so only its consumer declares it and + // the bindings that target it. + foreach (var queue in topology.Queues) + { + if (!queue.Exclusive && topology.ShouldProvision(queue)) + { + await queue.ProvisionAsync(channel, connection, cancellationToken); + } + } + + foreach (var binding in topology.Bindings) + { + if (binding is not RabbitMQQueueBinding { Destination.Exclusive: true } + && topology.ShouldProvision(binding)) + { + await binding.ProvisionAsync(channel, connection, cancellationToken); + } + } + } + + public async Task ProvisionQueueAsync( + RabbitMQQueue queue, + IChannel channel, + CancellationToken cancellationToken) + { + if (topology.ShouldProvision(queue)) + { + await queue.ProvisionAsync(channel, connection, cancellationToken); + } + + var bindings = new List(); + foreach (var binding in queue.Bindings) + { + if (topology.ShouldProvision(binding)) + { + bindings.Add(binding); + } + } + + // The list grows while it is read, which follows exchange-to-exchange bindings upstream of the queue. + var exchanges = new HashSet(); + for (var i = 0; i < bindings.Count; i++) + { + var exchange = bindings[i].Source; + if (!exchanges.Add(exchange)) + { + continue; + } + + if (topology.ShouldProvision(exchange)) + { + await exchange.ProvisionAsync(channel, connection, cancellationToken); + } + + foreach (var binding in exchange.Bindings) + { + if (binding is RabbitMQExchangeBinding exchangeBinding + && exchangeBinding.Destination == exchange + && topology.ShouldProvision(binding)) + { + bindings.Add(binding); + } + } + } + + foreach (var binding in bindings) + { + await binding.ProvisionAsync(channel, connection, cancellationToken); + } + } +} diff --git a/src/Mocha/src/Mocha/Consumers/Implementations/ReplyConsumer.cs b/src/Mocha/src/Mocha/Consumers/Implementations/ReplyConsumer.cs index 2af347d9b97..57ed5942157 100644 --- a/src/Mocha/src/Mocha/Consumers/Implementations/ReplyConsumer.cs +++ b/src/Mocha/src/Mocha/Consumers/Implementations/ReplyConsumer.cs @@ -66,6 +66,7 @@ protected override ValueTask ConsumeAsync(IConsumeContext context) { // Fault the waiting requester rather than leave it to time out. responseManager.SetException(correlationId, ex); + using var _ = _logger.BeginMessageScope(context); _logger.ReplyProcessingFailed(ex, correlationId, context.MessageId); } @@ -85,6 +86,7 @@ private void ReportUnmatchedReply(IConsumeContext context, string? correlationId return; } + using var _ = _logger.BeginMessageScope(context); _logger.ReplyDiscarded(correlationId, context.MessageId); } } diff --git a/src/Mocha/src/Mocha/DeferredResponseManager.cs b/src/Mocha/src/Mocha/DeferredResponseManager.cs index e3fafebf3c9..d613e32f0bf 100644 --- a/src/Mocha/src/Mocha/DeferredResponseManager.cs +++ b/src/Mocha/src/Mocha/DeferredResponseManager.cs @@ -1,5 +1,4 @@ using System.Collections.Concurrent; -using Mocha.Events; using static System.Threading.Tasks.TaskCreationOptions; namespace Mocha; @@ -20,23 +19,56 @@ public sealed class DeferredResponseManager(TimeProvider timeProvider) /// A that completes when the correlated response arrives or the timeout elapses. public TaskCompletionSource AddPromise(string correlationId, TimeSpan? timeout = null) { - timeout ??= _defaultTimeout; + var duration = timeout ?? _defaultTimeout; + var timeoutSource = new CancellationTokenSource(duration, timeProvider); + return AddPromise(correlationId, duration, timeoutSource.Token, timeoutSource); + } + + internal CancellationTokenSource CreateRequestTimeout() => new(_defaultTimeout, timeProvider); + + internal TaskCompletionSource AddPromise(string correlationId, CancellationToken timeoutToken) + => AddPromise(correlationId, _defaultTimeout, timeoutToken, ownedTimeout: null); + + private TaskCompletionSource AddPromise( + string correlationId, + TimeSpan timeout, + CancellationToken timeoutToken, + CancellationTokenSource? ownedTimeout) + { var tcs = new TaskCompletionSource(RunContinuationsAsynchronously); - var cts = new CancellationTokenSource(timeout.Value, timeProvider); - var promise = new Promise(tcs, cts, timeout.Value); + var promise = new Promise(tcs, ownedTimeout, timeout); + _matches.TryAdd(correlationId, promise); - cts.Token.Register(() => + timeoutToken.Register(() => { if (_matches.TryRemove(correlationId, out var p)) { - p.TaskCompletionSource.TrySetException(new ResponseTimeoutException(correlationId, p.Timeout)); + p.TaskCompletionSource.TrySetException(ThrowHelper.ResponseTimedOut(correlationId, p.Timeout)); + p.OwnedTimeout?.Dispose(); } }); - _matches.TryAdd(correlationId, promise); return tcs; } + internal async ValueTask ReleasePromiseAsync(string correlationId, Task completion) + { + if (_matches.TryRemove(correlationId, out var promise)) + { + promise.OwnedTimeout?.Dispose(); + promise.TaskCompletionSource.TrySetCanceled(); + } + + try + { + await completion; + } + catch + { + // The request has already reported its response or dispatch failure. + } + } + /// /// Awaits and returns the result of a previously registered promise. /// @@ -62,8 +94,7 @@ public void SetException(string correlationId, Exception exception) { if (_matches.TryRemove(correlationId, out var promise)) { - promise.Cts.Cancel(); - promise.Cts.Dispose(); + promise.OwnedTimeout?.Dispose(); promise.TaskCompletionSource.SetException(exception); } } @@ -78,8 +109,7 @@ public bool CompletePromise(string correlationId, object? response) { if (_matches.TryRemove(correlationId, out var promise)) { - promise.Cts.Cancel(); - promise.Cts.Dispose(); + promise.OwnedTimeout?.Dispose(); promise.TaskCompletionSource.SetResult(response); return true; } @@ -89,6 +119,6 @@ public bool CompletePromise(string correlationId, object? response) private sealed record Promise( TaskCompletionSource TaskCompletionSource, - CancellationTokenSource Cts, + CancellationTokenSource? OwnedTimeout, TimeSpan Timeout); } diff --git a/src/Mocha/src/Mocha/Endpoints/ReceiveEndpoint.cs b/src/Mocha/src/Mocha/Endpoints/ReceiveEndpoint.cs index 030efbffe76..cc184578987 100644 --- a/src/Mocha/src/Mocha/Endpoints/ReceiveEndpoint.cs +++ b/src/Mocha/src/Mocha/Endpoints/ReceiveEndpoint.cs @@ -128,6 +128,7 @@ public async ValueTask ExecuteAsync( catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { // exceptions should technically never bubble up here. + using var _ = logger.BeginMessageScope(context); logger.LogCritical(ex, "Error processing message"); } finally @@ -317,10 +318,13 @@ public async ValueTask StopAsync(IMessagingRuntimeContext context, CancellationT /// /// When overridden in a derived class, performs transport-specific shutdown logic - /// such as closing connections or unsubscribing from queues. + /// such as closing connections or unsubscribing from queues. Releasing the resources of the endpoint, + /// including temporary broker resources, must not depend on . /// /// The messaging runtime context. - /// Token to signal cancellation of the stop operation. + /// + /// A token that bounds the wait for running handlers. It can already be canceled. + /// protected abstract ValueTask OnStopAsync(IMessagingRuntimeContext context, CancellationToken cancellationToken); private void AssertUninitialized() diff --git a/src/Mocha/src/Mocha/Instrumentation/ActivityMessagingDiagnosticListener.cs b/src/Mocha/src/Mocha/Instrumentation/ActivityMessagingDiagnosticListener.cs index 2f5cb3bc78c..66699e8235a 100644 --- a/src/Mocha/src/Mocha/Instrumentation/ActivityMessagingDiagnosticListener.cs +++ b/src/Mocha/src/Mocha/Instrumentation/ActivityMessagingDiagnosticListener.cs @@ -82,7 +82,7 @@ public void Dispose() _activity? .EnrichMessageDefault() .SetMessageId(_context.MessageId ?? string.Empty) - .SetConversationId(_context.CorrelationId ?? string.Empty); + .SetConversationId(_context.ConversationId ?? string.Empty); _activity?.Dispose(); } diff --git a/src/Mocha/src/Mocha/Instrumentation/CircuitBreakerLogging.cs b/src/Mocha/src/Mocha/Instrumentation/CircuitBreakerLogging.cs new file mode 100644 index 00000000000..20347390b31 --- /dev/null +++ b/src/Mocha/src/Mocha/Instrumentation/CircuitBreakerLogging.cs @@ -0,0 +1,53 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Mocha.Middlewares; +using Polly.CircuitBreaker; + +namespace Mocha; + +internal static class CircuitBreakerLogging +{ + public static CircuitBreakerStrategyOptions WithLogging( + this CircuitBreakerStrategyOptions options, + ILogger logger, + ReceiveMiddlewareFactoryContext context, + string circuit) + { + var endpoint = context.Endpoint.Name; + var logContext = new Dictionary + { + ["InstanceId"] = context.Services.GetService()?.InstanceId, + ["Transport"] = context.Transport.Name, + ["Endpoint"] = endpoint, + ["Circuit"] = circuit + }; + options.OnOpened = args => + { + using (logger.BeginScope(logContext)) + { + logger.CircuitOpen(args.Outcome.Exception, circuit, endpoint, args.BreakDuration); + } + + return default; + }; + options.OnClosed = _ => + { + using (logger.BeginScope(logContext)) + { + logger.CircuitClosed(circuit, endpoint); + } + + return default; + }; + options.OnHalfOpened = _ => + { + using (logger.BeginScope(logContext)) + { + logger.CircuitHalfOpen(circuit, endpoint); + } + + return default; + }; + return options; + } +} diff --git a/src/Mocha/src/Mocha/Instrumentation/MessageLogging.cs b/src/Mocha/src/Mocha/Instrumentation/MessageLogging.cs new file mode 100644 index 00000000000..29d0a08de9e --- /dev/null +++ b/src/Mocha/src/Mocha/Instrumentation/MessageLogging.cs @@ -0,0 +1,40 @@ +using Microsoft.Extensions.Logging; +using Mocha.Middlewares; + +namespace Mocha; + +internal static class MessageLogging +{ + public static IDisposable? BeginMessageScope(this ILogger logger, IMessageContext context) + { + return logger.BeginScope(new Dictionary + { + ["InstanceId"] = (context as IExecutionContext)?.Runtime?.Host.InstanceId, + ["Transport"] = context.Transport?.Name, + ["Endpoint"] = context.Endpoint?.Name, + ["SourceAddress"] = context.Endpoint?.Source?.Address, + ["MessageId"] = context.MessageId, + ["MessageType"] = context.MessageType?.Identity ?? context.Envelope?.MessageType, + ["CorrelationId"] = context.CorrelationId, + ["ConversationId"] = context.ConversationId, + ["CausationId"] = context.CausationId, + ["DeliveryCount"] = context.DeliveryCount + }); + } + + public static IDisposable? BeginMessageScope(this ILogger logger, IDispatchContext context) + { + return logger.BeginScope(new Dictionary + { + ["InstanceId"] = context.Runtime?.Host.InstanceId, + ["Transport"] = context.Transport?.Name, + ["Endpoint"] = context.Endpoint?.Name, + ["DestinationAddress"] = context.DestinationAddress, + ["MessageId"] = context.Envelope?.MessageId ?? context.MessageId, + ["MessageType"] = context.Envelope?.MessageType ?? context.MessageType?.Identity, + ["CorrelationId"] = context.Envelope?.CorrelationId ?? context.CorrelationId, + ["ConversationId"] = context.Envelope?.ConversationId ?? context.ConversationId, + ["CausationId"] = context.Envelope?.CausationId ?? context.CausationId + }); + } +} diff --git a/src/Mocha/src/Mocha/Instrumentation/MessagingLogs.cs b/src/Mocha/src/Mocha/Instrumentation/MessagingLogs.cs new file mode 100644 index 00000000000..e08672c278a --- /dev/null +++ b/src/Mocha/src/Mocha/Instrumentation/MessagingLogs.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Logging; + +namespace Mocha; + +internal static partial class MessagingLogs +{ + [LoggerMessage(LogLevel.Information, "Retrying message {MessageId} in consumer {Consumer} (attempt {Attempt}) after {Delay}")] + public static partial void RetryingMessage(this ILogger logger, Exception exception, string? messageId, string consumer, int attempt, TimeSpan delay); + + [LoggerMessage(LogLevel.Warning, "Preparing redelivery of message {MessageId} (attempt {Attempt}) with requested delay {Delay}")] + public static partial void RedeliveringMessage(this ILogger logger, Exception exception, string? messageId, int attempt, TimeSpan delay); + + [LoggerMessage(LogLevel.Information, "Dispatched redelivery of message {MessageId} (attempt {Attempt}) with scheduled time {ScheduledTime}")] + public static partial void MessageRedelivered(this ILogger logger, string? messageId, int attempt, DateTimeOffset? scheduledTime); + + [LoggerMessage(LogLevel.Warning, "Discarded message {MessageId} according to its exception policy")] + public static partial void MessageDiscarded(this ILogger logger, Exception exception, string? messageId); + + [LoggerMessage(LogLevel.Error, "Message {MessageId} failed with fault {FaultId}; fault destination is {DestinationAddress}")] + public static partial void MessageFaulted(this ILogger logger, Exception exception, string? messageId, Guid faultId, string? destinationAddress); + + [LoggerMessage(LogLevel.Information, "Delivered fault {FaultId} for message {MessageId} to {DestinationAddress}")] + public static partial void FaultDelivered(this ILogger logger, string? messageId, Guid faultId, Uri destinationAddress); + + [LoggerMessage(LogLevel.Error, "Message {MessageId} failed and fault {FaultId} could not be sent to {DestinationAddress}")] + public static partial void FaultDeliveryFailed(this ILogger logger, Exception exception, string? messageId, Guid faultId, string? destinationAddress); + + [LoggerMessage(LogLevel.Warning, "Moved unhandled message {MessageId} to {DestinationAddress}")] + public static partial void MessageSkipped(this ILogger logger, string? messageId, Uri destinationAddress); + + [LoggerMessage(LogLevel.Warning, "{Circuit} circuit for endpoint {Endpoint} is open; recovery will be tested after {Delay}")] + public static partial void CircuitOpen(this ILogger logger, Exception? exception, string circuit, string endpoint, TimeSpan delay); + + [LoggerMessage(LogLevel.Information, "{Circuit} circuit for endpoint {Endpoint} closed; processing resumed")] + public static partial void CircuitClosed(this ILogger logger, string circuit, string endpoint); + + [LoggerMessage(LogLevel.Information, "{Circuit} circuit for endpoint {Endpoint} is testing recovery")] + public static partial void CircuitHalfOpen(this ILogger logger, string circuit, string endpoint); +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs index cd147362344..f80b709a4eb 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Mocha.Middlewares; namespace Mocha; @@ -12,7 +13,8 @@ namespace Mocha; internal sealed class ConsumerRetryMiddleware( ImmutableArray exceptionPolicyRules, Consumer consumer, - IConsumerExecutionStrategy executionStrategy) + IConsumerExecutionStrategy executionStrategy, + ILogger logger) { public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate next) { @@ -31,11 +33,20 @@ public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate nex await RetryExecutor.ExecuteAsync( exceptionPolicyRules, - (next, context, retryState, consumer, executionStrategy), + (next, context, retryState, consumer, executionStrategy, logger), static (s) => s.executionStrategy.ExecuteAsync( s.context, ct => ExecuteAttemptAsync(s.next, s.context, s.retryState, s.consumer, ct)), - onRetry: null, + onRetry: static (s, exception, attempt, delay) => + { + using var _ = s.logger.BeginMessageScope(s.context); + s.logger.RetryingMessage(exception, s.context.MessageId, s.consumer.Name, attempt, delay); + }, + onDiscard: static (s, exception) => + { + using var _ = s.logger.BeginMessageScope(s.context); + s.logger.MessageDiscarded(exception, s.context.MessageId); + }, context.CancellationToken); } @@ -99,7 +110,8 @@ public static ConsumerMiddlewareConfiguration Create() var middleware = new ConsumerRetryMiddleware( feature?.Rules.ToImmutableArray() ?? [], context.Consumer, - context.GetConsumerExecutionStrategy()); + context.GetConsumerExecutionStrategy(), + context.Services.GetRequiredService>()); return ctx => middleware.InvokeAsync(ctx, next); }, diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs index 620ee4cfac1..f7c09c32019 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs @@ -12,17 +12,19 @@ public static ValueTask ExecuteAsync( ImmutableArray rules, TState state, Func action, - Action? onRetry, + Action? onRetry, + Action? onDiscard, CancellationToken cancellationToken) { - return ExecuteAsync(rules, state, action, onRetry, TimeProvider.System, cancellationToken); + return ExecuteAsync(rules, state, action, onRetry, onDiscard, TimeProvider.System, cancellationToken); } public static async ValueTask ExecuteAsync( ImmutableArray rules, TState state, Func action, - Action? onRetry, + Action? onRetry, + Action? onDiscard, TimeProvider timeProvider, CancellationToken cancellationToken) { @@ -51,6 +53,7 @@ public static async ValueTask ExecuteAsync( // Discard: swallow the exception (always immediate, no retry chaining exists). if (rule.Terminal == TerminalAction.Discard) { + onDiscard?.Invoke(state, ex); return; } @@ -75,7 +78,7 @@ public static async ValueTask ExecuteAsync( var delay = CalculateDelay(attempts, retryConfig); // Notify caller of retry attempt. - onRetry?.Invoke(state, attempts); + onRetry?.Invoke(state, ex, attempts, delay); if (delay > TimeSpan.Zero) { diff --git a/src/Mocha/src/Mocha/Middlewares/DefaultMessageBus.cs b/src/Mocha/src/Mocha/Middlewares/DefaultMessageBus.cs index cf224417822..4ecdd173a22 100644 --- a/src/Mocha/src/Mocha/Middlewares/DefaultMessageBus.cs +++ b/src/Mocha/src/Mocha/Middlewares/DefaultMessageBus.cs @@ -154,9 +154,8 @@ public async ValueTask RequestAsync( /// Sends a typed request and asynchronously waits for the corresponding response using the specified send options. /// /// - /// A correlation ID is generated for the request, and a deferred response promise is registered - /// so the bus can match the incoming reply. The caller blocks until the response arrives or the - /// is triggered. + /// The two-minute response timeout includes time spent dispatching the request. + /// The cancels both dispatch and waiting for the response. /// /// The expected response event type. /// The request message that defines the expected response contract. @@ -262,42 +261,55 @@ private async ValueTask RequestAndWaitAsync( var replyEndpoint = options.ReplyEndpoint; var faultEndpoint = options.FaultEndpoint; - // var operationName = $"send {endpoint}"; var correlationId = Guid.NewGuid().ToString(); var headers = options.Headers; - var waitHandle = _deferredResponseManager.AddPromise(correlationId); + using var timeoutSource = _deferredResponseManager.CreateRequestTimeout(); + var timeoutToken = timeoutSource.Token; + var waitHandle = _deferredResponseManager.AddPromise(correlationId, timeoutToken); + using var requestCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutToken); - var context = _contextPool.Get(); try { - PropagateCorrelationIds(context); - context.CorrelationId = correlationId; - context.Initialize(services, endpoint, runtime, requestType, cancellationToken); + var context = _contextPool.Get(); + try + { + PropagateCorrelationIds(context); + context.CorrelationId = correlationId; + context.Initialize(services, endpoint, runtime, requestType, requestCts.Token); + + context.Message = message; + context.AddHeaders(headers); + context.Headers.SetMessageKind(MessageKind.Request); + context.ResponseAddress = replyEndpoint ?? endpoint.Transport.ReplyReceiveEndpoint?.Source.Address; + context.FaultAddress = faultEndpoint; + context.ScheduledTime = options.ScheduledTime; + context.DeliverBy = options.ExpirationTime; + + await endpoint.ExecuteAsync(context); + } + catch (OperationCanceledException) + when (timeoutToken.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + await waitHandle.Task; + } + finally + { + _contextPool.Return(context); + } - context.Message = message; - context.AddHeaders(headers); - context.Headers.SetMessageKind(MessageKind.Request); - context.ResponseAddress = replyEndpoint ?? endpoint.Transport.ReplyReceiveEndpoint?.Source.Address; - context.FaultAddress = faultEndpoint; - context.ScheduledTime = options.ScheduledTime; - context.DeliverBy = options.ExpirationTime; + if (await waitHandle.Task.WaitAsync(cancellationToken) is TResponse response) + { + return response; + } - await endpoint.ExecuteAsync(context); + throw ThrowHelper.UnexpectedResponseType(); } finally { - _contextPool.Return(context); + await _deferredResponseManager.ReleasePromiseAsync(correlationId, waitHandle.Task); } - - var result = await waitHandle.Task.WaitAsync(cancellationToken); - if (result is TResponse response) - { - return response; - } - - throw ThrowHelper.UnexpectedResponseType(); } /// diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/CircuitBreaker/ReceiveCircuitBreakerMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/CircuitBreaker/ReceiveCircuitBreakerMiddleware.cs index bc92aac05db..b41b93b99a4 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/CircuitBreaker/ReceiveCircuitBreakerMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/CircuitBreaker/ReceiveCircuitBreakerMiddleware.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Mocha.Middlewares; using Polly; using Polly.CircuitBreaker; @@ -66,6 +67,7 @@ public static ReceiveMiddlewareConfiguration Create() var breakDuration = context.GetConfiguration(f => f.BreakDuration) ?? CircuitBreakerOptions.Defaults.BreakDuration; + var logger = context.Services.GetRequiredService>(); var pipeline = new ResiliencePipelineBuilder() .AddCircuitBreaker( new CircuitBreakerStrategyOptions @@ -74,7 +76,7 @@ public static ReceiveMiddlewareConfiguration Create() MinimumThroughput = minimumThroughput, SamplingDuration = sampleDuration, BreakDuration = breakDuration - }) + }.WithLogging(logger, context, "Consumer")) .Build(); var timeProvider = context.Services.GetRequiredService(); diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveDeadLetterMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveDeadLetterMiddleware.cs index 40ced9bd7b3..23503793598 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveDeadLetterMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveDeadLetterMiddleware.cs @@ -29,6 +29,7 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next } catch (Exception ex) when (!context.CancellationToken.IsCancellationRequested) { + using var _ = logger.BeginMessageScope(context); logger.ExceptionOccurred(ex); } @@ -48,6 +49,8 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next dispatchContext.Envelope = context.Envelope; await deadLetterEndpoint.ExecuteAsync(dispatchContext); + using var _ = logger.BeginMessageScope(context); + logger.MessageSkipped(context.MessageId, deadLetterEndpoint.Destination?.Address ?? deadLetterEndpoint.Address); } finally { diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveFaultMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveFaultMiddleware.cs index 2524c0254df..27f007eb1ef 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveFaultMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveFaultMiddleware.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Mocha.Configuration.Faults; using Mocha.Events; using Mocha.Features; @@ -16,7 +17,8 @@ namespace Mocha.Middlewares; internal sealed class ReceiveFaultMiddleware( TimeProvider provider, DispatchEndpoint? errorEndpoint, - IMessagingPools pools) + IMessagingPools pools, + ILogger logger) { public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next) { @@ -29,15 +31,33 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next catch (Exception ex) when (!context.CancellationToken.IsCancellationRequested) { var fault = FaultInfo.From(Guid.NewGuid(), provider.GetUtcNow(), ex); - - if (context.TryCreateFaultOptions(out var options) - || context.TryCreateResponseOptions(out options)) + var reply = context.TryCreateFaultOptions(out var options) + || context.TryCreateResponseOptions(out options); + var destination = reply + ? options.ReplyAddress + : errorEndpoint?.Destination?.Address ?? errorEndpoint?.Address; + using var _ = logger.BeginMessageScope(context); + logger.MessageFaulted(ex, context.MessageId, fault.Id, destination?.ToString()); + try { - await ReplyToSenderAsync(context, options, fault); + if (reply) + { + await ReplyToSenderAsync(context, options, fault); + } + else + { + await SendToErrorEndpointAsync(context, context.Envelope, fault); + } + + if (destination is not null) + { + logger.FaultDelivered(context.MessageId, fault.Id, destination); + } } - else + catch (Exception deliveryException) { - await SendToErrorEndpointAsync(context, context.Envelope, fault); + logger.FaultDeliveryFailed(deliveryException, context.MessageId, fault.Id, destination?.ToString()); + throw; } feature.MessageConsumed = true; @@ -75,7 +95,6 @@ private async ValueTask SendToErrorEndpointAsync( return; } - // TODO unfortunately this can fail too.. so we need a way around this var dispatchContext = pools.DispatchContext.Get(); try { @@ -104,7 +123,11 @@ public static ReceiveMiddlewareConfiguration Create() var errorEndpoint = context.Endpoint.Features.Get()?.Endpoint; var pools = context.Services.GetRequiredService(); var timeProvider = context.Services.GetRequiredService(); - var middleware = new ReceiveFaultMiddleware(timeProvider, errorEndpoint, pools); + var middleware = new ReceiveFaultMiddleware( + timeProvider, + errorEndpoint, + pools, + context.Services.GetRequiredService>()); return ctx => middleware.InvokeAsync(ctx, next); }, "Fault"); diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs index 177906d6adb..2543b98ae0d 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Mocha.Features; using Mocha.Middlewares; using Mocha.Scheduling; @@ -19,7 +20,8 @@ namespace Mocha; internal sealed class ReceiveRedeliveryMiddleware( ImmutableArray exceptionPolicyRules, TimeProvider timeProvider, - IMessagingPools pools) + IMessagingPools pools, + ILogger logger) { public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next) { @@ -48,12 +50,26 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next switch (decision.Action) { case RedeliveryAction.Discard: + using (logger.BeginMessageScope(context)) + { + logger.MessageDiscarded(ex, context.MessageId); + } + context.Features.GetOrSet().MessageConsumed = true; return; case RedeliveryAction.Redeliver: var scheduledTime = timeProvider.GetUtcNow().Add(decision.Delay); - await DispatchRedeliveryAsync(context, delayedRetryCount, scheduledTime); + using (logger.BeginMessageScope(context)) + { + logger.RedeliveringMessage(ex, context.MessageId, delayedRetryCount + 1, decision.Delay); + var isScheduled = await DispatchRedeliveryAsync(context, delayedRetryCount, scheduledTime); + logger.MessageRedelivered( + context.MessageId, + delayedRetryCount + 1, + isScheduled ? scheduledTime : null); + } + context.Features.GetOrSet().MessageConsumed = true; return; @@ -63,7 +79,7 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next } } - private async ValueTask DispatchRedeliveryAsync( + private async ValueTask DispatchRedeliveryAsync( IReceiveContext context, int delayedRetryCount, DateTimeOffset scheduledTime) @@ -99,7 +115,8 @@ private async ValueTask DispatchRedeliveryAsync( var resolver = context.Services.GetRequiredService(); - if (resolver.CanSchedule(dispatchContext)) + var isScheduled = resolver.CanSchedule(dispatchContext); + if (isScheduled) { dispatchContext.ScheduledTime = scheduledTime; } @@ -110,6 +127,7 @@ private async ValueTask DispatchRedeliveryAsync( } await dispatchEndpoint.ExecuteAsync(dispatchContext); + return isScheduled; } finally { @@ -133,7 +151,11 @@ public static ReceiveMiddlewareConfiguration Create() var timeProvider = context.Services.GetRequiredService(); var pools = context.Services.GetRequiredService(); - var middleware = new ReceiveRedeliveryMiddleware(feature.Rules.ToImmutableArray(), timeProvider, pools); + var middleware = new ReceiveRedeliveryMiddleware( + feature.Rules.ToImmutableArray(), + timeProvider, + pools, + context.Services.GetRequiredService>()); return ctx => middleware.InvokeAsync(ctx, next); }, diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/TransportCircuitBreaker/TransportCircuitBreakerMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/TransportCircuitBreaker/TransportCircuitBreakerMiddleware.cs index ecf9d22d2fb..1277e32d69e 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/TransportCircuitBreaker/TransportCircuitBreakerMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/TransportCircuitBreaker/TransportCircuitBreakerMiddleware.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Polly; using Polly.CircuitBreaker; @@ -21,6 +22,9 @@ public static ReceiveMiddlewareConfiguration Create() var circuitBreaker = context.Transport.Options.CircuitBreaker; var breakDuration = circuitBreaker.BreakDuration; + var logger = context.Services + .GetRequiredService() + .CreateLogger(typeof(TransportCircuitBreakerMiddleware)); var pipeline = new ResiliencePipelineBuilder() .AddCircuitBreaker( new CircuitBreakerStrategyOptions @@ -29,7 +33,7 @@ public static ReceiveMiddlewareConfiguration Create() MinimumThroughput = circuitBreaker.MinimumThroughput, SamplingDuration = circuitBreaker.SamplingDuration, BreakDuration = circuitBreaker.BreakDuration - }) + }.WithLogging(logger, context, "Transport")) .Build(); var timeProvider = context.Services.GetRequiredService(); diff --git a/src/Mocha/src/Mocha/Runtime/MessagingRuntime.cs b/src/Mocha/src/Mocha/Runtime/MessagingRuntime.cs index e15e8711993..78131f5de47 100644 --- a/src/Mocha/src/Mocha/Runtime/MessagingRuntime.cs +++ b/src/Mocha/src/Mocha/Runtime/MessagingRuntime.cs @@ -180,9 +180,9 @@ public async ValueTask StopAsync(CancellationToken cancellationToken) } /// - /// Stops the runtime if it is still running, then disposes the consumers and transports. A failing - /// step does not prevent the remaining steps from running; the first failure is rethrown, except - /// for a stop failure after the runtime was already stopped, which that stop has reported. + /// Stops the runtime with cancellation requested, then disposes the consumers and transports. + /// All cleanup is attempted, and the first failure other than stop cancellation or a previously reported + /// stop failure is rethrown. /// public async ValueTask DisposeAsync() { @@ -197,7 +197,11 @@ public async ValueTask DisposeAsync() try { - await StopAsync(CancellationToken.None); + await StopAsync(new CancellationToken(canceled: true)); + } + catch (OperationCanceledException) + { + // Disposal cancels any remaining graceful shutdown work. } catch (Exception ex) { diff --git a/src/Mocha/src/Mocha/ThrowHelper.cs b/src/Mocha/src/Mocha/ThrowHelper.cs index 713153b72d1..9b80a712d13 100644 --- a/src/Mocha/src/Mocha/ThrowHelper.cs +++ b/src/Mocha/src/Mocha/ThrowHelper.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using Mocha.Events; using Mocha.Scheduling; namespace Mocha; @@ -176,6 +177,9 @@ public static Exception ResponseBodyNotSet() public static Exception PromiseNotFound() => new InvalidOperationException("Promise with correlation ID not found."); + public static Exception ResponseTimedOut(string correlationId, TimeSpan timeout) + => new ResponseTimeoutException(correlationId, timeout); + public static Exception EnvelopeRequired() => new InvalidOperationException("Envelope is required for deserialization"); diff --git a/src/Mocha/src/Mocha/Transport/MessagingTransport.cs b/src/Mocha/src/Mocha/Transport/MessagingTransport.cs index c364e0bca50..2c72bba86df 100644 --- a/src/Mocha/src/Mocha/Transport/MessagingTransport.cs +++ b/src/Mocha/src/Mocha/Transport/MessagingTransport.cs @@ -204,7 +204,39 @@ public virtual TransportDescription Describe() public bool IsStarted { get; private set; } /// - /// Starts the transport by invoking pre-start hooks and activating all receive endpoints. + /// Gets whether the transport is ready to send and receive messages. + /// + public bool IsHealthy => HealthFailureReason is null; + + /// + /// Gets the reason the transport is unhealthy, or null when it is healthy. + /// By default, the transport and all receive endpoints must be started. + /// + public virtual string? HealthFailureReason + { + get + { + if (!IsStarted) + { + return "The transport is not started."; + } + + foreach (var endpoint in _receiveEndpoints) + { + if (!endpoint.IsStarted) + { + return $"Receive endpoint '{endpoint.Name}' is not started."; + } + } + + return null; + } + } + + /// + /// Starts the transport by invoking pre-start hooks and starting the reply endpoints, then all other + /// receive endpoints. When an endpoint fails to start, the transport is stopped again without waiting + /// for running handlers, and the failure is rethrown. /// /// The runtime context providing access to services and configuration. /// A token to cancel the startup sequence. @@ -219,18 +251,37 @@ public async ValueTask StartAsync(IMessagingRuntimeContext context, Cancellation await OnBeforeStartAsync(context, cancellationToken); - foreach (var endpoint in ReceiveEndpoints) + try + { + foreach (var endpoint in ReceiveEndpoints) + { + if (endpoint.Kind == ReceiveEndpointKind.Reply) + { + await endpoint.StartAsync(context, cancellationToken); + } + } + + foreach (var endpoint in ReceiveEndpoints) + { + if (endpoint.Kind != ReceiveEndpointKind.Reply) + { + await endpoint.StartAsync(context, cancellationToken); + } + } + } + catch { - await endpoint.StartAsync(context, cancellationToken); + // A failure while stopping is dropped, so that the caller sees why the start failed. + await StopCoreAsync(context, new CancellationToken(canceled: true)); + throw; } IsStarted = true; } /// - /// Stops the transport by invoking pre-stop hooks and deactivating all receive endpoints. A failing - /// step does not prevent the remaining steps from running; the first failure is rethrown and the - /// transport stays started. + /// Stops application endpoints concurrently, then reply endpoints, between the pre-stop and post-stop hooks. + /// All steps are attempted; the first failure is rethrown and the transport stays started. /// /// The runtime context providing access to services and configuration. /// A token to cancel the shutdown sequence. @@ -242,6 +293,16 @@ public async ValueTask StopAsync(IMessagingRuntimeContext context, CancellationT throw ThrowHelper.TransportNotStarted(); } + var failure = await StopCoreAsync(context, cancellationToken); + failure?.Throw(); + + IsStarted = false; + } + + private async ValueTask StopCoreAsync( + IMessagingRuntimeContext context, + CancellationToken cancellationToken) + { ExceptionDispatchInfo? failure = null; try @@ -253,17 +314,13 @@ public async ValueTask StopAsync(IMessagingRuntimeContext context, CancellationT failure = ExceptionDispatchInfo.Capture(ex); } - foreach (var endpoint in ReceiveEndpoints) - { - try - { - await endpoint.StopAsync(context, cancellationToken); - } - catch (Exception ex) - { - failure ??= ExceptionDispatchInfo.Capture(ex); - } - } + await Task.WhenAll(ReceiveEndpoints + .Where(static endpoint => endpoint.Kind != ReceiveEndpointKind.Reply) + .Select(StopEndpointAsync)); + + await Task.WhenAll(ReceiveEndpoints + .Where(static endpoint => endpoint.Kind == ReceiveEndpointKind.Reply) + .Select(StopEndpointAsync)); try { @@ -274,9 +331,19 @@ public async ValueTask StopAsync(IMessagingRuntimeContext context, CancellationT failure ??= ExceptionDispatchInfo.Capture(ex); } - failure?.Throw(); + return failure; - IsStarted = false; + async Task StopEndpointAsync(ReceiveEndpoint endpoint) + { + try + { + await endpoint.StopAsync(context, cancellationToken); + } + catch (Exception ex) + { + Interlocked.CompareExchange(ref failure, ExceptionDispatchInfo.Capture(ex), null); + } + } } /// @@ -299,9 +366,12 @@ protected virtual ValueTask OnBeforeStartAsync( /// /// Called after receive endpoints are stopped, allowing derived transports to release resources - /// that the endpoints use while stopping. + /// that the endpoints use while stopping. Releasing them must not depend on + /// . /// - /// A token to cancel the post-stop operation. + /// + /// A token that bounds the wait for running handlers. It can already be canceled. + /// protected virtual ValueTask OnAfterStopAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask; /// diff --git a/src/Mocha/src/Mocha/Transport/TransportUnavailableException.cs b/src/Mocha/src/Mocha/Transport/TransportUnavailableException.cs new file mode 100644 index 00000000000..03aa762b2bd --- /dev/null +++ b/src/Mocha/src/Mocha/Transport/TransportUnavailableException.cs @@ -0,0 +1,18 @@ +namespace Mocha; + +/// +/// Thrown when a transport could not hand a message to its broker because the broker was unreachable or +/// did not answer in time. The broker may or may not have received the message. +/// +public sealed class TransportUnavailableException : Exception +{ + /// + /// Initializes a new instance of . + /// + /// The error message. + /// The failure of the last attempt, or null when there was none. + public TransportUnavailableException(string message, Exception? innerException) + : base(message, innerException) + { + } +} diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Broker/OutboxBrokerFixture.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Broker/OutboxBrokerFixture.cs new file mode 100644 index 00000000000..8b98f1de394 --- /dev/null +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Broker/OutboxBrokerFixture.cs @@ -0,0 +1,112 @@ +using CookieCrumble.Resources; +using Npgsql; +using RabbitMQ.Client; + +namespace Mocha.EntityFrameworkCore.Postgres.Tests.Broker; + +/// +/// Runs a Postgres server and a RabbitMQ broker that only the tests of one class use, so a test can +/// block or stop the broker. +/// +public sealed class OutboxBrokerFixture : IAsyncLifetime +{ + private readonly PostgreSqlResource _postgres = new(); + private readonly RabbitMQResource _broker = new(); + + public async ValueTask InitializeAsync() + { + await Task.WhenAll(_postgres.InitializeAsync().AsTask(), _broker.InitializeAsync().AsTask()); + } + + public async ValueTask DisposeAsync() + { + await _broker.DisposeAsync(); + await _postgres.DisposeAsync(); + } + + public async Task CreateDatabaseAsync() + { + var database = $"test_{Guid.NewGuid():N}"; + await _postgres.CreateDatabaseAsync(database); + return _postgres.GetConnectionString(database); + } + + /// + /// Creates a virtual host and returns the factory of the connections to it. + /// + public async Task CreateVirtualHostAsync() + { + var virtualHost = $"test_{Guid.NewGuid():N}"; + await _broker.InvokeCommandAsync(["rabbitmqctl", "add_vhost", virtualHost]); + await _broker.InvokeCommandAsync( + ["rabbitmqctl", "set_permissions", "-p", virtualHost, "guest", ".*", ".*", ".*"]); + + return new ConnectionFactory { Uri = new Uri(_broker.ConnectionString), VirtualHost = virtualHost }; + } + + /// + /// Raises the memory alarm, so the broker stops reading from connections that publish until the returned + /// handle is disposed. + /// + public async Task BlockPublishersAsync() + { + await _broker.InvokeCommandAsync(["rabbitmqctl", "set_vm_memory_high_watermark", "0"]); + + return new Restore(() => _broker.InvokeCommandAsync(["rabbitmqctl", "set_vm_memory_high_watermark", "0.4"])); + } + + /// + /// Stops the broker application, which closes every connection and refuses new ones until the returned + /// handle is disposed. + /// + public async Task StopBrokerAsync() + { + await _broker.InvokeCommandAsync(["rabbitmqctl", "stop_app"]); + + return new Restore(async () => + { + await _broker.InvokeCommandAsync(["rabbitmqctl", "start_app"]); + await _broker.InvokeCommandAsync(["rabbitmqctl", "await_startup"]); + }); + } + + /// + /// Waits until no other session of the database has an open transaction and returns the number of + /// sessions that had one when the wait ended. + /// + public static async Task WaitForNoOpenTransactionAsync( + string connectionString, + TimeSpan timeout, + CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + timeout; + + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken); + + await using var command = connection.CreateCommand(); + command.CommandText = + """ + SELECT COUNT(*) FROM pg_stat_activity + WHERE datname = current_database() + AND backend_type = 'client backend' + AND pid <> pg_backend_pid() + AND xact_start IS NOT NULL; + """; + + var sessions = (long)(await command.ExecuteScalarAsync(cancellationToken))!; + + while (sessions != 0 && DateTime.UtcNow < deadline) + { + await Task.Delay(20, cancellationToken); + sessions = (long)(await command.ExecuteScalarAsync(cancellationToken))!; + } + + return sessions; + } + + private sealed class Restore(Func restore) : IAsyncDisposable + { + public async ValueTask DisposeAsync() => await restore(); + } +} diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Broker/PostgresOutboxBrokerTests.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Broker/PostgresOutboxBrokerTests.cs new file mode 100644 index 00000000000..57c8548bf78 --- /dev/null +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Broker/PostgresOutboxBrokerTests.cs @@ -0,0 +1,217 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Mocha.EntityFrameworkCore.Postgres.Tests.Helpers; +using Mocha.Outbox; +using Mocha.Transport.RabbitMQ; +using Npgsql; + +namespace Mocha.EntityFrameworkCore.Postgres.Tests.Broker; + +public sealed class PostgresOutboxBrokerTests(OutboxBrokerFixture fixture) : IClassFixture +{ + // A request to the broker that is in flight can take the client 20 seconds to give up. + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(90); + private static CancellationToken TestToken => TestContext.Current.CancellationToken; + + [Fact] + public async Task Outbox_Should_KeepMessageAndNotCountAttempts_When_BrokerDoesNotConfirmPublish() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var deliveries = new DeliveryRecorder(); + var logs = RecordingLoggerProvider.For(LogLevel.Warning); + await using var host = await StartHostAsync( + connectionString, deliveries, logs, static o => o.DispatchTimeout = TimeSpan.FromSeconds(2)); + + // The broker blocks a connection at once only when the connection has published before. + await PublishAsync(host, "ORD-0"); + await deliveries.WaitForAsync(["ORD-0"], s_timeout, TestToken); + long openTransactionsWhileBlocked; + long rowsWhileBlocked; + int? timesSentWhileBlocked; + + // act + await using (await fixture.BlockPublishersAsync()) + { + await PublishAsync(host, "ORD-1"); + await logs.WaitForEntriesAsync(2, s_timeout, TestToken); + openTransactionsWhileBlocked = await OutboxBrokerFixture.WaitForNoOpenTransactionAsync( + connectionString, s_timeout, TestToken); + (rowsWhileBlocked, timesSentWhileBlocked) = await ReadOutboxAsync(connectionString); + } + + var delivered = await deliveries.WaitForAsync(["ORD-0", "ORD-1"], s_timeout, TestToken); + var remaining = await MessageRows.WaitForOutboxCountAsync(connectionString, 0, s_timeout, TestToken); + + // assert + Assert.Equal(0, openTransactionsWhileBlocked); + Assert.Equal(1, rowsWhileBlocked); + Assert.Equal(0, timesSentWhileBlocked); + Assert.Equal(["ORD-0", "ORD-1"], delivered); + Assert.Equal(0, remaining); + } + + [Fact] + public async Task Outbox_Should_DeliverEveryMessage_When_BrokerOutageOutlastsAllAttempts() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var deliveries = new DeliveryRecorder(); + var logs = RecordingLoggerProvider.For(LogLevel.Warning); + await using var host = await StartHostAsync(connectionString, deliveries, logs, static o => + { + o.DispatchTimeout = TimeSpan.FromSeconds(2); + o.MaxAttempts = 2; + o.MaxRetryDelay = TimeSpan.FromMilliseconds(200); + }); + await PublishAsync(host, "ORD-0"); + await deliveries.WaitForAsync(["ORD-0"], s_timeout, TestToken); + string[] expected = ["ORD-0", "ORD-1", "ORD-2"]; + long rowsDuringOutage; + + // act + await using (await fixture.StopBrokerAsync()) + { + foreach (var orderId in expected[1..]) + { + await PublishAsync(host, orderId); + } + + // More dispatches fail than the two messages have attempts together. + await logs.WaitForEntriesAsync(5, s_timeout, TestToken); + rowsDuringOutage = await MessageRows.CountOutboxAsync(connectionString, TestToken); + } + + var delivered = await deliveries.WaitForAsync(expected, s_timeout, TestToken); + var remaining = await MessageRows.WaitForOutboxCountAsync(connectionString, 0, s_timeout, TestToken); + + // assert + Assert.Equal(2, rowsDuringOutage); + Assert.Equal(expected, delivered); + Assert.Equal(0, remaining); + Assert.Equal([LogLevel.Warning], logs.Entries.Select(entry => entry.Level).Distinct()); + } + + private async Task StartHostAsync( + string connectionString, + DeliveryRecorder deliveries, + RecordingLoggerProvider logs, + Action configure) + { + var services = new ServiceCollection(); + services.AddLogging(b => b.SetMinimumLevel(LogLevel.Trace).AddProvider(logs)); + services.AddSingleton(await fixture.CreateVirtualHostAsync()); + services.AddSingleton(deliveries); + services.AddDbContext(o => o.UseTestNpgsql(connectionString)); + var builder = services.AddMessageBus(); + builder.AddEntityFramework(ef => ef.UsePostgresOutbox(o => + { + o.RequireTransaction = false; + configure(o); + })); + builder.AddEventHandler(); + builder.AddRabbitMQ(t => t.Endpoint("orders").Handler()); + + var provider = services.BuildServiceProvider(); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(TestToken); + await using (var scope = provider.CreateAsyncScope()) + { + await scope.ServiceProvider.GetRequiredService().Database.EnsureCreatedAsync(TestToken); + } + + var hostedServices = provider.GetServices().ToList(); + foreach (var service in hostedServices) + { + await service.StartAsync(TestToken); + } + + return new BrokerHost(provider, runtime, hostedServices); + } + + private static async Task PublishAsync(BrokerHost host, string orderId) + { + await using var scope = host.Services.CreateAsyncScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + await bus.PublishAsync(new OrderPlaced(orderId), TestToken); + } + + private static async Task<(long Rows, int? TimesSent)> ReadOutboxAsync(string connectionString) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(TestToken); + await using var command = connection.CreateCommand(); + command.CommandText = """SELECT COUNT(*), MAX("times_sent") FROM "outbox_messages";"""; + await using var reader = await command.ExecuteReaderAsync(TestToken); + await reader.ReadAsync(TestToken); + return (reader.GetInt64(0), reader.IsDBNull(1) ? null : reader.GetInt32(1)); + } + + public sealed record OrderPlaced(string OrderId); + + public sealed class OrderPlacedHandler(DeliveryRecorder deliveries) : IEventHandler + { + public ValueTask HandleAsync(OrderPlaced message, CancellationToken cancellationToken) + { + deliveries.Record(message.OrderId); + return default; + } + } + + public sealed class DeliveryRecorder + { + private readonly ConcurrentQueue _orderIds = new(); + + public void Record(string orderId) => _orderIds.Enqueue(orderId); + + /// + /// Waits until every expected order was delivered at least once and returns the distinct orders + /// delivered so far, in order. + /// + public async Task WaitForAsync( + string[] expected, + TimeSpan timeout, + CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + timeout; + + while (expected.Except(_orderIds).Any() && DateTime.UtcNow < deadline) + { + await Task.Delay(20, cancellationToken); + } + + return [.. _orderIds.Distinct().Order()]; + } + } + + private sealed class BrokerHost( + ServiceProvider provider, + MessagingRuntime runtime, + List hostedServices) : IAsyncDisposable + { + public IServiceProvider Services => provider; + + public async ValueTask DisposeAsync() + { + // A host stops with a shutdown timeout, after which work that still runs is cancelled. + using var shutdownTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + for (var i = hostedServices.Count - 1; i >= 0; i--) + { + await hostedServices[i].StopAsync(shutdownTimeout.Token); + } + + foreach (var transport in runtime.Transports) + { + if (transport.IsStarted) + { + await transport.StopAsync(runtime, shutdownTimeout.Token); + } + } + + await provider.DisposeAsync(); + } + } +} diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/MessageRows.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/MessageRows.cs index 3af82534389..451c6a87f5d 100644 --- a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/MessageRows.cs +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/MessageRows.cs @@ -29,6 +29,163 @@ public static async Task InsertOutboxAsync( return await InsertAsync(connectionString, sql, envelope, cancellationToken); } + /// + /// Inserts an outbox row whose envelope column holds as it is. + /// + public static async Task InsertOutboxAsync( + string connectionString, + string envelopeJson, + CancellationToken cancellationToken) + { + const string sql = + $""" + INSERT INTO "{OutboxTable}" ("id", "envelope", "times_sent", "created_at") + VALUES (@id, @envelope, 0, NOW()); + """; + + return await InsertAsync(connectionString, sql, envelopeJson, cancellationToken); + } + + /// + /// Returns the attempt count of the outbox row, or null when the row is gone. + /// + public static async Task GetOutboxTimesSentAsync( + string connectionString, + Guid id, + CancellationToken cancellationToken) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken); + + await using var command = connection.CreateCommand(); + command.CommandText = $"""SELECT "times_sent" FROM "{OutboxTable}" WHERE "id" = @id;"""; + command.Parameters.AddWithValue("@id", id); + + return (int?)await command.ExecuteScalarAsync(cancellationToken); + } + + /// + /// Waits until the outbox row has the expected attempt count and returns the last count that was read, + /// or null when the row is gone. + /// + public static async Task WaitForOutboxTimesSentAsync( + string connectionString, + Guid id, + int expected, + TimeSpan timeout, + CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + timeout; + var timesSent = await GetOutboxTimesSentAsync(connectionString, id, cancellationToken); + + while (timesSent != expected && timesSent is not null && DateTime.UtcNow < deadline) + { + await Task.Delay(20, cancellationToken); + timesSent = await GetOutboxTimesSentAsync(connectionString, id, cancellationToken); + } + + return timesSent; + } + + /// + /// Waits until the outbox row is gone and returns whether it is. + /// + public static async Task WaitForOutboxRemovalAsync( + string connectionString, + Guid id, + TimeSpan timeout, + CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + timeout; + var timesSent = await GetOutboxTimesSentAsync(connectionString, id, cancellationToken); + + while (timesSent is not null && DateTime.UtcNow < deadline) + { + await Task.Delay(20, cancellationToken); + timesSent = await GetOutboxTimesSentAsync(connectionString, id, cancellationToken); + } + + return timesSent is null; + } + + public static async Task CountOutboxAsync(string connectionString, CancellationToken cancellationToken) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken); + + await using var command = connection.CreateCommand(); + command.CommandText = $"""SELECT COUNT(*) FROM "{OutboxTable}";"""; + + return (long)(await command.ExecuteScalarAsync(cancellationToken))!; + } + + /// + /// Waits until the outbox holds the expected number of rows and returns the last count that was read. + /// + public static async Task WaitForOutboxCountAsync( + string connectionString, + long expected, + TimeSpan timeout, + CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + timeout; + var count = await CountOutboxAsync(connectionString, cancellationToken); + + while (count != expected && DateTime.UtcNow < deadline) + { + await Task.Delay(20, cancellationToken); + count = await CountOutboxAsync(connectionString, cancellationToken); + } + + return count; + } + + /// + /// Makes every outbox row eligible for dispatch, as if it had been due for . + /// + public static async Task MakeOutboxDueAsync( + string connectionString, + TimeSpan dueFor, + CancellationToken cancellationToken) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken); + + await using var command = connection.CreateCommand(); + command.CommandText = $"""UPDATE "{OutboxTable}" SET "created_at" = NOW() - @dueFor;"""; + command.Parameters.AddWithValue("@dueFor", dueFor); + + await command.ExecuteNonQueryAsync(cancellationToken); + } + + public static async Task SetOutboxTimesSentAsync( + string connectionString, + Guid id, + int timesSent, + CancellationToken cancellationToken) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken); + + await using var command = connection.CreateCommand(); + command.CommandText = $"""UPDATE "{OutboxTable}" SET "times_sent" = @timesSent WHERE "id" = @id;"""; + command.Parameters.AddWithValue("@timesSent", timesSent); + command.Parameters.AddWithValue("@id", id); + + await command.ExecuteNonQueryAsync(cancellationToken); + } + + public static async Task CountScheduledAsync(string connectionString, CancellationToken cancellationToken) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken); + + await using var command = connection.CreateCommand(); + command.CommandText = $"""SELECT COUNT(*) FROM "{ScheduledTable}";"""; + + return (long)(await command.ExecuteScalarAsync(cancellationToken))!; + } + public static async Task InsertScheduledAsync( string connectionString, MessageEnvelope envelope, @@ -65,11 +222,18 @@ public static async Task InsertScheduledAsync( CancellationToken cancellationToken) => WaitForRetryAsync(connectionString, ScheduledTable, id, timeout, cancellationToken); - private static async Task InsertAsync( + private static Task InsertAsync( string connectionString, string sql, MessageEnvelope envelope, CancellationToken cancellationToken) + => InsertAsync(connectionString, sql, Serialize(envelope), cancellationToken); + + private static async Task InsertAsync( + string connectionString, + string sql, + string envelopeJson, + CancellationToken cancellationToken) { var id = Guid.NewGuid(); @@ -79,7 +243,7 @@ private static async Task InsertAsync( await using var command = connection.CreateCommand(); command.CommandText = sql; command.Parameters.AddWithValue("@id", id); - command.Parameters.AddWithValue("@envelope", NpgsqlDbType.Json, Serialize(envelope)); + command.Parameters.AddWithValue("@envelope", NpgsqlDbType.Json, envelopeJson); await command.ExecuteNonQueryAsync(cancellationToken); diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/RecordingLoggerProvider.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/RecordingLoggerProvider.cs new file mode 100644 index 00000000000..54812a9174e --- /dev/null +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/RecordingLoggerProvider.cs @@ -0,0 +1,62 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Mocha.EntityFrameworkCore.Postgres.Tests.Helpers; + +internal sealed record LogEntry(LogLevel Level, string Message); + +/// +/// Records the log entries one category writes at or above a minimum level. +/// +internal sealed class RecordingLoggerProvider(string category, LogLevel minimumLevel) : ILoggerProvider +{ + private readonly ConcurrentQueue _entries = new(); + + public LogEntry[] Entries => [.. _entries]; + + public static RecordingLoggerProvider For(LogLevel minimumLevel = LogLevel.Information) + => new(typeof(TCategory).FullName!, minimumLevel); + + public ILogger CreateLogger(string categoryName) + => categoryName == category ? new RecordingLogger(_entries, minimumLevel) : NullLogger.Instance; + + /// + /// Waits until at least entries were recorded and returns the entries recorded so far. + /// + public async Task WaitForEntriesAsync(int count, TimeSpan timeout, CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + timeout; + + while (_entries.Count < count && DateTime.UtcNow < deadline) + { + await Task.Delay(20, cancellationToken); + } + + return Entries; + } + + public void Dispose() + { + } + + private sealed class RecordingLogger(ConcurrentQueue entries, LogLevel minimumLevel) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel >= minimumLevel; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (IsEnabled(logLevel)) + { + entries.Enqueue(new LogEntry(logLevel, formatter(state, exception))); + } + } + } +} diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/ScriptedDispatch.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/ScriptedDispatch.cs new file mode 100644 index 00000000000..253d620acaa --- /dev/null +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Helpers/ScriptedDispatch.cs @@ -0,0 +1,70 @@ +using System.Collections.Concurrent; +using Mocha.Middlewares; + +namespace Mocha.EntityFrameworkCore.Postgres.Tests.Helpers; + +/// +/// A dispatch middleware in front of the transport that lets a test decide what each dispatch attempt does. +/// +internal sealed class ScriptedDispatch(Func onDispatch) +{ + private readonly ConcurrentDictionary _attempts = new(); + + /// + /// Gets the ids of the messages the transport accepted, one entry per accepted dispatch. + /// + public ConcurrentQueue Accepted { get; } = new(); + + public DispatchMiddlewareConfiguration Middleware + => new((_, next) => context => DispatchAsync(context, next), "ScriptedDispatch"); + + public int AttemptsOf(string messageId) => _attempts.GetValueOrDefault(messageId); + + private ValueTask DispatchAsync(IDispatchContext context, DispatchDelegate next) + { + var messageId = context.Envelope!.MessageId!; + var number = _attempts.AddOrUpdate(messageId, 1, static (_, attempts) => attempts + 1); + + return onDispatch(new DispatchAttempt(this, context, next, messageId, number)); + } +} + +/// +/// One dispatch of a message as the sees it. +/// +internal sealed class DispatchAttempt( + ScriptedDispatch dispatch, + IDispatchContext context, + DispatchDelegate next, + string messageId, + int number) +{ + public string MessageId => messageId; + + /// + /// Gets the number of this dispatch of the message, starting at one. + /// + public int Number => number; + + public CancellationToken CancellationToken => context.CancellationToken; + + /// + /// Hands the message to the transport. + /// + public async ValueTask SendAsync() + { + await next(context); + dispatch.Accepted.Enqueue(messageId); + } + + /// + /// Waits until the dispatch is cancelled or completes, and returns without an + /// error in both cases, as a transport does that has already handed the message to its broker. + /// + public async Task WaitForCancellationAsync(Task release) + { + var cancelled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var registration = CancellationToken.Register(() => cancelled.TrySetResult()); + await Task.WhenAny(cancelled.Task, release); + } +} diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Mocha.EntityFrameworkCore.Postgres.Tests.csproj b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Mocha.EntityFrameworkCore.Postgres.Tests.csproj index 4d64f5b1923..9b085db1a9c 100644 --- a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Mocha.EntityFrameworkCore.Postgres.Tests.csproj +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/Mocha.EntityFrameworkCore.Postgres.Tests.csproj @@ -14,9 +14,11 @@ + + diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/OutboxServiceRegistrationTests.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/OutboxServiceRegistrationTests.cs index 5ebe57bf66d..54ebf732595 100644 --- a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/OutboxServiceRegistrationTests.cs +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/OutboxServiceRegistrationTests.cs @@ -11,6 +11,107 @@ public sealed class OutboxServiceRegistrationTests { private const string ConnectionString = "Host=localhost;Database=test"; + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void UsePostgresOutbox_Should_RejectInvalidConcurrency_When_OptionsResolved(int concurrency) + { + // arrange + var services = new ServiceCollection(); + services.AddMessageBus().AddEntityFramework( + ef => ef.UsePostgresOutbox(o => o.MaxConcurrentMessages = concurrency)); + using var provider = services.BuildServiceProvider(); + + // act + var exception = Assert.Throws(() => + provider.GetRequiredService>() + .Get(typeof(TestDbContext).FullName)); + + // assert + Assert.Equal(["MaxConcurrentMessages must be greater than zero."], exception.Failures); + } + + [Theory] + [InlineData(0)] + [InlineData(-2)] + [InlineData(25 * 24 * 60 * 60 * 1000L)] + public void UsePostgresOutbox_Should_RejectInvalidDispatchTimeout_When_OptionsResolved(long milliseconds) + { + // arrange + var services = new ServiceCollection(); + services.AddMessageBus().AddEntityFramework( + ef => ef.UsePostgresOutbox(o => o.DispatchTimeout = TimeSpan.FromMilliseconds(milliseconds))); + using var provider = services.BuildServiceProvider(); + + // act + var exception = Assert.Throws(() => + provider.GetRequiredService>() + .Get(typeof(TestDbContext).FullName)); + + // assert + Assert.Equal( + ["DispatchTimeout must be greater than zero and at most 24 days, or Timeout.InfiniteTimeSpan."], + exception.Failures); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void UsePostgresOutbox_Should_RejectInvalidMaxAttempts_When_OptionsResolved(int maxAttempts) + { + // arrange + var services = new ServiceCollection(); + services.AddMessageBus().AddEntityFramework( + ef => ef.UsePostgresOutbox(o => o.MaxAttempts = maxAttempts)); + using var provider = services.BuildServiceProvider(); + + // act + var exception = Assert.Throws(() => + provider.GetRequiredService>() + .Get(typeof(TestDbContext).FullName)); + + // assert + Assert.Equal(["MaxAttempts must be greater than zero."], exception.Failures); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void UsePostgresOutbox_Should_RejectInvalidMaxRetryDelay_When_OptionsResolved(int milliseconds) + { + // arrange + var services = new ServiceCollection(); + services.AddMessageBus().AddEntityFramework( + ef => ef.UsePostgresOutbox(o => o.MaxRetryDelay = TimeSpan.FromMilliseconds(milliseconds))); + using var provider = services.BuildServiceProvider(); + + // act + var exception = Assert.Throws(() => + provider.GetRequiredService>() + .Get(typeof(TestDbContext).FullName)); + + // assert + Assert.Equal(["MaxRetryDelay must be greater than zero."], exception.Failures); + } + + [Fact] + public void UsePostgresOutbox_Should_UseDefaults_When_OptionsAreNotConfigured() + { + // arrange + var services = new ServiceCollection(); + services.AddMessageBus().AddEntityFramework(ef => ef.UsePostgresOutbox()); + using var provider = services.BuildServiceProvider(); + + // act + var options = provider.GetRequiredService>() + .Get(typeof(TestDbContext).FullName); + + // assert + Assert.Equal(TimeSpan.FromSeconds(30), options.DispatchTimeout); + Assert.Equal(10, options.MaxAttempts); + Assert.Equal(TimeSpan.FromMinutes(10), options.MaxRetryDelay); + } + [Fact] public void UsePostgresOutbox_Should_DeferConfiguration_When_OptionsHaveNotBeenResolved() { diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresMessageOutboxTests.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresMessageOutboxTests.cs index 07358e8fc9a..6f23a223806 100644 --- a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresMessageOutboxTests.cs +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresMessageOutboxTests.cs @@ -143,7 +143,7 @@ public void Dispose_Should_NotThrow_When_Called() // Arrange var options = new DbContextOptionsBuilder().UseTestNpgsql("Host=localhost;Database=test").Options; using var context = new TestDbContext(options); - var queries = PostgresMessageOutboxQueries.From(new OutboxTableInfo()); + var queries = PostgresMessageOutboxQueries.From(new OutboxTableInfo(), new PostgresOutboxOptions()); var signal = new StubOutboxSignal(); var outbox = new PostgresMessageOutbox(context, signal, queries.InsertEnvelope, requireTransaction: false); @@ -161,7 +161,7 @@ public void Dispose_Should_NotThrow_When_Called() var options = new DbContextOptionsBuilder().UseTestNpgsql(connectionString).Options; var context = new TestDbContext(options); await context.Database.EnsureCreatedAsync(); - var queries = PostgresMessageOutboxQueries.From(new OutboxTableInfo()); + var queries = PostgresMessageOutboxQueries.From(new OutboxTableInfo(), new PostgresOutboxOptions()); var signal = new StubOutboxSignal(); var outbox = new PostgresMessageOutbox(context, signal, queries.InsertEnvelope, requireTransaction: false); return (context, outbox, signal); diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxConcurrencyTests.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxConcurrencyTests.cs new file mode 100644 index 00000000000..d8967a2f7d3 --- /dev/null +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxConcurrencyTests.cs @@ -0,0 +1,442 @@ +using System.Collections.Concurrent; +using System.Data; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Mocha.EntityFrameworkCore.Postgres.Tests.Helpers; +using Mocha.Middlewares; +using Mocha.Outbox; +using Mocha.Transport.InMemory; +using Npgsql; + +namespace Mocha.EntityFrameworkCore.Postgres.Tests; + +public sealed class PostgresOutboxConcurrencyTests(PostgresFixture fixture) : IClassFixture +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); + private static CancellationToken TestToken => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(null)] + [InlineData(1)] + [InlineData(4)] + public async Task ProcessAsync_Should_BoundConcurrentDispatches_When_MessagesArePending(int? concurrency) + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var probe = new DispatchProbe(concurrency ?? 1); + await using var provider = await CreateProviderAsync(connectionString, concurrency, probe); + await InsertMessagesAsync(provider, connectionString, 0, 12); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestToken); + var processor = provider.GetRequiredService(); + + // act + var processing = processor.ProcessAsync(cancellation.Token); + long pending; + try + { + await probe.Started.Task.WaitAsync(s_timeout, TestToken); + pending = await CountMessagesAsync(connectionString); + probe.Release.TrySetResult(); + await WaitForEmptyOutboxAsync(connectionString); + } + finally + { + await cancellation.CancelAsync(); + await processing.WaitAsync(s_timeout, TestToken); + } + + // assert + Assert.Equal(12, pending); + Assert.Equal(concurrency ?? 1, probe.PeakConcurrency); + Assert.Equal(Enumerable.Range(0, 12), probe.Completed.Order()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProcessAsync_Should_ResumeIdleWorkers_When_NewMessagesArriveDuringBlockedDispatch(bool signal) + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var idleWorkers = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var closedConnections = 0; + var probe = new DispatchProbe(4) + { + HoldFirst = true, + ConnectionCreated = connection => connection.StateChange += (_, args) => + { + if (args.CurrentState == ConnectionState.Closed + && Interlocked.Increment(ref closedConnections) == 3) + { + idleWorkers.TrySetResult(); + } + } + }; + await using var provider = await CreateProviderAsync(connectionString, 4, probe); + await InsertMessagesAsync(provider, connectionString, 0, 1); + if (signal) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(TestToken); + await using var command = connection.CreateCommand(); + // The blocked row must not schedule a poll within the test's timeout. + command.CommandText = "UPDATE outbox_messages SET created_at = NOW() - INTERVAL '1 hour'"; + await command.ExecuteNonQueryAsync(TestToken); + } + + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestToken); + var processor = provider.GetRequiredService(); + var processing = processor.ProcessAsync(cancellation.Token); + int[] completedWhileBlocked; + + // act + try + { + await probe.FirstStarted.Task.WaitAsync(s_timeout, TestToken); + await idleWorkers.Task.WaitAsync(s_timeout, TestToken); + await InsertMessagesAsync(provider, connectionString, 1, 300); + if (signal) + { + provider.GetRequiredService().Set(); + } + + await probe.Started.Task.WaitAsync(s_timeout, TestToken); + probe.Release.TrySetResult(); + await WaitForOutboxCountAsync(connectionString, 1); + + completedWhileBlocked = probe.Completed.Order().ToArray(); + probe.FirstRelease.TrySetResult(); + await WaitForEmptyOutboxAsync(connectionString); + } + finally + { + await cancellation.CancelAsync(); + await processing.WaitAsync(s_timeout, TestToken); + } + + // assert + Assert.Equal(Enumerable.Range(1, 300), completedWhileBlocked); + Assert.Equal(4, probe.PeakConcurrency); + Assert.Equal(Enumerable.Range(0, 301), probe.Completed.Order()); + } + + [Fact] + public async Task ProcessAsync_Should_DrainBacklog_When_OneDispatchRemainsBlocked() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var probe = new DispatchProbe(4) { HoldFirst = true }; + await using var provider = await CreateProviderAsync(connectionString, 4, probe); + await InsertMessagesAsync(provider, connectionString, 0, 501); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestToken); + var processor = provider.GetRequiredService(); + + // act + var processing = processor.ProcessAsync(cancellation.Token); + int[] completedWhileBlocked; + try + { + await probe.FirstStarted.Task.WaitAsync(s_timeout, TestToken); + await probe.Started.Task.WaitAsync(s_timeout, TestToken); + probe.Release.TrySetResult(); + await WaitForOutboxCountAsync(connectionString, 1); + + completedWhileBlocked = probe.Completed.Order().ToArray(); + } + finally + { + await cancellation.CancelAsync(); + await processing.WaitAsync(s_timeout, TestToken); + } + + // assert + Assert.Equal(Enumerable.Range(1, 500), completedWhileBlocked); + Assert.Equal(4, probe.PeakConcurrency); + } + + [Fact] + public async Task ProcessAsync_Should_ReleaseClaims_When_ConcurrentDispatchesAreCancelled() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var probe = new DispatchProbe(4); + await using var provider = await CreateProviderAsync(connectionString, 4, probe); + await InsertMessagesAsync(provider, connectionString, 0, 12); + var processor = provider.GetRequiredService(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestToken); + var processing = processor.ProcessAsync(cancellation.Token); + await probe.Started.Task.WaitAsync(s_timeout, TestToken); + + // act + await cancellation.CancelAsync(); + await processing.WaitAsync(s_timeout, TestToken); + var pendingAfterCancellation = await CountMessagesAsync(connectionString); + probe.Release.TrySetResult(); + using var restartCancellation = CancellationTokenSource.CreateLinkedTokenSource(TestToken); + var restarted = processor.ProcessAsync(restartCancellation.Token); + try + { + await WaitForEmptyOutboxAsync(connectionString); + } + finally + { + await restartCancellation.CancelAsync(); + await restarted.WaitAsync(s_timeout, TestToken); + } + + // assert + Assert.Equal(12, pendingAfterCancellation); + Assert.Equal(0, probe.ActiveDispatches); + Assert.Equal(Enumerable.Range(0, 12), probe.Completed.Order()); + } + + [Fact] + public async Task ProcessAsync_Should_KeepDispatchingOtherMessages_When_OneDispatchFails() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var probe = new DispatchProbe(4) { FailFirst = true }; + await using var provider = await CreateProviderAsync(connectionString, 4, probe); + await InsertMessagesAsync(provider, connectionString, 0, 12); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestToken); + var processor = provider.GetRequiredService(); + + // act + var processing = processor.ProcessAsync(cancellation.Token); + int[] completedWhileFailing; + bool stoppedByFailure; + try + { + await probe.Started.Task.WaitAsync(s_timeout, TestToken); + probe.Release.TrySetResult(); + await WaitForOutboxCountAsync(connectionString, 1); + completedWhileFailing = probe.Completed.Order().ToArray(); + stoppedByFailure = processing.IsCompleted; + } + finally + { + await cancellation.CancelAsync(); + await processing.WaitAsync(s_timeout, TestToken); + } + + // assert + Assert.Equal(Enumerable.Range(1, 11), completedWhileFailing); + Assert.False(stoppedByFailure); + Assert.Equal(0, probe.ActiveDispatches); + } + + [Fact] + public async Task Outbox_Should_DeliverMessageOnce_When_AnotherDispatchFailsBeforeItsDispatchReturns() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var acceptedA = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var returnA = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatch = new ScriptedDispatch(async attempt => + { + if (attempt is { MessageId: "B", Number: 1 }) + { + // B fails only after the transport accepted A. + await acceptedA.Task.WaitAsync(attempt.CancellationToken); + throw new InvalidOperationException("The dispatch of B failed."); + } + + await attempt.SendAsync(); + + if (attempt is { MessageId: "A", Number: 1 }) + { + // The transport has A, and the dispatch of A returns only after B has failed. + acceptedA.TrySetResult(); + await attempt.WaitForCancellationAsync(returnA.Task); + } + }); + await using var provider = await CreateProviderAsync(connectionString, 2, dispatch.Middleware, null); + var rowOfA = await InsertMessageAsync(provider, connectionString, "A", 0); + var rowOfB = await InsertMessageAsync(provider, connectionString, "B", 1); + await using var host = await MessagingTestHost.StartAsync(provider, TestToken); + + // act + var timesSentOfB = await MessageRows.WaitForOutboxTimesSentAsync( + connectionString, rowOfB, 1, s_timeout, TestToken); + returnA.TrySetResult(); + var removedA = await MessageRows.WaitForOutboxRemovalAsync(connectionString, rowOfA, s_timeout, TestToken); + await MessageRows.MakeOutboxDueAsync(connectionString, TimeSpan.FromHours(1), TestToken); + provider.GetRequiredService().Set(); + await WaitForEmptyOutboxAsync(connectionString); + + // assert + Assert.Equal(1, timesSentOfB); + Assert.True(removedA); + Assert.Equal(["A", "B"], dispatch.Accepted.Order()); + } + + private static Task CreateProviderAsync( + string connectionString, + int? concurrency, + DispatchProbe probe) + => CreateProviderAsync( + connectionString, + concurrency, + new DispatchMiddlewareConfiguration( + (_, next) => context => probe.DispatchAsync(context, next), + "DispatchProbe"), + probe.ConnectionCreated); + + private static async Task CreateProviderAsync( + string connectionString, + int? concurrency, + DispatchMiddlewareConfiguration dispatchMiddleware, + Action? connectionCreated) + { + var services = new ServiceCollection(); + services.AddDbContext(o => o.UseTestNpgsql(connectionString)); + var builder = services.AddMessageBus(); + builder.AddEntityFramework(ef => ef.UsePostgresOutbox(o => + { + if (concurrency is { } value) + { + o.MaxConcurrentMessages = value; + } + })); + builder.AddInMemory(t => + { + t.DeclareQueue("pending"); + t.DispatchEndpoint("pending").ToQueue("pending").Publish(); + }); + builder.ConfigureMessageBus(b => b.UseDispatch(dispatchMiddleware, after: "Serialization")); + + if (connectionCreated is { } onConnectionCreated) + { + services.PostConfigure(typeof(TestDbContext).FullName, options => + { + var createConnection = options.CreateConnection; + options.CreateConnection = sp => + { + var connection = createConnection(sp); + onConnectionCreated(connection); + return connection; + }; + }); + } + + var provider = services.BuildServiceProvider(); + await ((MessagingRuntime)provider.GetRequiredService()).StartAsync(TestToken); + await using var scope = provider.CreateAsyncScope(); + await scope.ServiceProvider.GetRequiredService().Database.EnsureCreatedAsync(TestToken); + return provider; + } + + private static async Task InsertMessagesAsync( + IServiceProvider provider, + string connectionString, + int startIndex, + int count) + { + for (var i = startIndex; i < startIndex + count; i++) + { + await InsertMessageAsync(provider, connectionString, i.ToString(), i); + } + } + + private static async Task InsertMessageAsync( + IServiceProvider provider, + string connectionString, + string messageId, + int index) + { + var runtime = provider.GetRequiredService(); + var type = runtime.GetMessageType(typeof(PendingEvent)); + return await MessageRows.InsertOutboxAsync( + connectionString, + new MessageEnvelope + { + MessageId = messageId, + MessageType = type.Identity, + ContentType = MessageContentType.Json.Value, + DestinationAddress = runtime.GetPublishEndpoint(type).Address.ToString(), + Body = JsonSerializer.SerializeToUtf8Bytes(new PendingEvent(index)) + }, + TestToken); + } + + private static async Task CountMessagesAsync(string connectionString) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(TestToken); + await using var command = connection.CreateCommand(); + command.CommandText = "SELECT COUNT(*) FROM outbox_messages"; + return (long)(await command.ExecuteScalarAsync(TestToken))!; + } + + private static Task WaitForEmptyOutboxAsync(string connectionString) + => WaitForOutboxCountAsync(connectionString, 0); + + private static async Task WaitForOutboxCountAsync(string connectionString, long count) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestToken); + timeout.CancelAfter(s_timeout); + while (await CountMessagesAsync(connectionString) != count) + { + await Task.Delay(20, timeout.Token); + } + } + + public sealed record PendingEvent(int Index); + + private sealed class DispatchProbe(int concurrency) + { + private int _active; + private int _peak; + + public bool FailFirst { get; set; } + public bool HoldFirst { get; set; } + public Action? ConnectionCreated { get; set; } + public int ActiveDispatches => Volatile.Read(ref _active); + public int PeakConcurrency => Volatile.Read(ref _peak); + public ConcurrentQueue Completed { get; } = new(); + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource FirstStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource FirstRelease { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async ValueTask DispatchAsync(IDispatchContext context, DispatchDelegate next) + { + var active = Interlocked.Increment(ref _active); + int previous; + do + { + previous = Volatile.Read(ref _peak); + } + while (active > previous && Interlocked.CompareExchange(ref _peak, active, previous) != previous); + + if (active >= concurrency) + { + Started.TrySetResult(); + } + + try + { + var index = int.Parse(context.Envelope!.MessageId!); + if (HoldFirst && index == 0) + { + FirstStarted.TrySetResult(); + await FirstRelease.Task.WaitAsync(context.CancellationToken); + } + + if (FailFirst && index == 0) + { + await Started.Task.WaitAsync(context.CancellationToken); + throw new InvalidOperationException("The first dispatch failed."); + } + + await Release.Task.WaitAsync(context.CancellationToken); + await next(context); + Completed.Enqueue(index); + } + finally + { + Interlocked.Decrement(ref _active); + } + } + } +} diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxConnectionLifetimeTests.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxConnectionLifetimeTests.cs index 8c5b965a267..b52363c863e 100644 --- a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxConnectionLifetimeTests.cs +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxConnectionLifetimeTests.cs @@ -191,7 +191,8 @@ public async Task PersistAsync_Should_RestoreConnectionState_When_InsertFails(bo await OpenConnectionAsync(throughEf: true); } - var queries = PostgresMessageOutboxQueries.From(new OutboxTableInfo { Table = "missing_outbox" }); + var queries = PostgresMessageOutboxQueries.From( + new OutboxTableInfo { Table = "missing_outbox" }, new PostgresOutboxOptions()); using var outbox = new PostgresMessageOutbox( _context, new StubOutboxSignal(), queries.InsertEnvelope, requireTransaction: false); @@ -353,7 +354,8 @@ private static ConnectionLifetimeDbContext CreateContext(NpgsqlDataSource dataSo private static PostgresMessageOutbox CreateOutbox(DbContext context) => new(context, new StubOutboxSignal(), - PostgresMessageOutboxQueries.From(new OutboxTableInfo()).InsertEnvelope, requireTransaction: false); + PostgresMessageOutboxQueries.From(new OutboxTableInfo(), new PostgresOutboxOptions()).InsertEnvelope, + requireTransaction: false); private static ServiceProvider CreateProvider(NpgsqlDataSource dataSource, bool pooled) { diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxDispatchOutcomeTests.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxDispatchOutcomeTests.cs new file mode 100644 index 00000000000..7397f2dd804 --- /dev/null +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresOutboxDispatchOutcomeTests.cs @@ -0,0 +1,615 @@ +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Mocha.EntityFrameworkCore.Postgres.Tests.Helpers; +using Mocha.Middlewares; +using Mocha.Outbox; +using Mocha.Transport.InMemory; +using Npgsql; + +namespace Mocha.EntityFrameworkCore.Postgres.Tests; + +public sealed class PostgresOutboxDispatchOutcomeTests(PostgresFixture fixture) : IClassFixture +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); + private static CancellationToken TestToken => TestContext.Current.CancellationToken; + + [Fact] + public async Task ProcessAsync_Should_ReleaseClaimAndWarn_When_TransportIsUnavailable() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var retrying = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var resume = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatch = new ScriptedDispatch(async attempt => + { + if (attempt.Number == 1) + { + throw new TransportUnavailableException("The broker is unreachable.", null); + } + + retrying.TrySetResult(); + await resume.Task.WaitAsync(attempt.CancellationToken); + await attempt.SendAsync(); + }); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync(connectionString, dispatch, logs, static _ => { }); + var envelope = CreateEnvelope(provider, "A"); + var id = await MessageRows.InsertOutboxAsync(connectionString, envelope, TestToken); + int? timesSentWhileRetrying; + long remaining; + + // act + await using (StartProcessor(provider)) + { + await retrying.Task.WaitAsync(s_timeout, TestToken); + timesSentWhileRetrying = await MessageRows.GetOutboxTimesSentAsync(connectionString, id, TestToken); + resume.TrySetResult(); + remaining = await MessageRows.WaitForOutboxCountAsync(connectionString, 0, s_timeout, TestToken); + } + + // assert + Assert.Equal(0, timesSentWhileRetrying); + Assert.Equal(0, remaining); + Assert.Equal(["A"], dispatch.Accepted); + Assert.Equal( + [ + new LogEntry( + LogLevel.Warning, + $"The dispatch of {Describe(id, envelope)} failed because the transport is unavailable. " + + "The attempt is not counted and the message is dispatched again.") + ], + logs.Entries); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProcessAsync_Should_ReleaseClaimAndWarn_When_DispatchExceedsDispatchTimeout( + bool afterCompletedDispatch) + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var dispatchTimeout = TimeSpan.FromSeconds(1); + var dispatch = new ScriptedDispatch(async attempt => + { + if (attempt is { MessageId: "A", Number: 1 }) + { + await Task.Delay(Timeout.InfiniteTimeSpan, attempt.CancellationToken); + } + + await attempt.SendAsync(); + }); + var logs = RecordingLoggerProvider.For(); + // One attempt is all a message has, so a counted timeout would leave the message in the outbox. + await using var provider = await CreateProviderAsync(connectionString, dispatch, logs, o => + { + o.DispatchTimeout = dispatchTimeout; + o.MaxAttempts = 1; + }); + string[] expectedAccepted = ["A"]; + if (afterCompletedDispatch) + { + await MessageRows.InsertOutboxAsync(connectionString, CreateEnvelope(provider, "earlier"), TestToken); + expectedAccepted = ["A", "earlier"]; + } + + var envelope = CreateEnvelope(provider, "A"); + var id = await MessageRows.InsertOutboxAsync(connectionString, envelope, TestToken); + long remaining; + + // act + await using (StartProcessor(provider)) + { + remaining = await MessageRows.WaitForOutboxCountAsync(connectionString, 0, s_timeout, TestToken); + } + + // assert + Assert.Equal(0, remaining); + Assert.Equal(expectedAccepted, dispatch.Accepted.Order()); + Assert.Equal( + [ + new LogEntry( + LogLevel.Warning, + $"The dispatch of {Describe(id, envelope)} did not complete within the dispatch timeout of " + + "00:00:01. The attempt is not counted and the message is dispatched again.") + ], + logs.Entries); + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + public async Task ProcessAsync_Should_ReleaseClaimWithoutWarning_When_HostStopsDuringDispatch( + bool afterCompletedDispatch, + bool withoutDispatchTimeout) + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var dispatching = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatch = new ScriptedDispatch(async attempt => + { + if (attempt.MessageId == "A") + { + dispatching.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, attempt.CancellationToken); + } + + await attempt.SendAsync(); + }); + var logs = RecordingLoggerProvider.For(LogLevel.Debug); + await using var provider = await CreateProviderAsync(connectionString, dispatch, logs, o => + { + if (withoutDispatchTimeout) + { + o.DispatchTimeout = Timeout.InfiniteTimeSpan; + } + }); + if (afterCompletedDispatch) + { + await MessageRows.InsertOutboxAsync(connectionString, CreateEnvelope(provider, "earlier"), TestToken); + } + + var envelope = CreateEnvelope(provider, "A"); + var id = await MessageRows.InsertOutboxAsync(connectionString, envelope, TestToken); + + // act + await using (StartProcessor(provider)) + { + await dispatching.Task.WaitAsync(s_timeout, TestToken); + } + + var timesSent = await MessageRows.GetOutboxTimesSentAsync(connectionString, id, TestToken); + + // assert + Assert.Equal(0, timesSent); + Assert.Equal( + [ + new LogEntry( + LogLevel.Debug, + $"The dispatch of {Describe(id, envelope)} was cancelled because the host is stopping. " + + "The attempt is not counted.") + ], + logs.Entries); + } + + [Fact] + public async Task ProcessAsync_Should_CountAttemptAndLogError_When_DispatchIsRefused() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var dispatch = new ScriptedDispatch(async attempt => + { + if (attempt.Number == 1) + { + throw new InvalidOperationException("The broker refused the message."); + } + + // No later attempt completes, so the outcome of the first one stays in the outbox. + await Task.Delay(Timeout.InfiniteTimeSpan, attempt.CancellationToken); + }); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync(connectionString, dispatch, logs, o => o.MaxAttempts = 3); + var envelope = CreateEnvelope(provider, "A"); + var id = await MessageRows.InsertOutboxAsync(connectionString, envelope, TestToken); + int? timesSent; + + // act + await using (StartProcessor(provider)) + { + timesSent = await MessageRows.WaitForOutboxTimesSentAsync(connectionString, id, 1, s_timeout, TestToken); + } + + // assert + Assert.Equal(1, timesSent); + Assert.Equal( + [new LogEntry(LogLevel.Error, $"The dispatch of {Describe(id, envelope)} failed on attempt 1 of 3.")], + logs.Entries); + } + + [Fact] + public async Task ProcessAsync_Should_LogCritical_When_RefusedDispatchWasTheLastAttempt() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var dispatch = new ScriptedDispatch( + static _ => throw new InvalidOperationException("The broker refused the message.")); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync(connectionString, dispatch, logs, o => o.MaxAttempts = 1); + var envelope = CreateEnvelope(provider, "A"); + var id = await MessageRows.InsertOutboxAsync(connectionString, envelope, TestToken); + + // act + await using (StartProcessor(provider)) + { + await logs.WaitForEntriesAsync(2, s_timeout, TestToken); + } + + var timesSent = await MessageRows.GetOutboxTimesSentAsync(connectionString, id, TestToken); + + // assert + Assert.Equal(1, timesSent); + Assert.Equal( + [ + new LogEntry(LogLevel.Error, $"The dispatch of {Describe(id, envelope)} failed on attempt 1 of 1."), + new LogEntry( + LogLevel.Critical, + $"Message with ID {id} exhausted all 1 attempts and will not be retried.") + ], + logs.Entries); + } + + [Fact] + public async Task ProcessAsync_Should_LogCritical_When_UnreadableMessageExhaustsItsAttempts() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync( + connectionString, new ScriptedDispatch(static attempt => attempt.SendAsync()), logs, o => o.MaxAttempts = 1); + var id = await MessageRows.InsertOutboxAsync(connectionString, "[]", TestToken); + + // act + await using (StartProcessor(provider)) + { + await logs.WaitForEntriesAsync(3, s_timeout, TestToken); + } + + var timesSent = await MessageRows.GetOutboxTimesSentAsync(connectionString, id, TestToken); + + // assert + Assert.Equal(1, timesSent); + Assert.Equal( + [ + new LogEntry(LogLevel.Error, "Error reading message envelope"), + new LogEntry( + LogLevel.Warning, + $"Could not deserialize message body for message with ID {id}. The message will be retried."), + new LogEntry( + LogLevel.Critical, + $"Message with ID {id} exhausted all 1 attempts and will not be retried.") + ], + logs.Entries); + } + + [Fact] + public async Task ProcessAsync_Should_LogCritical_When_MessageOfUnknownTypeExhaustsItsAttempts() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync( + connectionString, new ScriptedDispatch(static attempt => attempt.SendAsync()), logs, o => o.MaxAttempts = 1); + var envelope = new MessageEnvelope(CreateEnvelope(provider, "A")) + { + MessageType = "urn:message:Mocha.Tests:UnknownEvent" + }; + var id = await MessageRows.InsertOutboxAsync(connectionString, envelope, TestToken); + + // act + await using (StartProcessor(provider)) + { + await logs.WaitForEntriesAsync(2, s_timeout, TestToken); + } + + var timesSent = await MessageRows.GetOutboxTimesSentAsync(connectionString, id, TestToken); + + // assert + Assert.Equal(1, timesSent); + Assert.Equal( + [ + new LogEntry( + LogLevel.Warning, + "Could not resolve the message type urn:message:Mocha.Tests:UnknownEvent of message with ID " + + $"{id}. The message will be retried."), + new LogEntry( + LogLevel.Critical, + $"Message with ID {id} exhausted all 1 attempts and will not be retried.") + ], + logs.Entries); + } + + [Fact] + public async Task ProcessAsync_Should_LogCritical_When_MessageWithoutDestinationExhaustsItsAttempts() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync( + connectionString, new ScriptedDispatch(static attempt => attempt.SendAsync()), logs, o => o.MaxAttempts = 1); + var envelope = new MessageEnvelope(CreateEnvelope(provider, "A")) + { + DestinationAddress = "nowhere://unknown" + }; + var id = await MessageRows.InsertOutboxAsync(connectionString, envelope, TestToken); + + // act + await using (StartProcessor(provider)) + { + await logs.WaitForEntriesAsync(2, s_timeout, TestToken); + } + + var timesSent = await MessageRows.GetOutboxTimesSentAsync(connectionString, id, TestToken); + + // assert + Assert.Equal(1, timesSent); + Assert.Equal( + [ + new LogEntry( + LogLevel.Warning, + "Could not resolve a dispatch endpoint for the destination nowhere://unknown of message with ID " + + $"{id}. The message will be retried."), + new LogEntry( + LogLevel.Critical, + $"Message with ID {id} exhausted all 1 attempts and will not be retried.") + ], + logs.Entries); + } + + [Fact] + public async Task ProcessAsync_Should_StopDispatchingMessage_When_MaxAttemptsIsReached() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var dispatch = new ScriptedDispatch(attempt => attempt.MessageId == "A" + ? throw new InvalidOperationException("The broker refused the message.") + : attempt.SendAsync()); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync(connectionString, dispatch, logs, o => + { + o.MaxAttempts = 3; + o.MaxRetryDelay = TimeSpan.FromMilliseconds(50); + }); + var envelope = CreateEnvelope(provider, "A"); + var id = await MessageRows.InsertOutboxAsync(connectionString, envelope, TestToken); + + // act + await using (StartProcessor(provider)) + { + await logs.WaitForEntriesAsync(4, s_timeout, TestToken); + + // The exhausted message is due and older than the next message, so a claim would pick it first. + await MessageRows.MakeOutboxDueAsync(connectionString, TimeSpan.FromHours(1), TestToken); + await MessageRows.InsertOutboxAsync(connectionString, CreateEnvelope(provider, "B"), TestToken); + provider.GetRequiredService().Set(); + await MessageRows.WaitForOutboxCountAsync(connectionString, 1, s_timeout, TestToken); + } + + var timesSent = await MessageRows.GetOutboxTimesSentAsync(connectionString, id, TestToken); + + // assert + Assert.Equal(3, timesSent); + Assert.Equal(3, dispatch.AttemptsOf("A")); + Assert.Equal(["B"], dispatch.Accepted); + Assert.Equal( + [ + new LogEntry(LogLevel.Error, $"The dispatch of {Describe(id, envelope)} failed on attempt 1 of 3."), + new LogEntry(LogLevel.Error, $"The dispatch of {Describe(id, envelope)} failed on attempt 2 of 3."), + new LogEntry(LogLevel.Error, $"The dispatch of {Describe(id, envelope)} failed on attempt 3 of 3."), + new LogEntry( + LogLevel.Critical, + $"Message with ID {id} exhausted all 3 attempts and will not be retried.") + ], + logs.Entries); + } + + [Fact] + public async Task ProcessAsync_Should_RetryWithinMaxRetryDelay_When_AttemptsKeepFailing() + { + // arrange + // Without the cap the sixth attempt is due more than a minute after the first one. + var connectionString = await fixture.CreateDatabaseAsync(); + var dispatch = new ScriptedDispatch( + static _ => throw new InvalidOperationException("The broker refused the message.")); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync(connectionString, dispatch, logs, o => + { + o.MaxAttempts = 6; + o.MaxRetryDelay = TimeSpan.FromMilliseconds(100); + }); + var id = await MessageRows.InsertOutboxAsync(connectionString, CreateEnvelope(provider, "A"), TestToken); + int? timesSent; + + // act + await using (StartProcessor(provider)) + { + timesSent = await MessageRows.WaitForOutboxTimesSentAsync(connectionString, id, 6, s_timeout, TestToken); + } + + // assert + Assert.Equal(6, timesSent); + } + + [Fact] + public async Task ProcessAsync_Should_DispatchMessage_When_AttemptCountIsBeyondTheBackoffRange() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var dispatch = new ScriptedDispatch(static attempt => attempt.SendAsync()); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync( + connectionString, dispatch, logs, o => o.MaxAttempts = int.MaxValue); + var id = await MessageRows.InsertOutboxAsync(connectionString, CreateEnvelope(provider, "A"), TestToken); + await MessageRows.SetOutboxTimesSentAsync(connectionString, id, 5000, TestToken); + long remaining; + + // act + await using (StartProcessor(provider)) + { + remaining = await MessageRows.WaitForOutboxCountAsync(connectionString, 0, s_timeout, TestToken); + } + + // assert + Assert.Equal(0, remaining); + Assert.Equal(["A"], dispatch.Accepted); + Assert.Empty(logs.Entries); + } + + [Fact] + public async Task ProcessAsync_Should_LimitPollingInterval_When_ClaimedMessageHasBeenDueForMonths() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatch = new ScriptedDispatch(async attempt => + { + await release.Task.WaitAsync(attempt.CancellationToken); + await attempt.SendAsync(); + }); + var logs = RecordingLoggerProvider.For(LogLevel.Debug); + await using var provider = await CreateProviderAsync( + connectionString, dispatch, logs, static o => o.MaxConcurrentMessages = 2); + await MessageRows.InsertOutboxAsync(connectionString, CreateEnvelope(provider, "A"), TestToken); + await MessageRows.MakeOutboxDueAsync(connectionString, TimeSpan.FromDays(60), TestToken); + long remaining; + + // act + await using (StartProcessor(provider)) + { + // The worker without a message computes its polling interval while the other worker holds the claim. + await logs.WaitForEntriesAsync(1, s_timeout, TestToken); + release.TrySetResult(); + remaining = await MessageRows.WaitForOutboxCountAsync(connectionString, 0, s_timeout, TestToken); + } + + // assert + Assert.Equal(0, remaining); + Assert.Equal( + [new LogEntry(LogLevel.Debug, "Outbox processor is sleeping for 24.00:00:00.")], + logs.Entries); + } + + [Fact] + public async Task ProcessAsync_Should_KeepRunning_When_OutboxTableCannotBeRead() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var dispatch = new ScriptedDispatch(static attempt => attempt.SendAsync()); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync(connectionString, dispatch, logs, static _ => { }); + await ExecuteAsync(connectionString, "ALTER TABLE outbox_messages RENAME TO outbox_messages_moved"); + bool runningAfterFailure; + long remaining; + + // act + await using (var processor = StartProcessor(provider)) + { + await logs.WaitForEntriesAsync(1, s_timeout, TestToken); + runningAfterFailure = processor.IsRunning; + await ExecuteAsync(connectionString, "ALTER TABLE outbox_messages_moved RENAME TO outbox_messages"); + await MessageRows.InsertOutboxAsync(connectionString, CreateEnvelope(provider, "A"), TestToken); + remaining = await MessageRows.WaitForOutboxCountAsync(connectionString, 0, s_timeout, TestToken); + } + + // assert + Assert.True(runningAfterFailure); + Assert.Equal(0, remaining); + Assert.Equal(["A"], dispatch.Accepted); + Assert.Equal( + [new LogEntry(LogLevel.Error, "An unexpected error occurred while processing outbox event")], + logs.Entries.Distinct()); + } + + [Fact] + public async Task StopAsync_Should_DeleteMessage_When_TransportAcceptedItBeforeTheStop() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var accepted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatch = new ScriptedDispatch(async attempt => + { + await attempt.SendAsync(); + accepted.TrySetResult(); + await attempt.WaitForCancellationAsync(Task.Delay(s_timeout)); + }); + var logs = RecordingLoggerProvider.For(); + await using var provider = await CreateProviderAsync(connectionString, dispatch, logs, static _ => { }); + await MessageRows.InsertOutboxAsync(connectionString, CreateEnvelope(provider, "A"), TestToken); + var host = await MessagingTestHost.StartAsync(provider, TestToken); + await accepted.Task.WaitAsync(s_timeout, TestToken); + + // act + await host.DisposeAsync(); + var remaining = await MessageRows.CountOutboxAsync(connectionString, TestToken); + + // assert + Assert.Equal(0, remaining); + Assert.Equal(["A"], dispatch.Accepted); + Assert.Empty(logs.Entries); + } + + private static string Describe(Guid id, MessageEnvelope envelope) + => $"outbox message {id} (message ID {envelope.MessageId}, type {envelope.MessageType}) " + + $"to {envelope.DestinationAddress}"; + + private static MessageEnvelope CreateEnvelope(IServiceProvider provider, string messageId) + { + var runtime = provider.GetRequiredService(); + var type = runtime.GetMessageType(typeof(OutcomeEvent)); + return new MessageEnvelope + { + MessageId = messageId, + MessageType = type.Identity, + ContentType = MessageContentType.Json.Value, + DestinationAddress = runtime.GetPublishEndpoint(type).Address.ToString(), + Body = JsonSerializer.SerializeToUtf8Bytes(new OutcomeEvent(messageId)) + }; + } + + private static async Task CreateProviderAsync( + string connectionString, + ScriptedDispatch dispatch, + RecordingLoggerProvider logs, + Action configure) + { + var services = new ServiceCollection(); + services.AddLogging(b => b.SetMinimumLevel(LogLevel.Trace).AddProvider(logs)); + services.AddDbContext(o => o.UseTestNpgsql(connectionString)); + var builder = services.AddMessageBus(); + builder.AddEntityFramework(ef => ef.UsePostgresOutbox(configure)); + builder.AddInMemory(t => + { + t.DeclareQueue("outcome"); + t.DispatchEndpoint("outcome").ToQueue("outcome").Publish(); + }); + builder.ConfigureMessageBus(b => b.UseDispatch(dispatch.Middleware, after: "Serialization")); + + var provider = services.BuildServiceProvider(); + await ((MessagingRuntime)provider.GetRequiredService()).StartAsync(TestToken); + await using var scope = provider.CreateAsyncScope(); + await scope.ServiceProvider.GetRequiredService().Database.EnsureCreatedAsync(TestToken); + return provider; + } + + private static async Task ExecuteAsync(string connectionString, string sql) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(TestToken); + await using var command = connection.CreateCommand(); + command.CommandText = sql; + await command.ExecuteNonQueryAsync(TestToken); + } + + private static RunningProcessor StartProcessor(IServiceProvider provider) + { + var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestToken); + var processing = provider.GetRequiredService().ProcessAsync(cancellation.Token); + return new RunningProcessor(processing, cancellation); + } + + public sealed record OutcomeEvent(string Name); + + // Stops the processor like a host does and waits until it has returned. + private sealed class RunningProcessor(Task processing, CancellationTokenSource cancellation) : IAsyncDisposable + { + public bool IsRunning => !processing.IsCompleted; + + public async ValueTask DisposeAsync() + { + await cancellation.CancelAsync(); + await processing.WaitAsync(s_timeout, TestToken); + cancellation.Dispose(); + } + } +} diff --git a/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresScheduledMessageDispatcherTests.cs b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresScheduledMessageDispatcherTests.cs new file mode 100644 index 00000000000..bbef544d380 --- /dev/null +++ b/src/Mocha/test/Mocha.EntityFrameworkCore.Postgres.Tests/PostgresScheduledMessageDispatcherTests.cs @@ -0,0 +1,79 @@ +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Mocha.EntityFrameworkCore.Postgres.Tests.Helpers; +using Mocha.Middlewares; +using Mocha.Scheduling; +using Mocha.Transport.InMemory; + +namespace Mocha.EntityFrameworkCore.Postgres.Tests; + +public sealed class PostgresScheduledMessageDispatcherTests(PostgresFixture fixture) : IClassFixture +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); + private static CancellationToken TestToken => TestContext.Current.CancellationToken; + + [Fact] + public async Task ProcessAsync_Should_DeleteMessage_When_TransportAcceptedItBeforeTheStop() + { + // arrange + var connectionString = await fixture.CreateDatabaseAsync(); + var accepted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatch = new ScriptedDispatch(async attempt => + { + await attempt.SendAsync(); + accepted.TrySetResult(); + await attempt.WaitForCancellationAsync(Task.Delay(s_timeout)); + }); + await using var provider = await CreateProviderAsync(connectionString, dispatch); + await MessageRows.InsertScheduledAsync(connectionString, CreateEnvelope(provider, "A"), TestToken); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestToken); + var processing = provider.GetRequiredService().ProcessAsync(cancellation.Token); + await accepted.Task.WaitAsync(s_timeout, TestToken); + + // act + await cancellation.CancelAsync(); + await processing.WaitAsync(s_timeout, TestToken); + var remaining = await MessageRows.CountScheduledAsync(connectionString, TestToken); + + // assert + Assert.Equal(0, remaining); + Assert.Equal(["A"], dispatch.Accepted); + } + + private static MessageEnvelope CreateEnvelope(IServiceProvider provider, string messageId) + { + var runtime = provider.GetRequiredService(); + var type = runtime.GetMessageType(typeof(ScheduledEvent)); + return new MessageEnvelope + { + MessageId = messageId, + MessageType = type.Identity, + ContentType = MessageContentType.Json.Value, + DestinationAddress = runtime.GetPublishEndpoint(type).Address.ToString(), + Body = JsonSerializer.SerializeToUtf8Bytes(new ScheduledEvent(messageId)) + }; + } + + private static async Task CreateProviderAsync(string connectionString, ScriptedDispatch dispatch) + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddDbContext(o => o.UseTestNpgsql(connectionString)); + var builder = services.AddMessageBus(); + builder.AddEntityFramework(ef => ef.UsePostgresScheduling()); + builder.AddInMemory(t => + { + t.DeclareQueue("scheduled"); + t.DispatchEndpoint("scheduled").ToQueue("scheduled").Publish(); + }); + builder.ConfigureMessageBus(b => b.UseDispatch(dispatch.Middleware, after: "Serialization")); + + var provider = services.BuildServiceProvider(); + await ((MessagingRuntime)provider.GetRequiredService()).StartAsync(TestToken); + await using var scope = provider.CreateAsyncScope(); + await scope.ServiceProvider.GetRequiredService().Database.EnsureCreatedAsync(TestToken); + return provider; + } + + public sealed record ScheduledEvent(string Name); +} diff --git a/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckExtensionsTests.cs b/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckExtensionsTests.cs index 0b4c0288dbd..6d237b377e4 100644 --- a/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckExtensionsTests.cs +++ b/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckExtensionsTests.cs @@ -1,63 +1,140 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; +using Moq; namespace Mocha.Hosting.Tests.Health; public sealed class MessageBusHealthCheckExtensionsTests { [Fact] - public void AddMessageBus_Should_RegisterHealthCheck_When_Called() + public void AddMessageBus_Should_RegisterReadinessCheck_When_CalledWithDefaults() { - // Arrange + // arrange var services = new ServiceCollection(); services.AddLogging(); services.AddHealthChecks().AddMessageBus(); + using var provider = services.BuildServiceProvider(); - var provider = services.BuildServiceProvider(); + // act + var registrations = provider.GetRequiredService>().Value.Registrations; - // Act - var healthCheckOptions = provider.GetRequiredService>().Value; - - // Assert - var registration = Assert.Single(healthCheckOptions.Registrations, r => r.Name == "MessageBus"); - - Assert.Contains("ready", registration.Tags); - Assert.Contains("live", registration.Tags); + // assert + var registration = Assert.Single(registrations); + Assert.Equal("MessageBus", registration.Name); + Assert.Equal(HealthStatus.Unhealthy, registration.FailureStatus); + Assert.Equal(["ready"], registration.Tags); } [Fact] - public void AddMessageBus_Should_ConfigureEndpoint_When_EndpointProvided() + public void AddMessageBus_Should_RegisterCheckWithNameFailureStatusAndTags_When_Provided() { - // Arrange - var endpoint = new Uri("rabbitmq://health-endpoint"); + // arrange var services = new ServiceCollection(); services.AddLogging(); - services.AddHealthChecks().AddMessageBus(endpoint); + services.AddHealthChecks() + .AddMessageBus(name: "orders-bus", failureStatus: HealthStatus.Degraded, tags: ["live"]); + using var provider = services.BuildServiceProvider(); + + // act + var registrations = provider.GetRequiredService>().Value.Registrations; - var provider = services.BuildServiceProvider(); + // assert + var registration = Assert.Single(registrations); + Assert.Equal("orders-bus", registration.Name); + Assert.Equal(HealthStatus.Degraded, registration.FailureStatus); + Assert.Equal(["live"], registration.Tags); + } + + [Fact] + public async Task AddMessageBus_Should_SendToTheEndpointOfEachCheck_When_RegisteredTwice() + { + // arrange + var orders = new Uri("rabbitmq://orders-health"); + var bus = CreateBus(_ => new ValueTask(new HealthResponse("OK"))); + var services = CreateServices(bus.Object); + services.AddHealthChecks().AddMessageBus().AddMessageBus(orders, name: "orders-bus"); + await using var provider = services.BuildServiceProvider(); + var healthChecks = provider.GetRequiredService(); - // Act - var options = provider.GetRequiredService>().Value; + // act + var report = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken); - // Assert - Assert.Equal(endpoint, options.Endpoint); + // assert + Assert.Equal(HealthStatus.Healthy, report.Status); + bus.Verify(m => m.RequestAsync( + It.IsAny(), + It.Is(o => o.Endpoint == null), + It.IsAny()), Times.Once()); + bus.Verify(m => m.RequestAsync( + It.IsAny(), + It.Is(o => o.Endpoint == orders), + It.IsAny()), Times.Once()); } [Fact] - public void AddMessageBus_Should_NotConfigureEndpoint_When_EndpointIsNull() + public async Task AddMessageBus_Should_ReportFailureStatusAfterTimeout_When_BusDoesNotAnswer() { - // Arrange - var services = new ServiceCollection(); - services.AddLogging(); - services.AddHealthChecks().AddMessageBus(); + // arrange + var bus = CreateBus( + async cancellationToken => + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return new HealthResponse("OK"); + }); + var services = CreateServices(bus.Object); + services.AddHealthChecks() + .AddMessageBus(failureStatus: HealthStatus.Degraded, timeout: TimeSpan.FromMilliseconds(100)); + await using var provider = services.BuildServiceProvider(); + var healthChecks = provider.GetRequiredService(); - var provider = services.BuildServiceProvider(); + // act + var report = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - // Act - var options = provider.GetRequiredService>().Value; + // assert + var entry = report.Entries["MessageBus"]; + Assert.Equal(HealthStatus.Degraded, entry.Status); + Assert.Equal("Message Bus did not answer the health request within 00:00:00.1000000.", entry.Description); + } - // Assert - Assert.Null(options.Endpoint); + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void AddMessageBus_Should_Throw_When_TimeoutIsNotPositive(int seconds) + { + // arrange + var builder = new ServiceCollection().AddHealthChecks(); + + // act + var exception = Assert.Throws( + () => builder.AddMessageBus(timeout: TimeSpan.FromSeconds(seconds))); + + // assert + Assert.Equal("timeout", exception.ParamName); + } + + private static ServiceCollection CreateServices(IMessageBus bus) + { + var runtime = new Mock(); + runtime.SetupGet(r => r.Transports).Returns([]); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(bus); + services.AddSingleton(runtime.Object); + return services; + } + + private static Mock CreateBus(Func> respond) + { + var bus = new Mock(); + bus.Setup(m => m.RequestAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((IEventRequest _, SendOptions _, CancellationToken cancellationToken) + => respond(cancellationToken)); + return bus; } } diff --git a/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckIntegrationTests.cs b/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckIntegrationTests.cs index bac3eee41cb..ebb5a55e611 100644 --- a/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckIntegrationTests.cs +++ b/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckIntegrationTests.cs @@ -1,3 +1,6 @@ +using System.Net; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Mocha.Transport.InMemory; @@ -6,17 +9,134 @@ namespace Mocha.Hosting.Tests.Health; public sealed class MessageBusHealthCheckIntegrationTests { + [Fact] + public async Task HealthEndpoints_Should_KeepLivenessHealthy_When_TransportStopsAndRecovers() + { + // arrange + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseTestServer(); + builder.Services.AddMessageBus() + .AddHealthCheck() + .AddEventHandler() + .AddInMemory(t => t.Endpoint("orders").Handler()); + builder.Services.AddHealthChecks().AddMessageBus(); + await using var app = builder.Build(); + app.MapHealthChecks("/health/ready", new() { Predicate = check => check.Tags.Contains("ready") }); + app.MapHealthChecks("/health/live", new() { Predicate = check => check.Tags.Contains("live") }); + await app.StartAsync(TestContext.Current.CancellationToken); + using var client = app.GetTestClient(); + var runtime = (MessagingRuntime)app.Services.GetRequiredService(); + var endpoint = runtime.Transports.Single().ReceiveEndpoints.Single(e => e.Name == "orders"); + using var initial = await client.GetAsync("/health/ready", TestContext.Current.CancellationToken); + + // act + await endpoint.StopAsync(runtime, TestContext.Current.CancellationToken); + using var readiness = await client.GetAsync("/health/ready", TestContext.Current.CancellationToken); + using var liveness = await client.GetAsync("/health/live", TestContext.Current.CancellationToken); + await endpoint.StartAsync(runtime, TestContext.Current.CancellationToken); + using var recovered = await client.GetAsync("/health/ready", TestContext.Current.CancellationToken); + + // assert + Assert.Equal(HttpStatusCode.OK, initial.StatusCode); + Assert.Equal(HttpStatusCode.ServiceUnavailable, readiness.StatusCode); + Assert.Equal(HttpStatusCode.OK, liveness.StatusCode); + Assert.Equal("Healthy", await liveness.Content.ReadAsStringAsync(TestContext.Current.CancellationToken)); + Assert.Equal(HttpStatusCode.OK, recovered.StatusCode); + } + + [Fact] + public async Task CheckHealthAsync_Should_ReturnUnhealthy_When_TransportHasNotStarted() + { + // arrange + var services = new ServiceCollection(); + services.AddMessageBus().AddHealthCheck().AddInMemory(t => t.Name("orders")); + services.AddHealthChecks().AddMessageBus(); + await using var provider = services.BuildServiceProvider(); + var healthChecks = provider.GetRequiredService(); + + // act + var report = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + // assert + Assert.Equal(HealthStatus.Unhealthy, report.Status); + Assert.Equal( + "Transport 'orders' is unhealthy. The transport is not started.", + report.Entries["MessageBus"].Description); + } + + [Fact] + public async Task CheckHealthAsync_Should_ReportUnhealthyUntilRestart_When_ApplicationEndpointIsStopped() + { + // arrange + var services = new ServiceCollection(); + services.AddMessageBus() + .AddHealthCheck() + .AddEventHandler() + .AddInMemory(t => + { + t.Name("orders"); + t.Endpoint("orders").Handler(); + }); + services.AddHealthChecks().AddMessageBus(); + await using var provider = services.BuildServiceProvider(); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(TestContext.Current.CancellationToken); + var healthChecks = provider.GetRequiredService(); + var endpoint = runtime.Transports.Single().ReceiveEndpoints.Single(e => e.Name == "orders"); + var initial = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken); + + // act + await endpoint.StopAsync(runtime, TestContext.Current.CancellationToken); + var stopped = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken); + await endpoint.StartAsync(runtime, TestContext.Current.CancellationToken); + var restarted = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken); + + // assert + Assert.Equal( + [HealthStatus.Healthy, HealthStatus.Unhealthy, HealthStatus.Healthy], + new[] { initial.Status, stopped.Status, restarted.Status }); + Assert.Equal( + "Transport 'orders' is unhealthy. Receive endpoint 'orders' is not started.", + stopped.Entries["MessageBus"].Description); + Assert.Equal( + new Dictionary { ["orders"] = "Receive endpoint 'orders' is not started." }, + stopped.Entries["MessageBus"].Data); + } + + [Fact] + public async Task CheckHealthAsync_Should_ReportUnhealthyAfterTimeout_When_HealthRequestIsNotAnswered() + { + // arrange + var services = new ServiceCollection(); + services.AddMessageBus().AddRequestHandler().AddInMemory(); + services.AddHealthChecks().AddMessageBus(timeout: TimeSpan.FromMilliseconds(250)); + await using var provider = services.BuildServiceProvider(); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(TestContext.Current.CancellationToken); + var healthChecks = provider.GetRequiredService(); + + // act + var report = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + // assert + var entry = report.Entries["MessageBus"]; + Assert.Equal(HealthStatus.Unhealthy, entry.Status); + Assert.Equal("Message Bus did not answer the health request within 00:00:00.2500000.", entry.Description); + } + [Fact] public async Task CheckHealthAsync_Should_ReturnHealthy_When_BusIsRunning() { - // Arrange + // arrange await using var provider = await CreateBusWithHealthCheckAsync(); var healthCheckService = provider.GetRequiredService(); - // Act + // act var report = await healthCheckService.CheckHealthAsync(TestContext.Current.CancellationToken); - // Assert + // assert var entry = Assert.Contains("MessageBus", report.Entries); Assert.Equal(HealthStatus.Healthy, entry.Status); Assert.Equal("Message Bus is healthy.", entry.Description); @@ -25,17 +145,20 @@ public async Task CheckHealthAsync_Should_ReturnHealthy_When_BusIsRunning() [Fact] public async Task CheckHealthAsync_Should_ReturnHealthy_When_CalledMultipleTimes() { - // Arrange + // arrange await using var provider = await CreateBusWithHealthCheckAsync(); var healthCheckService = provider.GetRequiredService(); - // Act & Assert - for (var i = 0; i < 3; i++) + // act + var statuses = new HealthStatus[3]; + for (var i = 0; i < statuses.Length; i++) { var report = await healthCheckService.CheckHealthAsync(TestContext.Current.CancellationToken); - var entry = Assert.Contains("MessageBus", report.Entries); - Assert.Equal(HealthStatus.Healthy, entry.Status); + statuses[i] = report.Entries["MessageBus"].Status; } + + // assert + Assert.Equal([HealthStatus.Healthy, HealthStatus.Healthy, HealthStatus.Healthy], statuses); } private static async Task CreateBusWithHealthCheckAsync() @@ -49,4 +172,21 @@ private static async Task CreateBusWithHealthCheckAsync() await runtime.StartAsync(CancellationToken.None); return provider; } + + public sealed record OrderCreated; + + public sealed class OrderHandler : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + } + + public sealed class UnresponsiveHealthRequestHandler : IEventRequestHandler + { + public async ValueTask HandleAsync(HealthRequest request, CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return new HealthResponse("OK"); + } + } } diff --git a/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckTests.cs b/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckTests.cs index 1bf605fdff9..f221cac0c04 100644 --- a/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckTests.cs +++ b/src/Mocha/test/Mocha.Hosting.Tests/Health/MessageBusHealthCheckTests.cs @@ -1,93 +1,291 @@ using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Options; +using Mocha.Events; using Moq; namespace Mocha.Hosting.Tests.Health; public sealed class MessageBusHealthCheckTests { + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(10); + [Fact] public async Task CheckHealthAsync_Should_ReturnHealthy_When_BusRespondsOK() { - // Arrange - var busMock = new Mock(); - busMock.Setup(m => m.RequestAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new ValueTask(new HealthResponse("OK"))); - - var options = Options.Create(new MessageBusHealthCheckOptions()); - var healthCheck = new MessageBusHealthCheck(busMock.Object, options); + // arrange + var bus = CreateBus(new HealthResponse("OK")); + var healthCheck = CreateHealthCheck(bus.Object, CreateRuntime()); - // Act - var result = await healthCheck.CheckHealthAsync(new HealthCheckContext(), CancellationToken.None); + // act + var result = await healthCheck.CheckHealthAsync(CreateContext(), CancellationToken.None); - // Assert + // assert Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("Message Bus is healthy.", result.Description); } - [Fact] - public async Task CheckHealthAsync_Should_ReturnUnhealthy_When_BusRespondsNonOK() + [Theory] + [InlineData(HealthStatus.Unhealthy)] + [InlineData(HealthStatus.Degraded)] + public async Task CheckHealthAsync_Should_ReturnFailureStatus_When_BusRespondsNonOK(HealthStatus failureStatus) { - // Arrange - var busMock = new Mock(); - busMock.Setup(m => m.RequestAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new ValueTask(new HealthResponse("ERROR"))); - - var options = Options.Create(new MessageBusHealthCheckOptions()); - var healthCheck = new MessageBusHealthCheck(busMock.Object, options); + // arrange + var bus = CreateBus(new HealthResponse("ERROR")); + var healthCheck = CreateHealthCheck(bus.Object, CreateRuntime()); - // Act - var result = await healthCheck.CheckHealthAsync(new HealthCheckContext(), CancellationToken.None); + // act + var result = await healthCheck.CheckHealthAsync(CreateContext(failureStatus), CancellationToken.None); - // Assert - Assert.Equal(HealthStatus.Unhealthy, result.Status); + // assert + Assert.Equal(failureStatus, result.Status); Assert.Equal("Message Bus is unhealthy.", result.Description); } [Fact] public async Task CheckHealthAsync_Should_RouteToEndpoint_When_EndpointConfigured() { - // Arrange + // arrange var endpoint = new Uri("rabbitmq://health-endpoint"); - var busMock = new Mock(); - busMock.Setup(m => m.RequestAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new ValueTask(new HealthResponse("OK"))); - - var options = Options.Create(new MessageBusHealthCheckOptions { Endpoint = endpoint }); - var healthCheck = new MessageBusHealthCheck(busMock.Object, options); + var bus = CreateBus(new HealthResponse("OK")); + var healthCheck = CreateHealthCheck(bus.Object, CreateRuntime(), endpoint); - // Act - await healthCheck.CheckHealthAsync(new HealthCheckContext(), CancellationToken.None); + // act + await healthCheck.CheckHealthAsync(CreateContext(), CancellationToken.None); - // Assert - busMock.Verify(m => m.RequestAsync( + // assert + bus.Verify(m => m.RequestAsync( It.IsAny(), It.Is(o => o.Endpoint == endpoint), It.IsAny()), Times.Once()); } [Fact] - public async Task CheckHealthAsync_Should_UseDefaultSendOptions_When_NoEndpointConfigured() + public async Task CheckHealthAsync_Should_UseDefaultRouting_When_NoEndpointConfigured() { - // Arrange - var busMock = new Mock(); - busMock.Setup(m => m.RequestAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new ValueTask(new HealthResponse("OK"))); + // arrange + var bus = CreateBus(new HealthResponse("OK")); + var healthCheck = CreateHealthCheck(bus.Object, CreateRuntime()); - var options = Options.Create(new MessageBusHealthCheckOptions()); - var healthCheck = new MessageBusHealthCheck(busMock.Object, options); + // act + await healthCheck.CheckHealthAsync(CreateContext(), CancellationToken.None); - // Act - await healthCheck.CheckHealthAsync(new HealthCheckContext(), CancellationToken.None); - - // Assert - busMock.Verify(m => m.RequestAsync( + // assert + bus.Verify(m => m.RequestAsync( It.IsAny(), It.Is(o => o.Endpoint == null), It.IsAny()), Times.Once()); } + + [Fact] + public async Task CheckHealthAsync_Should_ExpireHealthRequestAfterTimeout_When_RequestIsSent() + { + // arrange + var now = new DateTimeOffset(2026, 1, 2, 3, 4, 5, TimeSpan.Zero); + SendOptions? sendOptions = null; + var bus = new Mock(); + bus.Setup(m => m.RequestAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((IEventRequest _, SendOptions options, CancellationToken _) => + { + sendOptions = options; + return new ValueTask(new HealthResponse("OK")); + }); + var healthCheck = new MessageBusHealthCheck( + bus.Object, + CreateRuntime(), + new FixedTimeProvider(now), + new MessageBusHealthCheckOptions { Timeout = TimeSpan.FromSeconds(7) }); + + // act + await healthCheck.CheckHealthAsync(CreateContext(), TestContext.Current.CancellationToken); + + // assert + Assert.Equal(now.AddSeconds(7), sendOptions?.ExpirationTime); + } + + [Fact] + public async Task CheckHealthAsync_Should_ReturnFailureStatus_When_BusDoesNotAnswerWithinTimeout() + { + // arrange + var bus = new Mock(); + bus.Setup(m => m.RequestAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((IEventRequest _, SendOptions _, CancellationToken cancellationToken) + => new ValueTask(WaitForCancellationAsync(cancellationToken))); + var healthCheck = CreateHealthCheck(bus.Object, CreateRuntime(), timeout: TimeSpan.FromMilliseconds(100)); + + // act + var result = await healthCheck + .CheckHealthAsync(CreateContext(HealthStatus.Degraded), TestContext.Current.CancellationToken) + .WaitAsync(s_timeout, TestContext.Current.CancellationToken); + + // assert + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("Message Bus did not answer the health request within 00:00:00.1000000.", result.Description); + } + + [Fact] + public async Task CheckHealthAsync_Should_ReturnFailureStatus_When_ResponseTimesOut() + { + // arrange + var failure = new ResponseTimeoutException("correlation-1", TimeSpan.FromMinutes(2)); + var bus = new Mock(); + bus.Setup(m => m.RequestAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(ValueTask.FromException(failure)); + var healthCheck = CreateHealthCheck(bus.Object, CreateRuntime()); + + // act + var result = await healthCheck.CheckHealthAsync( + CreateContext(HealthStatus.Degraded), + TestContext.Current.CancellationToken); + + // assert + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("Message Bus did not answer the health request within 00:02:00.", result.Description); + Assert.Same(failure, result.Exception); + } + + [Fact] + public async Task CheckHealthAsync_Should_PropagateCancellation_When_CallerCancels() + { + // arrange + using var caller = new CancellationTokenSource(); + var bus = new Mock(); + bus.Setup(m => m.RequestAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((IEventRequest _, SendOptions _, CancellationToken cancellationToken) => + { + caller.Cancel(); + return new ValueTask(WaitForCancellationAsync(cancellationToken)); + }); + var healthCheck = CreateHealthCheck(bus.Object, CreateRuntime()); + + // act + var exception = await Record.ExceptionAsync( + () => healthCheck.CheckHealthAsync(CreateContext(), caller.Token)); + + // assert + Assert.IsAssignableFrom(exception); + } + + [Fact] + public async Task CheckHealthAsync_Should_ReturnUnhealthy_When_TransportFailsDuringRequest() + { + // arrange + var transport = CreateTransport("orders", healthFailureReason: null); + var bus = new Mock(); + bus.Setup(m => m.RequestAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(() => + { + transport.SetupGet(t => t.HealthFailureReason).Returns("The dispatcher connection is unavailable."); + return new ValueTask(new HealthResponse("OK")); + }); + var healthCheck = CreateHealthCheck(bus.Object, CreateRuntime(transport.Object)); + + // act + var result = await healthCheck.CheckHealthAsync(CreateContext(), TestContext.Current.CancellationToken); + + // assert + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal( + "Transport 'orders' is unhealthy. The dispatcher connection is unavailable.", + result.Description); + } + + [Fact] + public async Task CheckHealthAsync_Should_ReportEveryUnhealthyTransport_When_SeveralTransportsAreUnhealthy() + { + // arrange + var orders = CreateTransport("orders", "The dispatcher connection is unavailable."); + var payments = CreateTransport("payments", healthFailureReason: null); + var shipping = CreateTransport("shipping", "Receive endpoint 'parcels' is not started."); + var bus = CreateBus(new HealthResponse("OK")); + var healthCheck = CreateHealthCheck( + bus.Object, + CreateRuntime(orders.Object, payments.Object, shipping.Object)); + + // act + var result = await healthCheck.CheckHealthAsync( + CreateContext(HealthStatus.Degraded), + TestContext.Current.CancellationToken); + + // assert + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal( + "Transport 'orders' is unhealthy. The dispatcher connection is unavailable.", + result.Description); + Assert.Equal( + new Dictionary + { + ["orders"] = "The dispatcher connection is unavailable.", + ["shipping"] = "Receive endpoint 'parcels' is not started." + }, + result.Data); + bus.VerifyNoOtherCalls(); + } + + private static MessageBusHealthCheck CreateHealthCheck( + IMessageBus bus, + IMessagingRuntime runtime, + Uri? endpoint = null, + TimeSpan? timeout = null) + => new( + bus, + runtime, + TimeProvider.System, + new MessageBusHealthCheckOptions { Endpoint = endpoint, Timeout = timeout ?? s_timeout }); + + private static HealthCheckContext CreateContext(HealthStatus failureStatus = HealthStatus.Unhealthy) + => new() + { + Registration = new HealthCheckRegistration("MessageBus", Mock.Of(), failureStatus, tags: null) + }; + + private static Mock CreateBus(HealthResponse response) + { + var bus = new Mock(); + bus.Setup(m => m.RequestAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(new ValueTask(response)); + return bus; + } + + private static Mock CreateTransport(string name, string? healthFailureReason) + { + var transport = new Mock(); + transport.Object.Name = name; + transport.SetupGet(t => t.HealthFailureReason).Returns(healthFailureReason); + return transport; + } + + private static IMessagingRuntime CreateRuntime(params MessagingTransport[] transports) + { + var runtime = new Mock(); + runtime.SetupGet(r => r.Transports).Returns([.. transports]); + return runtime.Object; + } + + private static async Task WaitForCancellationAsync(CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return new HealthResponse("OK"); + } + + private sealed class FixedTimeProvider(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + } } diff --git a/src/Mocha/test/Mocha.Sagas.Tests/ReplyDiagnosticsTests.cs b/src/Mocha/test/Mocha.Sagas.Tests/ReplyDiagnosticsTests.cs deleted file mode 100644 index d3ad1bf0d0d..00000000000 --- a/src/Mocha/test/Mocha.Sagas.Tests/ReplyDiagnosticsTests.cs +++ /dev/null @@ -1,195 +0,0 @@ -using System.Collections.Concurrent; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Mocha.Transport.InMemory; - -namespace Mocha.Sagas.Tests; - -/// -/// Tests that a reply reaching the shared reply endpoint without a pending request is reported when -/// nothing else claims it, and stays quiet when a saga route owns it. -/// -public class ReplyDiagnosticsTests -{ - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(10); - - [Fact] - public async Task ReplyConsumer_Should_WarnAboutDiscardedReply_When_NoRouteClaimsIt() - { - // A saga that handles only fault replies leaves a successful reply unclaimed: it fails the - // fault route's message type term and matches no request promise, so it is dropped. - - // arrange - var logs = new CapturingLoggerProvider(); - var handlerRan = new TaskCompletionSource(); - await using var provider = await CreateBusAsync(logs, b => - { - b.Services.AddSingleton(handlerRan); - b.AddRequestHandler(); - b.AddSaga(); - }); - - using var scope = provider.CreateScope(); - var bus = scope.ServiceProvider.GetRequiredService(); - - // act - await bus.PublishAsync(new StartDiagnosticsEvent(), CancellationToken.None); - await handlerRan.Task.WaitAsync(s_timeout, TestContext.Current.CancellationToken); - - // assert - var entry = await logs.WaitForAsync(LogLevel.Warning, s_timeout); - Assert.Contains("Discarded a reply", entry); - } - - [Fact] - public async Task ReplyConsumer_Should_StaySilent_When_SagaRouteClaimsTheReply() - { - // The happy path for every saga reply, so it must not warn. - - // arrange - var logs = new CapturingLoggerProvider(); - var handlerRan = new TaskCompletionSource(); - await using var provider = await CreateBusAsync(logs, b => - { - b.Services.AddSingleton(handlerRan); - b.AddRequestHandler(); - b.AddSaga(); - }); - - using var scope = provider.CreateScope(); - var bus = scope.ServiceProvider.GetRequiredService(); - var storage = provider.GetRequiredService(); - - // act - await bus.PublishAsync(new StartDiagnosticsEvent(), CancellationToken.None); - await handlerRan.Task.WaitAsync(s_timeout, TestContext.Current.CancellationToken); - - // wait for the saga to consume the reply and finalize, so the reply has been fully processed - var deadline = DateTime.UtcNow + s_timeout; - while (storage.Count != 0 && DateTime.UtcNow < deadline) - { - await Task.Delay(50, TestContext.Current.CancellationToken); - } - - // assert - the saga owned the reply, so the reply path reported nothing - Assert.Equal(0, storage.Count); - Assert.Equal( - [], - logs.Entries.Where(e => e.Level >= LogLevel.Warning).Select(e => e.Message).ToArray()); - } - - private static async Task CreateBusAsync( - CapturingLoggerProvider logs, - Action configure) - { - var services = new ServiceCollection(); - services.AddLogging(b => b.AddProvider(logs)); - services.AddInMemorySagas(); - var builder = services.AddMessageBus(); - configure(builder); - builder.AddInMemory(); - - var provider = services.BuildServiceProvider(); - var runtime = (MessagingRuntime)provider.GetRequiredService(); - await runtime.StartAsync(CancellationToken.None); - return provider; - } - - public sealed class DiagnosticsState : SagaStateBase; - - public sealed class StartDiagnosticsEvent; - - public sealed record DiagnosticsRequest : IEventRequest; - - public sealed record DiagnosticsResponse; - - public sealed class DiagnosticsRequestHandler(TaskCompletionSource handlerRan) - : IEventRequestHandler - { - public ValueTask HandleAsync( - DiagnosticsRequest request, - CancellationToken cancellationToken) - { - handlerRan.TrySetResult(); - return new(new DiagnosticsResponse()); - } - } - - /// - /// Handles only fault replies, so a successful reply matches no route on the reply endpoint. - /// - public sealed class FaultOnlySaga : Saga - { - protected override void Configure(ISagaDescriptor descriptor) - { - descriptor - .Initially() - .OnEvent() - .StateFactory(_ => new DiagnosticsState()) - .Send((_, _) => new DiagnosticsRequest()) - .TransitionTo("Awaiting"); - - descriptor.During("Awaiting").OnFault().TransitionTo("Failed"); - - descriptor.Finally("Failed"); - } - } - - public sealed class AnyReplySaga : Saga - { - protected override void Configure(ISagaDescriptor descriptor) - { - descriptor - .Initially() - .OnEvent() - .StateFactory(_ => new DiagnosticsState()) - .Send((_, _) => new DiagnosticsRequest()) - .TransitionTo("Awaiting"); - - descriptor.During("Awaiting").OnAnyReply().TransitionTo("Done"); - descriptor.During("Awaiting").OnFault().TransitionTo("Done"); - - descriptor.Finally("Done"); - } - } - - private sealed class CapturingLoggerProvider : ILoggerProvider - { - public ConcurrentQueue<(LogLevel Level, string Message)> Entries { get; } = new(); - - public ILogger CreateLogger(string categoryName) => new CapturingLogger(Entries); - - public void Dispose() { } - - public async Task WaitForAsync(LogLevel level, TimeSpan timeout) - { - var deadline = DateTime.UtcNow + timeout; - while (DateTime.UtcNow < deadline) - { - if (Entries.FirstOrDefault(e => e.Level == level) is { Message: { } message }) - { - return message; - } - - await Task.Delay(50, TestContext.Current.CancellationToken); - } - - throw new TimeoutException($"No {level} log entry was written within {timeout}."); - } - - private sealed class CapturingLogger(ConcurrentQueue<(LogLevel, string)> entries) : ILogger - { - public IDisposable? BeginScope(TState state) where TState : notnull => null; - - public bool IsEnabled(LogLevel logLevel) => true; - - public void Log( - LogLevel logLevel, - EventId eventId, - TState state, - Exception? exception, - Func formatter) - => entries.Enqueue((logLevel, formatter(state, exception))); - } - } -} diff --git a/src/Mocha/test/Mocha.Tests/CircuitBreaker/CircuitBreakerLoggingTests.cs b/src/Mocha/test/Mocha.Tests/CircuitBreaker/CircuitBreakerLoggingTests.cs new file mode 100644 index 00000000000..ab22279ab9e --- /dev/null +++ b/src/Mocha/test/Mocha.Tests/CircuitBreaker/CircuitBreakerLoggingTests.cs @@ -0,0 +1,176 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Mocha.Transport.InMemory; + +namespace Mocha.Tests; + +public sealed class CircuitBreakerLoggingTests +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); + + [Fact] + public async Task TransportCircuitBreaker_Should_LogTransportCircuitUnderOwnCategory_When_CircuitOpens() + { + // arrange + using var logs = new RecordingLoggerProvider("CircuitOpen"); + await using var provider = await CreateBusAsync( + logs, + builder => + { + builder.AddEventHandler(); + // the failure is raised inside the transport circuit breaker and outside the fault handling + builder.ConfigureMessageBus( + bus => bus.UseReceive(ThrowingMiddleware.Create(), after: "TransportCircuitBreaker")); + }, + transport => + { + transport.ModifyOptions( + options => options.CircuitBreaker = new TransportCircuitBreakerOptions + { + FailureRatio = 0.5, + MinimumThroughput = 2, + SamplingDuration = TimeSpan.FromSeconds(30), + BreakDuration = TimeSpan.FromSeconds(30) + }); + transport.Endpoint("orders").Handler(); + }); + + // act + await PublishAsync(provider, count: 2); + var entry = await logs.Entry.WaitAsync(s_timeout, TestContext.Current.CancellationToken); + + // assert + Assert.Equal( + new LogEntry( + "Mocha.TransportCircuitBreakerMiddleware", + LogLevel.Warning, + "Transport circuit for endpoint orders is open; recovery will be tested after 00:00:30"), + entry); + } + + [Fact] + public async Task ReceiveCircuitBreaker_Should_LogConsumerCircuitUnderOwnCategory_When_CircuitOpens() + { + // arrange + using var logs = new RecordingLoggerProvider("CircuitOpen"); + await using var provider = await CreateBusAsync( + logs, + builder => + { + builder.AddEventHandler(); + builder.AddCircuitBreaker( + options => + { + options.FailureRatio = 0.5; + options.MinimumThroughput = 2; + options.SamplingDuration = TimeSpan.FromSeconds(30); + options.BreakDuration = TimeSpan.FromSeconds(30); + }); + }, + transport => transport.Endpoint("orders").Handler()); + + // act + await PublishAsync(provider, count: 2); + var entry = await logs.Entry.WaitAsync(s_timeout, TestContext.Current.CancellationToken); + + // assert + Assert.Equal( + new LogEntry( + "Mocha.ReceiveCircuitBreakerMiddleware", + LogLevel.Warning, + "Consumer circuit for endpoint orders is open; recovery will be tested after 00:00:30"), + entry); + } + + private static async Task CreateBusAsync( + RecordingLoggerProvider logs, + Action configure, + Action configureTransport) + { + var services = new ServiceCollection(); + services.AddLogging(logging => logging.AddProvider(logs)); + var builder = services.AddMessageBus(); + configure(builder); + builder.AddInMemory(configureTransport); + + var provider = services.BuildServiceProvider(); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(TestContext.Current.CancellationToken); + return provider; + } + + private static async Task PublishAsync(IServiceProvider provider, int count) + { + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + for (var i = 0; i < count; i++) + { + await bus.PublishAsync(new OrderPlaced(), TestContext.Current.CancellationToken); + } + } + + public sealed class OrderPlaced; + + public sealed class NoOpHandler : IEventHandler + { + public ValueTask HandleAsync(OrderPlaced message, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + } + + public sealed class ThrowingHandler : IEventHandler + { + public ValueTask HandleAsync(OrderPlaced message, CancellationToken cancellationToken) + => throw new InvalidOperationException("The handler failed."); + } + + private static class ThrowingMiddleware + { + public static ReceiveMiddlewareConfiguration Create() + => new( + static (_, _) => static _ => throw new InvalidOperationException("The transport failed."), + "Throwing"); + } + + private sealed record LogEntry(string Category, LogLevel Level, string Message); + + /// + /// Records the first log entry with the given event name, together with the category of its logger. + /// + private sealed class RecordingLoggerProvider(string eventName) : ILoggerProvider + { + private readonly TaskCompletionSource _entry = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task Entry => _entry.Task; + + private string EventName => eventName; + + public ILogger CreateLogger(string categoryName) => new RecordingLogger(this, categoryName); + + public void Dispose() + { + } + + private sealed class RecordingLogger(RecordingLoggerProvider provider, string category) : ILogger + { + public IDisposable? BeginScope(TState state) + where TState : notnull + => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (eventId.Name == provider.EventName) + { + provider._entry.TrySetResult(new LogEntry(category, logLevel, formatter(state, exception))); + } + } + } + } +} diff --git a/src/Mocha/test/Mocha.Tests/Infrastructure/DeferredResponseManagerTests.cs b/src/Mocha/test/Mocha.Tests/Infrastructure/DeferredResponseManagerTests.cs index e1ed3d48c15..8196f331dd3 100644 --- a/src/Mocha/test/Mocha.Tests/Infrastructure/DeferredResponseManagerTests.cs +++ b/src/Mocha/test/Mocha.Tests/Infrastructure/DeferredResponseManagerTests.cs @@ -5,6 +5,45 @@ namespace Mocha.Tests; public class DeferredResponseManagerTests { + [Fact] + public async Task AddPromise_Should_CancelDispatch_When_DefaultTimeoutExpires() + { + // arrange + var timeProvider = new FakeTimeProvider(); + var manager = new DeferredResponseManager(timeProvider); + using var timeout = manager.CreateRequestTimeout(); + var promise = manager.AddPromise("request", timeout.Token); + + // act + timeProvider.Advance(TimeSpan.FromMinutes(2)); + + // assert + Assert.True(timeout.IsCancellationRequested); + var exception = await Assert.ThrowsAsync(() => promise.Task); + Assert.Equal("request", exception.CorrelationId); + Assert.Equal(TimeSpan.FromMinutes(2), exception.Timeout); + } + + [Fact] + public async Task CompletePromise_Should_PreserveRequestDeadline_When_ResponseArrivesBeforeDispatchCompletes() + { + // arrange + var timeProvider = new FakeTimeProvider(); + var manager = new DeferredResponseManager(timeProvider); + using var timeout = manager.CreateRequestTimeout(); + var promise = manager.AddPromise("request", timeout.Token); + + // act + manager.CompletePromise("request", "response"); + var cancelledBeforeDeadline = timeout.IsCancellationRequested; + timeProvider.Advance(TimeSpan.FromMinutes(2)); + + // assert + Assert.Equal("response", await promise.Task); + Assert.False(cancelledBeforeDeadline); + Assert.True(timeout.IsCancellationRequested); + } + [Fact] public async Task AddPromise_Should_CompletePromiseAndReturnResult_When_PromiseCompleted() { diff --git a/src/Mocha/test/Mocha.Tests/IntegrationTests/RequestTimeoutIntegrationTests.cs b/src/Mocha/test/Mocha.Tests/IntegrationTests/RequestTimeoutIntegrationTests.cs new file mode 100644 index 00000000000..c4b947a3809 --- /dev/null +++ b/src/Mocha/test/Mocha.Tests/IntegrationTests/RequestTimeoutIntegrationTests.cs @@ -0,0 +1,175 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using Mocha.Events; + +namespace Mocha.Tests.IntegrationTests; + +public sealed class RequestTimeoutIntegrationTests : ConsumerIntegrationTestsBase +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RequestAsync_Should_CompleteByDeadline_When_ResponseArrivesBeforeDispatchCompletes(bool fault) + { + // arrange + var timeProvider = new FakeTimeProvider(); + var responseReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var provider = await CreateBusAsync(builder => + { + builder.Services.AddSingleton(timeProvider); + builder.Services.AddSingleton(new MessageRecorder()); + if (fault) + { + builder.AddRequestHandler(); + } + else + { + builder.AddRequestHandler(); + } + + builder.ConfigureMessageBus(b => b.UseDispatch(new DispatchMiddlewareConfiguration( + (_, next) => async context => + { + if (context.Message is not GetOrderStatus) + { + await next(context); + return; + } + + var response = context.Services.GetRequiredService() + .GetPromise(context.CorrelationId!); + await next(context); + try + { + await response; + } + catch (RemoteErrorException) when (fault) + { + } + + responseReceived.TrySetResult(); + await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, context.CancellationToken); + }, + "DelayRequestDispatchCompletion"))); + }); + using var scope = provider.CreateScope(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var bus = scope.ServiceProvider.GetRequiredService(); + var request = bus.RequestAsync(new GetOrderStatus { OrderId = "ORD-1" }, cancellation.Token).AsTask(); + + try + { + await responseReceived.Task.WaitAsync(Timeout, TestContext.Current.CancellationToken); + Assert.False(request.IsCompleted); + + // act + timeProvider.Advance(TimeSpan.FromMinutes(2)); + + // assert + if (fault) + { + await Assert.ThrowsAsync( + () => request.WaitAsync(Timeout, TestContext.Current.CancellationToken)); + } + else + { + var response = await request.WaitAsync(Timeout, TestContext.Current.CancellationToken); + Assert.Equal("ORD-1", response.OrderId); + Assert.Equal("Shipped", response.Status); + } + } + finally + { + await cancellation.CancelAsync(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RequestAsync_Should_ReleasePendingResponse_When_DispatchFailsOrIsCancelled(bool cancel) + { + // arrange + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var provider = await CreateBusAsync(builder => + { + builder.Services.AddSingleton(new MessageRecorder()); + builder.AddRequestHandler(); + builder.ConfigureMessageBus(b => b.UseDispatch(new DispatchMiddlewareConfiguration( + (_, _) => async context => + { + started.TrySetResult(context.CorrelationId!); + if (cancel) + { + await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, context.CancellationToken); + } + + throw new InvalidOperationException("Dispatch failed."); + }, + "FailDispatch"))); + }); + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + var responses = provider.GetRequiredService(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + + // act + var request = bus.RequestAsync(new GetOrderStatus { OrderId = "ORD-1" }, cancellation.Token).AsTask(); + var correlationId = await started.Task.WaitAsync(Timeout, TestContext.Current.CancellationToken); + if (cancel) + { + await cancellation.CancelAsync(); + } + + // assert + if (cancel) + { + await Assert.ThrowsAnyAsync( + () => request.WaitAsync(Timeout, TestContext.Current.CancellationToken)); + } + else + { + var exception = await Assert.ThrowsAsync( + () => request.WaitAsync(Timeout, TestContext.Current.CancellationToken)); + Assert.Equal("Dispatch failed.", exception.Message); + } + + Assert.False(responses.CompletePromise(correlationId, new OrderStatusResponse { OrderId = "ORD-1", Status = "Late" })); + } + + [Fact] + public async Task RequestAsync_Should_Timeout_When_DispatchCompletesWithoutAResponse() + { + // arrange + var timeProvider = new FakeTimeProvider(); + var dispatched = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var provider = await CreateBusAsync(builder => + { + builder.Services.AddSingleton(timeProvider); + builder.Services.AddSingleton(new MessageRecorder()); + builder.AddRequestHandler(); + builder.ConfigureMessageBus(b => b.UseDispatch(new DispatchMiddlewareConfiguration( + (_, _) => context => + { + dispatched.TrySetResult(context.CorrelationId!); + return default; + }, + "DiscardRequest"))); + }); + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + var responses = provider.GetRequiredService(); + + // act + var request = bus.RequestAsync(new GetOrderStatus { OrderId = "ORD-1" }, TestContext.Current.CancellationToken).AsTask(); + var correlationId = await dispatched.Task.WaitAsync(Timeout, TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromMinutes(2)); + + // assert + var exception = await Assert.ThrowsAsync( + () => request.WaitAsync(Timeout, TestContext.Current.CancellationToken)); + Assert.Equal(correlationId, exception.CorrelationId); + Assert.Equal(TimeSpan.FromMinutes(2), exception.Timeout); + Assert.False(responses.CompletePromise(correlationId, "late")); + } +} diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs index b2d11e1b42b..9d2977dc006 100644 --- a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs @@ -22,6 +22,7 @@ await RetryExecutor.ExecuteAsync( return ValueTask.CompletedTask; }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); // assert @@ -42,6 +43,7 @@ await Assert.ThrowsAsync(() => 0, static (_) => throw new InvalidOperationException("no match"), onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -59,6 +61,7 @@ await RetryExecutor.ExecuteAsync( 0, static (_) => throw new InvalidOperationException("discard me"), onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); } @@ -76,6 +79,7 @@ await Assert.ThrowsAsync(() => 0, static (_) => throw new InvalidOperationException("dead letter"), onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -95,6 +99,7 @@ await Assert.ThrowsAsync(() => 0, static (_) => throw new InvalidOperationException("no retry"), onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -125,6 +130,7 @@ await RetryExecutor.ExecuteAsync( return ValueTask.CompletedTask; }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); // assert @@ -152,6 +158,7 @@ await Assert.ThrowsAsync(() => throw new InvalidOperationException("always fails"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -184,7 +191,8 @@ await RetryExecutor.ExecuteAsync( return ValueTask.CompletedTask; }, - onRetry: static (s, attempt) => s.Attempts.Add(attempt), + onRetry: static (s, _, attempt, _) => s.Attempts.Add(attempt), + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); // assert @@ -207,7 +215,8 @@ await RetryExecutor.ExecuteAsync( rules, 0, static (_) => ValueTask.CompletedTask, - onRetry: static (_, _) => throw new InvalidOperationException("should not be called"), + onRetry: static (_, _, _, _) => throw new InvalidOperationException("should not be called"), + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); } @@ -347,6 +356,7 @@ await RetryExecutor.ExecuteAsync( throw new ArgumentNullException("param"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); // assert @@ -376,6 +386,7 @@ await Assert.ThrowsAsync(() => throw new InvalidOperationException("dead letter"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -402,6 +413,7 @@ await Assert.ThrowsAsync(() => throw new ArgumentException("derived"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -432,6 +444,7 @@ await Assert.ThrowsAsync(() => throw new InvalidOperationException("specific"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -453,6 +466,7 @@ await RetryExecutor.ExecuteAsync( 0, static (_) => throw new InvalidOperationException("transient failure"), onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); } @@ -472,6 +486,7 @@ await Assert.ThrowsAsync(() => 0, static (_) => throw new InvalidOperationException("permanent failure"), onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -504,6 +519,7 @@ await RetryExecutor.ExecuteAsync( throw new ArgumentException("discard this"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); // assert @@ -535,6 +551,7 @@ await RetryExecutor.ExecuteAsync( return ValueTask.CompletedTask; }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); // assert @@ -555,6 +572,7 @@ await Assert.ThrowsAsync(() => 0, static (_) => throw new InvalidOperationException("no rules"), onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -579,6 +597,7 @@ await Assert.ThrowsAsync(() => throw new InvalidOperationException("always fails"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -605,7 +624,8 @@ await Assert.ThrowsAsync(() => s.Counter.Increment(); throw new InvalidOperationException("always fails"); }, - onRetry: static (s, attempt) => s.Attempts.Add(attempt), + onRetry: static (s, _, attempt, _) => s.Attempts.Add(attempt), + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -632,6 +652,7 @@ await Assert.ThrowsAsync(() => throw new InvalidOperationException("always fails"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -658,6 +679,7 @@ public async Task ExecuteAsync_Should_ThrowOperationCanceled_When_CancellationRe throw new InvalidOperationException("fail"); }, onRetry: null, + onDiscard: null, cts.Token).AsTask(); await cts.CancelAsync(); @@ -698,6 +720,7 @@ public async Task ExecuteAsync_Should_PropagateFailure_When_ActionCancelsProcess throw failure; }, onRetry: null, + onDiscard: null, cts.Token).AsTask()); // assert @@ -727,6 +750,7 @@ await Assert.ThrowsAsync(() => throw new InvalidOperationException("fail"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -762,6 +786,7 @@ await RetryExecutor.ExecuteAsync( return ValueTask.CompletedTask; }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); // assert @@ -782,6 +807,7 @@ public async Task ExecuteAsync_Should_PropagateOriginalException_When_RetriesExh 0, static (_) => throw new InvalidOperationException("specific message"), onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -808,6 +834,7 @@ await Assert.ThrowsAsync(() => throw new ArgumentException("caught by default"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -836,6 +863,7 @@ await RetryExecutor.ExecuteAsync( throw new InvalidOperationException("discarded despite default retry"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken); Assert.Equal(1, counter.Count); @@ -864,6 +892,7 @@ await Assert.ThrowsAsync(() => throw new ArgumentException("falls to default"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -893,6 +922,7 @@ await Assert.ThrowsAsync(() => throw new InvalidOperationException("exhaust then dead letter"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); @@ -926,6 +956,7 @@ await Assert.ThrowsAsync(() => throw new InvalidOperationException("full chain"); }, onRetry: null, + onDiscard: null, cancellationToken: TestContext.Current.CancellationToken) .AsTask() ); diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Receive/ReceiveFaultMiddlewareTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Receive/ReceiveFaultMiddlewareTests.cs index f5fc829b1aa..120f323b322 100644 --- a/src/Mocha/test/Mocha.Tests/Middlewares/Receive/ReceiveFaultMiddlewareTests.cs +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Receive/ReceiveFaultMiddlewareTests.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.ObjectPool; +using Microsoft.Extensions.Logging.Abstractions; using Mocha.Events; using Mocha.Features; using Mocha.Middlewares; @@ -453,7 +454,8 @@ public async Task InvokeAsync_Should_CopyInboundHeadersToFaultReply_When_Respons var middleware = new ReceiveFaultMiddleware( TimeProvider.System, errorEndpoint: null, - new UnusedMessagingPools()); + new UnusedMessagingPools(), + NullLogger.Instance); // act await middleware.InvokeAsync( @@ -483,7 +485,8 @@ public async Task InvokeAsync_Should_UseFaultAddress_When_FaultAndResponseAddres var middleware = new ReceiveFaultMiddleware( TimeProvider.System, errorEndpoint: null, - new UnusedMessagingPools()); + new UnusedMessagingPools(), + NullLogger.Instance); // act await middleware.InvokeAsync( diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Receive/ReceiveInstrumentationMiddlewareTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Receive/ReceiveInstrumentationMiddlewareTests.cs index 90244fdb618..c9cf7e263e9 100644 --- a/src/Mocha/test/Mocha.Tests/Middlewares/Receive/ReceiveInstrumentationMiddlewareTests.cs +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Receive/ReceiveInstrumentationMiddlewareTests.cs @@ -13,6 +13,31 @@ namespace Mocha.Tests.Middlewares.Receive; [Collection("OpenTelemetry")] public sealed class ReceiveInstrumentationMiddlewareTests : ReceiveMiddlewareTestBase { + [Fact] + public void Receive_Should_RecordConversationId_When_CorrelationIdDiffers() + { + // arrange + var activities = new ConcurrentBag(); + using var listener = CreateActivityListener(activities); + var observer = new ActivityMessagingDiagnosticListener(); + var context = new StubReceiveContext + { + Endpoint = new InMemoryReceiveEndpoint(new InMemoryMessagingTransport(_ => { })), + MessageId = "message-1", + CorrelationId = "request-1", + ConversationId = "conversation-1" + }; + + // act + var activityScope = observer.Receive(context); + activityScope.Dispose(); + + // assert + var activity = Assert.Single(activities); + Assert.Equal("message-1", activity.GetTagItem("messaging.message.id")); + Assert.Equal("conversation-1", activity.GetTagItem("messaging.message.conversation_id")); + } + [Fact] public async Task InvokeAsync_Should_CallObserverReceive_When_MiddlewareInvoked() { diff --git a/src/Mocha/test/Mocha.Tests/Threading/ExponentialBackoffTests.cs b/src/Mocha/test/Mocha.Tests/Threading/ExponentialBackoffTests.cs new file mode 100644 index 00000000000..22e3ec896a4 --- /dev/null +++ b/src/Mocha/test/Mocha.Tests/Threading/ExponentialBackoffTests.cs @@ -0,0 +1,99 @@ +using Mocha.Threading; + +namespace Mocha.Tests.Threading; + +public sealed class ExponentialBackoffTests +{ + [Theory] + [InlineData(0, 2_000)] + [InlineData(1, 2_000)] + [InlineData(2, 4_000)] + [InlineData(3, 8_000)] + [InlineData(4, 10_000)] + [InlineData(5, 10_000)] + [InlineData(40, 10_000)] + [InlineData(int.MaxValue, 10_000)] + public void CalculateDelay_Should_StayBetweenHalfAndFullCeiling_When_AttemptFails( + int attempt, + int ceilingMilliseconds) + { + // arrange + var initialDelay = TimeSpan.FromSeconds(1); + var maxDelay = TimeSpan.FromSeconds(10); + + // act + var delays = Enumerable.Range(0, 100) + .Select(_ => ExponentialBackoff.CalculateDelay(attempt, initialDelay, maxDelay).TotalMilliseconds) + .ToList(); + + // assert + Assert.All(delays, delay => Assert.InRange(delay, ceilingMilliseconds / 2d, ceilingMilliseconds)); + } + + [Theory] + [InlineData(1)] + [InlineData(5)] + [InlineData(40)] + public void CalculateDelay_Should_Vary_When_AttemptIsRepeated(int attempt) + { + // arrange + var initialDelay = TimeSpan.FromSeconds(1); + var maxDelay = TimeSpan.FromSeconds(10); + + // act + var delays = Enumerable.Range(0, 100) + .Select(_ => ExponentialBackoff.CalculateDelay(attempt, initialDelay, maxDelay)) + .Distinct() + .ToList(); + + // assert + Assert.True(delays.Count > 1, "The delays are not randomized."); + } + + [Fact] + public void CalculateDelay_Should_ReturnZero_When_DelayIsZero() + { + // arrange + var maxDelay = TimeSpan.FromSeconds(10); + + // act + var delay = ExponentialBackoff.CalculateDelay(int.MaxValue, TimeSpan.Zero, maxDelay); + + // assert + Assert.Equal(TimeSpan.Zero, delay); + } + + [Fact] + public void CalculateDelay_Should_Throw_When_DelayIsNegative() + { + // arrange + var delay = TimeSpan.FromSeconds(-1); + var maxDelay = TimeSpan.FromSeconds(10); + + // act + var exception = Assert.Throws( + () => ExponentialBackoff.CalculateDelay(1, delay, maxDelay)); + + // assert + Assert.Equal( + "The delay must not be negative. (Parameter 'delay')\nActual value was -00:00:01.", + exception.Message.ReplaceLineEndings("\n")); + } + + [Fact] + public void CalculateDelay_Should_Throw_When_MaxDelayIsNegative() + { + // arrange + var delay = TimeSpan.FromSeconds(1); + var maxDelay = TimeSpan.FromSeconds(-10); + + // act + var exception = Assert.Throws( + () => ExponentialBackoff.CalculateDelay(1, delay, maxDelay)); + + // assert + Assert.Equal( + "The delay must not be negative. (Parameter 'maxDelay')\nActual value was -00:00:10.", + exception.Message.ReplaceLineEndings("\n")); + } +} diff --git a/src/Mocha/test/Mocha.Tests/Transport/MessagingTransportLifecycleTests.cs b/src/Mocha/test/Mocha.Tests/Transport/MessagingTransportLifecycleTests.cs new file mode 100644 index 00000000000..23b3a990a3f --- /dev/null +++ b/src/Mocha/test/Mocha.Tests/Transport/MessagingTransportLifecycleTests.cs @@ -0,0 +1,350 @@ +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; + +namespace Mocha.Tests; + +public sealed class MessagingTransportLifecycleTests +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(5); + + [Fact] + public async Task DisposeAsync_Should_DisposeTransport_When_EndpointStopObservesCancellation() + { + // arrange + var transport = new TestTransport(); + await using var provider = CreateProvider(transport); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(TestContext.Current.CancellationToken); + var cancelledStops = new ConcurrentQueue(); + foreach (var endpoint in transport.ReceiveEndpoints.Cast()) + { + endpoint.StopCallback = ct => + { + cancelledStops.Enqueue(ct.IsCancellationRequested); + ct.ThrowIfCancellationRequested(); + return ValueTask.CompletedTask; + }; + } + + // act + await runtime.DisposeAsync(); + + // assert + Assert.Equal([true, true, true], cancelledStops); + Assert.True(transport.IsDisposed); + Assert.False(runtime.IsStarted); + } + + [Fact] + public async Task StopAsync_Should_DrainApplicationEndpointsConcurrentlyBeforeReplies_When_EndpointsAreRunning() + { + // arrange + var transport = new TestTransport(); + await using var provider = CreateProvider(transport); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(TestContext.Current.CancellationToken); + var first = transport.GetEndpoint("first"); + var second = transport.GetEndpoint("second"); + var reply = transport.GetEndpoint("reply"); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSecond = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + first.StopCallback = async ct => await releaseFirst.Task.WaitAsync(ct); + second.StopCallback = async ct => await releaseSecond.Task.WaitAsync(ct); + + // act + var stop = transport.StopAsync(runtime, TestContext.Current.CancellationToken).AsTask(); + bool replyStayedStarted; + bool replyWaitedForEveryApplication; + try + { + await Task.WhenAll(first.StopStarted.Task, second.StopStarted.Task) + .WaitAsync(s_timeout, TestContext.Current.CancellationToken); + replyStayedStarted = reply.IsStarted && reply.StopCount == 0; + releaseFirst.TrySetResult(); + await first.StopCompleted.Task.WaitAsync(s_timeout, TestContext.Current.CancellationToken); + replyWaitedForEveryApplication = reply.IsStarted && reply.StopCount == 0; + } + finally + { + releaseFirst.TrySetResult(); + releaseSecond.TrySetResult(); + await stop.WaitAsync(s_timeout, TestContext.Current.CancellationToken); + } + + // assert + Assert.True(replyStayedStarted); + Assert.True(replyWaitedForEveryApplication); + Assert.Equal(["before", "reply", "after"], transport.Stops.Where(name => name is not "first" and not "second")); + Assert.Equal([false, false, false], transport.ReceiveEndpoints.Select(endpoint => endpoint.IsStarted)); + Assert.False(transport.IsStarted); + } + + [Theory] + [InlineData("before")] + [InlineData("first")] + [InlineData("reply")] + [InlineData("after")] + public async Task StopAsync_Should_AttemptEveryStageAndRethrowFailure_When_StageFails(string stage) + { + // arrange + var failure = new InvalidOperationException("Stop failed."); + var transport = new TestTransport(); + await using var provider = CreateProvider(transport); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(TestContext.Current.CancellationToken); + if (stage == "before") + { + transport.BeforeStopFailure = failure; + } + else if (stage == "after") + { + transport.AfterStopFailure = failure; + } + else + { + transport.GetEndpoint(stage).StopFailure = failure; + } + + // act + var exception = await Assert.ThrowsAsync( + () => transport.StopAsync(runtime, TestContext.Current.CancellationToken).AsTask()); + + // assert + Assert.Same(failure, exception); + Assert.Equal(["after", "before", "first", "reply", "second"], transport.Stops.Order()); + Assert.True(transport.IsStarted); + } + + [Fact] + public async Task StopAsync_Should_PreserveFirstFailure_When_MultipleStagesFail() + { + // arrange + var firstFailure = new InvalidOperationException("Before stop failed."); + var transport = new TestTransport(); + await using var provider = CreateProvider(transport); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(TestContext.Current.CancellationToken); + transport.BeforeStopFailure = firstFailure; + transport.AfterStopFailure = new InvalidOperationException("After stop failed."); + foreach (var endpoint in transport.ReceiveEndpoints.Cast()) + { + endpoint.StopFailure = new InvalidOperationException("Endpoint stop failed."); + } + + // act + var exception = await Assert.ThrowsAsync( + () => transport.StopAsync(runtime, TestContext.Current.CancellationToken).AsTask()); + + // assert + Assert.Same(firstFailure, exception); + Assert.Equal(["after", "before", "first", "reply", "second"], transport.Stops.Order()); + Assert.Equal([true, true, true], transport.ReceiveEndpoints.Select(endpoint => endpoint.IsStarted)); + Assert.True(transport.IsStarted); + } + + [Fact] + public async Task StartAsync_Should_StopStartedEndpointsAndRethrow_When_EndpointStartFails() + { + // arrange + var failure = new InvalidOperationException("Start failed."); + var transport = new TestTransport(); + await using var provider = CreateProvider(transport); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + // the reply endpoint and one application endpoint start, the second application endpoint fails + transport.StartFailure = (Position: 3, failure); + var stopTokensCancelled = new ConcurrentQueue(); + foreach (var endpoint in transport.ReceiveEndpoints.Cast()) + { + endpoint.StopCallback = ct => + { + stopTokensCancelled.Enqueue(ct.IsCancellationRequested); + return ValueTask.CompletedTask; + }; + } + + // act + var exception = await Assert.ThrowsAsync( + () => transport.StartAsync(runtime, TestContext.Current.CancellationToken).AsTask()); + + // assert + Assert.Same(failure, exception); + Assert.Equal(["before", transport.Starts.ElementAt(1), "reply", "after"], transport.Stops); + Assert.Equal([true, true], stopTokensCancelled); + Assert.Equal([false, false, false], transport.ReceiveEndpoints.Select(endpoint => endpoint.IsStarted)); + Assert.False(transport.IsStarted); + } + + [Fact] + public async Task StartAsync_Should_RethrowStartFailure_When_StoppingStartedEndpointsFails() + { + // arrange + var failure = new InvalidOperationException("Start failed."); + var transport = new TestTransport(); + await using var provider = CreateProvider(transport); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + transport.StartFailure = (Position: 3, failure); + transport.BeforeStopFailure = new InvalidOperationException("Before stop failed."); + transport.AfterStopFailure = new InvalidOperationException("After stop failed."); + foreach (var endpoint in transport.ReceiveEndpoints.Cast()) + { + endpoint.StopFailure = new InvalidOperationException("Endpoint stop failed."); + } + + // act + var exception = await Assert.ThrowsAsync( + () => transport.StartAsync(runtime, TestContext.Current.CancellationToken).AsTask()); + + // assert + Assert.Same(failure, exception); + Assert.Equal(["before", transport.Starts.ElementAt(1), "reply", "after"], transport.Stops); + Assert.False(transport.IsStarted); + } + + private static ServiceProvider CreateProvider(TestTransport transport) + { + var builder = new ServiceCollection().AddMessageBus(); + builder.ConfigureMessageBus(bus => bus.AddTransport(transport)); + return builder.Services.BuildServiceProvider(); + } + + private sealed class TestTransport : MessagingTransport + { + public ConcurrentQueue Starts { get; } = new(); + + public ConcurrentQueue Stops { get; } = new(); + + public (int Position, Exception Exception)? StartFailure { get; set; } + + public bool IsDisposed { get; private set; } + + public Exception? BeforeStopFailure { get; set; } + + public Exception? AfterStopFailure { get; set; } + + public override MessagingTopology Topology { get; } + + public TestTransport() + { + Topology = new TestTopology(this); + } + + public TestReceiveEndpoint GetEndpoint(string name) + => ReceiveEndpoints.Cast().Single(endpoint => endpoint.Name == name); + + public override ValueTask DisposeAsync() + { + IsDisposed = true; + return ValueTask.CompletedTask; + } + + public override bool TryGetDispatchEndpoint(Uri address, [NotNullWhen(true)] out DispatchEndpoint? endpoint) + { + endpoint = null; + return false; + } + + protected override MessagingTransportConfiguration CreateConfiguration(IMessagingSetupContext context) + => new TestTransportConfiguration + { + Name = "test", + Schema = "test", + RoutingStrategyFactory = static _ => new TestRoutingStrategy(), + ReceiveEndpoints = + [ + new ReceiveEndpointConfiguration { Name = "first" }, + new ReceiveEndpointConfiguration { Name = "second" }, + new ReceiveEndpointConfiguration { Name = "reply", Kind = ReceiveEndpointKind.Reply } + ] + }; + + protected override ReceiveEndpoint CreateReceiveEndpoint() => new TestReceiveEndpoint(this); + + protected override DispatchEndpoint CreateDispatchEndpoint() => throw new NotSupportedException(); + + protected override ValueTask OnBeforeStopAsync(CancellationToken cancellationToken) + { + Stops.Enqueue("before"); + var failure = BeforeStopFailure; + BeforeStopFailure = null; + return failure is null ? ValueTask.CompletedTask : ValueTask.FromException(failure); + } + + protected override ValueTask OnAfterStopAsync(CancellationToken cancellationToken) + { + Stops.Enqueue("after"); + var failure = AfterStopFailure; + AfterStopFailure = null; + return failure is null ? ValueTask.CompletedTask : ValueTask.FromException(failure); + } + } + + private sealed class TestReceiveEndpoint(TestTransport transport) : ReceiveEndpoint(transport) + { + public TaskCompletionSource StopStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource StopCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Func StopCallback { get; set; } = static _ => ValueTask.CompletedTask; + + public Exception? StopFailure { get; set; } + + public int StopCount { get; private set; } + + protected override void OnInitialize( + IMessagingConfigurationContext context, + ReceiveEndpointConfiguration configuration) + { } + + protected override ValueTask OnStartAsync( + IMessagingRuntimeContext context, + CancellationToken cancellationToken) + { + transport.Starts.Enqueue(Name); + + return transport.StartFailure is { } failure && failure.Position == transport.Starts.Count + ? ValueTask.FromException(failure.Exception) + : ValueTask.CompletedTask; + } + + protected override async ValueTask OnStopAsync( + IMessagingRuntimeContext context, + CancellationToken cancellationToken) + { + transport.Stops.Enqueue(Name); + StopCount++; + StopStarted.TrySetResult(); + await StopCallback(cancellationToken); + var failure = StopFailure; + StopFailure = null; + if (failure is not null) + { + throw failure; + } + + StopCompleted.TrySetResult(); + } + } + + private sealed class TestTransportConfiguration : MessagingTransportConfiguration; + + private sealed class TestTopology(MessagingTransport transport) : MessagingTopology(transport, new Uri("test:///")); + + private sealed class TestRoutingStrategy : RoutingStrategy + { + public override DispatchEndpointConfiguration? CreateEndpointConfiguration( + IMessagingConfigurationContext context, + OutboundRoute route) + => null; + + public override DispatchEndpointConfiguration? CreateEndpointConfiguration( + IMessagingConfigurationContext context, + Uri address) + => null; + + public override ReceiveEndpointConfiguration? CreateEndpointConfiguration( + IMessagingConfigurationContext context, + InboundRoute route) + => null; + } +} diff --git a/src/Mocha/test/Mocha.Transport.AzureServiceBus.Tests/Behaviors/TemporaryEndpointCleanupTests.cs b/src/Mocha/test/Mocha.Transport.AzureServiceBus.Tests/Behaviors/TemporaryEndpointCleanupTests.cs index d5e2f46e2ff..7de18fbc2f0 100644 --- a/src/Mocha/test/Mocha.Transport.AzureServiceBus.Tests/Behaviors/TemporaryEndpointCleanupTests.cs +++ b/src/Mocha/test/Mocha.Transport.AzureServiceBus.Tests/Behaviors/TemporaryEndpointCleanupTests.cs @@ -65,6 +65,64 @@ public async Task StopAsync_Should_RemoveTemporaryEndpointResources_When_Stopped """); } + [Fact] + public async Task DisposeAsync_Should_RemoveTemporaryEndpointResources_When_RuntimeIsNotStopped() + { + // arrange + var recorder = new MessageRecorder(); + await using var context = fixture.CreateTestContext(); + var queueName = context.QueueName("temporary-orders"); + await using var provider = new ServiceCollection() + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddAzureServiceBus(t => + { + t.ConnectionString(context.ConnectionString); + t.AdministrationConnectionString(context.AdminConnectionString); + t.Endpoint(queueName).Handler().Temporary(); + }) + .Services.BuildServiceProvider(); + + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(Xunit.TestContext.Current.CancellationToken); + var transport = runtime.Transports.OfType().Single(); + var topology = (AzureServiceBusMessagingTopology)transport.Topology; + var subscription = topology.Subscriptions.Single(s => s.Destination.Name == queueName); + var administrationClient = new ServiceBusAdministrationClient(context.AdminConnectionString); + using var scope = provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + + // act + var result = await TemporaryEndpointCleanupScenario.ExecuteAsync( + messageBus, + recorder, + ct => InspectResourcesAsync( + administrationClient, + queueName, + subscription.Source.Name, + subscription.Name, + ct), + _ => runtime.DisposeAsync(), + Xunit.TestContext.Current.CancellationToken); + + // assert + result.MatchInlineSnapshot( + """ + { + "MessageDelivered": true, + "BeforeStop": { + "QueueExists": true, + "BindingExists": true + }, + "AfterStop": { + "QueueExists": false, + "BindingExists": false + } + } + """); + } + private static async Task InspectResourcesAsync( ServiceBusAdministrationClient administrationClient, string queueName, diff --git a/src/Mocha/test/Mocha.Transport.Postgres.Tests/Behaviors/ShutdownTests.cs b/src/Mocha/test/Mocha.Transport.Postgres.Tests/Behaviors/ShutdownTests.cs index b383ba7649d..d1a0828f9a2 100644 --- a/src/Mocha/test/Mocha.Transport.Postgres.Tests/Behaviors/ShutdownTests.cs +++ b/src/Mocha/test/Mocha.Transport.Postgres.Tests/Behaviors/ShutdownTests.cs @@ -91,6 +91,43 @@ public async Task StopAsync_Should_RecordFailure_When_DeleteFailsAfterHandlerCom Assert.Contains("delete blocked", message.ErrorReason); } + [Fact] + public async Task DisposeAsync_Should_UnregisterConsumer_When_HostIsDisposedWithoutStop() + { + // arrange + await using var db = await fixture.CreateDatabaseAsync("dispose_unregister"); + await using var provider = CreateProvider(db.ConnectionString, new HandlerGate()); + await StartAsync(provider.GetServices().ToArray()); + var consumersBeforeDispose = await CountConsumersAsync(db.ConnectionString); + + // act + await DisposeAsync(provider); + + // assert + Assert.Equal(1, consumersBeforeDispose); + Assert.Equal(0, await CountConsumersAsync(db.ConnectionString)); + } + + [Fact] + public async Task DisposeAsync_Should_ReleaseInterruptedMessages_When_HostIsDisposedWithoutStop() + { + // arrange + await using var db = await fixture.CreateDatabaseAsync("dispose_release"); + // the handler takes a second to unwind after the stop cancelled it + var gate = new HandlerGate { DelayAfterCancellation = TimeSpan.FromSeconds(1) }; + await using var provider = CreateProvider(db.ConnectionString, gate); + await StartAsync(provider.GetServices().ToArray()); + await PublishAsync(provider, count: 2); + await gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + // act + await DisposeAsync(provider); + + // assert + Assert.True(gate.Cancelled); + Assert.Equal([(false, null, 0, false), (false, null, 0, false)], await ReadMessagesAsync(db.ConnectionString)); + } + private static ServiceProvider CreateProvider( string connectionString, HandlerGate gate) @@ -123,6 +160,9 @@ await service.StopAsync(CancellationToken.None) } } + private static Task DisposeAsync(ServiceProvider provider) + => provider.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private static async Task PublishAsync(IServiceProvider provider, int count) { using var scope = provider.CreateScope(); @@ -189,6 +229,7 @@ public sealed class HandlerGate public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public bool Cancelled { get; set; } public bool FailAfterCancellation { get; init; } + public TimeSpan DelayAfterCancellation { get; init; } } public sealed class NoOpHandler : IEventHandler @@ -209,6 +250,7 @@ public async ValueTask HandleAsync(TestEvent message, CancellationToken cancella catch (OperationCanceledException) { gate.Cancelled = true; + await Task.Delay(gate.DelayAfterCancellation, CancellationToken.None); if (gate.FailAfterCancellation) { throw new InvalidOperationException("Handler failed after shutdown cancellation."); diff --git a/src/Mocha/test/Mocha.Transport.Postgres.Tests/Behaviors/TemporaryEndpointCleanupTests.cs b/src/Mocha/test/Mocha.Transport.Postgres.Tests/Behaviors/TemporaryEndpointCleanupTests.cs index 2ab365c6d42..9a695e3bbcc 100644 --- a/src/Mocha/test/Mocha.Transport.Postgres.Tests/Behaviors/TemporaryEndpointCleanupTests.cs +++ b/src/Mocha/test/Mocha.Transport.Postgres.Tests/Behaviors/TemporaryEndpointCleanupTests.cs @@ -56,6 +56,54 @@ public async Task StopAsync_Should_RemoveTemporaryEndpointResources_When_Stopped """); } + [Fact] + public async Task DisposeAsync_Should_RemoveTemporaryEndpointResources_When_RuntimeIsNotStopped() + { + // arrange + const string queueName = "temporary-orders"; + var recorder = new MessageRecorder(); + await using var db = await fixture.CreateDatabaseAsync(); + await using var provider = new ServiceCollection() + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddPostgres(t => + { + t.ConnectionString(db.ConnectionString); + t.Endpoint(queueName).Handler().Temporary(); + }) + .Services.BuildServiceProvider(); + + var runtime = (MessagingRuntime)provider.GetRequiredService(); + await runtime.StartAsync(TestContext.Current.CancellationToken); + using var scope = provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + + // act + var result = await TemporaryEndpointCleanupScenario.ExecuteAsync( + messageBus, + recorder, + ct => InspectResourcesAsync(db.ConnectionString, queueName, ct), + _ => runtime.DisposeAsync(), + TestContext.Current.CancellationToken); + + // assert + result.MatchInlineSnapshot( + """ + { + "MessageDelivered": true, + "BeforeStop": { + "QueueExists": true, + "BindingExists": true + }, + "AfterStop": { + "QueueExists": false, + "BindingExists": false + } + } + """); + } + private static async Task InspectResourcesAsync( string connectionString, string queueName, diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/AutoProvisionTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/AutoProvisionTests.cs index e4ab6fed720..d2e6e2b865d 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/AutoProvisionTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/AutoProvisionTests.cs @@ -260,7 +260,7 @@ public void Convention_Should_PropagateAutoProvision_When_ReceiveEndpointCreated var runtime = CreateRuntime(b => b.AddRequestHandler()); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act - find the reply queue (temporary, auto-provisioned) var replyEndpoint = transport.ReceiveEndpoints @@ -285,7 +285,7 @@ public void Descriptor_AutoProvision_Should_ReturnSelf_When_Chaining() .AutoProvision(true)); // assert - the last call wins - var exchange = ((RabbitMQMessagingTopology)transport.Topology) + var exchange = transport.Topology .Exchanges.Single(e => e.Name == "ex1"); Assert.True(exchange.AutoProvision); } @@ -305,7 +305,7 @@ private static ( }) .BuildRuntime(); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; return (runtime, transport, topology); } diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/AutoProvisionIntegrationTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/AutoProvisionIntegrationTests.cs index 91f5502c74d..1867f3dce25 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/AutoProvisionIntegrationTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/AutoProvisionIntegrationTests.cs @@ -5,10 +5,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class AutoProvisionIntegrationTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public AutoProvisionIntegrationTests(RabbitMQFixture fixture) @@ -37,7 +36,7 @@ public async Task PublishAsync_Should_Deliver_When_AutoProvisionEnabledByDefault await messageBus.PublishAsync(new OrderCreated { OrderId = "AP-1" }, CancellationToken.None); // assert - message is delivered because topology was auto-provisioned - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the event"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the event"); var order = Assert.IsType(Assert.Single(recorder.Messages)); Assert.Equal("AP-1", order.OrderId); } @@ -63,7 +62,7 @@ public async Task PublishAsync_Should_Deliver_When_AutoProvisionExplicitlyEnable await messageBus.PublishAsync(new OrderCreated { OrderId = "AP-2" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the event"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the event"); var order = Assert.IsType(Assert.Single(recorder.Messages)); Assert.Equal("AP-2", order.OrderId); } @@ -89,7 +88,7 @@ public async Task SendAsync_Should_Deliver_When_AutoProvisionEnabledByDefault() await messageBus.SendAsync(new ProcessPayment { OrderId = "AP-3", Amount = 42.00m }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the request"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the request"); var payment = Assert.IsType(Assert.Single(recorder.Messages)); Assert.Equal("AP-3", payment.OrderId); } @@ -125,7 +124,7 @@ public async Task ExplicitTopology_Should_Deliver_When_AutoProvisionEnabledOnRes await messageBus.PublishAsync(new OrderCreated { OrderId = "AP-4" }, CancellationToken.None); // assert - resources were explicitly enabled, so message should be delivered - Assert.True(await capture.WaitAsync(s_timeout), "Consumer did not receive the message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message"); var message = Assert.Single(capture.Messages); Assert.Equal("AP-4", message.OrderId); } @@ -186,7 +185,7 @@ await channel.QueueBindAsync( await messageBus.PublishAsync(new OrderCreated { OrderId = "AP-5" }, CancellationToken.None); // assert - pre-provisioned resources work with auto-provision disabled - Assert.True(await capture.WaitAsync(s_timeout), "Consumer did not receive the message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message"); var message = Assert.Single(capture.Messages); Assert.Equal("AP-5", message.OrderId); } @@ -236,7 +235,7 @@ await channel.ExchangeDeclareAsync( await messageBus.PublishAsync(new OrderCreated { OrderId = "AP-6" }, CancellationToken.None); // assert - Assert.True(await capture.WaitAsync(s_timeout), "Consumer did not receive the message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message"); var message = Assert.Single(capture.Messages); Assert.Equal("AP-6", message.OrderId); } diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BatchingTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BatchingTests.cs index 9311da15505..3f916b0a79f 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BatchingTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BatchingTests.cs @@ -3,10 +3,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Messaging)] public class BatchingTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(60); private readonly RabbitMQFixture _fixture; public BatchingTests(RabbitMQFixture fixture) @@ -35,7 +34,7 @@ public async Task Handler_Should_ReceiveBatch_When_SingleMessageSizeTrigger() await messageBus.PublishAsync(new OrderCreated { OrderId = "1" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Batch handler was not invoked within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Batch handler was not invoked within timeout"); var batch = Assert.IsAssignableFrom>(Assert.Single(recorder.Batches)); Assert.Single(batch); @@ -68,7 +67,7 @@ public async Task Handler_Should_ReceiveBatch_When_TimeoutExpires() await messageBus.PublishAsync(new OrderCreated { OrderId = "timeout-1" }, CancellationToken.None); // assert - batch should arrive via timeout with 1 message - Assert.True(await recorder.WaitAsync(s_timeout), "Batch handler was not invoked via timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Batch handler was not invoked via timeout"); var batch = Assert.IsAssignableFrom>(Assert.Single(recorder.Batches)); Assert.Equal(BatchCompletionMode.Time, batch.CompletionMode); @@ -113,7 +112,7 @@ public async Task Handler_Should_ReceiveMultiMessageBatch_When_ConcurrentDeliver } // assert - single batch containing all 5 messages - Assert.True(await recorder.WaitAsync(s_timeout), "Batch handler was not invoked within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Batch handler was not invoked within timeout"); var batch = Assert.IsAssignableFrom>(Assert.Single(recorder.Batches)); Assert.Equal(messageCount, batch.Count); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BrokerRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BrokerRecoveryTests.cs new file mode 100644 index 00000000000..328ec6c71ee --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BrokerRecoveryTests.cs @@ -0,0 +1,423 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class BrokerRecoveryTests(RabbitMQFixture fixture) +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Bus_Should_RedeliverAndResumePublishing_When_BrokerProcessIsKilled(bool automaticRecoveryEnabled) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.AutomaticRecoveryEnabled = automaticRecoveryEnabled; + var gate = new CrashHandlerGate(); + var recorder = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRequestHandler() + .AddRabbitMQ(t => + { + t.Endpoint("orders").Handler(); + t.Endpoint("order-status").Handler(); + }) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(45)); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.PublishAsync(new OrderCreated { OrderId = "before-crash" }, deadline.Token); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, deadline.Token); + + // act + Task publish; + await using (await fixture.Faults.KillBrokerAsync()) + { + await gate.Cancelled.Task.WaitAsync(TestTimeouts.Signal, deadline.Token); + publish = messageBus.PublishAsync(new OrderCreated { OrderId = "during-outage" }, deadline.Token).AsTask(); + await Assert.ThrowsAsync(() => publish.WaitAsync(TimeSpan.FromMilliseconds(200), deadline.Token)); + } + + await publish.WaitAsync(TestTimeouts.Signal, deadline.Token); + var recovered = await Condition.WaitAsync( + () => recorder.Messages.Cast().Select(m => m.OrderId).Distinct().Count() == 2, + TestTimeouts.Signal); + var response = await messageBus.RequestAsync(new GetOrderStatus { OrderId = "after-crash" }, deadline.Token); + var settled = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders"))?.Messages == 0, + TestTimeouts.Signal); + + // assert + Assert.True(recovered); + Assert.Equal(["before-crash", "during-outage"], + recorder.Messages.Cast().Select(m => m.OrderId).Distinct().Order()); + Assert.Equal("after-crash", response.OrderId); + Assert.True(settled); + } + + [Fact] + public async Task Bus_Should_DeliverEventsAndReplies_When_BrokerClosesConsumerConnection() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + var before = await WarmUpAsync(vhost, bus, recorder); + + // act + await vhost.Faults.CloseConsumerConnectionAsync("orders"); + await AssertRecoveredAsync(vhost, bus, recorder, before.Consumers, "ORD-2"); + + // assert + await AssertBrokerResourcesAsync(vhost, before.State); + } + + [Fact] + public async Task Bus_Should_DeliverEventsAndReplies_When_BrokerRestarts() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.NetworkRecoveryInterval = TimeSpan.FromSeconds(1); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + var before = await WarmUpAsync(vhost, bus, recorder); + + // act + var broker = await fixture.Faults.StopBrokerAsync(); + await broker.DisposeAsync(); + await AssertRecoveredAsync(vhost, bus, recorder, before.Consumers, "ORD-2"); + + // assert + await AssertBrokerResourcesAsync(vhost, before.State); + } + + [Fact] + public async Task Bus_Should_DeliverEventsAndReplies_When_HeartbeatsAreMissed() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedHeartbeat = TimeSpan.FromSeconds(1); + vhost.ConnectionSettings.NetworkRecoveryInterval = TimeSpan.FromSeconds(1); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + var before = await WarmUpAsync(vhost, bus, recorder); + + var connection = await bus.Transport.ConsumerManager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + + // act + // the broker stops sending heartbeats, so the client gives up on its connections after three intervals + bool detected; + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + detected = await Condition.WaitAsync(() => !connection.IsOpen, TestTimeouts.Signal); + } + + await AssertRecoveredAsync(vhost, bus, recorder, before.Consumers, "ORD-2"); + + // assert + Assert.True(detected); + await AssertBrokerResourcesAsync(vhost, before.State); + } + + [Fact] + public async Task DisposeAsync_Should_LeaveNoConnection_When_BrokerUnreachableDuringRecovery() + { + // arrange + // a recovery waits at most ten seconds between two attempts to connect + var longestReconnectDelay = TimeSpan.FromSeconds(10); + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.NetworkRecoveryInterval = TimeSpan.FromSeconds(1); + var recorder = new MessageRecorder(); + var bus = await CreateBusAsync(vhost, recorder); + var transport = bus.Transport; + await WarmUpAsync(vhost, bus, recorder); + + // act + await using (await fixture.Faults.StopBrokerAsync()) + { + await Wait.UntilAsync(() => !transport.ConsumerManager.IsConnected); + await bus.DisposeAsync().AsTask().WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + + // nothing signals that a disposed bus will never connect again, so this watches for a connection for + // longer than a recovery that outlived the dispose would wait before its next attempt + var reconnected = await Condition.WaitAsync( + async () => await vhost.Broker.GetConnectionCountAsync() > 0, + longestReconnectDelay + TimeSpan.FromSeconds(2)); + + // assert + Assert.False(reconnected); + } + + [Fact] + public async Task Bus_Should_Reconnect_When_PasswordRotated() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var credentials = new Credentials($"{vhost.VhostName}-rotating", "first-password"); + await using var user = await vhost.Faults.AddUserAsync(credentials.UserName, credentials.Password); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder, new UserConnectionProvider(vhost, credentials)); + var before = await WarmUpAsync(vhost, bus, recorder); + + // act + // open connections keep the old password, and the provider supplies the new one for every new connection + await vhost.Faults.ChangePasswordAsync(credentials.UserName, "second-password"); + credentials.Password = "second-password"; + await vhost.Faults.CloseConnectionsAsync(); + await AssertRecoveredAsync(vhost, bus, recorder, before.Consumers, "ORD-2"); + + // assert + await AssertBrokerResourcesAsync(vhost, before.State); + } + + [Fact] + public async Task Bus_Should_Reconnect_When_UserDeletedAndCreatedAgain() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var credentials = new Credentials($"{vhost.VhostName}-deleted", "password"); + await using var user = await vhost.Faults.AddUserAsync(credentials.UserName, credentials.Password); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder, new UserConnectionProvider(vhost, credentials)); + var before = await WarmUpAsync(vhost, bus, recorder); + + // act + // deleting the user closes its connections, and reconnects fail with an authentication error until the + // user exists again + await vhost.Faults.DeleteUserAsync(credentials.UserName); + var lost = await Condition.WaitAsync( + async () => await vhost.Broker.GetConnectionCountAsync() == 0, + TestTimeouts.Signal); + await using var recreated = await vhost.Faults.AddUserAsync(credentials.UserName, credentials.Password); + await AssertRecoveredAsync(vhost, bus, recorder, before.Consumers, "ORD-2"); + + // assert + Assert.True(lost); + await AssertBrokerResourcesAsync(vhost, before.State); + } + + [Fact] + public async Task Bus_Should_DeliverEventsAndReplies_When_BrokerRestartsRepeatedly() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.NetworkRecoveryInterval = TimeSpan.FromSeconds(1); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + var before = await WarmUpAsync(vhost, bus, recorder); + + // act + var consumers = before.Consumers; + for (var i = 0; i < 3; i++) + { + var broker = await fixture.Faults.StopBrokerAsync(); + await broker.DisposeAsync(); + consumers = await AssertRecoveredAsync(vhost, bus, recorder, consumers, $"ORD-{i + 2}"); + } + + // assert + await AssertBrokerResourcesAsync(vhost, before.State); + } + + [Fact] + public async Task Bus_Should_Resume_When_NodeDrainedAndRevived() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + var before = await WarmUpAsync(vhost, bus, recorder); + + // act + // maintenance mode closes every client connection with 320 and refuses new ones until the node is revived + bool closed; + await using (await fixture.Faults.DrainNodeAsync()) + { + closed = await Condition.WaitAsync( + async () => await vhost.Broker.GetConnectionCountAsync() == 0, + TestTimeouts.Signal); + } + + await AssertRecoveredAsync(vhost, bus, recorder, before.Consumers, "ORD-2"); + + // assert + Assert.True(closed); + await AssertBrokerResourcesAsync(vhost, before.State); + } + + [Fact] + public async Task Bus_Should_NotLeakConsumerResources_When_ConsumerConnectionClosesRepeatedly() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + var before = await WarmUpAsync(vhost, bus, recorder); + + // act + var consumers = before.Consumers; + for (var i = 0; i < 5; i++) + { + await vhost.Faults.CloseConsumerConnectionAsync("orders"); + consumers = await AssertRecoveredAsync(vhost, bus, recorder, consumers, $"ORD-{i + 2}"); + await AssertBrokerResourcesAsync(vhost, before.State); + } + + await bus.DisposeAsync(); + + // assert + await vhost.Broker.WaitForNoConnectionAsync(); + } + + private static Task CreateBusAsync( + VhostContext vhost, + MessageRecorder recorder, + IRabbitMQConnectionProvider? connectionProvider = null) + => CreateServices(vhost, recorder, connectionProvider).BuildTestBusAsync(); + + private static IMessageBusHostBuilder CreateServices( + VhostContext vhost, + MessageRecorder recorder, + IRabbitMQConnectionProvider? connectionProvider = null) + => new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRequestHandler() + .AddRabbitMQ(t => + { + if (connectionProvider is not null) + { + t.ConnectionProvider(_ => connectionProvider); + } + + t.Endpoint("orders").Handler(); + t.Endpoint("order-status").Handler(); + }); + + /// + /// Waits until both endpoints have a consumer that is not one of , sends an event and + /// a request through the bus, and returns the new consumers. + /// + private static async Task> AssertRecoveredAsync( + VhostContext vhost, + TestBus bus, + MessageRecorder recorder, + IReadOnlyList lost, + string orderId) + { + var orders = await vhost.Broker.WaitForConsumerAsync( + "orders", + otherThan: lost.Single(consumer => consumer.Queue == "orders")); + var orderStatus = await vhost.Broker.WaitForConsumerAsync( + "order-status", + otherThan: lost.Single(consumer => consumer.Queue == "order-status")); + await AssertRoundTripAsync(bus, recorder, orderId); + + return [orders, orderStatus]; + } + + private static async Task WarmUpAsync(VhostContext vhost, TestBus bus, MessageRecorder recorder) + { + await AssertRoundTripAsync(bus, recorder, "ORD-1"); + return new Baseline(await vhost.Broker.GetStateAsync(), await vhost.Broker.GetConsumersAsync()); + } + + private static async Task AssertRoundTripAsync(TestBus bus, MessageRecorder recorder, string orderId) + { + using var scope = bus.Provider.CreateScope(); + using var deadline = CreateTimeout(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.PublishAsync(new OrderCreated { OrderId = orderId }, deadline.Token); + await Wait.UntilAsync(() => recorder.Messages.Cast().Any(order => order.OrderId == orderId)); + var response = await messageBus.RequestAsync(new GetOrderStatus { OrderId = orderId }, deadline.Token); + Assert.Equal(orderId, response.OrderId); + Assert.Equal("Shipped", response.Status); + } + + private static async Task AssertBrokerResourcesAsync(VhostContext vhost, BrokerState expected) + => await Wait.UntilAsync( + vhost.Broker.GetStateAsync, + state => state with { IdleChannels = expected.IdleChannels } == expected, + $"the broker to hold the connections, consumer channels and consumers of {expected}"); + + private static CancellationTokenSource CreateTimeout() + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TestTimeouts.Signal); + return cts; + } + + private sealed record Baseline(BrokerState State, IReadOnlyList Consumers); + + public sealed class CrashHandlerGate + { + public int Invocations; + + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Cancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class CrashOrderHandler(CrashHandlerGate gate, MessageRecorder recorder) : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref gate.Invocations) == 1) + { + gate.Started.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + gate.Cancelled.TrySetResult(); + throw; + } + } + + recorder.Record(message); + } + } + + private sealed class Credentials(string userName, string password) + { + public string UserName => userName; + + public string Password { get; set; } = password; + } + + private sealed class UserConnectionProvider(VhostContext vhost, Credentials credentials) + : IRabbitMQConnectionProvider + { + public string Host => vhost.ConnectionSettings.HostName; + + public string VirtualHost => vhost.VhostName; + + public int Port => vhost.ConnectionSettings.Port; + + public async ValueTask CreateAsync(CancellationToken cancellationToken) + { + var factory = new ConnectionFactory + { + Uri = vhost.ConnectionSettings.Uri, + VirtualHost = vhost.VhostName, + UserName = credentials.UserName, + Password = credentials.Password + }; + + return await factory.CreateConnectionAsync(cancellationToken); + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BusDefaultsIntegrationTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BusDefaultsIntegrationTests.cs index a25d3a8f0a4..15f3c6504da 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BusDefaultsIntegrationTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/BusDefaultsIntegrationTests.cs @@ -4,10 +4,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class BusDefaultsIntegrationTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; private readonly ITestOutputHelper _output; @@ -65,7 +64,7 @@ public async Task ConfigureDefaults_Should_DeliverMessages_When_QuorumQueuesUsed await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-QUORUM" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the event within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the event within timeout"); var message = Assert.Single(recorder.Messages); var order = Assert.IsType(message); @@ -134,7 +133,7 @@ public async Task ConfigureDefaults_Should_DeliverMessages_When_CustomDefaultsAp await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-DEFAULTS" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the event within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the event within timeout"); var message = Assert.Single(recorder.Messages); var order = Assert.IsType(message); @@ -207,7 +206,7 @@ public async Task ConfigureDefaults_Should_NotOverrideExplicitQueue_When_Message await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-OVERRIDE" }, CancellationToken.None); // assert - Assert.True(await capture.WaitAsync(s_timeout), "Consumer did not receive the message within timeout"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message within timeout"); var message = Assert.Single(capture.Messages); Assert.Equal("ORD-OVERRIDE", message.OrderId); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConcurrencyLimiterTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConcurrencyLimiterTests.cs index e6b87404c14..da6de7b805c 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConcurrencyLimiterTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConcurrencyLimiterTests.cs @@ -3,10 +3,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Messaging)] public class ConcurrencyLimiterTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(60); private readonly RabbitMQFixture _fixture; public ConcurrencyLimiterTests(RabbitMQFixture fixture) @@ -44,7 +43,7 @@ public async Task Handler_Should_LimitConcurrency_When_ConcurrencyLimiterConfigu // assert - wait for all messages and check peak concurrency Assert.True( - await recorder.WaitAsync(s_timeout, expectedCount: messageCount), + await recorder.WaitAsync(TestTimeouts.Signal, expectedCount: messageCount), $"Handler did not process all {messageCount} messages within timeout"); Assert.Equal(1, tracker.PeakConcurrency); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConcurrencyTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConcurrencyTests.cs index 294cb7224c7..289d4152dfb 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConcurrencyTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConcurrencyTests.cs @@ -1,12 +1,12 @@ using Microsoft.Extensions.DependencyInjection; using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Messaging)] public class ConcurrencyTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(60); private readonly RabbitMQFixture _fixture; public ConcurrencyTests(RabbitMQFixture fixture) @@ -43,7 +43,7 @@ public async Task Handler_Should_LimitConcurrency_When_MaxConcurrencySetToOne() // assert Assert.True( - await recorder.WaitAsync(s_timeout, expectedCount: messageCount), + await recorder.WaitAsync(TestTimeouts.Signal, expectedCount: messageCount), $"Handler did not process all {messageCount} messages within timeout"); Assert.Equal(1, tracker.PeakConcurrency); @@ -78,13 +78,57 @@ public async Task Handler_Should_AllowParallelism_When_MaxConcurrencyGreaterThan // assert Assert.True( - await recorder.WaitAsync(s_timeout, expectedCount: messageCount), + await recorder.WaitAsync(TestTimeouts.Signal, expectedCount: messageCount), $"Handler did not process all {messageCount} messages within timeout"); Assert.True(tracker.PeakConcurrency > 1, $"Expected parallelism > 1, but peak was {tracker.PeakConcurrency}"); Assert.True(tracker.PeakConcurrency <= 5, $"Expected peak concurrency <= 5, but was {tracker.PeakConcurrency}"); } + [Fact] + public async Task Handler_Should_LimitConcurrency_When_BacklogIsProcessedAfterRecovery() + { + // arrange + var timeout = TestTimeouts.Signal; + const int messageCount = 10; + var tracker = new ConcurrencyTracker(); + var recorder = new MessageRecorder(); + await using var vhost = await _fixture.CreateVhostAsync(); + ((ConnectionFactory)vhost.ConnectionFactory).NetworkRecoveryInterval = TimeSpan.FromSeconds(2); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(tracker) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("recovery-orders") + .Handler() + .MaxConcurrency(1) + .MaxPrefetch(1)) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.PublishAsync(new OrderCreated { OrderId = "before" }, TestContext.Current.CancellationToken); + Assert.True(await recorder.WaitAsync(timeout)); + + // act + await using (await vhost.Faults.DenyDeclaresAsync()) + { + await vhost.Faults.CloseConsumerConnectionAsync("recovery-orders"); + await vhost.Broker.WaitForNoConsumerAsync("recovery-orders"); + for (var i = 0; i < messageCount; i++) + { + await messageBus.PublishAsync( + new OrderCreated { OrderId = $"ORD-{i}" }, + TestContext.Current.CancellationToken); + } + } + + // assert + Assert.True(await recorder.WaitAsync(timeout, expectedCount: messageCount)); + Assert.Equal(1, tracker.PeakConcurrency); + } + public sealed class SlowOrderHandler(ConcurrencyTracker tracker, MessageRecorder recorder) : IEventHandler { diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConnectionRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConnectionRecoveryTests.cs index 3abada90048..a76ba9dd72c 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConnectionRecoveryTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConnectionRecoveryTests.cs @@ -3,11 +3,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.ConsumerRecovery)] public class ConnectionRecoveryTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); - private static readonly TimeSpan s_recoveryTimeout = TimeSpan.FromSeconds(15); private readonly RabbitMQFixture _fixture; public ConnectionRecoveryTests(RabbitMQFixture fixture) @@ -28,22 +26,22 @@ public async Task Consumer_Should_ResumeReceiving_When_ConnectionRecovered() .AddEventHandler() .AddRabbitMQ() .BuildTestBusAsync(); + var queue = GetQueueName(bus); using var scope = bus.Provider.CreateScope(); var messageBus = scope.ServiceProvider.GetRequiredService(); - - // act - publish message 1, verify received await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-1" }, TestContext.Current.CancellationToken); - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive message 1"); - - // drop all connections - await _fixture.CloseAllConnectionsAsync("recovery-test"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive message 1"); - // wait for reconnection with polling - await WaitForRecoveryAsync(messageBus, recorder, expectedCount: 2); + // act + await DropConnectionsAsync(vhost, queue); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-2" }, TestContext.Current.CancellationToken); + var received = await recorder.WaitAsync(TestTimeouts.Signal); + await Wait.UntilAsync(() => bus.Transport.IsHealthy); - // assert - message 2 should have been received after recovery - Assert.True(recorder.Messages.Count >= 2, $"Expected at least 2 messages but got {recorder.Messages.Count}"); + // assert + Assert.True(received, "Handler did not receive message 2"); + Assert.Equal(["ORD-1", "ORD-2"], GetOrderIds(recorder)); } [Fact] @@ -59,24 +57,25 @@ public async Task Consumer_Should_HandleMultipleDisconnects_When_ConnectionDropp .AddEventHandler() .AddRabbitMQ() .BuildTestBusAsync(); + var queue = GetQueueName(bus); using var scope = bus.Provider.CreateScope(); var messageBus = scope.ServiceProvider.GetRequiredService(); - - // act - publish message 1 await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-1" }, TestContext.Current.CancellationToken); - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive message 1"); - - // drop connections twice - await _fixture.CloseAllConnectionsAsync("recovery-test-1"); - await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); - await _fixture.CloseAllConnectionsAsync("recovery-test-2"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive message 1"); - // wait for recovery and verify message 2 - await WaitForRecoveryAsync(messageBus, recorder, expectedCount: 2); + // act + await DropConnectionsAsync(vhost, queue); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-2" }, TestContext.Current.CancellationToken); + var receivedAfterFirstDrop = await recorder.WaitAsync(TestTimeouts.Signal); + await DropConnectionsAsync(vhost, queue); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-3" }, TestContext.Current.CancellationToken); + var receivedAfterSecondDrop = await recorder.WaitAsync(TestTimeouts.Signal); // assert - Assert.True(recorder.Messages.Count >= 2, $"Expected at least 2 messages but got {recorder.Messages.Count}"); + Assert.True(receivedAfterFirstDrop, "Handler did not receive message 2"); + Assert.True(receivedAfterSecondDrop, "Handler did not receive message 3"); + Assert.Equal(["ORD-1", "ORD-2", "ORD-3"], GetOrderIds(recorder)); } [Fact] @@ -92,79 +91,53 @@ public async Task ChannelPool_Should_RecoverGracefully_When_ConnectionLost() .AddRequestHandler() .AddRabbitMQ() .BuildTestBusAsync(); + var queue = GetQueueName(bus); using var scope = bus.Provider.CreateScope(); var messageBus = scope.ServiceProvider.GetRequiredService(); - - // send message 1 await messageBus.SendAsync( new ProcessPayment { OrderId = "ORD-1", Amount = 50.00m }, TestContext.Current.CancellationToken); - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive message 1"); - - // drop connections - await _fixture.CloseAllConnectionsAsync("channel-pool-recovery"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive message 1"); + var dispatcherConnection = await bus.Transport.Dispatcher.Connection.GetConnectionAsync( + TestContext.Current.CancellationToken); + var dispatcherConnectionLost = dispatcherConnection.WhenShutdownAsync(); - // wait for recovery, then send again - await WaitForDispatchRecoveryAsync(messageBus, recorder, expectedCount: 2); + // act + await DropConnectionsAsync(vhost, queue); + await dispatcherConnectionLost.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await messageBus.SendAsync( + new ProcessPayment { OrderId = "ORD-2", Amount = 1.00m }, + TestContext.Current.CancellationToken); + var received = await recorder.WaitAsync(TestTimeouts.Signal); // assert - Assert.True(recorder.Messages.Count >= 2, $"Expected at least 2 messages but got {recorder.Messages.Count}"); + Assert.True(received, "Handler did not receive message 2"); + Assert.Equal(["ORD-1", "ORD-2"], recorder.Messages.Cast().Select(p => p.OrderId).Order()); } - private static async Task WaitForRecoveryAsync(IMessageBus messageBus, MessageRecorder recorder, int expectedCount) + /// + /// Closes every connection to the vhost once no delivery of is in flight, and returns + /// when the queue has a consumer that attached after the close. + /// + private static async Task DropConnectionsAsync(VhostContext vhost, string queue) { - var deadline = DateTimeOffset.UtcNow.Add(s_recoveryTimeout); - while (DateTimeOffset.UtcNow < deadline) - { - try - { - await messageBus.PublishAsync( - new OrderCreated { OrderId = $"ORD-{expectedCount}" }, - default); - - if (await recorder.WaitAsync(TimeSpan.FromSeconds(2), expectedCount)) - { - return; - } - } - catch - { - // connection may not be recovered yet - } - - await Task.Delay(TimeSpan.FromSeconds(1)); - } - } + // a handler records its message before the acknowledgement reaches the broker + await vhost.Broker.WaitForEmptyQueueAsync(queue); + var before = await vhost.Broker.GetConsumerAsync(queue); - private static async Task WaitForDispatchRecoveryAsync( - IMessageBus messageBus, - MessageRecorder recorder, - int expectedCount) - { - var deadline = DateTimeOffset.UtcNow.Add(s_recoveryTimeout); - while (DateTimeOffset.UtcNow < deadline) - { - try - { - await messageBus.SendAsync( - new ProcessPayment { OrderId = $"ORD-{expectedCount}", Amount = 1.00m }, - default); - - if (await recorder.WaitAsync(TimeSpan.FromSeconds(2), expectedCount)) - { - return; - } - } - catch - { - // connection may not be recovered yet - } - - await Task.Delay(TimeSpan.FromSeconds(1)); - } + await vhost.Faults.CloseConnectionsAsync(); + await vhost.Broker.WaitForConsumerAsync(queue, otherThan: before); } + private static string GetQueueName(TestBus bus) + => bus.Transport.ReceiveEndpoints.OfType() + .Single(endpoint => endpoint.Kind == ReceiveEndpointKind.Default) + .Queue.Name; + + private static IEnumerable GetOrderIds(MessageRecorder recorder) + => recorder.Messages.Cast().Select(order => order.OrderId).Order(); + public sealed class PaymentHandler(MessageRecorder recorder) : IEventRequestHandler { public ValueTask HandleAsync(ProcessPayment request, CancellationToken cancellationToken) diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConnectionStartupTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConnectionStartupTests.cs new file mode 100644 index 00000000000..dbfde68a2dd --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConnectionStartupTests.cs @@ -0,0 +1,321 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class ConnectionStartupTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task StartAsync_Should_Complete_When_BrokerBecomesReachableDuringStart() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + var broker = await fixture.Faults.StopBrokerAsync(); + Task start; + + // act + try + { + start = CreateBusAsync(vhost, recorder); + await Assert.ThrowsAsync( + () => start.WaitAsync(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken)); + } + finally + { + await broker.DisposeAsync(); + } + + await using var bus = await start.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + await AssertRoundTripAsync(bus, recorder, "ORD-1"); + } + + [Fact] + public async Task StartAsync_Should_FailFast_When_VirtualHostAccessRefused() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var revoked = await vhost.Faults.RevokeAccessAsync(); + + // act + var exception = await Assert.ThrowsAsync( + () => StartAndDisposeAsync(CreateServices(vhost, recorder))); + + // assert + Assert.Equal(Constants.NotAllowed, exception.ShutdownReason?.ReplyCode); + } + + [Fact] + public async Task StartAsync_Should_FailFast_When_CredentialsRefused() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.Password = "wrong-password"; + var recorder = new MessageRecorder(); + var builder = CreateServices(vhost, recorder); + + // act + var exception = await Assert.ThrowsAsync( + () => StartAndDisposeAsync(builder)); + + // assert + Assert.IsType(exception.InnerException); + Assert.Equal(0, await vhost.Broker.GetConnectionCountAsync()); + } + + [Fact] + public async Task StartAsync_Should_FailFastWithoutLeakingConnections_When_ConfigurePermissionDenied() + { + // arrange + // the user may open the vhost and publish, but declares nothing + await using var vhost = await fixture.CreateVhostAsync(); + await vhost.Faults.SetPermissionsAsync("guest", configure: "^$", write: ".*", read: ".*"); + var recorder = new MessageRecorder(); + + // act + var exception = await Assert.ThrowsAsync( + () => StartAndDisposeAsync(CreateServices(vhost, recorder))); + + // assert + Assert.Equal(Constants.AccessRefused, exception.ShutdownReason?.ReplyCode); + Assert.True(await Condition.WaitAsync(async () => await vhost.Broker.GetConnectionCountAsync() == 0, TestTimeouts.Signal)); + } + + [Fact] + public async Task StartAsync_Should_Fail_When_BrokerUnreachableForConnectTimeout() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.HostName = "mocha-unreachable.invalid"; + await using var services = new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(new MessageRecorder()) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => + { + t.ConnectTimeout(TimeSpan.FromSeconds(3)); + t.Endpoint("orders").Handler(); + }) + .Services + .BuildServiceProvider(); + var runtime = (MessagingRuntime)services.GetRequiredService(); + + // act + await Assert.ThrowsAsync( + () => runtime.StartAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + + // assert + Assert.False(runtime.IsStarted); + Assert.Equal(0, await vhost.Broker.GetConnectionCountAsync()); + } + + [Fact] + public async Task StartAsync_Should_Complete_When_ConnectTimeoutIsZero() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + + // act + // a timeout of zero allows one attempt and no retry + await using var bus = await CreateServices(vhost, recorder, connectTimeout: TimeSpan.Zero).BuildTestBusAsync(); + + // assert + await AssertRoundTripAsync(bus, recorder, "ORD-1"); + } + + [Fact] + public async Task StartAsync_Should_FailFast_When_BrokerRefusesOneOfTheTwoConnections() + { + // arrange + // the broker completes the handshake of a connection above the limit and refuses it afterwards. It + // counts a connection with a delay, so the second connection waits until the first one is counted + await using var vhost = await fixture.CreateVhostAsync(); + await using var limit = await vhost.Faults.LimitConnectionsAsync(1); + var recorder = new MessageRecorder(); + var connectionProvider = new SequentialConnectionProvider(vhost); + + // act + var exception = await Assert.ThrowsAnyAsync( + () => StartAndDisposeAsync(CreateServices(vhost, recorder, connectionProvider))); + + // assert + Assert.Equal(Constants.NotAllowed, exception.ShutdownReason?.ReplyCode); + await vhost.Broker.WaitForNoConnectionAsync(); + } + + [Fact] + public async Task Connections_Should_BeNamedByRole_When_Started() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + + // act + var names = await vhost.Broker.GetConnectionNamesAsync(); + + // assert + Assert.Equal(["consumer", "dispatcher"], names.Select(name => name[(name.LastIndexOf(' ') + 1)..])); + } + + [Fact] + public async Task StartAsync_Should_Connect_When_FirstNodeUnreachable() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + var connectionProvider = new ClusterConnectionProvider(vhost); + + // act + await using var bus = await CreateBusAsync(vhost, recorder, connectionProvider); + + // assert + Assert.Equal( + [ + "mocha-unreachable.invalid", vhost.ConnectionSettings.HostName, + "mocha-unreachable.invalid", vhost.ConnectionSettings.HostName + ], + connectionProvider.ConnectionAttempts.SelectMany(attempt => attempt)); + await AssertRoundTripAsync(bus, recorder, "ORD-1"); + } + + private static Task CreateBusAsync( + VhostContext vhost, + MessageRecorder recorder, + IRabbitMQConnectionProvider? connectionProvider = null) + => CreateServices(vhost, recorder, connectionProvider).BuildTestBusAsync(); + + private static IMessageBusHostBuilder CreateServices( + VhostContext vhost, + MessageRecorder recorder, + IRabbitMQConnectionProvider? connectionProvider = null, + TimeSpan? connectTimeout = null) + => new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRequestHandler() + .AddRabbitMQ(t => + { + if (connectionProvider is not null) + { + t.ConnectionProvider(_ => connectionProvider); + } + + if (connectTimeout is { } timeout) + { + t.ConnectTimeout(timeout); + } + + t.Endpoint("orders").Handler(); + t.Endpoint("order-status").Handler(); + }); + + private static async Task StartAndDisposeAsync(IMessageBusHostBuilder builder) + { + await using var services = builder.Services.BuildServiceProvider(); + var runtime = (MessagingRuntime)services.GetRequiredService(); + await runtime.StartAsync(TestContext.Current.CancellationToken) + .AsTask().WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + + private static async Task AssertRoundTripAsync(TestBus bus, MessageRecorder recorder, string orderId) + { + using var scope = bus.Provider.CreateScope(); + using var deadline = CreateTimeout(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.PublishAsync(new OrderCreated { OrderId = orderId }, deadline.Token); + Assert.True(await Condition.WaitAsync( + () => recorder.Messages.Cast().Any(order => order.OrderId == orderId), + TestTimeouts.Signal)); + var response = await messageBus.RequestAsync(new GetOrderStatus { OrderId = orderId }, deadline.Token); + Assert.Equal(orderId, response.OrderId); + Assert.Equal("Shipped", response.Status); + } + + private static CancellationTokenSource CreateTimeout() + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TestTimeouts.Signal); + return cts; + } + + private sealed class ClusterConnectionProvider(VhostContext vhost) : IRabbitMQConnectionProvider + { + public ConcurrentQueue ConnectionAttempts { get; } = new(); + + public string Host => vhost.ConnectionSettings.HostName; + + public string VirtualHost => vhost.VhostName; + + public int Port => vhost.ConnectionSettings.Port; + + public async ValueTask CreateAsync(CancellationToken cancellationToken) + { + var factory = new ConnectionFactory { Uri = vhost.ConnectionSettings.Uri, VirtualHost = vhost.VhostName }; + AmqpTcpEndpoint[] nodes = + [ + new("mocha-unreachable.invalid", vhost.ConnectionSettings.Port), + new(vhost.ConnectionSettings.HostName, vhost.ConnectionSettings.Port) + ]; + + var resolver = new OrderedEndpointResolver(nodes); + var connection = await factory.CreateConnectionAsync(resolver, null, cancellationToken); + ConnectionAttempts.Enqueue(resolver.AttemptedHosts.ToArray()); + return connection; + } + } + + private sealed class SequentialConnectionProvider(VhostContext vhost) : IRabbitMQConnectionProvider + { + private readonly SemaphoreSlim _gate = new(1, 1); + private int _connections; + + public string Host => vhost.ConnectionSettings.HostName; + + public string VirtualHost => vhost.VhostName; + + public int Port => vhost.ConnectionSettings.Port; + + public async ValueTask CreateAsync(CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken); + try + { + await vhost.Broker.WaitForCountedConnectionsAsync(_connections); + var connection = await vhost.ConnectionFactory.CreateConnectionAsync(cancellationToken); + _connections++; + return connection; + } + finally + { + _gate.Release(); + } + } + } + + private sealed class OrderedEndpointResolver(AmqpTcpEndpoint[] endpoints) : IEndpointResolver + { + public List AttemptedHosts { get; } = []; + + public IEnumerable All() + { + foreach (var endpoint in endpoints) + { + AttemptedHosts.Add(endpoint.HostName); + yield return endpoint; + } + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConsumerLifecycleTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConsumerLifecycleTests.cs new file mode 100644 index 00000000000..8be196a30ce --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConsumerLifecycleTests.cs @@ -0,0 +1,137 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.Messaging)] +public sealed class ConsumerLifecycleTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task Consumer_Should_Recover_When_ReplyEndpointIsStopped() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + var transport = runtime.Transports.OfType().Single(); + var reply = transport.ReplyReceiveEndpoint!; + await reply.StopAsync(runtime, TestContext.Current.CancellationToken); + var before = await vhost.Broker.GetConsumerAsync("orders"); + + // act + // an endpoint attaches to a new connection by itself, whatever the state of the reply endpoint is + await vhost.Faults.CloseConsumerConnectionAsync("orders"); + await vhost.Broker.WaitForConsumerAsync("orders", otherThan: before); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + var received = await orders.WaitAsync(TestTimeouts.Signal); + + // assert + Assert.True(received); + Assert.Equal(["ORD-1"], orders.Messages.Cast().Select(order => order.OrderId)); + Assert.Null(transport.ConsumerManager.HealthFailureReason); + Assert.False(reply.IsStarted); + } + + [Fact] + public async Task StopAsync_Should_Complete_When_HandlerIgnoresCancellation() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var gate = new HandlerGate(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + + try + { + var runtime = bus.Provider.GetRequiredService(); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + using var stopTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + var stop = ((MessagingRuntime)runtime).StopAsync(stopTimeout.Token).AsTask(); + await stop.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var requeued = await Condition.WaitAsync( + async () => await vhost.Broker.GetQueueAsync("orders") is { Messages: 1, Consumers: 0 }, + TestTimeouts.Signal); + + // assert + Assert.True(requeued); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task StartAsync_Should_ConsumeOnce_When_TransportStoppedBefore() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + var transport = runtime.Transports.OfType().Single(); + + // act + await transport.StopAsync(runtime, TestContext.Current.CancellationToken); + await transport.StartAsync(runtime, TestContext.Current.CancellationToken); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + var received = await orders.WaitAsync(TestTimeouts.Signal); + var settled = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders"))?.Messages == 0, + TestTimeouts.Signal); + var state = await vhost.Broker.GetStateAsync(); + + // assert + Assert.True(received); + Assert.True(settled); + Assert.Single(orders.Messages); + Assert.Equal(2, state.Connections); + Assert.Equal(2, state.Consumers); + } + + private static async Task PublishAsync(TestBus bus, OrderCreated message) + { + using var scope = bus.Provider.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .PublishAsync(message, TestContext.Current.CancellationToken); + } + + public sealed class HandlerGate + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class StuckOrderHandler(HandlerGate gate) : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + gate.Started.TrySetResult(); + + // ignores its cancellation token, as a blocking call without a token would + await gate.Release.Task; + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConsumerRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConsumerRecoveryTests.cs new file mode 100644 index 00000000000..82a456772d8 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ConsumerRecoveryTests.cs @@ -0,0 +1,603 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Features; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConsumerRecovery)] +public sealed class ConsumerRecoveryTests(RabbitMQFixture fixture) +{ + private static readonly TimeSpan s_consumerTimeout = TimeSpan.FromSeconds(2); + + // Shorter than the 30 second connection health check, so the consumer has to attach by retrying, + // not by the periodic health check. + private static readonly TimeSpan s_beforeHealthCheck = TimeSpan.FromSeconds(8); + + [Fact] + public async Task Consumer_Should_RedeclareQueueAndBindings_When_QueueDeletedByOperator() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal)); + + // act + await vhost.Faults.DeleteQueueAsync("orders"); + var reattached = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders"))?.Consumers == 1, + TestTimeouts.Signal); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }); + var received = await recorder.WaitAsync(TestTimeouts.Signal); + + // assert + Assert.True(reattached); + Assert.True(received); + } + + [Fact] + public async Task Consumers_Should_KeepOtherEndpointsReceiving_When_ConsumerTimeoutExpiresForStuckHandler() + { + // arrange + // RabbitMQ 4.3 evaluates consumer timeouts only for quorum queues + await using var vhost = await fixture.CreateVhostAsync(); + await using var consumerTimeout = await fixture.Faults.SetConsumerTimeoutAsync(s_consumerTimeout); + var gate = new HandlerGate(); + var orders = new MessageRecorder(); + var payments = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddKeyedSingleton("orders", orders) + .AddKeyedSingleton("payments", payments) + .AddMessageBus() + .AddEventHandler() + .AddRequestHandler() + .AddRabbitMQ(t => + { + t.Queue("orders").Quorum().Handler().MaxConcurrency(1); + t.Endpoint("payments").Handler(); + }) + .BuildTestBusAsync(); + + try + { + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + // the broker closes the channel (before 4.3) or cancels the consumer (4.3) of the stuck handler + await gate.Cancelled.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await SendAsync(bus, new ProcessPayment { OrderId = "ORD-1", Amount = 1 }); + var paymentReceived = await payments.WaitAsync(TestTimeouts.Signal); + await gate.Cancelled.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var whileBlocked = (gate.Invocations, orders.Messages.Count, GetEndpoint(bus, "orders").IsHealthy); + gate.Release.TrySetResult(); + var orderRedelivered = await orders.WaitAsync(TestTimeouts.Signal); + + // assert + Assert.True(paymentReceived); + Assert.Equal(["ORD-1"], payments.Messages.Cast().Select(p => p.OrderId)); + Assert.Equal((1, 0, false), whileBlocked); + Assert.True(orderRedelivered); + Assert.Equal(["ORD-1"], orders.Messages.Cast().Select(o => o.OrderId)); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task Message_Should_BeHandledAgainAfterFirstHandlerIsCancelled_When_ConsumerTimeoutExpires() + { + // arrange + // RabbitMQ 4.3 evaluates consumer timeouts only for quorum queues + await using var vhost = await fixture.CreateVhostAsync(); + await using var consumerTimeout = await fixture.Faults.SetConsumerTimeoutAsync(s_consumerTimeout); + var gate = new HandlerGate(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddKeyedSingleton("orders", orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Queue("orders").Quorum().Handler().MaxConcurrency(1)) + .BuildTestBusAsync(); + + // act + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + var handled = await orders.WaitAsync(TestTimeouts.Signal); + var settled = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders"))?.Messages == 0, + TestTimeouts.Signal); + + // assert + Assert.True(handled); + Assert.True(settled); + Assert.Equal(2, gate.Invocations); + Assert.Single(orders.Messages); + Assert.True(gate.FirstCancelled); + } + + [Fact] + public async Task Consumer_Should_Reattach_When_ConnectionClosedWhileHandlerIgnoresCancellation() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.NetworkRecoveryInterval = TimeSpan.FromSeconds(1); + var gate = new HandlerGate(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddKeyedSingleton("orders", orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler().MaxConcurrency(1)) + .BuildTestBusAsync(); + + try + { + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + // the message is requeued when the connection closes, so a re-attached consumer receives it again + await vhost.Faults.CloseConnectionsAsync(); + await gate.Cancelled.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var whileBlocked = ( + Invocations: gate.Invocations, + Handled: orders.Messages.Count, + IsHealthy: GetEndpoint(bus, "orders").IsHealthy); + gate.Release.TrySetResult(); + var redelivered = await orders.WaitAsync(TestTimeouts.Signal); + var settled = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders"))?.Messages == 0, + TestTimeouts.Signal); + var queue = await vhost.Broker.GetQueueAsync("orders"); + + // assert + Assert.Equal((1, 0, false), whileBlocked); + Assert.True(redelivered); + Assert.True(settled); + Assert.Equal(1, queue?.Consumers); + Assert.Equal((1, true), (orders.Messages.Count, GetEndpoint(bus, "orders").IsHealthy)); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task Message_Should_RunHandlerAgain_When_HandlerCompletedAfterItsChannelClosed() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var gate = new HandlerGate(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddKeyedSingleton("orders", orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + + try + { + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + // the consumer cannot reconnect while access is revoked, so the handler completes before the + // broker redelivers its message + await using (await vhost.Faults.RevokeAccessAsync()) + { + await vhost.Faults.CloseConsumerConnectionAsync("orders"); + await gate.Cancelled.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + gate.Release.TrySetResult(); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + } + + var settled = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders"))?.Messages == 0, + TestTimeouts.Signal); + + // assert + Assert.True(settled); + Assert.Equal((2, 2), (gate.Invocations, orders.Messages.Count)); + Assert.Equal(0, (await vhost.Broker.GetQueueAsync("orders_error"))?.Messages); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task Consumer_Should_Attach_When_QueueReturnsAfterReconnect() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await DeclareQueueAsync(vhost, "orders"); + var orders = new MessageRecorder(); + var payments = new MessageRecorder(); + await using var bus = await CreateBusWithoutQueueProvisioningAsync(vhost, orders, payments); + var paymentsBefore = await vhost.Broker.GetConsumerAsync("payments"); + + // act + // the reconnect cannot attach while the queue is missing + await using (await vhost.Faults.RevokeAccessAsync()) + { + await vhost.Faults.CloseConnectionsAsync(); + await vhost.Faults.DeleteQueueAsync("orders"); + } + + await vhost.Broker.WaitForConsumerAsync("payments", otherThan: paymentsBefore); + await DeclareQueueAsync(vhost, "orders"); + var attached = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders"))?.Consumers == 1, + s_beforeHealthCheck); + + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-recovered" }); + + // assert + Assert.True(attached, "The consumer waited for the periodic health check instead of retrying."); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + Assert.Equal("ORD-recovered", Assert.Single(orders.Messages.Cast()).OrderId); + } + + [Fact] + public async Task Consumers_Should_KeepOtherQueuesFlowing_When_QueueDeletedWithoutAutoProvision() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await DeclareQueueAsync(vhost, "orders"); + var orders = new MessageRecorder(); + var payments = new MessageRecorder(); + await using var bus = await CreateBusWithoutQueueProvisioningAsync(vhost, orders, payments); + await SendAsync(bus, new ProcessPayment { OrderId = "ORD-0", Amount = 1 }); + + // act + // every re-attach attempt fails with 404, which closes only the channel of the attempt + await vhost.Faults.DeleteQueueAsync("orders"); + await vhost.Broker.WaitForNoConsumerAsync("orders"); + await SendAsync(bus, new ProcessPayment { OrderId = "ORD-1", Amount = 1 }); + var paymentsReceived = await payments.WaitAsync(TestTimeouts.Signal, expectedCount: 2); + + await DeclareQueueAsync(vhost, "orders"); + var reattached = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders"))?.Consumers == 1, + TestTimeouts.Signal); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }); + var orderReceived = await orders.WaitAsync(TestTimeouts.Signal); + + // assert + Assert.True(paymentsReceived); + Assert.Equal(["ORD-0", "ORD-1"], payments.Messages.Cast().Select(p => p.OrderId).Order()); + Assert.True(reattached); + Assert.True(orderReceived); + Assert.Equal(["ORD-2"], orders.Messages.Cast().Select(o => o.OrderId)); + } + + [Fact] + public async Task Consumer_Should_KeepPrefetch_When_Reattached() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddKeyedSingleton("orders", new MessageRecorder()) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler().MaxPrefetch(7)) + .BuildTestBusAsync(); + var before = await vhost.Broker.GetConsumerAsync("orders"); + + // act + await vhost.Faults.CloseConsumerConnectionAsync("orders"); + var after = await vhost.Broker.WaitForConsumerAsync("orders", otherThan: before); + + // assert + Assert.Equal(7, before.Prefetch); + Assert.Equal(7, after.Prefetch); + } + + [Fact] + public async Task Consumer_Should_KeepConsuming_When_RawMessageHasNoEnvelope() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddKeyedSingleton("orders", orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + + // act + // a message from a client that does not set any of the properties or headers the bus writes + await using (var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken)) + await using (var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken)) + { + await channel.BasicPublishAsync( + exchange: "", + routingKey: "orders", + body: "not an envelope"u8.ToArray(), + cancellationToken: TestContext.Current.CancellationToken); + } + + var drained = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders"))?.Messages == 0, + TestTimeouts.Signal); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + var nextHandled = await orders.WaitAsync(TestTimeouts.Signal); + + // assert + Assert.True(drained); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("orders_skipped"))?.Messages); + Assert.Equal(0, (await vhost.Broker.GetQueueAsync("orders_error"))?.Messages); + Assert.True(nextHandled); + } + + [Theory] + [InlineData(4)] + [InlineData(8)] + [InlineData(32)] + public async Task BatchHandler_Should_HandleEveryMessage_When_ConnectionClosesWhileBatchFills(int busConcurrency) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var batches = new BatchRecorder(); + using var buffered = new SemaphoreSlim(0); + var redeliveries = 0; + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(batches) + .AddMessageBus() + .AddConcurrencyLimiter(options => options.MaxConcurrency = busConcurrency) + .AddBatchHandler(options => + { + options.MaxBatchSize = 10; + options.BatchTimeout = TimeSpan.FromSeconds(30); + }) + .AddRabbitMQ(t => t.Queue("batch-orders") + .Handler() + .MaxConcurrency(10) + .AddConcurrencyLimiter(options => options.MaxConcurrency = 10) + .UseReceive(new ReceiveMiddlewareConfiguration( + (_, next) => context => + { + if (context.Features.Get() is { EventArgs.Redelivered: true }) + { + Interlocked.Increment(ref redeliveries); + } + + // A partial batch is buffered before the consumer returns its pending task. + var pending = next(context); + buffered.Release(); + return pending; + }, + "BatchBuffered"))) + .BuildTestBusAsync(); + for (var i = 1; i <= 6; i++) + { + await PublishAsync(bus, new OrderCreated { OrderId = $"ORD-{i}" }); + } + + await Task.WhenAll(Enumerable.Range(0, 6).Select(_ => buffered.WaitAsync(TestContext.Current.CancellationToken))) + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var before = await vhost.Broker.GetConsumerAsync("batch-orders"); + + // act + // the broker returns the six buffered messages to the queue and delivers them again after the re-attach + await vhost.Faults.CloseConsumerConnectionAsync("batch-orders"); + await vhost.Broker.WaitForConsumerAsync("batch-orders", otherThan: before); + for (var i = 7; i <= 10; i++) + { + await PublishAsync(bus, new OrderCreated { OrderId = $"ORD-{i}" }); + } + + await Wait.UntilAsync(() => batches.OrderIds.Count == 10); + await vhost.Broker.WaitForEmptyQueueAsync("batch-orders"); + + // assert + Assert.Equal(6, Volatile.Read(ref redeliveries)); + Assert.Equal(Enumerable.Range(1, 10).Select(i => $"ORD-{i}").Order(), batches.OrderIds.Order()); + Assert.Equal(1, batches.Count); + } + + private static async Task CreateBusWithoutQueueProvisioningAsync( + VhostContext vhost, + MessageRecorder orders, + MessageRecorder payments) + => await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddKeyedSingleton("orders", orders) + .AddKeyedSingleton("payments", payments) + .AddMessageBus() + .AddEventHandler() + .AddRequestHandler() + .AddRabbitMQ(t => + { + // an operator owns the queue, so the bus neither declares it nor declares it again + t.DeclareQueue("orders").AutoProvision(false); + t.Endpoint("orders").Handler(); + t.Endpoint("payments").Handler(); + }) + .BuildTestBusAsync(); + + private static async Task DeclareQueueAsync(VhostContext vhost, string queueName) + { + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync( + TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + queueName, + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + } + + private static RabbitMQReceiveEndpoint GetEndpoint(TestBus bus, string queueName) + => ((MessagingRuntime)bus.Provider.GetRequiredService()) + .Transports.OfType().Single() + .ReceiveEndpoints.OfType().Single(endpoint => endpoint.Queue.Name == queueName); + + private static async Task PublishAsync(TestBus bus, OrderCreated message) + { + using var scope = bus.Provider.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .PublishAsync(message, TestContext.Current.CancellationToken); + } + + private static async Task SendAsync(TestBus bus, ProcessPayment message) + { + using var scope = bus.Provider.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .SendAsync(message, TestContext.Current.CancellationToken); + } + + public sealed class HandlerGate + { + private int _invocations; + + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Cancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int Invocations => Volatile.Read(ref _invocations); + + public bool FirstCancelled { get; set; } + + public bool IsFirstInvocation() => Interlocked.Increment(ref _invocations) == 1; + } + + public sealed class StuckOrderHandler(HandlerGate gate, [FromKeyedServices("orders")] MessageRecorder recorder) + : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + if (gate.IsFirstInvocation()) + { + gate.Started.TrySetResult(); + await using var registration = cancellationToken.Register(() => gate.Cancelled.TrySetResult()); + + // ignores its cancellation token, as a blocking call without a token would + await gate.Release.Task; + return; + } + + recorder.Record(message); + } + } + + public sealed class CooperativeOrderHandler( + HandlerGate gate, + [FromKeyedServices("orders")] MessageRecorder recorder) + : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + if (gate.IsFirstInvocation()) + { + gate.Started.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + catch (OperationCanceledException) + { + gate.FirstCancelled = true; + throw; + } + } + + recorder.Record(message); + } + } + + public sealed class GatedOrderHandler(HandlerGate gate, [FromKeyedServices("orders")] MessageRecorder recorder) + : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + if (gate.IsFirstInvocation()) + { + gate.Started.TrySetResult(); + await using var registration = cancellationToken.Register(() => gate.Cancelled.TrySetResult()); + await gate.Release.Task; + } + + recorder.Record(message); + } + } + + public sealed class RecordingOrderHandler([FromKeyedServices("orders")] MessageRecorder recorder) + : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + recorder.Record(message); + return default; + } + } + + public sealed class BatchRecorder + { + private readonly ConcurrentDictionary _orderIds = new(); + private int _count; + + public ICollection OrderIds => _orderIds.Keys; + + public int Count => Volatile.Read(ref _count); + + public void Record(IEnumerable orders) + { + Interlocked.Increment(ref _count); + + foreach (var order in orders) + { + _orderIds.TryAdd(order.OrderId, 0); + } + } + } + + public sealed class OrderBatchHandler(BatchRecorder recorder) : IBatchEventHandler + { + public ValueTask HandleAsync(IMessageBatch batch, CancellationToken cancellationToken) + { + recorder.Record(batch); + return default; + } + } + + public sealed class PaymentHandler([FromKeyedServices("payments")] MessageRecorder recorder) + : IEventRequestHandler + { + public ValueTask HandleAsync(ProcessPayment request, CancellationToken cancellationToken) + { + recorder.Record(request); + return default; + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/CustomHeaderTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/CustomHeaderTests.cs index 2918f7a2d8e..7991ca213db 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/CustomHeaderTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/CustomHeaderTests.cs @@ -4,10 +4,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class CustomHeaderTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public CustomHeaderTests(RabbitMQFixture fixture) @@ -42,7 +41,7 @@ await messageBus.PublishAsync( CancellationToken.None); // assert - Assert.True(await capture.WaitAsync(s_timeout), "Consumer did not receive the published message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the published message"); var headers = Assert.Single(capture.CapturedHeaders); Assert.True(headers.TryGetValue("x-tenant", out var tenant), "Custom header 'x-tenant' not found"); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/EndpointMiddlewareTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/EndpointMiddlewareTests.cs index 87c7e8710bb..1d5ec28a44f 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/EndpointMiddlewareTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/EndpointMiddlewareTests.cs @@ -4,10 +4,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class EndpointMiddlewareTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public EndpointMiddlewareTests(RabbitMQFixture fixture) @@ -51,7 +50,7 @@ public async Task UseReceive_Should_InvokeMiddleware_When_MessageReceived() await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-MW" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Consumer did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message within timeout"); Assert.Contains("receive-mw", tracker.Invocations); } @@ -92,7 +91,7 @@ public async Task UseReceive_After_Should_InvokeMiddleware_When_MessageReceived( await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-MW2" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Consumer did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message within timeout"); Assert.Contains("append-receive-mw", tracker.Invocations); } @@ -133,7 +132,7 @@ public async Task UseReceive_Before_Should_InvokeMiddleware_When_MessageReceived await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-MW3" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Consumer did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message within timeout"); Assert.Contains("prepend-receive-mw", tracker.Invocations); } @@ -173,7 +172,7 @@ public async Task UseDispatch_Should_InvokeMiddleware_When_MessageDispatched() await messageBus.PublishAsync(new ProcessPayment { OrderId = "ORD-DM", Amount = 50.00m }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the message within timeout"); Assert.Contains("dispatch-mw", tracker.Invocations); } @@ -214,7 +213,7 @@ public async Task UseDispatch_After_Should_InvokeMiddleware_When_MessageDispatch await messageBus.PublishAsync(new ProcessPayment { OrderId = "ORD-DM2", Amount = 25.00m }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the message within timeout"); Assert.Contains("append-dispatch-mw", tracker.Invocations); } @@ -255,7 +254,7 @@ public async Task UseDispatch_Before_Should_InvokeMiddleware_When_MessageDispatc await messageBus.PublishAsync(new ProcessPayment { OrderId = "ORD-DM3", Amount = 10.00m }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the message within timeout"); Assert.Contains("prepend-dispatch-mw", tracker.Invocations); } diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ErrorQueueTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ErrorQueueTests.cs index 7a2901d9ecc..158adb8156a 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ErrorQueueTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ErrorQueueTests.cs @@ -4,10 +4,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class ErrorQueueTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public ErrorQueueTests(RabbitMQFixture fixture) @@ -45,7 +44,7 @@ public async Task PublishAsync_Should_RouteToErrorQueue_When_HandlerThrows() await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-FAULT" }, CancellationToken.None); // assert - Assert.True(await capture.WaitAsync(s_timeout), "Error queue consumer did not receive the faulted message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Error queue consumer did not receive the faulted message"); var headers = Assert.Single(capture.CapturedHeaders); Assert.Equal(FaultHeaders.ExpectedShape(), FaultHeaders.Shape(headers)); @@ -85,7 +84,7 @@ await messageBus.SendAsync( CancellationToken.None); // assert - Assert.True(await capture.WaitAsync(s_timeout), "Error queue consumer did not receive the faulted message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Error queue consumer did not receive the faulted message"); var headers = Assert.Single(capture.CapturedHeaders); Assert.True(headers.ContainsKey("fault-exception-type"), "Missing fault-exception-type header"); @@ -122,7 +121,7 @@ public async Task ErrorQueue_Should_PreserveOriginalBody_When_HandlerFaults() await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-PRESERVE" }, CancellationToken.None); // assert - Assert.True(await capture.WaitAsync(s_timeout), "Error queue consumer did not receive the faulted message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Error queue consumer did not receive the faulted message"); var message = Assert.Single(capture.Messages); Assert.Equal("ORD-PRESERVE", message.OrderId); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExclusiveQueueTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExclusiveQueueTests.cs new file mode 100644 index 00000000000..c5998133819 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExclusiveQueueTests.cs @@ -0,0 +1,100 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.Topology)] +public sealed class ExclusiveQueueTests(RabbitMQFixture fixture) +{ + private const string QueueName = "exclusive-orders"; + + [Fact] + public async Task StartAsync_Should_ConsumeFromExclusiveQueue_When_EndpointQueueIsExclusive() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + + // act + await PublishAsync(bus, "ORD-1"); + var received = await recorder.WaitAsync(TestTimeouts.Signal); + var queue = await vhost.Broker.GetQueueAsync(QueueName); + + // assert + Assert.True(received); + Assert.Equal(1, queue?.Consumers); + } + + [Fact] + public async Task Consumer_Should_RedeclareExclusiveQueue_When_ConsumerConnectionClosed() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + await PublishAsync(bus, "ORD-1"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal)); + var before = await vhost.Broker.GetConsumerAsync(QueueName); + + // act + // the broker deletes an exclusive queue together with the connection that declared it + await vhost.Faults.CloseConsumerConnectionAsync(QueueName); + await vhost.Broker.WaitForConsumerAsync(QueueName, otherThan: before); + await PublishAsync(bus, "ORD-2"); + var received = await recorder.WaitAsync(TestTimeouts.Signal); + + // assert + Assert.True(received); + Assert.Equal( + ["ORD-1", "ORD-2"], + recorder.Messages.Cast().Select(order => order.OrderId).Order()); + } + + [Fact] + public async Task StartAsync_Should_FailFast_When_ExclusiveQueueHeldByAnotherBus() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + + // act + await using var services = CreateServices(vhost, new MessageRecorder()).Services.BuildServiceProvider(); + var runtime = (MessagingRuntime)services.GetRequiredService(); + var exception = await Assert.ThrowsAnyAsync( + () => runtime.StartAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + + await PublishAsync(bus, "ORD-1"); + var firstStillReceives = await recorder.WaitAsync(TestTimeouts.Signal); + + // assert + Assert.Equal(Constants.ResourceLocked, exception.ShutdownReason?.ReplyCode); + Assert.True(firstStillReceives); + } + + private static IMessageBusHostBuilder CreateServices(VhostContext vhost, MessageRecorder recorder) + => new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => + { + t.DeclareQueue(QueueName).Exclusive(); + t.Endpoint(QueueName).Handler(); + }); + + private static Task CreateBusAsync(VhostContext vhost, MessageRecorder recorder) + => CreateServices(vhost, recorder).BuildTestBusAsync(); + + private static async Task PublishAsync(TestBus bus, string orderId) + { + using var scope = bus.Provider.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .PublishAsync(new OrderCreated { OrderId = orderId }, TestContext.Current.CancellationToken); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExplicitTopologyTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExplicitTopologyTests.cs index d51d7a80eca..84bc924c5e3 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExplicitTopologyTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExplicitTopologyTests.cs @@ -4,10 +4,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class ExplicitTopologyTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; private readonly ITestOutputHelper _output; @@ -48,7 +47,7 @@ public async Task PublishAsync_Should_RouteToQueue_When_ExplicitTopologyDeclared await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-TOPO" }, CancellationToken.None); // assert - Assert.True(await capture.WaitAsync(s_timeout), "Consumer on custom-q did not receive the published message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Consumer on custom-q did not receive the published message"); var message = Assert.Single(capture.Messages); Assert.Equal("ORD-TOPO", message.OrderId); @@ -85,7 +84,7 @@ public async Task PublishAsync_Should_RouteToQueue_When_ExplicitTopologyDeclared await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-TOPO" }, CancellationToken.None); // assert - Assert.True(await capture.WaitAsync(s_timeout), "Consumer on custom-q did not receive the published message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Consumer on custom-q did not receive the published message"); var message = Assert.Single(capture.Messages); Assert.Equal("ORD-TOPO", message.OrderId); @@ -123,7 +122,7 @@ public async Task PublishAsync_Should_RouteToQueue_When_ExplicitTopologyDeclared await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-RCV" }, CancellationToken.None); // assert - Assert.True(await capture.WaitAsync(s_timeout), "Consumer on custom-q did not receive the published message"); + Assert.True(await capture.WaitAsync(TestTimeouts.Signal), "Consumer on custom-q did not receive the published message"); var message = Assert.Single(capture.Messages); Assert.Equal("ORD-RCV", message.OrderId); @@ -168,7 +167,7 @@ public async Task ErrorQueue_Should_ProvisionRenamedQueue_When_FaultEndpointUses await messageBus.PublishAsync(new OrderCreated { OrderId = "ERROR-RENAME" }, CancellationToken.None); // assert - Assert.True(await faultCapture.WaitAsync(s_timeout), "Faulted message did not arrive in the renamed error queue"); + Assert.True(await faultCapture.WaitAsync(TestTimeouts.Signal), "Faulted message did not arrive in the renamed error queue"); Assert.Equal("ERROR-RENAME", Assert.Single(faultCapture.Messages).OrderId); } @@ -245,7 +244,7 @@ public async Task PublishAsync_Should_DeliverToConsumer_When_ExplicitDestination // assert Assert.True( - await capture.WaitAsync(s_timeout), + await capture.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message published to the explicit exchange destination"); var message = Assert.Single(capture.Messages); @@ -294,7 +293,7 @@ public async Task Queue_Endpoint_DeclareQueue_Should_ProvisionOneQueue_When_Same // assert: message is delivered, proving the provisioned queue is operational. Assert.True( - await capture.WaitAsync(s_timeout), + await capture.WaitAsync(TestTimeouts.Signal), "Consumer on the converged 'orders' queue did not receive the published message"); var received = Assert.Single(capture.Messages); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExternalDestinationTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExternalDestinationTests.cs index c0d3f11b708..2ca35df1eb2 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExternalDestinationTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ExternalDestinationTests.cs @@ -1,4 +1,4 @@ -using System.Text; +using System.Text.Json; using Microsoft.Extensions.DependencyInjection; using Mocha.Transport.RabbitMQ.Tests.Helpers; using RabbitMQ.Client; @@ -10,11 +10,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; /// Covers address-based sends to queues another service owns: the sending host binds explicitly, /// never declares the queue, and still has to reach it on the broker. /// -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class ExternalDestinationTests(RabbitMQFixture fixture) { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); - [Fact] public async Task SendAsync_Should_DeliverToForeignQueue_When_QueueNotDeclaredUnderExplicitBinding() { @@ -42,21 +40,27 @@ await channel.QueueDeclareAsync( var messageBus = scope.ServiceProvider.GetRequiredService(); // act - await messageBus.SendAsync( - new OrderCreated { OrderId = "EXTERNAL-1" }, - new SendOptions { Endpoint = new Uri($"rabbitmq:q/{queueName}") }, - cancellationToken); + await using (await vhost.Faults.DenyDeclaresAsync()) + { + await messageBus.SendAsync( + new OrderCreated { OrderId = "EXTERNAL-1" }, + new SendOptions { Endpoint = new Uri($"rabbitmq:q/{queueName}") }, + cancellationToken); + } // assert var delivery = await GetMessageAsync(channel, queueName, cancellationToken); Assert.NotNull(delivery); Assert.Equal("", delivery.Exchange); Assert.Equal(queueName, delivery.RoutingKey); - Assert.Contains("EXTERNAL-1", Encoding.UTF8.GetString(delivery.Body.Span)); + var order = JsonSerializer.Deserialize( + delivery.Body.Span, + new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + Assert.Equal("EXTERNAL-1", order?.OrderId); } [Fact] - public async Task SendAsync_Should_NotDeclareQueue_When_QueueNotDeclaredUnderExplicitBinding() + public async Task SendAsync_Should_ThrowWithoutDeclaringQueue_When_QueueNotDeclaredUnderExplicitBinding() { // arrange const string queueName = "reporting.missing"; @@ -72,10 +76,11 @@ public async Task SendAsync_Should_NotDeclareQueue_When_QueueNotDeclaredUnderExp var messageBus = scope.ServiceProvider.GetRequiredService(); // act - await messageBus.SendAsync( + // the broker returns a message sent to a missing queue instead of dropping it + var sendException = await Assert.ThrowsAsync(async () => await messageBus.SendAsync( new OrderCreated { OrderId = "EXTERNAL-2" }, new SendOptions { Endpoint = new Uri($"rabbitmq:q/{queueName}") }, - cancellationToken); + cancellationToken)); // assert // a passive declare only succeeds for a queue that exists, so a 404 proves the send @@ -84,15 +89,59 @@ await messageBus.SendAsync( await using var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken); var exception = await Assert.ThrowsAsync( () => channel.QueueDeclarePassiveAsync(queueName, cancellationToken)); + Assert.IsType(sendException.InnerException); Assert.Equal(404, exception.ShutdownReason?.ReplyCode); } + [Fact] + public async Task SendAsync_Should_DeliverToForeignExchange_When_SenderCannotDeclareResources() + { + // arrange + var cancellationToken = TestContext.Current.CancellationToken; + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(cancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken); + await channel.ExchangeDeclareAsync("reporting", "fanout", durable: true, cancellationToken: cancellationToken); + await channel.QueueDeclareAsync( + "reports", + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: cancellationToken); + await channel.QueueBindAsync("reports", "reporting", "", cancellationToken: cancellationToken); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => t.BindExplicitly()) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + + // act + await using (await vhost.Faults.DenyDeclaresAsync()) + { + await messageBus.SendAsync( + new OrderCreated { OrderId = "EXTERNAL-3" }, + new SendOptions { Endpoint = new Uri("rabbitmq:e/reporting") }, + cancellationToken); + } + + // assert + var delivery = await GetMessageAsync(channel, "reports", cancellationToken); + Assert.NotNull(delivery); + var order = JsonSerializer.Deserialize( + delivery.Body.Span, + new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + Assert.Equal("EXTERNAL-3", order?.OrderId); + Assert.Equal("reporting", delivery.Exchange); + } + private static async Task GetMessageAsync( IChannel channel, string queueName, CancellationToken cancellationToken) { - var deadline = DateTime.UtcNow + s_timeout; + var deadline = DateTime.UtcNow + TestTimeouts.Signal; while (true) { diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/FaultHandlingTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/FaultHandlingTests.cs index 9e4c542535d..4a72ed0a05d 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/FaultHandlingTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/FaultHandlingTests.cs @@ -4,10 +4,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class FaultHandlingTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public FaultHandlingTests(RabbitMQFixture fixture) @@ -63,15 +62,15 @@ public async Task PublishAsync_Should_NotAffectOtherHandlers_When_OneHandlerThro // act - publish event that triggers a throw await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-FAIL" }, CancellationToken.None); - // wait a bit for the throwing handler to process - await throwingRecorder.WaitAsync(TimeSpan.FromSeconds(2)); + // the throwing handler has processed the event + Assert.True(await throwingRecorder.WaitAsync(TestTimeouts.Signal), "Throwing handler did not receive the event"); // now publish a normal event await messageBus.PublishAsync(new ItemShipped { TrackingNumber = "TRK-1" }, CancellationToken.None); // assert - the second handler still works Assert.True( - await normalRecorder.WaitAsync(s_timeout), + await normalRecorder.WaitAsync(TestTimeouts.Signal), "Normal handler did not receive event after a previous handler threw"); } diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/HealthCheckTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/HealthCheckTests.cs new file mode 100644 index 00000000000..95bb4d72d23 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/HealthCheckTests.cs @@ -0,0 +1,109 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Mocha.Hosting; +using Mocha.Transport.RabbitMQ.Tests.Helpers; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.Messaging)] +public sealed class HealthCheckTests(RabbitMQFixture fixture) +{ + private const string BlockedReason = + "Queue 'orders': The consumer waits for a handler of a lost channel to return."; + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CheckHealthAsync_Should_ReportUnhealthyUntilRecovery_When_LostHandlerExhaustsCapacity( + bool loseConnection) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var gate = new HandlerGate(); + var services = new ServiceCollection(); + services.AddSingleton(vhost.ConnectionFactory); + services.AddSingleton(gate); + services.AddHealthChecks().AddMessageBus(); + await using var bus = await services.AddMessageBus() + .AddHealthCheck() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler().MaxConcurrency(1)) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + var healthChecks = bus.Provider.GetRequiredService(); + var transport = bus.Provider.GetRequiredService() + .Transports.OfType().Single(); + + try + { + var initial = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken); + await messageBus.PublishAsync(new OrderCreated { OrderId = "blocked" }, TestContext.Current.CancellationToken); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var busy = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken); + + // act + if (loseConnection) + { + await vhost.Faults.CloseConsumerConnectionAsync("orders"); + } + else + { + await vhost.Faults.DeleteQueueAsync("orders"); + } + + await gate.Cancelled.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await vhost.Broker.WaitForNoConsumerAsync("orders"); + // The independent health request and reply endpoints can still process messages. + var response = await messageBus.RequestAsync(new HealthRequest("Health Check"), TestContext.Current.CancellationToken); + await Wait.UntilAsync(() => transport.ConsumerManager.HealthFailureReason == BlockedReason); + var blocked = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken); + gate.Release.TrySetResult(); + await vhost.Broker.WaitForConsumerAsync("orders"); + var recovered = await healthChecks.CheckHealthAsync(TestContext.Current.CancellationToken); + await messageBus.PublishAsync(new OrderCreated { OrderId = "recovered" }, TestContext.Current.CancellationToken); + var order = await gate.Recovered.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + Assert.Equal( + [HealthStatus.Healthy, HealthStatus.Healthy, HealthStatus.Unhealthy, HealthStatus.Healthy], + new[] { initial.Status, busy.Status, blocked.Status, recovered.Status }); + Assert.Equal( + $"Transport '{transport.Name}' is unhealthy. {BlockedReason}", + blocked.Entries["MessageBus"].Description); + Assert.Equal("OK", response.Message); + Assert.Equal("recovered", order.OrderId); + } + finally + { + gate.Release.TrySetResult(); + } + } + + public sealed class HandlerGate + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Cancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Recovered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class StuckOrderHandler(HandlerGate gate) : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + if (message.OrderId == "recovered") + { + gate.Recovered.TrySetResult(message); + return; + } + + await using var registration = cancellationToken.Register(() => gate.Cancelled.TrySetResult()); + gate.Started.TrySetResult(); + await gate.Release.Task; + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/InboxTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/InboxTests.cs index 9329d9a22a1..62511a838f1 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/InboxTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/InboxTests.cs @@ -6,10 +6,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Messaging)] public class InboxTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public InboxTests(RabbitMQFixture fixture) @@ -58,8 +57,8 @@ public async Task Inbox_Should_DeduplicateMessage_When_SameMessageIdPublishedTwi await messageBus.PublishAsync(new InboxEvent { Payload = "first" }, CancellationToken.None); // Wait for the first message to be fully processed and recorded in the inbox - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the first event within timeout"); - await WaitUntilAsync(() => !inbox.RecordedEnvelopes.IsEmpty, s_timeout); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the first event within timeout"); + await WaitUntilAsync(() => !inbox.RecordedEnvelopes.IsEmpty, TestTimeouts.Signal); await messageBus.PublishAsync(new InboxEvent { Payload = "second" }, CancellationToken.None); @@ -97,7 +96,7 @@ public async Task Inbox_Should_ProcessBothMessages_When_DifferentMessageIds() // assert - both messages should be processed Assert.True( - await recorder.WaitAsync(s_timeout, expectedCount: 2), + await recorder.WaitAsync(TestTimeouts.Signal, expectedCount: 2), "Handler did not receive both events within timeout"); Assert.Equal(2, recorder.Messages.Count); @@ -165,7 +164,7 @@ public async Task Inbox_Should_ProcessMessage_When_SkipInboxIsSet() // assert - both messages should be processed even though they share the same MessageId Assert.True( - await recorder.WaitAsync(s_timeout, expectedCount: 2), + await recorder.WaitAsync(TestTimeouts.Signal, expectedCount: 2), "Handler should receive both messages when SkipInbox is set"); Assert.Equal(2, recorder.Messages.Count); @@ -213,7 +212,7 @@ public async Task Inbox_Should_ProcessMessage_When_MessageIdIsNull() // assert - both should be processed since null MessageId cannot be deduplicated Assert.True( - await recorder.WaitAsync(s_timeout, expectedCount: 2), + await recorder.WaitAsync(TestTimeouts.Signal, expectedCount: 2), "Handler should receive both messages when MessageId is null"); Assert.Equal(2, recorder.Messages.Count); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/MissingQueueRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/MissingQueueRecoveryTests.cs new file mode 100644 index 00000000000..bd0f486667a --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/MissingQueueRecoveryTests.cs @@ -0,0 +1,235 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.Topology)] +public sealed class MissingQueueRecoveryTests(RabbitMQFixture fixture) +{ + private const string QueueName = "temporary-orders"; + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task PublishAsync_Should_DeliverEvents_When_UpstreamExchangeBindingsAreDeletedDuringDisconnection( + bool automaticRecovery) + { + // arrange + var recorder = new MessageRecorder(); + await using var vhost = await fixture.CreateVhostAsync(); + var connectionFactory = (ConnectionFactory)vhost.ConnectionFactory; + connectionFactory.AutomaticRecoveryEnabled = automaticRecovery; + connectionFactory.NetworkRecoveryInterval = TimeSpan.FromSeconds(2); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(recorder) + .AddMessageBus() + .AddMessage(d => d.Publish(r => r.ToRabbitMQExchange("upstream"))) + .AddEventHandler() + .AddRabbitMQ(t => + { + t.DeclareExchange("upstream"); + t.DeclareExchange("intermediate").AutoDelete(); + t.DeclareExchange("downstream").AutoDelete(); + t.DeclareBinding("upstream", "intermediate").ToExchange("intermediate"); + t.DeclareBinding("intermediate", "downstream").ToExchange("downstream"); + t.Queue(QueueName) + .Handler() + .Temporary() + .BindExplicitly() + .BindFrom(new Uri("exchange:downstream")); + }) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-before" }, TestContext.Current.CancellationToken); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal)); + var before = await vhost.Broker.GetConsumerAsync(QueueName); + + // act + await vhost.Faults.CloseConsumerConnectionAsync(QueueName); + await vhost.Broker.WaitForConsumerAsync(QueueName, otherThan: before); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-after" }, TestContext.Current.CancellationToken); + + // assert + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal)); + Assert.Equal( + ["ORD-after", "ORD-before"], + recorder.Messages.Cast().Select(m => m.OrderId).Order()); + } + + [Fact] + public async Task PublishAsync_Should_DeliverEvents_When_ConsumerConnectionIsLost() + { + // arrange + var recorder = new MessageRecorder(); + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await CreateEventBusAsync(vhost, recorder); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-before" }, TestContext.Current.CancellationToken); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal)); + var before = await vhost.Broker.GetConsumerAsync(QueueName); + + // act + await vhost.Faults.CloseConsumerConnectionAsync(QueueName); + await vhost.Broker.WaitForConsumerAsync(QueueName, otherThan: before); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-after" }, TestContext.Current.CancellationToken); + + // assert + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal)); + Assert.Equal( + ["ORD-after", "ORD-before"], + recorder.Messages.Cast().Select(m => m.OrderId).Order()); + } + + [Fact] + public async Task PublishAsync_Should_DeliverEvent_When_QueueDeclarationPermissionIsRestored() + { + // arrange + var recorder = new MessageRecorder(); + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint(QueueName).Handler().Temporary()) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + + // act + await using (await vhost.Faults.DenyDeclaresAsync()) + { + await vhost.Faults.CloseConsumerConnectionAsync(QueueName); + await vhost.Broker.WaitForQueueDeletionAsync(QueueName); + await vhost.Broker.WaitForNoConsumerAsync(QueueName); + } + + await vhost.Broker.WaitForConsumerAsync(QueueName); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-recovered" }, TestContext.Current.CancellationToken); + + // assert + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal)); + Assert.Equal("ORD-recovered", Assert.Single(recorder.Messages.Cast()).OrderId); + } + + [Fact] + public async Task PublishAsync_Should_DeliverEvent_When_BindingPermissionIsRestored() + { + // arrange + var recorder = new MessageRecorder(); + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await CreateEventBusAsync(vhost, recorder); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await vhost.Faults.AllowReadingQueuesOnlyAsync(); + + // act + try + { + await vhost.Faults.CloseConsumerConnectionAsync(QueueName); + await vhost.Broker.WaitForQueueAsync(QueueName); + // Queue consumption is permitted, but exchange-to-queue binding is denied. + await vhost.Broker.WaitForNoBindingsAsync(QueueName); + } + finally + { + await vhost.Faults.SetReadPermissionAsync(".*"); + } + + await vhost.Broker.WaitForConsumerAsync(QueueName); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-bound" }, TestContext.Current.CancellationToken); + + // assert + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal)); + Assert.Equal("ORD-bound", Assert.Single(recorder.Messages.Cast()).OrderId); + } + + [Fact] + public async Task PublishAsync_Should_RetainEvent_When_QueueIsDeletedWhileHandlersBlockTheConsumer() + { + // arrange + var gate = new HandlerGate(); + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler().MaxConcurrency(1)) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.PublishAsync(new OrderCreated { OrderId = "stuck" }, TestContext.Current.CancellationToken); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + try + { + // act + // the handler ignores its cancellation, so the endpoint cannot consume again, yet its queue and + // bindings have to exist for what is published meanwhile + await vhost.Faults.DeleteQueueAsync("orders"); + await gate.Cancelled.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await Wait.UntilAsync( + () => bus.Transport.ConsumerManager.HealthFailureReason + == "Queue 'orders': The consumer waits for a handler of a lost channel to return."); + await messageBus.PublishAsync( + new OrderCreated { OrderId = "retained" }, + TestContext.Current.CancellationToken); + var whileBlocked = await vhost.Broker.GetQueueAsync("orders"); + gate.Release.TrySetResult(); + var handled = await gate.Handled.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + Assert.Equal(new QueueInfo(Consumers: 0, Ready: 1, Unacknowledged: 0), whileBlocked); + Assert.Equal("retained", handled.OrderId); + } + finally + { + gate.Release.TrySetResult(); + } + } + + private static Task CreateEventBusAsync(VhostContext vhost, MessageRecorder recorder) + { + ((ConnectionFactory)vhost.ConnectionFactory).NetworkRecoveryInterval = TimeSpan.FromSeconds(2); + + return new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint(QueueName).Handler().Temporary()) + .BuildTestBusAsync(); + } + + public sealed class HandlerGate + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Cancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Handled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class StuckOrderHandler(HandlerGate gate) : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + if (message.OrderId != "stuck") + { + gate.Handled.TrySetResult(message); + return; + } + + await using var registration = cancellationToken.Register(() => gate.Cancelled.TrySetResult()); + gate.Started.TrySetResult(); + await gate.Release.Task; + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishIsolationTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishIsolationTests.cs new file mode 100644 index 00000000000..7bea0e1dae9 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishIsolationTests.cs @@ -0,0 +1,230 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class PublishIsolationTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task SendAsync_Should_FailOnlyTheOversizedMessage_When_SentConcurrentlyWithOthers() + { + // arrange + const int rounds = 5; + const int smallPerRound = 100; + await using var maxMessageSize = await fixture.Faults.SetMaxMessageSizeAsync(64 * 1024); + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await CreatePaymentsBusAsync(vhost); + var smallFailures = new ConcurrentQueue(); + var oversizedFailures = new ConcurrentQueue(); + + // act + // the broker closes the channel of a message above its size limit + for (var round = 0; round < rounds; round++) + { + await Task.WhenAll(Enumerable.Range(0, smallPerRound + 1).Select(async i => + { + var isOversized = i == smallPerRound / 2; + var orderId = isOversized ? new string('x', 200_000) : $"ORD-{i}"; + try + { + await SendAsync(bus, new ProcessPayment { OrderId = orderId, Amount = 1 }); + } + catch (Exception ex) + { + (isOversized ? oversizedFailures : smallFailures).Enqueue(ex); + } + })); + } + + // assert + Assert.Empty(smallFailures); + Assert.Equal( + Enumerable.Repeat(Constants.PreconditionFailed, rounds), + oversizedFailures.Select(failure => (failure as AlreadyClosedException)?.ShutdownReason?.ReplyCode)); + Assert.Equal(rounds * smallPerRound, (await vhost.Broker.GetQueueAsync("payments"))?.Messages); + } + + [Fact] + public async Task SendAsync_Should_Complete_When_ConcurrentSendsAreCancelled() + { + // arrange + const int waves = 20; + const int sendsPerWave = 200; + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await CreatePaymentsBusAsync(vhost); + var failures = new ConcurrentQueue(); + + // act + // every second send is cancelled at a random moment of its publish, the others are not cancelled + for (var wave = 0; wave < waves; wave++) + { + await Task.WhenAll(Enumerable.Range(0, sendsPerWave).Select(i => Task.Run( + async () => + { + var isCancelled = i % 2 == 0; + var timeout = isCancelled + ? TimeSpan.FromMilliseconds(Random.Shared.Next(0, 6)) + : TestTimeouts.Signal; + try + { + await SendAsync(bus, new ProcessPayment { OrderId = $"ORD-{i}", Amount = 1 }, timeout); + } + catch (Exception ex) when (!isCancelled) + { + failures.Enqueue(ex); + } + catch (OperationCanceledException) + { + // the cancelled half + } + }, + TestContext.Current.CancellationToken))); + } + + // assert + // a cancelled send can have reached the broker, so the queue holds at least the other half + Assert.Empty(failures); + Assert.InRange( + (await vhost.Broker.GetQueueAsync("payments"))?.Messages ?? 0, + waves * sendsPerWave / 2, + waves * sendsPerWave); + } + + [Fact] + public async Task SendAsync_Should_FailWithBrokerReason_When_HeaderIsLargerThanAFrame() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await CreatePaymentsBusAsync(vhost); + await SendAsync(bus, new ProcessPayment { OrderId = "warmup", Amount = 1 }); + var options = new SendOptions { Headers = new() { ["x-big"] = new string('x', 200_000) } }; + + // act + // the broker closes the connection of a frame it cannot read, and every further attempt would do the same + using var scope = bus.Provider.CreateScope(); + var exception = await Assert.ThrowsAsync( + () => scope.ServiceProvider.GetRequiredService() + .SendAsync( + new ProcessPayment { OrderId = "ORD-1", Amount = 1 }, + options, + TestContext.Current.CancellationToken) + .AsTask() + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + await SendAsync(bus, new ProcessPayment { OrderId = "ORD-2", Amount = 1 }); + + // assert + Assert.Equal(Constants.FrameError, exception.ShutdownReason?.ReplyCode); + Assert.Equal(2, (await vhost.Broker.GetQueueAsync("payments"))?.Messages); + } + + [Fact] + public async Task SendAsync_Should_KeepConnectionAndDeliverOnce_When_BrokerBlocksLongerThanTheClientWaits() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.ContinuationTimeout = TimeSpan.FromSeconds(1); + await using var bus = await CreatePaymentsBusAsync(vhost); + await SendAsync(bus, new ProcessPayment { OrderId = "warmup", Amount = 1 }); + var dispatcher = bus.Transport.Dispatcher; + var original = await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + List sends; + + // act + // every send needs a channel of its own, and the broker answers no request on a connection it blocked + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + sends = [.. Enumerable.Range(0, 5).Select(i => SendAsync(bus, new ProcessPayment { OrderId = $"ORD-{i}", Amount = 1 }))]; + await vhost.Broker.WaitForBlockedPublisherAsync(); + sends.Add(SendAsync(bus, new ProcessPayment { OrderId = "ORD-late", Amount = 1 })); + + // the alarm lasts three times as long as the client waits for an answer to a request + await Task.Delay(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + } + + await Task.WhenAll(sends).WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var current = await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + + // assert + Assert.Same(original, current); + Assert.Equal(7, (await vhost.Broker.GetQueueAsync("payments"))?.Messages); + } + + [Fact] + public async Task SendAsync_Should_ThrowTransportUnavailable_When_BrokerDoesNotConfirmWithinPublishTimeout() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await CreatePaymentsBusAsync(vhost, TimeSpan.FromSeconds(2)); + await SendAsync(bus, new ProcessPayment { OrderId = "warmup", Amount = 1 }); + TransportUnavailableException exception; + + // act + // a broker under a memory alarm takes no message from a publishing connection + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + exception = await Assert.ThrowsAsync( + () => SendAsync(bus, new ProcessPayment { OrderId = "ORD-1", Amount = 1 })); + } + + // assert + Assert.Equal( + "The broker did not confirm the message within the publish timeout of 00:00:02. " + + "The broker may or may not have received the message.", + exception.Message); + } + + [Fact] + public async Task SendAsync_Should_ThrowTransportUnavailable_When_BrokerStaysDownLongerThanPublishTimeout() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await CreatePaymentsBusAsync(vhost, TimeSpan.FromSeconds(2)); + TransportUnavailableException exception; + + // act + await using (await fixture.Faults.StopBrokerAsync()) + { + exception = await Assert.ThrowsAsync( + () => SendAsync(bus, new ProcessPayment { OrderId = "ORD-1", Amount = 1 })); + } + + // the next attempt to reconnect can come later than the publish timeout allows a send to wait + await Wait.UntilAsync(() => bus.Transport.Dispatcher.IsConnected); + await SendAsync(bus, new ProcessPayment { OrderId = "ORD-2", Amount = 1 }); + + // assert + Assert.Equal( + "The broker did not confirm the message within the publish timeout of 00:00:02. " + + "The broker may or may not have received the message.", + exception.Message); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("payments"))?.Messages); + } + + private static Task CreatePaymentsBusAsync(VhostContext vhost, TimeSpan? publishTimeout = null) + => new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + if (publishTimeout is { } timeout) + { + t.PublishTimeout(timeout); + } + + t.DeclareQueue("payments"); + t.DispatchEndpoint("payment-send").ToQueue("payments").Send(); + }) + .BuildTestBusAsync(); + + private static async Task SendAsync(TestBus bus, ProcessPayment message, TimeSpan? timeout = null) + { + using var scope = bus.Provider.CreateScope(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(timeout ?? TestTimeouts.Signal); + await scope.ServiceProvider.GetRequiredService().SendAsync(message, cts.Token); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishSubscribeTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishSubscribeTests.cs index e7130c93b40..4d6e28bde49 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishSubscribeTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishSubscribeTests.cs @@ -3,10 +3,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Messaging)] public class PublishSubscribeTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public PublishSubscribeTests(RabbitMQFixture fixture) @@ -35,7 +34,7 @@ public async Task PublishAsync_Should_DeliverToHandler_When_SingleHandlerRegiste await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-1" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the event within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the event within timeout"); var message = Assert.Single(recorder.Messages); var order = Assert.IsType(message); @@ -66,8 +65,8 @@ public async Task PublishAsync_Should_FanOutToAllHandlers_When_MultipleHandlersR await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-1" }, CancellationToken.None); // assert - Assert.True(await recorder1.WaitAsync(s_timeout), "First handler did not receive the event"); - Assert.True(await recorder2.WaitAsync(s_timeout), "Second handler did not receive the event"); + Assert.True(await recorder1.WaitAsync(TestTimeouts.Signal), "First handler did not receive the event"); + Assert.True(await recorder2.WaitAsync(TestTimeouts.Signal), "Second handler did not receive the event"); Assert.Single(recorder1.Messages); Assert.Single(recorder2.Messages); @@ -97,7 +96,7 @@ public async Task PublishAsync_Should_DeliverAll_When_MultipleEventsSequential() // assert Assert.True( - await recorder.WaitAsync(s_timeout, expectedCount: 3), + await recorder.WaitAsync(TestTimeouts.Signal, expectedCount: 3), "Handler did not receive all 3 events within timeout"); Assert.Equal(3, recorder.Messages.Count); @@ -133,7 +132,7 @@ public async Task PublishAsync_Should_DeliverAll_When_RapidFire() // assert Assert.True( - await recorder.WaitAsync(s_timeout, expectedCount: messageCount), + await recorder.WaitAsync(TestTimeouts.Signal, expectedCount: messageCount), $"Handler did not receive all {messageCount} events within timeout"); Assert.Equal(messageCount, recorder.Messages.Count); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublisherRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublisherRecoveryTests.cs new file mode 100644 index 00000000000..8a57969b355 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublisherRecoveryTests.cs @@ -0,0 +1,297 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class PublisherRecoveryTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task PublishAsync_Should_Cancel_When_BrokerBlocksPublishing() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + + // act + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + await Assert.ThrowsAnyAsync( + () => PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }, TimeSpan.FromSeconds(2)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-3" }); + + // assert + Assert.True(await Condition.WaitAsync( + () => orders.Messages.Cast().Any(order => order.OrderId == "ORD-3"), + TestTimeouts.Signal)); + } + + [Fact] + public async Task PublishAsync_Should_Deliver_When_ConnectionClosedWhileAwaitingConfirm() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + var dispatcherConnectionLost = await ObserveDispatcherConnectionLossAsync(bus); + + // act + // the broker holds back the confirm of a blocked publish, so the connection closes while it is awaited + Task publish; + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + publish = PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }); + await vhost.Broker.WaitForBlockedPublisherAsync(); + await vhost.Faults.CloseConnectionsAsync(); + await dispatcherConnectionLost.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + + await publish.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await Wait.UntilAsync(() => orders.Messages.Cast().Any(order => order.OrderId == "ORD-2")); + + // assert + Assert.Equal( + ["ORD-1", "ORD-2"], + orders.Messages.Cast().Select(order => order.OrderId).Distinct().Order()); + } + + [Fact] + public async Task PublishAsync_Should_Cancel_When_BrokerRemainsOffline() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await CreateOrdersBusAsync(vhost, orders); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + + // act + await using (await fixture.Faults.StopBrokerAsync()) + { + await Assert.ThrowsAnyAsync( + () => PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }, TimeSpan.FromSeconds(3)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-3" }); + + // assert + Assert.True(await Condition.WaitAsync( + () => orders.Messages.Cast().Any(order => order.OrderId == "ORD-3"), + TestTimeouts.Signal)); + } + + [Fact] + public async Task PublishAsync_Should_CompletePendingPublish_When_BrokerReturns() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await CreateOrdersBusAsync(vhost, orders); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + + // act + Task publish; + await using (await fixture.Faults.StopBrokerAsync()) + { + publish = PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }); + await Assert.ThrowsAsync( + () => publish.WaitAsync(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken)); + } + + await publish.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + Assert.True(await Condition.WaitAsync( + () => orders.Messages.Cast().Any(order => order.OrderId == "ORD-2"), + TestTimeouts.Signal)); + } + + [Fact] + public async Task PublishAsync_Should_DeliverEveryMessage_When_ConnectionDropsDuringConcurrentPublishes() + { + // arrange + const int count = 500; + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await CreateOrdersBusAsync(vhost, orders); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-0" }); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + var dispatcherConnectionLost = await ObserveDispatcherConnectionLossAsync(bus); + + // act + Task[] publishes; + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + publishes = Enumerable.Range(1, count) + .Select(i => PublishAsync(bus, new OrderCreated { OrderId = $"ORD-{i}" })) + .ToArray(); + await vhost.Broker.WaitForBlockedPublisherAsync(); + await vhost.Faults.CloseConnectionsAsync(); + await dispatcherConnectionLost.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + + await Task.WhenAll(publishes).WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await Wait.UntilAsync( + () => orders.Messages.Cast().Select(o => o.OrderId).Distinct().Count() == count + 1); + + // assert + Assert.Equal( + Enumerable.Range(0, count + 1).Select(i => $"ORD-{i}").Order(), + orders.Messages.Cast().Select(o => o.OrderId).Distinct().Order()); + } + + [Fact] + public async Task PublishAsync_Should_Deliver_When_PreviousPublishWasCancelled() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.ContinuationTimeout = TimeSpan.FromSeconds(1); + var orders = new MessageRecorder(); + await using var bus = await CreateOrdersBusAsync(vhost, orders); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + + // act + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + await Assert.ThrowsAnyAsync( + () => PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }, TimeSpan.FromSeconds(2)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-3" }); + + // assert + Assert.True(await Condition.WaitAsync( + () => orders.Messages.Cast().Any(o => o.OrderId == "ORD-3"), + TestTimeouts.Signal)); + } + + [Fact] + public async Task PublishAsync_Should_Deliver_When_DispatchOnlyTransportRestarted() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var consumer = await CreateOrdersBusAsync(vhost, orders); + await using var publisher = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(_ => { }) + .BuildTestBusAsync(); + var runtime = (MessagingRuntime)publisher.Provider.GetRequiredService(); + var transport = runtime.Transports.OfType().Single(); + + // act + await transport.StopAsync(runtime, TestContext.Current.CancellationToken); + await transport.StartAsync(runtime, TestContext.Current.CancellationToken); + await PublishAsync(publisher, new OrderCreated { OrderId = "ORD-1" }); + var received = await orders.WaitAsync(TestTimeouts.Signal); + + // assert + Assert.True(received); + } + + [Fact] + public async Task Consumer_Should_KeepAcking_When_PublishingConnectionBlocked() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var gate = new HandlerGate(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler().MaxConcurrency(1)) + .BuildTestBusAsync(); + for (var i = 1; i <= 5; i++) + { + await PublishAsync(bus, new OrderCreated { OrderId = $"ORD-{i}" }); + } + + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + // the alarm blocks connections that publish, and the consumer connection only acks + QueueInfo? whileBlocked; + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + await vhost.Broker.WaitForAlarmAsync(); + gate.Release.TrySetResult(); + await vhost.Broker.WaitForEmptyQueueAsync("orders"); + whileBlocked = await vhost.Broker.GetQueueAsync("orders"); + } + + // assert + Assert.Equal(new QueueInfo(Consumers: 1, Ready: 0, Unacknowledged: 0), whileBlocked); + Assert.Equal(5, orders.Messages.Count); + } + + private static Task CreateOrdersBusAsync(VhostContext vhost, MessageRecorder orders) + => new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + + /// + /// Returns a task that completes once the client has seen the current connection of the dispatcher shut down. + /// + private static async Task ObserveDispatcherConnectionLossAsync(TestBus bus) + { + var connection = await bus.Transport.Dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + return connection.WhenShutdownAsync(); + } + + private static async Task PublishAsync(TestBus bus, OrderCreated message, TimeSpan? timeout = null) + { + using var scope = bus.Provider.CreateScope(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(timeout ?? TestTimeouts.Signal); + await scope.ServiceProvider.GetRequiredService().PublishAsync(message, cts.Token); + } + + public sealed class HandlerGate + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class GatedOrderHandler(HandlerGate gate, MessageRecorder recorder) : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + gate.Started.TrySetResult(); + await gate.Release.Task.WaitAsync(cancellationToken); + recorder.Record(message); + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishingConcurrencyTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishingConcurrencyTests.cs new file mode 100644 index 00000000000..6c12b4ba960 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishingConcurrencyTests.cs @@ -0,0 +1,129 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class PublishingConcurrencyTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task SendAsync_Should_StayWithinChannelLimit_When_ThousandsSendConcurrently() + { + // arrange + const int count = 3000; + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 128; + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue("burst"); + t.DispatchEndpoint("burst-send").ToQueue("burst").Send(); + }) + .BuildTestBusAsync(); + + // act + var sends = Enumerable.Range(0, count) + .Select(i => SendAsync(bus, new ProcessPayment { OrderId = $"ORD-{i}", Amount = 1 }, TestTimeouts.Signal)); + await Task.WhenAll(sends); + + // assert + Assert.Equal(count, (await vhost.Broker.GetQueueAsync("burst"))?.Messages); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task SendAsync_Should_Deliver_When_PreviousSendsWereCancelled(bool warmup) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 128; + var payments = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(payments) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ(t => + { + t.Endpoint("cancelled-sends").Handler(); + t.DispatchEndpoint("cancelled-send").ToQueue("cancelled-sends").Send(); + }) + .BuildTestBusAsync(); + if (warmup) + { + await SendAsync(bus, new ProcessPayment { OrderId = "warmup", Amount = 1 }); + } + + // act + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + for (var wave = 0; wave < 5; wave++) + { + await Task.WhenAll(Enumerable.Range(0, 100).Select(i => + Assert.ThrowsAnyAsync( + () => SendAsync( + bus, + new ProcessPayment { OrderId = $"{wave}-{i}", Amount = 1 }, + TimeSpan.FromMilliseconds(300))))); + } + } + + await SendAsync(bus, new ProcessPayment { OrderId = "resumed", Amount = 1 }); + + // assert + Assert.True(await Condition.WaitAsync( + () => payments.Messages.Cast().Any(p => p.OrderId == "resumed"), + TestTimeouts.Signal)); + Assert.Equal(1m, Assert.Single(payments.Messages.Cast(), p => p.OrderId == "resumed").Amount); + Assert.Equal(2, await vhost.Broker.GetConnectionCountAsync()); + } + + [Fact] + public async Task PublishAsync_Should_DeliverEveryMessage_When_FirstPublishesRunConcurrently() + { + // arrange + const int count = 25; + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + + // act + await Task.WhenAll(Enumerable.Range(0, count).Select(i => messageBus.PublishAsync( + new OrderCreated { OrderId = $"ORD-{i}" }, + TestContext.Current.CancellationToken).AsTask())); + + // assert + Assert.True(await orders.WaitAsync(TestTimeouts.Signal, count)); + Assert.Equal( + Enumerable.Range(0, count).Select(i => $"ORD-{i}").Order(), + orders.Messages.Cast().Select(order => order.OrderId).Order()); + } + + private static async Task SendAsync(TestBus bus, ProcessPayment message, TimeSpan? timeout = null) + { + using var scope = bus.Provider.CreateScope(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(timeout ?? TestTimeouts.Signal); + await scope.ServiceProvider.GetRequiredService().SendAsync(message, cts.Token); + } + + public sealed class PaymentHandler(MessageRecorder recorder) : IEventRequestHandler + { + public ValueTask HandleAsync(ProcessPayment request, CancellationToken cancellationToken) + { + recorder.Record(request); + return default; + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishingReliabilityTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishingReliabilityTests.cs new file mode 100644 index 00000000000..21e361d973c --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/PublishingReliabilityTests.cs @@ -0,0 +1,270 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class PublishingReliabilityTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task SendAsync_Should_DeliverOnce_When_ConcurrentPublishTargetsMissingExchange() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareExchange("orders"); + t.DeclareQueue("healthy"); + t.DispatchEndpoint("orders-publish").ToExchange("orders").Publish(); + t.DispatchEndpoint("healthy-send").ToQueue("healthy").Send(); + }) + .BuildTestBusAsync(); + await PublishAsync(bus, new OrderCreated { OrderId = "warmup" }); + await SendAsync(bus, new ProcessPayment { OrderId = "warmup", Amount = 1 }); + var dispatcher = bus.Provider.GetRequiredService() + .Transports.OfType().Single().Dispatcher; + var first = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + var second = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + dispatcher.Return(first, reusable: true); + dispatcher.Return(second, reusable: true); + await DeleteExchangeAsync(vhost, "orders"); + + // act + Task publish; + Task send; + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + publish = PublishAsync(bus, new OrderCreated { OrderId = "missing-exchange" }); + send = SendAsync(bus, new ProcessPayment { OrderId = "healthy", Amount = 1 }); + } + + var exception = await Assert.ThrowsAsync(() => publish); + await send; + + // assert + Assert.Equal(Constants.NotFound, exception.ShutdownReason?.ReplyCode); + Assert.Equal(2, (await vhost.Broker.GetQueueAsync("healthy"))?.Messages); + } + + [Fact] + public async Task SendAsync_Should_Throw_When_BrokerRejectsMessage() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue("bounded") + .WithArgument("x-max-length", 1) + .WithArgument("x-overflow", "reject-publish"); + t.DispatchEndpoint("bounded-send").ToQueue("bounded").Send(); + }) + .BuildTestBusAsync(); + + // act + await SendAsync(bus, new ProcessPayment { OrderId = "ORD-1", Amount = 1 }); + var exception = await Assert.ThrowsAsync( + () => SendAsync(bus, new ProcessPayment { OrderId = "ORD-2", Amount = 1 })); + + // assert + Assert.False(exception.IsReturn); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("bounded"))?.Messages); + } + + [Fact] + public async Task SendAsync_Should_Deliver_When_PreviousPublishMadeBrokerCloseChannel() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var payments = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(payments) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ(t => + { + t.DispatchEndpoint("orders-publish").ToExchange("orders").Publish(); + t.Endpoint("payments").Handler(); + }) + .BuildTestBusAsync(); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + await SendAsync(bus, new ProcessPayment { OrderId = "ORD-1", Amount = 1 }); + Assert.True(await payments.WaitAsync(TestTimeouts.Signal)); + await DeleteExchangeAsync(vhost, "orders"); + + // act + var exception = await Assert.ThrowsAsync( + () => PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" })); + await SendAsync(bus, new ProcessPayment { OrderId = "ORD-3", Amount = 1 }); + + // assert + Assert.Equal(Constants.NotFound, exception.ShutdownReason?.ReplyCode); + Assert.True(await payments.WaitAsync(TestTimeouts.Signal)); + Assert.Equal(["ORD-1", "ORD-3"], payments.Messages.Cast().Select(p => p.OrderId).Order()); + } + + [Fact] + public async Task PublishAsync_Should_FailFast_When_MessageLargerThanMaxMessageSize() + { + // arrange + await using var maxMessageSize = await fixture.Faults.SetMaxMessageSizeAsync(4096); + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await CreateOrdersBusAsync(vhost, orders); + + // act + var exception = await Assert.ThrowsAsync( + () => PublishAsync(bus, new OrderCreated { OrderId = new string('x', 100_000) }) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }); + + // assert + Assert.Equal(Constants.PreconditionFailed, exception.ShutdownReason?.ReplyCode); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + Assert.Equal("ORD-2", Assert.Single(orders.Messages.Cast()).OrderId); + } + + [Fact] + public async Task Fault_Should_ReachErrorQueue_When_ErrorQueueDeletedByOperator() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + Assert.True(await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders_error"))?.Messages == 1, + TestTimeouts.Signal)); + + // act + // without mandatory, the broker confirms a message it cannot route and drops it + await vhost.Faults.DeleteQueueAsync("orders_error"); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }); + var deadLettered = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders_error"))?.Messages == 1, + TestTimeouts.Signal); + var pending = (await vhost.Broker.GetQueueAsync("orders"))?.Messages; + + // assert + Assert.True(deadLettered); + Assert.Equal(0, pending); + } + + [Fact] + public async Task Handler_Should_FaultMessageAndKeepConsuming_When_PublishInHandlerFails() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var orders = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => + { + t.DispatchEndpoint("invoices-publish").ToExchange("invoices").Publish(); + t.Endpoint("orders").Handler(); + }) + .BuildTestBusAsync(); + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-1" }); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + await DeleteExchangeAsync(vhost, "invoices"); + + // act + // the handler's publish to the deleted exchange makes the broker close its channel with 404 + await PublishAsync(bus, new OrderCreated { OrderId = "ORD-2" }); + var faulted = await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync("orders_error"))?.Messages == 1, + TestTimeouts.Signal); + await PublishAsync(bus, new OrderCreated { OrderId = InvoicingOrderHandler.WithoutInvoice }); + var nextHandled = await orders.WaitAsync(TestTimeouts.Signal); + var queue = await vhost.Broker.GetQueueAsync("orders"); + + // assert + Assert.True(faulted); + Assert.True(nextHandled); + Assert.Equal(1, queue?.Consumers); + } + + private static Task CreateOrdersBusAsync(VhostContext vhost, MessageRecorder orders) + => new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + + private static async Task DeleteExchangeAsync(VhostContext vhost, string exchange) + { + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync( + TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + cancellationToken: TestContext.Current.CancellationToken); + await channel.ExchangeDeleteAsync(exchange, cancellationToken: TestContext.Current.CancellationToken); + } + + private static async Task SendAsync(TestBus bus, ProcessPayment message, TimeSpan? timeout = null) + { + using var scope = bus.Provider.CreateScope(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(timeout ?? TestTimeouts.Signal); + await scope.ServiceProvider.GetRequiredService().SendAsync(message, cts.Token); + } + + private static async Task PublishAsync(TestBus bus, OrderCreated message, TimeSpan? timeout = null) + { + using var scope = bus.Provider.CreateScope(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(timeout ?? TestTimeouts.Signal); + await scope.ServiceProvider.GetRequiredService().PublishAsync(message, cts.Token); + } + + public sealed class InvoiceRequested + { + public required string OrderId { get; init; } + } + + public sealed class FailingOrderHandler : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + => throw new InvalidOperationException($"Order {message.OrderId} failed."); + } + + public sealed class InvoicingOrderHandler(IMessageBus bus, MessageRecorder recorder) : IEventHandler + { + public const string WithoutInvoice = "ORD-WITHOUT-INVOICE"; + + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + if (message.OrderId != WithoutInvoice) + { + await bus.PublishAsync(new InvoiceRequested { OrderId = message.OrderId }, cancellationToken); + } + + recorder.Record(message); + } + } + + public sealed class PaymentHandler(MessageRecorder recorder) : IEventRequestHandler + { + public ValueTask HandleAsync(ProcessPayment request, CancellationToken cancellationToken) + { + recorder.Record(request); + return default; + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/QuorumFailoverTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/QuorumFailoverTests.cs new file mode 100644 index 00000000000..e2e8531d656 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/QuorumFailoverTests.cs @@ -0,0 +1,259 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +public sealed class QuorumFailoverTests(RabbitMQClusterFixture fixture) : IClassFixture +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Bus_Should_DeliverEventsAndReplies_When_ConnectedClusterNodeIsKilled(bool automaticRecoveryEnabled) + { + // arrange + var queue = $"cluster-orders-{automaticRecoveryEnabled}"; + var primary = fixture.CreateConnectionFactory(0); + primary.AutomaticRecoveryEnabled = automaticRecoveryEnabled; + var secondary = fixture.CreateConnectionFactory(1); + var provider = new ClusterConnectionProvider(primary, [primary.Endpoint, secondary.Endpoint]); + var recorder = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRequestHandler() + .AddRabbitMQ(t => + { + t.ConnectionProvider(_ => provider); + t.ConfigureDefaults(defaults => defaults.Queue.QueueType = RabbitMQQueueType.Quorum); + t.Endpoint(queue).Handler(); + t.Endpoint($"cluster-requests-{automaticRecoveryEnabled}").Handler(); + }) + .BuildTestBusAsync(); + var transport = bus.Provider.GetRequiredService().Transports + .OfType().Single(); + using var scope = bus.Provider.CreateScope(); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(45)); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.RequestAsync(new GetOrderStatus { OrderId = "before-failover" }, deadline.Token); + var before = await transport.ConsumerManager.Connection.GetConnectionAsync(deadline.Token); + Assert.Equal(primary.Port, before.Endpoint.Port); + + // act + await using var stopped = await fixture.KillNodeAsync(0); + await messageBus.PublishAsync(new OrderCreated { OrderId = "after-failover" }, deadline.Token); + var response = await messageBus.RequestAsync(new GetOrderStatus { OrderId = "after-failover" }, deadline.Token); + var settled = await Condition.WaitAsync( + async () => + !recorder.Messages.IsEmpty + && (await fixture.GetQueueStateAsync(1, queue))?.Messages == 0 + && transport.ConsumerManager.IsHealthy, + TestTimeouts.Signal); + var after = await transport.ConsumerManager.Connection.GetConnectionAsync(deadline.Token); + + // assert + // Recovery may redeliver messages whose acknowledgements were lost. + Assert.Equal(["after-failover"], + recorder.Messages.Cast().Select(message => message.OrderId).Distinct()); + Assert.True(settled); + Assert.Equal("after-failover", response.OrderId); + Assert.Equal(secondary.Port, after.Endpoint.Port); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Bus_Should_PreserveMessagesAndKeepConsuming_When_QuorumLeaderIsKilled(bool automaticRecoveryEnabled) + { + // arrange + var queue = $"orders-{automaticRecoveryEnabled}"; + var leaderFactory = fixture.CreateConnectionFactory(0); + await using (var connection = await leaderFactory.CreateConnectionAsync(TestContext.Current.CancellationToken)) + await using (var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken)) + { + var arguments = new Dictionary + { + ["x-queue-type"] = "quorum", + ["x-queue-leader-locator"] = "client-local" + }; + await channel.QueueDeclareAsync( + queue, + durable: true, + exclusive: false, + autoDelete: false, + arguments: arguments, + cancellationToken: TestContext.Current.CancellationToken); + } + + var before = await fixture.GetQueueStateAsync(1, queue); + Assert.Equal(fixture.GetNodeName(0), before?.Leader); + Assert.Equal(Enumerable.Range(0, 3).Select(fixture.GetNodeName), before!.Members.Order()); + var consumerFactory = fixture.CreateConnectionFactory(1); + consumerFactory.AutomaticRecoveryEnabled = automaticRecoveryEnabled; + var gate = new HandlerGate(); + var recorder = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(consumerFactory) + .AddSingleton(gate) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => + { + t.ConfigureDefaults(defaults => defaults.Queue.QueueType = RabbitMQQueueType.Quorum); + t.DeclareQueue(queue).WithArgument("x-queue-leader-locator", "client-local"); + t.Endpoint(queue).Handler().MaxConcurrency(1).MaxPrefetch(1); + }) + .BuildTestBusAsync(); + var transport = bus.Provider.GetRequiredService().Transports + .OfType().Single(); + using var scope = bus.Provider.CreateScope(); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromMinutes(1)); + var messageBus = scope.ServiceProvider.GetRequiredService(); + for (var i = 0; i < 25; i++) + { + await messageBus.PublishAsync(new OrderCreated { OrderId = $"ORD-{i}" }, deadline.Token); + } + + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, deadline.Token); + + // act + await using var stopped = await fixture.KillNodeAsync(0); + var elected = await Condition.WaitAsync( + async () => + await fixture.GetQueueStateAsync(1, queue) is { Leader: { Length: > 0 } leader } + && leader != before.Leader, + TestTimeouts.Signal); + gate.Release.TrySetResult(); + await messageBus.PublishAsync(new OrderCreated { OrderId = "after-failover" }, deadline.Token); + var settled = await Condition.WaitAsync( + async () => + recorder.Messages.Cast().Select(m => m.OrderId).Distinct().Count() == 26 + && (await fixture.GetQueueStateAsync(1, queue))?.Messages == 0 + && transport.ConsumerManager.IsHealthy, + TestTimeouts.Signal); + + // assert + Assert.True(elected); + Assert.Equal( + Enumerable.Range(0, 25).Select(i => $"ORD-{i}").Append("after-failover").Order(), + recorder.Messages.Cast().Select(m => m.OrderId).Distinct().Order()); + Assert.True(settled); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Bus_Should_RedeliverUnacknowledgedEvent_When_ConsumerNodeIsKilled(bool automaticRecoveryEnabled) + { + // arrange + var queue = $"unacknowledged-orders-{automaticRecoveryEnabled}"; + var secondary = fixture.CreateConnectionFactory(1); + await using (var connection = await secondary.CreateConnectionAsync(TestContext.Current.CancellationToken)) + await using (var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken)) + { + await channel.QueueDeclareAsync( + queue, + durable: true, + exclusive: false, + autoDelete: false, + arguments: new Dictionary + { + ["x-queue-type"] = "quorum", + ["x-queue-leader-locator"] = "client-local" + }, + cancellationToken: TestContext.Current.CancellationToken); + } + + var primary = fixture.CreateConnectionFactory(0); + primary.AutomaticRecoveryEnabled = automaticRecoveryEnabled; + var provider = new ClusterConnectionProvider(primary, [primary.Endpoint, secondary.Endpoint]); + var gate = new HandlerGate(); + var recorder = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(gate) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => + { + t.ConnectionProvider(_ => provider); + t.ConfigureDefaults(defaults => defaults.Queue.QueueType = RabbitMQQueueType.Quorum); + t.DeclareQueue(queue).WithArgument("x-queue-leader-locator", "client-local"); + t.Endpoint(queue).Handler().MaxConcurrency(1).MaxPrefetch(1); + }) + .BuildTestBusAsync(); + var transport = bus.Provider.GetRequiredService().Transports + .OfType().Single(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.PublishAsync(new OrderCreated { OrderId = "unacknowledged" }, TestContext.Current.CancellationToken); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var before = await transport.ConsumerManager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + Assert.Equal(primary.Port, before.Endpoint.Port); + Assert.Equal(fixture.GetNodeName(1), (await fixture.GetQueueStateAsync(1, queue))?.Leader); + + // act + // the first delivery cannot be acknowledged on the killed node, so the event is delivered a second time + await using var stopped = await fixture.KillNodeAsync(0); + gate.Release.TrySetResult(); + var settled = await Condition.WaitAsync( + async () => + gate.Invocations == 2 + && !recorder.Messages.IsEmpty + && (await fixture.GetQueueStateAsync(1, queue))?.Messages == 0 + && transport.ConsumerManager.IsHealthy, + TestTimeouts.Signal); + + // assert + Assert.True(settled); + Assert.Equal(2, gate.Invocations); + Assert.Equal(["unacknowledged"], + recorder.Messages.Cast().Select(message => message.OrderId).Distinct()); + } + + public sealed class HandlerGate + { + private int _invocations; + + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int Invocations => Volatile.Read(ref _invocations); + + public void HandlerInvoked() => Interlocked.Increment(ref _invocations); + } + + public sealed class QuorumOrderHandler(HandlerGate gate, MessageRecorder recorder) : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + gate.HandlerInvoked(); + gate.Started.TrySetResult(); + await gate.Release.Task.WaitAsync(cancellationToken); + recorder.Record(message); + } + } + + private sealed class ClusterConnectionProvider(ConnectionFactory factory, AmqpTcpEndpoint[] endpoints) + : IRabbitMQConnectionProvider + { + public string Host => factory.HostName; + + public string VirtualHost => factory.VirtualHost; + + public int Port => factory.Port; + + public async ValueTask CreateAsync(CancellationToken cancellationToken) + => await factory.CreateConnectionAsync(new OrderedEndpointResolver(endpoints), null, cancellationToken); + } + + private sealed class OrderedEndpointResolver(AmqpTcpEndpoint[] endpoints) : IEndpointResolver + { + public IEnumerable All() => endpoints; + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RecoveryLogTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RecoveryLogTests.cs new file mode 100644 index 00000000000..1410d8d537f --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RecoveryLogTests.cs @@ -0,0 +1,477 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Mocha.Transport.RabbitMQ.Tests.Helpers; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +/// +/// Pins what an operator reads at Information level and above for each incident. +/// +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class RecoveryLogTests(RabbitMQFixture fixture) +{ + private const string Template = "{OriginalFormat}"; + + [Fact] + public async Task Log_Should_ReportLossAttemptsAndRecoveryPerConnection_When_BrokerRestarts() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var logs = new LogRecorder(); + var orders = new MessageRecorder(); + await using var bus = await CreateOrdersBusAsync(vhost, logs, orders); + await PublishAsync(bus, "ORD-1"); + Assert.True(await orders.WaitAsync(TestTimeouts.Signal)); + var before = await vhost.Broker.GetConsumerAsync("orders"); + logs.Clear(); + + // act + await using (await fixture.Faults.StopBrokerAsync()) + { + await WaitForFailedAttemptOfBothConnectionsAsync(logs); + } + + await vhost.Broker.WaitForConsumerAsync("orders", otherThan: before); + await Wait.UntilAsync(() => bus.Transport.HealthFailureReason is null); + var entries = logs.GetTransportEntries(); + + // assert + Assert.Equal( + [ + "Warning ConnectionLost", + "Warning ConnectionAttemptFailed", + "Information ConnectionReady", + "Information ConsumerAttached" + ], + Summarize(entries, "consumer")); + Assert.Equal( + ["Warning ConnectionLost", "Warning ConnectionAttemptFailed", "Information ConnectionReady"], + Summarize(entries, "dispatcher")); + } + + [Fact] + public async Task Log_Should_ReportOneErrorPerAttemptAndConnection_When_AccessIsRevoked() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var logs = new LogRecorder(); + var orders = new MessageRecorder(); + await using var bus = await CreateOrdersBusAsync(vhost, logs, orders); + var before = await vhost.Broker.GetConsumerAsync("orders"); + logs.Clear(); + + // act + await using (await vhost.Faults.RevokeAccessAsync()) + { + await vhost.Faults.CloseConnectionsAsync(); + await WaitForFailedAttemptOfBothConnectionsAsync(logs); + } + + await vhost.Broker.WaitForConsumerAsync("orders", otherThan: before); + await Wait.UntilAsync(() => bus.Transport.HealthFailureReason is null); + var entries = logs.GetTransportEntries(); + + // assert + Assert.Equal( + [ + "Warning ConnectionLost", + "Error ConnectionAttemptFailed", + "Information ConnectionReady", + "Information ConsumerAttached" + ], + Summarize(entries, "consumer")); + Assert.Equal( + ["Warning ConnectionLost", "Error ConnectionAttemptFailed", "Information ConnectionReady"], + Summarize(entries, "dispatcher")); + Assert.Equal( + "RabbitMQ connection {ConnectionName} is unavailable for {Elapsed}: attempt {Attempt} failed. " + + "Retrying in {Delay}", + logs.GetEntries("ConnectionAttemptFailed")[0].Properties[Template]); + } + + [Fact] + public async Task Log_Should_NameTheMissedHeartbeat_When_ConnectionStopsAnswering() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedHeartbeat = TimeSpan.FromSeconds(2); + var logs = new LogRecorder(); + await using var bus = await CreateOrdersBusAsync(vhost, logs, new MessageRecorder()); + logs.Clear(); + LogEntry lost; + + // act + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + lost = await logs.WaitForAsync("ConnectionLost"); + } + + // assert + Assert.Equal(LogLevel.Warning, lost.Level); + Assert.Equal("Heartbeat missing with heartbeat == 00:00:02 seconds", lost.Properties["Cause"]); + Assert.Equal( + "RabbitMQ connection {ConnectionName} was lost: {ReplyCode} {ReplyText} (initiator: {Initiator}, " + + "cause: {Cause}). Reconnecting in the background", + lost.Properties[Template]); + } + + [Theory] + [InlineData(1, 1)] + [InlineData(4, 4)] + [InlineData(4, 1)] + public async Task Log_Should_CountCancelledHandlersWithoutAnError_When_HandlersObserveCancellation( + int maxConcurrency, + int runningHandlers) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var logs = new LogRecorder(); + var gate = new HandlerGate(); + await using var bus = await CreateServices(vhost, logs) + .AddSingleton(gate) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler().MaxConcurrency(maxConcurrency)) + .BuildTestBusAsync(); + for (var i = 0; i < runningHandlers; i++) + { + await PublishAsync(bus, $"ORD-{i}"); + } + + await Wait.UntilAsync(() => gate.StartedHandlers == runningHandlers); + var before = await vhost.Broker.GetConsumerAsync("orders"); + logs.Clear(); + + // act + await vhost.Faults.CloseConsumerConnectionAsync("orders"); + await vhost.Broker.WaitForConsumerAsync("orders", otherThan: before); + gate.Release.TrySetResult(); + var lost = Assert.Single(logs.GetEntries("ConsumerChannelLost"), entry => IsAbout(entry, "orders")); + + // assert + Assert.Empty(logs.GetTransportEntries(LogLevel.Error)); + Assert.Equal(LogLevel.Warning, lost.Level); + Assert.Equal(runningHandlers, lost.Properties["CancelledHandlers"]); + Assert.Equal( + "The consumer for queue {Queue} on {ConnectionName} lost its channel: {ReplyCode} {ReplyText} " + + "(initiator: {Initiator}). {CancelledHandlers} running handlers were cancelled, and the broker " + + "delivers their messages again", + lost.Properties[Template]); + } + + [Fact] + public async Task Log_Should_ReportBlockedConsumerAsError_When_HandlersIgnoreCancellationPastTheGracePeriod() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var logs = new LogRecorder(); + var gate = new HandlerGate(); + await using var bus = await CreateStuckOrdersBusAsync(vhost, logs, gate); + bus.Transport.ConsumerManager.BlockedGracePeriod = TimeSpan.FromSeconds(1); + await PublishAsync(bus, "stuck"); + await Wait.UntilAsync(() => gate.StartedHandlers == 1); + logs.Clear(); + + try + { + // act + await vhost.Faults.DeleteQueueAsync("orders"); + var blocked = await logs.WaitForAsync("ConsumerBlockedByHandlers"); + gate.Release.TrySetResult(); + await vhost.Broker.WaitForConsumerAsync("orders"); + await Wait.UntilAsync(() => bus.Transport.HealthFailureReason is null); + + // assert + Assert.Equal( + [ + "Warning ConsumerCancelled", + "Error ConsumerBlockedByHandlers", + "Information ConsumerUnblocked", + "Information ConsumerAttached" + ], + Summarize(logs.GetTransportEntries().Where(entry => IsAbout(entry, "orders")), "consumer")); + Assert.Equal( + "The consumer for queue {Queue} on {ConnectionName} cannot attach: all {MaxConcurrency} handlers " + + "of a lost channel are still running {GracePeriod} after they were cancelled. The endpoint " + + "consumes again when one of them returns", + blocked.Properties[Template]); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task Log_Should_ReportDrain_When_BusStopsWhileAHandlerRuns() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var logs = new LogRecorder(); + var gate = new HandlerGate(); + await using var bus = await CreateStuckOrdersBusAsync(vhost, logs, gate); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + await PublishAsync(bus, "stuck"); + await Wait.UntilAsync(() => gate.StartedHandlers == 1); + logs.Clear(); + + // act + var stop = runtime.StopAsync(TestContext.Current.CancellationToken).AsTask(); + await logs.WaitForAsync("ConsumerDrainStarted"); + gate.Release.TrySetResult(); + await stop.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var entries = logs.GetTransportEntries(); + + // assert + Assert.Equal( + ["Information ConsumerDrainStarted", "Information ConsumerStopped"], + Summarize(entries.Where(entry => IsAbout(entry, "orders")), "consumer")); + Assert.Equal( + ["Information ConsumerStopped"], + Summarize(entries.Where(entry => !IsAbout(entry, "orders")), "consumer")); + } + + [Fact] + public async Task Log_Should_SayThatHandlersWereCancelled_When_StopDeadlineElapses() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var logs = new LogRecorder(); + var gate = new HandlerGate(); + await using var bus = await CreateStuckOrdersBusAsync(vhost, logs, gate); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + await PublishAsync(bus, "stuck"); + await Wait.UntilAsync(() => gate.StartedHandlers == 1); + logs.Clear(); + + try + { + // act + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + await runtime.StopAsync(deadline.Token).AsTask() + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var entries = logs.GetTransportEntries(); + + // assert + Assert.Equal( + [ + "Information ConsumerDrainStarted", + "Warning ConsumerDrainTimedOut", + "Information ConsumerStopped" + ], + Summarize(entries.Where(entry => IsAbout(entry, "orders")), "consumer")); + Assert.Equal( + ["Information ConsumerStopped"], + Summarize(entries.Where(entry => !IsAbout(entry, "orders")), "consumer")); + Assert.Equal( + "The consumer for queue {Queue} on {ConnectionName} stopped waiting for its handlers after " + + "{Elapsed}. {RunningHandlers} handlers were cancelled, and the broker returns their " + + "unacknowledged messages to the queue", + logs.GetEntries("ConsumerDrainTimedOut")[0].Properties[Template]); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task Log_Should_ReportReturnAndRepair_When_QueueOfSentMessageIsMissing() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var logs = new LogRecorder(); + await using var bus = await CreatePaymentsBusAsync(vhost, logs); + await SendAsync(bus, "warmup"); + await vhost.Faults.DeleteQueueAsync("payments"); + logs.Clear(); + + // act + await SendAsync(bus, "ORD-1"); + var entries = logs.GetTransportEntries(); + + // assert + Assert.Equal(["Warning MessageReturned", "Information QueueRedeclared"], Summarize(entries, "dispatcher")); + Assert.Equal( + [ + "The broker returned message {MessageId} because queue {Queue} does not exist on " + + "{ConnectionName}. Declaring the queue and its bindings again", + "Declared queue {Queue} and its bindings again on {ConnectionName}. Publishing message " + + "{MessageId} again" + ], + entries.Select(entry => entry.Properties[Template])); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("payments"))?.Messages); + } + + [Fact] + public async Task Log_Should_ReportStalledPublishesInOneLine_When_BrokerBlocksThePublisher() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var logs = new LogRecorder(); + await using var bus = await CreatePaymentsBusAsync(vhost, logs); + await SendAsync(bus, "warmup"); + bus.Transport.Dispatcher.StalledPublishInterval = TimeSpan.FromSeconds(1); + logs.Clear(); + Task[] sends; + LogEntry stalled; + + // act + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + sends = [SendAsync(bus, "ORD-1"), SendAsync(bus, "ORD-2"), SendAsync(bus, "ORD-3")]; + stalled = (await Wait.UntilAsync( + () => Task.FromResult( + logs.GetEntries("PublishesStalled").FirstOrDefault(entry => Equals(entry.Properties["Count"], 3L))), + entry => entry is not null, + "one report about three stalled publishes"))!; + } + + await Task.WhenAll(sends).WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + Assert.Equal(LogLevel.Warning, stalled.Level); + Assert.Equal("blocked by the broker", stalled.Properties["ConnectionStatus"]); + Assert.Equal( + "{Count} publishes on RabbitMQ connection {ConnectionName} have waited for the broker for at least " + + "{Interval}. The connection is {ConnectionStatus}", + stalled.Properties[Template]); + } + + [Fact] + public async Task Log_Should_ExplainPossibleDuplicate_When_PublishIsAttemptedAgain() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var logs = new LogRecorder(); + await using var bus = await CreatePaymentsBusAsync(vhost, logs); + await SendAsync(bus, "warmup"); + logs.Clear(); + Task send; + LogEntry retry; + + // act + // the broker keeps the publish unconfirmed, then its connection is closed under it + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + send = SendAsync(bus, "ORD-1"); + await vhost.Broker.WaitForBlockedPublisherAsync(); + await vhost.Faults.CloseConnectionsAsync(); + retry = await logs.WaitForAsync("PublishRetryScheduled"); + } + + await send.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + Assert.Equal(LogLevel.Information, retry.Level); + Assert.Equal(1, retry.Properties["Attempt"]); + Assert.Equal( + "Publishing message {MessageId} to endpoint {Endpoint} on {ConnectionName} failed (attempt {Attempt}) " + + "and is attempted again in {Delay}. The broker delivers the message twice if it already has it. " + + "Reason: {Reason}", + retry.Properties[Template]); + } + + private static string[] Summarize(IEnumerable entries, string connectionRole) + => + [ + .. entries + .Where(entry => Equals(entry.Properties.GetValueOrDefault("ConnectionRole"), connectionRole)) + .Select(entry => $"{entry.Level} {entry.EventName}") + .Distinct() + ]; + + private static bool IsAbout(LogEntry entry, string queue) + => Equals(entry.Properties.GetValueOrDefault("Queue"), queue); + + private static Task WaitForFailedAttemptOfBothConnectionsAsync(LogRecorder logs) + => Wait.UntilAsync( + () => logs.GetEntries("ConnectionAttemptFailed") + .Select(entry => entry.Properties.GetValueOrDefault("ConnectionRole")) + .Distinct() + .Count() == 2); + + private static IServiceCollection CreateServices(VhostContext vhost, LogRecorder logs) + => new ServiceCollection() + .AddLogging(builder => builder.SetMinimumLevel(LogLevel.Debug).AddProvider(logs)) + .AddSingleton(vhost.ConnectionFactory); + + private static Task CreateOrdersBusAsync(VhostContext vhost, LogRecorder logs, MessageRecorder orders) + => CreateServices(vhost, logs) + .AddSingleton(orders) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + + private static Task CreateStuckOrdersBusAsync(VhostContext vhost, LogRecorder logs, HandlerGate gate) + => CreateServices(vhost, logs) + .AddSingleton(gate) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler().MaxConcurrency(1)) + .BuildTestBusAsync(); + + private static Task CreatePaymentsBusAsync(VhostContext vhost, LogRecorder logs) + => CreateServices(vhost, logs) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue("payments"); + t.DispatchEndpoint("payment-send").ToQueue("payments").Send(); + }) + .BuildTestBusAsync(); + + private static async Task PublishAsync(TestBus bus, string orderId) + { + using var scope = bus.Provider.CreateScope(); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TestTimeouts.Signal); + await scope.ServiceProvider.GetRequiredService() + .PublishAsync(new OrderCreated { OrderId = orderId }, deadline.Token); + } + + private static async Task SendAsync(TestBus bus, string orderId) + { + using var scope = bus.Provider.CreateScope(); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TestTimeouts.Signal); + await scope.ServiceProvider.GetRequiredService() + .SendAsync(new ProcessPayment { OrderId = orderId, Amount = 1 }, deadline.Token); + } + + public sealed class HandlerGate + { + private int _startedHandlers; + + public int StartedHandlers => Volatile.Read(ref _startedHandlers); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void OnStarted() => Interlocked.Increment(ref _startedHandlers); + } + + /// + /// Runs until it is released or cancelled. + /// + public sealed class CooperativeOrderHandler(HandlerGate gate) : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + gate.OnStarted(); + await gate.Release.Task.WaitAsync(cancellationToken); + } + } + + /// + /// Runs until it is released, and ignores its cancellation. + /// + public sealed class StuckOrderHandler(HandlerGate gate) : IEventHandler + { + public async ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + gate.OnStarted(); + await gate.Release.Task; + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ReplyQueueCleanupTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ReplyQueueCleanupTests.cs new file mode 100644 index 00000000000..8b8bfca51f3 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ReplyQueueCleanupTests.cs @@ -0,0 +1,157 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.Topology)] +public sealed class ReplyQueueCleanupTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task StopAsync_Should_DeleteReplyQueue_When_QueueIsGenerated() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ() + .BuildTestBusAsync(); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + var transport = runtime.Transports.OfType().Single(); + var queue = transport.ReceiveEndpoints.OfType() + .Single(endpoint => endpoint.Kind == ReceiveEndpointKind.Reply).Queue; + var beforeStop = (await vhost.Broker.GetQueueAsync(queue.Name))?.Consumers; + + // act + await transport.StopAsync(runtime, TestContext.Current.CancellationToken); + var afterStop = (await vhost.Broker.GetQueueAsync(queue.Name))?.Consumers; + + // assert + Assert.Equal(1, beforeStop); + Assert.Null(afterStop); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StopAsync_Should_PreserveReplyQueue_When_QueueIsPersistent(bool autoProvision) + { + // arrange + const string queueName = "persistent-replies"; + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync( + TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + queueName, + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue(queueName).AutoProvision(autoProvision); + t.Endpoint(queueName).Kind(ReceiveEndpointKind.Reply); + }) + .BuildTestBusAsync(); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + var transport = runtime.Transports.OfType().Single(); + var beforeStop = (await vhost.Broker.GetQueueAsync(queueName))?.Consumers; + + // act + await transport.StopAsync(runtime, TestContext.Current.CancellationToken); + var afterStop = (await vhost.Broker.GetQueueAsync(queueName))?.Consumers; + + // assert + Assert.Equal(1, beforeStop); + Assert.Equal(0, afterStop); + } + + [Fact] + public async Task StopAsync_Should_PreserveTemporaryReplyQueue_When_AutoProvisionIsDisabled() + { + // arrange + const string queueName = "external-replies"; + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync( + TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + queueName, + durable: true, + exclusive: false, + autoDelete: false, + arguments: new Dictionary { ["x-expires"] = 60000 }, + cancellationToken: TestContext.Current.CancellationToken); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.AutoProvision(false); + t.Endpoint(queueName).Kind(ReceiveEndpointKind.Reply).Temporary(); + }) + .BuildTestBusAsync(); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + var transport = runtime.Transports.OfType().Single(); + var beforeStop = (await vhost.Broker.GetQueueAsync(queueName))?.Consumers; + + // act + await transport.StopAsync(runtime, TestContext.Current.CancellationToken); + var afterStop = (await vhost.Broker.GetQueueAsync(queueName))?.Consumers; + + // assert + Assert.Equal(1, beforeStop); + Assert.Equal(0, afterStop); + } + + [Fact] + public async Task StopAsync_Should_PreserveDeclaredReplyQueue_When_AnotherConsumerIsAttached() + { + // arrange + const string queueName = "shared-replies"; + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync( + TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + queueName, + durable: true, + exclusive: false, + autoDelete: true, + cancellationToken: TestContext.Current.CancellationToken); + await channel.BasicConsumeAsync( + queueName, + autoAck: true, + consumer: new AsyncDefaultBasicConsumer(channel), + cancellationToken: TestContext.Current.CancellationToken); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue(queueName).AutoDelete(); + t.Endpoint(queueName).Kind(ReceiveEndpointKind.Reply).Temporary(); + }) + .BuildTestBusAsync(); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + var transport = runtime.Transports.OfType().Single(); + var beforeStop = (await vhost.Broker.GetQueueAsync(queueName))?.Consumers; + + // act + await transport.StopAsync(runtime, TestContext.Current.CancellationToken); + var afterStop = (await vhost.Broker.GetQueueAsync(queueName))?.Consumers; + + // assert + Assert.Equal(2, beforeStop); + Assert.Equal(1, afterStop); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ReplyRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ReplyRecoveryTests.cs new file mode 100644 index 00000000000..7f6b8e2cf7b --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ReplyRecoveryTests.cs @@ -0,0 +1,255 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using Mocha.Events; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class ReplyRecoveryTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task RequestAsync_Should_Cancel_When_BrokerStoppedAndCallerCancels() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var recorder = new MessageRecorder(); + await using var bus = await CreateBusAsync(vhost, recorder); + await AssertRoundTripAsync(bus, recorder, "ORD-1"); + + // act + await using (await fixture.Faults.StopBrokerAsync()) + { + for (var i = 0; i < 3; i++) + { + using var scope = bus.Provider.CreateScope(); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(2)); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await Assert.ThrowsAnyAsync( + () => messageBus.RequestAsync(new GetOrderStatus { OrderId = $"ORD-{i + 2}" }, deadline.Token) + .AsTask().WaitAsync(TimeSpan.FromSeconds(4), TestContext.Current.CancellationToken)); + } + } + + // assert + await AssertRoundTripAsync(bus, recorder, "ORD-recovered"); + } + + [Fact] + public async Task RequestAsync_Should_TimeoutWithoutPublishing_When_BrokerRemainsOffline() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var timeProvider = new FakeTimeProvider(); + var recorder = new MessageRecorder(); + await using var bus = await CreateRequestBusAsync(vhost, recorder, timeProvider); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + + // act + await using (await fixture.Faults.StopBrokerAsync()) + { + Assert.True(await Condition.WaitAsync(() => !GetTransport(bus).ConsumerManager.IsConnected, TestTimeouts.Signal)); + var request = messageBus.RequestAsync(new GetOrderStatus { OrderId = "expired" }, cancellation.Token).AsTask(); + try + { + timeProvider.Advance(TimeSpan.FromMinutes(2)); + + // assert + var exception = await Assert.ThrowsAsync( + () => request.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + Assert.Equal(TimeSpan.FromMinutes(2), exception.Timeout); + } + finally + { + await cancellation.CancelAsync(); + } + } + + using var deadline = CreateTimeout(); + var response = await messageBus.RequestAsync(new GetOrderStatus { OrderId = "recovered" }, deadline.Token); + Assert.Equal("recovered", response.OrderId); + Assert.Equal(["recovered"], recorder.Messages.Cast().Select(request => request.OrderId)); + } + + [Fact] + public async Task RequestAsync_Should_Timeout_When_BrokerBlocksPublishing() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var timeProvider = new FakeTimeProvider(); + await using var bus = await CreateRequestBusAsync(vhost, new MessageRecorder(), timeProvider); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + using (var deadline = CreateTimeout()) + { + await messageBus.RequestAsync(new GetOrderStatus { OrderId = "warmup" }, deadline.Token); + } + + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + + // act + await using (await fixture.Faults.RaiseMemoryAlarmAsync()) + { + var request = messageBus.RequestAsync(new GetOrderStatus { OrderId = "blocked" }, cancellation.Token).AsTask(); + try + { + await vhost.Broker.WaitForBlockedPublisherAsync(); + timeProvider.Advance(TimeSpan.FromMinutes(2)); + + // assert + var exception = await Assert.ThrowsAsync( + () => request.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + Assert.Equal(TimeSpan.FromMinutes(2), exception.Timeout); + } + finally + { + await cancellation.CancelAsync(); + } + } + + using var recoveryDeadline = CreateTimeout(); + var response = await messageBus.RequestAsync(new GetOrderStatus { OrderId = "recovered" }, recoveryDeadline.Token); + Assert.Equal("recovered", response.OrderId); + Assert.Equal("Shipped", response.Status); + } + + [Fact] + public async Task RequestAsync_Should_ReceiveReply_When_RequesterConsumerConnectionDropsBeforeReply() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var requesterFactory = new ConnectionFactory + { + Uri = vhost.ConnectionSettings.Uri, + VirtualHost = vhost.VhostName, + UserName = $"{vhost.VhostName}-requester", + Password = "password" + }; + await using var user = await vhost.Faults.AddUserAsync(requesterFactory.UserName, requesterFactory.Password); + var gate = new ReplyGate(); + await using var responder = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ(t => t.Endpoint("order-status").Handler()) + .BuildTestBusAsync(); + await using var requester = await new ServiceCollection() + .AddSingleton(requesterFactory) + .AddMessageBus() + .AddRabbitMQ() + .BuildTestBusAsync(); + var replyQueue = ((RabbitMQReceiveEndpoint)GetTransport(requester).ReplyReceiveEndpoint!).Queue.Name; + + // act + using var scope = requester.Provider.CreateScope(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TestTimeouts.Signal); + var request = scope.ServiceProvider.GetRequiredService().RequestAsync( + new GetOrderStatus { OrderId = "ORD-1" }, + new SendOptions { Endpoint = new Uri("rabbitmq:q/order-status") }, + cts.Token).AsTask(); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // the requester cannot attach again until its permissions are restored, so the reply is sent while its + // reply queue has no consumer + await vhost.Faults.SetPermissionsAsync(requesterFactory.UserName, configure: "^$", write: "^$", read: "^$"); + await vhost.Faults.CloseConsumerConnectionAsync(replyQueue); + await vhost.Broker.WaitForNoConsumerAsync(replyQueue); + gate.Release.TrySetResult(); + await vhost.Broker.WaitForMessagesAsync(replyQueue, 1); + await vhost.Faults.SetPermissionsAsync(requesterFactory.UserName, configure: ".*", write: ".*", read: ".*"); + + var reply = await request; + + // assert + Assert.Equal("ORD-1", reply.OrderId); + Assert.Equal("Shipped", reply.Status); + } + + private static Task CreateBusAsync(VhostContext vhost, MessageRecorder recorder) + => new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRequestHandler() + .AddRabbitMQ(t => + { + t.Endpoint("orders").Handler(); + t.Endpoint("order-status").Handler(); + }) + .BuildTestBusAsync(); + + private static RabbitMQMessagingTransport GetTransport(TestBus bus) + => bus.Provider.GetRequiredService() + .Transports.OfType().Single(); + + private static Task CreateRequestBusAsync( + VhostContext vhost, + MessageRecorder recorder, + TimeProvider timeProvider) + => new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(timeProvider) + .AddSingleton(recorder) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ(t => t.Endpoint("order-status").Handler()) + .BuildTestBusAsync(); + + public sealed class RecordingOrderStatusHandler(MessageRecorder recorder) + : IEventRequestHandler + { + public ValueTask HandleAsync(GetOrderStatus request, CancellationToken cancellationToken) + { + recorder.Record(request); + return new(new OrderStatusResponse { OrderId = request.OrderId, Status = "Shipped" }); + } + } + + private static async Task AssertRoundTripAsync(TestBus bus, MessageRecorder recorder, string orderId) + { + using var scope = bus.Provider.CreateScope(); + using var deadline = CreateTimeout(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await messageBus.PublishAsync(new OrderCreated { OrderId = orderId }, deadline.Token); + Assert.True(await Condition.WaitAsync( + () => recorder.Messages.Cast().Any(order => order.OrderId == orderId), + TestTimeouts.Signal)); + var response = await messageBus.RequestAsync(new GetOrderStatus { OrderId = orderId }, deadline.Token); + Assert.Equal(orderId, response.OrderId); + Assert.Equal("Shipped", response.Status); + } + + private static CancellationTokenSource CreateTimeout() + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TestTimeouts.Signal); + return cts; + } + + public sealed class ReplyGate + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class GatedOrderStatusHandler(ReplyGate gate) + : IEventRequestHandler + { + public async ValueTask HandleAsync( + GetOrderStatus request, + CancellationToken cancellationToken) + { + gate.Started.TrySetResult(); + await gate.Release.Task.WaitAsync(cancellationToken); + return new OrderStatusResponse { OrderId = request.OrderId, Status = "Shipped" }; + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RequestReadinessTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RequestReadinessTests.cs new file mode 100644 index 00000000000..07577681bdc --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RequestReadinessTests.cs @@ -0,0 +1,167 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConsumerRecovery)] +public sealed class RequestReadinessTests(RabbitMQFixture fixture) +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RequestAsync_Should_WaitForReplyConsumer_When_ReplyQueueRecoversOnSeparateBus(bool cancel) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var requesterFactory = new ConnectionFactory + { + Uri = vhost.ConnectionSettings.Uri, + VirtualHost = vhost.VhostName, + UserName = $"{vhost.VhostName}-requester", + Password = "password" + }; + await using var user = await vhost.Faults.AddUserAsync(requesterFactory.UserName, requesterFactory.Password); + await using var requester = await new ServiceCollection() + .AddSingleton(requesterFactory) + .AddMessageBus() + .AddRabbitMQ() + .BuildTestBusAsync(); + var replyQueue = ((RabbitMQReceiveEndpoint)requester.Transport.ReplyReceiveEndpoint!).Queue.Name; + var handled = new MessageRecorder(); + var replyConsumers = new ReplyConsumerProbe(vhost.Broker, replyQueue); + await using var responder = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(handled) + .AddSingleton(replyConsumers) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ(t => t.Endpoint("order-status").Handler()) + .BuildTestBusAsync(); + using var scope = requester.Provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + var options = new SendOptions { Endpoint = new Uri("rabbitmq:q/order-status") }; + await bus.RequestAsync(new GetOrderStatus { OrderId = "before" }, options, TestContext.Current.CancellationToken); + var replyConsumer = await vhost.Broker.GetConsumerAsync(replyQueue); + var consumerConnection = await requester.Transport.ConsumerManager.Connection.GetConnectionAsync( + TestContext.Current.CancellationToken); + var consumerConnectionLost = consumerConnection.WhenShutdownAsync(); + + // the requester cannot declare its reply queue again, so its reply consumer stays unavailable + await vhost.Faults.SetPermissionsAsync(requesterFactory.UserName, configure: "^$", write: ".*", read: ".*"); + await vhost.Faults.CloseConsumerConnectionAsync(replyQueue); + await consumerConnectionLost.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await vhost.Faults.DeleteQueueAsync(replyQueue); + + // act + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TestTimeouts.Signal); + var pending = bus.RequestAsync(new GetOrderStatus { OrderId = "after" }, options, deadline.Token).AsTask(); + try + { + var requestQueueWhileUnavailable = await vhost.Broker.GetQueueAsync("order-status"); + var handledWhileUnavailable = handled.Messages.Cast().Select(m => m.OrderId).ToArray(); + if (cancel) + { + await deadline.CancelAsync(); + await Assert.ThrowsAnyAsync(() => pending); + } + + await vhost.Faults.SetPermissionsAsync(requesterFactory.UserName, configure: ".*", write: ".*", read: ".*"); + await vhost.Broker.WaitForConsumerAsync(replyQueue, otherThan: replyConsumer); + var reply = cancel + ? null + : await pending.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + await bus.RequestAsync(new GetOrderStatus { OrderId = "probe" }, options, deadline.Token.IsCancellationRequested + ? TestContext.Current.CancellationToken + : deadline.Token); + + // assert + Assert.Equal(cancel ? null : "after", reply?.OrderId); + Assert.Equal(new QueueInfo(Consumers: 1, Ready: 0, Unacknowledged: 0), requestQueueWhileUnavailable); + Assert.Equal(["before"], handledWhileUnavailable); + + // every request was handled while its reply queue had a consumer, so "after" was held back until then + Assert.Equal( + cancel + ? [("before", 1), ("probe", 1)] + : new (string, int?)[] { ("after", 1), ("before", 1), ("probe", 1) }, + replyConsumers.AtHandleTime.OrderBy(request => request.OrderId)); + } + finally + { + await vhost.Faults.SetPermissionsAsync(requesterFactory.UserName, configure: ".*", write: ".*", read: ".*"); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SendAsync_Should_Deliver_When_LocalReplyConsumerIsUnavailable(bool customReplyAddress) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var sender = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => t.DeclareQueue("outbound")) + .BuildTestBusAsync(); + var transport = sender.Transport; + var replyQueue = ((RabbitMQReceiveEndpoint)transport.ReplyReceiveEndpoint!).Queue.Name; + using var scope = sender.Provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + var options = new SendOptions + { + Endpoint = new Uri("rabbitmq:q/outbound"), + ReplyEndpoint = customReplyAddress ? new Uri("rabbitmq://remote/q/replies") : null + }; + await bus.SendAsync(new OrderCreated { OrderId = "before" }, options, TestContext.Current.CancellationToken); + await using var declaresDenied = await vhost.Faults.DenyDeclaresAsync(); + await vhost.Faults.CloseConsumerConnectionAsync(replyQueue); + await vhost.Faults.DeleteQueueAsync(replyQueue); + + // act + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TestTimeouts.Signal); + await bus.SendAsync(new OrderCreated { OrderId = "after" }, options, deadline.Token); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(deadline.Token); + await using var channel = await connection.CreateChannelAsync(cancellationToken: deadline.Token); + var first = await channel.BasicGetAsync("outbound", true, deadline.Token); + var second = await channel.BasicGetAsync("outbound", true, deadline.Token); + var remaining = await channel.MessageCountAsync("outbound", deadline.Token); + + // assert + Assert.False(transport.ConsumerManager.IsHealthy); + Assert.Equal(options.ReplyEndpoint?.ToString(), first!.BasicProperties.ReplyTo); + Assert.Equal(options.ReplyEndpoint?.ToString(), second!.BasicProperties.ReplyTo); + Assert.Equal(0u, remaining); + } + + /// + /// Records how many consumers the reply queue of the requester has at the moment a request is handled. + /// + public sealed class ReplyConsumerProbe(BrokerView broker, string replyQueue) + { + private readonly ConcurrentQueue<(string OrderId, int? ReplyConsumers)> _atHandleTime = new(); + + public IEnumerable<(string OrderId, int? ReplyConsumers)> AtHandleTime => _atHandleTime; + + public async Task ObserveAsync(string orderId) + => _atHandleTime.Enqueue((orderId, (await broker.GetQueueAsync(replyQueue))?.Consumers)); + } + + public sealed class StatusHandler(MessageRecorder recorder, ReplyConsumerProbe replyConsumers) + : IEventRequestHandler + { + public async ValueTask HandleAsync( + GetOrderStatus request, + CancellationToken cancellationToken) + { + recorder.Record(request); + await replyConsumers.ObserveAsync(request.OrderId); + return new OrderStatusResponse { OrderId = request.OrderId, Status = "Shipped" }; + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RequestReplyTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RequestReplyTests.cs index 99b8bcb54c3..ce0638aedb7 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RequestReplyTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RequestReplyTests.cs @@ -1,13 +1,13 @@ using System.Text.Json; using Microsoft.Extensions.DependencyInjection; using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Messaging)] public class RequestReplyTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public RequestReplyTests(RabbitMQFixture fixture) @@ -95,7 +95,7 @@ await messageBus.RequestAsync( CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the request within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the request within timeout"); } [Fact] @@ -131,7 +131,7 @@ public async Task RequestAsync_Should_ReturnCorrectResponse_When_MultipleRequest } [Fact] - public async Task Transport_Should_DeclareDurableAutoDeleteReplyQueue_When_Started() + public async Task Transport_Should_DeclareDurableExpiringReplyQueue_When_Started() { // arrange // The broker denies non-durable, non-exclusive queues by default since RabbitMQ 4.3, so a @@ -173,8 +173,90 @@ public async Task Transport_Should_DeclareDurableAutoDeleteReplyQueue_When_Start // assert Assert.True(replyQueue.GetProperty("durable").GetBoolean()); Assert.False(replyQueue.GetProperty("exclusive").GetBoolean()); - Assert.True(replyQueue.GetProperty("auto_delete").GetBoolean()); - Assert.Equal(1800000, expires); + Assert.False(replyQueue.GetProperty("auto_delete").GetBoolean()); + Assert.Equal(60000, expires); + } + + [Fact] + public async Task RequestAsync_Should_ReturnResponse_When_ConsumerConnectionIsLost() + { + // arrange + const string queueName = "order-status"; + await using var vhost = await _fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ(t => t.Endpoint(queueName).Handler()) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TestTimeouts.Signal); + var before = await messageBus.RequestAsync(new GetOrderStatus { OrderId = "ORD-before" }, cts.Token); + var replyQueue = await vhost.Broker.GetReplyQueueNameAsync(); + + // act + // the reply queue expires while its consumer is disconnected + await using (await vhost.Faults.RevokeAccessAsync()) + { + await vhost.Faults.CloseConsumerConnectionAsync(queueName); + await vhost.Faults.DeleteQueueAsync(replyQueue); + } + + var after = await messageBus.RequestAsync(new GetOrderStatus { OrderId = "ORD-after" }, cts.Token); + + // assert + Assert.Equal(("ORD-before", "Shipped"), (before.OrderId, before.Status)); + Assert.Equal(("ORD-after", "Shipped"), (after.OrderId, after.Status)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task RequestAsync_Should_ReturnAllResponses_When_RequestsWaitForReplyRecovery( + bool automaticRecovery) + { + // arrange + const string queueName = "order-status"; + var timeout = TimeSpan.FromSeconds(60); + await using var vhost = await _fixture.CreateVhostAsync(); + var connectionFactory = (ConnectionFactory)vhost.ConnectionFactory; + connectionFactory.AutomaticRecoveryEnabled = automaticRecovery; + connectionFactory.NetworkRecoveryInterval = TimeSpan.FromSeconds(2); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ(t => t.Endpoint(queueName).Handler()) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(timeout); + await messageBus.RequestAsync(new GetOrderStatus { OrderId = "ORD-before" }, cts.Token); + var replyQueue = await vhost.Broker.GetReplyQueueNameAsync(); + Task[] pendingRequests; + + // act + await using (await vhost.Faults.DenyDeclaresAsync()) + { + // the reply queue expires while its consumer is disconnected + await vhost.Faults.CloseConsumerConnectionAsync(queueName); + await vhost.Faults.DeleteQueueAsync(replyQueue); + pendingRequests = Enumerable.Range(1, 3) + .Select(i => messageBus.RequestAsync(new GetOrderStatus { OrderId = $"ORD-{i}" }, cts.Token).AsTask()) + .ToArray(); + Assert.Equal(0, (await vhost.Broker.GetQueueAsync(queueName))?.Messages); + Assert.All(pendingRequests, request => Assert.False(request.IsCompleted)); + } + + var responses = await Task.WhenAll(pendingRequests).WaitAsync(cts.Token); + + // assert + Assert.Equal( + [("ORD-1", "Shipped"), ("ORD-2", "Shipped"), ("ORD-3", "Shipped")], + responses.Select(r => (r.OrderId, r.Status))); } public sealed class ProcessPaymentHandler(MessageRecorder recorder) : IEventRequestHandler diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RoutingKeyTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RoutingKeyTests.cs index 34471755ac9..cd48f7c820c 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RoutingKeyTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/RoutingKeyTests.cs @@ -4,10 +4,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class RoutingKeyTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private static readonly TimeSpan s_negativeTimeout = TimeSpan.FromSeconds(2); private readonly RabbitMQFixture _fixture; @@ -56,7 +55,7 @@ await messageBus.PublishAsync( CancellationToken.None); // assert - Assert.True(await usRecorder.WaitAsync(s_timeout), "US handler did not receive the event"); + Assert.True(await usRecorder.WaitAsync(TestTimeouts.Signal), "US handler did not receive the event"); var usMessage = Assert.Single(usRecorder.Messages); var usEvent = Assert.IsType(usMessage); Assert.Equal("us.east", usEvent.Region); @@ -109,7 +108,7 @@ await messageBus.PublishAsync( // assert Assert.True( - await recorder.WaitAsync(s_timeout, expectedCount: 3), + await recorder.WaitAsync(TestTimeouts.Signal, expectedCount: 3), "Catch-all handler did not receive all 3 events within timeout"); Assert.Equal(3, recorder.Messages.Count); @@ -215,7 +214,7 @@ await messageBus.PublishAsync( CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Consumer did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message within timeout"); var capturedKey = Assert.Single(tracker.CapturedKeys); Assert.Equal("ap.southeast", capturedKey); @@ -278,10 +277,10 @@ await messageBus.PublishAsync( // assert Assert.True( - await usRecorder.WaitAsync(s_timeout, expectedCount: 2), + await usRecorder.WaitAsync(TestTimeouts.Signal, expectedCount: 2), "Both bindings to the us queue should deliver their routing keys"); Assert.True( - await euRecorder.WaitAsync(s_timeout, expectedCount: 2), + await euRecorder.WaitAsync(TestTimeouts.Signal, expectedCount: 2), "Both bindings to the eu queue should deliver their routing keys"); var usPayloads = usRecorder diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SchedulingTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SchedulingTests.cs index d0c226daefc..048ff7f7edb 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SchedulingTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SchedulingTests.cs @@ -9,7 +9,7 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; /// instead of being silently delivered immediately. To enable scheduling for RabbitMQ /// register a fallback scheduling store (e.g. UsePostgresScheduling()). /// -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Messaging)] public class SchedulingTests { private readonly RabbitMQFixture _fixture; diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SendTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SendTests.cs index cf67487381a..d4af52fb878 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SendTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SendTests.cs @@ -3,10 +3,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Messaging)] public class SendTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public SendTests(RabbitMQFixture fixture) @@ -35,7 +34,7 @@ public async Task SendAsync_Should_DeliverToHandler_When_RequestHandlerRegistere await messageBus.SendAsync(new ProcessPayment { OrderId = "ORD-1", Amount = 99.99m }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the request"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the request"); var message = Assert.Single(recorder.Messages); var payment = Assert.IsType(message); @@ -67,7 +66,7 @@ public async Task SendAsync_Should_DeliverToCorrectHandler_When_MultipleQueuesEx await messageBus.SendAsync(new ProcessPayment { OrderId = "ORD-1", Amount = 50.00m }, CancellationToken.None); // assert - Assert.True(await paymentRecorder.WaitAsync(s_timeout), "Payment handler did not receive the send message"); + Assert.True(await paymentRecorder.WaitAsync(TestTimeouts.Signal), "Payment handler did not receive the send message"); var msg = Assert.Single(paymentRecorder.Messages); Assert.IsType(msg); @@ -103,8 +102,8 @@ public async Task SendAsync_Should_DeliverToEachHandler_When_SendingToDifferentQ await messageBus.SendAsync(new ProcessRefund { OrderId = "ORD-2", Amount = 25.00m }, CancellationToken.None); // assert - Assert.True(await paymentRecorder.WaitAsync(s_timeout), "Payment handler did not receive the message"); - Assert.True(await refundRecorder.WaitAsync(s_timeout), "Refund handler did not receive the message"); + Assert.True(await paymentRecorder.WaitAsync(TestTimeouts.Signal), "Payment handler did not receive the message"); + Assert.True(await refundRecorder.WaitAsync(TestTimeouts.Signal), "Refund handler did not receive the message"); var payment = Assert.IsType(Assert.Single(paymentRecorder.Messages)); Assert.Equal("ORD-1", payment.OrderId); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ShutdownTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ShutdownTests.cs index 7117164d9a1..ed22f19b361 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ShutdownTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/ShutdownTests.cs @@ -1,51 +1,331 @@ +using System.Collections.Concurrent; using Microsoft.Extensions.DependencyInjection; using Mocha.Transport.RabbitMQ.Tests.Helpers; namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.ConsumerRecovery)] public sealed class ShutdownTests(RabbitMQFixture fixture) { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisposeAsync_Should_CancelHandlerAndRequeueMessage_When_StoppedWithoutGracePeriod( + bool disposeProvider) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var gate = new HandlerGate(); + await using var bus = await CreateBusAsync(vhost, gate); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + await PublishAsync(bus); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + var dispose = disposeProvider ? bus.Provider.DisposeAsync().AsTask() : runtime.DisposeAsync().AsTask(); + try + { + await dispose.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await gate.CancellationObserved.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + Assert.True(gate.Cancelled); + Assert.False(gate.Completed); + Assert.False(runtime.IsStarted); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("shutdown"))?.Messages); + } + finally + { + gate.Release.TrySetResult(); + await dispose.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + } + [Fact] - public async Task StopAsync_Should_CancelHandlerAndRequeueMessage_When_EndpointStops() + public async Task StopAsync_Should_LetHandlerFinishAndAck_When_StopTokenNotCancelled() { // arrange await using var vhost = await fixture.CreateVhostAsync(); var gate = new HandlerGate(); - await using var bus = await new ServiceCollection() - .AddSingleton(vhost.ConnectionFactory) - .AddSingleton(gate) - .AddMessageBus() - .AddEventHandler() - .AddRabbitMQ(t => t.Endpoint("shutdown").Handler()) - .BuildTestBusAsync(); + await using var bus = await CreateBusAsync(vhost, gate); var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); - using var scope = bus.Provider.CreateScope(); - await scope.ServiceProvider.GetRequiredService() - .PublishAsync(new TestEvent(), TestContext.Current.CancellationToken); - await gate.Started.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await PublishAsync(bus); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + var stop = runtime.StopAsync(CancellationToken.None).AsTask(); + await vhost.Broker.WaitForNoConsumerAsync("shutdown"); + var stoppedBeforeHandlerFinished = stop.IsCompleted; + gate.Release.TrySetResult(); + await stop.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var messages = (await vhost.Broker.GetQueueAsync("shutdown"))?.Messages; + + // assert + Assert.False(stoppedBeforeHandlerFinished); + Assert.True(gate.Completed); + Assert.False(gate.Cancelled); + Assert.Equal(0, messages); + Assert.False(runtime.IsStarted); + } + + [Fact] + public async Task StopAsync_Should_CancelHandlerAndRequeueMessage_When_StopTokenCancelled() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var gate = new HandlerGate(); + await using var bus = await CreateBusAsync(vhost, gate); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + await PublishAsync(bus); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); // act - await runtime.StopAsync(CancellationToken.None).AsTask() - .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + using var stopTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + await runtime.StopAsync(stopTimeout.Token).AsTask() + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); // assert await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); var message = await channel.BasicGetAsync("shutdown", autoAck: true, TestContext.Current.CancellationToken); + Assert.False(gate.Completed); Assert.True(gate.Cancelled); Assert.True(message?.Redelivered); + // a handler cancelled by the shutdown did not fail, so its message is not dead-lettered + Assert.Equal(0, (await vhost.Broker.GetQueueAsync("shutdown_error"))?.Messages); + Assert.False(runtime.IsStarted); + } + + [Fact] + public async Task StopAsync_Should_SendReply_When_HandlerFinishesDuringStop() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var gate = new HandlerGate(); + await using var responder = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ(t => t.Endpoint("order-status").Handler()) + .BuildTestBusAsync(); + await using var requester = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(_ => { }) + .BuildTestBusAsync(); + var runtime = (MessagingRuntime)responder.Provider.GetRequiredService(); + using var scope = requester.Provider.CreateScope(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TestTimeouts.Signal); + var request = scope.ServiceProvider.GetRequiredService().RequestAsync( + new GetOrderStatus { OrderId = "ORD-1" }, + new SendOptions { Endpoint = new Uri("rabbitmq:q/order-status") }, + cts.Token).AsTask(); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + using var stopTimeout = new CancellationTokenSource(TestTimeouts.Signal); + var stop = runtime.StopAsync(stopTimeout.Token).AsTask(); + await vhost.Broker.WaitForNoConsumerAsync("order-status"); + var stoppedBeforeHandlerFinished = stop.IsCompleted; + gate.Release.TrySetResult(); + var reply = await request; + await stop.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + Assert.False(stoppedBeforeHandlerFinished); + Assert.Equal("Shipped", reply.Status); + Assert.False(gate.Cancelled); + Assert.False(runtime.IsStarted); + } + + [Fact] + public async Task StopAsync_Should_KeepReceivingReplies_When_ApplicationHandlersAreDraining() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var responder = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRequestHandler() + .AddRabbitMQ(t => t.Endpoint("nested-responder").Handler()) + .BuildTestBusAsync(); + var gate = new NestedRequestGate(); + await using var requester = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => + { + t.Endpoint("nested-first").Handler(); + t.Endpoint("nested-second").Handler(); + }) + .BuildTestBusAsync(); + var runtime = (MessagingRuntime)requester.Provider.GetRequiredService(); + var transport = runtime.Transports.OfType().Single(); + using var scope = requester.Provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + foreach (var queue in new[] { "nested-first", "nested-second" }) + { + await bus.SendAsync( + new NestedRequest(queue), + new SendOptions { Endpoint = new Uri($"rabbitmq:q/{queue}") }, + TestContext.Current.CancellationToken); + } + + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + using var stopTimeout = new CancellationTokenSource(TestTimeouts.Signal); + var stop = runtime.StopAsync(stopTimeout.Token).AsTask(); + bool replyWasHealthy; + try + { + await vhost.Broker.WaitForNoConsumerAsync("nested-first"); + await vhost.Broker.WaitForNoConsumerAsync("nested-second"); + replyWasHealthy = ((RabbitMQReceiveEndpoint)transport.ReplyReceiveEndpoint!).IsHealthy; + } + finally + { + gate.Release.TrySetResult(); + } + + await stop.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var firstMessages = (await vhost.Broker.GetQueueAsync("nested-first"))?.Messages; + var secondMessages = (await vhost.Broker.GetQueueAsync("nested-second"))?.Messages; + + // assert + Assert.True(replyWasHealthy); + Assert.Equal(["nested-first", "nested-second"], gate.Responses.Order()); + Assert.Equal(0, firstMessages); + Assert.Equal(0, secondMessages); Assert.False(runtime.IsStarted); } + [Fact] + public async Task StopAsync_Should_Complete_When_ConnectionClosedDuringStop() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + var gate = new HandlerGate(); + var bus = await CreateBusAsync(vhost, gate); + var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); + await PublishAsync(bus); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + using var stopTimeout = new CancellationTokenSource(TestTimeouts.Signal); + var stop = runtime.StopAsync(stopTimeout.Token).AsTask(); + await vhost.Faults.CloseConnectionsAsync(); + await gate.CancellationObserved.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + gate.Release.TrySetResult(); + await stop.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await bus.DisposeAsync(); + var closed = await Condition.WaitAsync( + async () => await vhost.Broker.GetConnectionCountAsync() == 0, + TestTimeouts.Signal); + + // the handler completed, but its delivery could not be acked on the closed channel + var requeued = (await vhost.Broker.GetQueueAsync("shutdown"))?.Messages; + + // assert + Assert.False(gate.Completed); + Assert.True(gate.Cancelled); + Assert.True(closed); + Assert.Equal(1, requeued); + } + + private static async Task CreateBusAsync(VhostContext vhost, HandlerGate gate) + => await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddSingleton(gate) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("shutdown").Handler()) + .BuildTestBusAsync(); + + private static async Task PublishAsync(TestBus bus) + { + using var scope = bus.Provider.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .PublishAsync(new TestEvent(), TestContext.Current.CancellationToken); + } + public sealed class TestEvent; + public sealed record NestedRequest(string OrderId); + + public sealed class NestedRequestGate + { + private int _started; + + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ConcurrentBag Responses { get; } = []; + + public void HandlerStarted() + { + if (Interlocked.Increment(ref _started) == 2) + { + Started.TrySetResult(); + } + } + } + + public sealed class NestedRequestHandler(IMessageBus bus, NestedRequestGate gate) : IEventHandler + { + public async ValueTask HandleAsync(NestedRequest message, CancellationToken cancellationToken) + { + gate.HandlerStarted(); + await gate.Release.Task.WaitAsync(cancellationToken); + var response = await bus.RequestAsync( + new GetOrderStatus { OrderId = message.OrderId }, + new SendOptions { Endpoint = new Uri("rabbitmq:q/nested-responder") }, + cancellationToken); + gate.Responses.Add(response.OrderId); + } + } + public sealed class HandlerGate { public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource CancellationObserved { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool Completed { get; set; } + public bool Cancelled { get; set; } } + public sealed class GatedOrderStatusHandler(HandlerGate gate) + : IEventRequestHandler + { + public async ValueTask HandleAsync( + GetOrderStatus request, + CancellationToken cancellationToken) + { + gate.Started.TrySetResult(); + try + { + await gate.Release.Task.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) + { + gate.Cancelled = true; + gate.CancellationObserved.TrySetResult(); + throw; + } + + return new OrderStatusResponse { OrderId = request.OrderId, Status = "Shipped" }; + } + } + public sealed class BlockingHandler(HandlerGate gate) : IEventHandler { public async ValueTask HandleAsync(TestEvent message, CancellationToken cancellationToken) @@ -53,11 +333,13 @@ public async ValueTask HandleAsync(TestEvent message, CancellationToken cancella gate.Started.TrySetResult(); try { - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + await gate.Release.Task.WaitAsync(cancellationToken); + gate.Completed = true; } catch (OperationCanceledException) { gate.Cancelled = true; + gate.CancellationObserved.TrySetResult(); throw; } } diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SilentEndpointRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SilentEndpointRecoveryTests.cs new file mode 100644 index 00000000000..7d12bfb744b --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/SilentEndpointRecoveryTests.cs @@ -0,0 +1,62 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class SilentEndpointRecoveryTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task Bus_Should_Recover_When_EndpointAcceptsConnectionsWithoutAnswering() + { + // arrange + // the bus reaches the broker through a proxy, as it would through a load balancer or a sidecar + await using var vhost = await fixture.CreateVhostAsync(); + await using var proxy = new TcpProxy(vhost.ConnectionSettings.HostName, vhost.ConnectionSettings.Port); + var throughProxy = new ConnectionFactory + { + Uri = vhost.ConnectionSettings.Uri, + VirtualHost = vhost.VhostName, + HostName = IPAddress.Loopback.ToString(), + Port = proxy.Port + }; + var recorder = new MessageRecorder(); + await using var bus = await new ServiceCollection() + .AddSingleton(throughProxy) + .AddSingleton(recorder) + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(t => t.Endpoint("orders").Handler()) + .BuildTestBusAsync(); + bus.Transport.Dispatcher.Connection.AttemptTimeout = TimeSpan.FromSeconds(2); + bus.Transport.ConsumerManager.Connection.AttemptTimeout = TimeSpan.FromSeconds(2); + await PublishAsync(bus, "ORD-1"); + await Wait.UntilAsync(() => recorder.Messages.Count == 1); + + // act + // both connections are lost, and the first attempt of each to reconnect gets no answer + proxy.IsSilent = true; + proxy.DropConnections(); + await Wait.UntilAsync(() => proxy.UnansweredConnections >= 2); + proxy.IsSilent = false; + await PublishAsync(bus, "ORD-2"); + await Wait.UntilAsync(() => recorder.Messages.Count == 2); + await Wait.UntilAsync(() => bus.Transport.HealthFailureReason is null); + + // assert + Assert.Equal( + ["ORD-1", "ORD-2"], + recorder.Messages.Cast().Select(order => order.OrderId).Order()); + } + + private static async Task PublishAsync(TestBus bus, string orderId) + { + using var scope = bus.Provider.CreateScope(); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TestTimeouts.Signal); + await scope.ServiceProvider.GetRequiredService() + .PublishAsync(new OrderCreated { OrderId = orderId }, deadline.Token); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TemporaryEndpointCleanupTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TemporaryEndpointCleanupTests.cs index f1ed8a632f5..1d2b2887145 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TemporaryEndpointCleanupTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TemporaryEndpointCleanupTests.cs @@ -4,7 +4,7 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public sealed class TemporaryEndpointCleanupTests(RabbitMQFixture fixture) { [Fact] @@ -24,7 +24,7 @@ public async Task StopAsync_Should_RemoveTemporaryEndpointResources_When_Stopped var runtime = (MessagingRuntime)bus.Provider.GetRequiredService(); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; var binding = topology.Bindings .OfType() .Single(b => b.Destination.Name == queueName); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TopologyRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TopologyRecoveryTests.cs new file mode 100644 index 00000000000..bbc78bddba4 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TopologyRecoveryTests.cs @@ -0,0 +1,100 @@ +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; + +[Collection(RabbitMQCollections.Topology)] +public sealed class TopologyRecoveryTests(RabbitMQFixture fixture) +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SendAsync_Should_RestoreQueueAndUpstreamBindings_When_QueueIsDeleted(bool sendBeforeDeletion) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareExchange("orders"); + t.DeclareExchange("routed-orders").AutoDelete(); + t.DeclareQueue("archive"); + t.DeclareBinding("orders", "routed-orders").ToExchange("routed-orders"); + t.DeclareBinding("routed-orders", "archive"); + t.DispatchEndpoint("orders-publish").ToExchange("orders").Publish(); + t.DispatchEndpoint("orders-send").ToQueue("archive").Send(); + }) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + var dispatcher = bus.Provider.GetRequiredService() + .Transports.OfType().Single().Dispatcher; + var publisherConnection = await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + await messageBus.PublishAsync(new OrderCreated { OrderId = "warmup" }, TestContext.Current.CancellationToken); + if (sendBeforeDeletion) + { + await messageBus.SendAsync(new OrderCreated { OrderId = "warmup-send" }, TestContext.Current.CancellationToken); + } + + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + + // act + // No application consumer recreates the deleted queue, exchange, or bindings. + await channel.QueueDeleteAsync("archive", cancellationToken: TestContext.Current.CancellationToken); + await messageBus.SendAsync(new OrderCreated { OrderId = "direct" }, TestContext.Current.CancellationToken); + await messageBus.PublishAsync(new OrderCreated { OrderId = "published" }, TestContext.Current.CancellationToken); + var orders = new List<(string? OrderId, string Exchange)>(); + var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; + while (await channel.BasicGetAsync("archive", true, TestContext.Current.CancellationToken) is { } delivery) + { + var order = JsonSerializer.Deserialize(delivery.Body.Span, options); + orders.Add((order?.OrderId, delivery.Exchange)); + } + + // assert + Assert.Equal([("direct", ""), ("published", "orders")], orders); + Assert.Same(publisherConnection, await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task PublishAsync_Should_RestoreQueueAndBinding_When_PublisherReconnects() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareExchange("orders"); + t.DeclareQueue("archive"); + t.DeclareBinding("orders", "archive"); + t.DispatchEndpoint("orders").ToExchange("orders").Publish(); + }) + .BuildTestBusAsync(); + using var scope = bus.Provider.CreateScope(); + var messageBus = scope.ServiceProvider.GetRequiredService(); + await vhost.Faults.DeleteQueueAsync("archive"); + + // act + // No application consumer recreates this queue or its binding. + await vhost.Faults.CloseConnectionsAsync(); + await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-1" }, TestContext.Current.CancellationToken); + + // assert + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + var delivery = await channel.BasicGetAsync("archive", true, TestContext.Current.CancellationToken); + Assert.NotNull(delivery); + var order = JsonSerializer.Deserialize( + delivery.Body.Span, + new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + Assert.Equal("ORD-1", order?.OrderId); + Assert.Equal("orders", delivery.Exchange); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TransportMiddlewareTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TransportMiddlewareTests.cs index 4522cfd3dff..914985cd85a 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TransportMiddlewareTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/TransportMiddlewareTests.cs @@ -4,10 +4,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class TransportMiddlewareTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public TransportMiddlewareTests(RabbitMQFixture fixture) @@ -55,7 +54,7 @@ public async Task UseReceive_Should_InvokeOnAllEndpoints_When_ConfiguredAtTransp await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-T1" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Consumer did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message within timeout"); Assert.Contains("transport-receive-mw", tracker.Invocations); } @@ -100,7 +99,7 @@ public async Task UseReceive_After_Should_InvokeOnAllEndpoints_When_ConfiguredAt await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-T2" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Consumer did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message within timeout"); Assert.Contains("transport-append-receive", tracker.Invocations); } @@ -145,7 +144,7 @@ public async Task UseReceive_Before_Should_InvokeOnAllEndpoints_When_ConfiguredA await messageBus.PublishAsync(new OrderCreated { OrderId = "ORD-T3" }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Consumer did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Consumer did not receive the message within timeout"); Assert.Contains("transport-prepend-receive", tracker.Invocations); } @@ -183,7 +182,7 @@ public async Task UseDispatch_Should_InvokeOnAllEndpoints_When_ConfiguredAtTrans await messageBus.SendAsync(new ProcessPayment { OrderId = "ORD-TD1", Amount = 50.00m }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the message within timeout"); Assert.Contains("transport-dispatch-mw", tracker.Invocations); } @@ -222,7 +221,7 @@ public async Task UseDispatch_After_Should_InvokeOnAllEndpoints_When_ConfiguredA await messageBus.SendAsync(new ProcessPayment { OrderId = "ORD-TD2", Amount = 25.00m }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the message within timeout"); Assert.Contains("transport-append-dispatch", tracker.Invocations); } @@ -261,7 +260,7 @@ public async Task UseDispatch_Before_Should_InvokeOnAllEndpoints_When_Configured await messageBus.SendAsync(new ProcessPayment { OrderId = "ORD-TD3", Amount = 10.00m }, CancellationToken.None); // assert - Assert.True(await recorder.WaitAsync(s_timeout), "Handler did not receive the message within timeout"); + Assert.True(await recorder.WaitAsync(TestTimeouts.Signal), "Handler did not receive the message within timeout"); Assert.Contains("transport-prepend-dispatch", tracker.Invocations); } diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/UnifiedQueueBehaviorTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/UnifiedQueueBehaviorTests.cs index c6e072d9d69..18ee9c5f578 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/UnifiedQueueBehaviorTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Behaviors/UnifiedQueueBehaviorTests.cs @@ -8,10 +8,9 @@ namespace Mocha.Transport.RabbitMQ.Tests.Behaviors; /// End-to-end Docker-gated tests for the unified t.Queue(name) API. /// Verifies that consumer placement via the Queue() API routes messages to the consumer. /// -[Collection("RabbitMQ")] +[Collection(RabbitMQCollections.Topology)] public class UnifiedQueueBehaviorTests { - private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(30); private readonly RabbitMQFixture _fixture; public UnifiedQueueBehaviorTests(RabbitMQFixture fixture) @@ -49,7 +48,7 @@ public async Task PublishAsync_Should_RouteToConsumer_When_PlacedViaUnifiedQueue // assert Assert.True( - await capture.WaitAsync(s_timeout), + await capture.WaitAsync(TestTimeouts.Signal), "Consumer placed via the unified Queue() API did not receive the published message"); var message = Assert.Single(capture.Messages); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConnectionManagerBaseTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConnectionManagerBaseTests.cs deleted file mode 100644 index 7c5f143d541..00000000000 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConnectionManagerBaseTests.cs +++ /dev/null @@ -1,213 +0,0 @@ -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using RabbitMQ.Client; - -namespace Mocha.Transport.RabbitMQ.Tests.Connection; - -public class RabbitMQConnectionManagerBaseTests -{ - [Fact] - public async Task GetConnectionAsync_Should_CreateConnection_When_NotConnected() - { - // arrange - var connectionMock = CreateOpenConnection(); - await using var manager = CreateManager(connectionMock.Object); - - // act - var result = await manager.GetConnectionAsync(CancellationToken.None); - - // assert - Assert.Same(connectionMock.Object, result); - } - - [Fact] - public async Task GetConnectionAsync_Should_ReuseConnection_When_AlreadyConnected() - { - // arrange - var factoryCallCount = 0; - var connectionMock = CreateOpenConnection(); - await using var manager = new TestConnectionManager( - NullLoggerFactory.Instance.CreateLogger(), - _ => - { - factoryCallCount++; - return new ValueTask(connectionMock.Object); - }); - - // act - var first = await manager.GetConnectionAsync(CancellationToken.None); - var second = await manager.GetConnectionAsync(CancellationToken.None); - - // assert - Assert.Same(first, second); - Assert.Equal(1, factoryCallCount); - } - - [Fact] - public async Task GetConnectionAsync_Should_ThrowObjectDisposed_When_Disposed() - { - // arrange - var connectionMock = CreateOpenConnection(); - var manager = CreateManager(connectionMock.Object); - await manager.DisposeAsync(); - - // act & assert - await Assert.ThrowsAsync(() => - manager.GetConnectionAsync(CancellationToken.None).AsTask() - ); - } - - [Fact] - public async Task EnsureConnectedAsync_Should_CreateConnection_When_NotConnected() - { - // arrange - var connectionMock = CreateOpenConnection(); - await using var manager = CreateManager(connectionMock.Object); - - // act - await manager.EnsureConnectedAsync(TestContext.Current.CancellationToken); - - // assert - Assert.True(manager.IsConnected); - } - - [Fact] - public async Task DisposeAsync_Should_CloseAndDisposeConnection_When_Connected() - { - // arrange - var connectionMock = CreateOpenConnection(); - var manager = CreateManager(connectionMock.Object); - - await manager.EnsureConnectedAsync(TestContext.Current.CancellationToken); - - // act - await manager.DisposeAsync(); - - // assert - connectionMock.Verify(c => c.CloseAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny()), Times.AtLeastOnce()); - connectionMock.Verify(c => c.DisposeAsync(), Times.AtLeastOnce()); - } - - [Fact] - public async Task DisposeAsync_Should_BeIdempotent_When_CalledTwice() - { - // arrange - var connectionMock = CreateOpenConnection(); - var manager = CreateManager(connectionMock.Object); - await manager.EnsureConnectedAsync(TestContext.Current.CancellationToken); - - // act - await manager.DisposeAsync(); - var exception = await Record.ExceptionAsync(() => manager.DisposeAsync().AsTask()); - - // assert - Assert.Null(exception); - } - - [Fact] - public async Task Lifecycle_Should_InvokeBeforeConnectionCreated_When_Connecting() - { - // arrange - var connectionMock = CreateOpenConnection(); - await using var manager = CreateManager(connectionMock.Object); - - // act - await manager.EnsureConnectedAsync(TestContext.Current.CancellationToken); - - // assert - Assert.Equal(1, manager.BeforeConnectionCreatedCount); - } - - [Fact] - public async Task Lifecycle_Should_InvokeAfterConnectionCreated_When_Connecting() - { - // arrange - var connectionMock = CreateOpenConnection(); - await using var manager = CreateManager(connectionMock.Object); - - // act - await manager.EnsureConnectedAsync(TestContext.Current.CancellationToken); - - // assert - Assert.Equal(1, manager.AfterConnectionCreatedCount); - } - - [Fact] - public async Task Lifecycle_Should_InvokeConnectionEstablished_When_Connecting() - { - // arrange - var connectionMock = CreateOpenConnection(); - await using var manager = CreateManager(connectionMock.Object); - - // act - await manager.EnsureConnectedAsync(TestContext.Current.CancellationToken); - - // assert - Assert.Equal(1, manager.ConnectionEstablishedCount); - } - - private static Mock CreateOpenConnection() - { - var connectionMock = new Mock(); - connectionMock.SetupGet(c => c.IsOpen).Returns(true); - connectionMock.SetupGet(c => c.ClientProvidedName).Returns("test-connection"); - return connectionMock; - } - - private static TestConnectionManager CreateManager(IConnection connection) - { - return new TestConnectionManager( - NullLoggerFactory.Instance.CreateLogger(), - _ => new ValueTask(connection)); - } - - private sealed class TestConnectionManager : RabbitMQConnectionManagerBase - { - public int BeforeConnectionCreatedCount { get; private set; } - public int AfterConnectionCreatedCount { get; private set; } - public int ConnectionLostCount { get; private set; } - public int ConnectionRecoveredCount { get; private set; } - public int ConnectionEstablishedCount { get; private set; } - - public TestConnectionManager(ILogger logger, Func> factory) - : base(logger, factory) { } - - protected override Task OnBeforeConnectionCreatedAsync(CancellationToken cancellationToken) - { - BeforeConnectionCreatedCount++; - return Task.CompletedTask; - } - - protected override Task OnAfterConnectionCreatedAsync( - IConnection connection, - CancellationToken cancellationToken) - { - AfterConnectionCreatedCount++; - return Task.CompletedTask; - } - - protected override Task OnConnectionLostAsync() - { - ConnectionLostCount++; - return Task.CompletedTask; - } - - protected override Task OnConnectionRecoveredAsync(CancellationToken cancellationToken) - { - ConnectionRecoveredCount++; - return Task.CompletedTask; - } - - protected override Task OnConnectionEstablished(IConnection connection, CancellationToken cancellationToken) - { - ConnectionEstablishedCount++; - return Task.CompletedTask; - } - } -} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConnectionManagerIntegrationTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConnectionManagerIntegrationTests.cs new file mode 100644 index 00000000000..b8731148cfc --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConnectionManagerIntegrationTests.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Mocha.Transport.RabbitMQ.Tests.Helpers; + +namespace Mocha.Transport.RabbitMQ.Tests.Connection; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class RabbitMQConnectionManagerIntegrationTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task DisposeAsync_Should_CloseConnection_When_ConnectionAttemptCompletesAfterDisposal() + { + // arrange + // a connection provider that does not observe cancellation hands its connection over after the disposal + await using var vhost = await fixture.CreateVhostAsync(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var manager = new RabbitMQConnectionManager( + NullLogger.Instance, + "late", + TimeProvider.System, + async _ => + { + entered.TrySetResult(); + await release.Task; + return await vhost.ConnectionFactory.CreateConnectionAsync(CancellationToken.None); + }, + static (_, _) => Task.CompletedTask); + var starting = manager.StartAsync(TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + var disposing = manager.DisposeAsync().AsTask(); + var connectionsBeforeRelease = await vhost.Broker.GetConnectionCountAsync(); + var disposeWaitedForAttempt = !disposing.IsCompleted; + release.TrySetResult(); + await disposing.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + Assert.True(disposeWaitedForAttempt); + Assert.Equal(0, connectionsBeforeRelease); + await Assert.ThrowsAsync(() => starting); + await vhost.Broker.WaitForNoConnectionAsync(); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConnectionManagerTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConnectionManagerTests.cs new file mode 100644 index 00000000000..07910420bcf --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConnectionManagerTests.cs @@ -0,0 +1,147 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Connection; + +public sealed class RabbitMQConnectionManagerTests +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(5); + + [Fact] + public async Task GetConnectionAsync_Should_ThrowObjectDisposed_When_Disposed() + { + // arrange + var manager = CreateManager(NeverConnectAsync); + await manager.DisposeAsync(); + + // act + var exception = await Record.ExceptionAsync( + () => manager.GetConnectionAsync(TestContext.Current.CancellationToken).AsTask()); + + // assert + Assert.IsType(exception); + } + + [Fact] + public async Task DisposeAsync_Should_FailPendingStart_When_ConnectionAttemptIsPending() + { + // arrange + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var manager = CreateManager( + async ct => + { + entered.TrySetResult(); + return await NeverConnectAsync(ct); + }); + var starting = manager.StartAsync(TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(s_timeout, TestContext.Current.CancellationToken); + + // act + await manager.DisposeAsync().AsTask().WaitAsync(s_timeout, TestContext.Current.CancellationToken); + + // assert + await Assert.ThrowsAsync(() => starting); + Assert.False(manager.IsConnected); + } + + [Fact] + public async Task StartAsync_Should_TimeOut_When_ConnectionAttemptDoesNotComplete() + { + // arrange + await using var manager = CreateManager(NeverConnectAsync); + manager.ConnectTimeout = TimeSpan.FromMilliseconds(100); + + // act + var exception = await Assert.ThrowsAsync( + () => manager.StartAsync(TestContext.Current.CancellationToken).WaitAsync(s_timeout, TestContext.Current.CancellationToken)); + + // assert + Assert.Equal("Connecting to RabbitMQ did not complete within 00:00:00.1000000.", exception.Message); + Assert.False(TestContext.Current.CancellationToken.IsCancellationRequested); + } + + [Fact] + public async Task StartAsync_Should_DisposeConnection_When_SetUpDoesNotComplete() + { + // arrange + var connection = new Mock(); + connection.SetupGet(c => c.IsOpen).Returns(true); + await using var manager = CreateManager( + _ => new ValueTask(connection.Object), + static (_, ct) => Task.Delay(Timeout.InfiniteTimeSpan, ct)); + manager.ConnectTimeout = TimeSpan.FromMilliseconds(100); + + // act + await Assert.ThrowsAsync( + () => manager.StartAsync(TestContext.Current.CancellationToken).WaitAsync(s_timeout, TestContext.Current.CancellationToken)); + + // assert + connection.Verify(c => c.DisposeAsync(), Times.Once); + Assert.False(manager.IsConnected); + } + + [Fact] + public async Task DisposeAsync_Should_DisposeConnection_When_CloseFails() + { + // arrange + var connection = new Mock(); + connection.SetupGet(c => c.IsOpen).Returns(true); + connection + .Setup(c => c.CloseAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new IOException("The socket is gone.")); + var manager = CreateManager(_ => new ValueTask(connection.Object)); + await manager.StartAsync(TestContext.Current.CancellationToken); + + // act + await manager.DisposeAsync(); + + // assert + connection.Verify(c => c.DisposeAsync(), Times.Once); + } + + [Theory] + [InlineData(typeof(OperationCanceledException))] + [InlineData(typeof(TimeoutException))] + public async Task CloseChannelAsync_Should_DisposeChannel_When_CloseFails(Type failure) + { + // arrange + var channel = new Mock(); + channel.SetupGet(c => c.IsOpen).Returns(true); + channel + .Setup(c => c.CloseAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync((Exception)Activator.CreateInstance(failure)!); + + // act + await RabbitMQConnectionManager.CloseChannelAsync(channel.Object); + + // assert + channel.Verify(c => c.DisposeAsync(), Times.Once); + } + + private static RabbitMQConnectionManager CreateManager( + Func> connectionFactory, + Func? setUpConnectionAsync = null) + => new( + NullLogger.Instance, + "test", + TimeProvider.System, + connectionFactory, + setUpConnectionAsync ?? (static (_, _) => Task.CompletedTask)); + + private static async ValueTask NeverConnectAsync(CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + throw new UnreachableException(); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerAttachmentIntegrationTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerAttachmentIntegrationTests.cs new file mode 100644 index 00000000000..eb576d2ddd7 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerAttachmentIntegrationTests.cs @@ -0,0 +1,342 @@ +using System.Text; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ.Tests.Connection; + +[Collection(RabbitMQCollections.ConsumerRecovery)] +public sealed class RabbitMQConsumerAttachmentIntegrationTests(RabbitMQFixture fixture) +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Consumer_Should_Recover_When_ChannelOpenTimesOut(bool automaticRecoveryEnabled) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 1; + vhost.ConnectionSettings.ContinuationTimeout = TimeSpan.FromSeconds(1); + vhost.ConnectionSettings.AutomaticRecoveryEnabled = automaticRecoveryEnabled; + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + new CreateChannelOptions(publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true), + TestContext.Current.CancellationToken); + await DeclareQueueAsync(channel, "orders"); + await channel.BasicPublishAsync("", "orders", "retained"u8.ToArray(), TestContext.Current.CancellationToken); + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var manager = CreateManager(vhost); + var original = await manager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + + // act + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + await manager.RegisterConsumerAsync( + "orders", + (consumerChannel, delivery, ct) => RecordDeliveryAsync(received, consumerChannel, delivery, ct), + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + } + + var delivered = await received.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var current = await manager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + + // assert + Assert.Equal("retained", delivered); + Assert.NotSame(original, current); + } + + [Fact] + public async Task RegisterConsumerAsync_Should_EndPromptly_When_CancelledWhileTheBrokerDoesNotAnswer() + { + // arrange + // the client holds a request the broker does not answer for 20 seconds, whatever the caller's token says + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await DeclareQueueAsync(channel, "orders"); + await using var manager = CreateManager(vhost); + await manager.StartAsync(TestContext.Current.CancellationToken); + var original = await manager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + using var cancellation = new CancellationTokenSource(); + Task registration; + + // act + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + registration = manager.RegisterConsumerAsync( + "orders", + static (_, _, _) => default, + maxPrefetch: 1, + maxConcurrency: 1, + cancellation.Token); + await cancellation.CancelAsync(); + await Assert.ThrowsAnyAsync( + () => registration.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + + var current = await manager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + + // assert + Assert.Same(original, current); + await vhost.Broker.WaitForNoConsumerAsync("orders"); + } + + [Fact] + public async Task RegisterConsumerAsync_Should_End_When_ManagerIsDisposedDuringTheFirstAttempt() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await DeclareQueueAsync(channel, "orders"); + var manager = CreateManager(vhost); + await manager.StartAsync(TestContext.Current.CancellationToken); + Task registration; + + // act + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + registration = manager.RegisterConsumerAsync( + "orders", + static (_, _, _) => default, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + await manager.DisposeAsync().AsTask().WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + + // assert + await Assert.ThrowsAsync( + () => registration.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task RegisterConsumerAsync_Should_PreserveOtherConsumers_When_ChannelLimitReached() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 1; + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + new CreateChannelOptions(publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true), + TestContext.Current.CancellationToken); + await DeclareQueueAsync(channel, "orders"); + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var manager = CreateManager(vhost); + var original = await manager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + await manager.RegisterConsumerAsync( + "orders", + (consumerChannel, delivery, ct) => RecordDeliveryAsync(received, consumerChannel, delivery, ct), + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + + // act + await Assert.ThrowsAsync(() => manager.RegisterConsumerAsync( + "another-queue", + static (_, _, _) => default, + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken)); + await channel.BasicPublishAsync("", "orders", "available"u8.ToArray(), TestContext.Current.CancellationToken); + var delivered = await received.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var current = await manager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + + // assert + Assert.Equal("available", delivered); + Assert.Same(original, current); + } + + [Fact] + public async Task Consumer_Should_DeliverFromOtherQueue_When_OneTopologyTimesOut() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + new CreateChannelOptions(publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true), + TestContext.Current.CancellationToken); + await DeclareQueueAsync(channel, "slow"); + await DeclareQueueAsync(channel, "healthy"); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var healthyDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var manager = CreateManager(vhost); + manager.AttachTimeout = TimeSpan.FromSeconds(1); + await manager.RegisterConsumerAsync( + "slow", + async (_, consumerChannel, ct) => + { + await release.Task.WaitAsync(ct); + await DeclareQueueAsync(consumerChannel, "slow"); + }, + (consumerChannel, delivery, ct) => RecordDeliveryAsync(slowDelivery, consumerChannel, delivery, ct), + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + await manager.RegisterConsumerAsync( + "healthy", + (consumerChannel, delivery, ct) => RecordDeliveryAsync(healthyDelivery, consumerChannel, delivery, ct), + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + + // act + try + { + await manager.StartAsync(TestContext.Current.CancellationToken) + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "healthy", "available"u8.ToArray(), TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "slow", "retained"u8.ToArray(), TestContext.Current.CancellationToken); + + // assert + Assert.Equal("available", await healthyDelivery.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + Assert.Equal(1u, (await channel.QueueDeclarePassiveAsync("slow", TestContext.Current.CancellationToken)).MessageCount); + } + finally + { + release.TrySetResult(); + } + + Assert.Equal("retained", await slowDelivery.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task Consumer_Should_WaitForReplyQueue_When_DependencyTopologyIsDelayed() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + new CreateChannelOptions(publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true), + TestContext.Current.CancellationToken); + await DeclareQueueAsync(channel, "requests"); + var replyStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var requestStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var response = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var manager = CreateManager(vhost); + var replies = await manager.RegisterConsumerAsync( + "replies", + async (_, consumerChannel, ct) => + { + replyStarted.TrySetResult(); + await release.Task.WaitAsync(ct); + await DeclareQueueAsync(consumerChannel, "replies"); + }, + (consumerChannel, delivery, ct) => RecordDeliveryAsync(response, consumerChannel, delivery, ct), + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + await manager.RegisterConsumerAsync( + "requests", + async (_, _, ct) => + { + requestStarted.TrySetResult(); + await replies.WaitUntilAttachedAsync(ct); + }, + async (consumerChannel, delivery, ct) => + { + await consumerChannel.BasicPublishAsync("", "replies", delivery.Body, ct); + await consumerChannel.BasicAckAsync(delivery.DeliveryTag, false, ct); + }, + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + + // act + var connecting = manager.StartAsync(TestContext.Current.CancellationToken); + QueueDeclareOk pending; + try + { + await Task.WhenAll(replyStarted.Task, requestStarted.Task) + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "requests", "ORD-1"u8.ToArray(), TestContext.Current.CancellationToken); + pending = await channel.QueueDeclarePassiveAsync("requests", TestContext.Current.CancellationToken); + } + finally + { + release.TrySetResult(); + await connecting.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + + // assert + Assert.Equal(0u, pending.ConsumerCount); + Assert.Equal(1u, pending.MessageCount); + Assert.Equal("ORD-1", await response.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task StopAsync_Should_RetainQueuedMessage_When_ConsumerStopsDuringProvisioning() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + new CreateChannelOptions(publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true), + TestContext.Current.CancellationToken); + await DeclareQueueAsync(channel, "orders"); + await channel.BasicPublishAsync("", "orders", "ORD-1"u8.ToArray(), TestContext.Current.CancellationToken); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var manager = CreateManager(vhost); + var consumer = await manager.RegisterConsumerAsync( + "orders", + async (_, _, ct) => + { + started.TrySetResult(); + await release.Task.WaitAsync(ct); + }, + static (_, _, _) => default, + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + + // act + var connecting = manager.StartAsync(TestContext.Current.CancellationToken); + try + { + await started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await consumer.StopAsync(TestContext.Current.CancellationToken) + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + finally + { + release.TrySetResult(); + await connecting.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + + var queue = await channel.QueueDeclarePassiveAsync("orders", TestContext.Current.CancellationToken); + var retained = await channel.BasicGetAsync("orders", true, TestContext.Current.CancellationToken); + + // assert + Assert.Equal(0u, queue.ConsumerCount); + Assert.Equal(1u, queue.MessageCount); + Assert.Equal("ORD-1", Encoding.UTF8.GetString(retained!.Body.Span)); + } + + private static RabbitMQConsumerManager CreateManager(VhostContext vhost) + => TestManagers.CreateConsumerManager(async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct)); + + private static async Task DeclareQueueAsync(IChannel channel, string queue) + => await channel.QueueDeclareAsync( + queue, + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + + private static async ValueTask RecordDeliveryAsync( + TaskCompletionSource received, + IChannel channel, + BasicDeliverEventArgs delivery, + CancellationToken cancellationToken) + { + var value = Encoding.UTF8.GetString(delivery.Body.Span); + await channel.BasicAckAsync(delivery.DeliveryTag, false, cancellationToken); + received.TrySetResult(value); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerCapacityTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerCapacityTests.cs new file mode 100644 index 00000000000..a34502c560e --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerCapacityTests.cs @@ -0,0 +1,322 @@ +using System.Collections.Concurrent; +using System.Text; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; + +namespace Mocha.Transport.RabbitMQ.Tests.Connection; + +[Collection(RabbitMQCollections.Messaging)] +public sealed class RabbitMQConsumerCapacityTests(RabbitMQFixture fixture) +{ + private const string BlockedReason = + "Queue 'orders': The consumer waits for a handler of a lost channel to return."; + + [Fact] + public async Task Consumer_Should_ResumeReceiving_When_BlockedHandlerExits() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + "orders", + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + await using var manager = TestManagers.CreateConsumerManager( + async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct)); + await manager.StartAsync(TestContext.Current.CancellationToken); + var gate = new HandlerGate(blockedHandlers: 1); + var registration = await manager.RegisterConsumerAsync( + "orders", + gate.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "orders", "order"u8.ToArray(), TestContext.Current.CancellationToken); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + try + { + // act + await registration.Channel!.BasicAckAsync(9999, multiple: false, TestContext.Current.CancellationToken); + var blocked = await Condition.WaitAsync( + () => gate.CancelledHandlers == 1 && manager.HealthFailureReason == BlockedReason, + TestTimeouts.Signal); + gate.Release.TrySetResult(); + var recovered = await Condition.WaitAsync( + () => gate.Delivered.Count == 1 && manager.HealthFailureReason is null, + TestTimeouts.Signal); + + // assert + Assert.True(blocked); + Assert.True(recovered); + Assert.Equal(1, gate.PeakConcurrency); + Assert.Equal(["order"], gate.Delivered); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task Consumer_Should_UseRemainingCapacity_When_OneLostHandlerIgnoresCancellation() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + "orders", + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + await using var manager = TestManagers.CreateConsumerManager( + async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct)); + await manager.StartAsync(TestContext.Current.CancellationToken); + var gate = new HandlerGate(blockedHandlers: 1); + var registration = await manager.RegisterConsumerAsync( + "orders", + gate.HandleAsync, + maxPrefetch: 2, + maxConcurrency: 2, + TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "orders", "order-0"u8.ToArray(), TestContext.Current.CancellationToken); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "orders", "order-1"u8.ToArray(), TestContext.Current.CancellationToken); + Assert.True(await Condition.WaitAsync(() => gate.Delivered.Count == 1 && gate.ActiveHandlers == 1, TestTimeouts.Signal)); + + try + { + // act + var lostChannel = registration.Channel!; + await lostChannel.BasicAckAsync(9999, multiple: false, TestContext.Current.CancellationToken); + var redelivered = await Condition.WaitAsync( + () => gate.Delivered.Count == 2 && gate.CancelledHandlers == 1 && gate.ActiveHandlers == 1, + TestTimeouts.Signal); + + // assert + Assert.True(redelivered); + Assert.Equal( + new HandlerState(ActiveHandlers: 1, PeakConcurrency: 2, Invocations: 3), + gate.State); + Assert.Null(manager.HealthFailureReason); + Assert.Equal(["order-0", "order-1"], gate.Delivered.Order()); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task Consumer_Should_WaitForCapacity_When_AllLostHandlersIgnoreCancellation() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync("orders", durable: true, exclusive: false, autoDelete: false, cancellationToken: TestContext.Current.CancellationToken); + await using var manager = TestManagers.CreateConsumerManager( + async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct)); + await manager.StartAsync(TestContext.Current.CancellationToken); + var gate = new HandlerGate(blockedHandlers: 2); + var registration = await manager.RegisterConsumerAsync( + "orders", + gate.HandleAsync, + maxPrefetch: 4, + maxConcurrency: 2, + TestContext.Current.CancellationToken); + for (var i = 0; i < 4; i++) + { + await channel.BasicPublishAsync("", "orders", Encoding.UTF8.GetBytes($"order-{i}"), TestContext.Current.CancellationToken); + } + + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + try + { + // act + await registration.Channel!.BasicAckAsync(9999, multiple: false, TestContext.Current.CancellationToken); + var cancelled = await Condition.WaitAsync(() => gate.CancelledHandlers == 2, TestTimeouts.Signal); + + // the consumer must not consume, and so not prefetch, while no handler can take a message + await Wait.UntilAsync(() => manager.HealthFailureReason == BlockedReason); + await vhost.Broker.WaitForNoConsumerAsync("orders"); + var whileBlocked = (registration.IsAttached, gate.Invocations, gate.ActiveHandlers); + gate.Release.TrySetResult(); + var delivered = await Condition.WaitAsync( + () => gate.Delivered.Count == 4 && manager.HealthFailureReason is null, + TestTimeouts.Signal); + + // assert + Assert.Equal((false, 2, 2), whileBlocked); + Assert.True(cancelled); + Assert.True(delivered); + Assert.Equal(2, gate.PeakConcurrency); + Assert.Equal(["order-0", "order-1", "order-2", "order-3"], gate.Delivered.Order()); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task StopAsync_Should_CompleteWithinDeadline_When_LostChannelHandlerIgnoresCancellation() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync("orders", durable: true, exclusive: false, autoDelete: false, cancellationToken: TestContext.Current.CancellationToken); + await using var manager = TestManagers.CreateConsumerManager( + async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct)); + await manager.StartAsync(TestContext.Current.CancellationToken); + var gate = new HandlerGate(blockedHandlers: 1); + var registration = await manager.RegisterConsumerAsync( + "orders", + gate.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "orders", "order"u8.ToArray(), TestContext.Current.CancellationToken); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + try + { + // act + var lostChannel = registration.Channel!; + await lostChannel.BasicAckAsync(9999, multiple: false, TestContext.Current.CancellationToken); + var cancelled = await Condition.WaitAsync(() => gate.CancelledHandlers == 1, TestTimeouts.Signal); + using var deadline = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + await registration.StopAsync(deadline.Token).WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var activeAtStop = gate.ActiveHandlers; + gate.Release.TrySetResult(); + var finished = await Condition.WaitAsync(() => gate.ActiveHandlers == 0, TestTimeouts.Signal); + + // assert + Assert.True(cancelled); + Assert.Equal(1, activeAtStop); + Assert.True(finished); + Assert.Equal(1, gate.Invocations); + Assert.False(registration.IsAttached); + } + finally + { + gate.Release.TrySetResult(); + } + } + + [Fact] + public async Task DisposeAsync_Should_EndGracefulStop_When_HandlerIgnoresCancellation() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + "orders", + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + await using var manager = TestManagers.CreateConsumerManager( + async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct)); + await manager.StartAsync(TestContext.Current.CancellationToken); + var gate = new HandlerGate(blockedHandlers: 1); + var registration = await manager.RegisterConsumerAsync( + "orders", + gate.HandleAsync, + 1, + 1, + TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "orders", "order"u8.ToArray(), TestContext.Current.CancellationToken); + await gate.Started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + try + { + // act + var gracefulStop = registration.StopAsync(CancellationToken.None); + await vhost.Broker.WaitForNoConsumerAsync("orders"); + var draining = !gracefulStop.IsCompleted; + await registration.DisposeAsync().AsTask().WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await gracefulStop.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await vhost.Broker.WaitForMessagesAsync("orders", 1); + + // assert + Assert.True(draining); + Assert.True(await Condition.WaitAsync(() => gate.CancelledHandlers == 1, TestTimeouts.Signal)); + Assert.False(registration.IsAttached); + Assert.Equal(1, gate.ActiveHandlers); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("orders"))?.Messages); + } + finally + { + gate.Release.TrySetResult(); + } + } + + private sealed record HandlerState(int ActiveHandlers, int PeakConcurrency, int Invocations); + + private sealed class HandlerGate(int blockedHandlers) + { + private int _invocations; + private int _activeHandlers; + private int _peakConcurrency; + private int _cancelledHandlers; + + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public ConcurrentQueue Delivered { get; } = new(); + public int Invocations => Volatile.Read(ref _invocations); + public int ActiveHandlers => Volatile.Read(ref _activeHandlers); + public int PeakConcurrency => Volatile.Read(ref _peakConcurrency); + public int CancelledHandlers => Volatile.Read(ref _cancelledHandlers); + public HandlerState State => new(ActiveHandlers, PeakConcurrency, Invocations); + + public async ValueTask HandleAsync(IChannel channel, BasicDeliverEventArgs delivery, CancellationToken cancellationToken) + { + var invocation = Interlocked.Increment(ref _invocations); + var active = Interlocked.Increment(ref _activeHandlers); + var peak = Volatile.Read(ref _peakConcurrency); + while (active > peak) + { + var previous = Interlocked.CompareExchange(ref _peakConcurrency, active, peak); + if (previous == peak) + { + break; + } + + peak = previous; + } + + try + { + if (invocation <= blockedHandlers) + { + await using var registration = cancellationToken.Register(() => Interlocked.Increment(ref _cancelledHandlers)); + if (invocation == blockedHandlers) + { + Started.TrySetResult(); + } + + await Release.Task; + } + else + { + Delivered.Enqueue(Encoding.UTF8.GetString(delivery.Body.Span)); + await channel.BasicAckAsync(delivery.DeliveryTag, false, CancellationToken.None); + } + } + finally + { + Interlocked.Decrement(ref _activeHandlers); + } + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerManagerRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerManagerRecoveryTests.cs new file mode 100644 index 00000000000..cbafe745ca5 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerManagerRecoveryTests.cs @@ -0,0 +1,526 @@ +using System.Text; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ.Tests.Connection; + +[Collection(RabbitMQCollections.ConsumerRecovery)] +public class RabbitMQConsumerManagerRecoveryTests +{ + private const string QueueName = "recovery-queue"; + + // A consumer reacts to a loss at once, so a recovery that needs the general signal timeout is too slow. + private static readonly TimeSpan s_promptly = TimeSpan.FromSeconds(10); + + private readonly RabbitMQFixture _fixture; + + public RabbitMQConsumerManagerRecoveryTests(RabbitMQFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task Consumer_Should_BeReattached_When_BrokerClosesChannel() + { + // arrange + await using var vhost = await _fixture.CreateVhostAsync(); + await using var admin = await AdminChannel.CreateAsync(vhost); + await using var manager = await CreateManagerAsync(vhost); + var received = new DeliveryRecorder(); + + var registration = await manager.RegisterConsumerAsync( + QueueName, + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + + await admin.PublishAsync("A"); + Assert.Equal("A", await received.NextAsync()); + + // act + // acking an unknown delivery tag makes the broker close the channel with 406 PRECONDITION_FAILED, + // which is what a consumer timeout does before RabbitMQ 4.3 + await CloseChannelByBrokerAsync(registration); + await admin.PublishAsync("B"); + + // assert + Assert.Equal("B", await received.NextAsync()); + } + + [Fact] + public async Task Consumer_Should_BeReattached_When_BrokerCancelsConsumer() + { + // arrange + await using var vhost = await _fixture.CreateVhostAsync(); + await using var admin = await AdminChannel.CreateAsync(vhost); + await using var manager = await CreateManagerAsync(vhost); + var received = new DeliveryRecorder(); + + await manager.RegisterConsumerAsync( + QueueName, + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + + await admin.PublishAsync("A"); + Assert.Equal("A", await received.NextAsync()); + + // act + // deleting the queue makes the broker send basic.cancel and keep the channel open, + // which is what a consumer timeout does from RabbitMQ 4.3 on + await admin.Channel.QueueDeleteAsync(QueueName, cancellationToken: TestContext.Current.CancellationToken); + await admin.DeclareQueueAsync(); + await admin.PublishAsync("B"); + + // assert + Assert.Equal("B", await received.NextAsync()); + } + + [Fact] + public async Task Consumer_Should_ReceiveReturnedMessageFirst_When_BrokerClosesChannelDuringHandler() + { + // arrange + await using var vhost = await _fixture.CreateVhostAsync(); + await using var admin = await AdminChannel.CreateAsync(vhost); + await using var manager = await CreateManagerAsync(vhost); + var received = new DeliveryRecorder(); + var abandonedHandlerCancelled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var registration = await manager.RegisterConsumerAsync( + QueueName, + async (channel, eventArgs, cancellationToken) => + { + // the first delivery of A blocks without being acked, as a handler retrying in place would + if (!eventArgs.Redelivered && Encoding.UTF8.GetString(eventArgs.Body.Span) == "A") + { + received.Record(eventArgs); + + try + { + await Task.Delay(Timeout.Infinite, cancellationToken); + } + catch (OperationCanceledException) + { + abandonedHandlerCancelled.TrySetResult(); + } + + return; + } + + await received.HandleAsync(channel, eventArgs, cancellationToken); + }, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + + await admin.PublishAsync("A"); + Assert.Equal("A", await received.NextAsync()); + + // act + await CloseChannelByBrokerAsync(registration); + await admin.PublishAsync("B"); + + // assert + Assert.Equal("A", await received.NextAsync()); + Assert.Equal("B", await received.NextAsync()); + + await abandonedHandlerCancelled.Task.WaitAsync(s_promptly, TestContext.Current.CancellationToken); + Assert.Null(manager.HealthFailureReason); + } + + [Fact] + public async Task Consumer_Should_NotBeReattached_When_Disposed() + { + // arrange + const string witnessQueue = "witness-queue"; + await using var vhost = await _fixture.CreateVhostAsync(); + await using var admin = await AdminChannel.CreateAsync(vhost); + await admin.DeclareQueueAsync(witnessQueue); + await using var manager = await CreateManagerAsync(vhost); + var received = new DeliveryRecorder(); + + var registration = await manager.RegisterConsumerAsync( + QueueName, + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + + // a second consumer of the same manager shows when the consumers attached to the new connection + await manager.RegisterConsumerAsync( + witnessQueue, + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + var witness = await vhost.Broker.GetConsumerAsync(witnessQueue); + + // act + await registration.DisposeAsync(); + var afterDispose = await vhost.Broker.GetQueueAsync(QueueName); + await vhost.Faults.CloseConsumerConnectionAsync(witnessQueue); + await vhost.Broker.WaitForConsumerAsync(witnessQueue, otherThan: witness); + var afterReconnect = await vhost.Broker.GetQueueAsync(QueueName); + + // assert + Assert.Equal(new QueueInfo(Consumers: 0, Ready: 0, Unacknowledged: 0), afterDispose); + Assert.Equal(new QueueInfo(Consumers: 0, Ready: 0, Unacknowledged: 0), afterReconnect); + } + + [Fact] + public async Task Consumer_Should_StayDetached_When_DisposedDuringReattachBackoff() + { + // arrange + await using var vhost = await _fixture.CreateVhostAsync(); + await using var admin = await AdminChannel.CreateAsync(vhost); + await using var manager = await CreateManagerAsync(vhost); + var received = new DeliveryRecorder(); + + var registration = await manager.RegisterConsumerAsync( + QueueName, + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + + // the queue stays missing, so every re-attach attempt fails and the consumer backs off + await vhost.Faults.DeleteQueueAsync(QueueName); + Assert.True( + await Condition.WaitAsync( + async () => await vhost.Broker.GetQueueAsync(QueueName) is null, + s_promptly)); + + // act + await registration.DisposeAsync().AsTask().WaitAsync(s_promptly, TestContext.Current.CancellationToken); + await admin.DeclareQueueAsync(); + + // assert + // nothing signals that a disposed consumer will never attach again, so this watches for a consumer for + // longer than the first delays of a re-attach that outlived the dispose + Assert.False( + await Condition.WaitAsync( + async () => (await vhost.Broker.GetQueueAsync(QueueName))?.Consumers > 0, + TimeSpan.FromSeconds(4))); + } + + [Fact] + public async Task RegisterConsumerAsync_Should_LeaveNoConsumerBehind_When_FirstAttachFails() + { + // arrange + await using var vhost = await _fixture.CreateVhostAsync(); + await using var admin = await AdminChannel.CreateAsync(vhost); + await using var manager = await CreateManagerAsync(vhost); + var received = new DeliveryRecorder(); + await manager.RegisterConsumerAsync( + QueueName, + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + + // act + var exception = await Assert.ThrowsAsync(() => manager.RegisterConsumerAsync( + "missing-queue", + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None)); + await admin.Channel.QueueDeclareAsync( + "missing-queue", + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + + // every consumer attaches to a new connection by itself, so a registration that was left behind has a + // consumer once the other consumer of the manager is attached again + var before = await vhost.Broker.GetConsumerAsync(QueueName); + await vhost.Faults.CloseConsumerConnectionAsync(QueueName); + await vhost.Broker.WaitForConsumerAsync(QueueName, otherThan: before); + var missingQueue = await vhost.Broker.GetQueueAsync("missing-queue"); + + // assert + Assert.Equal(Constants.NotFound, exception.ShutdownReason?.ReplyCode); + Assert.Equal(new QueueInfo(Consumers: 0, Ready: 0, Unacknowledged: 0), missingQueue); + } + + [Fact] + public async Task RegisterConsumerAsync_Should_KeepOtherConsumersAttached_When_RegistrationIsCancelled() + { + // arrange + const string otherQueue = "other-queue"; + await using var vhost = await _fixture.CreateVhostAsync(); + await using var admin = await AdminChannel.CreateAsync(vhost); + await admin.DeclareQueueAsync(otherQueue); + await using var manager = await CreateManagerAsync(vhost); + var received = new DeliveryRecorder(); + await manager.RegisterConsumerAsync( + QueueName, + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + var original = await manager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + var before = await vhost.Broker.GetConsumerAsync(QueueName); + var cancelledRegistrations = 0; + + // act + // a registration is cancelled at a random moment while its channel opens and its consumer attaches + for (var i = 0; i < 100; i++) + { + using var cancellation = new CancellationTokenSource(); + var registration = manager.RegisterConsumerAsync( + otherQueue, + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + cancellation.Token); + Thread.SpinWait(Random.Shared.Next(0, 80_000)); + await cancellation.CancelAsync(); + + try + { + await (await registration).DisposeAsync(); + } + catch (OperationCanceledException) + { + cancelledRegistrations++; + } + } + + var current = await manager.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + var after = await vhost.Broker.GetConsumerAsync(QueueName); + + // assert + Assert.True(cancelledRegistrations > 0, "No registration was cancelled while it was pending."); + Assert.Same(original, current); + Assert.Equal(before, after); + } + + [Fact] + public async Task Consumers_Should_AllReattach_When_ConnectionClosedWithManyQueues() + { + // arrange + const int count = 50; + await using var vhost = await _fixture.CreateVhostAsync(); + await using (var admin = await AdminChannel.CreateAsync(vhost)) + { + for (var i = 0; i < count; i++) + { + await admin.Channel.QueueDeclareAsync( + $"many-{i}", + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + } + } + + await using var manager = await CreateManagerAsync(vhost); + var received = new DeliveryRecorder(); + for (var i = 0; i < count; i++) + { + await manager.RegisterConsumerAsync( + $"many-{i}", + received.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + CancellationToken.None); + } + + var before = await vhost.Broker.GetConsumersAsync(); + var lostTags = before.Select(consumer => consumer.ConsumerTag).ToHashSet(); + + // act + await vhost.Faults.CloseConnectionsAsync(); + var after = await Wait.UntilAsync( + vhost.Broker.GetConsumersAsync, + consumers => consumers.Count == count && !consumers.Any(consumer => lostTags.Contains(consumer.ConsumerTag)), + $"{count} consumers that attached after the connection was closed", + within: s_promptly); + var state = await vhost.Broker.GetStateAsync(); + + // assert + Assert.Equal(count, lostTags.Count); + Assert.Equal( + Enumerable.Range(0, count).Select(i => $"many-{i}").Order(), + after.Select(consumer => consumer.Queue).Order()); + Assert.Equal(1, state.Connections); + Assert.Equal(count, state.ConsumerChannels); + } + + [Fact] + public async Task Consumer_Should_DeliverReply_When_ReplyQueueIsRecreatedAfterConnectionLoss() + { + // arrange + await using var vhost = await _fixture.CreateVhostAsync(); + await using var admin = await AdminChannel.CreateAsync(vhost); + await using var manager = await CreateManagerAsync(vhost); + var replies = new DeliveryRecorder(); + var firstDelivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var provisioning = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var attempts = 0; + var replyConsumer = await manager.RegisterConsumerAsync( + "replies", + async (_, channel, ct) => + { + if (Interlocked.Increment(ref attempts) > 1) + { + provisioning.TrySetResult(); + await release.Task.WaitAsync(ct); + } + + await channel.QueueDeclareAsync( + "replies", + durable: false, + exclusive: true, + autoDelete: true, + cancellationToken: ct); + }, + replies.HandleAsync, + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + await manager.RegisterConsumerAsync( + QueueName, + async (_, channel, ct) => + { + await channel.QueueDeclareAsync( + QueueName, + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: ct); + await replyConsumer.WaitUntilAttachedAsync(ct); + }, + async (channel, delivery, ct) => + { + if (!delivery.Redelivered) + { + firstDelivery.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, ct); + return; + } + + await channel.BasicPublishAsync("", "replies", delivery.Body, ct); + await channel.BasicAckAsync(delivery.DeliveryTag, false, ct); + }, + maxPrefetch: 1, + maxConcurrency: 1, + TestContext.Current.CancellationToken); + await admin.PublishAsync("ORD-1"); + await firstDelivery.Task.WaitAsync(s_promptly, TestContext.Current.CancellationToken); + + // act + QueueDeclareOk pending; + try + { + await vhost.Faults.CloseConsumerConnectionAsync(QueueName); + await provisioning.Task.WaitAsync(s_promptly, TestContext.Current.CancellationToken); + pending = await admin.Channel.QueueDeclarePassiveAsync(QueueName, TestContext.Current.CancellationToken); + } + finally + { + release.TrySetResult(); + } + + // assert + Assert.Equal(0u, pending.ConsumerCount); + Assert.Equal(1u, pending.MessageCount); + Assert.Equal("ORD-1", await replies.NextAsync()); + } + + private static async Task CreateManagerAsync(VhostContext vhost) + { + var manager = TestManagers.CreateConsumerManager( + async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct)); + await manager.StartAsync(TestContext.Current.CancellationToken); + return manager; + } + + private static async Task CloseChannelByBrokerAsync(RabbitMQConsumer registration) + { + var channel = registration.Channel!; + await channel.BasicAckAsync(deliveryTag: 9999, multiple: false, TestContext.Current.CancellationToken); + + var deadline = DateTimeOffset.UtcNow.Add(s_promptly); + while (channel.IsOpen && DateTimeOffset.UtcNow < deadline) + { + await Task.Delay(50, TestContext.Current.CancellationToken); + } + + Assert.False(channel.IsOpen, "The broker did not close the consumer channel."); + } + + private sealed class DeliveryRecorder + { + private readonly System.Threading.Channels.Channel _bodies = + System.Threading.Channels.Channel.CreateUnbounded(); + + public ValueTask HandleAsync(IChannel channel, BasicDeliverEventArgs eventArgs, CancellationToken _) + { + Record(eventArgs); + return channel.BasicAckAsync(eventArgs.DeliveryTag, false, CancellationToken.None); + } + + public void Record(BasicDeliverEventArgs eventArgs) + => _bodies.Writer.TryWrite(Encoding.UTF8.GetString(eventArgs.Body.Span)); + + public async Task NextAsync() + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(s_promptly); + + return await _bodies.Reader.ReadAsync(cts.Token); + } + } + + private sealed class AdminChannel(IConnection connection, IChannel channel) : IAsyncDisposable + { + public IChannel Channel => channel; + + public static async Task CreateAsync(VhostContext vhost) + { + var connection = await vhost.ConnectionFactory.CreateConnectionAsync( + TestContext.Current.CancellationToken); + var channel = await connection.CreateChannelAsync( + new CreateChannelOptions(publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true), + TestContext.Current.CancellationToken); + var admin = new AdminChannel(connection, channel); + await admin.DeclareQueueAsync(); + return admin; + } + + public Task DeclareQueueAsync() => DeclareQueueAsync(QueueName); + + public async Task DeclareQueueAsync(string queueName) + { + await channel.QueueDeclareAsync( + queueName, + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + } + + public async Task PublishAsync(string body) + { + await channel.BasicPublishAsync( + exchange: "", + routingKey: QueueName, + body: Encoding.UTF8.GetBytes(body), + cancellationToken: TestContext.Current.CancellationToken); + } + + public async ValueTask DisposeAsync() + { + await channel.DisposeAsync(); + await connection.DisposeAsync(); + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerManagerTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerManagerTests.cs deleted file mode 100644 index de3ced3a5af..00000000000 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQConsumerManagerTests.cs +++ /dev/null @@ -1,126 +0,0 @@ -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using RabbitMQ.Client; - -namespace Mocha.Transport.RabbitMQ.Tests.Connection; - -public class RabbitMQConsumerManagerTests -{ - [Fact] - public async Task RegisterConsumerAsync_Should_TrackConsumer_When_Called() - { - // arrange - await using var manager = CreateManager(); - - // act - var registration = await manager.RegisterConsumerAsync( - "test-queue", - (_, _, _) => default, - prefetchCount: 10, - consumerDispatchConcurrency: 1, - CancellationToken.None); - - // assert - Assert.NotNull(registration); - Assert.IsAssignableFrom(registration); - } - - [Fact] - public async Task AddConsumerAsync_Should_BeThreadSafe_When_ConcurrentCalls() - { - // arrange - await using var manager = CreateManager(); - var tasks = new Task[10]; - - // act - for (var i = 0; i < 10; i++) - { - var queueName = $"queue-{i}"; - tasks[i] = manager.RegisterConsumerAsync( - queueName, - (_, _, _) => default, - prefetchCount: 10, - consumerDispatchConcurrency: 1, - CancellationToken.None); - } - - var results = await Task.WhenAll(tasks); - - // assert - Assert.Equal(10, results.Length); - Assert.All(results, Assert.NotNull); - } - - [Fact] - public async Task DisposeAsync_Should_DisposeAllConsumers_When_Called() - { - // arrange - var manager = CreateManager(); - - var registrations = new IAsyncDisposable[3]; - for (var i = 0; i < 3; i++) - { - registrations[i] = await manager.RegisterConsumerAsync( - $"queue-{i}", - (_, _, _) => default, - prefetchCount: 10, - consumerDispatchConcurrency: 1, - CancellationToken.None); - } - - // act - await manager.DisposeAsync(); - - // assert - registering after dispose via the internal method should still work - // (the manager clears its list during dispose). - // We verify dispose completes without error. - // The consumers' internal state (ConsumerTag/Channel) should be null after dispose. - foreach (var reg in registrations) - { - var consumer = (RabbitMQConsumerManager.RegisteredConsumer)reg; - Assert.Null(consumer.ConsumerTag); - Assert.Null(consumer.Channel); - } - } - - [Fact] - public async Task DisposeAsync_Should_HandleConsumerDisposeFailure_When_ConsumerThrows() - { - // arrange - var channelMock = new Mock(); - channelMock.SetupGet(c => c.IsOpen).Returns(true); - channelMock - .Setup(c => c.BasicCancelAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(Task.FromException(new InvalidOperationException("cancel failed"))); - - var manager = CreateManager(); - - var registration = await manager.RegisterConsumerAsync( - "test-queue", - (_, _, _) => default, - prefetchCount: 10, - consumerDispatchConcurrency: 1, - CancellationToken.None); - - // Manually set channel to simulate a connected consumer - var consumer = (RabbitMQConsumerManager.RegisteredConsumer)registration; - consumer.Channel = channelMock.Object; - consumer.ConsumerTag = "tag-1"; - - // act - should not throw despite consumer dispose failure - var exception = await Record.ExceptionAsync(() => manager.DisposeAsync().AsTask()); - - // assert - Assert.Null(exception); - } - - private static RabbitMQConsumerManager CreateManager() - { - var connectionMock = new Mock(); - connectionMock.SetupGet(c => c.IsOpen).Returns(false); - - return new RabbitMQConsumerManager( - NullLogger.Instance, - _ => new ValueTask(connectionMock.Object)); - } -} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQDispatcherIntegrationTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQDispatcherIntegrationTests.cs new file mode 100644 index 00000000000..c26dc820ff3 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQDispatcherIntegrationTests.cs @@ -0,0 +1,438 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ.Tests.Connection; + +[Collection(RabbitMQCollections.ConnectionRecovery)] +public sealed class RabbitMQDispatcherIntegrationTests(RabbitMQFixture fixture) +{ + [Fact] + public async Task SendAsync_Should_WaitForChannel_When_EveryChannelIsLeased() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 1; + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue("leased"); + t.DispatchEndpoint("leased-send").ToQueue("leased").Send(); + }) + .BuildTestBusAsync(); + await SendAsync(bus, new ProcessPayment { OrderId = "warmup", Amount = 1 }); + var dispatcher = bus.Transport.Dispatcher; + var lease = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + + // act + var sends = Enumerable.Range(0, 8) + .Select(i => SendAsync(bus, new ProcessPayment { OrderId = $"ORD-{i}", Amount = 1 })) + .ToArray(); + var whileLeased = await vhost.Broker.GetQueueAsync("leased"); + var allWaiting = sends.All(send => !send.IsCompleted); + dispatcher.Return(lease, reusable: true); + await Task.WhenAll(sends).WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + Assert.True(allWaiting); + Assert.Equal(1, whileLeased?.Messages); + Assert.Equal(9, (await vhost.Broker.GetQueueAsync("leased"))?.Messages); + Assert.Equal(1, (await vhost.Broker.GetStateAsync()).IdleChannels); + } + + [Fact] + public async Task SendAsync_Should_Complete_When_AnotherSendIsCancelledWhileTheBrokerDoesNotAnswer() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 2; + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue("payments"); + t.DispatchEndpoint("payment-send").ToQueue("payments").Send(); + }) + .BuildTestBusAsync(); + await SendAsync(bus, new ProcessPayment { OrderId = "warmup", Amount = 1 }); + using var cancellation = new CancellationTokenSource(); + using var scope = bus.Provider.CreateScope(); + Task surviving; + + // act + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + var cancelled = scope.ServiceProvider.GetRequiredService() + .SendAsync(new ProcessPayment { OrderId = "cancelled", Amount = 1 }, cancellation.Token).AsTask(); + surviving = SendAsync(bus, new ProcessPayment { OrderId = "surviving", Amount = 1 }); + await cancellation.CancelAsync(); + await Assert.ThrowsAnyAsync(() => cancelled); + } + + await surviving.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // assert + // the cancelled message is in the queue as well when it was written before the cancellation + Assert.InRange((await vhost.Broker.GetQueueAsync("payments"))?.Messages ?? 0, 2, 3); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PublishAsync_Should_WaitForTopology_When_PublisherConnectionRecovers(bool automaticRecoveryEnabled) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 2; + vhost.ConnectionSettings.AutomaticRecoveryEnabled = automaticRecoveryEnabled; + var recoveryStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseRecovery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var setups = 0; + await using var dispatcher = TestManagers.CreateDispatcher( + async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct), + async (connection, ct) => + { + await using var channel = await connection.CreateChannelAsync(cancellationToken: ct); + if (Interlocked.Increment(ref setups) > 1) + { + recoveryStarted.TrySetResult(); + await releaseRecovery.Task.WaitAsync(ct); + } + + await channel.ExchangeDeclareAsync("events", "fanout", durable: true, cancellationToken: ct); + await channel.QueueDeclareAsync( + "orders", + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: ct); + await channel.QueueBindAsync("orders", "events", "", cancellationToken: ct); + }); + await dispatcher.StartAsync(TestContext.Current.CancellationToken); + await vhost.Faults.DeleteQueueAsync("orders"); + + // act + await vhost.Faults.CloseConnectionsAsync(); + await recoveryStarted.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var publish = PublishToAsync(dispatcher, exchange: "events", routingKey: ""); + bool publishWaited; + try + { + using var deadline = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + await Assert.ThrowsAnyAsync( + () => dispatcher.RentChannelAsync(deadline.Token).AsTask()); + + publishWaited = !publish.IsCompleted; + } + finally + { + releaseRecovery.TrySetResult(); + await publish.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + + // assert + Assert.True(publishWaited); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("orders"))?.Messages); + } + + [Fact] + public async Task SendAsync_Should_KeepPublisherConnection_When_NegotiatedChannelLimitReached() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 2; + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue("payments"); + t.DispatchEndpoint("payment-send").ToQueue("payments").Send(); + }) + .BuildTestBusAsync(); + var dispatcher = bus.Transport.Dispatcher; + var connection = await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + var first = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + var second = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + + // act + using var deadline = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + await Assert.ThrowsAnyAsync( + () => dispatcher.RentChannelAsync(deadline.Token).AsTask()); + var channelsStayedOpen = first.Channel.IsOpen && second.Channel.IsOpen; + dispatcher.Return(first, reusable: true); + dispatcher.Return(second, reusable: true); + await SendAsync(bus, new ProcessPayment { OrderId = "ORD-1", Amount = 1 }); + var current = await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + + // assert + Assert.True(channelsStayedOpen); + Assert.Same(connection, current); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("payments"))?.Messages); + } + + [Fact] + public async Task SendAsync_Should_StayWithinChannelLimit_When_ChannelRentIsCancelledRepeatedly() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 3; + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue("pending-channel"); + t.DispatchEndpoint("pending-send").ToQueue("pending-channel").Send(); + }) + .BuildTestBusAsync(); + var dispatcher = bus.Transport.Dispatcher; + + // act + // the first three rents each leave a channel that is still opening, the other two wait for one of them + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + for (var i = 0; i < 5; i++) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + await Assert.ThrowsAnyAsync( + () => dispatcher.RentChannelAsync(timeout.Token).AsTask() + .WaitAsync(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken)); + } + } + + await SendAsync(bus, new ProcessPayment { OrderId = "resumed", Amount = 1 }); + var state = await Wait.UntilAsync( + vhost.Broker.GetStateAsync, + state => state.IdleChannels == 3, + "the three channels that were opening to be open and idle"); + + // assert + Assert.Equal(2, state.Connections); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("pending-channel"))?.Messages); + } + + [Fact] + public async Task SendAsync_Should_Reconnect_When_PublisherChannelOpenTimesOut() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.ContinuationTimeout = TimeSpan.FromSeconds(30); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + t.DeclareQueue("channel-timeout"); + t.DispatchEndpoint("timeout-send").ToQueue("channel-timeout").Send(); + }) + .BuildTestBusAsync(); + var dispatcher = bus.Transport.Dispatcher; + var original = await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + dispatcher.ChannelOpenTimeout = TimeSpan.FromMilliseconds(250); + TimeoutException exception; + + // act + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + exception = await Assert.ThrowsAsync( + () => dispatcher.RentChannelAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken)); + } + + dispatcher.ChannelOpenTimeout = TestTimeouts.Signal; + await SendAsync(bus, new ProcessPayment { OrderId = "resumed", Amount = 1 }); + var current = await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + + // assert + Assert.Equal("The broker did not answer the request to open a channel.", exception.Message); + Assert.NotSame(original, current); + Assert.Equal(1, (await vhost.Broker.GetQueueAsync("channel-timeout"))?.Messages); + } + + [Fact] + public async Task RentChannelAsync_Should_PreserveOtherPublishes_When_ChannelAllocationFails() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 2; + await using var dispatcher = CreateDispatcher(vhost); + await dispatcher.StartAsync(TestContext.Current.CancellationToken); + var connection = await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + var external = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + var rented = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + + // act + try + { + await Assert.ThrowsAsync( + () => dispatcher.RentChannelAsync(TestContext.Current.CancellationToken).AsTask()); + } + finally + { + await external.DisposeAsync(); + dispatcher.Return(rented, reusable: true); + } + + // assert + Assert.True(connection.IsOpen); + await AssertCanPublishAsync(dispatcher, vhost); + } + + [Fact] + public async Task RentChannelAsync_Should_WaitForChannelClose_When_ChannelIsNotReusable() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 2; + await using var dispatcher = CreateDispatcher(vhost); + await dispatcher.StartAsync(TestContext.Current.CancellationToken); + var first = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + var second = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + + // act + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + dispatcher.Return(first, reusable: false); + using var deadline = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + await Assert.ThrowsAnyAsync( + () => dispatcher.RentChannelAsync(deadline.Token).AsTask()); + } + + var replacement = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + dispatcher.Return(replacement, reusable: true); + dispatcher.Return(second, reusable: true); + + // assert + Assert.NotSame(first.Channel, replacement.Channel); + await AssertCanPublishAsync(dispatcher, vhost); + } + + [Fact] + public async Task RentChannelAsync_Should_UseNewCapacity_When_ConnectionReplacedWhileWaiting() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 1; + await using var dispatcher = CreateDispatcher(vhost); + await dispatcher.StartAsync(TestContext.Current.CancellationToken); + var original = await dispatcher.Connection.GetConnectionAsync(TestContext.Current.CancellationToken); + var first = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + var waiting = dispatcher.RentChannelAsync(TestContext.Current.CancellationToken).AsTask(); + + // act + vhost.ConnectionSettings.RequestedChannelMax = 2; + await original.AbortAsync(TestContext.Current.CancellationToken); + var second = await waiting.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + var third = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + dispatcher.Return(first, reusable: true); + dispatcher.Return(second, reusable: true); + dispatcher.Return(third, reusable: true); + + // assert + Assert.NotSame(second.Channel, third.Channel); + Assert.NotSame(original, second.Connection); + await AssertCanPublishAsync(dispatcher, vhost); + } + + [Fact] + public async Task DisposeAsync_Should_FailWaitingRent_When_AllChannelsAreLeased() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + vhost.ConnectionSettings.RequestedChannelMax = 1; + await using var dispatcher = CreateDispatcher(vhost); + await dispatcher.StartAsync(TestContext.Current.CancellationToken); + var rented = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + var waiting = dispatcher.RentChannelAsync(TestContext.Current.CancellationToken).AsTask(); + + // act + await dispatcher.DisposeAsync(); + + // assert + await Assert.ThrowsAsync(() => waiting); + dispatcher.Return(rented, reusable: true); + await vhost.Broker.WaitForNoConnectionAsync(); + } + + [Fact] + public async Task DisposeAsync_Should_FailPendingRent_When_ChannelIsOpening() + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var dispatcher = CreateDispatcher(vhost); + await dispatcher.StartAsync(TestContext.Current.CancellationToken); + Task pending; + + // act + await using (await vhost.Faults.FreezeConnectionsAsync()) + { + pending = dispatcher.RentChannelAsync(TestContext.Current.CancellationToken).AsTask(); + await dispatcher.DisposeAsync().AsTask().WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + } + + // assert + await Assert.ThrowsAsync(() => pending); + await vhost.Broker.WaitForNoConnectionAsync(); + } + + private static RabbitMQDispatcher CreateDispatcher(VhostContext vhost) + => TestManagers.CreateDispatcher( + async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct), + async (connection, ct) => + { + await using var channel = await connection.CreateChannelAsync(cancellationToken: ct); + await channel.QueueDeclareAsync( + "orders", + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: ct); + }); + + private static async Task AssertCanPublishAsync(RabbitMQDispatcher dispatcher, VhostContext vhost) + { + await PublishToAsync(dispatcher, exchange: "", routingKey: "orders"); + + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var reader = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + var result = await reader.BasicGetAsync("orders", true, TestContext.Current.CancellationToken); + Assert.Equal("order", Encoding.UTF8.GetString(result!.Body.Span)); + } + + private static async Task PublishToAsync(RabbitMQDispatcher dispatcher, string exchange, string routingKey) + { + var lease = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); + var confirmed = false; + try + { + await lease.Channel.BasicPublishAsync( + exchange, + routingKey, + false, + new BasicProperties(), + "order"u8.ToArray(), + TestContext.Current.CancellationToken); + confirmed = true; + } + finally + { + dispatcher.Return(lease, reusable: confirmed); + } + } + + private static async Task SendAsync(TestBus bus, ProcessPayment message) + { + using var scope = bus.Provider.CreateScope(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TestTimeouts.Signal); + await scope.ServiceProvider.GetRequiredService().SendAsync(message, cts.Token); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQDispatcherTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQDispatcherTests.cs deleted file mode 100644 index 295c586f253..00000000000 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQDispatcherTests.cs +++ /dev/null @@ -1,266 +0,0 @@ -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using RabbitMQ.Client; - -namespace Mocha.Transport.RabbitMQ.Tests.Connection; - -public class RabbitMQDispatcherTests -{ - [Fact] - public async Task RentChannelAsync_Should_CreateNewChannel_When_PoolEmpty() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - await using var dispatcher = CreateDispatcher(connectionMock); - - // act - var rented = await dispatcher.RentChannelAsync(CancellationToken.None); - - // assert - Assert.Same(channelMock.Object, rented); - } - - [Fact] - public async Task RentChannelAsync_Should_ReuseChannel_When_PoolHasOpenChannel() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - await using var dispatcher = CreateDispatcher(connectionMock); - - var first = await dispatcher.RentChannelAsync(CancellationToken.None); - await dispatcher.ReturnChannelAsync(first); - - // act - var second = await dispatcher.RentChannelAsync(CancellationToken.None); - - // assert - Assert.Same(first, second); - } - - [Fact] - public async Task RentChannelAsync_Should_SkipClosedChannels_When_PoolHasClosedChannel() - { - // arrange - var closedChannelMock = CreateOpenChannel(); - var freshChannelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(closedChannelMock, freshChannelMock); - await using var dispatcher = CreateDispatcher(connectionMock); - - var rented = await dispatcher.RentChannelAsync(CancellationToken.None); - Assert.Same(closedChannelMock.Object, rented); - - // Simulate the channel closing while in the pool - closedChannelMock.SetupGet(c => c.IsOpen).Returns(false); - await dispatcher.ReturnChannelAsync(rented); - - // act - var next = await dispatcher.RentChannelAsync(CancellationToken.None); - - // assert - Assert.Same(freshChannelMock.Object, next); - } - - [Fact] - public async Task RentChannelAsync_Should_ThrowObjectDisposed_When_Disposed() - { - // arrange - var connectionMock = CreateOpenConnection(); - var dispatcher = CreateDispatcher(connectionMock); - await dispatcher.DisposeAsync(); - - // act & assert - await Assert.ThrowsAsync(() => - dispatcher.RentChannelAsync(CancellationToken.None).AsTask() - ); - } - - [Fact] - public async Task ReturnChannelAsync_Should_AddToPool_When_ChannelOpenAndPoolNotFull() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - await using var dispatcher = CreateDispatcher(connectionMock); - - var rented = await dispatcher.RentChannelAsync(CancellationToken.None); - - // act - await dispatcher.ReturnChannelAsync(rented); - - // assert - renting again should return the same pooled channel - var second = await dispatcher.RentChannelAsync(CancellationToken.None); - Assert.Same(channelMock.Object, second); - } - - [Fact] - public async Task ReturnChannelAsync_Should_DisposeChannel_When_PoolFull() - { - // arrange - var channelMocks = Enumerable.Range(0, 11).Select(_ => CreateOpenChannel()).ToArray(); - var connectionMock = CreateOpenConnection(channelMocks); - await using var dispatcher = CreateDispatcher(connectionMock); - - // Rent all 11 channels - var rented = new IChannel[11]; - for (var i = 0; i < 11; i++) - { - rented[i] = await dispatcher.RentChannelAsync(CancellationToken.None); - } - - // Return first 10 - fills the pool - for (var i = 0; i < 10; i++) - { - await dispatcher.ReturnChannelAsync(rented[i]); - } - - // act - return the 11th, which should be disposed (pool full) - await dispatcher.ReturnChannelAsync(rented[10]); - - // assert - the 11th channel should have been disposed - channelMocks[10].Verify(c => c.DisposeAsync(), Times.AtLeastOnce()); - } - - [Fact] - public async Task ReturnChannelAsync_Should_DisposeChannel_When_ChannelClosed() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - await using var dispatcher = CreateDispatcher(connectionMock); - - var rented = await dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); - channelMock.SetupGet(c => c.IsOpen).Returns(false); - - // act - await dispatcher.ReturnChannelAsync(rented); - - // assert - channelMock.Verify(c => c.DisposeAsync(), Times.AtLeastOnce()); - } - - [Fact] - public async Task ReturnChannelAsync_Should_DisposeChannel_When_DispatcherDisposed() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - var dispatcher = CreateDispatcher(connectionMock); - - var rented = await dispatcher.RentChannelAsync(CancellationToken.None); - await dispatcher.DisposeAsync(); - - // Clear previous calls from dispose - channelMock.Invocations.Clear(); - channelMock.SetupGet(c => c.IsOpen).Returns(true); - - // act - await dispatcher.ReturnChannelAsync(rented); - - // assert - channelMock.Verify(c => c.DisposeAsync(), Times.AtLeastOnce()); - } - - [Fact] - public async Task DisposeAsync_Should_ClearAllChannels_When_Called() - { - // arrange - var channelMocks = Enumerable.Range(0, 3).Select(_ => CreateOpenChannel()).ToArray(); - var connectionMock = CreateOpenConnection(channelMocks); - var dispatcher = CreateDispatcher(connectionMock); - - // Rent all channels first, then return them to populate pool - var rented = new IChannel[3]; - for (var i = 0; i < 3; i++) - { - rented[i] = await dispatcher.RentChannelAsync(CancellationToken.None); - } - - for (var i = 0; i < 3; i++) - { - await dispatcher.ReturnChannelAsync(rented[i]); - } - - // act - await dispatcher.DisposeAsync(); - - // assert - all channels should have been disposed - foreach (var ch in channelMocks) - { - ch.Verify(c => c.DisposeAsync(), Times.AtLeastOnce()); - } - } - - [Fact] - public async Task OnConnectionEstablished_Should_InvokeCallback_When_ConnectionCreated() - { - // arrange - var connectionMock = CreateOpenConnection(); - var callbackInvoked = false; - IConnection? receivedConnection = null; - - await using var dispatcher = new RabbitMQDispatcher( - NullLogger.Instance, - _ => new ValueTask(connectionMock.Object), - (conn, _) => - { - callbackInvoked = true; - receivedConnection = conn; - return Task.CompletedTask; - }); - - // act - await dispatcher.EnsureConnectedAsync(CancellationToken.None); - - // assert - Assert.True(callbackInvoked); - Assert.Same(connectionMock.Object, receivedConnection); - } - - private static Mock CreateOpenChannel() - { - var channelMock = new Mock(); - channelMock.SetupGet(c => c.IsOpen).Returns(true); - return channelMock; - } - - private static Mock CreateOpenConnection(params Mock[] channelsToReturn) - { - var connectionMock = new Mock(); - connectionMock.SetupGet(c => c.IsOpen).Returns(true); - connectionMock.SetupGet(c => c.ClientProvidedName).Returns("test-connection"); - - if (channelsToReturn.Length > 0) - { - var queue = new Queue>(channelsToReturn); - connectionMock - .Setup(c => c.CreateChannelAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(() => - { - if (queue.Count == 0) - { - return CreateOpenChannel().Object; - } - - return queue.Dequeue().Object; - }); - } - else - { - connectionMock - .Setup(c => c.CreateChannelAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(() => CreateOpenChannel().Object); - } - - return connectionMock; - } - - private static RabbitMQDispatcher CreateDispatcher(Mock connectionMock) - { - return new RabbitMQDispatcher( - NullLogger.Instance, - _ => new ValueTask(connectionMock.Object), - (_, _) => Task.CompletedTask); - } -} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQFailureTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQFailureTests.cs new file mode 100644 index 00000000000..a364c2f4196 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQFailureTests.cs @@ -0,0 +1,162 @@ +using System.Net.Sockets; +using System.Security.Authentication; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ.Tests.Connection; + +public sealed class RabbitMQFailureTests +{ + [Theory] + [MemberData(nameof(TransientFailures))] + public void Classify_Should_ReturnTransient_When_LaterAttemptCanSucceed(string shape, Exception exception) + { + // act + var kind = RabbitMQFailure.Classify(exception); + + // assert + Assert.True(kind == RabbitMQFailureKind.Transient, shape); + } + + [Theory] + [MemberData(nameof(PermanentFailures))] + public void Classify_Should_ReturnPermanent_When_EveryAttemptFailsTheSameWay(string shape, Exception exception) + { + // act + var kind = RabbitMQFailure.Classify(exception); + + // assert + Assert.True(kind == RabbitMQFailureKind.Permanent, shape); + } + + [Fact] + public void GetLogLevel_Should_ReturnWarning_When_FailureIsTransient() + { + // arrange + var exception = new AlreadyClosedException(Shutdown(ShutdownInitiator.Library, 0, "End of stream")); + + // act + var level = RabbitMQFailure.GetLogLevel(exception); + + // assert + Assert.Equal(LogLevel.Warning, level); + } + + [Fact] + public void GetLogLevel_Should_ReturnError_When_FailureIsPermanent() + { + // arrange + var exception = new BrokerUnreachableException(new AuthenticationFailureException("ACCESS_REFUSED")); + + // act + var level = RabbitMQFailure.GetLogLevel(exception); + + // assert + Assert.Equal(LogLevel.Error, level); + } + + // Every entry is built to an exception chain that RabbitMQ.Client 7.2.2 produced against a broker. + public static TheoryData TransientFailures() + => new() + { + { + "the broker forced the connection closed (320)", + new AlreadyClosedException(Shutdown(ShutdownInitiator.Peer, Constants.ConnectionForced, "CONNECTION_FORCED")) + }, + { + "the broker reported an internal error (541)", + new AlreadyClosedException(Shutdown(ShutdownInitiator.Peer, Constants.InternalError, "INTERNAL_ERROR")) + }, + { + "the socket ended or a heartbeat was missed", + new AlreadyClosedException(Shutdown(ShutdownInitiator.Library, 0, "End of stream")) + }, + { + "the application closed the connection", + new AlreadyClosedException(Shutdown(ShutdownInitiator.Application, 200, "Goodbye")) + }, + { "an operation was interrupted without a reason", new OperationInterruptedException() }, + { + "no node could be reached", + new BrokerUnreachableException( + new AggregateException( + new ConnectFailureException("Connection failed", new SocketException(111)))) + }, + { + "one node refused TLS and another one could not be reached", + new BrokerUnreachableException( + new AggregateException( + new AuthenticationException("The remote certificate is invalid"), + new ConnectFailureException("Connection failed", new SocketException(111)))) + }, + { + "the connection dropped during the handshake", + new BrokerUnreachableException( + new PossibleAuthenticationFailureException( + "Possibly caused by authentication failure", + new OperationInterruptedException(Shutdown(ShutdownInitiator.Library, 0, "End of stream")))) + }, + { "an operation timed out", new TimeoutException("Connecting to RabbitMQ did not complete") }, + { "the client timed out a request", new TaskCanceledException() }, + { "a channel or connection was disposed", new ObjectDisposedException("channel") }, + { "the frame writer of the client closed", new ChannelClosedException() }, + { "the socket failed", new IOException("Unable to write data to the transport connection") }, + { "the socket was reset", new SocketException(104) } + }; + + public static TheoryData PermanentFailures() + => new() + { + { + "the credentials were refused", + new BrokerUnreachableException(new AuthenticationFailureException("ACCESS_REFUSED")) + }, + { + "the virtual host was refused (530)", + new OperationInterruptedException(Shutdown(ShutdownInitiator.Peer, Constants.NotAllowed, "NOT_ALLOWED")) + }, + { + "a declaration did not match (406)", + new AlreadyClosedException( + Shutdown(ShutdownInitiator.Peer, Constants.PreconditionFailed, "PRECONDITION_FAILED")) + }, + { + "a queue or exchange does not exist (404)", + new OperationInterruptedException(Shutdown(ShutdownInitiator.Peer, Constants.NotFound, "NOT_FOUND")) + }, + { + "a queue is locked by another connection (405)", + new OperationInterruptedException( + Shutdown(ShutdownInitiator.Peer, Constants.ResourceLocked, "RESOURCE_LOCKED")) + }, + { + "access to a resource was refused (403)", + new OperationInterruptedException( + Shutdown(ShutdownInitiator.Peer, Constants.AccessRefused, "ACCESS_REFUSED")) + }, + { + "a frame was larger than the broker accepts (501)", + new AlreadyClosedException(Shutdown(ShutdownInitiator.Peer, Constants.FrameError, "FRAME_ERROR")) + }, + { + "every node refused TLS", + new BrokerUnreachableException( + new AggregateException(new AuthenticationException("The remote certificate is invalid"))) + }, + { "the broker rejected the message", new PublishException(1, isReturn: false) }, + { + "the broker returned the message", + new InvalidOperationException( + "The broker returned the message because no queue named 'orders' exists.", + new PublishException(1, isReturn: true)) + }, + { "the connection has no free channel", new ChannelAllocationException() }, + { "the failure is unknown", new InvalidOperationException("Something else") } + }; + + private static ShutdownEventArgs Shutdown(ShutdownInitiator initiator, ushort replyCode, string replyText) + => new(initiator, replyCode, replyText); +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQLifecycleTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQLifecycleTests.cs new file mode 100644 index 00000000000..ee114eec312 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQLifecycleTests.cs @@ -0,0 +1,184 @@ +using System.Diagnostics; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Connection; + +public sealed class RabbitMQLifecycleTests +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(5); + + [Fact] + public async Task DisposeAsync_Should_FailWaitingRents_When_ConnectionAttemptIsPending() + { + // arrange + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatcher = TestManagers.CreateDispatcher( + async ct => + { + entered.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, ct); + } + finally + { + await release.Task; + } + + throw new UnreachableException(); + }, + static (_, _) => Task.CompletedTask); + var first = dispatcher.RentChannelAsync(CancellationToken.None).AsTask(); + await entered.Task.WaitAsync(s_timeout, TestContext.Current.CancellationToken); + var second = dispatcher.RentChannelAsync(CancellationToken.None).AsTask(); + + // act + var disposing = dispatcher.DisposeAsync().AsTask(); + bool disposeWaitedForAttempt; + try + { + await Assert.ThrowsAsync( + () => second.WaitAsync(s_timeout, TestContext.Current.CancellationToken)); + disposeWaitedForAttempt = !disposing.IsCompleted; + } + finally + { + release.TrySetResult(); + await disposing.WaitAsync(s_timeout, TestContext.Current.CancellationToken); + } + + // assert + Assert.True(disposeWaitedForAttempt); + await Assert.ThrowsAsync(() => first); + Assert.False(dispatcher.IsConnected); + } + + [Fact] + public async Task RentChannelAsync_Should_ThrowObjectDisposed_When_Disposed() + { + // arrange + var dispatcher = TestManagers.CreateDispatcher(NeverConnectAsync, static (_, _) => Task.CompletedTask); + await dispatcher.DisposeAsync(); + + // act + var exception = await Record.ExceptionAsync( + () => dispatcher.RentChannelAsync(TestContext.Current.CancellationToken).AsTask()); + + // assert + Assert.IsType(exception); + } + + [Fact] + public async Task RegisterConsumerAsync_Should_RejectZeroConcurrency_When_NoHandlerCouldRun() + { + // arrange + await using var manager = TestManagers.CreateConsumerManager(NeverConnectAsync); + + // act + var exception = await Assert.ThrowsAsync(() => manager.RegisterConsumerAsync( + "orders", + static (_, _, _) => default, + 1, + 0, + TestContext.Current.CancellationToken)); + + // assert + Assert.Equal("maxConcurrency", exception.ParamName); + } + + [Fact] + public async Task RegisterConsumerAsync_Should_RejectRegistration_When_ManagerDisposed() + { + // arrange + var manager = TestManagers.CreateConsumerManager(NeverConnectAsync); + await manager.DisposeAsync(); + + // act + var exception = await Record.ExceptionAsync(() => manager.RegisterConsumerAsync( + "orders", + static (_, _, _) => default, + 1, + 1, + TestContext.Current.CancellationToken)); + + // assert + Assert.IsType(exception); + Assert.Equal("The consumer connection is unavailable.", manager.HealthFailureReason); + } + + [Fact] + public async Task RegisterConsumerAsync_Should_ReturnDetachedConsumer_When_NoConnectionIsReady() + { + // arrange + await using var manager = TestManagers.CreateConsumerManager(NeverConnectAsync); + + // act + var consumer = await manager.RegisterConsumerAsync( + "orders", + static (_, _, _) => default, + 1, + 1, + TestContext.Current.CancellationToken); + + // assert + Assert.False(consumer.IsAttached); + Assert.Equal("The consumer is not attached.", consumer.HealthFailureReason); + } + + [Fact] + public async Task WaitUntilAttachedAsync_Should_EndEveryWait_When_ConsumerStops() + { + // arrange + await using var manager = TestManagers.CreateConsumerManager(NeverConnectAsync); + var consumer = await manager.RegisterConsumerAsync( + "orders", + static (_, _, _) => default, + 1, + 1, + TestContext.Current.CancellationToken); + using var cancelled = new CancellationTokenSource(); + var first = consumer.WaitUntilAttachedAsync(cancelled.Token).AsTask(); + var second = consumer.WaitUntilAttachedAsync(CancellationToken.None).AsTask(); + + // act + await cancelled.CancelAsync(); + await Assert.ThrowsAnyAsync(() => first); + var otherWaiterPending = !second.IsCompleted; + await consumer.StopAsync(TestContext.Current.CancellationToken); + + // assert + Assert.True(otherWaiterPending); + var exception = await Assert.ThrowsAsync( + () => second.WaitAsync(s_timeout, TestContext.Current.CancellationToken)); + Assert.Equal("The consumer for queue 'orders' is stopped.", exception.Message); + } + + [Fact] + public async Task DisposeAsync_Should_BeIdempotent_When_ConsumerWasStopped() + { + // arrange + await using var manager = TestManagers.CreateConsumerManager(NeverConnectAsync); + var consumer = await manager.RegisterConsumerAsync( + "orders", + static (_, _, _) => default, + 1, + 1, + TestContext.Current.CancellationToken); + await consumer.StopAsync(TestContext.Current.CancellationToken); + + // act + var exception = await Record.ExceptionAsync(() => consumer.DisposeAsync().AsTask()); + + // assert + Assert.Null(exception); + Assert.Equal("The consumer is stopped.", consumer.HealthFailureReason); + } + + private static async ValueTask NeverConnectAsync(CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + throw new UnreachableException(); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQQuorumRecoveryTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQQuorumRecoveryTests.cs new file mode 100644 index 00000000000..daddfca2c93 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Connection/RabbitMQQuorumRecoveryTests.cs @@ -0,0 +1,225 @@ +using System.Collections.Concurrent; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Connection; + +[Collection(RabbitMQCollections.ConsumerRecovery)] +public sealed class RabbitMQQuorumRecoveryTests(RabbitMQFixture fixture) +{ + // The header in which a quorum queue counts the earlier deliveries of a message. +#if RABBITMQ_3 + private const string EarlierDeliveriesHeader = "x-delivery-count"; +#else + private const string EarlierDeliveriesHeader = "x-acquired-count"; +#endif + + private const string BlockedReason = + "Queue 'orders': The consumer waits for a handler of a lost channel to return."; + + [Fact] + public async Task Consumer_Should_RetainUnprocessedMessages_When_LostHandlerUsesAllCapacity() + { + // arrange + await using var timeout = await fixture.Faults.SetConsumerTimeoutAsync(TimeSpan.FromSeconds(1)); + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + "orders", + durable: true, + exclusive: false, + autoDelete: false, + arguments: new Dictionary { ["x-queue-type"] = "quorum", ["x-delivery-limit"] = 2 }, + cancellationToken: TestContext.Current.CancellationToken); + await using var manager = TestManagers.CreateConsumerManager( + async ct => await vhost.ConnectionFactory.CreateConnectionAsync(ct)); + await manager.StartAsync(TestContext.Current.CancellationToken); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancelled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var invocations = 0; + var delivered = new ConcurrentQueue<(string Body, long EarlierDeliveries)>(); + var consumer = await manager.RegisterConsumerAsync( + "orders", + async (consumerChannel, delivery, ct) => + { + if (Interlocked.Increment(ref invocations) == 1) + { + await using var registration = ct.Register(() => cancelled.TrySetResult()); + started.TrySetResult(); + await release.Task; + ct.ThrowIfCancellationRequested(); + } + + var earlierDeliveries = delivery.BasicProperties.Headers is { } headers + && headers.TryGetValue(EarlierDeliveriesHeader, out var count) + ? Convert.ToInt64(count) + : 0L; + delivered.Enqueue((Encoding.UTF8.GetString(delivery.Body.Span), earlierDeliveries)); + await consumerChannel.BasicAckAsync(delivery.DeliveryTag, false, ct); + }, + 2, + 1, + TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "orders", "first"u8.ToArray(), TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "orders", "second"u8.ToArray(), TestContext.Current.CancellationToken); + await started.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + + // act + try + { + await cancelled.Task.WaitAsync(TestTimeouts.Signal, TestContext.Current.CancellationToken); + await vhost.Broker.WaitForNoConsumerAsync("orders"); + + // A consumer that attached would take both messages again without a free handler, until the broker + // took them back. + await Wait.UntilAsync(() => manager.HealthFailureReason == BlockedReason); + var queueWhileBlocked = await vhost.Broker.GetQueueAsync("orders"); + var whileBlocked = (queueWhileBlocked?.Consumers, queueWhileBlocked?.Messages, invocations); + release.TrySetResult(); + await Wait.UntilAsync(() => delivered.Count == 2); + await vhost.Broker.WaitForEmptyQueueAsync("orders"); + + // assert + Assert.Equal((0, 2, 1), whileBlocked); + + // each message was delivered once before, to the consumer that the broker took away + Assert.Equal([("first", 1L), ("second", 1L)], delivered.Order()); + Assert.Null(manager.HealthFailureReason); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(-1, 25, false)] + [InlineData(-1, 25, true)] + [InlineData(3, 4, false)] + [InlineData(3, 4, true)] +#if RABBITMQ_3 + [InlineData(null, 25, false)] + [InlineData(null, 25, true)] +#else + [InlineData(null, 21, false)] + [InlineData(null, 21, true)] +#endif + public async Task QuorumQueue_Should_RespectDeliveryLimit_When_ChannelsCloseRepeatedly( + int? limit, + int expectedDeliveries, + bool useBusDefault) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + var queue = t.DeclareQueue("orders"); + if (useBusDefault) + { + t.ConfigureDefaults(d => + { + d.Queue.QueueType = RabbitMQQueueType.Quorum; + if (limit is { } deliveryLimit) + { + d.Queue.Arguments["x-delivery-limit"] = deliveryLimit; + } + }); + } + else + { + queue.QueueType(RabbitMQQueueType.Quorum); + if (limit is { } deliveryLimit) + { + queue.MaxDeliveryLimit(deliveryLimit); + } + } + }) + .BuildTestBusAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var publisher = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await publisher.BasicPublishAsync("", "orders", "order"u8.ToArray(), TestContext.Current.CancellationToken); + + // act + var deliveries = new List(); + for (var i = 0; i < 25; i++) + { + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + BasicGetResult? delivery = null; + var available = await Condition.WaitAsync( + async () => + { + delivery = await channel.BasicGetAsync("orders", false, TestContext.Current.CancellationToken); + return delivery is not null; + }, + TimeSpan.FromSeconds(2)); + if (!available) + { + break; + } + + deliveries.Add(Encoding.UTF8.GetString(delivery!.Body.Span)); + if (i == 24) + { + await channel.BasicAckAsync(delivery.DeliveryTag, false, TestContext.Current.CancellationToken); + } + } + + var settled = await Condition.WaitAsync(async () => (await vhost.Broker.GetQueueAsync("orders"))?.Messages == 0, TestTimeouts.Signal); + + // assert + Assert.Equal(Enumerable.Repeat("order", expectedDeliveries), deliveries); + Assert.True(settled); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StartAsync_Should_PreserveMessages_When_ExistingQuorumQueueHasBrokerDeliveryLimit( + bool useVirtualHostDefault) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + if (useVirtualHostDefault) + { + await fixture.InvokeCommandAsync( + ["rabbitmqctl", "update_vhost_metadata", vhost.VhostName, "--default-queue-type", "quorum"]); + } + + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync(cancellationToken: TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + "orders", + durable: true, + exclusive: false, + autoDelete: false, + arguments: new Dictionary { ["x-queue-type"] = "quorum" }, + cancellationToken: TestContext.Current.CancellationToken); + await channel.BasicPublishAsync("", "orders", "retained"u8.ToArray(), TestContext.Current.CancellationToken); + var builder = new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + var queue = t.DeclareQueue("orders"); + if (!useVirtualHostDefault) + { + queue.QueueType(RabbitMQQueueType.Quorum); + } + }); + + // act + await using var bus = await builder.BuildTestBusAsync(); + var delivery = await channel.BasicGetAsync("orders", true, TestContext.Current.CancellationToken); + + // assert + Assert.Equal("retained", Encoding.UTF8.GetString(delivery!.Body.Span)); + Assert.Equal(0u, await channel.MessageCountAsync("orders", TestContext.Current.CancellationToken)); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQDescriptorTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQDescriptorTests.cs index 2cbb93a6687..a076d692f36 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQDescriptorTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQDescriptorTests.cs @@ -6,6 +6,98 @@ namespace Mocha.Transport.RabbitMQ.Tests.Descriptors; public class RabbitMQDescriptorTests { + [Theory] + [InlineData(-1L)] + [InlineData(4_294_967_295L)] + public void ConnectTimeout_Should_Throw_When_TimeoutOutsideSupportedRange(long milliseconds) + { + // arrange + var timeout = TimeSpan.FromMilliseconds(milliseconds); + + // act + var exception = Record.Exception(() => CreateRuntime(t => t.ConnectTimeout(timeout))); + + // assert + var error = Assert.IsType(exception); + Assert.Equal("timeout", error.ParamName); + } + + [Theory] + [InlineData(0L)] + [InlineData(30_000L)] + [InlineData(4_294_967_294L)] + public void ConnectTimeout_Should_ConfigureBothConnections_When_TimeoutWithinSupportedRange(long milliseconds) + { + // arrange + var timeout = TimeSpan.FromMilliseconds(milliseconds); + + // act + var runtime = CreateRuntime(t => t.ConnectTimeout(timeout)); + var transport = runtime.Transports.OfType().Single(); + + // assert + Assert.Equal(timeout, transport.Dispatcher.Connection.ConnectTimeout); + Assert.Equal(timeout, transport.ConsumerManager.Connection.ConnectTimeout); + } + + [Fact] + public void ConnectTimeout_Should_BeTwoMinutes_When_NotConfigured() + { + // act + var runtime = CreateRuntime(_ => { }); + var transport = runtime.Transports.OfType().Single(); + + // assert + Assert.Equal(TimeSpan.FromMinutes(2), transport.Dispatcher.Connection.ConnectTimeout); + Assert.Equal(TimeSpan.FromMinutes(2), transport.ConsumerManager.Connection.ConnectTimeout); + } + + [Theory] + [InlineData(0L)] + [InlineData(-2L)] + [InlineData(4_294_967_295L)] + public void PublishTimeout_Should_Throw_When_TimeoutOutsideSupportedRange(long milliseconds) + { + // arrange + var timeout = TimeSpan.FromMilliseconds(milliseconds); + + // act + var exception = Record.Exception(() => CreateRuntime(t => t.PublishTimeout(timeout))); + + // assert + var error = Assert.IsType(exception); + Assert.Equal("timeout", error.ParamName); + } + + [Theory] + [InlineData(1L)] + [InlineData(30_000L)] + [InlineData(4_294_967_294L)] + [InlineData(-1L)] + public void PublishTimeout_Should_ConfigureDispatcher_When_TimeoutWithinSupportedRangeOrInfinite(long milliseconds) + { + // arrange + var timeout = TimeSpan.FromMilliseconds(milliseconds); + + // act + var runtime = CreateRuntime(t => t.PublishTimeout(timeout)); + var transport = runtime.Transports.OfType().Single(); + + // assert + Assert.Equal(timeout, transport.Dispatcher.PublishTimeout); + } + + [Fact] + public void PublishTimeout_Should_BeTwoMinutes_When_NotConfigured() + { + // act + var runtime = CreateRuntime(_ => { }); + var transport = runtime.Transports.OfType().Single(); + + // assert + Assert.Equal(TimeSpan.FromMinutes(2), transport.Dispatcher.PublishTimeout); + } + [Fact] public void Transport_Should_UseCustomSchema_When_SchemaConfigured() { @@ -187,7 +279,7 @@ public void ReceiveEndpoint_Should_PreserveFaultQueueAutoProvisionFalse_When_Fau t.Queue("q").AutoProvision(true).Handler().FaultEndpoint(new Uri("queue:q_error")); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // assert var errorQueue = topology.Queues.Single(q => q.Name == "q_error"); @@ -205,7 +297,7 @@ public void ReceiveEndpoint_Should_PreserveFaultQueueAutoProvisionNull_When_Faul t.Queue("q").AutoProvision(true).Handler().FaultEndpoint(new Uri("queue:q_error")); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // assert var errorQueue = topology.Queues.Single(q => q.Name == "q_error"); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQQueueLoweringTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQQueueLoweringTests.cs index cf669a8b8d1..7600db9c0e7 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQQueueLoweringTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQQueueLoweringTests.cs @@ -70,7 +70,7 @@ public void Describe_Should_ApplyQuorumArgument_When_QueueIsQuorum() t.Queue("audit").Quorum(); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name == "audit"); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQUnifiedQueueTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQUnifiedQueueTests.cs index b0f31385db7..5dda418c1cf 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQUnifiedQueueTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Descriptors/RabbitMQUnifiedQueueTests.cs @@ -152,7 +152,7 @@ public void QueueEndpointDeclareQueue_Should_ConvergeToOneEntity_When_SameName() t.DeclareQueue("orders").AutoProvision(false); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var description = transport.Describe(); @@ -257,7 +257,7 @@ public void Queue_Should_ApplyQueueShapeArguments_When_WithArgumentCalled() t.Queue("audit").WithArgument("x-message-ttl", 30_000); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.SingleOrDefault(q => q.Name == "audit"); @@ -310,7 +310,7 @@ public void Queue_Should_MaterializeDurableAutoDeleteQueueWithExpiry_When_Tempor t.Queue("temp-audit").Temporary(); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name == "temp-audit"); @@ -334,7 +334,7 @@ public void Queue_Should_UseConfiguredExpiry_When_TemporaryCalledWithExpiry() t.Queue("temp-audit").Temporary(TimeSpan.FromMinutes(5)); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name == "temp-audit"); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/BrokerFaults.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/BrokerFaults.cs new file mode 100644 index 00000000000..870dcc2bdd2 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/BrokerFaults.cs @@ -0,0 +1,312 @@ +using System.Net; +using System.Text.RegularExpressions; + +namespace Mocha.Transport.RabbitMQ.Tests.Helpers; + +/// +/// Faults that affect the whole broker, and so every test of the collection. +/// +public sealed class BrokerFaults +{ + private readonly RabbitMQFixture _fixture; + private readonly ManagementRabbitMQResource _resource; + + internal BrokerFaults(RabbitMQFixture fixture, ManagementRabbitMQResource resource) + { + _fixture = fixture; + _resource = resource; + } + + /// + /// Makes the broker close consumer channels that hold a delivery longer than , + /// for channels opened until the returned handle is disposed. + /// + public async Task SetConsumerTimeoutAsync(TimeSpan timeout) + { + // RabbitMQ 3.11 has no consumer-timeout policy, and it checks the timeout on the channel tick + await _fixture.EvalAsync( + $"application:set_env(rabbit, consumer_timeout, {(int)timeout.TotalMilliseconds}), " + + "application:set_env(rabbit, channel_tick_interval, 250)."); + + return new AsyncDisposable(() => _fixture.EvalAsync( + "application:set_env(rabbit, consumer_timeout, 1800000), " + + "application:set_env(rabbit, channel_tick_interval, 60000).")); + } + + /// + /// Raises a memory alarm so the broker blocks every publishing connection until the returned handle is disposed. + /// + public async Task RaiseMemoryAlarmAsync() + { + await _fixture.InvokeCommandAsync(["rabbitmqctl", "set_vm_memory_high_watermark", "0"]); + + return new AsyncDisposable(() => _fixture.InvokeCommandAsync( + ["rabbitmqctl", "set_vm_memory_high_watermark", "0.4"])); + } + + /// + /// Stops the broker application, which closes every connection and refuses new ones until the returned + /// handle is disposed. + /// + public async Task StopBrokerAsync() + { + await _fixture.InvokeCommandAsync(["rabbitmqctl", "stop_app"]); + + return new AsyncDisposable(async () => + { + await _fixture.InvokeCommandAsync(["rabbitmqctl", "start_app"]); + await _fixture.InvokeCommandAsync(["rabbitmqctl", "await_startup"]); + }); + } + + /// + /// Kills the broker process, and starts it again when the returned handle is disposed. + /// + public Task KillBrokerAsync() => _resource.KillAsync(); + + /// + /// Lowers the largest message the broker accepts until the returned handle is disposed. + /// + public async Task SetMaxMessageSizeAsync(int bytes) + { + // RabbitMQ 3.13 reads the limit from the application environment when a channel opens, so the bus must + // start after this call. RabbitMQ 4.3 reads it from a persistent term that is set at boot. + await _fixture.EvalAsync( + $$""" + persistent_term:put( + mocha_max_message_size, + {application:get_env(rabbit, max_message_size), persistent_term:get(max_message_size, undefined)}), + application:set_env(rabbit, max_message_size, {{bytes}}), + persistent_term:put(max_message_size, {{bytes}}). + """); + + return new AsyncDisposable(() => _fixture.EvalAsync( + """ + {Env, Term} = persistent_term:get(mocha_max_message_size), + case Env of + {ok, Value} -> application:set_env(rabbit, max_message_size, Value); + undefined -> application:unset_env(rabbit, max_message_size) + end, + case Term of + undefined -> persistent_term:erase(max_message_size); + _ -> persistent_term:put(max_message_size, Term) + end, + persistent_term:erase(mocha_max_message_size). + """)); + } + + /// + /// Puts the node into maintenance mode, which closes every client connection and refuses new ones until the + /// returned handle is disposed. + /// + public async Task DrainNodeAsync() + { + try + { + await _fixture.InvokeCommandAsync(["rabbitmq-upgrade", "drain"]); + } + catch (InvalidOperationException) + { + // The command fails when a connection exits while the node closes it, and the node is in maintenance + // mode anyway, so the returned handle must still revive it. + } + + return new AsyncDisposable(() => _fixture.InvokeCommandAsync(["rabbitmq-upgrade", "revive"])); + } +} + +/// +/// Faults that affect one vhost, and so only the test that owns it. +/// +public sealed class VhostFaults +{ + private readonly RabbitMQFixture _fixture; + private readonly string _vhostName; + private readonly BrokerView _broker; + + internal VhostFaults(RabbitMQFixture fixture, string vhostName, BrokerView broker) + { + _fixture = fixture; + _vhostName = vhostName; + _broker = broker; + } + + /// + /// Force-closes every connection to the vhost, and returns the broker's identifiers of the closed connections + /// once the broker no longer lists them. Throws when the vhost has no connection. + /// + public async Task> CloseConnectionsAsync() + { + var connections = (await _broker.GetConnectionsAsync()).Select(connection => connection.Pid).ToArray(); + if (connections.Length == 0) + { + throw new InvalidOperationException($"Vhost '{_vhostName}' has no connection to close."); + } + + await _fixture.InvokeCommandAsync(["rabbitmqctl", "close_all_connections", "-p", _vhostName, "test"]); + await WaitForClosedAsync(connections); + + return connections; + } + + /// + /// Force-closes the connection that consumes , leaves the others open, and returns + /// the broker's identifier of the closed connection once the broker no longer lists it. Throws when the queue + /// does not have exactly one consumer. + /// + public async Task> CloseConsumerConnectionAsync(string queueName) + { + var consumer = await _broker.GetConsumerAsync(queueName); + var channels = await _fixture.ListAsync("list_channels", "pid", "connection"); + var connection = channels.Single(row => row["pid"] == consumer.ChannelPid)["connection"]; + + await _fixture.InvokeCommandAsync(["rabbitmqctl", "close_connection", connection, "test"]); + await WaitForClosedAsync([connection]); + + return [connection]; + } + + /// + /// Deletes a queue the way an operator would, together with its bindings and messages. + /// + public async Task DeleteQueueAsync(string queueName) + { + await _fixture.DeleteAsync($"queues/{Uri.EscapeDataString(_vhostName)}/{Uri.EscapeDataString(queueName)}"); + } + + /// + /// Freezes the broker side of every connection to the vhost, so that the broker neither reads nor sends + /// frames, including heartbeats, until the returned handle is disposed. + /// + public async Task FreezeConnectionsAsync() + { + // A suspension ends when the suspending process exits, so a process that waits for "resume" holds it. + // Connections of other tests can close during the scan, so lookups on them are allowed to fail. + var suspender = $"mocha_suspender_{_vhostName}"; + await _fixture.EvalAsync( + $$""" + Tree = fun Walk(Sup) -> + case catch supervisor:which_children(Sup) of + Children when is_list(Children) -> + lists:append([case Type of supervisor -> Walk(Pid); _ -> [{Id, Pid}] end + || {Id, Pid, Type, _} <- Children, is_pid(Pid)]); + _ -> [] + end + end, + Readers = [R || R <- rabbit_networking:connections(), + (catch rabbit_reader:info(R, [vhost])) =:= [{vhost, <<"{{_vhostName}}">>}]], + Sups = [Sup || R <- Readers, + {dictionary, Dictionary} <- [process_info(R, dictionary)], + [Sup | _] <- [proplists:get_value('$ancestors', Dictionary, [])]], + Pids = [Pid || Sup <- Sups, + {Id, Pid} <- Tree(Sup), + lists:member(Id, [reader, writer, heartbeat_sender, heartbeat_receiver])], + Holder = spawn(fun() -> [erlang:suspend_process(P) || P <- Pids], receive resume -> ok end end), + register('{{suspender}}', Holder), + length(Pids). + """); + + return new AsyncDisposable(() => _fixture.EvalAsync($"'{suspender}' ! resume, ok.")); + } + + /// + /// Removes the guest user's access to the vhost, so that new connections are refused, until the returned + /// handle is disposed. + /// + public async Task RevokeAccessAsync() + { + await _fixture.DeleteAsync($"permissions/{Uri.EscapeDataString(_vhostName)}/guest"); + + return new AsyncDisposable(() => SetPermissionsAsync("guest", ".*", ".*", ".*")); + } + + /// + /// Limits the number of connections the vhost accepts, until the returned handle is disposed. The broker + /// refuses a connection above the limit with 530 NOT_ALLOWED after the handshake. + /// + public async Task LimitConnectionsAsync(int maxConnections) + { + var path = $"vhost-limits/{Uri.EscapeDataString(_vhostName)}/max-connections"; + await _fixture.PutAsync(path, new { value = maxConnections }); + + return new AsyncDisposable(() => _fixture.DeleteAsync(path)); + } + + /// + /// Removes the guest user's permission to declare or delete queues, exchanges and bindings in the vhost, + /// until the returned handle is disposed. + /// + public async Task DenyDeclaresAsync() + { + await SetPermissionsAsync("guest", "^$", ".*", ".*"); + + return new AsyncDisposable(() => SetPermissionsAsync("guest", ".*", ".*", ".*")); + } + + /// + /// Creates a user with full permissions on the vhost, and deletes it when the returned handle is disposed. + /// + public async Task AddUserAsync(string userName, string password) + { + await _fixture.PutAsync($"users/{Uri.EscapeDataString(userName)}", new { password, tags = "" }); + await SetPermissionsAsync(userName, ".*", ".*", ".*"); + + return new AsyncDisposable(async () => + { + try + { + await DeleteUserAsync(userName); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // The test deleted the user already. + } + }); + } + + /// + /// Deletes a user the way an operator would, which closes every connection of the user. + /// + public Task DeleteUserAsync(string userName) + => _fixture.DeleteAsync($"users/{Uri.EscapeDataString(userName)}"); + + /// + /// Changes the password of a user. Open connections of the user stay open. + /// + public Task ChangePasswordAsync(string userName, string password) + => _fixture.PutAsync($"users/{Uri.EscapeDataString(userName)}", new { password, tags = "" }); + + /// + /// Sets the configure, write and read permissions of a user on the vhost, each a regular expression of + /// resource names. + /// + public Task SetPermissionsAsync(string userName, string configure, string write, string read) + => _fixture.PutAsync( + $"permissions/{Uri.EscapeDataString(_vhostName)}/{Uri.EscapeDataString(userName)}", + new { configure, write, read }); + + public Task SetReadPermissionAsync(string pattern) + => SetPermissionsAsync("guest", ".*", ".*", pattern); + + public async Task AllowReadingQueuesOnlyAsync() + { + var queues = await _broker.GetQueuesAsync(); + var pattern = $"^({string.Join("|", queues.Keys.Select(Regex.Escape))})$"; + await SetReadPermissionAsync(pattern); + } + + private Task WaitForClosedAsync(string[] connections) + => Wait.UntilAsync( + async () => + { + var open = await _fixture.ListAsync("list_connections", "pid"); + return open.Select(row => row["pid"]).Intersect(connections).ToArray(); + }, + remaining => remaining.Length == 0, + $"the broker to close the connections [{string.Join(", ", connections)}] to vhost '{_vhostName}'"); +} + +internal sealed class AsyncDisposable(Func dispose) : IAsyncDisposable +{ + public async ValueTask DisposeAsync() => await dispose(); +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/BrokerView.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/BrokerView.cs new file mode 100644 index 00000000000..f39a2af9869 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/BrokerView.cs @@ -0,0 +1,328 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace Mocha.Transport.RabbitMQ.Tests.Helpers; + +/// +/// Reads the state the broker holds for one vhost, and waits for it to change. +/// +public sealed partial class BrokerView +{ + private readonly RabbitMQFixture _fixture; + private readonly string _vhostName; + + internal BrokerView(RabbitMQFixture fixture, string vhostName) + { + _fixture = fixture; + _vhostName = vhostName; + } + + /// + /// Gets the queues of the vhost by name. + /// + public async Task> GetQueuesAsync() + { + var queues = await _fixture.ListAsync( + "list_queues", "-p", _vhostName, "name", "consumers", "messages_ready", "messages_unacknowledged"); + + return queues.ToDictionary( + row => row["name"], + row => new QueueInfo( + ParseCount(row["consumers"]), + ParseCount(row["messages_ready"]), + ParseCount(row["messages_unacknowledged"]))); + } + + /// + /// Gets the queue , or when the queue does not exist. + /// + public async Task GetQueueAsync(string queueName) + => (await GetQueuesAsync()).GetValueOrDefault(queueName); + + /// + /// Gets the consumers on all queues of the vhost. + /// + public async Task> GetConsumersAsync() + { + var consumers = await _fixture.ListAsync( + "list_consumers", "-p", _vhostName, "queue_name", "consumer_tag", "channel_pid", "prefetch_count"); + + return consumers + .Select(row => new ConsumerInfo( + row["queue_name"], + row["consumer_tag"], + row["channel_pid"], + ParseCount(row["prefetch_count"]))) + .ToArray(); + } + + /// + /// Gets the consumer of the queue , and throws when the queue does not have + /// exactly one consumer. + /// + public async Task GetConsumerAsync(string queueName) + { + var consumers = await GetConsumersOfAsync(queueName); + + return consumers is [var consumer] + ? consumer + : throw new InvalidOperationException( + $"Expected queue '{queueName}' to have one consumer, but observed [{string.Join(", ", consumers)}]."); + } + + /// + /// Gets the connections to the vhost. + /// + public async Task> GetConnectionsAsync() + { + var connections = await _fixture.ListAsync("list_connections", "pid", "vhost", "state"); + + return connections + .Where(row => row["vhost"] == _vhostName) + .Select(row => new ConnectionInfo(row["pid"], row["state"])) + .ToArray(); + } + + /// + /// Gets the number of open connections to the vhost. + /// + public async Task GetConnectionCountAsync() => (await GetConnectionsAsync()).Count; + + /// + /// Gets the client-provided names of the connections to the vhost. + /// + public async Task> GetConnectionNamesAsync() + { + var connections = await _fixture.ListAsync("list_connections", "vhost", "client_properties"); + + return connections + .Where(row => row["vhost"] == _vhostName) + .Select(row => ConnectionNameProperty().Match(row["client_properties"])) + .Where(match => match.Success) + .Select(match => match.Groups["name"].Value) + .Order(StringComparer.Ordinal) + .ToList(); + } + + /// + /// Gets the connections, channels and consumers the broker holds for the vhost. + /// + public async Task GetStateAsync() + { + var output = await _fixture.EvalAsync( + $$""" + VHost = <<"{{_vhostName}}">>, + QueueCounts = [proplists:get_value(consumers, Info, 0) + || Q <- rabbit_amqqueue:list(VHost), + Info <- [catch rabbit_amqqueue:info(Q, [consumers])], is_list(Info)], + Connections = [C || C <- rabbit_networking:connections(), + (catch rabbit_reader:info(C, [vhost])) =:= [{vhost, VHost}]], + ChannelCounts = [proplists:get_value(consumer_count, Info, 0) + || C <- rabbit_channel:list(), + Info <- [catch rabbit_channel:info(C, [vhost, consumer_count])], + is_list(Info), proplists:get_value(vhost, Info) =:= VHost], + [length(Connections), + length([N || N <- ChannelCounts, N > 0]), + length([N || N <- ChannelCounts, N =:= 0]), + lists:sum(QueueCounts), + length([N || N <- QueueCounts, N > 0])]. + """); + var counts = JsonSerializer.Deserialize(output)!; + + return new BrokerState(counts[0], counts[1], counts[2], counts[3], counts[4]); + } + + /// + /// Waits until the broker counts the given number of connections against the connection limit of the + /// vhost. The count follows an open connection with a delay. + /// + public Task WaitForCountedConnectionsAsync(int count) + => Wait.UntilAsync( + async () => int.Parse( + await _fixture.EvalAsync( + $$"""rabbit_connection_tracking:count_tracked_items_in({vhost, <<"{{_vhostName}}">>})."""), + CultureInfo.InvariantCulture), + counted => counted == count, + $"the broker to count {count} connections against the limit of vhost '{_vhostName}'"); + + /// + /// Gets the name of the reply queue of the bus. + /// + public async Task GetReplyQueueNameAsync() + => (await GetQueuesAsync()).Keys.Single(name => name.StartsWith("response-", StringComparison.Ordinal)); + + /// + /// Waits until the queue has exactly one consumer, and returns it. + /// + public async Task WaitForConsumerAsync(string queueName) + { + var consumers = await Wait.UntilAsync( + () => GetConsumersOfAsync(queueName), + found => found.Count == 1, + $"queue '{queueName}' to have one consumer"); + + return consumers[0]; + } + + /// + /// Waits until the queue has exactly one consumer and that consumer is not + /// , and returns it. + /// + public async Task WaitForConsumerAsync(string queueName, ConsumerInfo otherThan) + { + var consumers = await Wait.UntilAsync( + () => GetConsumersOfAsync(queueName), + found => found is [var consumer] && consumer.ConsumerTag != otherThan.ConsumerTag, + $"queue '{queueName}' to have one consumer other than {otherThan.ConsumerTag}"); + + return consumers[0]; + } + + /// + /// Waits until the queue has no consumer. A queue that does not exist has none. + /// + public Task WaitForNoConsumerAsync(string queueName) + => Wait.UntilAsync( + () => GetQueueAsync(queueName), + queue => queue is null or { Consumers: 0 }, + $"queue '{queueName}' to have no consumer"); + + /// + /// Waits until the queue exists. + /// + public Task WaitForQueueAsync(string queueName) + => Wait.UntilAsync( + () => GetQueueAsync(queueName), + queue => queue is not null, + $"queue '{queueName}' to exist"); + + /// + /// Waits until the queue does not exist. + /// + public Task WaitForQueueDeletionAsync(string queueName) + => Wait.UntilAsync( + () => GetQueueAsync(queueName), + queue => queue is null, + $"queue '{queueName}' to be deleted"); + + /// + /// Waits until no exchange is bound to the queue . + /// + public Task WaitForNoBindingsAsync(string queueName) + => Wait.UntilAsync( + async () => + { + var bindings = await _fixture.ListAsync( + "list_bindings", "-p", _vhostName, "source_name", "destination_name"); + + // every queue has a binding to the default exchange, whose name is empty + return bindings + .Where(row => row["source_name"].Length > 0 && row["destination_name"] == queueName) + .Select(row => row["source_name"]) + .ToArray(); + }, + sources => sources.Length == 0, + $"no exchange to be bound to queue '{queueName}'"); + + /// + /// Waits until the queue holds messages that are + /// ready for delivery. + /// + public Task WaitForMessagesAsync(string queueName, int count) + => Wait.UntilAsync( + () => GetQueueAsync(queueName), + queue => queue?.Ready == count, + $"queue '{queueName}' to hold {count} ready messages"); + + /// + /// Waits until the queue holds no message, neither ready nor delivered but + /// not yet acknowledged. + /// + public Task WaitForEmptyQueueAsync(string queueName) + => Wait.UntilAsync( + () => GetQueueAsync(queueName), + queue => queue is { Ready: 0, Unacknowledged: 0 }, + $"queue '{queueName}' to hold no messages"); + + /// + /// Waits until the broker blocks a publishing connection to the vhost. + /// + public Task WaitForBlockedPublisherAsync() + => Wait.UntilAsync( + GetConnectionsAsync, + connections => connections.Any(connection => connection.State == "blocked"), + $"a blocked connection to vhost '{_vhostName}'"); + + /// + /// Waits until a resource alarm of the broker applies to a connection to the vhost. Such a connection is + /// blocked, or is blocked as soon as it publishes. + /// + public Task WaitForAlarmAsync() + => Wait.UntilAsync( + GetConnectionsAsync, + connections => connections.Any(connection => connection.State is "blocking" or "blocked"), + $"a resource alarm to apply to a connection to vhost '{_vhostName}'"); + + /// + /// Waits until the vhost has no open connection. + /// + public Task WaitForNoConnectionAsync() + => Wait.UntilAsync( + GetConnectionsAsync, + connections => connections.Count == 0, + $"no connection to vhost '{_vhostName}'"); + + private async Task> GetConsumersOfAsync(string queueName) + => (await GetConsumersAsync()).Where(consumer => consumer.Queue == queueName).ToArray(); + + private static int ParseCount(string value) => int.Parse(value, CultureInfo.InvariantCulture); + + [GeneratedRegex("\\{\"connection_name\",\"(?[^\"]*)\"\\}")] + private static partial Regex ConnectionNameProperty(); +} + +/// +/// A queue as the broker reports it. +/// +/// The number of consumers on the queue. +/// The number of messages that are ready for delivery. +/// The number of messages that are delivered but not yet acknowledged. +public sealed record QueueInfo(int Consumers, int Ready, int Unacknowledged) +{ + /// + /// Gets the number of messages in the queue, both ready and delivered but not yet acknowledged. + /// + public int Messages => Ready + Unacknowledged; +} + +/// +/// A consumer as the broker reports it. Every attach gets a new consumer tag from the broker. +/// +/// The name of the queue the consumer consumes from. +/// The consumer tag. +/// The broker's identifier of the channel the consumer is on. +/// The prefetch count of the consumer. +public sealed record ConsumerInfo(string Queue, string ConsumerTag, string ChannelPid, int Prefetch); + +/// +/// A connection as the broker reports it. +/// +/// The broker's identifier of the connection. +/// The state of the connection, such as running or blocked. +public sealed record ConnectionInfo(string Pid, string State); + +/// +/// The connections, channels and consumers the broker holds for one vhost. +/// +/// The number of open connections. +/// The number of open channels that have a consumer. +/// The number of open channels without a consumer, such as pooled publish channels. +/// The number of consumers across all queues. +/// The number of queues that have at least one consumer. +public sealed record BrokerState( + int Connections, + int ConsumerChannels, + int IdleChannels, + int Consumers, + int ConsumedQueues); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/LogRecorder.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/LogRecorder.cs new file mode 100644 index 00000000000..ed0563c2d84 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/LogRecorder.cs @@ -0,0 +1,113 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; + +namespace Mocha.Transport.RabbitMQ.Tests.Helpers; + +/// +/// One recorded log entry. +/// +/// The level of the entry. +/// The logger category. +/// The name of the log event, which is the name of its log method. +/// The rendered message. +/// The logged exception, or null when there was none. +/// The structured properties of the message and of every scope around it. +public sealed record LogEntry( + LogLevel Level, + string Category, + string EventName, + string Message, + Exception? Exception, + IReadOnlyDictionary Properties) +{ + /// + public override string ToString() => $"{Level} {EventName}: {Message}"; +} + +/// +/// A logger provider that keeps every entry in memory, in the order it was written. +/// +public sealed class LogRecorder : ILoggerProvider, ISupportExternalScope +{ + private const string TransportCategory = "Mocha.Transport.RabbitMQ"; + + private readonly ConcurrentQueue _entries = new(); + private IExternalScopeProvider _scopes = new LoggerExternalScopeProvider(); + + /// + /// Gets a snapshot of the entries the RabbitMQ transport wrote at or above. + /// + public IReadOnlyList GetTransportEntries(LogLevel minimumLevel = LogLevel.Information) + => [.. _entries.Where(e => e.Level >= minimumLevel && e.Category.StartsWith(TransportCategory, StringComparison.Ordinal))]; + + /// + /// Gets a snapshot of the entries of one log event. + /// + public IReadOnlyList GetEntries(string eventName) => [.. _entries.Where(e => e.EventName == eventName)]; + + /// + /// Waits until an entry of the log event was written, and returns the first one. + /// + public Task WaitForAsync(string eventName) + => Wait.UntilAsync( + () => Task.FromResult(_entries.FirstOrDefault(e => e.EventName == eventName)), + entry => entry is not null, + $"a log entry '{eventName}'")!; + + /// + /// Removes every recorded entry. + /// + public void Clear() => _entries.Clear(); + + /// + public ILogger CreateLogger(string categoryName) => new Logger(this, categoryName); + + /// + public void SetScopeProvider(IExternalScopeProvider scopeProvider) => _scopes = scopeProvider; + + /// + public void Dispose() + { + } + + private sealed class Logger(LogRecorder owner, string category) : ILogger + { + public IDisposable? BeginScope(TState state) + where TState : notnull + => owner._scopes.Push(state); + + public bool IsEnabled(LogLevel logLevel) => logLevel != LogLevel.None; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var properties = new Dictionary(); + owner._scopes.ForEachScope(static (scope, properties) => Add(properties, scope), properties); + Add(properties, state); + + owner._entries.Enqueue( + new LogEntry( + logLevel, + category, + eventId.Name ?? string.Empty, + formatter(state, exception), + exception, + properties)); + } + + private static void Add(Dictionary properties, object? state) + { + if (state is IEnumerable> pairs) + { + foreach (var pair in pairs) + { + properties[pair.Key] = pair.Value; + } + } + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/RabbitMQClusterFixture.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/RabbitMQClusterFixture.cs new file mode 100644 index 00000000000..2a319626dde --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/RabbitMQClusterFixture.cs @@ -0,0 +1,119 @@ +using System.Text.Json; +using CookieCrumble.Resources; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Networks; +using RabbitMQ.Client; +using Testcontainers.RabbitMq; +using ContainerWait = DotNet.Testcontainers.Builders.Wait; + +namespace Mocha.Transport.RabbitMQ.Tests.Helpers; + +public sealed class RabbitMQClusterFixture : IAsyncLifetime +{ + private readonly INetwork _network = new NetworkBuilder().Build(); + private readonly ClusterNode[] _nodes; + + public RabbitMQClusterFixture() + { + _nodes = Enumerable.Range(0, 3) + .Select(index => new ClusterNode(_network, $"mocha-rabbit-{index}")) + .ToArray(); + } + + public async ValueTask InitializeAsync() + { + await _network.CreateAsync(TestContext.Current.CancellationToken); + foreach (var node in _nodes) + { + await node.InitializeAsync(); + if (!ReferenceEquals(node, _nodes[0])) + { + await node.ExecAsync(["rabbitmqctl", "stop_app"]); + await node.ExecAsync(["rabbitmqctl", "reset"]); + await node.ExecAsync(["rabbitmqctl", "join_cluster", GetNodeName(0)]); + await node.ExecAsync(["rabbitmqctl", "start_app"]); + } + } + } + + public string GetNodeName(int index) => $"rabbit@mocha-rabbit-{index}"; + + public ConnectionFactory CreateConnectionFactory(int index) + => new() { Uri = new Uri(_nodes[index].ConnectionString) }; + + /// + /// Kills the node once every quorum queue keeps a majority of online replicas without it. + /// + public async Task KillNodeAsync(int index) + { + await _nodes[index].ExecAsync( + ["rabbitmq-upgrade", "await_online_quorum_plus_one", "--timeout", "30"]); + await _nodes[index].KillAsync(); + return new AsyncDisposable(() => _nodes[index].RestartAsync()); + } + + public async Task GetQueueStateAsync(int node, string queue) + { + var output = await _nodes[node].ExecAsync( + ["rabbitmqctl", "list_queues", "name", "leader", "members", "messages", "--formatter", "json", "--quiet"]); + using var document = JsonDocument.Parse(output); + foreach (var state in document.RootElement.EnumerateArray()) + { + if (state.GetProperty("name").GetString() == queue) + { + return new QueueState( + state.GetProperty("leader").GetString()!, + state.GetProperty("members").EnumerateArray().Select(member => member.GetString()!).ToArray(), + state.GetProperty("messages").GetInt32()); + } + } + + return null; + } + + public async ValueTask DisposeAsync() + { + try + { + await Task.WhenAll(_nodes.Select(node => node.DisposeAsync().AsTask())); + } + finally + { + await _network.DisposeAsync(); + } + } + + public sealed record QueueState(string Leader, string[] Members, int Messages); + + private sealed class ClusterNode(INetwork network, string hostname) : RabbitMQResource + { + public Task ExecAsync(string[] command) => Container.ExecWithTimeoutAsync(command); + + public Task KillAsync() => Container.StopAsync(TestContext.Current.CancellationToken); + + public Task RestartAsync() => Container.StartAsync(TestContext.Current.CancellationToken); + + protected override RabbitMqContainer Build() + { +#if RABBITMQ_3 + const string image = "rabbitmq:3.13"; +#else + const string image = "rabbitmq:4.3"; +#endif + return new RabbitMqBuilder(image) + .WithUsername("guest") + .WithPassword("guest") + .WithNetwork(network) + .WithNetworkAliases(hostname) + .WithHostname(hostname) + .WithEnvironment("RABBITMQ_ERLANG_COOKIE", "mocha-cluster-tests") + // Return deliveries held by a disconnected quorum consumer after one second. + .WithEnvironment("RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS", "-rabbit consumer_disconnected_timeout 1000") + .WithWaitStrategy(ContainerWait.ForUnixContainer() + .UntilMessageIsLogged("Server startup complete") + .UntilCommandIsCompleted("rabbitmq-diagnostics", "-q", "check_running")) + .WithCreateParameterModifier(parameters => parameters.StopSignal = "SIGKILL") + .Build(); + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/RabbitMQFixture.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/RabbitMQFixture.cs index dd405ee70e5..51d3582ad43 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/RabbitMQFixture.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/RabbitMQFixture.cs @@ -1,58 +1,138 @@ +using System.Net.Http.Headers; +#if !NET11_0_OR_GREATER +using System.Net.Http.Json; +#endif using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; using CookieCrumble.Resources; +using DotNet.Testcontainers.Containers; using RabbitMQ.Client; +using Testcontainers.RabbitMq; +using ContainerWait = DotNet.Testcontainers.Builders.Wait; namespace Mocha.Transport.RabbitMQ.Tests.Helpers; public sealed class RabbitMQFixture : IAsyncLifetime { + private const string ManagementUser = "mocha-tests"; + private const string ManagementPassword = "mocha-tests"; + #if RABBITMQ_3 - private readonly RabbitMQResource _resource = new() { Image = "rabbitmq:3.13" }; + private readonly ManagementRabbitMQResource _resource = new() { Image = "rabbitmq:3.13" }; #else - private readonly RabbitMQResource _resource = new(); + private readonly ManagementRabbitMQResource _resource = new(); #endif + private readonly HttpClient _management = new(); + + public RabbitMQFixture() + { + Faults = new BrokerFaults(this, _resource); + } public async ValueTask InitializeAsync() { await _resource.InitializeAsync(); + _management.BaseAddress = new UriBuilder("http", _resource.Hostname, _resource.ManagementPort, "api/").Uri; + _management.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String("guest:guest"u8)); + await PutAsync($"users/{ManagementUser}", new { password = ManagementPassword, tags = "administrator" }); + _management.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{ManagementUser}:{ManagementPassword}"))); + + // Quorum queues refresh the consumer and message counts that list_queues reports on each tick, so tests + // that read those counts need a short tick. + await EvalAsync("application:set_env(rabbit, quorum_tick_interval, 250)."); } public async ValueTask DisposeAsync() { + _management.Dispose(); await _resource.DisposeAsync(); } public string ConnectionString => _resource.ConnectionString; + /// + /// Gets the faults that affect the whole broker. A fault of a single vhost is on . + /// + public BrokerFaults Faults { get; } + public async Task CreateVhostAsync( [CallerMemberName] string testName = "", [CallerFilePath] string filePath = "") { var vhostName = GenerateVhostName(testName, filePath); - await _resource.InvokeCommandAsync(["rabbitmqctl", "add_vhost", vhostName]); - await _resource.InvokeCommandAsync([ - "rabbitmqctl", "set_permissions", "-p", vhostName, "guest", ".*", ".*", ".*" - ]); + await PutAsync($"vhosts/{Uri.EscapeDataString(vhostName)}", new { }); + await PutAsync( + $"permissions/{Uri.EscapeDataString(vhostName)}/guest", + new { configure = ".*", write = ".*", read = ".*" }); + await PutAsync( + $"permissions/{Uri.EscapeDataString(vhostName)}/{ManagementUser}", + new { configure = ".*", write = ".*", read = ".*" }); return new VhostContext(this, vhostName); } - internal async Task InvokeCommandAsync(string[] command) + /// + /// Runs a command inside the broker container and returns its output. Throws a + /// when the command does not complete within . + /// + internal Task InvokeCommandAsync(string[] command) => _resource.ExecAsync(command); + + internal Task EvalAsync(string expression) => InvokeCommandAsync(["rabbitmqctl", "eval", expression]); + + /// + /// Runs a rabbitmqctl list command and returns its rows, each keyed by column name. + /// + internal async Task>> ListAsync(params string[] command) + { + // rabbitmqctl prints the columns of some lists in its own order, so the header row names them + var output = await InvokeCommandAsync(["rabbitmqctl", .. command, "--quiet"]); + var lines = output.Split('\n', StringSplitOptions.RemoveEmptyEntries); + if (lines.Length == 0) + { + return []; + } + + var columns = SplitRow(lines[0]); + var rows = new List>(lines.Length - 1); + foreach (var line in lines.Skip(1)) + { + var cells = SplitRow(line); + var row = new Dictionary(columns.Length); + for (var i = 0; i < columns.Length; i++) + { + // a row can end before its trailing empty cells + row[columns[i]] = i < cells.Length ? cells[i] : ""; + } + + rows.Add(row); + } + + return rows; + } + + internal async Task DeleteVhostAsync(string vhostName) { - return await _resource.InvokeCommandAsync(command); + await DeleteAsync($"vhosts/{Uri.EscapeDataString(vhostName)}"); } - internal async Task CloseAllConnectionsAsync(string reason = "test") + internal async Task PutAsync(string path, object value) { - await _resource.InvokeCommandAsync(["rabbitmqctl", "close_all_connections", reason]); + using var response = await _management.PutAsJsonAsync(path, value, TestContext.Current.CancellationToken); + response.EnsureSuccessStatusCode(); } - internal async Task DeleteVhostAsync(string vhostName) + internal async Task DeleteAsync(string path) { - await _resource.InvokeCommandAsync(["rabbitmqctl", "delete_vhost", vhostName]); + using var response = await _management.DeleteAsync(path, TestContext.Current.CancellationToken); + response.EnsureSuccessStatusCode(); } + private static string[] SplitRow(string line) => line.TrimEnd('\r').Split('\t'); + private static string GenerateVhostName(string testName, string filePath) { var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(filePath)))[..8]; @@ -68,15 +148,117 @@ public VhostContext(RabbitMQFixture fixture, string vhostName) { _fixture = fixture; VhostName = vhostName; - ConnectionFactory = new ConnectionFactory { Uri = new Uri(fixture.ConnectionString), VirtualHost = vhostName }; + ConnectionSettings = new ConnectionFactory + { + Uri = new Uri(fixture.ConnectionString), + VirtualHost = vhostName + }; + Broker = new BrokerView(fixture, vhostName); + Faults = new VhostFaults(fixture, vhostName, Broker); } public string VhostName { get; } - public IConnectionFactory ConnectionFactory { get; } + public IConnectionFactory ConnectionFactory => ConnectionSettings; + + /// + /// Gets the connection factory as its concrete type, so a test can shorten heartbeats and timeouts. + /// + public ConnectionFactory ConnectionSettings { get; } + + /// + /// Gets the state the broker holds for the vhost. + /// + public BrokerView Broker { get; } + + /// + /// Gets the faults that affect only this vhost. + /// + public VhostFaults Faults { get; } public async ValueTask DisposeAsync() => await _fixture.DeleteVhostAsync(VhostName); } -[CollectionDefinition("RabbitMQ")] -public class RabbitMQCollection : ICollectionFixture; +public static class RabbitMQCollections +{ + public const string ConnectionRecovery = "RabbitMQ Connection Recovery"; + public const string ConsumerRecovery = "RabbitMQ Consumer Recovery"; + public const string Messaging = "RabbitMQ Messaging"; + public const string Topology = "RabbitMQ Topology"; +} + +[CollectionDefinition(RabbitMQCollections.ConnectionRecovery)] +public sealed class RabbitMQConnectionRecoveryCollection : ICollectionFixture; + +[CollectionDefinition(RabbitMQCollections.ConsumerRecovery)] +public sealed class RabbitMQConsumerRecoveryCollection : ICollectionFixture; + +[CollectionDefinition(RabbitMQCollections.Messaging)] +public sealed class RabbitMQMessagingCollection : ICollectionFixture; + +[CollectionDefinition(RabbitMQCollections.Topology)] +public sealed class RabbitMQTopologyCollection : ICollectionFixture; + +internal sealed class ManagementRabbitMQResource : RabbitMQResource +{ + public ushort ManagementPort => Container.GetMappedPublicPort(15672); + + public Task ExecAsync(string[] command) => Container.ExecWithTimeoutAsync(command); + + public async Task KillAsync() + { + await ExecAsync(["sh", "-c", "kill -KILL \"$(cat /tmp/rabbitmq.pid)\""]); + await ExecAsync(["rm", "/tmp/rabbitmq.pid"]); + return new AsyncDisposable(async () => + { + await ExecAsync(["docker-entrypoint.sh", "rabbitmq-server", "-detached"]); + await ExecAsync(["rabbitmqctl", "wait", "/tmp/rabbitmq.pid"]); + }); + } + + protected override RabbitMqBuilder Configure(RabbitMqBuilder builder) + => builder + .WithEnvironment("RABBITMQ_PID_FILE", "/tmp/rabbitmq.pid") + .WithEntrypoint("/bin/sh", "-c") + .WithCommand("docker-entrypoint.sh rabbitmq-server -detached && exec sleep infinity") + .WithResourceMapping("[rabbitmq_management]."u8.ToArray(), "/etc/rabbitmq/enabled_plugins") + .WithPortBinding(15672, true) + .WithWaitStrategy(ContainerWait.ForUnixContainer().UntilHttpRequestIsSucceeded(request => request + .ForPort(15672) + .ForPath("/api/overview") + .WithBasicAuthentication("guest", "guest"))); +} + +internal static class ContainerCommands +{ + /// + /// Runs a command inside the container and returns its output. Throws a when + /// the command does not complete within , and an + /// when it exits with an error. + /// + public static async Task ExecWithTimeoutAsync(this IContainer container, string[] command) + { + var testAborted = TestContext.Current.CancellationToken; + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(testAborted); + deadline.CancelAfter(TestTimeouts.Command); + + ExecResult result; + try + { + result = await container.ExecAsync(command, deadline.Token).WaitAsync(deadline.Token); + } + catch (OperationCanceledException) when (!testAborted.IsCancellationRequested) + { + throw new TimeoutException( + $"'{string.Join(' ', command)}' did not complete within {TestTimeouts.Command.TotalSeconds:0} s."); + } + + if (result.ExitCode != 0) + { + throw new InvalidOperationException( + $"'{string.Join(' ', command)}' failed with exit code {result.ExitCode}: {result.Stderr}"); + } + + return result.Stdout; + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TcpProxy.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TcpProxy.cs new file mode 100644 index 00000000000..aebdadcffb8 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TcpProxy.cs @@ -0,0 +1,118 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; + +namespace Mocha.Transport.RabbitMQ.Tests.Helpers; + +/// +/// A TCP proxy in front of the broker that can drop its connections, and can accept new connections +/// without answering them. +/// +public sealed class TcpProxy : IAsyncDisposable +{ + private readonly TcpListener _listener = new(IPAddress.Loopback, 0); + private readonly CancellationTokenSource _stop = new(); + private readonly ConcurrentDictionary _sessions = new(); + private readonly string _brokerHost; + private readonly int _brokerPort; + private readonly Task _accept; + private int _unanswered; + + public TcpProxy(string brokerHost, int brokerPort) + { + _brokerHost = brokerHost; + _brokerPort = brokerPort; + _listener.Start(); + Port = ((IPEndPoint)_listener.LocalEndpoint).Port; + _accept = AcceptAsync(); + } + + /// + /// Gets the local port the proxy listens on. + /// + public int Port { get; } + + /// + /// Gets or sets whether new connections are accepted and then never answered. + /// + public bool IsSilent { get; set; } + + /// + /// Gets the number of connections that were accepted and not answered. + /// + public int UnansweredConnections => _unanswered; + + /// + /// Closes every connection through the proxy. + /// + public void DropConnections() + { + foreach (var (client, upstream) in _sessions) + { + client.Close(); + upstream.Close(); + } + } + + /// + public async ValueTask DisposeAsync() + { + await _stop.CancelAsync(); + _listener.Stop(); + DropConnections(); + await _accept; + _stop.Dispose(); + } + + private async Task AcceptAsync() + { + var sessions = new List(); + try + { + while (true) + { + var client = await _listener.AcceptTcpClientAsync(_stop.Token); + sessions.Add(ForwardAsync(client)); + } + } + catch (Exception ex) + when (ex is OperationCanceledException or SocketException or ObjectDisposedException + or InvalidOperationException) + { + // The proxy was stopped. + } + + await Task.WhenAll(sessions); + } + + private async Task ForwardAsync(TcpClient client) + { + var upstream = new TcpClient(); + _sessions[client] = upstream; + try + { + if (IsSilent) + { + // The connection stays open, and the peer never receives a byte. + Interlocked.Increment(ref _unanswered); + await Task.Delay(Timeout.Infinite, _stop.Token); + } + + await upstream.ConnectAsync(_brokerHost, _brokerPort, _stop.Token); + var toBroker = client.GetStream().CopyToAsync(upstream.GetStream(), _stop.Token); + var toClient = upstream.GetStream().CopyToAsync(client.GetStream(), _stop.Token); + await Task.WhenAny(toBroker, toClient); + } + catch (Exception ex) + when (ex is IOException or SocketException or ObjectDisposedException or OperationCanceledException) + { + // The session ended. + } + finally + { + _sessions.TryRemove(client, out _); + client.Dispose(); + upstream.Dispose(); + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TestBus.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TestBus.cs index 1a46912c022..3e57d337830 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TestBus.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TestBus.cs @@ -6,13 +6,21 @@ public sealed class TestBus(ServiceProvider provider, MessagingRuntime runtime) { public ServiceProvider Provider => provider; + /// + /// Gets the RabbitMQ transport of the bus. + /// + public RabbitMQMessagingTransport Transport => runtime.Transports.OfType().Single(); + public async ValueTask DisposeAsync() { + // A host stops with a shutdown timeout, after which handlers that still run are cancelled. + using var shutdownTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + foreach (var transport in runtime.Transports) { if (transport.IsStarted) { - await transport.StopAsync(runtime, CancellationToken.None); + await transport.StopAsync(runtime, shutdownTimeout.Token); } } @@ -26,8 +34,16 @@ public static async Task BuildTestBusAsync(this IMessageBusHostBuilder { var provider = builder.Services.BuildServiceProvider(); var runtime = (MessagingRuntime)provider.GetRequiredService(); - await runtime.StartAsync(CancellationToken.None); - return new TestBus(provider, runtime); + try + { + await runtime.StartAsync(TestContext.Current.CancellationToken); + return new TestBus(provider, runtime); + } + catch + { + await provider.DisposeAsync(); + throw; + } } public static MessagingRuntime BuildRuntime(this IMessageBusHostBuilder builder) diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TestManagers.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TestManagers.cs new file mode 100644 index 00000000000..a40daae28af --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/TestManagers.cs @@ -0,0 +1,65 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; + +namespace Mocha.Transport.RabbitMQ.Tests.Helpers; + +/// +/// Creates the transport's connection owners without a transport around them. +/// +internal static class TestManagers +{ + public static RabbitMQConsumerManager CreateConsumerManager( + Func> connectionFactory, + string connectionName = "test consumer", + ILoggerFactory? loggerFactory = null) + { + loggerFactory ??= NullLoggerFactory.Instance; + var logger = loggerFactory.CreateLogger(); + return new RabbitMQConsumerManager( + logger, + new RabbitMQConnectionManager( + logger, + connectionName, + TimeProvider.System, + connectionFactory, + RabbitMQConsumerManager.SetUpConnectionAsync)); + } + + public static RabbitMQDispatcher CreateDispatcher( + Func> connectionFactory, + Func setUpConnectionAsync, + string connectionName = "test dispatcher", + ILoggerFactory? loggerFactory = null) + { + loggerFactory ??= NullLoggerFactory.Instance; + var logger = loggerFactory.CreateLogger(); + return new RabbitMQDispatcher( + logger, + new RabbitMQConnectionManager( + logger, + connectionName, + TimeProvider.System, + connectionFactory, + setUpConnectionAsync)); + } + + /// + /// Registers a consumer that consumes at once, without preparing its channel. + /// + public static Task RegisterConsumerAsync( + this RabbitMQConsumerManager manager, + string queueName, + Func messageHandler, + ushort maxPrefetch, + ushort maxConcurrency, + CancellationToken cancellationToken) + => manager.RegisterConsumerAsync( + queueName, + prepareAsync: null, + messageHandler, + maxPrefetch, + maxConcurrency, + cancellationToken); +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/Wait.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/Wait.cs new file mode 100644 index 00000000000..5236a3ef100 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Helpers/Wait.cs @@ -0,0 +1,184 @@ +using System.Collections; +using System.Runtime.CompilerServices; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Helpers; + +internal static class TestTimeouts +{ + /// + /// The ceiling for anything a test expects to happen. It is never an expected duration. + /// + public static readonly TimeSpan Signal = TimeSpan.FromSeconds(30); + + /// + /// The ceiling for one command that runs inside a broker container, which includes a broker restart. + /// + public static readonly TimeSpan Command = TimeSpan.FromSeconds(60); +} + +internal static class Wait +{ + private static readonly TimeSpan s_pollInterval = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan s_localPollInterval = TimeSpan.FromMilliseconds(25); + + /// + /// Polls until it holds, and throws a that names + /// the condition when it does not hold within . + /// + public static async Task UntilAsync( + Func condition, + [CallerArgumentExpression(nameof(condition))] string expectation = "") + { + var deadline = DateTimeOffset.UtcNow.Add(TestTimeouts.Signal); + + while (!condition()) + { + if (DateTimeOffset.UtcNow >= deadline) + { + throw new TimeoutException($"Expected {expectation} to hold within {Ceiling}."); + } + + await Task.Delay(s_localPollInterval, TestContext.Current.CancellationToken); + } + } + + /// + /// Polls until it holds, and throws a that names + /// the condition when it does not hold within . + /// + public static async Task UntilAsync( + Func> condition, + [CallerArgumentExpression(nameof(condition))] string expectation = "") + { + var deadline = DateTimeOffset.UtcNow.Add(TestTimeouts.Signal); + + while (!await condition()) + { + if (DateTimeOffset.UtcNow >= deadline) + { + throw new TimeoutException($"Expected {expectation} to hold within {Ceiling}."); + } + + await Task.Delay(s_pollInterval, TestContext.Current.CancellationToken); + } + } + + /// + /// Reads a state with until holds for it, and returns + /// that state. Throws a with the last observed state when the condition does + /// not hold within . + /// + public static Task UntilAsync(Func> observe, Func condition, string expectation) + => UntilAsync(observe, condition, expectation, TestTimeouts.Signal); + + /// + /// Reads a state with until holds for it, and returns + /// that state. Throws a with the last observed state when the condition does + /// not hold within , a bound that has a meaning of its own in the test. + /// + public static async Task UntilAsync( + Func> observe, + Func condition, + string expectation, + TimeSpan within) + { + var deadline = DateTimeOffset.UtcNow.Add(within); + + while (true) + { + var state = await observe(); + if (condition(state)) + { + return state; + } + + if (DateTimeOffset.UtcNow >= deadline) + { + throw new TimeoutException( + $"Expected {expectation} within {within.TotalSeconds:0} s, but last observed {Describe(state)}."); + } + + await Task.Delay(s_pollInterval, TestContext.Current.CancellationToken); + } + } + + private static string Ceiling => $"{TestTimeouts.Signal.TotalSeconds:0} s"; + + private static string Describe(object? state) + => state switch + { + null => "nothing", + string text => text, + IEnumerable items => $"[{string.Join(", ", items.Cast().Select(Describe))}]", + _ => state.ToString() ?? "nothing" + }; +} + +internal static class Condition +{ + /// + /// Polls until it holds or elapses, and returns + /// whether it held. For something that must happen, use instead. + /// + public static async Task WaitAsync(Func condition, TimeSpan window) + { + var deadline = DateTimeOffset.UtcNow.Add(window); + + while (!condition()) + { + if (DateTimeOffset.UtcNow >= deadline) + { + return false; + } + + await Task.Delay(50, TestContext.Current.CancellationToken); + } + + return true; + } + + /// + /// Polls until it holds or elapses, and returns + /// whether it held. For something that must happen, use instead. + /// + public static async Task WaitAsync(Func> condition, TimeSpan window) + { + var deadline = DateTimeOffset.UtcNow.Add(window); + + while (!await condition()) + { + if (DateTimeOffset.UtcNow >= deadline) + { + return false; + } + + await Task.Delay(100, TestContext.Current.CancellationToken); + } + + return true; + } +} + +internal static class ConnectionExtensions +{ + /// + /// Returns a task that completes once the client has seen shut down. + /// + public static Task WhenShutdownAsync(this IConnection connection) + { + var shutdown = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + connection.ConnectionShutdownAsync += (_, _) => + { + shutdown.TrySetResult(); + return Task.CompletedTask; + }; + + if (!connection.IsOpen) + { + shutdown.TrySetResult(); + } + + return shutdown.Task; + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Middlewares/RabbitMQAcknowledgementMiddlewareTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Middlewares/RabbitMQAcknowledgementMiddlewareTests.cs new file mode 100644 index 00000000000..0ffb60579f6 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Middlewares/RabbitMQAcknowledgementMiddlewareTests.cs @@ -0,0 +1,137 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Mocha.Middlewares; +using Mocha.Transport.RabbitMQ.Features; +using Mocha.Transport.RabbitMQ.Middlewares; +using Moq; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Exceptions; + +namespace Mocha.Transport.RabbitMQ.Tests.Middlewares; + +public sealed class RabbitMQAcknowledgementMiddlewareTests +{ + private static readonly ShutdownEventArgs s_connectionForced = + new(ShutdownInitiator.Peer, 320, "CONNECTION_FORCED - test"); + + [Fact] + public async Task InvokeAsync_Should_Complete_When_ChannelClosesDuringAck() + { + // arrange + // the channel reports open, and closes before the ack reaches it + var channel = new Mock(); + channel.SetupGet(c => c.IsOpen).Returns(true); + channel + .Setup(c => c.BasicAckAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => ValueTask.FromException(new AlreadyClosedException(s_connectionForced))); + var context = CreateContext(channel.Object, redelivered: false); + + // act + await new RabbitMQAcknowledgementMiddleware(NullLogger.Instance).InvokeAsync(context, _ => default); + + // assert + channel.Verify(c => c.BasicAckAsync(7, false, It.IsAny()), Times.Once()); + } + + [Fact] + public async Task InvokeAsync_Should_RunHandler_When_MessageIsRedelivered() + { + // arrange + var channel = CreateOpenChannel(); + var context = CreateContext(channel.Object, redelivered: true); + var handled = false; + + // act + await new RabbitMQAcknowledgementMiddleware(NullLogger.Instance).InvokeAsync( + context, + _ => + { + handled = true; + return default; + }); + + // assert + Assert.True(handled); + channel.Verify(c => c.BasicAckAsync(7, false, It.IsAny()), Times.Once()); + } + + [Fact] + public async Task InvokeAsync_Should_RunHandler_When_MessageIsNotRedelivered() + { + // arrange + var channel = CreateOpenChannel(); + var context = CreateContext(channel.Object, redelivered: false); + var handled = false; + + // act + await new RabbitMQAcknowledgementMiddleware(NullLogger.Instance).InvokeAsync( + context, + _ => + { + handled = true; + return default; + }); + + // assert + Assert.True(handled); + channel.Verify(c => c.BasicAckAsync(7, false, It.IsAny()), Times.Once()); + } + + [Fact] + public async Task InvokeAsync_Should_RethrowHandlerException_When_NackFailsOnClosedChannel() + { + // arrange + var channel = new Mock(); + channel.SetupGet(c => c.IsOpen).Returns(true); + channel + .Setup(c => c.BasicNackAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(() => ValueTask.FromException(new AlreadyClosedException(s_connectionForced))); + var context = CreateContext(channel.Object, redelivered: false); + + // act + var expected = new InvalidOperationException("handler failed"); + var exception = await Assert.ThrowsAsync( + async () => await new RabbitMQAcknowledgementMiddleware(NullLogger.Instance).InvokeAsync(context, _ => throw expected)); + + // assert + Assert.Same(expected, exception); + } + + private static Mock CreateOpenChannel() + { + var channel = new Mock(); + channel.SetupGet(c => c.IsOpen).Returns(true); + channel + .Setup(c => c.BasicAckAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(ValueTask.CompletedTask); + return channel; + } + + private static ReceiveContext CreateContext( + IChannel channel, + bool redelivered) + { + var services = new ServiceCollection() + .AddLogging() + .BuildServiceProvider(); + var context = new ReceiveContext { Services = services }; + + var feature = context.Features.GetOrSet(); + feature.Channel = channel; + feature.EventArgs = new BasicDeliverEventArgs( + consumerTag: "consumer-1", + deliveryTag: 7, + redelivered: redelivered, + exchange: "", + routingKey: "orders", + properties: new BasicProperties { MessageId = "message-1" }, + body: ReadOnlyMemory.Empty); + + return context; + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Mocha.Transport.RabbitMQ.Tests.csproj b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Mocha.Transport.RabbitMQ.Tests.csproj index ceedb6bef2a..308a6f89a86 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Mocha.Transport.RabbitMQ.Tests.csproj +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Mocha.Transport.RabbitMQ.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQAddressResolutionTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQAddressResolutionTests.cs index 900ec31f887..9c8df2e2056 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQAddressResolutionTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQAddressResolutionTests.cs @@ -142,7 +142,7 @@ private static (MessagingRuntime Runtime, RabbitMQMessagingTopology Topology) Cr .BuildRuntime(); var transport = runtime.Transports.OfType().Single(); - return (runtime, (RabbitMQMessagingTopology)transport.Topology); + return (runtime, transport.Topology); } private sealed class VirtualHostConnectionProvider(string virtualHost) : IRabbitMQConnectionProvider diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQDispatchProvisioningTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQDispatchProvisioningTests.cs deleted file mode 100644 index 5da4bd2b0d5..00000000000 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQDispatchProvisioningTests.cs +++ /dev/null @@ -1,418 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Mocha.Transport.RabbitMQ.Tests.Helpers; -using Moq; -using RabbitMQ.Client; -using RabbitMQ.Client.Events; - -namespace Mocha.Transport.RabbitMQ.Tests; - -/// -/// Covers dispatch-endpoint provisioning races, topology reprovisioning on reconnect, and -/// the mandatory-publish contract as exposed by RabbitMQ.Client, all against a mocked -/// connection so the outcomes are deterministic and do not depend on a live broker. -/// -public class RabbitMQDispatchProvisioningTests -{ - [Fact] - public async Task PublishAsync_Should_SettleProvisioning_When_ConcurrentFirstDispatchesRace() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - await using var bus = await BuildTestBusAsync(connectionMock.Object, t => - { - t.DeclareExchange("concurrent-ex").AutoProvision(true); - t.DispatchEndpoint("concurrent-ep").ToExchange("concurrent-ex").Publish(); - }); - - using var scope = bus.Provider.CreateScope(); - var messageBus = scope.ServiceProvider.GetRequiredService(); - - // the runtime eagerly declares the whole topology once on start; only the racy - // per-endpoint provisioning that happens on first dispatch is under test here - channelMock.Invocations.Clear(); - - // gate the exchange declare so every dispatch that reads the endpoint's - // unsynchronized _isProvisioned field before the gate opens is genuinely still - // in flight when the next one starts, instead of each finishing before the next begins - var declareGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var declareCount = 0; - channelMock - .Setup(c => c.ExchangeDeclareAsync( - "concurrent-ex", - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny())) - .Returns(async () => - { - Interlocked.Increment(ref declareCount); - await declareGate.Task; - }); - - // act - start the first dispatches without awaiting them so each one reaches the - // gated declare call while _isProvisioned is still false, proving they overlap, - // then release them together so they genuinely race on that check - var raceTasks = Enumerable.Range(0, 8) - .Select(i => messageBus.PublishAsync( - new OrderCreated { OrderId = $"RACE-{i}" }, - TestContext.Current.CancellationToken).AsTask()) - .ToArray(); - - Assert.All(raceTasks, task => Assert.False(task.IsCompleted)); - - declareGate.SetResult(); - - await Task.WhenAll(raceTasks); - - // assert - the endpoint's unsynchronized _isProvisioned field lets more than one of - // the racing dispatches observe it as false, so the exchange may be declared more - // than once during the race; the settling below is what has teeth - Assert.InRange(declareCount, 1, 8); - - // act - once the race has settled, provisioning must not repeat on later dispatches - channelMock.Invocations.Clear(); - - await messageBus.PublishAsync(new OrderCreated { OrderId = "RACE-SETTLED" }, TestContext.Current.CancellationToken); - - // assert - channelMock.Verify( - c => c.ExchangeDeclareAsync( - "concurrent-ex", - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny()), - Times.Never()); - } - - [Fact] - public async Task Dispatcher_Should_ReprovisionFullTopology_When_ConnectionRecovers() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - var recoveryHandlers = new List>(); - connectionMock - .SetupAdd(c => c.RecoverySucceededAsync += It.IsAny>()) - .Callback>(recoveryHandlers.Add); - - await using var bus = await BuildTestBusAsync(connectionMock.Object, t => - { - t.DeclareExchange("recover-ex").AutoProvision(true); - t.DeclareQueue("recover-q").AutoProvision(true); - t.DeclareBinding("recover-ex", "recover-q").AutoProvision(true); - t.DispatchEndpoint("recover-ep").ToExchange("recover-ex").Publish(); - }); - - channelMock.Invocations.Clear(); - - // act - await the recovery handlers directly instead of racing a real-time delay - // against the fire-and-forget RabbitMQ.Client event, which is how automatic topology - // recovery presents itself to the transport - await Task.WhenAll(recoveryHandlers.Select( - handler => handler(connectionMock.Object, AsyncEventArgs.Empty))); - - // assert - the exchange, queue, and binding are all redeclared, not just the resources - // touched by a prior dispatch - channelMock.Verify( - c => c.ExchangeDeclareAsync( - "recover-ex", - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny()), - Times.AtLeastOnce()); - - channelMock.Verify( - c => c.QueueDeclareAsync( - "recover-q", - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny()), - Times.AtLeastOnce()); - - channelMock.Verify( - c => c.QueueBindAsync( - "recover-q", - "recover-ex", - It.IsAny(), - It.IsAny>(), - It.IsAny(), - It.IsAny()), - Times.AtLeastOnce()); - } - - [Fact] - public async Task PublishAsync_Should_NotReprovision_When_ResourceDeletedWhileConnectionStaysOpen() - { - // arrange - every rented channel is tracked as it is created, so the assertions - // follow whichever channel the dispatcher actually publishes on rather than - // assuming a fixed rent order - var exchangeDeclares = new List(); - var publishedOn = new List>(); - var connectionMock = CreateTrackingConnection(exchangeDeclares, publishedOn); - await using var bus = await BuildTestBusAsync(connectionMock.Object, t => - { - t.DeclareExchange("deleted-ex").AutoProvision(true); - t.DispatchEndpoint("deleted-ep").ToExchange("deleted-ex").Publish(); - }); - - using var scope = bus.Provider.CreateScope(); - var messageBus = scope.ServiceProvider.GetRequiredService(); - - await messageBus.PublishAsync(new OrderCreated { OrderId = "DEL-1" }, TestContext.Current.CancellationToken); - - var channelUsedForFirstSend = Assert.Single(publishedOn); - - // the broker closed the channel after the exchange was deleted; the dispatcher - // connection itself remains open - channelUsedForFirstSend.SetupGet(c => c.IsOpen).Returns(false); - exchangeDeclares.Clear(); - publishedOn.Clear(); - - // act - the next send must rent a fresh channel from the still-open connection - // because the pool discards the one that now reports closed - await messageBus.PublishAsync(new OrderCreated { OrderId = "DEL-2" }, TestContext.Current.CancellationToken); - - // assert - the endpoint's provisioning latch is permanent for the process lifetime, - // so the deleted exchange is never redeclared; the publish is still attempted blind - // on a channel distinct from the one that was closed - Assert.Empty(exchangeDeclares); - var channelUsedForSecondSend = Assert.Single(publishedOn); - Assert.NotSame(channelUsedForFirstSend, channelUsedForSecondSend); - } - - [Fact] - public async Task PublishAsync_Should_CompleteWithoutObservingBasicReturn_When_MessageIsMandatory() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - await using var bus = await BuildTestBusAsync(connectionMock.Object, t => - { - t.DeclareExchange("mandatory-ex").AutoProvision(true); - t.DispatchEndpoint("mandatory-ep").ToExchange("mandatory-ex").Publish(); - }); - - using var scope = bus.Provider.CreateScope(); - var messageBus = scope.ServiceProvider.GetRequiredService(); - - // act - no BasicReturnAsync handler is attached to dispatch channels, and publisher - // confirmations are not enabled on them either, so an unroutable mandatory publish - // would go unnoticed; the mandatory flag is dropped rather than left as dead signal - await messageBus.PublishAsync(new OrderCreated { OrderId = "MANDATORY-1" }, TestContext.Current.CancellationToken); - - // assert - the publish is sent with mandatory=false, and no BasicReturnAsync handler - // is ever attached to the channel - channelMock.Verify( - c => c.BasicPublishAsync( - It.IsAny(), - It.IsAny(), - false, - It.IsAny(), - It.IsAny>(), - It.IsAny()), - Times.Once()); - - channelMock.VerifyAdd( - c => c.BasicReturnAsync += It.IsAny>(), - Times.Never()); - } - - [Fact] - public async Task SendAsync_Should_PublishToDefaultExchangeByQueueName_When_QueueOutsideTopology() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - await using var bus = await BuildTestBusAsync(connectionMock.Object, t => t.BindExplicitly()); - - using var scope = bus.Provider.CreateScope(); - var messageBus = scope.ServiceProvider.GetRequiredService(); - - channelMock.Invocations.Clear(); - - // act - await messageBus.SendAsync( - new OrderCreated { OrderId = "EXTERNAL-1" }, - new SendOptions { Endpoint = new Uri("rabbitmq:q/reporting.events") }, - TestContext.Current.CancellationToken); - - // assert - the message is routed through the default exchange with the queue name as - // routing key, and the queue the sender does not own is never declared - channelMock.Verify( - c => c.BasicPublishAsync( - It.Is(exchange => exchange.Value == ""), - It.Is(routingKey => routingKey.Value == "reporting.events"), - false, - It.IsAny(), - It.IsAny>(), - It.IsAny()), - Times.Once()); - - channelMock.Verify( - c => c.QueueDeclareAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny()), - Times.Never()); - } - - [Fact] - public async Task SendAsync_Should_PublishToExchangeByName_When_ExchangeOutsideTopology() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - await using var bus = await BuildTestBusAsync(connectionMock.Object, t => t.BindExplicitly()); - - using var scope = bus.Provider.CreateScope(); - var messageBus = scope.ServiceProvider.GetRequiredService(); - - channelMock.Invocations.Clear(); - - // act - await messageBus.SendAsync( - new OrderCreated { OrderId = "EXTERNAL-2" }, - new SendOptions { Endpoint = new Uri("rabbitmq:e/reporting-events") }, - TestContext.Current.CancellationToken); - - // assert - channelMock.Verify( - c => c.BasicPublishAsync( - It.Is(exchange => exchange.Value == "reporting-events"), - It.Is(routingKey => routingKey.Value == ""), - false, - It.IsAny(), - It.IsAny>(), - It.IsAny()), - Times.Once()); - - channelMock.Verify( - c => c.ExchangeDeclareAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny()), - Times.Never()); - } - - private static Mock CreateOpenChannel() - { - var channelMock = new Mock(); - channelMock.SetupGet(c => c.IsOpen).Returns(true); - return channelMock; - } - - private static Mock CreateOpenConnection(Mock channelToReturn) - { - var connectionMock = new Mock(); - connectionMock.SetupGet(c => c.IsOpen).Returns(true); - connectionMock.SetupGet(c => c.ClientProvidedName).Returns("test-connection"); - connectionMock - .Setup(c => c.CreateChannelAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(channelToReturn.Object); - - return connectionMock; - } - - /// - /// Builds a connection that hands out a fresh, tracked channel mock on every rent, so - /// tests can follow which channel actually declares or publishes without assuming a - /// fixed rent order. - /// - private static Mock CreateTrackingConnection( - List exchangeDeclares, - List> publishedOn) - { - var connectionMock = new Mock(); - connectionMock.SetupGet(c => c.IsOpen).Returns(true); - connectionMock.SetupGet(c => c.ClientProvidedName).Returns("test-connection"); - connectionMock - .Setup(c => c.CreateChannelAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(() => - { - var channelMock = CreateOpenChannel(); - channelMock - .Setup(c => c.ExchangeDeclareAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny(), - It.IsAny(), - It.IsAny())) - .Callback?, bool, bool, CancellationToken>( - (exchange, _, _, _, _, _, _, _) => exchangeDeclares.Add(exchange)) - .Returns(Task.CompletedTask); - channelMock - .Setup(c => c.BasicPublishAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny())) - .Callback(() => publishedOn.Add(channelMock)) - .Returns(ValueTask.CompletedTask); - return channelMock.Object; - }); - - return connectionMock; - } - - private static async Task BuildTestBusAsync( - IConnection connection, - Action configure) - { - var services = new ServiceCollection(); - services.AddSingleton(new MessageRecorder()); - var builder = services.AddMessageBus(); - var provider = builder - .AddRabbitMQ(t => - { - t.ConnectionProvider(_ => new FakeConnectionProvider(connection)); - configure(t); - }) - .Services - .BuildServiceProvider(); - - var runtime = (MessagingRuntime)provider.GetRequiredService(); - await runtime.StartAsync(CancellationToken.None); - - return new TestBus(provider, runtime); - } - - private sealed class FakeConnectionProvider(IConnection connection) : IRabbitMQConnectionProvider - { - public string Host => "localhost"; - public string VirtualHost => "/"; - public int Port => 5672; - - public ValueTask CreateAsync(CancellationToken cancellationToken) => new(connection); - } -} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQEndpointQueueOwnershipTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQEndpointQueueOwnershipTests.cs index b0737ad616e..f839b19daeb 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQEndpointQueueOwnershipTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQEndpointQueueOwnershipTests.cs @@ -85,7 +85,7 @@ public void EndpointQueue_Should_PreserveDeclaredQueue_When_CollidesWithDeclareQ t.Queue("orders").AutoProvision(true).Consumer(); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name == "orders"); @@ -109,7 +109,7 @@ public void EndpointQueue_Should_PreserveDeclaredArguments_When_CollidesWithDecl t.Queue("orders").WithArgument("x-dead-letter-exchange", "orders_dlx").Consumer(); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name == "orders"); @@ -133,7 +133,7 @@ public void EndpointQueue_Should_MaterializeDurableAutoDeleteQueueWithExpiry_Whe t.Endpoint("temp-orders").Temporary().Consumer(); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name == "temp-orders"); @@ -157,7 +157,7 @@ public void EndpointQueue_Should_UseConfiguredExpiry_When_TemporaryCalledWithExp t.Endpoint("temp-orders").Temporary(TimeSpan.FromMinutes(5)).Consumer(); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name == "temp-orders"); @@ -207,7 +207,7 @@ public void EndpointQueue_Should_MaterializeTemporaryQueue_When_SharedQueueDisco } }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; var secondConfiguration = new RabbitMQReceiveEndpointConfiguration { Name = "second", @@ -266,7 +266,7 @@ public void EndpointQueue_Should_ReuseDeclaredQueue_When_DeclaredAutoDeleteAndEn t.Endpoint("orders").Temporary().Consumer(); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name == "orders"); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQReceiveEndpointBindFromTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQReceiveEndpointBindFromTests.cs index 856eea2d03f..1e36c254c23 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQReceiveEndpointBindFromTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQReceiveEndpointBindFromTests.cs @@ -51,7 +51,7 @@ public void OnDiscoverTopology_Should_AddTwoBindings_When_TwoRoutingKeysDeclared .BindFrom(new Uri("exchange:topic-exchange"), "order.created.us"); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var description = transport.Describe(); @@ -84,7 +84,7 @@ public void OnDiscoverTopology_Should_InheritQueueAutoProvision_When_QueueOptsIn .BindFrom(new Uri("exchange:source-fanout-exchange"), "order.created"); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var binding = topology.Bindings @@ -112,7 +112,7 @@ public void OnDiscoverTopology_Should_LeaveBindingAutoProvisionUnset_When_QueueD .BindFrom(new Uri("exchange:source-fanout-exchange"), "order.created"); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var binding = topology.Bindings @@ -140,7 +140,7 @@ public void OnDiscoverTopology_Should_KeepBindingOptOut_When_QueueOptsIn() .BindFrom(new Uri("exchange:source-fanout-exchange"), "order.created", false); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var binding = topology.Bindings @@ -167,7 +167,7 @@ public void OnDiscoverTopology_Should_ProvisionBinding_When_OnlyBindingOptsIn() .BindFrom(new Uri("exchange:source-fanout-exchange"), "order.created", autoProvision: true); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var binding = topology.Bindings @@ -195,7 +195,7 @@ public void OnDiscoverTopology_Should_InheritQueueOptOut_When_TransportProvision .BindFrom(new Uri("exchange:source-fanout-exchange"), "order.created"); }); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var binding = topology.Bindings diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQTransportTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQTransportTests.cs index 856d8c41ce9..ca6845bb9c1 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQTransportTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/RabbitMQTransportTests.cs @@ -1,8 +1,7 @@ +using CookieCrumble; using Microsoft.Extensions.DependencyInjection; using Mocha.Features; using Mocha.Transport.RabbitMQ.Tests.Helpers; -using Moq; -using RabbitMQ.Client; namespace Mocha.Transport.RabbitMQ.Tests; @@ -92,7 +91,7 @@ public void TryGetDispatchEndpoint_Should_Resolve_When_TopologyBaseAddressUsed() // arrange var runtime = CreateRuntime(b => b.AddRequestHandler()); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; var dispatchEndpoint = transport.DispatchEndpoints.FirstOrDefault(e => topology.Address.IsBaseOf(e.Destination.Address) @@ -128,7 +127,7 @@ public void TryGetDispatchEndpoint_Should_ReturnFalse_When_UnknownUri() } [Fact] - public void Describe_Should_ReturnDescription_When_EventHandlerRegistered() + public void Describe_Should_ListExchangesQueuesAndBindings_When_EventHandlerRegistered() { // arrange var runtime = CreateRuntime(b => b.AddEventHandler()); @@ -138,62 +137,7 @@ public void Describe_Should_ReturnDescription_When_EventHandlerRegistered() var description = transport.Describe(); // assert - Assert.NotNull(description); - Assert.Equal("rabbitmq", description.Schema); - Assert.Equal("RabbitMQMessagingTransport", description.TransportType); - Assert.NotNull(description.Topology); - } - - [Fact] - public void Describe_Should_IncludeExchangeEntities_When_EventHandlerRegistered() - { - // arrange - var runtime = CreateRuntime(b => b.AddEventHandler()); - var transport = runtime.Transports.OfType().Single(); - - // act - var description = transport.Describe(); - - // assert - Assert.NotNull(description.Topology); - Assert.Contains(description.Topology.Entities, e => e.Kind == "exchange"); - } - - [Fact] - public void Describe_Should_IncludeQueueEntities_When_EventHandlerRegistered() - { - // arrange - var runtime = CreateRuntime(b => b.AddEventHandler()); - var transport = runtime.Transports.OfType().Single(); - - // act - var description = transport.Describe(); - - // assert - Assert.NotNull(description.Topology); - Assert.Contains(description.Topology.Entities, e => e.Kind == "queue"); - } - - [Fact] - public void Describe_Should_IncludeBindingLinks_When_EventHandlerRegistered() - { - // arrange - var runtime = CreateRuntime(b => b.AddEventHandler()); - var transport = runtime.Transports.OfType().Single(); - - // act - var description = transport.Describe(); - - // assert - Assert.NotNull(description.Topology); - Assert.NotEmpty(description.Topology.Links); - Assert.All( - description.Topology.Links, - link => - { - Assert.Equal("bind", link.Kind); - Assert.Equal("forward", link.Direction); - }); + RabbitMQDescribeSnapshot.Create(description).MatchSnapshot(); } [Fact] @@ -394,7 +338,7 @@ public void CreateEndpointConfiguration_Should_CreateQueueConfig_When_TopologyBa var runtime = CreateRuntime(b => b.AddRequestHandler()); var transport = runtime.Transports.OfType().Single(); var context = (IMessagingConfigurationContext)runtime; - var topologyAddress = ((RabbitMQMessagingTopology)transport.Topology).Address; + var topologyAddress = transport.Topology.Address; // act - use the full topology base address with q/ path var config = transport.CreateEndpointConfiguration(context, new Uri(topologyAddress, "q/my-queue")); @@ -413,7 +357,7 @@ public void CreateEndpointConfiguration_Should_CreateExchangeConfig_When_Topolog var runtime = CreateRuntime(b => b.AddEventHandler()); var transport = runtime.Transports.OfType().Single(); var context = (IMessagingConfigurationContext)runtime; - var topologyAddress = ((RabbitMQMessagingTopology)transport.Topology).Address; + var topologyAddress = transport.Topology.Address; // act - use the full topology base address with e/ path var config = transport.CreateEndpointConfiguration(context, new Uri(topologyAddress, "e/my-exchange")); @@ -440,105 +384,17 @@ public void CreateEndpointConfiguration_Should_ReturnNull_When_UnknownScheme() Assert.Null(config); } - [Fact] - public async Task DisposeAsync_Should_DisposeConsumerManager_When_Called() - { - // arrange - var runtime = CreateRuntime(b => b.AddEventHandler()); - var transport = runtime.Transports.OfType().Single(); - - // act & assert - no exception thrown - await transport.DisposeAsync(); - } - - [Fact] - public async Task DisposeAsync_Should_BeIdempotent_When_CalledTwice() - { - // arrange - var runtime = CreateRuntime(b => b.AddEventHandler()); - var transport = runtime.Transports.OfType().Single(); - - // act - call twice - await transport.DisposeAsync(); - await transport.DisposeAsync(); - - // assert - no exception - } - - [Fact] - public async Task DisposeAsync_Should_DisposeDispatcherConnectionAndPooledChannels_When_Called() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - var runtime = CreateRuntime(_ => { }, new FakeConnectionProvider(connectionMock.Object)); - var transport = runtime.Transports.OfType().Single(); - - await transport.Dispatcher.EnsureConnectedAsync(TestContext.Current.CancellationToken); - var rented = await transport.Dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); - await transport.Dispatcher.ReturnChannelAsync(rented); - - channelMock.Invocations.Clear(); - connectionMock.Invocations.Clear(); - - // act - await transport.DisposeAsync(); - - // assert - channelMock.Verify(c => c.DisposeAsync(), Times.AtLeastOnce()); - connectionMock.Verify(c => c.DisposeAsync(), Times.AtLeastOnce()); - } - - [Fact] - public async Task DisposeAsync_Should_DisposeBothConsumerManagerAndDispatcher_When_ConnectedAndRepeated() - { - // arrange - var channelMock = CreateOpenChannel(); - var connectionMock = CreateOpenConnection(channelMock); - var runtime = CreateRuntime(_ => { }, new FakeConnectionProvider(connectionMock.Object)); - var transport = runtime.Transports.OfType().Single(); - - await transport.ConsumerManager.EnsureConnectedAsync(TestContext.Current.CancellationToken); - await transport.Dispatcher.EnsureConnectedAsync(TestContext.Current.CancellationToken); - var rented = await transport.Dispatcher.RentChannelAsync(TestContext.Current.CancellationToken); - await transport.Dispatcher.ReturnChannelAsync(rented); - - // act - dispose twice - await transport.DisposeAsync(); - await transport.DisposeAsync(); - - // assert - dispatcher is disposed and rejects further use - await Assert.ThrowsAsync(() => - transport.Dispatcher.RentChannelAsync(TestContext.Current.CancellationToken).AsTask()); - } - [Fact] public async Task DisposeAsync_Should_NotThrow_When_TransportNeverInitialized() { // arrange var transport = new RabbitMQMessagingTransport(_ => { }); - // act & assert - no exception thrown even though OnAfterInitialized never ran - await transport.DisposeAsync(); - } - - private static Mock CreateOpenChannel() - { - var channelMock = new Mock(); - channelMock.SetupGet(c => c.IsOpen).Returns(true); - return channelMock; - } + // act + var exception = await Record.ExceptionAsync(() => transport.DisposeAsync().AsTask()); - private static Mock CreateOpenConnection(Mock channelToReturn) - { - var connectionMock = new Mock(); - connectionMock.SetupGet(c => c.IsOpen).Returns(true); - connectionMock.SetupGet(c => c.ClientProvidedName).Returns("test-connection"); - connectionMock - .Setup(c => c.CreateChannelAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(channelToReturn.Object); - - return connectionMock; + // assert + Assert.Null(exception); } private static MessagingRuntime CreateRuntime(Action configure) @@ -551,29 +407,6 @@ private static MessagingRuntime CreateRuntime(Action con return runtime; } - private static MessagingRuntime CreateRuntime( - Action configure, - IRabbitMQConnectionProvider connectionProvider) - { - var services = new ServiceCollection(); - services.AddSingleton(new MessageRecorder()); - var builder = services.AddMessageBus(); - configure(builder); - var runtime = builder - .AddRabbitMQ(t => t.ConnectionProvider(_ => connectionProvider).AutoProvision(false)) - .BuildRuntime(); - return runtime; - } - - private sealed class FakeConnectionProvider(IConnection connection) : IRabbitMQConnectionProvider - { - public string Host => "localhost"; - public string VirtualHost => "/"; - public int Port => 5672; - - public ValueTask CreateAsync(CancellationToken cancellationToken) => new(connection); - } - public sealed class ProcessPaymentHandler(MessageRecorder recorder) : IEventRequestHandler { public ValueTask HandleAsync(ProcessPayment request, CancellationToken cancellationToken) diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Routing/RabbitMQReceiveTopologyConventionTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Routing/RabbitMQReceiveTopologyConventionTests.cs index c7c92975d02..16fa695cd08 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Routing/RabbitMQReceiveTopologyConventionTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Routing/RabbitMQReceiveTopologyConventionTests.cs @@ -125,7 +125,7 @@ public void DiscoverTopology_Should_NotConsultResolver_When_RouteIsReply() }) .BuildRuntime(); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; var replyExchange = runtime.Naming.GetPublishEndpointName(typeof(StockInfoResult)); // act @@ -136,11 +136,11 @@ public void DiscoverTopology_Should_NotConsultResolver_When_RouteIsReply() } [Fact] - public void DiscoverTopology_Should_MaterializeDurableAutoDeleteQueueWithExpiry_When_RouteIsReply() + public void DiscoverTopology_Should_MaterializeDurableExpiringQueue_When_RouteIsReply() { // arrange - // The generated reply endpoint sets IsTemporary, and the reply queue must go through the - // same durable, auto-delete plus queue expiry lifecycle path as any other temporary endpoint. + // The reply queue expires instead of being deleted with its last consumer, so it survives a + // consumer that re-attaches. var services = new ServiceCollection(); services.AddInMemorySagas(); var builder = services.AddMessageBus(); @@ -153,7 +153,7 @@ public void DiscoverTopology_Should_MaterializeDurableAutoDeleteQueueWithExpiry_ }) .BuildRuntime(); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; var replyEndpoint = transport.ReceiveEndpoints .OfType() .Single(e => e.Kind == ReceiveEndpointKind.Reply); @@ -164,8 +164,8 @@ public void DiscoverTopology_Should_MaterializeDurableAutoDeleteQueueWithExpiry_ // assert Assert.True(queue.Durable); Assert.False(queue.Exclusive); - Assert.True(queue.AutoDelete); - Assert.Equal(1800000, queue.Arguments["x-expires"]); + Assert.False(queue.AutoDelete); + Assert.Equal(new Dictionary { ["x-expires"] = 60000 }, queue.Arguments); } [Fact] @@ -245,7 +245,7 @@ private static (RabbitMQMessagingTopology Topology, string QueueName) ResolveCon var endpoint = transport.ReceiveEndpoints .OfType() .First(e => e.Kind == ReceiveEndpointKind.Default); - return ((RabbitMQMessagingTopology)transport.Topology, endpoint.Queue.Name); + return (transport.Topology, endpoint.Queue.Name); } private static MessagingRuntime CreateRuntime( diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQBusDefaultsTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQBusDefaultsTests.cs index 53d1885c7bd..00b948d5278 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQBusDefaultsTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQBusDefaultsTests.cs @@ -129,7 +129,7 @@ public void AddQueue_Should_CombineDefaultAndExplicitArguments_When_BothConfigur } [Fact] - public void AddQueue_Should_ApplyNoDefaults_When_NoDefaultsConfigured() + public void AddQueue_Should_LeaveQueueTypeUnspecified_When_NoDefaultsConfigured() { // arrange var (_, _, topology) = CreateTopology(_ => { }); @@ -137,10 +137,10 @@ public void AddQueue_Should_ApplyNoDefaults_When_NoDefaultsConfigured() // act var queue = topology.AddQueue(new RabbitMQQueueConfiguration { Name = "test-queue" }); - // assert - only standard defaults, no x-queue-type argument + // assert Assert.True(queue.Durable); Assert.False(queue.AutoDelete); - Assert.DoesNotContain("x-queue-type", queue.Arguments.Keys); + Assert.Empty(queue.Arguments); } [Fact] @@ -272,11 +272,13 @@ public void AddExchange_Should_NotOverrideArguments_When_ExplicitlySet() } [Fact] - public void ConfigureDefaults_Should_SkipQueueType_When_AutoDeleteIsTrue() + public void ConfigureDefaults_Should_SkipQueueType_When_QuorumDefaultConflictsWithAutoDelete() { - // arrange - auto-delete queues (e.g. reply queues) are incompatible with quorum + // arrange var (_, _, topology) = CreateTopology(t => t.ConfigureDefaults(d => d.Queue.QueueType = RabbitMQQueueType.Quorum)); + + // act var queue = topology.AddQueue(new RabbitMQQueueConfiguration { Name = "reply-queue", @@ -284,15 +286,17 @@ public void ConfigureDefaults_Should_SkipQueueType_When_AutoDeleteIsTrue() }); // assert - Assert.DoesNotContain("x-queue-type", queue.Arguments.Keys); + Assert.Empty(queue.Arguments); } [Fact] - public void ConfigureDefaults_Should_SkipQueueType_When_ExclusiveIsTrue() + public void ConfigureDefaults_Should_SkipQueueType_When_QuorumDefaultConflictsWithExclusive() { - // arrange - exclusive queues are incompatible with quorum + // arrange var (_, _, topology) = CreateTopology(t => t.ConfigureDefaults(d => d.Queue.QueueType = RabbitMQQueueType.Quorum)); + + // act var queue = topology.AddQueue(new RabbitMQQueueConfiguration { Name = "exclusive-queue", @@ -300,15 +304,17 @@ public void ConfigureDefaults_Should_SkipQueueType_When_ExclusiveIsTrue() }); // assert - Assert.DoesNotContain("x-queue-type", queue.Arguments.Keys); + Assert.Empty(queue.Arguments); } [Fact] - public void ConfigureDefaults_Should_SkipStreamQueueType_When_AutoDeleteIsTrue() + public void ConfigureDefaults_Should_SkipQueueType_When_StreamDefaultConflictsWithAutoDelete() { - // arrange - auto-delete queues are incompatible with stream + // arrange var (_, _, topology) = CreateTopology(t => t.ConfigureDefaults(d => d.Queue.QueueType = RabbitMQQueueType.Stream)); + + // act var queue = topology.AddQueue(new RabbitMQQueueConfiguration { Name = "reply-queue", @@ -316,14 +322,13 @@ public void ConfigureDefaults_Should_SkipStreamQueueType_When_AutoDeleteIsTrue() }); // assert - Assert.DoesNotContain("x-queue-type", queue.Arguments.Keys); + Assert.Empty(queue.Arguments); } [Fact] - public void ConfigureDefaults_Should_SkipAllDefaults_When_QueueIsIncompatible() + public void ConfigureDefaults_Should_OmitQuorumArguments_When_QueueIsAutoDelete() { - // arrange - both queue type and arguments are skipped for incompatible queues - // since default arguments may be queue-type-specific (e.g. x-delivery-limit) + // arrange var (_, _, topology) = CreateTopology(t => t.ConfigureDefaults(d => { @@ -331,15 +336,137 @@ public void ConfigureDefaults_Should_SkipAllDefaults_When_QueueIsIncompatible() d.Queue.Arguments["x-delivery-limit"] = 5; })); + // act var queue = topology.AddQueue(new RabbitMQQueueConfiguration { Name = "reply-queue", AutoDelete = true }); - // assert - both queue type and arguments are skipped - Assert.DoesNotContain("x-queue-type", queue.Arguments.Keys); - Assert.DoesNotContain("x-delivery-limit", queue.Arguments.Keys); + // assert + Assert.Empty(queue.Arguments); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ConfigureDefaults_Should_ApplyClassicDefaults_When_QueueIsNonDurable(bool useArgument) + { + // arrange + var (_, _, topology) = CreateTopology(t => + t.ConfigureDefaults(d => + { + if (useArgument) + { + d.Queue.Arguments["x-queue-type"] = RabbitMQQueueType.Classic; + } + else + { + d.Queue.QueueType = RabbitMQQueueType.Classic; + } + + d.Queue.Arguments["x-message-ttl"] = 1000; + d.Queue.Arguments["x-max-length"] = 10; + })); + + // act + var queue = topology.AddQueue(new RabbitMQQueueConfiguration + { + Name = "test-queue", + Durable = false, + Arguments = new Dictionary { ["x-message-ttl"] = 2000 } + }); + + // assert + Assert.False(queue.Durable); + Assert.Equal( + new Dictionary + { + ["x-max-length"] = 10, + ["x-message-ttl"] = 2000, + ["x-queue-type"] = RabbitMQQueueType.Classic + }, + queue.Arguments); + } + + [Fact] + public void ConfigureDefaults_Should_ApplyGenericDefaults_When_QueueIsNonDurableWithoutDefaultQueueType() + { + // arrange + var (_, _, topology) = CreateTopology(t => + t.ConfigureDefaults(d => + { + d.Queue.Durable = false; + d.Queue.Arguments["x-message-ttl"] = 1000; + d.Queue.Arguments["x-max-length"] = 10; + })); + + // act + var queue = topology.AddQueue(new RabbitMQQueueConfiguration + { + Name = "test-queue", + Arguments = new Dictionary { ["x-message-ttl"] = 2000 } + }); + + // assert + Assert.False(queue.Durable); + Assert.Equal( + new Dictionary + { + ["x-max-length"] = 10, + ["x-message-ttl"] = 2000 + }, + queue.Arguments); + } + + [Theory] + [InlineData(RabbitMQQueueType.Quorum, false)] + [InlineData(RabbitMQQueueType.Quorum, true)] + [InlineData(RabbitMQQueueType.Stream, false)] + [InlineData(RabbitMQQueueType.Stream, true)] + public void ConfigureDefaults_Should_SkipIncompatibleDefaults_When_QueueIsNonDurable( + string queueType, + bool useArgument) + { + // arrange + var (_, _, topology) = CreateTopology(t => + t.ConfigureDefaults(d => + { + if (useArgument) + { + d.Queue.Arguments["x-queue-type"] = queueType; + } + else + { + d.Queue.QueueType = queueType; + } + + d.Queue.Arguments["x-message-ttl"] = 1000; + d.Queue.Arguments["x-max-length"] = 10; + d.Queue.Arguments["x-delivery-limit"] = 5; + })); + + // act + var queue = topology.AddQueue(new RabbitMQQueueConfiguration + { + Name = "test-queue", + Durable = false, + Arguments = new Dictionary + { + ["x-message-ttl"] = 2000, + ["x-queue-type"] = RabbitMQQueueType.Classic + } + }); + + // assert + Assert.False(queue.Durable); + Assert.Equal( + new Dictionary + { + ["x-message-ttl"] = 2000, + ["x-queue-type"] = RabbitMQQueueType.Classic + }, + queue.Arguments); } [Fact] @@ -488,7 +615,7 @@ private static ( }) .BuildRuntime(); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; return (runtime, transport, topology); } } diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQExplicitTopologyTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQExplicitTopologyTests.cs index 6e3d4a0dcd3..62c421c07d4 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQExplicitTopologyTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQExplicitTopologyTests.cs @@ -446,7 +446,7 @@ private static ( }) .BuildRuntime(); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; return (runtime, transport, topology); } @@ -527,7 +527,7 @@ public void Describe_Should_ResolveProducerAndConsumerToSameEntity_When_Conventi b => b.AddConsumer(), t => t.BindImplicitly()); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; var publishExchangeName = transport.Naming.GetPublishEndpointName(typeof(OrderCreated)); // act: both paths are exercised during BuildRuntime above; inspect the result. @@ -591,7 +591,7 @@ public void Build_Should_Allow_When_ConsumedTypeHasExplicitQueueDestination() }) .BuildRuntime(); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; var endpoint = Assert.IsType( runtime.GetPublishEndpoint(runtime.GetMessageType(typeof(OrderCreated)))); diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQMessagingTopologyTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQMessagingTopologyTests.cs index c2c9965e5b1..aa9ed68d6f6 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQMessagingTopologyTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQMessagingTopologyTests.cs @@ -563,7 +563,7 @@ private static ( configure(builder); var runtime = builder.AddRabbitMQ(t => t.ConnectionProvider(_ => new StubConnectionProvider())).BuildRuntime(); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; return (runtime, transport, topology); } } diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQQueueProvisioningTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQQueueProvisioningTests.cs new file mode 100644 index 00000000000..98d53645aec --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQQueueProvisioningTests.cs @@ -0,0 +1,153 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.RabbitMQ.Tests.Helpers; +using RabbitMQ.Client; + +namespace Mocha.Transport.RabbitMQ.Tests.Topology; + +[Collection(RabbitMQCollections.Topology)] +public sealed class RabbitMQQueueProvisioningTests(RabbitMQFixture fixture) +{ + [Theory] + [InlineData(null, RabbitMQQueueType.Quorum)] + [InlineData(RabbitMQQueueType.Classic, RabbitMQQueueType.Classic)] + [InlineData(RabbitMQQueueType.Quorum, RabbitMQQueueType.Quorum)] + public async Task StartAsync_Should_UseExplicitOrBrokerQueueType_When_VirtualHostDefaultsToQuorum( + string? queueType, + string expectedType) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await fixture.InvokeCommandAsync( + ["rabbitmqctl", "update_vhost_metadata", vhost.VhostName, "--default-queue-type", "quorum"]); + + // act + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + var queue = t.DeclareQueue("orders"); + if (queueType is not null) + { + queue.QueueType(queueType); + } + }) + .BuildTestBusAsync(); + var output = await fixture.InvokeCommandAsync( + ["rabbitmqctl", "list_queues", "-p", vhost.VhostName, "name", "type", "--formatter", "json", "--quiet"]); + using var document = JsonDocument.Parse(output); + var actualType = document.RootElement.EnumerateArray() + .Single(queue => queue.GetProperty("name").GetString() == "orders").GetProperty("type").GetString(); + + // assert + Assert.Equal(expectedType, actualType); + } + + [Theory] + [InlineData(null)] + [InlineData(RabbitMQQueueType.Classic)] + public async Task StartAsync_Should_PreserveMessages_When_ExistingClassicQueueHasNoDeliveryLimit(string? queueType) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + new CreateChannelOptions(true, true), + TestContext.Current.CancellationToken); + await channel.QueueDeclareAsync( + "orders", + durable: true, + exclusive: false, + autoDelete: false, + cancellationToken: TestContext.Current.CancellationToken); + await channel.BasicPublishAsync( + "", + "orders", + true, + new BasicProperties(), + "retained"u8.ToArray(), + TestContext.Current.CancellationToken); + + // act + await using var bus = await new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + var queue = t.DeclareQueue("orders"); + if (queueType is not null) + { + queue.QueueType(queueType); + } + }) + .BuildTestBusAsync(); + var message = await channel.BasicGetAsync("orders", true, TestContext.Current.CancellationToken); + + // assert + Assert.Equal("retained", Encoding.UTF8.GetString(message!.Body.Span)); + Assert.Equal(0u, await channel.MessageCountAsync("orders", TestContext.Current.CancellationToken)); + } + + [Theory] + [InlineData(true, null)] + [InlineData(false, null)] + [InlineData(true, -1)] + [InlineData(false, -1)] + [InlineData(true, 5)] + [InlineData(false, 5)] + public async Task ProvisionAsync_Should_PreserveMessages_When_CombinedWithAutomaticProvisioning( + bool provisionBeforeStart, + int? deliveryLimit) + { + // arrange + await using var vhost = await fixture.CreateVhostAsync(); + await using var provider = new ServiceCollection() + .AddSingleton(vhost.ConnectionFactory) + .AddMessageBus() + .AddRabbitMQ(t => + { + var queue = t.DeclareQueue("orders").QueueType(RabbitMQQueueType.Quorum); + if (deliveryLimit is { } limit) + { + queue.MaxDeliveryLimit(limit); + } + }) + .Services.BuildServiceProvider(); + var runtime = (MessagingRuntime)provider.GetRequiredService(); + var transport = runtime.Transports.OfType().Single(); + var queue = transport.Topology.Queues.Single(queue => queue.Name == "orders"); + await using var connection = await vhost.ConnectionFactory.CreateConnectionAsync(TestContext.Current.CancellationToken); + await using var channel = await connection.CreateChannelAsync( + new CreateChannelOptions(true, true), + TestContext.Current.CancellationToken); + if (!provisionBeforeStart) + { + await runtime.StartAsync(TestContext.Current.CancellationToken); + } + + // act + await queue.ProvisionAsync(channel, connection, TestContext.Current.CancellationToken); + await channel.BasicPublishAsync( + "", + "orders", + true, + new BasicProperties(), + "order"u8.ToArray(), + TestContext.Current.CancellationToken); + if (provisionBeforeStart) + { + await runtime.StartAsync(TestContext.Current.CancellationToken); + } + + await queue.ProvisionAsync(channel, connection, TestContext.Current.CancellationToken); + var message = await channel.BasicGetAsync("orders", true, TestContext.Current.CancellationToken); + + // assert + Assert.True(channel.IsOpen); + Assert.NotNull(message); + Assert.Equal("order", Encoding.UTF8.GetString(message.Body.Span)); + Assert.Equal(0u, await channel.MessageCountAsync("orders", TestContext.Current.CancellationToken)); + } +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQReplyQueueTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQReplyQueueTests.cs index 5d396b441ea..e0f0af8d5d9 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQReplyQueueTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQReplyQueueTests.cs @@ -5,15 +5,34 @@ namespace Mocha.Transport.RabbitMQ.Tests.Topology; public class RabbitMQReplyQueueTests { - [Fact] - public void ReplyQueue_Should_BeDurableAutoDeleteWithExpiry_When_RequestHandlerRegistered() + [Theory] + [InlineData(1_000)] + [InlineData(600_000)] + public void ReplyQueue_Should_PreserveExplicitExpiry_When_TemporaryExpiryConfigured(int milliseconds) { // arrange - // RabbitMQ 4.3 denies non-durable, non-exclusive queues by default, so the per-instance - // reply queue must stay durable and rely on auto-delete plus a queue expiry. - var runtime = CreateRuntime(_ => { }); + var runtime = CreateRuntime(t => t.Endpoint("custom-replies") + .Kind(ReceiveEndpointKind.Reply) + .Temporary(TimeSpan.FromMilliseconds(milliseconds))); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + + // act + var queue = transport.Topology.Queues.Single(q => q.Name == "custom-replies"); + + // assert + Assert.Equal(milliseconds, queue.Arguments["x-expires"]); + } + + [Theory] + [InlineData(null)] + [InlineData(RabbitMQQueueType.Classic)] + [InlineData(RabbitMQQueueType.Stream)] + public void ReplyQueue_Should_LeaveQueueTypeUnspecified_When_DefaultIsNotQuorum(string? queueType) + { + // arrange + var runtime = CreateRuntime(t => t.ConfigureDefaults(d => d.Queue.QueueType = queueType)); + var transport = runtime.Transports.OfType().Single(); + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name.StartsWith("response-", StringComparison.Ordinal)); @@ -21,22 +40,36 @@ public void ReplyQueue_Should_BeDurableAutoDeleteWithExpiry_When_RequestHandlerR // assert Assert.True(queue.Durable); Assert.False(queue.Exclusive); - Assert.True(queue.AutoDelete); - Assert.Equal(1800000, queue.Arguments["x-expires"]); + Assert.False(queue.AutoDelete); + Assert.Equal(new Dictionary { ["x-expires"] = 60000 }, queue.Arguments); } - [Fact] - public void ReplyQueue_Should_StayClassic_When_QuorumDefaultsConfigured() + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ReplyQueue_Should_InheritOnlyReplicationSettings_When_QuorumDefaultsConfigured(bool useArgument) { // arrange - // Quorum queues reject auto-delete, so bus defaults must not reach the reply queue. var runtime = CreateRuntime(t => t.ConfigureDefaults(d => { - d.Queue.QueueType = RabbitMQQueueType.Quorum; + if (useArgument) + { + d.Queue.Arguments["x-queue-type"] = RabbitMQQueueType.Quorum; + } + else + { + d.Queue.QueueType = RabbitMQQueueType.Quorum; + } + + d.Queue.Durable = false; + d.Queue.AutoDelete = true; + d.Queue.Arguments["x-quorum-initial-group-size"] = 5; d.Queue.Arguments["x-delivery-limit"] = 5; + d.Queue.Arguments["x-message-ttl"] = 1000; + d.Queue.Arguments["x-expires"] = 1000; })); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; // act var queue = topology.Queues.Single(q => q.Name.StartsWith("response-", StringComparison.Ordinal)); @@ -44,9 +77,15 @@ public void ReplyQueue_Should_StayClassic_When_QuorumDefaultsConfigured() // assert Assert.True(queue.Durable); Assert.False(queue.Exclusive); - Assert.True(queue.AutoDelete); - Assert.Equal(["x-expires"], queue.Arguments.Keys); - Assert.Equal(1800000, queue.Arguments["x-expires"]); + Assert.False(queue.AutoDelete); + Assert.Equal( + new Dictionary + { + ["x-expires"] = 60000, + ["x-queue-type"] = RabbitMQQueueType.Quorum, + ["x-quorum-initial-group-size"] = 5 + }, + queue.Arguments); } private static MessagingRuntime CreateRuntime(Action configureTransport) diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQTopologyDescriptorTests.cs b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQTopologyDescriptorTests.cs index 99d79163b3d..fa7f8f95bbe 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQTopologyDescriptorTests.cs +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/Topology/RabbitMQTopologyDescriptorTests.cs @@ -468,7 +468,7 @@ private static ( }) .BuildRuntime(); var transport = runtime.Transports.OfType().Single(); - var topology = (RabbitMQMessagingTopology)transport.Topology; + var topology = transport.Topology; return (runtime, transport, topology); } } diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/__snapshots__/RabbitMQTransportTests.Describe_Should_ListExchangesQueuesAndBindings_When_EventHandlerRegistered.snap b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/__snapshots__/RabbitMQTransportTests.Describe_Should_ListExchangesQueuesAndBindings_When_EventHandlerRegistered.snap new file mode 100644 index 00000000000..d01f892ca77 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ.Tests/__snapshots__/RabbitMQTransportTests.Describe_Should_ListExchangesQueuesAndBindings_When_EventHandlerRegistered.snap @@ -0,0 +1,52 @@ +{ + "Schema": "rabbitmq", + "TransportType": "RabbitMQMessagingTransport", + "Entities": [ + { + "Kind": "exchange", + "Name": "mocha.test-helpers.order-created", + "AutoProvision": true, + "Origin": "convention" + }, + { + "Kind": "exchange", + "Name": "order-created", + "AutoProvision": true, + "Origin": "convention" + }, + { + "Kind": "queue", + "Name": "mocha.transport.rabbit-m-q.tests.order-created", + "AutoProvision": true, + "Origin": "endpoint" + }, + { + "Kind": "queue", + "Name": "mocha.transport.rabbit-m-q.tests.order-created_error", + "AutoProvision": true, + "Origin": "convention" + }, + { + "Kind": "queue", + "Name": "mocha.transport.rabbit-m-q.tests.order-created_skipped", + "AutoProvision": true, + "Origin": "convention" + } + ], + "Links": [ + { + "Kind": "bind", + "From": "rabbitmq://localhost:5672/e/mocha.test-helpers.order-created", + "To": "rabbitmq://localhost:5672/e/order-created", + "AutoProvision": true, + "Origin": "convention" + }, + { + "Kind": "bind", + "From": "rabbitmq://localhost:5672/e/order-created", + "To": "rabbitmq://localhost:5672/q/mocha.transport.rabbit-m-q.tests.order-created", + "AutoProvision": true, + "Origin": "convention" + } + ] +} diff --git a/src/Mocha/test/Mocha.Transport.RabbitMQ3.Tests/Mocha.Transport.RabbitMQ3.Tests.csproj b/src/Mocha/test/Mocha.Transport.RabbitMQ3.Tests/Mocha.Transport.RabbitMQ3.Tests.csproj index e1f5fc95053..c78458f57ed 100644 --- a/src/Mocha/test/Mocha.Transport.RabbitMQ3.Tests/Mocha.Transport.RabbitMQ3.Tests.csproj +++ b/src/Mocha/test/Mocha.Transport.RabbitMQ3.Tests/Mocha.Transport.RabbitMQ3.Tests.csproj @@ -26,6 +26,7 @@ + diff --git a/website/content/docs/mocha/hosting.md b/website/content/docs/mocha/hosting.md index a064b19794e..2f84a1b1210 100644 --- a/website/content/docs/mocha/hosting.md +++ b/website/content/docs/mocha/hosting.md @@ -50,7 +50,31 @@ app.MapHealthChecks("/health"); app.Run(); ``` -A `GET /health` request will now include the message bus status. If the bus cannot process and reply to the health request within the timeout, the check reports `Unhealthy`. +A `GET /health` request includes the message bus status. The check fails if any local transport is unhealthy or the bus does not answer the health request within the timeout. The `Data` of the health check entry lists each unhealthy transport by name with its reason. RabbitMQ transport health includes broker connectivity, consumer attachment, and capacity occupied by canceled handlers that have not exited. Normally busy handlers do not make the transport unhealthy. + +## Configure the health check + +`AddMessageBus()` takes optional parameters: + +| Parameter | Default | Description | +| --------------- | -------------- | ------------------------------------------------------------------------- | +| `endpoint` | `null` | Destination of the health request. `null` uses the bus's default routing. | +| `name` | `"MessageBus"` | Name of the health check entry. | +| `failureStatus` | `Unhealthy` | Status reported when the check fails. | +| `tags` | `["ready"]` | Tags of the health check. | +| `timeout` | 10 seconds | Time within which the bus must answer the health request. | + +```csharp +builder.Services + .AddHealthChecks() + .AddMessageBus( + name: "orders-bus", + failureStatus: HealthStatus.Degraded, + tags: ["ready", "messaging"], + timeout: TimeSpan.FromSeconds(5)); +``` + +The health request expires after the timeout. A request that was not handled by then is dropped when it is received. ## Target a specific endpoint @@ -62,7 +86,11 @@ builder.Services .AddMessageBus(new Uri("queue://my-service-health")); ``` -The health check is registered with the `"ready"` and `"live"` tags, so you can use tag-based filtering to separate readiness from liveness probes: +The endpoint URI selects the destination of the health request. Every local transport must still be healthy. + +## Readiness and liveness + +The message bus check has the `"ready"` tag unless you pass other `tags`. Use separate, explicitly filtered endpoints for readiness and liveness: ```csharp app.MapHealthChecks("/health/ready", new() @@ -76,6 +104,8 @@ app.MapHealthChecks("/health/live", new() }); ``` +The liveness endpoint runs only checks registered with the `"live"` tag. With none registered, it returns healthy while the application can serve the request. Broker outages and consumer recovery affect readiness without failing liveness. An unfiltered health endpoint includes the bus check and must not be used as a process liveness probe. + # Next steps - [Observability](./observability.md) - Add OpenTelemetry tracing and metrics to the bus. diff --git a/website/content/docs/mocha/observability.md b/website/content/docs/mocha/observability.md index 6026188e03c..c293bf60fed 100644 --- a/website/content/docs/mocha/observability.md +++ b/website/content/docs/mocha/observability.md @@ -5,7 +5,7 @@ description: "Instrument Mocha with OpenTelemetry for distributed tracing, metri Mocha integrates with OpenTelemetry to give you distributed traces, metrics, and structured logging across every stage of the messaging pipeline. When a message is dispatched, received, or consumed, Mocha emits spans and metrics that let you trace message flows end-to-end across services, measure throughput, and diagnose failures. -Observability is opt-in. Call `.AddInstrumentation()` on the bus builder to activate the built-in `OpenTelemetryDiagnosticObserver`. Without it, Mocha uses a no-op observer that adds zero overhead. +Tracing and metrics are opt-in through `.AddInstrumentation()`. Operational logs use `ILogger` independently of instrumentation. # Enable tracing and metrics @@ -192,6 +192,8 @@ When OpenTelemetry is configured with `AddOpenTelemetry().WithLogging()`, your ` The Aspire configuration block above already includes `builder.Logging.AddOpenTelemetry(...)`, which enables this behavior. No additional code is needed in your handlers. +Keep `IncludeScopes = true` in your OpenTelemetry logging configuration to export Mocha's message and connection correlation fields. + For more detail on .NET distributed tracing concepts and how `Activity` maps to spans, see [.NET Distributed Tracing Concepts](https://learn.microsoft.com/en-us/dotnet/core/diagnostics/distributed-tracing-concepts). # Next steps diff --git a/website/content/docs/mocha/reliability.md b/website/content/docs/mocha/reliability.md index 9f0140c8be8..b299740a05b 100644 --- a/website/content/docs/mocha/reliability.md +++ b/website/content/docs/mocha/reliability.md @@ -823,7 +823,71 @@ builder.Services The outbox processor is a background hosted service (`IHostedService`). When the EF Core interceptors detect a `SaveChanges` or transaction commit, they signal the processor through `IOutboxSignal`. Inside an ambient `TransactionScope`, the signal is raised when the ambient transaction completes. The processor reads pending `OutboxMessage` rows, deserializes the envelope, and dispatches each message through the bus's dispatch pipeline. -The `TimesSent` column tracks dispatch attempts. If dispatch fails, the processor retries on the next signal. Messages are deleted from the outbox table after successful dispatch. +The processor deletes a message from the outbox table after the transport has accepted it. When a dispatch does not succeed, the message stays in the table and is dispatched again as described in [Configure outbox retries](#configure-outbox-retries). + +When the host stops, a dispatch that is in progress is cancelled. If the transport has already accepted the message, the processor deletes the row before it stops. + +## Configure outbox retries + +Three options on `PostgresOutboxOptions` control what the processor does when a dispatch does not succeed: + +| Option | Default | Description | +| ----------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `DispatchTimeout` | 30 seconds | The longest time a single dispatch may take. When it elapses, the dispatch is cancelled and the message is released. The value must be greater than zero and at most 24 days. `Timeout.InfiniteTimeSpan` switches the timeout off. | +| `MaxAttempts` | 10 | The number of counted attempts after which a message is no longer dispatched. The value must be at least 1. | +| `MaxRetryDelay` | 10 minutes | The upper bound of the delay before a message is dispatched again after a counted attempt. The value must be greater than zero. | + +```csharp +builder.Services + .AddMessageBus() + .AddEntityFramework(p => + p.UsePostgresOutbox(o => + { + o.DispatchTimeout = TimeSpan.FromSeconds(10); + o.MaxAttempts = 20; + o.MaxRetryDelay = TimeSpan.FromMinutes(1); + })); +``` + +The `TimesSent` column holds the number of counted attempts of a message. How a dispatch ends decides whether it counts: + +| Outcome of the dispatch | Counted | What the processor does | Log level | +| ---------------------------------------------------------------------------------------------------------------------- | ------- | ------------------------------------------------------------------------- | -------------------------------------- | +| The transport accepted the message. | | Deletes the row. | | +| The transport threw `TransportUnavailableException`, the dispatch exceeded `DispatchTimeout`, or the host is stopping. | No | Releases the message unchanged, pauses, and dispatches the message again. | Warning, or Debug when the host stops. | +| The dispatch threw any other exception. | Yes | Increments `TimesSent` and delays the next attempt. | Error | +| The message cannot be read, its message type is unknown, or no endpoint matches its destination. | Yes | Increments `TimesSent` and delays the next attempt. | Warning | + +A message is eligible again one second after its first counted attempt. The delay doubles with every further counted attempt, up to `MaxRetryDelay`. + +A released message keeps its attempt count and its position in the outbox. After a release, the worker that dispatched the message pauses before it dispatches again. The pause starts at about 100 milliseconds, doubles with every further release up to 10 seconds, and starts over after a successful dispatch. + +The log entry of a dispatch that was released or threw carries the row ID (`Id`), the message ID (`MessageId`), the message type (`MessageType`), and the destination (`DestinationAddress`) as structured properties. + +When a message reaches `MaxAttempts` counted attempts, the processor writes a log entry at `Critical` level: + +```text +Message with ID 0197f3c2-5b7e-7000-8a1c-3d2e9f4b6a10 exhausted all 10 attempts and will not be retried. +``` + +The ID is the `Id` of the row. The row stays in the outbox table and is not dispatched again. To dispatch it again, set its `TimesSent` to a value below `MaxAttempts`. The processor picks the row up the next time it reads the outbox, for example when another message is written to it. + +## Configure outbox dispatch concurrency + +The Postgres outbox sends one stored message at a time by default. Set `MaxConcurrentMessages` to allow it to send several messages concurrently. With RabbitMQ, this lets the outbox send another message while an earlier one waits for the broker's [publisher confirmation](./transports/rabbitmq.md#publisher-confirms): + +```csharp +builder.Services + .AddMessageBus() + .AddEntityFramework(p => + p.UsePostgresOutbox(o => o.MaxConcurrentMessages = 32)); +``` + +The value must be greater than zero and applies to each outbox processor in each application instance. Each concurrent send needs a database connection. When increasing the limit, leave enough connections in the database pool for the rest of your application. + +Concurrent sends are independent. A send that fails, times out, or is released does not cancel or delay the other sends. + +With a value greater than one, messages may arrive in a different order from the order in which they were stored. A value of one keeps sends sequential within that processor, but retries and multiple application instances can still change delivery order. ## Skip the outbox for specific dispatches diff --git a/website/content/docs/mocha/transports/rabbitmq.md b/website/content/docs/mocha/transports/rabbitmq.md index 27d7652c829..283c38ca6b8 100644 --- a/website/content/docs/mocha/transports/rabbitmq.md +++ b/website/content/docs/mocha/transports/rabbitmq.md @@ -128,9 +128,89 @@ Send a POST request to `/orders` and check your application logs. You should see # Two connections per broker transport -Mocha opens two connections to the broker: one for consuming and one for dispatching. +Mocha opens two connections to RabbitMQ: one for receiving messages and one for sending them. -This design prevents back-pressure from slow consumers from blocking outbound message publishing. When a consumer processes messages slowly, the RabbitMQ client applies back-pressure on that connection. Without separation, a slow consumer could prevent your application from publishing new messages entirely. With separate connections, each direction operates independently. +In the RabbitMQ management UI, these connections appear as ` consumer` and ` dispatcher`. Mocha uses the host's service name, or the process name if no service name is available. Set `ClientProvidedName` on the connection factory to use a custom name. + +# Connection recovery + +Mocha reconnects automatically when it loses a connection to RabbitMQ. It retries with increasing delays of about 1 to 10 seconds between attempts, and abandons an attempt that the broker does not answer within 30 seconds. After reconnecting, Mocha restores message consumption and recreates any missing queues and bindings that it manages through [auto-provisioning](#control-auto-provisioning). + +RabbitMQ can also stop delivering messages to an individual receive endpoint while the connection remains open, for example if its queue is deleted or processing exceeds a broker timeout. Mocha retries that endpoint independently, with increasing delays of about 1 to 30 seconds between attempts. + +Every receive endpoint recovers on its own. A slow or unavailable queue, including the reply queue, does not prevent other endpoints from recovering. An attempt to resume an endpoint that RabbitMQ does not answer within 60 seconds is abandoned and retried. + +If RabbitMQ does not answer a request to open a channel within 60 seconds, Mocha replaces the connection. + +Mocha manages this recovery regardless of the connection factory's `AutomaticRecoveryEnabled` setting. + +For quorum queues, a surviving majority can elect a new leader and continue delivering messages after a node fails. Classic queues that are not replicated remain unavailable while their node is down. Configure your connection provider to reach surviving cluster nodes if the node serving its AMQP connection fails. + +## Heartbeats + +RabbitMQ detects a connection whose network path failed silently through heartbeats. The connection counts as lost after about three missed heartbeat intervals. With the client's default interval of 60 seconds that takes about three minutes, during which consumers receive nothing and publishes wait. Set `RequestedHeartbeat` on the connection factory to 10 to 20 seconds to detect such a failure within a minute: + +```csharp +builder.Services.AddSingleton( + new ConnectionFactory + { + Uri = new Uri(connectionString), + RequestedHeartbeat = TimeSpan.FromSeconds(15) + }); +``` + +The client also gives up on a single request that the broker does not answer within `ContinuationTimeout`, 20 seconds by default. + +## Recovery logs + +Logs under `Mocha.Transport.RabbitMQ` name the connection in every message. `ConnectionName` identifies the service, the transport, and the consumer or dispatcher role. Consumer events include the queue name, and loss events include the broker's reply code and reason. + +| Level | Events | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Information | `ConnectionReady`, `ConnectionUnblocked`, `ConnectionStopped`, `ConsumerAttached`, `ConsumerUnblocked`, `ConsumerDrainStarted`, `ConsumerStopped`, `PublishRetryScheduled`, `QueueRedeclared`, `SettlementSkipped` | +| Warning | `ConnectionLost`, `ConnectionAttemptFailed`, `ConnectionAborted`, `ConnectionBlocked`, `ConsumerChannelLost`, `ConsumerCancelled`, `ConsumerAttachFailed`, `ConsumerDrainTimedOut`, `PublishesStalled`, `PublishConfirmedSlowly`, `MessageReturned`, `QueueDeleteFailed` | +| Error | `ConnectionFailed`, `ConnectionAttemptFailed` and `ConsumerAttachFailed` for a failure that repeats on every attempt, `ConsumerRegistrationFailed`, `ConsumerBlockedByHandlers`, `PublishFailed`, `DeliveryFailed`, `ConsumerStopFailed`, `HandlerCancellationFailed`, `ConnectionCallbackFailed` | +| Critical | `ConsumerFailed` | +| Debug | `ConnectionAttemptStarted`, `ConnectionCloseFailed`, `PublishConfirmed`, `PublishCancelled`, `PublisherChannelOpened`, and `ConsumerChannelLost` when the whole connection was lost and no handler was running | + +Every failed attempt to recover a connection or a consumer is logged with its attempt number, the time since the loss, and the delay before the next attempt. Connectivity failures are logged as warnings. Failures that repeat on every attempt, such as refused credentials or a rejected declaration, are logged as errors. + +`PublishRetryScheduled` and `SettlementSkipped` explain a message that is delivered twice: the first names a publish that is attempted again after a connection loss, the second a handled message whose acknowledgement could not be sent because its channel closed. + +`PublishesStalled` reports once per 30 seconds how many publishes have waited for the broker for at least that long, and whether the connection is ready, blocked by the broker, or unavailable. + +`ConsumerBlockedByHandlers` means that every handler of a lost channel is still running 30 seconds after it was cancelled. The endpoint consumes again when one of them returns. + +## Startup connection timeout + +If RabbitMQ is unavailable when the application starts, Mocha retries the connection for up to two minutes by default. Startup fails if it cannot connect within that time. Invalid credentials, a failed TLS handshake, denied access to the configured virtual host, or queue and exchange declarations rejected by the broker cause startup to fail without waiting for the timeout. + +`ConnectTimeout` applies to startup only. After startup, Mocha keeps reconnecting until the application stops. + +Set `ConnectTimeout` to change how long startup can wait for RabbitMQ: + +```csharp +builder.Services + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(transport => transport.ConnectTimeout(TimeSpan.FromSeconds(30))); +``` + +## Message handling after a connection loss + +When a connection or channel is lost, Mocha requests cancellation through the `CancellationToken` passed to each affected handler. Handlers should observe this token and pass it to asynchronous operations such as database queries and HTTP requests. Cancellation does not forcibly stop a handler or undo work it has already completed. + +RabbitMQ can [deliver a message again](https://www.rabbitmq.com/docs/confirms#automatic-requeueing) if it did not receive confirmation that processing finished. This includes messages whose handlers saved their changes just before the connection failed, as well as messages that had not started processing. Make handlers safe to run more than once for the same message, or configure the [transactional inbox](../reliability.md#deduplicate-messages-with-the-transactional-inbox) to skip messages that were already processed. + +The endpoint's `MaxConcurrency` setting limits how many messages it processes at once. Work that started before the connection failed still counts towards this limit until it finishes, even after Mocha reconnects. + +If handlers ignore cancellation and continue running at the concurrency limit, the endpoint cannot start processing new messages and reports itself as unhealthy. Its queue and bindings are declared again in the meantime, so messages published to it are kept. If no handler has returned after 30 seconds, the endpoint logs `ConsumerBlockedByHandlers` as an error. Processing resumes when a handler finishes. Other endpoints recover and continue processing independently. + +## Graceful shutdown + +When the application shuts down, Mocha stops receiving new messages and waits for running handlers to finish. If the host's shutdown timeout expires, Mocha requests cancellation through the handlers' `CancellationToken`. + +RabbitMQ returns messages that have not been acknowledged to their queues when Mocha closes their channel. Closing RabbitMQ resources can continue after the shutdown timeout, so the transport may take longer than that timeout to stop. # How topology works @@ -149,14 +229,43 @@ graph LR **Request/reply:** The transport creates a temporary reply queue per service instance. The reply address is embedded in the request message so the responder knows where to send the reply. +When the transport's default queue type is quorum, generated reply queues also use quorum and inherit `x-quorum-initial-group-size` if configured. They remain durable, expire after one minute without consumers by default, and do not inherit message limits or other queue lifecycle defaults. + > [!WARNING] -> Messages published before the transport completes its Start phase may be lost if no queue is bound to the exchange yet. During deployment, ensure consuming services start before publishing services, or use [publisher confirms](https://www.rabbitmq.com/docs/reliability#publisher-confirms) to detect lost messages. -> -> If a message is published to an exchange with no bound queue - for example, when no consumer has started - that message is dropped. Mocha auto-provisions topology, but the window between exchange creation and queue binding is a real operational risk. +> RabbitMQ drops a published message if no queue binding matches it, even when the publish call succeeds. When Mocha creates your queues and bindings at startup, start the receiving services before services that publish to them. ## Publisher confirms -Mocha's RabbitMQ transport does not use publisher confirms on dispatch. A publish completes once the message is handed to the broker, and a message the broker cannot route is dropped. See the [RabbitMQ Reliability Guide](https://www.rabbitmq.com/docs/reliability) for a full treatment of delivery guarantees. +A publisher confirmation is RabbitMQ's acknowledgement that it accepted a message. Every send and publish through the RabbitMQ transport waits for this confirmation. It does not mean that an application handler has processed the message. + +If RabbitMQ rejects a message, the call throws `PublishException`. This can happen when a queue is full and configured to reject new messages with `x-overflow: reject-publish`. + +If the connection is unavailable or fails before the confirmation arrives, Mocha waits for the next connection and sends the message again, with delays of about 250 milliseconds to 5 seconds between attempts. The message may already have reached RabbitMQ before the failure, so this retry can produce a duplicate. Broker rejections and messages that exceed a broker limit are reported to the caller without retrying. + +A send or publish waits for its confirmation for up to two minutes by default, including the wait for a connection and every retry. After that it throws `TransportUnavailableException`. Set `PublishTimeout` to change the limit, or pass `Timeout.InfiniteTimeSpan` to wait until the caller cancels: + +```csharp +builder.Services + .AddMessageBus() + .AddEventHandler() + .AddRabbitMQ(transport => transport.PublishTimeout(TimeSpan.FromSeconds(30))); +``` + +RabbitMQ blocks publishing connections while a memory or disk alarm is active. Sends and publishes then wait for their confirmation, up to the publish timeout. Mocha keeps the connection during the alarm and does not send the waiting messages again. + +Each send or publish uses its own channel until RabbitMQ answered, so a message that RabbitMQ rejects does not affect other messages. The transport allows up to 100 sends and publishes in flight per transport instance, or fewer if the broker allows fewer channels per connection. Additional calls wait until one completes. Awaiting each call before starting the next sends messages one at a time. The Postgres outbox has a separate setting for [dispatching messages concurrently](../reliability.md#configure-outbox-dispatch-concurrency). + +A request is sent only while the reply queue of the service instance exists and is consumed. RabbitMQ drops a reply whose requester has stopped in the meantime. + +### Missing queues and bindings + +When you send directly to a queue that no longer exists, Mocha restores the queue and its configured bindings and retries once if auto-provisioning is enabled and the queue is not exclusive. Otherwise, or if the queue is still missing after the retry, the send throws `InvalidOperationException`. If RabbitMQ refuses to declare the queue, the send fails with the broker's reason. + +Publishing to an exchange behaves differently: if no queue binding matches the message, RabbitMQ drops it and the publish can still complete successfully. Publisher confirmations do not detect missing subscribers. + +If Mocha cannot forward a failed message to its error queue, it returns the original message to its source queue for redelivery. + +See the [RabbitMQ Reliability Guide](https://www.rabbitmq.com/docs/reliability) for a full treatment of delivery guarantees. ## Default topology for event handlers @@ -187,9 +296,8 @@ builder.Services { transport.ConfigureDefaults(defaults => { - // All queues will be quorum with a delivery limit of 5 + // Durable queues use quorum defaults.Queue.QueueType = RabbitMQQueueType.Quorum; - defaults.Queue.Arguments["x-delivery-limit"] = 5; // All exchanges will use topic routing defaults.Exchange.Type = RabbitMQExchangeType.Topic; @@ -248,6 +356,42 @@ Defaults never override explicitly configured values. If you declare a queue wit # Configure queues +When no queue type is configured, Mocha omits `x-queue-type` and RabbitMQ selects the type using its broker or virtual-host default. Use `Queue("name").Quorum()`, `DeclareQueue("name").QueueType(...)`, or `ConfigureDefaults` to select a type explicitly. + +## Quorum delivery limits + +Mocha leaves delivery limits to RabbitMQ unless you configure `x-delivery-limit` explicitly. RabbitMQ 4 defaults quorum queues to a delivery limit of 20; RabbitMQ 3 defaults to unlimited deliveries. + +A finite delivery limit can cause RabbitMQ to discard or dead-letter messages after repeated connection or channel failures. Without a configured dead-letter exchange, messages that exceed the limit are discarded. Mocha's fault queue does not automatically capture messages discarded by the broker. See [RabbitMQ's poison message handling](https://www.rabbitmq.com/docs/quorum-queues#poison-message-handling) for how delivery limits apply. + +To disable the limit for a quorum queue: + +```csharp +transport.DeclareQueue("orders") + .QueueType(RabbitMQQueueType.Quorum) + .MaxDeliveryLimit(-1); +``` + +To configure unlimited deliveries for application queues through transport defaults: + +```csharp +transport.ConfigureDefaults(defaults => +{ + defaults.Queue.QueueType = RabbitMQQueueType.Quorum; + defaults.Queue.Arguments["x-delivery-limit"] = -1; +}); +``` + +On RabbitMQ 3, an explicit `-1` is omitted from the declaration because that broker version already defaults to unlimited deliveries. Broker policies can also affect the effective delivery limit. + +When auto-provisioning is disabled, configure the queue type and delivery limit on the broker yourself. Mocha does not inspect existing queue settings through the management API. + +### Existing queues + +Leaving the queue type and delivery limit unspecified preserves declarations that use broker defaults. If you explicitly change arguments that RabbitMQ checks for equivalence, an existing queue can reject the declaration and fail startup. Use a broker policy where supported, drain and recreate the queue, or migrate to a new queue; Mocha does not delete or replace existing queues. + +## Queue configuration + Use `transport.Queue("name")` when you need to customize a RabbitMQ queue. The queue builder is the primary API for queue names, queue type, arguments, handler assignment, source bindings, prefetch, concurrency, fault queues, and skipped queues. ```csharp @@ -289,6 +433,8 @@ transport.Queue("eu-orders") `BindExplicitly()` on the queue suppresses convention-derived exchange bindings for that queue, so only the `BindFrom(...)` sources are used. +An [exclusive queue](https://www.rabbitmq.com/docs/queues#exclusive-queues), configured with `DeclareQueue("name").Exclusive()`, can only be consumed through the connection that created it. RabbitMQ deletes the queue and its messages when that connection closes. With auto-provisioning enabled, Mocha recreates the queue and its bindings after reconnecting. An application using another connection cannot consume from the same exclusive queue; RabbitMQ rejects the attempt with `405 RESOURCE_LOCKED`. + # Declare topology resources Mocha auto-provisions topology from registered handlers and queue builders by default. @@ -342,6 +488,8 @@ RabbitMQ 4.3 and later deny non-durable, non-exclusive queues by default. Tempor If the queue is already explicitly declared without auto-delete, for example through `DeclareQueue(...)`, startup fails with an explicit configuration error instead of silently ignoring `Temporary()`. +For request/reply messaging, Mocha creates a separate queue to receive responses. This queue remains available during a brief disconnection, so replies can wait for Mocha to reconnect. By default, RabbitMQ deletes it after it has had no consumers for one minute. Mocha also deletes this automatically created queue during graceful shutdown. + # Control auto-provisioning By default, the transport auto-provisions all topology resources (exchanges, queues, bindings) on the broker at startup. In production environments where infrastructure is managed externally - for example by Terraform, Ansible, or the [RabbitMQ Messaging Topology Operator](https://www.rabbitmq.com/kubernetes/operator/install-topology-operator) on Kubernetes - you can disable auto-provisioning so the transport expects resources to already exist. @@ -480,7 +628,7 @@ transport.Handler() For full control over the queue name, queue type, source bindings, and handler assignment, use `Queue("name")` instead. -**MaxPrefetch** controls how many unacknowledged messages RabbitMQ delivers to the consumer at once. Default: `100`. Lower values reduce memory pressure under high load. Higher values improve throughput for fast handlers. +**MaxPrefetch** controls how many messages RabbitMQ can deliver before Mocha acknowledges them, including messages being processed and messages waiting for a handler. Default: `100`. Setting it to `0` removes this prefetch limit. Lower values reduce memory pressure under high load. Higher values improve throughput for fast handlers. **MaxConcurrency** controls how many messages the endpoint processes in parallel. Set this based on your handler's throughput characteristics. @@ -506,6 +654,10 @@ builder.Services }); ``` +## Batch handler timeouts + +For batch handlers, messages wait until the batch fills or its `BatchTimeout` expires before processing starts. If RabbitMQ enforces a [delivery acknowledgement timeout](https://www.rabbitmq.com/docs/consumers#acknowledgement-timeout), configure it to allow enough time for both this wait and processing the batch. When the broker's timeout expires, RabbitMQ interrupts consumption and returns the messages to the queue for redelivery. + # Auto-provisioned resource naming | Resource | Naming convention | Created when |