From cfb5f00f78a4c557c90b29b3e47bfae85e9f1a3a Mon Sep 17 00:00:00 2001 From: Petr Onderka Date: Wed, 25 Feb 2026 17:47:44 +0100 Subject: [PATCH 1/3] Propagate BackgroundService exceptions from the host --- .../src/Internal/Host.cs | 32 ++- .../src/Internal/HostingLoggerExtensions.cs | 13 +- .../src/Internal/LoggerEventIds.cs | 1 + ...BackgroundServiceExceptionBehaviorTests.cs | 28 -- .../BackgroundServiceExceptionTests.cs | 244 ++++++++++++++++++ 5 files changed, 285 insertions(+), 33 deletions(-) delete mode 100644 src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionBehaviorTests.cs create mode 100644 src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs diff --git a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs index 12066ef8814d27..1afda3579f578e 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs @@ -30,6 +30,7 @@ internal sealed class Host : IHost, IAsyncDisposable private IEnumerable? _hostedLifecycleServices; private bool _hostStarting; private bool _hostStopped; + private readonly List _backgroundServiceExceptions = new(); public Host(IServiceProvider services, IHostEnvironment hostEnvironment, @@ -142,7 +143,7 @@ await ForeachService(_hostedLifecycleServices, cancellationToken, concurrent, ab // Exceptions in StartedAsync cause startup to be aborted. LogAndRethrow(); - // Call IHostApplicationLifetime.Started + // Cancel IHostApplicationLifetime.Started // This catches all exceptions and does not re-throw. _applicationLifetime.NotifyStarted(); @@ -197,6 +198,10 @@ private async Task TryExecuteBackgroundServiceAsync(BackgroundService background if (_options.BackgroundServiceExceptionBehavior == BackgroundServiceExceptionBehavior.StopHost) { _logger.BackgroundServiceStoppingHost(ex); + lock (_backgroundServiceExceptions) + { + _backgroundServiceExceptions.Add(ExceptionDispatchInfo.Capture(ex)); + } // This catches all exceptions and does not re-throw. _applicationLifetime.StopApplication(); @@ -231,7 +236,7 @@ public async Task StopAsync(CancellationToken cancellationToken = default) if (!_hostStarting) // Started? { - // Call IHostApplicationLifetime.ApplicationStopping. + // Cancel IHostApplicationLifetime.ApplicationStopping. // This catches all exceptions and does not re-throw. _applicationLifetime.StopApplication(); } @@ -251,7 +256,7 @@ await ForeachService(reversedLifetimeServices, cancellationToken, concurrent, ab (service, token) => service.StoppingAsync(token)).ConfigureAwait(false); } - // Call IHostApplicationLifetime.ApplicationStopping. + // Cancel IHostApplicationLifetime.ApplicationStopping. // This catches all exceptions and does not re-throw. _applicationLifetime.StopApplication(); @@ -267,7 +272,7 @@ await ForeachService(reversedLifetimeServices, cancellationToken, concurrent, ab } } - // Call IHostApplicationLifetime.Stopped + // Cancel IHostApplicationLifetime.Stopped // This catches all exceptions and does not re-throw. _applicationLifetime.NotifyStopped(); @@ -302,6 +307,25 @@ await ForeachService(reversedLifetimeServices, cancellationToken, concurrent, ab } _logger.Stopped(); + + // If background services faulted and caused the host to stop, rethrow the exceptions + // so they propagate and cause a non-zero exit code. + lock (_backgroundServiceExceptions) + { + if (_backgroundServiceExceptions.Count == 1) + { + _logger.BackgroundServiceExceptionsPropagating(_backgroundServiceExceptions[0].SourceException); + _backgroundServiceExceptions[0].Throw(); + } + else if (_backgroundServiceExceptions.Count > 1) + { + var aggregateException = new AggregateException( + "One or more background services threw an exception.", + _backgroundServiceExceptions.Select(edi => edi.SourceException)); + _logger.BackgroundServiceExceptionsPropagating(aggregateException); + throw aggregateException; + } + } } private static async Task ForeachService( diff --git a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/HostingLoggerExtensions.cs b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/HostingLoggerExtensions.cs index 71354c63819cac..4d7a2b3d0756b3 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/HostingLoggerExtensions.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/HostingLoggerExtensions.cs @@ -69,7 +69,7 @@ public static void Stopped(this ILogger logger) } } - public static void StoppedWithException(this ILogger logger, Exception? ex) + public static void StoppedWithException(this ILogger logger, Exception ex) { if (logger.IsEnabled(LogLevel.Debug)) { @@ -112,5 +112,16 @@ public static void HostedServiceStartupFaulted(this ILogger logger, Exception? e message: "Hosting failed to start"); } } + + public static void BackgroundServiceExceptionsPropagating(this ILogger logger, Exception ex) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug( + eventId: LoggerEventIds.BackgroundServiceExceptionsPropagating, + exception: ex, + message: "Propagating background service exceptions"); + } + } } } diff --git a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/LoggerEventIds.cs b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/LoggerEventIds.cs index 64666cd09a0097..7af886965ec2a4 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/LoggerEventIds.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/LoggerEventIds.cs @@ -18,5 +18,6 @@ internal static class LoggerEventIds public static readonly EventId BackgroundServiceFaulted = new EventId(9, nameof(BackgroundServiceFaulted)); public static readonly EventId BackgroundServiceStoppingHost = new EventId(10, nameof(BackgroundServiceStoppingHost)); public static readonly EventId HostedServiceStartupFaulted = new EventId(11, nameof(HostedServiceStartupFaulted)); + public static readonly EventId BackgroundServiceExceptionsPropagating = new EventId(12, nameof(BackgroundServiceExceptionsPropagating)); } } diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionBehaviorTests.cs b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionBehaviorTests.cs deleted file mode 100644 index e925cbb343464c..00000000000000 --- a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionBehaviorTests.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Xunit; - -namespace Microsoft.Extensions.Hosting.Tests -{ - public class BackgroundServiceExceptionBehaviorTests - { - [Fact] - public void EnumValues_HaveExpectedValues() - { - Assert.Equal(0, (int)BackgroundServiceExceptionBehavior.StopHost); - Assert.Equal(1, (int)BackgroundServiceExceptionBehavior.Ignore); - } - - [Fact] - public void CanCompareValues() - { - var stopHost = BackgroundServiceExceptionBehavior.StopHost; - var ignore = BackgroundServiceExceptionBehavior.Ignore; - - Assert.True(stopHost == BackgroundServiceExceptionBehavior.StopHost); - Assert.True(ignore == BackgroundServiceExceptionBehavior.Ignore); - Assert.False(stopHost == ignore); - } - } -} diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs new file mode 100644 index 00000000000000..a1475254fe12a9 --- /dev/null +++ b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs @@ -0,0 +1,244 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Microsoft.Extensions.Hosting.Tests +{ + public class BackgroundServiceExceptionTests + { + /// + /// Tests that when a BackgroundService throws an exception synchronously (without await), + /// the host propagates the exception. + /// + [Fact] + public async Task BackgroundService_SynchronousException_ThrowsException() + { + var builder = new HostBuilder() + .ConfigureServices(services => + { + services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + }); + services.AddHostedService(); + }); + + await Assert.ThrowsAsync(async () => + { + await builder.Build().RunAsync(); + }); + } + + /// + /// Tests that when a BackgroundService throws an exception asynchronously (after an await), + /// the host propagates the exception. + /// This is the main issue that was reported in GitHub issue #67146. + /// + [Fact] + public async Task BackgroundService_AsynchronousException_ThrowsException() + { + var builder = new HostBuilder() + .ConfigureServices(services => + { + services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + }); + services.AddHostedService(); + }); + + await Assert.ThrowsAsync(async () => + { + await builder.Build().RunAsync(); + }); + } + + /// + /// Tests that when a BackgroundService throws an exception asynchronously, + /// StopAsync propagates the exception when StopHost behavior is configured. + /// + [Fact] + public async Task BackgroundService_AsynchronousException_StopAsync_ThrowsException() + { + var builder = new HostBuilder() + .ConfigureServices(services => + { + services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + }); + services.AddHostedService(); + }); + + var host = builder.Build(); + await host.StartAsync(); + + // Wait for the background service to fail + await Task.Delay(TimeSpan.FromMilliseconds(500)); + + await Assert.ThrowsAsync(async () => + { + await host.StopAsync(); + }); + } + + /// + /// Tests that when multiple BackgroundServices throw exceptions, + /// the host aggregates them into an AggregateException. + /// + [Fact] + public async Task BackgroundService_MultipleExceptions_ThrowsAggregateException() + { + var builder = new HostBuilder() + .ConfigureServices(services => + { + services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + }); + services.AddHostedService(); + services.AddHostedService(); + services.AddHostedService(); + }); + + var aggregateException = await Assert.ThrowsAsync(async () => + { + await builder.Build().RunAsync(); + }); + + Assert.Equal(3, aggregateException.InnerExceptions.Count); + + Assert.All(aggregateException.InnerExceptions, ex => + Assert.IsType(ex)); + } + + /// + /// Tests that when a BackgroundService throws an exception with Ignore behavior, + /// the host does not throw and continues to run until stopped. + /// + [Fact] + public async Task BackgroundService_IgnoreException_DoesNotThrow() + { + var builder = new HostBuilder() + .ConfigureServices(services => + { + services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.Ignore; + }); + services.AddHostedService(); + services.AddHostedService(); + }); + + await builder.Build().RunAsync(); + } + + /// + /// Tests that when a BackgroundService is configured to Ignore exceptions, + /// the host does not throw even when the service fails. + /// + [Fact] + public async Task BackgroundService_IgnoreException_StopAsync_DoesNotThrow() + { + var builder = new HostBuilder() + .ConfigureServices(services => + { + services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.Ignore; + options.ShutdownTimeout = TimeSpan.FromSeconds(1); + }); + services.AddHostedService(); + }); + + var host = builder.Build(); + await host.StartAsync(); + + // Wait a bit for the background service to fail + await Task.Delay(TimeSpan.FromSeconds(2)); + + await host.StopAsync(); + } + + /// + /// Tests that when a BackgroundService completes successfully, + /// the host does not throw an exception. + /// + [Fact] + public async Task BackgroundService_SuccessfulCompletion_DoesNotThrow() + { + var builder = new HostBuilder() + .ConfigureServices(services => + { + services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + }); + services.AddHostedService(); + }); + + await builder.Build().RunAsync(); + } + + private class SynchronousFailureService : BackgroundService + { + protected override Task ExecuteAsync(CancellationToken stoppingToken) + { + // Throw synchronously (no await before the exception) + throw new InvalidOperationException("Synchronous failure"); + } + } + + private class AsynchronousFailureService : BackgroundService + { + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + // Await before throwing to make the exception asynchronous + await Task.Delay(TimeSpan.FromMilliseconds(100), stoppingToken); + throw new InvalidOperationException("Asynchronous failure"); + } + } + + private class SecondAsynchronousFailureService : BackgroundService + { + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + // Ignore the cancellation token to ensure this service throws even if the host is trying to shut down + await Task.Delay(TimeSpan.FromMilliseconds(150)); + throw new InvalidOperationException("Second asynchronous failure"); + } + } + + private class ThirdAsynchronousFailureService : BackgroundService + { + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + // Ignore the cancellation token to ensure this service throws even if the host is trying to shut down + await Task.Delay(TimeSpan.FromMilliseconds(200)); + throw new InvalidOperationException("Third asynchronous failure"); + } + } + + private class SuccessfulService : BackgroundService + { + private readonly IHostApplicationLifetime _lifetime; + + public SuccessfulService(IHostApplicationLifetime lifetime) + { + _lifetime = lifetime; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await Task.Delay(TimeSpan.FromMilliseconds(100), stoppingToken); + // Exit normally without throwing - signal the host to stop + _lifetime.StopApplication(); + } + } + } +} From 74d5fa30741aceeae3936be375a3f9a3bb800113 Mon Sep 17 00:00:00 2001 From: Petr Onderka Date: Wed, 25 Feb 2026 18:03:09 +0100 Subject: [PATCH 2/3] Shorter delays --- .../tests/UnitTests/BackgroundServiceExceptionTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs index a1475254fe12a9..834b8c1363c797 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs @@ -79,7 +79,7 @@ public async Task BackgroundService_AsynchronousException_StopAsync_ThrowsExcept await host.StartAsync(); // Wait for the background service to fail - await Task.Delay(TimeSpan.FromMilliseconds(500)); + await Task.Delay(TimeSpan.FromMilliseconds(200)); await Assert.ThrowsAsync(async () => { @@ -160,7 +160,7 @@ public async Task BackgroundService_IgnoreException_StopAsync_DoesNotThrow() await host.StartAsync(); // Wait a bit for the background service to fail - await Task.Delay(TimeSpan.FromSeconds(2)); + await Task.Delay(TimeSpan.FromMilliseconds(200)); await host.StopAsync(); } From 3469239e4bc543e33e617ac2efd30b075549c801 Mon Sep 17 00:00:00 2001 From: Petr Onderka Date: Thu, 26 Feb 2026 17:43:58 +0100 Subject: [PATCH 3/3] Ignore stopping token in all concurrently stopping services --- .../tests/UnitTests/BackgroundServiceExceptionTests.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs index 834b8c1363c797..082cbdbc277adb 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs @@ -113,7 +113,7 @@ public async Task BackgroundService_MultipleExceptions_ThrowsAggregateException( Assert.Equal(3, aggregateException.InnerExceptions.Count); - Assert.All(aggregateException.InnerExceptions, ex => + Assert.All(aggregateException.InnerExceptions, ex => Assert.IsType(ex)); } @@ -199,7 +199,8 @@ private class AsynchronousFailureService : BackgroundService protected override async Task ExecuteAsync(CancellationToken stoppingToken) { // Await before throwing to make the exception asynchronous - await Task.Delay(TimeSpan.FromMilliseconds(100), stoppingToken); + // Ignore the cancellation token to ensure this service throws even if the host is trying to shut down + await Task.Delay(TimeSpan.FromMilliseconds(100)); throw new InvalidOperationException("Asynchronous failure"); } }