Propagate BackgroundService exceptions from the host

src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs

@@ -30,6 +30,7 @@ internal sealed class Host : IHost, IAsyncDisposable
         private IEnumerable<IHostedLifecycleService>? _hostedLifecycleServices;
         private bool _hostStarting;
         private bool _hostStopped;
+        private readonly List<ExceptionDispatchInfo> _backgroundServiceExceptions = new();
 
         public Host(IServiceProvider services,
                     IHostEnvironment hostEnvironment,
@@ -142,7 +143,7 @@ await ForeachService(_hostedLifecycleServices, cancellationToken, concurrent, ab
                 // Exceptions in StartedAsync cause startup to be aborted.
                 LogAndRethrow();
 
-                // Call IHostApplicationLifetime.Started
+                // Cancel IHostApplicationLifetime.Started
                 // This catches all exceptions and does not re-throw.
                 _applicationLifetime.NotifyStarted();
 
@@ -197,6 +198,10 @@ private async Task TryExecuteBackgroundServiceAsync(BackgroundService background
                 if (_options.BackgroundServiceExceptionBehavior == BackgroundServiceExceptionBehavior.StopHost)
                 {
                     _logger.BackgroundServiceStoppingHost(ex);
+                    lock (_backgroundServiceExceptions)
+                    {
+                        _backgroundServiceExceptions.Add(ExceptionDispatchInfo.Capture(ex));
+                    }
 
                     // This catches all exceptions and does not re-throw.
                     _applicationLifetime.StopApplication();
@@ -231,7 +236,7 @@ public async Task StopAsync(CancellationToken cancellationToken = default)
                 if (!_hostStarting) // Started?
                 {
 
-                    // Call IHostApplicationLifetime.ApplicationStopping.
+                    // Cancel IHostApplicationLifetime.ApplicationStopping.
                     // This catches all exceptions and does not re-throw.
                     _applicationLifetime.StopApplication();
                 }
@@ -251,7 +256,7 @@ await ForeachService(reversedLifetimeServices, cancellationToken, concurrent, ab
                             (service, token) => service.StoppingAsync(token)).ConfigureAwait(false);
                     }
 
-                    // Call IHostApplicationLifetime.ApplicationStopping.
+                    // Cancel IHostApplicationLifetime.ApplicationStopping.
                     // This catches all exceptions and does not re-throw.
                     _applicationLifetime.StopApplication();
 
@@ -267,7 +272,7 @@ await ForeachService(reversedLifetimeServices, cancellationToken, concurrent, ab
                     }
                 }
 
-                // Call IHostApplicationLifetime.Stopped
+                // Cancel IHostApplicationLifetime.Stopped
                 // This catches all exceptions and does not re-throw.
                 _applicationLifetime.NotifyStopped();
 
@@ -302,6 +307,25 @@ await ForeachService(reversedLifetimeServices, cancellationToken, concurrent, ab
             }
 
             _logger.Stopped();
+
+            // If background services faulted and caused the host to stop, rethrow the exceptions
+            // so they propagate and cause a non-zero exit code.
+            lock (_backgroundServiceExceptions)
+            {
+                if (_backgroundServiceExceptions.Count == 1)
+                {
+                    _logger.BackgroundServiceExceptionsPropagating(_backgroundServiceExceptions[0].SourceException);
+                    _backgroundServiceExceptions[0].Throw();
+                }
+                else if (_backgroundServiceExceptions.Count > 1)
+                {
+                    var aggregateException = new AggregateException(
+                        "One or more background services threw an exception.",
+                        _backgroundServiceExceptions.Select(edi => edi.SourceException));
+                    _logger.BackgroundServiceExceptionsPropagating(aggregateException);
+                    throw aggregateException;
+                }
+            }
         }
 
         private static async Task ForeachService<T>(

src/libraries/Microsoft.Extensions.Hosting/src/Internal/HostingLoggerExtensions.cs

@@ -69,7 +69,7 @@ public static void Stopped(this ILogger logger)
             }
         }
 
-        public static void StoppedWithException(this ILogger logger, Exception? ex)
+        public static void StoppedWithException(this ILogger logger, Exception ex)
         {
             if (logger.IsEnabled(LogLevel.Debug))
             {
@@ -112,5 +112,16 @@ public static void HostedServiceStartupFaulted(this ILogger logger, Exception? e
                    message: "Hosting failed to start");
             }
         }
+
+        public static void BackgroundServiceExceptionsPropagating(this ILogger logger, Exception ex)
+        {
+            if (logger.IsEnabled(LogLevel.Debug))
+            {
+                logger.LogDebug(
+                    eventId: LoggerEventIds.BackgroundServiceExceptionsPropagating,
+                    exception: ex,
+                    message: "Propagating background service exceptions");
+            }
+        }
     }
 }

src/libraries/Microsoft.Extensions.Hosting/src/Internal/LoggerEventIds.cs

@@ -18,5 +18,6 @@ internal static class LoggerEventIds
         public static readonly EventId BackgroundServiceFaulted = new EventId(9, nameof(BackgroundServiceFaulted));
         public static readonly EventId BackgroundServiceStoppingHost = new EventId(10, nameof(BackgroundServiceStoppingHost));
         public static readonly EventId HostedServiceStartupFaulted = new EventId(11, nameof(HostedServiceStartupFaulted));
+        public static readonly EventId BackgroundServiceExceptionsPropagating = new EventId(12, nameof(BackgroundServiceExceptionsPropagating));
     }
 }

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs

@@ -0,0 +1,245 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Xunit;
+
+namespace Microsoft.Extensions.Hosting.Tests
+{
+    public class BackgroundServiceExceptionTests
+    {
+        /// <summary>
+        /// Tests that when a BackgroundService throws an exception synchronously (without await),
+        /// the host propagates the exception.
+        /// </summary>
+        [Fact]
+        public async Task BackgroundService_SynchronousException_ThrowsException()
+        {
+            var builder = new HostBuilder()
+                .ConfigureServices(services =>
+                {
+                    services.Configure<HostOptions>(options =>
+                    {
+                        options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost;
+                    });
+                    services.AddHostedService<SynchronousFailureService>();
+                });
+
+            await Assert.ThrowsAsync<InvalidOperationException>(async () =>
+            {
+                await builder.Build().RunAsync();
+            });
+        }
+
+        /// <summary>
+        /// Tests that when a BackgroundService throws an exception asynchronously (after an await),
+        /// the host propagates the exception.
+        /// This is the main issue that was reported in GitHub issue #67146.
+        /// </summary>
+        [Fact]
+        public async Task BackgroundService_AsynchronousException_ThrowsException()
+        {
+            var builder = new HostBuilder()
+                .ConfigureServices(services =>
+                {
+                    services.Configure<HostOptions>(options =>
+                    {
+                        options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost;
+                    });
+                    services.AddHostedService<AsynchronousFailureService>();
+                });
+
+            await Assert.ThrowsAsync<InvalidOperationException>(async () =>
+            {
+                await builder.Build().RunAsync();
+            });
+        }
+
+        /// <summary>
+        /// Tests that when a BackgroundService throws an exception asynchronously,
+        /// StopAsync propagates the exception when StopHost behavior is configured.
+        /// </summary>
+        [Fact]
+        public async Task BackgroundService_AsynchronousException_StopAsync_ThrowsException()
+        {
+            var builder = new HostBuilder()
+                .ConfigureServices(services =>
+                {
+                    services.Configure<HostOptions>(options =>
+                    {
+                        options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost;
+                    });
+                    services.AddHostedService<AsynchronousFailureService>();
+                });
+
+            var host = builder.Build();
+            await host.StartAsync();
+
+            // Wait for the background service to fail
+            await Task.Delay(TimeSpan.FromMilliseconds(200));
+
+            await Assert.ThrowsAsync<InvalidOperationException>(async () =>
+            {
+                await host.StopAsync();
+            });
+        }
+
+        /// <summary>
+        /// Tests that when multiple BackgroundServices throw exceptions,
+        /// the host aggregates them into an AggregateException.
+        /// </summary>
+        [Fact]
+        public async Task BackgroundService_MultipleExceptions_ThrowsAggregateException()
+        {
+            var builder = new HostBuilder()
+                .ConfigureServices(services =>
+                {
+                    services.Configure<HostOptions>(options =>
+                    {
+                        options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost;
+                    });
+                    services.AddHostedService<AsynchronousFailureService>();
+                    services.AddHostedService<SecondAsynchronousFailureService>();
+                    services.AddHostedService<ThirdAsynchronousFailureService>();
+                });
+
+            var aggregateException = await Assert.ThrowsAsync<AggregateException>(async () =>
+            {
+                await builder.Build().RunAsync();
+            });
+
+            Assert.Equal(3, aggregateException.InnerExceptions.Count);
+
+            Assert.All(aggregateException.InnerExceptions, ex =>
+                Assert.IsType<InvalidOperationException>(ex));
+        }
+
+        /// <summary>
+        /// Tests that when a BackgroundService throws an exception with Ignore behavior,
+        /// the host does not throw and continues to run until stopped.
+        /// </summary>
+        [Fact]
+        public async Task BackgroundService_IgnoreException_DoesNotThrow()
+        {
+            var builder = new HostBuilder()
+                .ConfigureServices(services =>
+                {
+                    services.Configure<HostOptions>(options =>
+                    {
+                        options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.Ignore;
+                    });
+                    services.AddHostedService<AsynchronousFailureService>();
+                    services.AddHostedService<SuccessfulService>();
+                });
+
+            await builder.Build().RunAsync();
+        }
+
+        /// <summary>
+        /// Tests that when a BackgroundService is configured to Ignore exceptions,
+        /// the host does not throw even when the service fails.
+        /// </summary>
+        [Fact]
+        public async Task BackgroundService_IgnoreException_StopAsync_DoesNotThrow()
+        {
+            var builder = new HostBuilder()
+                .ConfigureServices(services =>
+                {
+                    services.Configure<HostOptions>(options =>
+                    {
+                        options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.Ignore;
+                        options.ShutdownTimeout = TimeSpan.FromSeconds(1);
+                    });
+                    services.AddHostedService<AsynchronousFailureService>();
+                });
+
+            var host = builder.Build();
+            await host.StartAsync();
+
+            // Wait a bit for the background service to fail
+            await Task.Delay(TimeSpan.FromMilliseconds(200));
+
+            await host.StopAsync();
+        }
+
+        /// <summary>
+        /// Tests that when a BackgroundService completes successfully,
+        /// the host does not throw an exception.
+        /// </summary>
+        [Fact]
+        public async Task BackgroundService_SuccessfulCompletion_DoesNotThrow()
+        {
+            var builder = new HostBuilder()
+                .ConfigureServices(services =>
+                {
+                    services.Configure<HostOptions>(options =>
+                    {
+                        options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost;
+                    });
+                    services.AddHostedService<SuccessfulService>();
+                });
+
+            await builder.Build().RunAsync();
+        }
+
+        private class SynchronousFailureService : BackgroundService
+        {
+            protected override Task ExecuteAsync(CancellationToken stoppingToken)
+            {
+                // Throw synchronously (no await before the exception)
+                throw new InvalidOperationException("Synchronous failure");
+            }
+        }
+
+        private class AsynchronousFailureService : BackgroundService
+        {
+            protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+            {
+                // Await before throwing to make the exception asynchronous
+                // Ignore the cancellation token to ensure this service throws even if the host is trying to shut down
+                await Task.Delay(TimeSpan.FromMilliseconds(100));
+                throw new InvalidOperationException("Asynchronous failure");
+            }
+        }
+
+        private class SecondAsynchronousFailureService : BackgroundService
+        {
+            protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+            {
+                // Ignore the cancellation token to ensure this service throws even if the host is trying to shut down
+                await Task.Delay(TimeSpan.FromMilliseconds(150));
+                throw new InvalidOperationException("Second asynchronous failure");
+            }
+        }
+
+        private class ThirdAsynchronousFailureService : BackgroundService
+        {
+            protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+            {
+                // Ignore the cancellation token to ensure this service throws even if the host is trying to shut down
+                await Task.Delay(TimeSpan.FromMilliseconds(200));
+                throw new InvalidOperationException("Third asynchronous failure");
+            }
+        }
+
+        private class SuccessfulService : BackgroundService
+        {
+            private readonly IHostApplicationLifetime _lifetime;
+
+            public SuccessfulService(IHostApplicationLifetime lifetime)
+            {
+                _lifetime = lifetime;
+            }
+
+            protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+            {
+                await Task.Delay(TimeSpan.FromMilliseconds(100), stoppingToken);
+                // Exit normally without throwing - signal the host to stop
+                _lifetime.StopApplication();
+            }
+        }
+    }
+}