c# - Graceful shutdown with Generic Host in .NET Core 2.1

Question

.NET Core 2.1 introduced new Generic Host, which allows to host non-HTTP workloads with all benefits of Web Host. Currently, there is no much information and recipes with it, but I used following articles as a starting point:

https://docs.microsoft.com/en-us/aspnet/core/fundamentals/host/generic-host?view=aspnetcore-2.1

https://docs.microsoft.com/en-us/aspnet/core/fundamentals/host/hosted-services?view=aspnetcore-2.1

https://docs.microsoft.com/en-us/dotnet/standard/microservices-architecture/multi-container-microservice-net-applications/background-tasks-with-ihostedservice

My .NET Core application starts, listens for new requests via RabbitMQ message broker and shuts down by user request (usually by Ctrl+C in console). However, shutdown is not graceful - application still have unfinished background threads while it returns control to OS. I see it by console messages - when I press Ctrl+C in console I see few lines of console output from my application, then OS command prompt and then again console output from my application.

Here is my code:

Program.cs

public class Program
{
    public static async Task Main(string[] args)
    {
        var host = new HostBuilder()
            .ConfigureHostConfiguration(config =>
            {
                config.SetBasePath(AppContext.BaseDirectory);
                config.AddEnvironmentVariables(prefix: "ASPNETCORE_");
                config.AddJsonFile("hostsettings.json", optional: true);
            })
            .ConfigureAppConfiguration((context, config) =>
            {
                var env = context.HostingEnvironment;
                config.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
                config.AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true);
                if (env.IsProduction())
                    config.AddDockerSecrets();
                config.AddEnvironmentVariables();
            })
            .ConfigureServices((context, services) =>
            {
                services.AddLogging();
                services.AddHostedService<WorkerPoolHostedService>();
                // ... other services
            })
            .ConfigureLogging((context, logging) =>
            {
                if (context.HostingEnvironment.IsDevelopment())
                    logging.AddDebug();

                logging.AddSerilog(dispose: true);

                Log.Logger = new LoggerConfiguration()
                    .ReadFrom.Configuration(context.Configuration)
                    .CreateLogger();
            })
            .UseConsoleLifetime()
            .Build();

        await host.RunAsync();
    }
}

WorkerPoolHostedService.cs

internal class WorkerPoolHostedService : IHostedService
{
    private IList<VideoProcessingWorker> _workers;
    private CancellationTokenSource _stoppingCts = new CancellationTokenSource();

    protected WorkerPoolConfiguration WorkerPoolConfiguration { get; }
    protected RabbitMqConfiguration RabbitMqConfiguration { get; }
    protected IServiceProvider ServiceProvider { get; }
    protected ILogger<WorkerPoolHostedService> Logger { get; }

    public WorkerPoolHostedService(
        IConfiguration configuration,
        IServiceProvider serviceProvider,
        ILogger<WorkerPoolHostedService> logger)
    {
        this.WorkerPoolConfiguration = new WorkerPoolConfiguration(configuration);
        this.RabbitMqConfiguration = new RabbitMqConfiguration(configuration);
        this.ServiceProvider = serviceProvider;
        this.Logger = logger;
    }

    public async Task StartAsync(CancellationToken cancellationToken)
    {
        var connectionFactory = new ConnectionFactory
        {
            AutomaticRecoveryEnabled = true,
            UserName = this.RabbitMqConfiguration.Username,
            Password = this.RabbitMqConfiguration.Password,
            HostName = this.RabbitMqConfiguration.Hostname,
            Port = this.RabbitMqConfiguration.Port,
            VirtualHost = this.RabbitMqConfiguration.VirtualHost
        };

        _workers = Enumerable.Range(0, this.WorkerPoolConfiguration.WorkerCount)
            .Select(i => new VideoProcessingWorker(
                connectionFactory: connectionFactory,
                serviceScopeFactory: this.ServiceProvider.GetRequiredService<IServiceScopeFactory>(),
                logger: this.ServiceProvider.GetRequiredService<ILogger<VideoProcessingWorker>>(),
                cancellationToken: _stoppingCts.Token))
            .ToList();

        this.Logger.LogInformation("Worker pool started with {0} workers.", this.WorkerPoolConfiguration.WorkerCount);
    }

    public async Task StopAsync(CancellationToken cancellationToken)
    {
        this.Logger.LogInformation("Stopping working pool...");

        try
        {
            _stoppingCts.Cancel();
            await Task.WhenAll(_workers.SelectMany(w => w.ActiveTasks).ToArray());
        }
        catch (AggregateException ae)
        {
            ae.Handle((Exception exc) =>
            {
                this.Logger.LogError(exc, "Error while cancelling workers");
                return true;
            });
        }
        finally
        {
            if (_workers != null)
            {
                foreach (var worker in _workers)
                    worker.Dispose();
                _workers = null;
            }
        }
    }
}

VideoProcessingWorker.cs

internal class VideoProcessingWorker : IDisposable
{
    private readonly Guid _id = Guid.NewGuid();
    private bool _disposed = false;

    protected IConnection Connection { get; }
    protected IModel Channel { get; }
    protected IServiceScopeFactory ServiceScopeFactory { get; }
    protected ILogger<VideoProcessingWorker> Logger { get; }
    protected CancellationToken CancellationToken { get; }

    public VideoProcessingWorker(
        IConnectionFactory connectionFactory,
        IServiceScopeFactory serviceScopeFactory,
        ILogger<VideoProcessingWorker> logger,
        CancellationToken cancellationToken)
    {
        this.Connection = connectionFactory.CreateConnection();
        this.Channel = this.Connection.CreateModel();
        this.Channel.BasicQos(prefetchSize: 0, prefetchCount: 1, global: false);
        this.ServiceScopeFactory = serviceScopeFactory;
        this.Logger = logger;
        this.CancellationToken = cancellationToken;

        #region [ Declare ]

        // ...

        #endregion

        #region [ Consume ]

        // ...

        #endregion
    }

    // ... worker logic ...

    public void Dispose()
    {
        if (!_disposed)
        {
            this.Channel.Close(200, "Goodbye");
            this.Channel.Dispose();
            this.Connection.Close();
            this.Connection.Dispose();
            this.Logger.LogDebug("Worker {0}: disposed.", _id);
        }
        _disposed = true;
    }
}

So, when I press Ctrl+C I see following output in console (when there is no request processing):

Stopping working pool...
command prompt
Worker id: disposed.

How to shutdown gracefully?

Answer 1

You need IApplicationLifetime. This provides you with all the needed information about application start and shutdown. You can even trigger the shutdown with it via appLifetime.StopApplication();

Look at https://github.com/aspnet/Docs/blob/66916c2ed3874ed9b000dfd1cab53ef68e84a0f7/aspnetcore/fundamentals/host/generic-host/samples/2.x/GenericHostSample/LifetimeEventsHostedService.cs

Snippet(if the link becomes invalid):

public Task StartAsync(CancellationToken cancellationToken)
{
    appLifetime.ApplicationStarted.Register(OnStarted);
    appLifetime.ApplicationStopping.Register(OnStopping);
    appLifetime.ApplicationStopped.Register(OnStopped);

    return Task.CompletedTask;
}