2026-10-03 11:10:42 +02:00
using PrivaPub.Infrastructure.Http ;
2026-10-01 11:09:35 +02:00
using PrivaPub.Models.Jobs ;
using System.Collections.Concurrent ;
namespace PrivaPub.Infrastructure.Jobs
{
public interface IJobHandler
{
JobKind Kind { get ; }
int Concurrency { get ; }
int MaxAttempts { get ; }
int PerHostLimit { get ; }
Task < JobOutcome > Handle ( Job job , CancellationToken token );
}
2026-11-10 01:24:43 +01:00
public class JobWorker (
IJobQueue jobQueue ,
IEnumerable < IJobHandler > handlers ,
ILogger < JobWorker > logger ) : BackgroundService
2026-10-01 11:09:35 +02:00
{
static readonly TimeSpan Poll = TimeSpan . FromSeconds ( 5 );
static readonly TimeSpan ReapInterval = TimeSpan . FromSeconds ( 30 );
protected override async Task ExecuteAsync ( CancellationToken stoppingToken )
{
2026-11-10 01:24:43 +01:00
var loops = handlers . SelectMany ( handler =>
2026-10-01 11:09:35 +02:00
{
var inFlight = new ConcurrentDictionary < string , int >( StringComparer . OrdinalIgnoreCase );
return Enumerable . Range ( 0 , handler . Concurrency ). Select ( _ => Run ( handler , inFlight , stoppingToken ));
}). Append ( Reap ( stoppingToken ));
await Task . WhenAll ( loops );
}
async Task Run ( IJobHandler handler , ConcurrentDictionary < string , int > inFlight , CancellationToken stoppingToken )
{
while (! stoppingToken . IsCancellationRequested )
{
2026-11-10 01:24:43 +01:00
var job = default ( Job );
2026-10-01 11:09:35 +02:00
try
{
var busy = inFlight . Where ( h => h . Value >= handler . PerHostLimit ). Select ( h => h . Key ). ToList ();
2026-11-10 01:24:43 +01:00
job = await jobQueue . Lease ( handler . Kind , busy , stoppingToken );
2026-10-01 11:09:35 +02:00
}
catch ( OperationCanceledException ) when ( stoppingToken . IsCancellationRequested )
{
return ;
}
catch ( Exception ex )
{
2026-11-10 01:24:43 +01:00
logger . LogError ( ex , "{Worker} could not lease a {Kind} job" , nameof ( JobWorker ), handler . Kind );
2026-10-01 11:09:35 +02:00
await Delay ( Poll , stoppingToken );
continue ;
}
if ( job == default )
{
2026-11-10 01:24:43 +01:00
await jobQueue . WaitForWork ( handler . Kind , Poll , stoppingToken );
2026-10-01 11:09:35 +02:00
continue ;
}
var host = job . Host ?? string . Empty ;
inFlight . AddOrUpdate ( host , 1 , ( _ , count ) => count + 1 );
2026-10-07 10:20:24 +02:00
// the lease is renewed while the handler runs; once it is lost (another worker leased the job again) the
// handler is cancelled and its outcome dropped
using var running = CancellationTokenSource . CreateLinkedTokenSource ( stoppingToken );
var renewing = KeepLease ( job , running );
2026-10-01 11:09:35 +02:00
try
{
2026-11-10 01:24:43 +01:00
var outcome = default ( JobOutcome );
2026-10-01 11:09:35 +02:00
try
{
2026-10-03 11:10:42 +02:00
using var scope = HttpScope . Triggered ( handler . Kind . ToString (). ToLowerInvariant ());
2026-10-07 10:20:24 +02:00
outcome = await handler . Handle ( job , running . Token );
2026-10-01 11:09:35 +02:00
}
catch ( OperationCanceledException ) when ( stoppingToken . IsCancellationRequested )
{
return ;
}
2026-10-07 10:20:24 +02:00
catch ( OperationCanceledException ) when ( running . IsCancellationRequested )
{
2026-11-10 01:24:43 +01:00
logger . LogWarning ( "{Kind} job {Id} lost its lease and was stopped" , handler . Kind , job . ID );
2026-10-07 10:20:24 +02:00
continue ;
}
2026-10-01 11:09:35 +02:00
catch ( Exception ex )
{
2026-11-10 01:24:43 +01:00
logger . LogError ( ex , "{Kind} job {Id} threw" , handler . Kind , job . ID );
2026-10-01 11:09:35 +02:00
outcome = JobOutcome . Retry ( ex . GetType (). Name );
}
if ( outcome . Result != JobResult . Done && job . Attempts >= handler . MaxAttempts && outcome . Result == JobResult . Retry )
2026-11-10 01:24:43 +01:00
logger . LogWarning ( "{Kind} job {Id} for {Host} is dead after {Attempts} attempts: {Error}" ,
2026-10-01 11:09:35 +02:00
handler . Kind , job . ID , job . Host , job . Attempts , outcome . Error );
2026-11-10 01:24:43 +01:00
if (! await jobQueue . Finish ( job , outcome , handler . MaxAttempts , CancellationToken . None ))
logger . LogWarning ( "{Kind} job {Id} lost its lease before it finished; its outcome is dropped" , handler . Kind , job . ID );
2026-10-01 11:09:35 +02:00
}
catch ( Exception ex )
{
2026-11-10 01:24:43 +01:00
logger . LogError ( ex , "{Kind} job {Id} could not be finished; its lease will expire" , handler . Kind , job . ID );
2026-10-01 11:09:35 +02:00
}
finally
{
2026-10-07 10:20:24 +02:00
await running . CancelAsync ();
await renewing ;
2026-10-01 11:09:35 +02:00
inFlight . AddOrUpdate ( host , 0 , ( _ , count ) => count - 1 );
}
}
}
2026-10-07 10:20:24 +02:00
// renews the lease every third of its length until the handler ends; a lease found lost cancels the handler
2026-11-10 01:24:43 +01:00
[System.Diagnostics.CodeAnalysis.SuppressMessage("HouseRules", "HR0003", Justification = "Takes the source it cancels when the lease is lost; its token already follows the worker's.")]
2026-10-07 10:20:24 +02:00
async Task KeepLease ( Job job , CancellationTokenSource running )
{
2026-11-10 01:24:43 +01:00
var every = jobQueue . LeaseFor / 3 ;
2026-10-07 10:20:24 +02:00
while (! running . IsCancellationRequested )
{
try
{
await Task . Delay ( every , running . Token );
}
catch ( OperationCanceledException )
{
return ;
}
try
{
2026-11-10 01:24:43 +01:00
if ( await jobQueue . Renew ( job , CancellationToken . None ))
2026-10-07 10:20:24 +02:00
continue ;
await running . CancelAsync ();
return ;
}
catch ( Exception ex )
{
// a passing database error: the lease has two more thirds to go, so try again at the next turn
2026-11-10 01:24:43 +01:00
logger . LogWarning ( ex , "{Worker} could not renew the lease of job {Id}" , nameof ( JobWorker ), job . ID );
2026-10-07 10:20:24 +02:00
}
}
}
2026-10-01 11:09:35 +02:00
async Task Reap ( CancellationToken stoppingToken )
{
while (! stoppingToken . IsCancellationRequested )
{
try
{
2026-11-10 01:24:43 +01:00
var reaped = await jobQueue . Reap ( stoppingToken );
2026-10-01 11:09:35 +02:00
if ( reaped > 0 )
2026-11-10 01:24:43 +01:00
logger . LogWarning ( "{Worker} returned {Count} expired leases to the queue" , nameof ( JobWorker ), reaped );
2026-10-01 11:09:35 +02:00
}
catch ( OperationCanceledException ) when ( stoppingToken . IsCancellationRequested )
{
return ;
}
catch ( Exception ex )
{
2026-11-10 01:24:43 +01:00
logger . LogError ( ex , "{Worker} reaper failed" , nameof ( JobWorker ));
2026-10-01 11:09:35 +02:00
}
await Delay ( ReapInterval , stoppingToken );
}
}
static async Task Delay ( TimeSpan delay , CancellationToken token )
{
try
{
await Task . Delay ( delay , token );
}
catch ( OperationCanceledException )
{
}
}
}
}