Build / Build (push) Successful in 30s
- Services, controllers and jobs take their dependencies through primary constructors and use them directly; where a method already named a local after a dependency, the dependency takes its type's name. - Every async method takes a CancellationToken last and passes it down to MongoDB, HTTP and the job queue. The setup claim is still given back when a request is cancelled. - The federation and media options are read through IOptionsMonitor. - The Mastodon entities keep sending "" for empty strings, and every model field where null means something says so. - The inbox's 503 with Retry-After, which other servers retry, is the one server error left, and says why. - Arasaka.HouseRules.Analyzers checks all of this in every build, here and in PrivaPub.ClientModels as decePubClient builds it. Checked against GoToSocial in the pasture: 64 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
176 lines
5.4 KiB
C#
176 lines
5.4 KiB
C#
using PrivaPub.Infrastructure.Http;
|
|
using PrivaPub.Models.Jobs;
|
|
|
|
using System.Collections.Concurrent;
|
|
|
|
namespace PrivaPub.Infrastructure.Jobs
|
|
{
|
|
public interface IJobHandler
|
|
{
|
|
JobKind Kind { get; }
|
|
int Concurrency { get; }
|
|
int MaxAttempts { get; }
|
|
int PerHostLimit { get; }
|
|
Task<JobOutcome> Handle(Job job, CancellationToken token);
|
|
}
|
|
|
|
public class JobWorker(
|
|
IJobQueue jobQueue,
|
|
IEnumerable<IJobHandler> handlers,
|
|
ILogger<JobWorker> logger) : BackgroundService
|
|
{
|
|
static readonly TimeSpan Poll = TimeSpan.FromSeconds(5);
|
|
static readonly TimeSpan ReapInterval = TimeSpan.FromSeconds(30);
|
|
|
|
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
{
|
|
var loops = handlers.SelectMany(handler =>
|
|
{
|
|
var inFlight = new ConcurrentDictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
|
return Enumerable.Range(0, handler.Concurrency).Select(_ => Run(handler, inFlight, stoppingToken));
|
|
}).Append(Reap(stoppingToken));
|
|
await Task.WhenAll(loops);
|
|
}
|
|
|
|
async Task Run(IJobHandler handler, ConcurrentDictionary<string, int> inFlight, CancellationToken stoppingToken)
|
|
{
|
|
while (!stoppingToken.IsCancellationRequested)
|
|
{
|
|
var job = default(Job);
|
|
try
|
|
{
|
|
var busy = inFlight.Where(h => h.Value >= handler.PerHostLimit).Select(h => h.Key).ToList();
|
|
job = await jobQueue.Lease(handler.Kind, busy, stoppingToken);
|
|
}
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
{
|
|
return;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogError(ex, "{Worker} could not lease a {Kind} job", nameof(JobWorker), handler.Kind);
|
|
await Delay(Poll, stoppingToken);
|
|
continue;
|
|
}
|
|
|
|
if (job == default)
|
|
{
|
|
await jobQueue.WaitForWork(handler.Kind, Poll, stoppingToken);
|
|
continue;
|
|
}
|
|
|
|
var host = job.Host ?? string.Empty;
|
|
inFlight.AddOrUpdate(host, 1, (_, count) => count + 1);
|
|
// the lease is renewed while the handler runs; once it is lost (another worker leased the job again) the
|
|
// handler is cancelled and its outcome dropped
|
|
using var running = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
|
|
var renewing = KeepLease(job, running);
|
|
try
|
|
{
|
|
var outcome = default(JobOutcome);
|
|
try
|
|
{
|
|
using var scope = HttpScope.Triggered(handler.Kind.ToString().ToLowerInvariant());
|
|
outcome = await handler.Handle(job, running.Token);
|
|
}
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
{
|
|
return;
|
|
}
|
|
catch (OperationCanceledException) when (running.IsCancellationRequested)
|
|
{
|
|
logger.LogWarning("{Kind} job {Id} lost its lease and was stopped", handler.Kind, job.ID);
|
|
continue;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogError(ex, "{Kind} job {Id} threw", handler.Kind, job.ID);
|
|
outcome = JobOutcome.Retry(ex.GetType().Name);
|
|
}
|
|
|
|
if (outcome.Result != JobResult.Done && job.Attempts >= handler.MaxAttempts && outcome.Result == JobResult.Retry)
|
|
logger.LogWarning("{Kind} job {Id} for {Host} is dead after {Attempts} attempts: {Error}",
|
|
handler.Kind, job.ID, job.Host, job.Attempts, outcome.Error);
|
|
if (!await jobQueue.Finish(job, outcome, handler.MaxAttempts, CancellationToken.None))
|
|
logger.LogWarning("{Kind} job {Id} lost its lease before it finished; its outcome is dropped", handler.Kind, job.ID);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogError(ex, "{Kind} job {Id} could not be finished; its lease will expire", handler.Kind, job.ID);
|
|
}
|
|
finally
|
|
{
|
|
await running.CancelAsync();
|
|
await renewing;
|
|
inFlight.AddOrUpdate(host, 0, (_, count) => count - 1);
|
|
}
|
|
}
|
|
}
|
|
|
|
// renews the lease every third of its length until the handler ends; a lease found lost cancels the handler
|
|
[System.Diagnostics.CodeAnalysis.SuppressMessage("HouseRules", "HR0003", Justification = "Takes the source it cancels when the lease is lost; its token already follows the worker's.")]
|
|
async Task KeepLease(Job job, CancellationTokenSource running)
|
|
{
|
|
var every = jobQueue.LeaseFor / 3;
|
|
while (!running.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await Task.Delay(every, running.Token);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
if (await jobQueue.Renew(job, CancellationToken.None))
|
|
continue;
|
|
await running.CancelAsync();
|
|
return;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// a passing database error: the lease has two more thirds to go, so try again at the next turn
|
|
logger.LogWarning(ex, "{Worker} could not renew the lease of job {Id}", nameof(JobWorker), job.ID);
|
|
}
|
|
}
|
|
}
|
|
|
|
async Task Reap(CancellationToken stoppingToken)
|
|
{
|
|
while (!stoppingToken.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
var reaped = await jobQueue.Reap(stoppingToken);
|
|
if (reaped > 0)
|
|
logger.LogWarning("{Worker} returned {Count} expired leases to the queue", nameof(JobWorker), reaped);
|
|
}
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
{
|
|
return;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogError(ex, "{Worker} reaper failed", nameof(JobWorker));
|
|
}
|
|
await Delay(ReapInterval, stoppingToken);
|
|
}
|
|
}
|
|
|
|
static async Task Delay(TimeSpan delay, CancellationToken token)
|
|
{
|
|
try
|
|
{
|
|
await Task.Delay(delay, token);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
}
|
|
}
|
|
}
|
|
}
|