House rules: primary constructors, a cancellation token on every async call, and the rules fail the build
Build / Build (push) Successful in 30s
Build / Build (push) Successful in 30s
- Services, controllers and jobs take their dependencies through primary constructors and use them directly; where a method already named a local after a dependency, the dependency takes its type's name. - Every async method takes a CancellationToken last and passes it down to MongoDB, HTTP and the job queue. The setup claim is still given back when a request is cancelled. - The federation and media options are read through IOptionsMonitor. - The Mastodon entities keep sending "" for empty strings, and every model field where null means something says so. - The inbox's 503 with Retry-After, which other servers retry, is the one server error left, and says why. - Arasaka.HouseRules.Analyzers checks all of this in every build, here and in PrivaPub.ClientModels as decePubClient builds it. Checked against GoToSocial in the pasture: 64 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
This commit is contained in:
1 parent
f0a14c737f
commit
0f73f69239
176 files changed
+2789
-3974
No files matched your search
@@ -27,7 +27,12 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
public const string Served = "served";
|
||||
}
|
||||
|
||||
public class InteractionLedger : BackgroundService, IInteractionLedger
|
||||
public class InteractionLedger(
|
||||
InteractionSalts salts,
|
||||
IOptionsMonitor<StatisticsOptions> options,
|
||||
ILogger<InteractionLedger> logger,
|
||||
IJobQueue queue = default,
|
||||
IDomainBlocks blocks = default) : BackgroundService, IInteractionLedger
|
||||
{
|
||||
const int Capacity = 10_000;
|
||||
const int BatchSize = 1000;
|
||||
@@ -46,11 +51,6 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
static readonly TimeSpan TouchInterval = TimeSpan.FromHours(1);
|
||||
static readonly HashSet<string> TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context", "replies" };
|
||||
|
||||
readonly InteractionSalts _salts;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly ILogger<InteractionLedger> _logger;
|
||||
readonly IJobQueue _queue;
|
||||
readonly IDomainBlocks _blocks;
|
||||
readonly ConcurrentDictionary<string, byte> _touches = new(StringComparer.Ordinal);
|
||||
readonly ConcurrentDictionary<string, DateTime> _touchedAt = new(StringComparer.Ordinal);
|
||||
readonly SemaphoreSlim _flushing = new(1, 1);
|
||||
@@ -63,36 +63,26 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
long _written;
|
||||
long _failed;
|
||||
|
||||
public InteractionLedger(InteractionSalts salts, IOptionsMonitor<StatisticsOptions> options, ILogger<InteractionLedger> logger,
|
||||
IJobQueue queue = default, IDomainBlocks blocks = default)
|
||||
{
|
||||
_salts = salts;
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
_queue = queue;
|
||||
_blocks = blocks;
|
||||
}
|
||||
|
||||
public long Dropped => Interlocked.Read(ref _dropped);
|
||||
|
||||
public void Record(InteractionEvent interaction, string actorUri = default, bool hostClaimed = false)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (interaction == default || !_options.CurrentValue.Enabled)
|
||||
if (interaction == default || !options.CurrentValue.Enabled)
|
||||
return;
|
||||
if (!_channel.Writer.TryWrite(new Pending(interaction, actorUri, hostClaimed)))
|
||||
Interlocked.Increment(ref _dropped);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "An interaction could not be recorded");
|
||||
logger.LogDebug(ex, "An interaction could not be recorded");
|
||||
}
|
||||
}
|
||||
|
||||
public void Count(string host, string key, long bytes = 0)
|
||||
{
|
||||
if (!_options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
|
||||
if (!options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
|
||||
return;
|
||||
var day = DateTime.UtcNow.Date;
|
||||
host = Interactions.Host(host);
|
||||
@@ -107,7 +97,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
|
||||
public void CountServer(string section, string key, int? latencyMs = default)
|
||||
{
|
||||
if (!_options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
|
||||
if (!options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
|
||||
return;
|
||||
var day = DateTime.UtcNow.Date;
|
||||
var field = section == ServerSections.Served ? nameof(ServerDay.Served) : nameof(ServerDay.Client);
|
||||
@@ -145,7 +135,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "The interaction ledger could not flush");
|
||||
logger.LogWarning(ex, "The interaction ledger could not flush");
|
||||
await Task.Delay(BatchWait, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
@@ -161,7 +151,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "The interaction ledger could not flush on shutdown");
|
||||
logger.LogWarning(ex, "The interaction ledger could not flush on shutdown");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,10 +199,10 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
.Modify(i => i.LastSeenAt, now)
|
||||
.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token));
|
||||
.ExecuteAsync(token), token);
|
||||
_knownHosts.Add(host);
|
||||
if (_queue != default && _blocks?.IsSuspended(host) != true)
|
||||
await _queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
|
||||
if (queue != default && blocks?.IsSuspended(host) != true)
|
||||
await queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
|
||||
}
|
||||
if (_touchedAt.Count > 100_000)
|
||||
_touchedAt.Clear();
|
||||
@@ -238,7 +228,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
Interlocked.Add(ref _failed, batch.Count);
|
||||
_logger.LogWarning(ex, "{Count} interactions could not be written", batch.Count);
|
||||
logger.LogWarning(ex, "{Count} interactions could not be written", batch.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -254,7 +244,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
if (pending.HostClaimed && interaction.Host != Interactions.Unknown && !(await KnownHosts(token)).Contains(interaction.Host))
|
||||
interaction.Host = Interactions.Unknown;
|
||||
if (pending.ActorUri != default)
|
||||
interaction.ActorHash = InteractionSalts.Hash(await _salts.For(interaction.At, token), pending.ActorUri);
|
||||
interaction.ActorHash = InteractionSalts.Hash(await salts.For(interaction.At, token), pending.ActorUri);
|
||||
if (interaction.Host != Interactions.Unknown && Touches(interaction, pending.ActorUri != default))
|
||||
_touches[interaction.Host] = 0;
|
||||
return interaction;
|
||||
@@ -272,7 +262,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
|
||||
// two flushes upserting a new day's document at once: one inserts it and the other's insert fails on the unique
|
||||
// key; the second try finds the document and adds to it (MongoDB retries this only for single-document updates)
|
||||
static async Task Upsert(Func<Task> write)
|
||||
static async Task Upsert(Func<Task> write, CancellationToken token = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -294,7 +284,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
.Match(d => d.Day == key.Day && d.Host == key.Host)
|
||||
.Modify(b => b.Inc($"{nameof(InstanceDay.Reads)}.{key.Key}", count))
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token));
|
||||
.ExecuteAsync(token), token);
|
||||
}
|
||||
|
||||
var day = DateTime.UtcNow.Date;
|
||||
@@ -311,7 +301,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
.Modify(b => b.Inc(d => d.LedgerFailed, failed))
|
||||
.Modify(b => b.Inc(d => d.LedgerDropped, newlyDropped))
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token));
|
||||
.ExecuteAsync(token), token);
|
||||
foreach (var key in server)
|
||||
{
|
||||
if (!_server.TryRemove(key, out var count) || count == 0)
|
||||
@@ -320,7 +310,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
.Match(d => d.Day == key.Day)
|
||||
.Modify(b => b.Inc($"{key.Field}.{key.Key}", count))
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token));
|
||||
.ExecuteAsync(token), token);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
day = day.Date;
|
||||
if (_known.TryGetValue(day, out var cached))
|
||||
return cached;
|
||||
InteractionSalt salt;
|
||||
var salt = default(InteractionSalt);
|
||||
if (day == DateTime.UtcNow.Date)
|
||||
salt = await DB.Default.UpdateAndGet<InteractionSalt>()
|
||||
.Match(s => s.Day == day)
|
||||
|
||||
@@ -11,13 +11,11 @@ using System.Globalization;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Statistics
|
||||
{
|
||||
public class RollupJob : IJobHandler
|
||||
public class RollupJob(
|
||||
IJobQueue queue) : IJobHandler
|
||||
{
|
||||
public static readonly TimeSpan After = TimeSpan.FromMinutes(30);
|
||||
|
||||
readonly IJobQueue _queue;
|
||||
|
||||
public RollupJob(IJobQueue queue) => _queue = queue;
|
||||
|
||||
public JobKind Kind => JobKind.RollupDay;
|
||||
public int Concurrency => 1;
|
||||
@@ -47,7 +45,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
await DB.Default.DeleteAsync<InteractionSalt>(s => s.Day == day);
|
||||
var next = end;
|
||||
if (next < DateTime.UtcNow.Date.AddDays(1))
|
||||
await _queue.EnqueueMany(new[] { For(next) }, token);
|
||||
await queue.EnqueueMany(new[] { For(next) }, token);
|
||||
return JobOutcome.Done;
|
||||
}
|
||||
|
||||
@@ -95,21 +93,14 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
}
|
||||
}
|
||||
|
||||
public class StatisticsSchedule : BackgroundService
|
||||
public class StatisticsSchedule(
|
||||
IJobQueue queue,
|
||||
ILogger<StatisticsSchedule> logger,
|
||||
IOptionsMonitor<StatisticsOptions> options = default) : BackgroundService
|
||||
{
|
||||
static readonly TimeSpan Interval = TimeSpan.FromHours(1);
|
||||
const int CatchUpDays = 7;
|
||||
|
||||
readonly IJobQueue _queue;
|
||||
readonly ILogger<StatisticsSchedule> _logger;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
|
||||
public StatisticsSchedule(IJobQueue queue, ILogger<StatisticsSchedule> logger, IOptionsMonitor<StatisticsOptions> options = default)
|
||||
{
|
||||
_queue = queue;
|
||||
_logger = logger;
|
||||
_options = options;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
@@ -121,7 +112,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Statistics rollups could not be planned");
|
||||
logger.LogWarning(ex, "Statistics rollups could not be planned");
|
||||
}
|
||||
try
|
||||
{
|
||||
@@ -146,12 +137,12 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
if (!rolled && await DB.Default.Find<InteractionEvent>().Match(e => e.At >= day && e.At < end).ExecuteAnyAsync(token))
|
||||
jobs.Add(RollupJob.For(day));
|
||||
}
|
||||
if (_options?.CurrentValue.Crawler.Enabled == true)
|
||||
if (options?.CurrentValue.Crawler.Enabled == true)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
jobs.Add(Federation.Crawler.CrawlPlanner.For(new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Utc)));
|
||||
}
|
||||
await _queue.EnqueueMany(jobs, token);
|
||||
await queue.EnqueueMany(jobs, token);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user