House rules: primary constructors, a cancellation token on every async call, and the rules fail the build
Build / Build (push) Successful in 30s

- Services, controllers and jobs take their dependencies through primary
  constructors and use them directly; where a method already named a local
  after a dependency, the dependency takes its type's name.
- Every async method takes a CancellationToken last and passes it down to
  MongoDB, HTTP and the job queue. The setup claim is still given back when
  a request is cancelled.
- The federation and media options are read through IOptionsMonitor.
- The Mastodon entities keep sending "" for empty strings, and every model
  field where null means something says so.
- The inbox's 503 with Retry-After, which other servers retry, is the one
  server error left, and says why.
- Arasaka.HouseRules.Analyzers checks all of this in every build, here and in
  PrivaPub.ClientModels as decePubClient builds it.

Checked against GoToSocial in the pasture: 64 passed, 0 failed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
This commit is contained in:
thepraandClaude Opus 5.5 committed 2026-11-10 01:24:43 +01:00
1 parent f0a14c737f
commit 0f73f69239
176 files changed
+2789 -3974

No files matched your search

@@ -54,12 +54,13 @@ namespace PrivaPub.Infrastructure.Backup
public sealed class Held : IAsyncDisposable
{
readonly string _owner;
readonly Timer _heartbeat;
string Owner { get; }
public Held(string owner)
{
_owner = owner;
Owner = owner;
_heartbeat = new Timer(_ => _ = DB.Default.Update<MaintenanceLock>().Match(l => l.ID == Name && l.Owner == owner)
.Modify(l => l.Heartbeat, DateTime.UtcNow).ExecuteAsync(), default, Beat, Beat);
}
@@ -67,7 +68,7 @@ namespace PrivaPub.Infrastructure.Backup
public async ValueTask DisposeAsync()
{
await _heartbeat.DisposeAsync();
await DB.Default.Update<MaintenanceLock>().Match(l => l.ID == Name && l.Owner == _owner)
await DB.Default.Update<MaintenanceLock>().Match(l => l.ID == Name && l.Owner == Owner)
.Modify(l => l.Owner, null).Modify(l => l.Heartbeat, DateTime.MinValue).ExecuteAsync();
}
}
@@ -101,7 +101,7 @@ namespace PrivaPub.Infrastructure.Backup
file.Seek(offset, SeekOrigin.Begin);
var buffer = new byte[81920];
var total = 0L;
int read;
var read = default(int);
while ((read = await piece.ReadAsync(buffer, token)) > 0)
{
total += read;
+50 -50
View File
@@ -41,36 +41,36 @@ namespace PrivaPub.Infrastructure.Cli
/// <summary>Whether a command runs before migrations: backups are of the database as it was.</summary>
public static bool BeforeMigrations(string[] args) => args is ["backup", ..] or ["backups"] or ["restore", ..];
public static async Task<int> Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default)
public static async Task<int> Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default, CancellationToken token = default)
{
input ??= Console.In;
output ??= Console.Out;
switch (args)
{
case [("promote" or "demote") and var verb, var userName]:
return await Role(services, userName, verb == "promote" ? RootRole.Admin : RootRole.User, output);
return await Role(services, userName, verb == "promote" ? RootRole.Admin : RootRole.User, output, token: token);
case ["role", var userName, var role] when RootUser.TryParseRole(role, out var parsed):
return await Role(services, userName, parsed, output);
return await Role(services, userName, parsed, output, token: token);
case ["create-root", var login, .. var flags] when flags.All(f => f == "--admin"):
return await CreateRoot(services, login, input.ReadLine(), flags.Contains("--admin"), output);
return await CreateRoot(services, login, input.ReadLine(), flags.Contains("--admin"), output, token);
case ["move-persona", var persona, var root]:
return await MovePersona(services, persona, root, output);
return await MovePersona(services, persona, root, output, token);
case ["delete-root", var root]:
return await DeleteRoot(services, root, output);
return await DeleteRoot(services, root, output, token);
case ["reset-totp", var root]:
return await ResetTotp(services, root, output);
return await ResetTotp(services, root, output, token);
case ["media", "audit", .. var flags] when flags.All(f => f == "--fix"):
return await AuditMedia(services, flags.Contains("--fix"), output);
return await AuditMedia(services, flags.Contains("--fix"), output, token);
case ["backup", "verify", var id]:
return await VerifyBackup(services, id, output);
return await VerifyBackup(services, id, output, token);
case ["backup", .. var flags] when BackupFlags(flags, out var kind, out var dbOnly):
return await BackUp(services, kind, dbOnly, output);
return await BackUp(services, kind, dbOnly, output, token);
case ["backups"]:
return ListBackups(services, output);
case ["restore", "--status"]:
return await RestoreStatus(services, output);
return await RestoreStatus(services, output, token);
case ["restore", var id] when !id.StartsWith('-'):
return await Restore(services, id, output);
return await Restore(services, id, output, token);
default:
Console.Error.WriteLine(Usage);
return 2;
@@ -98,7 +98,7 @@ namespace PrivaPub.Infrastructure.Cli
return true;
}
static async Task<int> BackUp(IServiceProvider services, string kind, bool dbOnly, TextWriter output)
static async Task<int> BackUp(IServiceProvider services, string kind, bool dbOnly, TextWriter output, CancellationToken token = default)
{
var (made, error) = await services.GetRequiredService<Backups>().Create(kind, dbOnly, CancellationToken.None);
if (made == default)
@@ -123,7 +123,7 @@ namespace PrivaPub.Infrastructure.Cli
return 0;
}
static async Task<int> VerifyBackup(IServiceProvider services, string id, TextWriter output)
static async Task<int> VerifyBackup(IServiceProvider services, string id, TextWriter output, CancellationToken token = default)
{
var problems = await services.GetRequiredService<Backups>().Verify(id, CancellationToken.None);
foreach (var problem in problems)
@@ -134,7 +134,7 @@ namespace PrivaPub.Infrastructure.Cli
return 0;
}
static async Task<int> Restore(IServiceProvider services, string id, TextWriter output)
static async Task<int> Restore(IServiceProvider services, string id, TextWriter output, CancellationToken token = default)
{
var backups = services.GetRequiredService<Backups>();
var refused = await ServerRestore.Request(backups.Context(), id, "cli", CancellationToken.None);
@@ -143,18 +143,18 @@ namespace PrivaPub.Infrastructure.Cli
Console.Error.WriteLine(refused);
return 1;
}
await Log(services, AdminActions.BackupRestore, AdminTargets.Backup, id, id);
await Log(services, AdminActions.BackupRestore, AdminTargets.Backup, id, id, token: token);
output.WriteLine($"{id} will be restored: the running service stops within seconds and restores it when it starts again");
output.WriteLine("(systemd starts it again; follow it with journalctl -u privapub, then PrivaPub admin restore --status)");
return 0;
}
static async Task<int> RestoreStatus(IServiceProvider services, TextWriter output)
static async Task<int> RestoreStatus(IServiceProvider services, TextWriter output, CancellationToken token = default)
{
var waiting = RestoreMarker.Read(services.GetRequiredService<Backups>().Root);
if (waiting != default)
output.WriteLine($"waiting: {waiting.Backup}, {waiting.State}, attempt {waiting.Attempts} of {RestoreMarker.MaxAttempts}, asked by {waiting.RequestedBy} at {waiting.RequestedAt:u}{(waiting.Error == default ? string.Empty : $": {waiting.Error}")}");
var last = await DB.Default.Find<RestoreRecord>().Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync();
var last = await DB.Default.Find<RestoreRecord>().Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync(token);
if (last == default)
{
if (waiting == default)
@@ -177,7 +177,7 @@ namespace PrivaPub.Infrastructure.Cli
static string Names(List<string> names) => names.Count == 0 ? "none" : string.Join(", ", names);
static async Task<int> AuditMedia(IServiceProvider services, bool fix, TextWriter output)
static async Task<int> AuditMedia(IServiceProvider services, bool fix, TextWriter output, CancellationToken token = default)
{
var report = await Domain.Media.MediaAudit.Run(services.GetRequiredService<Domain.Media.IMediaService>(), fix, CancellationToken.None);
output.WriteLine($"{report.Files} files served, {report.Held} rows holding media");
@@ -194,15 +194,15 @@ namespace PrivaPub.Infrastructure.Cli
// every act below is kept in the staff's log as the command line's ("cli"): a login's by its name alone, a persona's
// without the login it went to
static Task Log(IServiceProvider services, string action, string kind, string id, string label, string note = default) =>
services.GetRequiredService<IModerationLog>().Write(AdminAction.Cli, action, kind, id, label, note: note);
static Task Log(IServiceProvider services, string action, string kind, string id, string label, string note = default, CancellationToken token = default) =>
services.GetRequiredService<IModerationLog>().Write(AdminAction.Cli, action, kind, id, label, note: note, token: token);
// The command line is the way back, so it may leave the server without an administrator (the client's route refuses
// to); it says so, and `role <root> admin` gives one back.
static async Task<int> Role(IServiceProvider services, string userName, RootRole role, TextWriter output, bool created = false)
static async Task<int> Role(IServiceProvider services, string userName, RootRole role, TextWriter output, bool created = false, CancellationToken token = default)
{
userName = userName.ToLowerInvariant();
var user = await DB.Default.Find<RootUser>().Match(u => u.UserName == userName && u.DeletedAt == null).ExecuteFirstAsync();
var user = await DB.Default.Find<RootUser>().Match(u => u.UserName == userName && u.DeletedAt == null).ExecuteFirstAsync(token);
if (user == default)
{
Console.Error.WriteLine($"no root user '{userName}'");
@@ -211,17 +211,17 @@ namespace PrivaPub.Infrastructure.Cli
var was = user.Role;
user.SetRole(role);
await DB.Default.Update<RootUser>().MatchID(user.ID).Modify(u => u.Policies, user.Policies).Modify(u => u.UpdatedAt, user.UpdatedAt).ExecuteAsync();
await DB.Default.Update<RootUser>().MatchID(user.ID).Modify(u => u.Policies, user.Policies).Modify(u => u.UpdatedAt, user.UpdatedAt).ExecuteAsync(token);
if (role == RootRole.Admin)
await ServerSetup.Seal(user.ID, default);//an administrator exists: the client's setup is closed
if (!created && was != role)
await Log(services, role == RootRole.Admin && was == RootRole.User ? AdminActions.RootPromote
: role == RootRole.User && was == RootRole.Admin ? AdminActions.RootDemote
: AdminActions.RootRole, AdminTargets.Root, user.ID, userName, $"{Name(was)} → {Name(role)}");
: AdminActions.RootRole, AdminTargets.Root, user.ID, userName, $"{Name(was)} → {Name(role)}", token);
output.WriteLine($"{userName}: {string.Join(", ", user.Policies)}");
if (was == RootRole.Admin && role != RootRole.Admin
&& !await DB.Default.Find<RootUser>().Match(u => u.DeletedAt == null && !u.IsBanned && u.Policies.Contains(Policies.IsAdmin)).ExecuteAnyAsync())
&& !await DB.Default.Find<RootUser>().Match(u => u.DeletedAt == null && !u.IsBanned && u.Policies.Contains(Policies.IsAdmin)).ExecuteAnyAsync(token))
Console.Error.WriteLine("no administrator is left: `PrivaPub admin role <root> admin` makes one");
return 0;
}
@@ -230,7 +230,7 @@ namespace PrivaPub.Infrastructure.Cli
// With sign-up closed, this or the client's setup (while no administrator exists) makes the first root; group invitations
// make the rest.
static async Task<int> CreateRoot(IServiceProvider services, string login, string password, bool admin, TextWriter output)
static async Task<int> CreateRoot(IServiceProvider services, string login, string password, bool admin, TextWriter output, CancellationToken token = default)
{
var form = new LoginForm { UserName = login, Password = password?.Trim() };
var problems = new List<ValidationResult>();
@@ -239,18 +239,18 @@ namespace PrivaPub.Infrastructure.Cli
Console.Error.WriteLine(string.Join(Environment.NewLine, problems.Select(p => p.ErrorMessage)));
return 1;
}
var created = await services.GetRequiredService<IRootUsersService>().SignUpAsync(form);
var created = await services.GetRequiredService<IRootUsersService>().SignUpAsync(form, token);
if (!created.IsValid)
{
Console.Error.WriteLine(created.ErrorMessage);
return 1;
}
var made = await DB.Default.Find<RootUser>().Match(u => u.UserName == login.ToLowerInvariant()).ExecuteFirstAsync();
await Log(services, AdminActions.RootCreate, AdminTargets.Root, made?.ID, login.ToLowerInvariant(), admin ? "administrator" : default);
return admin ? await Role(services, login, RootRole.Admin, output, created: true) : await Report(login, output);
var made = await DB.Default.Find<RootUser>().Match(u => u.UserName == login.ToLowerInvariant()).ExecuteFirstAsync(token);
await Log(services, AdminActions.RootCreate, AdminTargets.Root, made?.ID, login.ToLowerInvariant(), admin ? "administrator" : default, token);
return admin ? await Role(services, login, RootRole.Admin, output, created: true, token) : await Report(login, output, token);
}
static Task<int> Report(string login, TextWriter output)
static Task<int> Report(string login, TextWriter output, CancellationToken token = default)
{
output.WriteLine($"{login.ToLowerInvariant()}: created");
return Task.FromResult(0);
@@ -259,23 +259,23 @@ namespace PrivaPub.Infrastructure.Cli
// A persona given to another login (owner decision 2026-10-17, made for @thepra): its link moves first, so the old
// login can no longer exchange a token for it, then its Mastodon tokens and authorizations end. Its actor, posts and
// followers stay as they are, and nobody outside is told: nothing they hold changes.
static async Task<int> MovePersona(IServiceProvider services, string personaName, string login, TextWriter output)
static async Task<int> MovePersona(IServiceProvider services, string personaName, string login, TextWriter output, CancellationToken token = default)
{
personaName = personaName.TrimStart('@').ToLowerInvariant();
login = login.ToLowerInvariant();
var persona = await DB.Default.Find<Avatar>().Match(a => a.UserName == personaName && !a.DeletionAt.HasValue).ExecuteFirstAsync();
var persona = await DB.Default.Find<Avatar>().Match(a => a.UserName == personaName && !a.DeletionAt.HasValue).ExecuteFirstAsync(token);
if (persona == default)
{
Console.Error.WriteLine($"no persona '@{personaName}'");
return 1;
}
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login).ExecuteFirstAsync();
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login).ExecuteFirstAsync(token);
if (root == default || root.DeletedAt.HasValue || root.IsBanned)
{
Console.Error.WriteLine($"no root user '{login}' that can take it");
return 1;
}
var link = await DB.Default.Find<RootToAvatar>().Match(ra => ra.AvatarId == persona.ID).ExecuteFirstAsync();
var link = await DB.Default.Find<RootToAvatar>().Match(ra => ra.AvatarId == persona.ID).ExecuteFirstAsync(token);
if (link == default)
{
Console.Error.WriteLine($"'@{personaName}' belongs to no login");
@@ -287,52 +287,52 @@ namespace PrivaPub.Infrastructure.Cli
return 0;
}
await DB.Default.Update<RootToAvatar>().MatchID(link.ID).Modify(ra => ra.RootId, root.ID).ExecuteAsync();
await services.GetRequiredService<IOpenIddictTokenManager>().RevokeBySubjectAsync(persona.ID);
await services.GetRequiredService<IOpenIddictAuthorizationManager>().RevokeBySubjectAsync(persona.ID);
await Domain.Social.WebPush.Forget(new[] { persona.ID });//its Web Push subscriptions go with its tokens (R9)
await Log(services, AdminActions.PersonaMove, AdminTargets.Persona, persona.ID, "@" + personaName, "moved to another login");
await DB.Default.Update<RootToAvatar>().MatchID(link.ID).Modify(ra => ra.RootId, root.ID).ExecuteAsync(token);
await services.GetRequiredService<IOpenIddictTokenManager>().RevokeBySubjectAsync(persona.ID, token);
await services.GetRequiredService<IOpenIddictAuthorizationManager>().RevokeBySubjectAsync(persona.ID, token);
await Domain.Social.WebPush.Forget(new[] { persona.ID }, token);//its Web Push subscriptions go with its tokens (R9)
await Log(services, AdminActions.PersonaMove, AdminTargets.Persona, persona.ID, "@" + personaName, "moved to another login", token);
output.WriteLine($"@{personaName}: moved to {login}");
return 0;
}
// The way back for a login that lost its authenticator and its recovery codes: its two-factor sign-in goes, its
// password stays.
static async Task<int> ResetTotp(IServiceProvider services, string login, TextWriter output)
static async Task<int> ResetTotp(IServiceProvider services, string login, TextWriter output, CancellationToken token = default)
{
login = login.ToLowerInvariant();
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login && u.DeletedAt == null).ExecuteFirstAsync();
var updated = root == default ? default : await DB.Default.Update<RootUser>().MatchID(root.ID).Modify(u => u.Totp, null).ExecuteAsync();
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login && u.DeletedAt == null).ExecuteFirstAsync(token);
var updated = root == default ? default : await DB.Default.Update<RootUser>().MatchID(root.ID).Modify(u => u.Totp, null).ExecuteAsync(token);
if (updated == default || updated.MatchedCount == 0)
{
Console.Error.WriteLine($"no root user '{login}'");
return 1;
}
await Log(services, AdminActions.RootResetTotp, AdminTargets.Root, root.ID, login);
await Log(services, AdminActions.RootResetTotp, AdminTargets.Root, root.ID, login, token: token);
output.WriteLine($"{login}: two-factor sign-in is off");
return 0;
}
// A login deleted from the box as the client deletes its own (RootRemoval): every persona it still has goes with it,
// announced deleted, and its sessions end.
static async Task<int> DeleteRoot(IServiceProvider services, string login, TextWriter output)
static async Task<int> DeleteRoot(IServiceProvider services, string login, TextWriter output, CancellationToken token = default)
{
login = login.ToLowerInvariant();
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login && u.DeletedAt == null).ExecuteFirstAsync();
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login && u.DeletedAt == null).ExecuteFirstAsync(token);
if (root == default)
{
Console.Error.WriteLine($"no root user '{login}'");
return 1;
}
var owned = (await DB.Default.Find<RootToAvatar>().Match(ra => ra.RootId == root.ID).ExecuteAsync()).Select(ra => ra.AvatarId).ToList();
var personas = (await DB.Default.Find<Avatar>().Match(a => owned.Contains(a.ID) && !a.DeletionAt.HasValue).ExecuteAsync())
var owned = (await DB.Default.Find<RootToAvatar>().Match(ra => ra.RootId == root.ID).ExecuteAsync(token)).Select(ra => ra.AvatarId).ToList();
var personas = (await DB.Default.Find<Avatar>().Match(a => owned.Contains(a.ID) && !a.DeletionAt.HasValue).ExecuteAsync(token))
.Select(a => "@" + a.UserName).ToList();
if (!await services.GetRequiredService<IRootRemoval>().Remove(root.ID, default))
{
Console.Error.WriteLine($"'{login}' could not be deleted");
return 1;
}
await Log(services, AdminActions.RootDelete, AdminTargets.Root, root.ID, login);
await Log(services, AdminActions.RootDelete, AdminTargets.Root, root.ID, login, token: token);
output.WriteLine($"{login}: deleted{(personas.Count > 0 ? ", with " + string.Join(", ", personas) : string.Empty)}");
return 0;
}
+1
View File
@@ -256,6 +256,7 @@ namespace PrivaPub.Infrastructure.Data
})
.CreateAsync(token);
[System.Diagnostics.CodeAnalysis.SuppressMessage("HouseRules", "HR0003", Justification = "The index keys are a params array, which has to come last.")]
static async Task Plain<T>(CancellationToken token, params System.Linq.Expressions.Expression<Func<T, object>>[] keys) where T : IEntity
{
var index = DB.Default.Index<T>();
@@ -43,17 +43,17 @@ namespace PrivaPub.Infrastructure.Data.Migrations
await DB.Default.SaveAsync(reservations);
}
static async Task DeleteAllBut<T>(IEnumerable<IOrderedEnumerable<T>> groups, Func<T, Task> delete)
static async Task DeleteAllBut<T>(IEnumerable<IOrderedEnumerable<T>> groups, Func<T, CancellationToken, Task> delete, CancellationToken token = default)
{
foreach (var group in groups)
foreach (var duplicate in group.Skip(1))
await delete(duplicate);
await delete(duplicate, token);
}
static Task DeleteForeignAvatar(ForeignAvatar a) => DB.Default.DeleteAsync<ForeignAvatar>(a.ID);
static Task DeletePost(Post p) => DB.Default.DeleteAsync<Post>(p.ID);
static Task DeleteDmPost(DmPost p) => DB.Default.DeleteAsync<DmPost>(p.ID);
static Task DeleteFollower(Follower f) => DB.Default.DeleteAsync<Follower>(f.ID);
static Task DeleteLink(RootToAvatar r) => DB.Default.DeleteAsync<RootToAvatar>(r.ID);
static Task DeleteForeignAvatar(ForeignAvatar a, CancellationToken token = default) => DB.Default.DeleteAsync<ForeignAvatar>(a.ID);
static Task DeletePost(Post p, CancellationToken token = default) => DB.Default.DeleteAsync<Post>(p.ID);
static Task DeleteDmPost(DmPost p, CancellationToken token = default) => DB.Default.DeleteAsync<DmPost>(p.ID);
static Task DeleteFollower(Follower f, CancellationToken token = default) => DB.Default.DeleteAsync<Follower>(f.ID);
static Task DeleteLink(RootToAvatar r, CancellationToken token = default) => DB.Default.DeleteAsync<RootToAvatar>(r.ID);
}
}
@@ -61,7 +61,7 @@ namespace PrivaPub.Infrastructure.Data.Migrations
static string Remote(string reference) =>
reference != default && reference.Contains("://") ? reference : default;
static async Task<string> ForeignAccountId(string actorUri) =>
(await DB.Default.Find<ForeignAvatar>().Match(a => a.ActorURI == actorUri).ExecuteFirstAsync())?.ID;
static async Task<string> ForeignAccountId(string actorUri, CancellationToken token = default) =>
(await DB.Default.Find<ForeignAvatar>().Match(a => a.ActorURI == actorUri).ExecuteFirstAsync(token))?.ID;
}
}
+1 -1
View File
@@ -89,7 +89,7 @@ namespace PrivaPub.Infrastructure.Geo
// The networks a published list holds; an entry that is not an address or a network is an error, never skipped.
public static List<IPNetwork> Parse(CdnListFormat format, string body)
{
IEnumerable<string> entries = format switch
var entries = format switch
{
CdnListFormat.Lines => body.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Where(l => !l.StartsWith('#')),
CdnListFormat.StringArray => Root(body),
+1 -1
View File
@@ -38,7 +38,7 @@ namespace PrivaPub.Infrastructure.Geo
return default;
if (address.IsIPv4MappedToIPv6)
address = address.MapToIPv4();
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> sets;
var sets = default(Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)>);
lock (_gate)
sets = _sets;
foreach (var (provider, networks, _) in sets.Values)
+12 -20
View File
@@ -12,7 +12,11 @@ namespace PrivaPub.Infrastructure.Geo
// Keeps the CDNs' published address ranges current by itself (owner decision 2026-10-04): at start it loads the stored
// ones, then once a day it downloads each list again. A list that fails to download or to parse, or comes back empty,
// leaves the stored one in place and is tried the next day. Like GeoUpdater, plain HTTPS to fixed hosts, not federation.
public sealed class CdnUpdater : BackgroundService
public sealed class CdnUpdater(
IHttpClientFactory http,
IOptionsMonitor<StatisticsOptions> options,
ICdnRanges cdnRanges,
ILogger<CdnUpdater> logger) : BackgroundService
{
public const string ClientName = "cdn";
static readonly TimeSpan FirstWait = TimeSpan.FromMinutes(1);
@@ -20,28 +24,16 @@ namespace PrivaPub.Infrastructure.Geo
const int MaxBytes = 16 * 1024 * 1024;
const int MaxRanges = 200_000;
readonly IHttpClientFactory _http;
readonly IOptionsMonitor<StatisticsOptions> _options;
readonly ICdnRanges _ranges;
readonly ILogger<CdnUpdater> _logger;
public CdnUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, ICdnRanges ranges, ILogger<CdnUpdater> logger)
{
_http = http;
_options = options;
_ranges = ranges;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
await _ranges.Load(stoppingToken);
await cdnRanges.Load(stoppingToken);
await Task.Delay(FirstWait, stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
if (_options.CurrentValue.Cdn.AutoUpdate)
if (options.CurrentValue.Cdn.AutoUpdate)
await Update(CdnCatalog.All, stoppingToken);
await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
}
@@ -51,7 +43,7 @@ namespace PrivaPub.Infrastructure.Geo
}
catch (Exception ex)
{
_logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
}
}
@@ -76,13 +68,13 @@ namespace PrivaPub.Infrastructure.Geo
set.Sources = provider.Lists.Select(l => l.Url).ToList();
set.FetchedAt = DateTime.UtcNow;
await DB.Default.SaveAsync(set, token);
_ranges.Replace(set);
cdnRanges.Replace(set);
replaced++;
}
catch (Exception ex) when (ex is HttpRequestException or InvalidDataException or FormatException or System.Text.Json.JsonException
or TaskCanceledException && !token.IsCancellationRequested)
{
_logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
}
}
return replaced;
@@ -94,14 +86,14 @@ namespace PrivaPub.Infrastructure.Geo
request.Headers.Accept.ParseAdd("application/json, text/plain;q=0.9");
if (list.Body != default)
request.Content = new StringContent(list.Body, Encoding.UTF8, "application/x-www-form-urlencoded");
using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
using var response = await http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
response.EnsureSuccessStatusCode();
if (response.Content.Headers.ContentLength > MaxBytes)
throw new InvalidDataException($"more than {MaxBytes} bytes");
await using var stream = await response.Content.ReadAsStreamAsync(token);
using var buffer = new MemoryStream();
var chunk = new byte[81920];
int read;
var read = default(int);
while ((read = await stream.ReadAsync(chunk, token)) > 0)
{
if (buffer.Length + read > MaxBytes)
+9 -14
View File
@@ -23,14 +23,14 @@ namespace PrivaPub.Infrastructure.Geo
public string Source => default;
}
public sealed class DbIpLocator : IGeoLocator, IDisposable
public sealed class DbIpLocator(
IOptionsMonitor<StatisticsOptions> options,
ILogger<DbIpLocator> logger) : IGeoLocator, IDisposable
{
public static readonly string CityFile = GeoUpdater.FileOf("city");
public static readonly string AsnFile = GeoUpdater.FileOf("asn");
static readonly TimeSpan CheckInterval = TimeSpan.FromMinutes(10);
readonly IOptionsMonitor<StatisticsOptions> _options;
readonly ILogger<DbIpLocator> _logger;
readonly object _lock = new();
Reader _city;
Reader _asn;
@@ -39,12 +39,6 @@ namespace PrivaPub.Infrastructure.Geo
DateTime _checkedAt = DateTime.MinValue;
bool _warned;
public DbIpLocator(IOptionsMonitor<StatisticsOptions> options, ILogger<DbIpLocator> logger)
{
_options = options;
_logger = logger;
}
public string Source
{
get
@@ -60,7 +54,8 @@ namespace PrivaPub.Infrastructure.Geo
if (address == default || !IpRangeGuard.IsPublic(address))
return default;
Refresh();
Dictionary<string, object> place, network;
var place = default(Dictionary<string, object>);
var network = default(Dictionary<string, object>);
try
{
// under the lock, so a reload cannot dispose a reader in the middle of a lookup; lookups are rare
@@ -82,7 +77,7 @@ namespace PrivaPub.Infrastructure.Geo
}
catch (Exception ex) when (ex is InvalidDatabaseException or ArgumentException)
{
_logger.LogWarning(ex, "The geolocation databases could not be read");
logger.LogWarning(ex, "The geolocation databases could not be read");
return default;
}
}
@@ -104,13 +99,13 @@ namespace PrivaPub.Infrastructure.Geo
if (DateTime.UtcNow - _checkedAt < CheckInterval)
return;
_checkedAt = DateTime.UtcNow;
var directory = _options.CurrentValue.GeoDirectory;
var directory = options.CurrentValue.GeoDirectory;
(_city, _cityStamp) = Open(Path.Combine(directory ?? string.Empty, CityFile), _city, _cityStamp);
(_asn, _asnStamp) = Open(Path.Combine(directory ?? string.Empty, AsnFile), _asn, _asnStamp);
if (_city == default && _asn == default && !_warned)
{
_warned = true;
_logger.LogInformation("No geolocation databases in {Directory}; servers are not located", directory);
logger.LogInformation("No geolocation databases in {Directory}; servers are not located", directory);
}
}
}
@@ -133,7 +128,7 @@ namespace PrivaPub.Infrastructure.Geo
}
catch (Exception ex) when (ex is InvalidDatabaseException or IOException)
{
_logger.LogWarning(ex, "{Path} is not a readable geolocation database", path);
logger.LogWarning(ex, "{Path} is not a readable geolocation database", path);
return (current, stamp);
}
}
+13 -21
View File
@@ -13,25 +13,17 @@ namespace PrivaPub.Infrastructure.Geo
// whether each database was built this month and, if not, fetches this month's (or, early in the month, last month's),
// checks it opens as the right kind of database and swaps it in. This, CdnUpdater and SMTP are the server's only traffic
// that is not federation, which is why each has its own client instead of IFederationHttp.
public sealed class GeoUpdater : BackgroundService
public sealed class GeoUpdater(
IHttpClientFactory http,
IOptionsMonitor<StatisticsOptions> statisticsOptions,
IGeoLocator geoLocator,
ILogger<GeoUpdater> logger) : BackgroundService
{
public const string ClientName = "geo";
static readonly string[] Kinds = { "city", "asn" };
static readonly TimeSpan FirstWait = TimeSpan.FromSeconds(30);
static readonly TimeSpan Interval = TimeSpan.FromHours(24);
readonly IHttpClientFactory _http;
readonly IOptionsMonitor<StatisticsOptions> _options;
readonly IGeoLocator _locator;
readonly ILogger<GeoUpdater> _logger;
public GeoUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, IGeoLocator locator, ILogger<GeoUpdater> logger)
{
_http = http;
_options = options;
_locator = locator;
_logger = logger;
}
public static string FileOf(string kind) => $"dbip-{kind}-lite.mmdb";
@@ -42,7 +34,7 @@ namespace PrivaPub.Infrastructure.Geo
await Task.Delay(FirstWait, stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
if (_options.CurrentValue.Geo.AutoUpdate)
if (statisticsOptions.CurrentValue.Geo.AutoUpdate)
await Update(DateTime.UtcNow, stoppingToken);
await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
}
@@ -55,7 +47,7 @@ namespace PrivaPub.Infrastructure.Geo
// The number of databases replaced. A failure keeps the database already there and is tried again the next day.
public async Task<int> Update(DateTime now, CancellationToken token)
{
var options = _options.CurrentValue;
var options = statisticsOptions.CurrentValue;
var replaced = 0;
try
{
@@ -63,7 +55,7 @@ namespace PrivaPub.Infrastructure.Geo
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_logger.LogWarning(ex, "Cannot create {Directory}; servers are not located", options.GeoDirectory);
logger.LogWarning(ex, "Cannot create {Directory}; servers are not located", options.GeoDirectory);
return 0;
}
foreach (var kind in Kinds)
@@ -79,19 +71,19 @@ namespace PrivaPub.Infrastructure.Geo
if (await Install(kind, month, path, options, token))
{
replaced++;
_logger.LogInformation("Installed DB-IP Lite {Kind} {Month:yyyy-MM}", kind, month);
logger.LogInformation("Installed DB-IP Lite {Kind} {Month:yyyy-MM}", kind, month);
break;
}
}
catch (Exception ex) when (ex is HttpRequestException or IOException or InvalidDatabaseException or InvalidDataException
or TaskCanceledException && !token.IsCancellationRequested)
{
_logger.LogWarning(ex, "Could not install DB-IP Lite {Kind} {Month:yyyy-MM}", kind, month);
logger.LogWarning(ex, "Could not install DB-IP Lite {Kind} {Month:yyyy-MM}", kind, month);
break;
}
}
}
if (replaced > 0 && _locator is DbIpLocator locator)
if (replaced > 0 && geoLocator is DbIpLocator locator)
locator.Reload();
return replaced;
}
@@ -121,7 +113,7 @@ namespace PrivaPub.Infrastructure.Geo
var unpacked = path + ".new";
try
{
using (var response = await _http.CreateClient(ClientName).GetAsync(url, HttpCompletionOption.ResponseHeadersRead, token))
using (var response = await http.CreateClient(ClientName).GetAsync(url, HttpCompletionOption.ResponseHeadersRead, token))
{
if (response.StatusCode == HttpStatusCode.NotFound)
return false;
@@ -150,7 +142,7 @@ namespace PrivaPub.Infrastructure.Geo
{
var buffer = new byte[81920];
long total = 0;
int read;
var read = default(int);
while ((read = await source.ReadAsync(buffer, token)) > 0)
{
total += read;
+3 -6
View File
@@ -38,17 +38,14 @@ namespace PrivaPub.Infrastructure.Http
// On the federation client: reads each response's CDN fingerprint for its host. A response without one forgets the
// host's hint, so a server that leaves its CDN is seen as itself again.
public class EdgeHintsHandler : DelegatingHandler
public class EdgeHintsHandler(
IEdgeHints hints) : DelegatingHandler
{
readonly IEdgeHints _hints;
public EdgeHintsHandler(IEdgeHints hints) => _hints = hints;
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var response = await base.SendAsync(request, cancellationToken);
if (request.RequestUri is { } uri && (int)response.StatusCode is >= 200 and < 500)
_hints.Remember(uri.Host, CdnCatalog.FromResponse(response));
hints.Remember(uri.Host, CdnCatalog.FromResponse(response));
return response;
}
}
+33 -49
View File
@@ -42,7 +42,13 @@ namespace PrivaPub.Infrastructure.Http
Task<IReadOnlyList<string>> GetStringArray(string url, int maxItems, CancellationToken token);
}
public class FederationHttp : IFederationHttp
public class FederationHttp(
IHttpClientFactory httpClientFactory,
IMemoryCache cache,
IOptionsMonitor<FederationOptions> federationOptions,
IDomainBlocks domainBlocks,
ILogger<FederationHttp> logger,
IInteractionLedger ledger = default) : IFederationHttp
{
public const string ClientName = "Federation";
public const int MaxResponseBytes = 1024 * 1024;
@@ -58,34 +64,12 @@ namespace PrivaPub.Infrastructure.Http
"application/json"
};
readonly IHttpClientFactory _httpClientFactory;
readonly IMemoryCache _cache;
readonly IOptionsMonitor<FederationOptions> _options;
readonly IDomainBlocks _domainBlocks;
readonly ILogger<FederationHttp> _logger;
readonly IInteractionLedger _ledger;
public FederationHttp(IHttpClientFactory httpClientFactory, IMemoryCache cache, IOptionsMonitor<FederationOptions> options,
IDomainBlocks domainBlocks, ILogger<FederationHttp> logger, IInteractionLedger ledger = default)
sealed class Exchange(string url, string purpose)
{
_httpClientFactory = httpClientFactory;
_cache = cache;
_options = options;
_domainBlocks = domainBlocks;
_logger = logger;
_ledger = ledger;
}
public string Host { get; } = Interactions.HostOf(url);
public string Purpose { get; } = purpose;
sealed class Exchange
{
public Exchange(string url, string purpose)
{
Host = Interactions.HostOf(url);
Purpose = purpose;
}
public readonly string Host;
public readonly string Purpose;
public readonly long Started = System.Diagnostics.Stopwatch.GetTimestamp();
public int? Status;
public long? Bytes;
@@ -102,15 +86,15 @@ namespace PrivaPub.Infrastructure.Http
void Record(Exchange exchange)
{
if (_ledger == default)
if (ledger == default)
return;
var trigger = HttpScope.Trigger;
if (trigger == HttpScope.Request)
{
_ledger.Count(exchange.Host, $"http:{exchange.Purpose}:{exchange.Outcome}", exchange.Bytes ?? 0);
ledger.Count(exchange.Host, $"http:{exchange.Purpose}:{exchange.Outcome}", exchange.Bytes ?? 0);
return;
}
_ledger.Record(new InteractionEvent
ledger.Record(new InteractionEvent
{
Channel = Interactions.Http,
Host = exchange.Host,
@@ -151,7 +135,7 @@ namespace PrivaPub.Infrastructure.Http
for (var hop = 0; hop <= MaxRedirects; hop++)
{
using var request = Request(target, "text/plain");
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
using var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
exchange.Answered(response);
if (IsRedirect(response.StatusCode))
{
@@ -207,7 +191,7 @@ namespace PrivaPub.Infrastructure.Http
for (var hop = 0; hop <= MaxRedirects; hop++)
{
using var request = Request(target, "application/json");
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
using var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
exchange.Answered(response);
if (IsRedirect(response.StatusCode))
{
@@ -278,9 +262,9 @@ namespace PrivaPub.Infrastructure.Http
{
if (target is not { IsAbsoluteUri: true } || !string.IsNullOrEmpty(target.UserInfo))
return false;
if (_domainBlocks?.IsSuspended(target.Host) == true)
if (domainBlocks?.IsSuspended(target.Host) == true)
return false;
var options = _options.CurrentValue;
var options = federationOptions.CurrentValue;
if (target.Scheme != Uri.UriSchemeHttps && !(options.AllowPlainHttp && target.Scheme == Uri.UriSchemeHttp))
return false;
if (options.AllowPrivateNetworks)
@@ -322,7 +306,7 @@ namespace PrivaPub.Infrastructure.Http
// status; 0 when it never answered)
public async Task<(int Status, FetchedJson Json)> GetJsonStatus(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token)
{
if (Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out Refusal remembered))
if (Uri.TryCreate(url, UriKind.Absolute, out var target) && cache.TryGetValue(NegativeKey(target), out Refusal remembered))
return (remembered.Status, default);
var exchange = new Exchange(url, HttpScope.Purpose ?? "object");
try
@@ -344,7 +328,7 @@ namespace PrivaPub.Infrastructure.Http
return default;
}
var negativeKey = NegativeKey(target);
if (_cache.TryGetValue(negativeKey, out _))
if (cache.TryGetValue(negativeKey, out _))
{
exchange.Refused("remembered");
return default;
@@ -361,7 +345,7 @@ namespace PrivaPub.Infrastructure.Http
request.Headers.TryAddWithoutValidation("If-None-Match", etag);
sign?.Invoke(request);
using var response = await _httpClientFactory.CreateClient(ClientName)
using var response = await httpClientFactory.CreateClient(ClientName)
.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
exchange.Answered(response);
if (response.StatusCode == HttpStatusCode.NotModified && !string.IsNullOrEmpty(etag))
@@ -412,7 +396,7 @@ namespace PrivaPub.Infrastructure.Http
}
public bool FailedTemporarily(string url) =>
Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out Refusal refusal) && refusal.Transient;
Uri.TryCreate(url, UriKind.Absolute, out var target) && cache.TryGetValue(NegativeKey(target), out Refusal refusal) && refusal.Transient;
public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token)
{
@@ -460,7 +444,7 @@ namespace PrivaPub.Infrastructure.Http
{
using var request = new HttpRequestMessage(HttpMethod.Get, target);
request.Headers.Accept.ParseAdd("image/*, video/*, audio/*");
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
using var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
exchange.Answered(response);
if (IsRedirect(response.StatusCode))
{
@@ -507,7 +491,7 @@ namespace PrivaPub.Infrastructure.Http
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
{
exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
_logger.LogInformation("Media {Url} refused: {Reason}", url, ex.Message);
logger.LogInformation("Media {Url} refused: {Reason}", url, ex.Message);
return default;
}
}
@@ -524,7 +508,7 @@ namespace PrivaPub.Infrastructure.Http
{
using var request = new HttpRequestMessage(HttpMethod.Get, target);
request.Headers.Accept.ParseAdd("text/html, application/xhtml+xml");
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
using var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
if (IsRedirect(response.StatusCode))
{
var location = response.Headers.Location;
@@ -554,7 +538,7 @@ namespace PrivaPub.Infrastructure.Http
}
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
{
_logger.LogInformation("Page {Url} refused: {Reason}", url, ex.Message);
logger.LogInformation("Page {Url} refused: {Reason}", url, ex.Message);
return default;
}
}
@@ -590,7 +574,7 @@ namespace PrivaPub.Infrastructure.Http
using var request = new HttpRequestMessage(HttpMethod.Get, target);
request.Headers.Accept.ParseAdd("video/*, audio/*, image/*");
request.Headers.Range = range;
var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
exchange.Answered(response);
if (IsRedirect(response.StatusCode))
{
@@ -624,7 +608,7 @@ namespace PrivaPub.Infrastructure.Http
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
{
exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
_logger.LogInformation("Media stream {Url} refused: {Reason}", url, ex.Message);
logger.LogInformation("Media stream {Url} refused: {Reason}", url, ex.Message);
return default;
}
}
@@ -636,7 +620,7 @@ namespace PrivaPub.Infrastructure.Http
await using var stream = await content.ReadAsStreamAsync(token);
var buffer = new byte[limit];
var total = 0;
int read;
var read = default(int);
while (total < limit && (read = await stream.ReadAsync(buffer.AsMemory(total, limit - total), token)) > 0)
total += read;
return buffer[..total];
@@ -646,7 +630,7 @@ namespace PrivaPub.Infrastructure.Http
{
if (!IsAllowed(request.RequestUri))
throw new BlockedDestinationException(request.RequestUri?.Host);
return await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
return await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
}
// copies at most limit bytes: how many, or -1 once there are more
@@ -655,7 +639,7 @@ namespace PrivaPub.Infrastructure.Http
await using var stream = await content.ReadAsStreamAsync(token);
var chunk = new byte[64 * 1024];
var copied = 0L;
int read;
var read = default(int);
while ((read = await stream.ReadAsync(chunk, token)) > 0)
{
copied += read;
@@ -671,7 +655,7 @@ namespace PrivaPub.Infrastructure.Http
await using var stream = await content.ReadAsStreamAsync(token);
using var buffer = new MemoryStream();
var chunk = new byte[16 * 1024];
int read;
var read = default(int);
while ((read = await stream.ReadAsync(chunk, token)) > 0)
{
if (buffer.Length + read > limit)
@@ -704,8 +688,8 @@ namespace PrivaPub.Infrastructure.Http
exchange.Failed(code);
else
exchange.Refused(code);
_cache.Set(negativeKey, new Refusal(transient, exchange.Status ?? 0), NegativeCacheLifetime);
_logger.LogInformation("GET {Url} refused: {Reason}", url, reason);
cache.Set(negativeKey, new Refusal(transient, exchange.Status ?? 0), NegativeCacheLifetime);
logger.LogInformation("GET {Url} refused: {Reason}", url, reason);
return default;
}
}
+3 -6
View File
@@ -28,13 +28,10 @@ namespace PrivaPub.Infrastructure.Http
return new Restore(previous);
}
sealed class Restore : IDisposable
sealed class Restore(
Tags previous) : IDisposable
{
readonly Tags _previous;
public Restore(Tags previous) => _previous = previous;
public void Dispose() => current.Value = _previous;
public void Dispose() => current.Value = previous;
}
}
}
@@ -23,11 +23,12 @@ namespace PrivaPub.Infrastructure.Http
ConnectTimeout = connectTimeout ?? TimeSpan.FromSeconds(10),
PooledConnectionLifetime = TimeSpan.FromMinutes(2),
MaxResponseHeadersLength = 64,
ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token, connected)
ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, connected, token)
};
public static async ValueTask<Stream> Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token,
IConnectedAddresses connected = default)
public static async ValueTask<Stream> Connect(DnsEndPoint endPoint, bool allowPrivateNetworks,
IConnectedAddresses connected = default,
CancellationToken token = default)
{
var addresses = await Resolve(endPoint.Host, token);
if (addresses.Length == 0 || !allowPrivateNetworks && !addresses.All(IpRangeGuard.IsPublic))
@@ -13,17 +13,12 @@ namespace PrivaPub.Infrastructure.Jobs
Task Failed(string host, string error, CancellationToken token);
}
public class HostCircuitBreaker : IHostCircuitBreaker
public class HostCircuitBreaker(
IMemoryCache cache) : IHostCircuitBreaker
{
public const int Threshold = 10;
static readonly TimeSpan CacheLifetime = TimeSpan.FromSeconds(30);
readonly IMemoryCache _cache;
public HostCircuitBreaker(IMemoryCache cache)
{
_cache = cache;
}
public async Task<DateTime?> UnavailableUntil(string host, CancellationToken token)
{
@@ -43,7 +38,7 @@ namespace PrivaPub.Infrastructure.Jobs
.Modify(i => i.LastSuccessAt, DateTime.UtcNow)
.Option(o => o.IsUpsert = true)
.ExecuteAsync(token);
_cache.Remove(Key(host));
cache.Remove(Key(host));
}
public async Task Failed(string host, string error, CancellationToken token)
@@ -61,11 +56,11 @@ namespace PrivaPub.Infrastructure.Jobs
await DB.Default.Update<RemoteInstance>().MatchID(instance.ID)
.Modify(i => i.UnavailableUntil, now + quarantine)
.ExecuteAsync(token);
_cache.Remove(Key(host));
cache.Remove(Key(host));
}
async Task<RemoteInstance> Instance(string host, CancellationToken token) =>
await _cache.GetOrCreateAsync(Key(host), async entry =>
await cache.GetOrCreateAsync(Key(host), async entry =>
{
entry.AbsoluteExpirationRelativeToNow = CacheLifetime;
return await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
+23 -29
View File
@@ -14,25 +14,18 @@ namespace PrivaPub.Infrastructure.Jobs
Task<JobOutcome> Handle(Job job, CancellationToken token);
}
public class JobWorker : BackgroundService
public class JobWorker(
IJobQueue jobQueue,
IEnumerable<IJobHandler> handlers,
ILogger<JobWorker> logger) : BackgroundService
{
static readonly TimeSpan Poll = TimeSpan.FromSeconds(5);
static readonly TimeSpan ReapInterval = TimeSpan.FromSeconds(30);
readonly IJobQueue _queue;
readonly IEnumerable<IJobHandler> _handlers;
readonly ILogger<JobWorker> _logger;
public JobWorker(IJobQueue queue, IEnumerable<IJobHandler> handlers, ILogger<JobWorker> logger)
{
_queue = queue;
_handlers = handlers;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var loops = _handlers.SelectMany(handler =>
var loops = handlers.SelectMany(handler =>
{
var inFlight = new ConcurrentDictionary<string, int>(StringComparer.OrdinalIgnoreCase);
return Enumerable.Range(0, handler.Concurrency).Select(_ => Run(handler, inFlight, stoppingToken));
@@ -44,11 +37,11 @@ namespace PrivaPub.Infrastructure.Jobs
{
while (!stoppingToken.IsCancellationRequested)
{
Job job;
var job = default(Job);
try
{
var busy = inFlight.Where(h => h.Value >= handler.PerHostLimit).Select(h => h.Key).ToList();
job = await _queue.Lease(handler.Kind, busy, stoppingToken);
job = await jobQueue.Lease(handler.Kind, busy, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
@@ -56,14 +49,14 @@ namespace PrivaPub.Infrastructure.Jobs
}
catch (Exception ex)
{
_logger.LogError(ex, "{Worker} could not lease a {Kind} job", nameof(JobWorker), handler.Kind);
logger.LogError(ex, "{Worker} could not lease a {Kind} job", nameof(JobWorker), handler.Kind);
await Delay(Poll, stoppingToken);
continue;
}
if (job == default)
{
await _queue.WaitForWork(handler.Kind, Poll, stoppingToken);
await jobQueue.WaitForWork(handler.Kind, Poll, stoppingToken);
continue;
}
@@ -75,7 +68,7 @@ namespace PrivaPub.Infrastructure.Jobs
var renewing = KeepLease(job, running);
try
{
JobOutcome outcome;
var outcome = default(JobOutcome);
try
{
using var scope = HttpScope.Triggered(handler.Kind.ToString().ToLowerInvariant());
@@ -87,24 +80,24 @@ namespace PrivaPub.Infrastructure.Jobs
}
catch (OperationCanceledException) when (running.IsCancellationRequested)
{
_logger.LogWarning("{Kind} job {Id} lost its lease and was stopped", handler.Kind, job.ID);
logger.LogWarning("{Kind} job {Id} lost its lease and was stopped", handler.Kind, job.ID);
continue;
}
catch (Exception ex)
{
_logger.LogError(ex, "{Kind} job {Id} threw", handler.Kind, job.ID);
logger.LogError(ex, "{Kind} job {Id} threw", handler.Kind, job.ID);
outcome = JobOutcome.Retry(ex.GetType().Name);
}
if (outcome.Result != JobResult.Done && job.Attempts >= handler.MaxAttempts && outcome.Result == JobResult.Retry)
_logger.LogWarning("{Kind} job {Id} for {Host} is dead after {Attempts} attempts: {Error}",
logger.LogWarning("{Kind} job {Id} for {Host} is dead after {Attempts} attempts: {Error}",
handler.Kind, job.ID, job.Host, job.Attempts, outcome.Error);
if (!await _queue.Finish(job, outcome, handler.MaxAttempts, CancellationToken.None))
_logger.LogWarning("{Kind} job {Id} lost its lease before it finished; its outcome is dropped", handler.Kind, job.ID);
if (!await jobQueue.Finish(job, outcome, handler.MaxAttempts, CancellationToken.None))
logger.LogWarning("{Kind} job {Id} lost its lease before it finished; its outcome is dropped", handler.Kind, job.ID);
}
catch (Exception ex)
{
_logger.LogError(ex, "{Kind} job {Id} could not be finished; its lease will expire", handler.Kind, job.ID);
logger.LogError(ex, "{Kind} job {Id} could not be finished; its lease will expire", handler.Kind, job.ID);
}
finally
{
@@ -116,9 +109,10 @@ namespace PrivaPub.Infrastructure.Jobs
}
// renews the lease every third of its length until the handler ends; a lease found lost cancels the handler
[System.Diagnostics.CodeAnalysis.SuppressMessage("HouseRules", "HR0003", Justification = "Takes the source it cancels when the lease is lost; its token already follows the worker's.")]
async Task KeepLease(Job job, CancellationTokenSource running)
{
var every = _queue.LeaseFor / 3;
var every = jobQueue.LeaseFor / 3;
while (!running.IsCancellationRequested)
{
try
@@ -132,7 +126,7 @@ namespace PrivaPub.Infrastructure.Jobs
try
{
if (await _queue.Renew(job, CancellationToken.None))
if (await jobQueue.Renew(job, CancellationToken.None))
continue;
await running.CancelAsync();
return;
@@ -140,7 +134,7 @@ namespace PrivaPub.Infrastructure.Jobs
catch (Exception ex)
{
// a passing database error: the lease has two more thirds to go, so try again at the next turn
_logger.LogWarning(ex, "{Worker} could not renew the lease of job {Id}", nameof(JobWorker), job.ID);
logger.LogWarning(ex, "{Worker} could not renew the lease of job {Id}", nameof(JobWorker), job.ID);
}
}
}
@@ -151,9 +145,9 @@ namespace PrivaPub.Infrastructure.Jobs
{
try
{
var reaped = await _queue.Reap(stoppingToken);
var reaped = await jobQueue.Reap(stoppingToken);
if (reaped > 0)
_logger.LogWarning("{Worker} returned {Count} expired leases to the queue", nameof(JobWorker), reaped);
logger.LogWarning("{Worker} returned {Count} expired leases to the queue", nameof(JobWorker), reaped);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
@@ -161,7 +155,7 @@ namespace PrivaPub.Infrastructure.Jobs
}
catch (Exception ex)
{
_logger.LogError(ex, "{Worker} reaper failed", nameof(JobWorker));
logger.LogError(ex, "{Worker} reaper failed", nameof(JobWorker));
}
await Delay(ReapInterval, stoppingToken);
}
+33 -46
View File
@@ -42,7 +42,15 @@ namespace PrivaPub.Infrastructure.Push
// safe handler, with the pasture's exceptions where Federation says so).
// A 404 or 410 means the subscription is gone: it is deleted. 408, 429, a 5xx, a timeout or a broken connection retry on
// the jobs' backoff; any other answer drops the push (logged by the push service's host alone).
public class WebPushJob : IJobHandler
public class WebPushJob(
IHttpClientFactory http,
IVapid vapid,
IServiceScopeFactory scopes,
IInstanceSettings settings,
ILocalActorService localActors,
IOptionsMonitor<FederationOptions> federation,
IStringLocalizer<GenericRes> localizer,
ILogger<WebPushJob> logger) : IJobHandler
{
public const string ClientName = "push";
public static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(15);
@@ -52,27 +60,6 @@ namespace PrivaPub.Infrastructure.Push
const string Urgency = "normal";
const int BodyLength = 140;
readonly IHttpClientFactory _http;
readonly IVapid _vapid;
readonly IServiceScopeFactory _scopes;
readonly IInstanceSettings _settings;
readonly ILocalActorService _localActors;
readonly IOptionsMonitor<FederationOptions> _federation;
readonly IStringLocalizer<GenericRes> _localizer;
readonly ILogger<WebPushJob> _logger;
public WebPushJob(IHttpClientFactory http, IVapid vapid, IServiceScopeFactory scopes, IInstanceSettings settings, ILocalActorService localActors,
IOptionsMonitor<FederationOptions> federation, IStringLocalizer<GenericRes> localizer, ILogger<WebPushJob> logger)
{
_http = http;
_vapid = vapid;
_scopes = scopes;
_settings = settings;
_localActors = localActors;
_federation = federation;
_localizer = localizer;
_logger = logger;
}
public JobKind Kind => JobKind.SendPush;
public int Concurrency => 4;
@@ -96,22 +83,22 @@ namespace PrivaPub.Infrastructure.Push
var notification = await DB.Default.Find<Notification>().Match(n => n.ID == notificationId && n.AvatarId == avatarId).ExecuteFirstAsync(token);
if (notification == default || !await WebPush.Pushable(subscription, notification, token))
return JobOutcome.Done;
if (!Uri.TryCreate(subscription.Endpoint, UriKind.Absolute, out var endpoint) || !IsAllowed(endpoint, _federation.CurrentValue))
if (!Uri.TryCreate(subscription.Endpoint, UriKind.Absolute, out var endpoint) || !IsAllowed(endpoint, federation.CurrentValue))
return JobOutcome.Dead("endpoint");
byte[] json;
using (var scope = _scopes.CreateScope())
var json = default(byte[]);
using (var scope = scopes.CreateScope())
{
// the token the subscription was made with must still be valid, and the persona's: it goes into the payload
if (!await TokenHolds(scope.ServiceProvider.GetRequiredService<IOpenIddictTokenManager>(), subscription, token))
{
await WebPush.ForgetToken(subscription.TokenId);
await WebPush.ForgetToken(subscription.TokenId, token);
return JobOutcome.Done;
}
var avatar = await DB.Default.Find<Avatar>().Match(a => a.ID == avatarId && a.DeletionAt == null).ExecuteFirstAsync(token);
if (avatar == default)
{
await WebPush.Forget(new[] { avatarId });
await WebPush.Forget(new[] { avatarId }, token);
return JobOutcome.Done;
}
var mapped = (await scope.ServiceProvider.GetRequiredService<MastodonMapper>().Notifications(new List<Notification> { notification }, avatarId, token))
@@ -130,7 +117,7 @@ namespace PrivaPub.Infrastructure.Push
});
}
var key = await _vapid.Key(token);
var key = await vapid.Key(token);
var p256dh = WebPushCrypto.FromBase64Url(subscription.P256dh);
var auth = WebPushCrypto.FromBase64Url(subscription.Auth);
if (!WebPushCrypto.AreUsable(p256dh, auth))
@@ -154,24 +141,24 @@ namespace PrivaPub.Infrastructure.Push
try
{
using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
using var response = await http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
var status = (int)response.StatusCode;
if (status is >= 200 and < 300)
return JobOutcome.Done;
if (response.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Gone)
{
await DB.Default.DeleteAsync<PushSubscription>(subscription.ID);
_logger.LogInformation("{Kind}: {Host} answered {Status}, the subscription is gone", Kind, endpoint.Host, status);
logger.LogInformation("{Kind}: {Host} answered {Status}, the subscription is gone", Kind, endpoint.Host, status);
return JobOutcome.Done;
}
if (response.StatusCode is HttpStatusCode.RequestTimeout or HttpStatusCode.TooManyRequests || status >= 500)
return JobOutcome.Retry($"status {status}");
_logger.LogWarning("{Kind}: {Host} refused a push with {Status}; it is dropped", Kind, endpoint.Host, status);
logger.LogWarning("{Kind}: {Host} refused a push with {Status}; it is dropped", Kind, endpoint.Host, status);
return JobOutcome.Dead($"status {status}");
}
catch (BlockedDestinationException)
{
_logger.LogWarning("{Kind}: {Host} does not resolve to a public address; the push is dropped", Kind, endpoint.Host);
logger.LogWarning("{Kind}: {Host} does not resolve to a public address; the push is dropped", Kind, endpoint.Host);
return JobOutcome.Dead("blocked destination");
}
catch (TaskCanceledException) when (!token.IsCancellationRequested)
@@ -180,7 +167,7 @@ namespace PrivaPub.Infrastructure.Push
}
catch (HttpRequestException ex) when (ex.InnerException is BlockedDestinationException)
{
_logger.LogWarning("{Kind}: {Host} does not resolve to a public address; the push is dropped", Kind, endpoint.Host);
logger.LogWarning("{Kind}: {Host} does not resolve to a public address; the push is dropped", Kind, endpoint.Host);
return JobOutcome.Dead("blocked destination");
}
catch (Exception ex) when (ex is HttpRequestException or IOException or SocketException)
@@ -212,10 +199,10 @@ namespace PrivaPub.Infrastructure.Push
// so it names a mailbox at its host: RFC 8292 takes only mailto: and https:.)
string Subject()
{
var email = _settings.Current.ContactEmail?.Trim();
var email = settings.Current.ContactEmail?.Trim();
if (!string.IsNullOrEmpty(email) && !email.Any(char.IsWhiteSpace) && email.Contains('@'))
return "mailto:" + email;
var address = _localActors.BaseAddress;
var address = localActors.BaseAddress;
return Uri.TryCreate(address, UriKind.Absolute, out var base_) && base_.Scheme != Uri.UriSchemeHttps
? "mailto:privapub@" + base_.Host
: address;
@@ -234,17 +221,17 @@ namespace PrivaPub.Infrastructure.Push
var name = string.IsNullOrWhiteSpace(mapped.Account?.DisplayName) ? mapped.Account?.Username ?? string.Empty : mapped.Account.DisplayName;
return notification.Type switch
{
NotificationType.Mention => string.Format(_localizer["You were mentioned by {0}"], name),
NotificationType.Status => string.Format(_localizer["{0} just posted"], name),
NotificationType.Reblog => string.Format(_localizer["{0} boosted your post"], name),
NotificationType.Follow => string.Format(_localizer["{0} is now following you"], name),
NotificationType.FollowRequest => string.Format(_localizer["Pending follower: {0}"], name),
NotificationType.Favourite => string.Format(_localizer["{0} favorited your post"], name),
NotificationType.Poll => string.Format(_localizer["A poll by {0} has ended"], name),
NotificationType.Update => string.Format(_localizer["{0} edited a post"], name),
NotificationType.Quote => string.Format(_localizer["{0} quoted your post"], name),
NotificationType.Reaction => string.Format(_localizer["{0} reacted to your post with {1}"], name, notification.Emoji),
NotificationType.ModerationWarning => _localizer["You have received a moderation warning"],
NotificationType.Mention => string.Format(localizer["You were mentioned by {0}"], name),
NotificationType.Status => string.Format(localizer["{0} just posted"], name),
NotificationType.Reblog => string.Format(localizer["{0} boosted your post"], name),
NotificationType.Follow => string.Format(localizer["{0} is now following you"], name),
NotificationType.FollowRequest => string.Format(localizer["Pending follower: {0}"], name),
NotificationType.Favourite => string.Format(localizer["{0} favorited your post"], name),
NotificationType.Poll => string.Format(localizer["A poll by {0} has ended"], name),
NotificationType.Update => string.Format(localizer["{0} edited a post"], name),
NotificationType.Quote => string.Format(localizer["{0} quoted your post"], name),
NotificationType.Reaction => string.Format(localizer["{0} reacted to your post with {1}"], name, notification.Emoji),
NotificationType.ModerationWarning => localizer["You have received a moderation warning"],
_ => name
};
}
@@ -27,7 +27,12 @@ namespace PrivaPub.Infrastructure.Statistics
public const string Served = "served";
}
public class InteractionLedger : BackgroundService, IInteractionLedger
public class InteractionLedger(
InteractionSalts salts,
IOptionsMonitor<StatisticsOptions> options,
ILogger<InteractionLedger> logger,
IJobQueue queue = default,
IDomainBlocks blocks = default) : BackgroundService, IInteractionLedger
{
const int Capacity = 10_000;
const int BatchSize = 1000;
@@ -46,11 +51,6 @@ namespace PrivaPub.Infrastructure.Statistics
static readonly TimeSpan TouchInterval = TimeSpan.FromHours(1);
static readonly HashSet<string> TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context", "replies" };
readonly InteractionSalts _salts;
readonly IOptionsMonitor<StatisticsOptions> _options;
readonly ILogger<InteractionLedger> _logger;
readonly IJobQueue _queue;
readonly IDomainBlocks _blocks;
readonly ConcurrentDictionary<string, byte> _touches = new(StringComparer.Ordinal);
readonly ConcurrentDictionary<string, DateTime> _touchedAt = new(StringComparer.Ordinal);
readonly SemaphoreSlim _flushing = new(1, 1);
@@ -63,36 +63,26 @@ namespace PrivaPub.Infrastructure.Statistics
long _written;
long _failed;
public InteractionLedger(InteractionSalts salts, IOptionsMonitor<StatisticsOptions> options, ILogger<InteractionLedger> logger,
IJobQueue queue = default, IDomainBlocks blocks = default)
{
_salts = salts;
_options = options;
_logger = logger;
_queue = queue;
_blocks = blocks;
}
public long Dropped => Interlocked.Read(ref _dropped);
public void Record(InteractionEvent interaction, string actorUri = default, bool hostClaimed = false)
{
try
{
if (interaction == default || !_options.CurrentValue.Enabled)
if (interaction == default || !options.CurrentValue.Enabled)
return;
if (!_channel.Writer.TryWrite(new Pending(interaction, actorUri, hostClaimed)))
Interlocked.Increment(ref _dropped);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "An interaction could not be recorded");
logger.LogDebug(ex, "An interaction could not be recorded");
}
}
public void Count(string host, string key, long bytes = 0)
{
if (!_options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
if (!options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
return;
var day = DateTime.UtcNow.Date;
host = Interactions.Host(host);
@@ -107,7 +97,7 @@ namespace PrivaPub.Infrastructure.Statistics
public void CountServer(string section, string key, int? latencyMs = default)
{
if (!_options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
if (!options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
return;
var day = DateTime.UtcNow.Date;
var field = section == ServerSections.Served ? nameof(ServerDay.Served) : nameof(ServerDay.Client);
@@ -145,7 +135,7 @@ namespace PrivaPub.Infrastructure.Statistics
}
catch (Exception ex)
{
_logger.LogWarning(ex, "The interaction ledger could not flush");
logger.LogWarning(ex, "The interaction ledger could not flush");
await Task.Delay(BatchWait, CancellationToken.None);
}
}
@@ -161,7 +151,7 @@ namespace PrivaPub.Infrastructure.Statistics
}
catch (Exception ex)
{
_logger.LogWarning(ex, "The interaction ledger could not flush on shutdown");
logger.LogWarning(ex, "The interaction ledger could not flush on shutdown");
}
}
@@ -209,10 +199,10 @@ namespace PrivaPub.Infrastructure.Statistics
.Modify(i => i.LastSeenAt, now)
.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
.Option(o => o.IsUpsert = true)
.ExecuteAsync(token));
.ExecuteAsync(token), token);
_knownHosts.Add(host);
if (_queue != default && _blocks?.IsSuspended(host) != true)
await _queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
if (queue != default && blocks?.IsSuspended(host) != true)
await queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
}
if (_touchedAt.Count > 100_000)
_touchedAt.Clear();
@@ -238,7 +228,7 @@ namespace PrivaPub.Infrastructure.Statistics
catch (Exception ex) when (ex is not OperationCanceledException)
{
Interlocked.Add(ref _failed, batch.Count);
_logger.LogWarning(ex, "{Count} interactions could not be written", batch.Count);
logger.LogWarning(ex, "{Count} interactions could not be written", batch.Count);
}
}
}
@@ -254,7 +244,7 @@ namespace PrivaPub.Infrastructure.Statistics
if (pending.HostClaimed && interaction.Host != Interactions.Unknown && !(await KnownHosts(token)).Contains(interaction.Host))
interaction.Host = Interactions.Unknown;
if (pending.ActorUri != default)
interaction.ActorHash = InteractionSalts.Hash(await _salts.For(interaction.At, token), pending.ActorUri);
interaction.ActorHash = InteractionSalts.Hash(await salts.For(interaction.At, token), pending.ActorUri);
if (interaction.Host != Interactions.Unknown && Touches(interaction, pending.ActorUri != default))
_touches[interaction.Host] = 0;
return interaction;
@@ -272,7 +262,7 @@ namespace PrivaPub.Infrastructure.Statistics
// two flushes upserting a new day's document at once: one inserts it and the other's insert fails on the unique
// key; the second try finds the document and adds to it (MongoDB retries this only for single-document updates)
static async Task Upsert(Func<Task> write)
static async Task Upsert(Func<Task> write, CancellationToken token = default)
{
try
{
@@ -294,7 +284,7 @@ namespace PrivaPub.Infrastructure.Statistics
.Match(d => d.Day == key.Day && d.Host == key.Host)
.Modify(b => b.Inc($"{nameof(InstanceDay.Reads)}.{key.Key}", count))
.Option(o => o.IsUpsert = true)
.ExecuteAsync(token));
.ExecuteAsync(token), token);
}
var day = DateTime.UtcNow.Date;
@@ -311,7 +301,7 @@ namespace PrivaPub.Infrastructure.Statistics
.Modify(b => b.Inc(d => d.LedgerFailed, failed))
.Modify(b => b.Inc(d => d.LedgerDropped, newlyDropped))
.Option(o => o.IsUpsert = true)
.ExecuteAsync(token));
.ExecuteAsync(token), token);
foreach (var key in server)
{
if (!_server.TryRemove(key, out var count) || count == 0)
@@ -320,7 +310,7 @@ namespace PrivaPub.Infrastructure.Statistics
.Match(d => d.Day == key.Day)
.Modify(b => b.Inc($"{key.Field}.{key.Key}", count))
.Option(o => o.IsUpsert = true)
.ExecuteAsync(token));
.ExecuteAsync(token), token);
}
}
@@ -17,7 +17,7 @@ namespace PrivaPub.Infrastructure.Statistics
day = day.Date;
if (_known.TryGetValue(day, out var cached))
return cached;
InteractionSalt salt;
var salt = default(InteractionSalt);
if (day == DateTime.UtcNow.Date)
salt = await DB.Default.UpdateAndGet<InteractionSalt>()
.Match(s => s.Day == day)
+10 -19
View File
@@ -11,13 +11,11 @@ using System.Globalization;
namespace PrivaPub.Infrastructure.Statistics
{
public class RollupJob : IJobHandler
public class RollupJob(
IJobQueue queue) : IJobHandler
{
public static readonly TimeSpan After = TimeSpan.FromMinutes(30);
readonly IJobQueue _queue;
public RollupJob(IJobQueue queue) => _queue = queue;
public JobKind Kind => JobKind.RollupDay;
public int Concurrency => 1;
@@ -47,7 +45,7 @@ namespace PrivaPub.Infrastructure.Statistics
await DB.Default.DeleteAsync<InteractionSalt>(s => s.Day == day);
var next = end;
if (next < DateTime.UtcNow.Date.AddDays(1))
await _queue.EnqueueMany(new[] { For(next) }, token);
await queue.EnqueueMany(new[] { For(next) }, token);
return JobOutcome.Done;
}
@@ -95,21 +93,14 @@ namespace PrivaPub.Infrastructure.Statistics
}
}
public class StatisticsSchedule : BackgroundService
public class StatisticsSchedule(
IJobQueue queue,
ILogger<StatisticsSchedule> logger,
IOptionsMonitor<StatisticsOptions> options = default) : BackgroundService
{
static readonly TimeSpan Interval = TimeSpan.FromHours(1);
const int CatchUpDays = 7;
readonly IJobQueue _queue;
readonly ILogger<StatisticsSchedule> _logger;
readonly IOptionsMonitor<StatisticsOptions> _options;
public StatisticsSchedule(IJobQueue queue, ILogger<StatisticsSchedule> logger, IOptionsMonitor<StatisticsOptions> options = default)
{
_queue = queue;
_logger = logger;
_options = options;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
@@ -121,7 +112,7 @@ namespace PrivaPub.Infrastructure.Statistics
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Statistics rollups could not be planned");
logger.LogWarning(ex, "Statistics rollups could not be planned");
}
try
{
@@ -146,12 +137,12 @@ namespace PrivaPub.Infrastructure.Statistics
if (!rolled && await DB.Default.Find<InteractionEvent>().Match(e => e.At >= day && e.At < end).ExecuteAnyAsync(token))
jobs.Add(RollupJob.For(day));
}
if (_options?.CurrentValue.Crawler.Enabled == true)
if (options?.CurrentValue.Crawler.Enabled == true)
{
var now = DateTime.UtcNow;
jobs.Add(Federation.Crawler.CrawlPlanner.For(new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Utc)));
}
await _queue.EnqueueMany(jobs, token);
await queue.EnqueueMany(jobs, token);
}
}
}