House rules: primary constructors, a cancellation token on every async call, and the rules fail the build
Build / Build (push) Successful in 30s
Build / Build (push) Successful in 30s
- Services, controllers and jobs take their dependencies through primary constructors and use them directly; where a method already named a local after a dependency, the dependency takes its type's name. - Every async method takes a CancellationToken last and passes it down to MongoDB, HTTP and the job queue. The setup claim is still given back when a request is cancelled. - The federation and media options are read through IOptionsMonitor. - The Mastodon entities keep sending "" for empty strings, and every model field where null means something says so. - The inbox's 503 with Retry-After, which other servers retry, is the one server error left, and says why. - Arasaka.HouseRules.Analyzers checks all of this in every build, here and in PrivaPub.ClientModels as decePubClient builds it. Checked against GoToSocial in the pasture: 64 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
This commit is contained in:
1 parent
f0a14c737f
commit
0f73f69239
176 files changed
+2789
-3974
No files matched your search
@@ -54,12 +54,13 @@ namespace PrivaPub.Infrastructure.Backup
|
||||
|
||||
public sealed class Held : IAsyncDisposable
|
||||
{
|
||||
readonly string _owner;
|
||||
readonly Timer _heartbeat;
|
||||
|
||||
string Owner { get; }
|
||||
|
||||
public Held(string owner)
|
||||
{
|
||||
_owner = owner;
|
||||
Owner = owner;
|
||||
_heartbeat = new Timer(_ => _ = DB.Default.Update<MaintenanceLock>().Match(l => l.ID == Name && l.Owner == owner)
|
||||
.Modify(l => l.Heartbeat, DateTime.UtcNow).ExecuteAsync(), default, Beat, Beat);
|
||||
}
|
||||
@@ -67,7 +68,7 @@ namespace PrivaPub.Infrastructure.Backup
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _heartbeat.DisposeAsync();
|
||||
await DB.Default.Update<MaintenanceLock>().Match(l => l.ID == Name && l.Owner == _owner)
|
||||
await DB.Default.Update<MaintenanceLock>().Match(l => l.ID == Name && l.Owner == Owner)
|
||||
.Modify(l => l.Owner, null).Modify(l => l.Heartbeat, DateTime.MinValue).ExecuteAsync();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ namespace PrivaPub.Infrastructure.Backup
|
||||
file.Seek(offset, SeekOrigin.Begin);
|
||||
var buffer = new byte[81920];
|
||||
var total = 0L;
|
||||
int read;
|
||||
var read = default(int);
|
||||
while ((read = await piece.ReadAsync(buffer, token)) > 0)
|
||||
{
|
||||
total += read;
|
||||
|
||||
@@ -41,36 +41,36 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
/// <summary>Whether a command runs before migrations: backups are of the database as it was.</summary>
|
||||
public static bool BeforeMigrations(string[] args) => args is ["backup", ..] or ["backups"] or ["restore", ..];
|
||||
|
||||
public static async Task<int> Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default)
|
||||
public static async Task<int> Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default, CancellationToken token = default)
|
||||
{
|
||||
input ??= Console.In;
|
||||
output ??= Console.Out;
|
||||
switch (args)
|
||||
{
|
||||
case [("promote" or "demote") and var verb, var userName]:
|
||||
return await Role(services, userName, verb == "promote" ? RootRole.Admin : RootRole.User, output);
|
||||
return await Role(services, userName, verb == "promote" ? RootRole.Admin : RootRole.User, output, token: token);
|
||||
case ["role", var userName, var role] when RootUser.TryParseRole(role, out var parsed):
|
||||
return await Role(services, userName, parsed, output);
|
||||
return await Role(services, userName, parsed, output, token: token);
|
||||
case ["create-root", var login, .. var flags] when flags.All(f => f == "--admin"):
|
||||
return await CreateRoot(services, login, input.ReadLine(), flags.Contains("--admin"), output);
|
||||
return await CreateRoot(services, login, input.ReadLine(), flags.Contains("--admin"), output, token);
|
||||
case ["move-persona", var persona, var root]:
|
||||
return await MovePersona(services, persona, root, output);
|
||||
return await MovePersona(services, persona, root, output, token);
|
||||
case ["delete-root", var root]:
|
||||
return await DeleteRoot(services, root, output);
|
||||
return await DeleteRoot(services, root, output, token);
|
||||
case ["reset-totp", var root]:
|
||||
return await ResetTotp(services, root, output);
|
||||
return await ResetTotp(services, root, output, token);
|
||||
case ["media", "audit", .. var flags] when flags.All(f => f == "--fix"):
|
||||
return await AuditMedia(services, flags.Contains("--fix"), output);
|
||||
return await AuditMedia(services, flags.Contains("--fix"), output, token);
|
||||
case ["backup", "verify", var id]:
|
||||
return await VerifyBackup(services, id, output);
|
||||
return await VerifyBackup(services, id, output, token);
|
||||
case ["backup", .. var flags] when BackupFlags(flags, out var kind, out var dbOnly):
|
||||
return await BackUp(services, kind, dbOnly, output);
|
||||
return await BackUp(services, kind, dbOnly, output, token);
|
||||
case ["backups"]:
|
||||
return ListBackups(services, output);
|
||||
case ["restore", "--status"]:
|
||||
return await RestoreStatus(services, output);
|
||||
return await RestoreStatus(services, output, token);
|
||||
case ["restore", var id] when !id.StartsWith('-'):
|
||||
return await Restore(services, id, output);
|
||||
return await Restore(services, id, output, token);
|
||||
default:
|
||||
Console.Error.WriteLine(Usage);
|
||||
return 2;
|
||||
@@ -98,7 +98,7 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
return true;
|
||||
}
|
||||
|
||||
static async Task<int> BackUp(IServiceProvider services, string kind, bool dbOnly, TextWriter output)
|
||||
static async Task<int> BackUp(IServiceProvider services, string kind, bool dbOnly, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
var (made, error) = await services.GetRequiredService<Backups>().Create(kind, dbOnly, CancellationToken.None);
|
||||
if (made == default)
|
||||
@@ -123,7 +123,7 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
return 0;
|
||||
}
|
||||
|
||||
static async Task<int> VerifyBackup(IServiceProvider services, string id, TextWriter output)
|
||||
static async Task<int> VerifyBackup(IServiceProvider services, string id, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
var problems = await services.GetRequiredService<Backups>().Verify(id, CancellationToken.None);
|
||||
foreach (var problem in problems)
|
||||
@@ -134,7 +134,7 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
return 0;
|
||||
}
|
||||
|
||||
static async Task<int> Restore(IServiceProvider services, string id, TextWriter output)
|
||||
static async Task<int> Restore(IServiceProvider services, string id, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
var backups = services.GetRequiredService<Backups>();
|
||||
var refused = await ServerRestore.Request(backups.Context(), id, "cli", CancellationToken.None);
|
||||
@@ -143,18 +143,18 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
Console.Error.WriteLine(refused);
|
||||
return 1;
|
||||
}
|
||||
await Log(services, AdminActions.BackupRestore, AdminTargets.Backup, id, id);
|
||||
await Log(services, AdminActions.BackupRestore, AdminTargets.Backup, id, id, token: token);
|
||||
output.WriteLine($"{id} will be restored: the running service stops within seconds and restores it when it starts again");
|
||||
output.WriteLine("(systemd starts it again; follow it with journalctl -u privapub, then PrivaPub admin restore --status)");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static async Task<int> RestoreStatus(IServiceProvider services, TextWriter output)
|
||||
static async Task<int> RestoreStatus(IServiceProvider services, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
var waiting = RestoreMarker.Read(services.GetRequiredService<Backups>().Root);
|
||||
if (waiting != default)
|
||||
output.WriteLine($"waiting: {waiting.Backup}, {waiting.State}, attempt {waiting.Attempts} of {RestoreMarker.MaxAttempts}, asked by {waiting.RequestedBy} at {waiting.RequestedAt:u}{(waiting.Error == default ? string.Empty : $": {waiting.Error}")}");
|
||||
var last = await DB.Default.Find<RestoreRecord>().Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync();
|
||||
var last = await DB.Default.Find<RestoreRecord>().Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync(token);
|
||||
if (last == default)
|
||||
{
|
||||
if (waiting == default)
|
||||
@@ -177,7 +177,7 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
|
||||
static string Names(List<string> names) => names.Count == 0 ? "none" : string.Join(", ", names);
|
||||
|
||||
static async Task<int> AuditMedia(IServiceProvider services, bool fix, TextWriter output)
|
||||
static async Task<int> AuditMedia(IServiceProvider services, bool fix, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
var report = await Domain.Media.MediaAudit.Run(services.GetRequiredService<Domain.Media.IMediaService>(), fix, CancellationToken.None);
|
||||
output.WriteLine($"{report.Files} files served, {report.Held} rows holding media");
|
||||
@@ -194,15 +194,15 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
|
||||
// every act below is kept in the staff's log as the command line's ("cli"): a login's by its name alone, a persona's
|
||||
// without the login it went to
|
||||
static Task Log(IServiceProvider services, string action, string kind, string id, string label, string note = default) =>
|
||||
services.GetRequiredService<IModerationLog>().Write(AdminAction.Cli, action, kind, id, label, note: note);
|
||||
static Task Log(IServiceProvider services, string action, string kind, string id, string label, string note = default, CancellationToken token = default) =>
|
||||
services.GetRequiredService<IModerationLog>().Write(AdminAction.Cli, action, kind, id, label, note: note, token: token);
|
||||
|
||||
// The command line is the way back, so it may leave the server without an administrator (the client's route refuses
|
||||
// to); it says so, and `role <root> admin` gives one back.
|
||||
static async Task<int> Role(IServiceProvider services, string userName, RootRole role, TextWriter output, bool created = false)
|
||||
static async Task<int> Role(IServiceProvider services, string userName, RootRole role, TextWriter output, bool created = false, CancellationToken token = default)
|
||||
{
|
||||
userName = userName.ToLowerInvariant();
|
||||
var user = await DB.Default.Find<RootUser>().Match(u => u.UserName == userName && u.DeletedAt == null).ExecuteFirstAsync();
|
||||
var user = await DB.Default.Find<RootUser>().Match(u => u.UserName == userName && u.DeletedAt == null).ExecuteFirstAsync(token);
|
||||
if (user == default)
|
||||
{
|
||||
Console.Error.WriteLine($"no root user '{userName}'");
|
||||
@@ -211,17 +211,17 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
|
||||
var was = user.Role;
|
||||
user.SetRole(role);
|
||||
await DB.Default.Update<RootUser>().MatchID(user.ID).Modify(u => u.Policies, user.Policies).Modify(u => u.UpdatedAt, user.UpdatedAt).ExecuteAsync();
|
||||
await DB.Default.Update<RootUser>().MatchID(user.ID).Modify(u => u.Policies, user.Policies).Modify(u => u.UpdatedAt, user.UpdatedAt).ExecuteAsync(token);
|
||||
if (role == RootRole.Admin)
|
||||
await ServerSetup.Seal(user.ID, default);//an administrator exists: the client's setup is closed
|
||||
if (!created && was != role)
|
||||
await Log(services, role == RootRole.Admin && was == RootRole.User ? AdminActions.RootPromote
|
||||
: role == RootRole.User && was == RootRole.Admin ? AdminActions.RootDemote
|
||||
: AdminActions.RootRole, AdminTargets.Root, user.ID, userName, $"{Name(was)} → {Name(role)}");
|
||||
: AdminActions.RootRole, AdminTargets.Root, user.ID, userName, $"{Name(was)} → {Name(role)}", token);
|
||||
|
||||
output.WriteLine($"{userName}: {string.Join(", ", user.Policies)}");
|
||||
if (was == RootRole.Admin && role != RootRole.Admin
|
||||
&& !await DB.Default.Find<RootUser>().Match(u => u.DeletedAt == null && !u.IsBanned && u.Policies.Contains(Policies.IsAdmin)).ExecuteAnyAsync())
|
||||
&& !await DB.Default.Find<RootUser>().Match(u => u.DeletedAt == null && !u.IsBanned && u.Policies.Contains(Policies.IsAdmin)).ExecuteAnyAsync(token))
|
||||
Console.Error.WriteLine("no administrator is left: `PrivaPub admin role <root> admin` makes one");
|
||||
return 0;
|
||||
}
|
||||
@@ -230,7 +230,7 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
|
||||
// With sign-up closed, this or the client's setup (while no administrator exists) makes the first root; group invitations
|
||||
// make the rest.
|
||||
static async Task<int> CreateRoot(IServiceProvider services, string login, string password, bool admin, TextWriter output)
|
||||
static async Task<int> CreateRoot(IServiceProvider services, string login, string password, bool admin, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
var form = new LoginForm { UserName = login, Password = password?.Trim() };
|
||||
var problems = new List<ValidationResult>();
|
||||
@@ -239,18 +239,18 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
Console.Error.WriteLine(string.Join(Environment.NewLine, problems.Select(p => p.ErrorMessage)));
|
||||
return 1;
|
||||
}
|
||||
var created = await services.GetRequiredService<IRootUsersService>().SignUpAsync(form);
|
||||
var created = await services.GetRequiredService<IRootUsersService>().SignUpAsync(form, token);
|
||||
if (!created.IsValid)
|
||||
{
|
||||
Console.Error.WriteLine(created.ErrorMessage);
|
||||
return 1;
|
||||
}
|
||||
var made = await DB.Default.Find<RootUser>().Match(u => u.UserName == login.ToLowerInvariant()).ExecuteFirstAsync();
|
||||
await Log(services, AdminActions.RootCreate, AdminTargets.Root, made?.ID, login.ToLowerInvariant(), admin ? "administrator" : default);
|
||||
return admin ? await Role(services, login, RootRole.Admin, output, created: true) : await Report(login, output);
|
||||
var made = await DB.Default.Find<RootUser>().Match(u => u.UserName == login.ToLowerInvariant()).ExecuteFirstAsync(token);
|
||||
await Log(services, AdminActions.RootCreate, AdminTargets.Root, made?.ID, login.ToLowerInvariant(), admin ? "administrator" : default, token);
|
||||
return admin ? await Role(services, login, RootRole.Admin, output, created: true, token) : await Report(login, output, token);
|
||||
}
|
||||
|
||||
static Task<int> Report(string login, TextWriter output)
|
||||
static Task<int> Report(string login, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
output.WriteLine($"{login.ToLowerInvariant()}: created");
|
||||
return Task.FromResult(0);
|
||||
@@ -259,23 +259,23 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
// A persona given to another login (owner decision 2026-10-17, made for @thepra): its link moves first, so the old
|
||||
// login can no longer exchange a token for it, then its Mastodon tokens and authorizations end. Its actor, posts and
|
||||
// followers stay as they are, and nobody outside is told: nothing they hold changes.
|
||||
static async Task<int> MovePersona(IServiceProvider services, string personaName, string login, TextWriter output)
|
||||
static async Task<int> MovePersona(IServiceProvider services, string personaName, string login, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
personaName = personaName.TrimStart('@').ToLowerInvariant();
|
||||
login = login.ToLowerInvariant();
|
||||
var persona = await DB.Default.Find<Avatar>().Match(a => a.UserName == personaName && !a.DeletionAt.HasValue).ExecuteFirstAsync();
|
||||
var persona = await DB.Default.Find<Avatar>().Match(a => a.UserName == personaName && !a.DeletionAt.HasValue).ExecuteFirstAsync(token);
|
||||
if (persona == default)
|
||||
{
|
||||
Console.Error.WriteLine($"no persona '@{personaName}'");
|
||||
return 1;
|
||||
}
|
||||
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login).ExecuteFirstAsync();
|
||||
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login).ExecuteFirstAsync(token);
|
||||
if (root == default || root.DeletedAt.HasValue || root.IsBanned)
|
||||
{
|
||||
Console.Error.WriteLine($"no root user '{login}' that can take it");
|
||||
return 1;
|
||||
}
|
||||
var link = await DB.Default.Find<RootToAvatar>().Match(ra => ra.AvatarId == persona.ID).ExecuteFirstAsync();
|
||||
var link = await DB.Default.Find<RootToAvatar>().Match(ra => ra.AvatarId == persona.ID).ExecuteFirstAsync(token);
|
||||
if (link == default)
|
||||
{
|
||||
Console.Error.WriteLine($"'@{personaName}' belongs to no login");
|
||||
@@ -287,52 +287,52 @@ namespace PrivaPub.Infrastructure.Cli
|
||||
return 0;
|
||||
}
|
||||
|
||||
await DB.Default.Update<RootToAvatar>().MatchID(link.ID).Modify(ra => ra.RootId, root.ID).ExecuteAsync();
|
||||
await services.GetRequiredService<IOpenIddictTokenManager>().RevokeBySubjectAsync(persona.ID);
|
||||
await services.GetRequiredService<IOpenIddictAuthorizationManager>().RevokeBySubjectAsync(persona.ID);
|
||||
await Domain.Social.WebPush.Forget(new[] { persona.ID });//its Web Push subscriptions go with its tokens (R9)
|
||||
await Log(services, AdminActions.PersonaMove, AdminTargets.Persona, persona.ID, "@" + personaName, "moved to another login");
|
||||
await DB.Default.Update<RootToAvatar>().MatchID(link.ID).Modify(ra => ra.RootId, root.ID).ExecuteAsync(token);
|
||||
await services.GetRequiredService<IOpenIddictTokenManager>().RevokeBySubjectAsync(persona.ID, token);
|
||||
await services.GetRequiredService<IOpenIddictAuthorizationManager>().RevokeBySubjectAsync(persona.ID, token);
|
||||
await Domain.Social.WebPush.Forget(new[] { persona.ID }, token);//its Web Push subscriptions go with its tokens (R9)
|
||||
await Log(services, AdminActions.PersonaMove, AdminTargets.Persona, persona.ID, "@" + personaName, "moved to another login", token);
|
||||
output.WriteLine($"@{personaName}: moved to {login}");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// The way back for a login that lost its authenticator and its recovery codes: its two-factor sign-in goes, its
|
||||
// password stays.
|
||||
static async Task<int> ResetTotp(IServiceProvider services, string login, TextWriter output)
|
||||
static async Task<int> ResetTotp(IServiceProvider services, string login, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
login = login.ToLowerInvariant();
|
||||
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login && u.DeletedAt == null).ExecuteFirstAsync();
|
||||
var updated = root == default ? default : await DB.Default.Update<RootUser>().MatchID(root.ID).Modify(u => u.Totp, null).ExecuteAsync();
|
||||
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login && u.DeletedAt == null).ExecuteFirstAsync(token);
|
||||
var updated = root == default ? default : await DB.Default.Update<RootUser>().MatchID(root.ID).Modify(u => u.Totp, null).ExecuteAsync(token);
|
||||
if (updated == default || updated.MatchedCount == 0)
|
||||
{
|
||||
Console.Error.WriteLine($"no root user '{login}'");
|
||||
return 1;
|
||||
}
|
||||
await Log(services, AdminActions.RootResetTotp, AdminTargets.Root, root.ID, login);
|
||||
await Log(services, AdminActions.RootResetTotp, AdminTargets.Root, root.ID, login, token: token);
|
||||
output.WriteLine($"{login}: two-factor sign-in is off");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// A login deleted from the box as the client deletes its own (RootRemoval): every persona it still has goes with it,
|
||||
// announced deleted, and its sessions end.
|
||||
static async Task<int> DeleteRoot(IServiceProvider services, string login, TextWriter output)
|
||||
static async Task<int> DeleteRoot(IServiceProvider services, string login, TextWriter output, CancellationToken token = default)
|
||||
{
|
||||
login = login.ToLowerInvariant();
|
||||
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login && u.DeletedAt == null).ExecuteFirstAsync();
|
||||
var root = await DB.Default.Find<RootUser>().Match(u => u.UserName == login && u.DeletedAt == null).ExecuteFirstAsync(token);
|
||||
if (root == default)
|
||||
{
|
||||
Console.Error.WriteLine($"no root user '{login}'");
|
||||
return 1;
|
||||
}
|
||||
var owned = (await DB.Default.Find<RootToAvatar>().Match(ra => ra.RootId == root.ID).ExecuteAsync()).Select(ra => ra.AvatarId).ToList();
|
||||
var personas = (await DB.Default.Find<Avatar>().Match(a => owned.Contains(a.ID) && !a.DeletionAt.HasValue).ExecuteAsync())
|
||||
var owned = (await DB.Default.Find<RootToAvatar>().Match(ra => ra.RootId == root.ID).ExecuteAsync(token)).Select(ra => ra.AvatarId).ToList();
|
||||
var personas = (await DB.Default.Find<Avatar>().Match(a => owned.Contains(a.ID) && !a.DeletionAt.HasValue).ExecuteAsync(token))
|
||||
.Select(a => "@" + a.UserName).ToList();
|
||||
if (!await services.GetRequiredService<IRootRemoval>().Remove(root.ID, default))
|
||||
{
|
||||
Console.Error.WriteLine($"'{login}' could not be deleted");
|
||||
return 1;
|
||||
}
|
||||
await Log(services, AdminActions.RootDelete, AdminTargets.Root, root.ID, login);
|
||||
await Log(services, AdminActions.RootDelete, AdminTargets.Root, root.ID, login, token: token);
|
||||
output.WriteLine($"{login}: deleted{(personas.Count > 0 ? ", with " + string.Join(", ", personas) : string.Empty)}");
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -256,6 +256,7 @@ namespace PrivaPub.Infrastructure.Data
|
||||
})
|
||||
.CreateAsync(token);
|
||||
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage("HouseRules", "HR0003", Justification = "The index keys are a params array, which has to come last.")]
|
||||
static async Task Plain<T>(CancellationToken token, params System.Linq.Expressions.Expression<Func<T, object>>[] keys) where T : IEntity
|
||||
{
|
||||
var index = DB.Default.Index<T>();
|
||||
|
||||
@@ -43,17 +43,17 @@ namespace PrivaPub.Infrastructure.Data.Migrations
|
||||
await DB.Default.SaveAsync(reservations);
|
||||
}
|
||||
|
||||
static async Task DeleteAllBut<T>(IEnumerable<IOrderedEnumerable<T>> groups, Func<T, Task> delete)
|
||||
static async Task DeleteAllBut<T>(IEnumerable<IOrderedEnumerable<T>> groups, Func<T, CancellationToken, Task> delete, CancellationToken token = default)
|
||||
{
|
||||
foreach (var group in groups)
|
||||
foreach (var duplicate in group.Skip(1))
|
||||
await delete(duplicate);
|
||||
await delete(duplicate, token);
|
||||
}
|
||||
|
||||
static Task DeleteForeignAvatar(ForeignAvatar a) => DB.Default.DeleteAsync<ForeignAvatar>(a.ID);
|
||||
static Task DeletePost(Post p) => DB.Default.DeleteAsync<Post>(p.ID);
|
||||
static Task DeleteDmPost(DmPost p) => DB.Default.DeleteAsync<DmPost>(p.ID);
|
||||
static Task DeleteFollower(Follower f) => DB.Default.DeleteAsync<Follower>(f.ID);
|
||||
static Task DeleteLink(RootToAvatar r) => DB.Default.DeleteAsync<RootToAvatar>(r.ID);
|
||||
static Task DeleteForeignAvatar(ForeignAvatar a, CancellationToken token = default) => DB.Default.DeleteAsync<ForeignAvatar>(a.ID);
|
||||
static Task DeletePost(Post p, CancellationToken token = default) => DB.Default.DeleteAsync<Post>(p.ID);
|
||||
static Task DeleteDmPost(DmPost p, CancellationToken token = default) => DB.Default.DeleteAsync<DmPost>(p.ID);
|
||||
static Task DeleteFollower(Follower f, CancellationToken token = default) => DB.Default.DeleteAsync<Follower>(f.ID);
|
||||
static Task DeleteLink(RootToAvatar r, CancellationToken token = default) => DB.Default.DeleteAsync<RootToAvatar>(r.ID);
|
||||
}
|
||||
}
|
||||
@@ -61,7 +61,7 @@ namespace PrivaPub.Infrastructure.Data.Migrations
|
||||
static string Remote(string reference) =>
|
||||
reference != default && reference.Contains("://") ? reference : default;
|
||||
|
||||
static async Task<string> ForeignAccountId(string actorUri) =>
|
||||
(await DB.Default.Find<ForeignAvatar>().Match(a => a.ActorURI == actorUri).ExecuteFirstAsync())?.ID;
|
||||
static async Task<string> ForeignAccountId(string actorUri, CancellationToken token = default) =>
|
||||
(await DB.Default.Find<ForeignAvatar>().Match(a => a.ActorURI == actorUri).ExecuteFirstAsync(token))?.ID;
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
// The networks a published list holds; an entry that is not an address or a network is an error, never skipped.
|
||||
public static List<IPNetwork> Parse(CdnListFormat format, string body)
|
||||
{
|
||||
IEnumerable<string> entries = format switch
|
||||
var entries = format switch
|
||||
{
|
||||
CdnListFormat.Lines => body.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Where(l => !l.StartsWith('#')),
|
||||
CdnListFormat.StringArray => Root(body),
|
||||
|
||||
@@ -38,7 +38,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
return default;
|
||||
if (address.IsIPv4MappedToIPv6)
|
||||
address = address.MapToIPv4();
|
||||
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> sets;
|
||||
var sets = default(Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)>);
|
||||
lock (_gate)
|
||||
sets = _sets;
|
||||
foreach (var (provider, networks, _) in sets.Values)
|
||||
|
||||
@@ -12,7 +12,11 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
// Keeps the CDNs' published address ranges current by itself (owner decision 2026-10-04): at start it loads the stored
|
||||
// ones, then once a day it downloads each list again. A list that fails to download or to parse, or comes back empty,
|
||||
// leaves the stored one in place and is tried the next day. Like GeoUpdater, plain HTTPS to fixed hosts, not federation.
|
||||
public sealed class CdnUpdater : BackgroundService
|
||||
public sealed class CdnUpdater(
|
||||
IHttpClientFactory http,
|
||||
IOptionsMonitor<StatisticsOptions> options,
|
||||
ICdnRanges cdnRanges,
|
||||
ILogger<CdnUpdater> logger) : BackgroundService
|
||||
{
|
||||
public const string ClientName = "cdn";
|
||||
static readonly TimeSpan FirstWait = TimeSpan.FromMinutes(1);
|
||||
@@ -20,28 +24,16 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
const int MaxBytes = 16 * 1024 * 1024;
|
||||
const int MaxRanges = 200_000;
|
||||
|
||||
readonly IHttpClientFactory _http;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly ICdnRanges _ranges;
|
||||
readonly ILogger<CdnUpdater> _logger;
|
||||
|
||||
public CdnUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, ICdnRanges ranges, ILogger<CdnUpdater> logger)
|
||||
{
|
||||
_http = http;
|
||||
_options = options;
|
||||
_ranges = ranges;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _ranges.Load(stoppingToken);
|
||||
await cdnRanges.Load(stoppingToken);
|
||||
await Task.Delay(FirstWait, stoppingToken);
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
if (_options.CurrentValue.Cdn.AutoUpdate)
|
||||
if (options.CurrentValue.Cdn.AutoUpdate)
|
||||
await Update(CdnCatalog.All, stoppingToken);
|
||||
await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
|
||||
}
|
||||
@@ -51,7 +43,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
|
||||
logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,13 +68,13 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
set.Sources = provider.Lists.Select(l => l.Url).ToList();
|
||||
set.FetchedAt = DateTime.UtcNow;
|
||||
await DB.Default.SaveAsync(set, token);
|
||||
_ranges.Replace(set);
|
||||
cdnRanges.Replace(set);
|
||||
replaced++;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or InvalidDataException or FormatException or System.Text.Json.JsonException
|
||||
or TaskCanceledException && !token.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
|
||||
logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
|
||||
}
|
||||
}
|
||||
return replaced;
|
||||
@@ -94,14 +86,14 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
request.Headers.Accept.ParseAdd("application/json, text/plain;q=0.9");
|
||||
if (list.Body != default)
|
||||
request.Content = new StringContent(list.Body, Encoding.UTF8, "application/x-www-form-urlencoded");
|
||||
using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
using var response = await http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
response.EnsureSuccessStatusCode();
|
||||
if (response.Content.Headers.ContentLength > MaxBytes)
|
||||
throw new InvalidDataException($"more than {MaxBytes} bytes");
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(token);
|
||||
using var buffer = new MemoryStream();
|
||||
var chunk = new byte[81920];
|
||||
int read;
|
||||
var read = default(int);
|
||||
while ((read = await stream.ReadAsync(chunk, token)) > 0)
|
||||
{
|
||||
if (buffer.Length + read > MaxBytes)
|
||||
|
||||
@@ -23,14 +23,14 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
public string Source => default;
|
||||
}
|
||||
|
||||
public sealed class DbIpLocator : IGeoLocator, IDisposable
|
||||
public sealed class DbIpLocator(
|
||||
IOptionsMonitor<StatisticsOptions> options,
|
||||
ILogger<DbIpLocator> logger) : IGeoLocator, IDisposable
|
||||
{
|
||||
public static readonly string CityFile = GeoUpdater.FileOf("city");
|
||||
public static readonly string AsnFile = GeoUpdater.FileOf("asn");
|
||||
static readonly TimeSpan CheckInterval = TimeSpan.FromMinutes(10);
|
||||
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly ILogger<DbIpLocator> _logger;
|
||||
readonly object _lock = new();
|
||||
Reader _city;
|
||||
Reader _asn;
|
||||
@@ -39,12 +39,6 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
DateTime _checkedAt = DateTime.MinValue;
|
||||
bool _warned;
|
||||
|
||||
public DbIpLocator(IOptionsMonitor<StatisticsOptions> options, ILogger<DbIpLocator> logger)
|
||||
{
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public string Source
|
||||
{
|
||||
get
|
||||
@@ -60,7 +54,8 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
if (address == default || !IpRangeGuard.IsPublic(address))
|
||||
return default;
|
||||
Refresh();
|
||||
Dictionary<string, object> place, network;
|
||||
var place = default(Dictionary<string, object>);
|
||||
var network = default(Dictionary<string, object>);
|
||||
try
|
||||
{
|
||||
// under the lock, so a reload cannot dispose a reader in the middle of a lookup; lookups are rare
|
||||
@@ -82,7 +77,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDatabaseException or ArgumentException)
|
||||
{
|
||||
_logger.LogWarning(ex, "The geolocation databases could not be read");
|
||||
logger.LogWarning(ex, "The geolocation databases could not be read");
|
||||
return default;
|
||||
}
|
||||
}
|
||||
@@ -104,13 +99,13 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
if (DateTime.UtcNow - _checkedAt < CheckInterval)
|
||||
return;
|
||||
_checkedAt = DateTime.UtcNow;
|
||||
var directory = _options.CurrentValue.GeoDirectory;
|
||||
var directory = options.CurrentValue.GeoDirectory;
|
||||
(_city, _cityStamp) = Open(Path.Combine(directory ?? string.Empty, CityFile), _city, _cityStamp);
|
||||
(_asn, _asnStamp) = Open(Path.Combine(directory ?? string.Empty, AsnFile), _asn, _asnStamp);
|
||||
if (_city == default && _asn == default && !_warned)
|
||||
{
|
||||
_warned = true;
|
||||
_logger.LogInformation("No geolocation databases in {Directory}; servers are not located", directory);
|
||||
logger.LogInformation("No geolocation databases in {Directory}; servers are not located", directory);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -133,7 +128,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDatabaseException or IOException)
|
||||
{
|
||||
_logger.LogWarning(ex, "{Path} is not a readable geolocation database", path);
|
||||
logger.LogWarning(ex, "{Path} is not a readable geolocation database", path);
|
||||
return (current, stamp);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,25 +13,17 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
// whether each database was built this month and, if not, fetches this month's (or, early in the month, last month's),
|
||||
// checks it opens as the right kind of database and swaps it in. This, CdnUpdater and SMTP are the server's only traffic
|
||||
// that is not federation, which is why each has its own client instead of IFederationHttp.
|
||||
public sealed class GeoUpdater : BackgroundService
|
||||
public sealed class GeoUpdater(
|
||||
IHttpClientFactory http,
|
||||
IOptionsMonitor<StatisticsOptions> statisticsOptions,
|
||||
IGeoLocator geoLocator,
|
||||
ILogger<GeoUpdater> logger) : BackgroundService
|
||||
{
|
||||
public const string ClientName = "geo";
|
||||
static readonly string[] Kinds = { "city", "asn" };
|
||||
static readonly TimeSpan FirstWait = TimeSpan.FromSeconds(30);
|
||||
static readonly TimeSpan Interval = TimeSpan.FromHours(24);
|
||||
|
||||
readonly IHttpClientFactory _http;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly IGeoLocator _locator;
|
||||
readonly ILogger<GeoUpdater> _logger;
|
||||
|
||||
public GeoUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, IGeoLocator locator, ILogger<GeoUpdater> logger)
|
||||
{
|
||||
_http = http;
|
||||
_options = options;
|
||||
_locator = locator;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public static string FileOf(string kind) => $"dbip-{kind}-lite.mmdb";
|
||||
|
||||
@@ -42,7 +34,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
await Task.Delay(FirstWait, stoppingToken);
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
if (_options.CurrentValue.Geo.AutoUpdate)
|
||||
if (statisticsOptions.CurrentValue.Geo.AutoUpdate)
|
||||
await Update(DateTime.UtcNow, stoppingToken);
|
||||
await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
|
||||
}
|
||||
@@ -55,7 +47,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
// The number of databases replaced. A failure keeps the database already there and is tried again the next day.
|
||||
public async Task<int> Update(DateTime now, CancellationToken token)
|
||||
{
|
||||
var options = _options.CurrentValue;
|
||||
var options = statisticsOptions.CurrentValue;
|
||||
var replaced = 0;
|
||||
try
|
||||
{
|
||||
@@ -63,7 +55,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Cannot create {Directory}; servers are not located", options.GeoDirectory);
|
||||
logger.LogWarning(ex, "Cannot create {Directory}; servers are not located", options.GeoDirectory);
|
||||
return 0;
|
||||
}
|
||||
foreach (var kind in Kinds)
|
||||
@@ -79,19 +71,19 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
if (await Install(kind, month, path, options, token))
|
||||
{
|
||||
replaced++;
|
||||
_logger.LogInformation("Installed DB-IP Lite {Kind} {Month:yyyy-MM}", kind, month);
|
||||
logger.LogInformation("Installed DB-IP Lite {Kind} {Month:yyyy-MM}", kind, month);
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or IOException or InvalidDatabaseException or InvalidDataException
|
||||
or TaskCanceledException && !token.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not install DB-IP Lite {Kind} {Month:yyyy-MM}", kind, month);
|
||||
logger.LogWarning(ex, "Could not install DB-IP Lite {Kind} {Month:yyyy-MM}", kind, month);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (replaced > 0 && _locator is DbIpLocator locator)
|
||||
if (replaced > 0 && geoLocator is DbIpLocator locator)
|
||||
locator.Reload();
|
||||
return replaced;
|
||||
}
|
||||
@@ -121,7 +113,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
var unpacked = path + ".new";
|
||||
try
|
||||
{
|
||||
using (var response = await _http.CreateClient(ClientName).GetAsync(url, HttpCompletionOption.ResponseHeadersRead, token))
|
||||
using (var response = await http.CreateClient(ClientName).GetAsync(url, HttpCompletionOption.ResponseHeadersRead, token))
|
||||
{
|
||||
if (response.StatusCode == HttpStatusCode.NotFound)
|
||||
return false;
|
||||
@@ -150,7 +142,7 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
{
|
||||
var buffer = new byte[81920];
|
||||
long total = 0;
|
||||
int read;
|
||||
var read = default(int);
|
||||
while ((read = await source.ReadAsync(buffer, token)) > 0)
|
||||
{
|
||||
total += read;
|
||||
|
||||
@@ -38,17 +38,14 @@ namespace PrivaPub.Infrastructure.Http
|
||||
|
||||
// On the federation client: reads each response's CDN fingerprint for its host. A response without one forgets the
|
||||
// host's hint, so a server that leaves its CDN is seen as itself again.
|
||||
public class EdgeHintsHandler : DelegatingHandler
|
||||
public class EdgeHintsHandler(
|
||||
IEdgeHints hints) : DelegatingHandler
|
||||
{
|
||||
readonly IEdgeHints _hints;
|
||||
|
||||
public EdgeHintsHandler(IEdgeHints hints) => _hints = hints;
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var response = await base.SendAsync(request, cancellationToken);
|
||||
if (request.RequestUri is { } uri && (int)response.StatusCode is >= 200 and < 500)
|
||||
_hints.Remember(uri.Host, CdnCatalog.FromResponse(response));
|
||||
hints.Remember(uri.Host, CdnCatalog.FromResponse(response));
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +42,13 @@ namespace PrivaPub.Infrastructure.Http
|
||||
Task<IReadOnlyList<string>> GetStringArray(string url, int maxItems, CancellationToken token);
|
||||
}
|
||||
|
||||
public class FederationHttp : IFederationHttp
|
||||
public class FederationHttp(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IMemoryCache cache,
|
||||
IOptionsMonitor<FederationOptions> federationOptions,
|
||||
IDomainBlocks domainBlocks,
|
||||
ILogger<FederationHttp> logger,
|
||||
IInteractionLedger ledger = default) : IFederationHttp
|
||||
{
|
||||
public const string ClientName = "Federation";
|
||||
public const int MaxResponseBytes = 1024 * 1024;
|
||||
@@ -58,34 +64,12 @@ namespace PrivaPub.Infrastructure.Http
|
||||
"application/json"
|
||||
};
|
||||
|
||||
readonly IHttpClientFactory _httpClientFactory;
|
||||
readonly IMemoryCache _cache;
|
||||
readonly IOptionsMonitor<FederationOptions> _options;
|
||||
readonly IDomainBlocks _domainBlocks;
|
||||
readonly ILogger<FederationHttp> _logger;
|
||||
readonly IInteractionLedger _ledger;
|
||||
|
||||
public FederationHttp(IHttpClientFactory httpClientFactory, IMemoryCache cache, IOptionsMonitor<FederationOptions> options,
|
||||
IDomainBlocks domainBlocks, ILogger<FederationHttp> logger, IInteractionLedger ledger = default)
|
||||
sealed class Exchange(string url, string purpose)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_cache = cache;
|
||||
_options = options;
|
||||
_domainBlocks = domainBlocks;
|
||||
_logger = logger;
|
||||
_ledger = ledger;
|
||||
}
|
||||
public string Host { get; } = Interactions.HostOf(url);
|
||||
public string Purpose { get; } = purpose;
|
||||
|
||||
sealed class Exchange
|
||||
{
|
||||
public Exchange(string url, string purpose)
|
||||
{
|
||||
Host = Interactions.HostOf(url);
|
||||
Purpose = purpose;
|
||||
}
|
||||
|
||||
public readonly string Host;
|
||||
public readonly string Purpose;
|
||||
public readonly long Started = System.Diagnostics.Stopwatch.GetTimestamp();
|
||||
public int? Status;
|
||||
public long? Bytes;
|
||||
@@ -102,15 +86,15 @@ namespace PrivaPub.Infrastructure.Http
|
||||
|
||||
void Record(Exchange exchange)
|
||||
{
|
||||
if (_ledger == default)
|
||||
if (ledger == default)
|
||||
return;
|
||||
var trigger = HttpScope.Trigger;
|
||||
if (trigger == HttpScope.Request)
|
||||
{
|
||||
_ledger.Count(exchange.Host, $"http:{exchange.Purpose}:{exchange.Outcome}", exchange.Bytes ?? 0);
|
||||
ledger.Count(exchange.Host, $"http:{exchange.Purpose}:{exchange.Outcome}", exchange.Bytes ?? 0);
|
||||
return;
|
||||
}
|
||||
_ledger.Record(new InteractionEvent
|
||||
ledger.Record(new InteractionEvent
|
||||
{
|
||||
Channel = Interactions.Http,
|
||||
Host = exchange.Host,
|
||||
@@ -151,7 +135,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
for (var hop = 0; hop <= MaxRedirects; hop++)
|
||||
{
|
||||
using var request = Request(target, "text/plain");
|
||||
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
using var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
exchange.Answered(response);
|
||||
if (IsRedirect(response.StatusCode))
|
||||
{
|
||||
@@ -207,7 +191,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
for (var hop = 0; hop <= MaxRedirects; hop++)
|
||||
{
|
||||
using var request = Request(target, "application/json");
|
||||
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
using var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
exchange.Answered(response);
|
||||
if (IsRedirect(response.StatusCode))
|
||||
{
|
||||
@@ -278,9 +262,9 @@ namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
if (target is not { IsAbsoluteUri: true } || !string.IsNullOrEmpty(target.UserInfo))
|
||||
return false;
|
||||
if (_domainBlocks?.IsSuspended(target.Host) == true)
|
||||
if (domainBlocks?.IsSuspended(target.Host) == true)
|
||||
return false;
|
||||
var options = _options.CurrentValue;
|
||||
var options = federationOptions.CurrentValue;
|
||||
if (target.Scheme != Uri.UriSchemeHttps && !(options.AllowPlainHttp && target.Scheme == Uri.UriSchemeHttp))
|
||||
return false;
|
||||
if (options.AllowPrivateNetworks)
|
||||
@@ -322,7 +306,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
// status; 0 when it never answered)
|
||||
public async Task<(int Status, FetchedJson Json)> GetJsonStatus(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token)
|
||||
{
|
||||
if (Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out Refusal remembered))
|
||||
if (Uri.TryCreate(url, UriKind.Absolute, out var target) && cache.TryGetValue(NegativeKey(target), out Refusal remembered))
|
||||
return (remembered.Status, default);
|
||||
var exchange = new Exchange(url, HttpScope.Purpose ?? "object");
|
||||
try
|
||||
@@ -344,7 +328,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
return default;
|
||||
}
|
||||
var negativeKey = NegativeKey(target);
|
||||
if (_cache.TryGetValue(negativeKey, out _))
|
||||
if (cache.TryGetValue(negativeKey, out _))
|
||||
{
|
||||
exchange.Refused("remembered");
|
||||
return default;
|
||||
@@ -361,7 +345,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
request.Headers.TryAddWithoutValidation("If-None-Match", etag);
|
||||
sign?.Invoke(request);
|
||||
|
||||
using var response = await _httpClientFactory.CreateClient(ClientName)
|
||||
using var response = await httpClientFactory.CreateClient(ClientName)
|
||||
.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
exchange.Answered(response);
|
||||
if (response.StatusCode == HttpStatusCode.NotModified && !string.IsNullOrEmpty(etag))
|
||||
@@ -412,7 +396,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
}
|
||||
|
||||
public bool FailedTemporarily(string url) =>
|
||||
Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out Refusal refusal) && refusal.Transient;
|
||||
Uri.TryCreate(url, UriKind.Absolute, out var target) && cache.TryGetValue(NegativeKey(target), out Refusal refusal) && refusal.Transient;
|
||||
|
||||
public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token)
|
||||
{
|
||||
@@ -460,7 +444,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, target);
|
||||
request.Headers.Accept.ParseAdd("image/*, video/*, audio/*");
|
||||
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
using var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
exchange.Answered(response);
|
||||
if (IsRedirect(response.StatusCode))
|
||||
{
|
||||
@@ -507,7 +491,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
|
||||
{
|
||||
exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
|
||||
_logger.LogInformation("Media {Url} refused: {Reason}", url, ex.Message);
|
||||
logger.LogInformation("Media {Url} refused: {Reason}", url, ex.Message);
|
||||
return default;
|
||||
}
|
||||
}
|
||||
@@ -524,7 +508,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, target);
|
||||
request.Headers.Accept.ParseAdd("text/html, application/xhtml+xml");
|
||||
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
using var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
if (IsRedirect(response.StatusCode))
|
||||
{
|
||||
var location = response.Headers.Location;
|
||||
@@ -554,7 +538,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogInformation("Page {Url} refused: {Reason}", url, ex.Message);
|
||||
logger.LogInformation("Page {Url} refused: {Reason}", url, ex.Message);
|
||||
return default;
|
||||
}
|
||||
}
|
||||
@@ -590,7 +574,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, target);
|
||||
request.Headers.Accept.ParseAdd("video/*, audio/*, image/*");
|
||||
request.Headers.Range = range;
|
||||
var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
var response = await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
exchange.Answered(response);
|
||||
if (IsRedirect(response.StatusCode))
|
||||
{
|
||||
@@ -624,7 +608,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
|
||||
{
|
||||
exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
|
||||
_logger.LogInformation("Media stream {Url} refused: {Reason}", url, ex.Message);
|
||||
logger.LogInformation("Media stream {Url} refused: {Reason}", url, ex.Message);
|
||||
return default;
|
||||
}
|
||||
}
|
||||
@@ -636,7 +620,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
await using var stream = await content.ReadAsStreamAsync(token);
|
||||
var buffer = new byte[limit];
|
||||
var total = 0;
|
||||
int read;
|
||||
var read = default(int);
|
||||
while (total < limit && (read = await stream.ReadAsync(buffer.AsMemory(total, limit - total), token)) > 0)
|
||||
total += read;
|
||||
return buffer[..total];
|
||||
@@ -646,7 +630,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
if (!IsAllowed(request.RequestUri))
|
||||
throw new BlockedDestinationException(request.RequestUri?.Host);
|
||||
return await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
return await httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
}
|
||||
|
||||
// copies at most limit bytes: how many, or -1 once there are more
|
||||
@@ -655,7 +639,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
await using var stream = await content.ReadAsStreamAsync(token);
|
||||
var chunk = new byte[64 * 1024];
|
||||
var copied = 0L;
|
||||
int read;
|
||||
var read = default(int);
|
||||
while ((read = await stream.ReadAsync(chunk, token)) > 0)
|
||||
{
|
||||
copied += read;
|
||||
@@ -671,7 +655,7 @@ namespace PrivaPub.Infrastructure.Http
|
||||
await using var stream = await content.ReadAsStreamAsync(token);
|
||||
using var buffer = new MemoryStream();
|
||||
var chunk = new byte[16 * 1024];
|
||||
int read;
|
||||
var read = default(int);
|
||||
while ((read = await stream.ReadAsync(chunk, token)) > 0)
|
||||
{
|
||||
if (buffer.Length + read > limit)
|
||||
@@ -704,8 +688,8 @@ namespace PrivaPub.Infrastructure.Http
|
||||
exchange.Failed(code);
|
||||
else
|
||||
exchange.Refused(code);
|
||||
_cache.Set(negativeKey, new Refusal(transient, exchange.Status ?? 0), NegativeCacheLifetime);
|
||||
_logger.LogInformation("GET {Url} refused: {Reason}", url, reason);
|
||||
cache.Set(negativeKey, new Refusal(transient, exchange.Status ?? 0), NegativeCacheLifetime);
|
||||
logger.LogInformation("GET {Url} refused: {Reason}", url, reason);
|
||||
return default;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,13 +28,10 @@ namespace PrivaPub.Infrastructure.Http
|
||||
return new Restore(previous);
|
||||
}
|
||||
|
||||
sealed class Restore : IDisposable
|
||||
sealed class Restore(
|
||||
Tags previous) : IDisposable
|
||||
{
|
||||
readonly Tags _previous;
|
||||
|
||||
public Restore(Tags previous) => _previous = previous;
|
||||
|
||||
public void Dispose() => current.Value = _previous;
|
||||
public void Dispose() => current.Value = previous;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -23,11 +23,12 @@ namespace PrivaPub.Infrastructure.Http
|
||||
ConnectTimeout = connectTimeout ?? TimeSpan.FromSeconds(10),
|
||||
PooledConnectionLifetime = TimeSpan.FromMinutes(2),
|
||||
MaxResponseHeadersLength = 64,
|
||||
ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token, connected)
|
||||
ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, connected, token)
|
||||
};
|
||||
|
||||
public static async ValueTask<Stream> Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token,
|
||||
IConnectedAddresses connected = default)
|
||||
public static async ValueTask<Stream> Connect(DnsEndPoint endPoint, bool allowPrivateNetworks,
|
||||
IConnectedAddresses connected = default,
|
||||
CancellationToken token = default)
|
||||
{
|
||||
var addresses = await Resolve(endPoint.Host, token);
|
||||
if (addresses.Length == 0 || !allowPrivateNetworks && !addresses.All(IpRangeGuard.IsPublic))
|
||||
|
||||
@@ -13,17 +13,12 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
Task Failed(string host, string error, CancellationToken token);
|
||||
}
|
||||
|
||||
public class HostCircuitBreaker : IHostCircuitBreaker
|
||||
public class HostCircuitBreaker(
|
||||
IMemoryCache cache) : IHostCircuitBreaker
|
||||
{
|
||||
public const int Threshold = 10;
|
||||
static readonly TimeSpan CacheLifetime = TimeSpan.FromSeconds(30);
|
||||
|
||||
readonly IMemoryCache _cache;
|
||||
|
||||
public HostCircuitBreaker(IMemoryCache cache)
|
||||
{
|
||||
_cache = cache;
|
||||
}
|
||||
|
||||
public async Task<DateTime?> UnavailableUntil(string host, CancellationToken token)
|
||||
{
|
||||
@@ -43,7 +38,7 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
.Modify(i => i.LastSuccessAt, DateTime.UtcNow)
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token);
|
||||
_cache.Remove(Key(host));
|
||||
cache.Remove(Key(host));
|
||||
}
|
||||
|
||||
public async Task Failed(string host, string error, CancellationToken token)
|
||||
@@ -61,11 +56,11 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
await DB.Default.Update<RemoteInstance>().MatchID(instance.ID)
|
||||
.Modify(i => i.UnavailableUntil, now + quarantine)
|
||||
.ExecuteAsync(token);
|
||||
_cache.Remove(Key(host));
|
||||
cache.Remove(Key(host));
|
||||
}
|
||||
|
||||
async Task<RemoteInstance> Instance(string host, CancellationToken token) =>
|
||||
await _cache.GetOrCreateAsync(Key(host), async entry =>
|
||||
await cache.GetOrCreateAsync(Key(host), async entry =>
|
||||
{
|
||||
entry.AbsoluteExpirationRelativeToNow = CacheLifetime;
|
||||
return await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
|
||||
|
||||
@@ -14,25 +14,18 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
Task<JobOutcome> Handle(Job job, CancellationToken token);
|
||||
}
|
||||
|
||||
public class JobWorker : BackgroundService
|
||||
public class JobWorker(
|
||||
IJobQueue jobQueue,
|
||||
IEnumerable<IJobHandler> handlers,
|
||||
ILogger<JobWorker> logger) : BackgroundService
|
||||
{
|
||||
static readonly TimeSpan Poll = TimeSpan.FromSeconds(5);
|
||||
static readonly TimeSpan ReapInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
readonly IJobQueue _queue;
|
||||
readonly IEnumerable<IJobHandler> _handlers;
|
||||
readonly ILogger<JobWorker> _logger;
|
||||
|
||||
public JobWorker(IJobQueue queue, IEnumerable<IJobHandler> handlers, ILogger<JobWorker> logger)
|
||||
{
|
||||
_queue = queue;
|
||||
_handlers = handlers;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var loops = _handlers.SelectMany(handler =>
|
||||
var loops = handlers.SelectMany(handler =>
|
||||
{
|
||||
var inFlight = new ConcurrentDictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
||||
return Enumerable.Range(0, handler.Concurrency).Select(_ => Run(handler, inFlight, stoppingToken));
|
||||
@@ -44,11 +37,11 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
Job job;
|
||||
var job = default(Job);
|
||||
try
|
||||
{
|
||||
var busy = inFlight.Where(h => h.Value >= handler.PerHostLimit).Select(h => h.Key).ToList();
|
||||
job = await _queue.Lease(handler.Kind, busy, stoppingToken);
|
||||
job = await jobQueue.Lease(handler.Kind, busy, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -56,14 +49,14 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "{Worker} could not lease a {Kind} job", nameof(JobWorker), handler.Kind);
|
||||
logger.LogError(ex, "{Worker} could not lease a {Kind} job", nameof(JobWorker), handler.Kind);
|
||||
await Delay(Poll, stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (job == default)
|
||||
{
|
||||
await _queue.WaitForWork(handler.Kind, Poll, stoppingToken);
|
||||
await jobQueue.WaitForWork(handler.Kind, Poll, stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -75,7 +68,7 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
var renewing = KeepLease(job, running);
|
||||
try
|
||||
{
|
||||
JobOutcome outcome;
|
||||
var outcome = default(JobOutcome);
|
||||
try
|
||||
{
|
||||
using var scope = HttpScope.Triggered(handler.Kind.ToString().ToLowerInvariant());
|
||||
@@ -87,24 +80,24 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
}
|
||||
catch (OperationCanceledException) when (running.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning("{Kind} job {Id} lost its lease and was stopped", handler.Kind, job.ID);
|
||||
logger.LogWarning("{Kind} job {Id} lost its lease and was stopped", handler.Kind, job.ID);
|
||||
continue;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "{Kind} job {Id} threw", handler.Kind, job.ID);
|
||||
logger.LogError(ex, "{Kind} job {Id} threw", handler.Kind, job.ID);
|
||||
outcome = JobOutcome.Retry(ex.GetType().Name);
|
||||
}
|
||||
|
||||
if (outcome.Result != JobResult.Done && job.Attempts >= handler.MaxAttempts && outcome.Result == JobResult.Retry)
|
||||
_logger.LogWarning("{Kind} job {Id} for {Host} is dead after {Attempts} attempts: {Error}",
|
||||
logger.LogWarning("{Kind} job {Id} for {Host} is dead after {Attempts} attempts: {Error}",
|
||||
handler.Kind, job.ID, job.Host, job.Attempts, outcome.Error);
|
||||
if (!await _queue.Finish(job, outcome, handler.MaxAttempts, CancellationToken.None))
|
||||
_logger.LogWarning("{Kind} job {Id} lost its lease before it finished; its outcome is dropped", handler.Kind, job.ID);
|
||||
if (!await jobQueue.Finish(job, outcome, handler.MaxAttempts, CancellationToken.None))
|
||||
logger.LogWarning("{Kind} job {Id} lost its lease before it finished; its outcome is dropped", handler.Kind, job.ID);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "{Kind} job {Id} could not be finished; its lease will expire", handler.Kind, job.ID);
|
||||
logger.LogError(ex, "{Kind} job {Id} could not be finished; its lease will expire", handler.Kind, job.ID);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -116,9 +109,10 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
}
|
||||
|
||||
// renews the lease every third of its length until the handler ends; a lease found lost cancels the handler
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage("HouseRules", "HR0003", Justification = "Takes the source it cancels when the lease is lost; its token already follows the worker's.")]
|
||||
async Task KeepLease(Job job, CancellationTokenSource running)
|
||||
{
|
||||
var every = _queue.LeaseFor / 3;
|
||||
var every = jobQueue.LeaseFor / 3;
|
||||
while (!running.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
@@ -132,7 +126,7 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
|
||||
try
|
||||
{
|
||||
if (await _queue.Renew(job, CancellationToken.None))
|
||||
if (await jobQueue.Renew(job, CancellationToken.None))
|
||||
continue;
|
||||
await running.CancelAsync();
|
||||
return;
|
||||
@@ -140,7 +134,7 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
catch (Exception ex)
|
||||
{
|
||||
// a passing database error: the lease has two more thirds to go, so try again at the next turn
|
||||
_logger.LogWarning(ex, "{Worker} could not renew the lease of job {Id}", nameof(JobWorker), job.ID);
|
||||
logger.LogWarning(ex, "{Worker} could not renew the lease of job {Id}", nameof(JobWorker), job.ID);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -151,9 +145,9 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
{
|
||||
try
|
||||
{
|
||||
var reaped = await _queue.Reap(stoppingToken);
|
||||
var reaped = await jobQueue.Reap(stoppingToken);
|
||||
if (reaped > 0)
|
||||
_logger.LogWarning("{Worker} returned {Count} expired leases to the queue", nameof(JobWorker), reaped);
|
||||
logger.LogWarning("{Worker} returned {Count} expired leases to the queue", nameof(JobWorker), reaped);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -161,7 +155,7 @@ namespace PrivaPub.Infrastructure.Jobs
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "{Worker} reaper failed", nameof(JobWorker));
|
||||
logger.LogError(ex, "{Worker} reaper failed", nameof(JobWorker));
|
||||
}
|
||||
await Delay(ReapInterval, stoppingToken);
|
||||
}
|
||||
|
||||
@@ -42,7 +42,15 @@ namespace PrivaPub.Infrastructure.Push
|
||||
// safe handler, with the pasture's exceptions where Federation says so).
|
||||
// A 404 or 410 means the subscription is gone: it is deleted. 408, 429, a 5xx, a timeout or a broken connection retry on
|
||||
// the jobs' backoff; any other answer drops the push (logged by the push service's host alone).
|
||||
public class WebPushJob : IJobHandler
|
||||
public class WebPushJob(
|
||||
IHttpClientFactory http,
|
||||
IVapid vapid,
|
||||
IServiceScopeFactory scopes,
|
||||
IInstanceSettings settings,
|
||||
ILocalActorService localActors,
|
||||
IOptionsMonitor<FederationOptions> federation,
|
||||
IStringLocalizer<GenericRes> localizer,
|
||||
ILogger<WebPushJob> logger) : IJobHandler
|
||||
{
|
||||
public const string ClientName = "push";
|
||||
public static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(15);
|
||||
@@ -52,27 +60,6 @@ namespace PrivaPub.Infrastructure.Push
|
||||
const string Urgency = "normal";
|
||||
const int BodyLength = 140;
|
||||
|
||||
readonly IHttpClientFactory _http;
|
||||
readonly IVapid _vapid;
|
||||
readonly IServiceScopeFactory _scopes;
|
||||
readonly IInstanceSettings _settings;
|
||||
readonly ILocalActorService _localActors;
|
||||
readonly IOptionsMonitor<FederationOptions> _federation;
|
||||
readonly IStringLocalizer<GenericRes> _localizer;
|
||||
readonly ILogger<WebPushJob> _logger;
|
||||
|
||||
public WebPushJob(IHttpClientFactory http, IVapid vapid, IServiceScopeFactory scopes, IInstanceSettings settings, ILocalActorService localActors,
|
||||
IOptionsMonitor<FederationOptions> federation, IStringLocalizer<GenericRes> localizer, ILogger<WebPushJob> logger)
|
||||
{
|
||||
_http = http;
|
||||
_vapid = vapid;
|
||||
_scopes = scopes;
|
||||
_settings = settings;
|
||||
_localActors = localActors;
|
||||
_federation = federation;
|
||||
_localizer = localizer;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public JobKind Kind => JobKind.SendPush;
|
||||
public int Concurrency => 4;
|
||||
@@ -96,22 +83,22 @@ namespace PrivaPub.Infrastructure.Push
|
||||
var notification = await DB.Default.Find<Notification>().Match(n => n.ID == notificationId && n.AvatarId == avatarId).ExecuteFirstAsync(token);
|
||||
if (notification == default || !await WebPush.Pushable(subscription, notification, token))
|
||||
return JobOutcome.Done;
|
||||
if (!Uri.TryCreate(subscription.Endpoint, UriKind.Absolute, out var endpoint) || !IsAllowed(endpoint, _federation.CurrentValue))
|
||||
if (!Uri.TryCreate(subscription.Endpoint, UriKind.Absolute, out var endpoint) || !IsAllowed(endpoint, federation.CurrentValue))
|
||||
return JobOutcome.Dead("endpoint");
|
||||
|
||||
byte[] json;
|
||||
using (var scope = _scopes.CreateScope())
|
||||
var json = default(byte[]);
|
||||
using (var scope = scopes.CreateScope())
|
||||
{
|
||||
// the token the subscription was made with must still be valid, and the persona's: it goes into the payload
|
||||
if (!await TokenHolds(scope.ServiceProvider.GetRequiredService<IOpenIddictTokenManager>(), subscription, token))
|
||||
{
|
||||
await WebPush.ForgetToken(subscription.TokenId);
|
||||
await WebPush.ForgetToken(subscription.TokenId, token);
|
||||
return JobOutcome.Done;
|
||||
}
|
||||
var avatar = await DB.Default.Find<Avatar>().Match(a => a.ID == avatarId && a.DeletionAt == null).ExecuteFirstAsync(token);
|
||||
if (avatar == default)
|
||||
{
|
||||
await WebPush.Forget(new[] { avatarId });
|
||||
await WebPush.Forget(new[] { avatarId }, token);
|
||||
return JobOutcome.Done;
|
||||
}
|
||||
var mapped = (await scope.ServiceProvider.GetRequiredService<MastodonMapper>().Notifications(new List<Notification> { notification }, avatarId, token))
|
||||
@@ -130,7 +117,7 @@ namespace PrivaPub.Infrastructure.Push
|
||||
});
|
||||
}
|
||||
|
||||
var key = await _vapid.Key(token);
|
||||
var key = await vapid.Key(token);
|
||||
var p256dh = WebPushCrypto.FromBase64Url(subscription.P256dh);
|
||||
var auth = WebPushCrypto.FromBase64Url(subscription.Auth);
|
||||
if (!WebPushCrypto.AreUsable(p256dh, auth))
|
||||
@@ -154,24 +141,24 @@ namespace PrivaPub.Infrastructure.Push
|
||||
|
||||
try
|
||||
{
|
||||
using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
using var response = await http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
var status = (int)response.StatusCode;
|
||||
if (status is >= 200 and < 300)
|
||||
return JobOutcome.Done;
|
||||
if (response.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Gone)
|
||||
{
|
||||
await DB.Default.DeleteAsync<PushSubscription>(subscription.ID);
|
||||
_logger.LogInformation("{Kind}: {Host} answered {Status}, the subscription is gone", Kind, endpoint.Host, status);
|
||||
logger.LogInformation("{Kind}: {Host} answered {Status}, the subscription is gone", Kind, endpoint.Host, status);
|
||||
return JobOutcome.Done;
|
||||
}
|
||||
if (response.StatusCode is HttpStatusCode.RequestTimeout or HttpStatusCode.TooManyRequests || status >= 500)
|
||||
return JobOutcome.Retry($"status {status}");
|
||||
_logger.LogWarning("{Kind}: {Host} refused a push with {Status}; it is dropped", Kind, endpoint.Host, status);
|
||||
logger.LogWarning("{Kind}: {Host} refused a push with {Status}; it is dropped", Kind, endpoint.Host, status);
|
||||
return JobOutcome.Dead($"status {status}");
|
||||
}
|
||||
catch (BlockedDestinationException)
|
||||
{
|
||||
_logger.LogWarning("{Kind}: {Host} does not resolve to a public address; the push is dropped", Kind, endpoint.Host);
|
||||
logger.LogWarning("{Kind}: {Host} does not resolve to a public address; the push is dropped", Kind, endpoint.Host);
|
||||
return JobOutcome.Dead("blocked destination");
|
||||
}
|
||||
catch (TaskCanceledException) when (!token.IsCancellationRequested)
|
||||
@@ -180,7 +167,7 @@ namespace PrivaPub.Infrastructure.Push
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.InnerException is BlockedDestinationException)
|
||||
{
|
||||
_logger.LogWarning("{Kind}: {Host} does not resolve to a public address; the push is dropped", Kind, endpoint.Host);
|
||||
logger.LogWarning("{Kind}: {Host} does not resolve to a public address; the push is dropped", Kind, endpoint.Host);
|
||||
return JobOutcome.Dead("blocked destination");
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or IOException or SocketException)
|
||||
@@ -212,10 +199,10 @@ namespace PrivaPub.Infrastructure.Push
|
||||
// so it names a mailbox at its host: RFC 8292 takes only mailto: and https:.)
|
||||
string Subject()
|
||||
{
|
||||
var email = _settings.Current.ContactEmail?.Trim();
|
||||
var email = settings.Current.ContactEmail?.Trim();
|
||||
if (!string.IsNullOrEmpty(email) && !email.Any(char.IsWhiteSpace) && email.Contains('@'))
|
||||
return "mailto:" + email;
|
||||
var address = _localActors.BaseAddress;
|
||||
var address = localActors.BaseAddress;
|
||||
return Uri.TryCreate(address, UriKind.Absolute, out var base_) && base_.Scheme != Uri.UriSchemeHttps
|
||||
? "mailto:privapub@" + base_.Host
|
||||
: address;
|
||||
@@ -234,17 +221,17 @@ namespace PrivaPub.Infrastructure.Push
|
||||
var name = string.IsNullOrWhiteSpace(mapped.Account?.DisplayName) ? mapped.Account?.Username ?? string.Empty : mapped.Account.DisplayName;
|
||||
return notification.Type switch
|
||||
{
|
||||
NotificationType.Mention => string.Format(_localizer["You were mentioned by {0}"], name),
|
||||
NotificationType.Status => string.Format(_localizer["{0} just posted"], name),
|
||||
NotificationType.Reblog => string.Format(_localizer["{0} boosted your post"], name),
|
||||
NotificationType.Follow => string.Format(_localizer["{0} is now following you"], name),
|
||||
NotificationType.FollowRequest => string.Format(_localizer["Pending follower: {0}"], name),
|
||||
NotificationType.Favourite => string.Format(_localizer["{0} favorited your post"], name),
|
||||
NotificationType.Poll => string.Format(_localizer["A poll by {0} has ended"], name),
|
||||
NotificationType.Update => string.Format(_localizer["{0} edited a post"], name),
|
||||
NotificationType.Quote => string.Format(_localizer["{0} quoted your post"], name),
|
||||
NotificationType.Reaction => string.Format(_localizer["{0} reacted to your post with {1}"], name, notification.Emoji),
|
||||
NotificationType.ModerationWarning => _localizer["You have received a moderation warning"],
|
||||
NotificationType.Mention => string.Format(localizer["You were mentioned by {0}"], name),
|
||||
NotificationType.Status => string.Format(localizer["{0} just posted"], name),
|
||||
NotificationType.Reblog => string.Format(localizer["{0} boosted your post"], name),
|
||||
NotificationType.Follow => string.Format(localizer["{0} is now following you"], name),
|
||||
NotificationType.FollowRequest => string.Format(localizer["Pending follower: {0}"], name),
|
||||
NotificationType.Favourite => string.Format(localizer["{0} favorited your post"], name),
|
||||
NotificationType.Poll => string.Format(localizer["A poll by {0} has ended"], name),
|
||||
NotificationType.Update => string.Format(localizer["{0} edited a post"], name),
|
||||
NotificationType.Quote => string.Format(localizer["{0} quoted your post"], name),
|
||||
NotificationType.Reaction => string.Format(localizer["{0} reacted to your post with {1}"], name, notification.Emoji),
|
||||
NotificationType.ModerationWarning => localizer["You have received a moderation warning"],
|
||||
_ => name
|
||||
};
|
||||
}
|
||||
|
||||
@@ -27,7 +27,12 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
public const string Served = "served";
|
||||
}
|
||||
|
||||
public class InteractionLedger : BackgroundService, IInteractionLedger
|
||||
public class InteractionLedger(
|
||||
InteractionSalts salts,
|
||||
IOptionsMonitor<StatisticsOptions> options,
|
||||
ILogger<InteractionLedger> logger,
|
||||
IJobQueue queue = default,
|
||||
IDomainBlocks blocks = default) : BackgroundService, IInteractionLedger
|
||||
{
|
||||
const int Capacity = 10_000;
|
||||
const int BatchSize = 1000;
|
||||
@@ -46,11 +51,6 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
static readonly TimeSpan TouchInterval = TimeSpan.FromHours(1);
|
||||
static readonly HashSet<string> TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context", "replies" };
|
||||
|
||||
readonly InteractionSalts _salts;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly ILogger<InteractionLedger> _logger;
|
||||
readonly IJobQueue _queue;
|
||||
readonly IDomainBlocks _blocks;
|
||||
readonly ConcurrentDictionary<string, byte> _touches = new(StringComparer.Ordinal);
|
||||
readonly ConcurrentDictionary<string, DateTime> _touchedAt = new(StringComparer.Ordinal);
|
||||
readonly SemaphoreSlim _flushing = new(1, 1);
|
||||
@@ -63,36 +63,26 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
long _written;
|
||||
long _failed;
|
||||
|
||||
public InteractionLedger(InteractionSalts salts, IOptionsMonitor<StatisticsOptions> options, ILogger<InteractionLedger> logger,
|
||||
IJobQueue queue = default, IDomainBlocks blocks = default)
|
||||
{
|
||||
_salts = salts;
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
_queue = queue;
|
||||
_blocks = blocks;
|
||||
}
|
||||
|
||||
public long Dropped => Interlocked.Read(ref _dropped);
|
||||
|
||||
public void Record(InteractionEvent interaction, string actorUri = default, bool hostClaimed = false)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (interaction == default || !_options.CurrentValue.Enabled)
|
||||
if (interaction == default || !options.CurrentValue.Enabled)
|
||||
return;
|
||||
if (!_channel.Writer.TryWrite(new Pending(interaction, actorUri, hostClaimed)))
|
||||
Interlocked.Increment(ref _dropped);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "An interaction could not be recorded");
|
||||
logger.LogDebug(ex, "An interaction could not be recorded");
|
||||
}
|
||||
}
|
||||
|
||||
public void Count(string host, string key, long bytes = 0)
|
||||
{
|
||||
if (!_options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
|
||||
if (!options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
|
||||
return;
|
||||
var day = DateTime.UtcNow.Date;
|
||||
host = Interactions.Host(host);
|
||||
@@ -107,7 +97,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
|
||||
public void CountServer(string section, string key, int? latencyMs = default)
|
||||
{
|
||||
if (!_options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
|
||||
if (!options.CurrentValue.Enabled || string.IsNullOrEmpty(key))
|
||||
return;
|
||||
var day = DateTime.UtcNow.Date;
|
||||
var field = section == ServerSections.Served ? nameof(ServerDay.Served) : nameof(ServerDay.Client);
|
||||
@@ -145,7 +135,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "The interaction ledger could not flush");
|
||||
logger.LogWarning(ex, "The interaction ledger could not flush");
|
||||
await Task.Delay(BatchWait, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
@@ -161,7 +151,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "The interaction ledger could not flush on shutdown");
|
||||
logger.LogWarning(ex, "The interaction ledger could not flush on shutdown");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,10 +199,10 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
.Modify(i => i.LastSeenAt, now)
|
||||
.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token));
|
||||
.ExecuteAsync(token), token);
|
||||
_knownHosts.Add(host);
|
||||
if (_queue != default && _blocks?.IsSuspended(host) != true)
|
||||
await _queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
|
||||
if (queue != default && blocks?.IsSuspended(host) != true)
|
||||
await queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
|
||||
}
|
||||
if (_touchedAt.Count > 100_000)
|
||||
_touchedAt.Clear();
|
||||
@@ -238,7 +228,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
Interlocked.Add(ref _failed, batch.Count);
|
||||
_logger.LogWarning(ex, "{Count} interactions could not be written", batch.Count);
|
||||
logger.LogWarning(ex, "{Count} interactions could not be written", batch.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -254,7 +244,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
if (pending.HostClaimed && interaction.Host != Interactions.Unknown && !(await KnownHosts(token)).Contains(interaction.Host))
|
||||
interaction.Host = Interactions.Unknown;
|
||||
if (pending.ActorUri != default)
|
||||
interaction.ActorHash = InteractionSalts.Hash(await _salts.For(interaction.At, token), pending.ActorUri);
|
||||
interaction.ActorHash = InteractionSalts.Hash(await salts.For(interaction.At, token), pending.ActorUri);
|
||||
if (interaction.Host != Interactions.Unknown && Touches(interaction, pending.ActorUri != default))
|
||||
_touches[interaction.Host] = 0;
|
||||
return interaction;
|
||||
@@ -272,7 +262,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
|
||||
// two flushes upserting a new day's document at once: one inserts it and the other's insert fails on the unique
|
||||
// key; the second try finds the document and adds to it (MongoDB retries this only for single-document updates)
|
||||
static async Task Upsert(Func<Task> write)
|
||||
static async Task Upsert(Func<Task> write, CancellationToken token = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -294,7 +284,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
.Match(d => d.Day == key.Day && d.Host == key.Host)
|
||||
.Modify(b => b.Inc($"{nameof(InstanceDay.Reads)}.{key.Key}", count))
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token));
|
||||
.ExecuteAsync(token), token);
|
||||
}
|
||||
|
||||
var day = DateTime.UtcNow.Date;
|
||||
@@ -311,7 +301,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
.Modify(b => b.Inc(d => d.LedgerFailed, failed))
|
||||
.Modify(b => b.Inc(d => d.LedgerDropped, newlyDropped))
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token));
|
||||
.ExecuteAsync(token), token);
|
||||
foreach (var key in server)
|
||||
{
|
||||
if (!_server.TryRemove(key, out var count) || count == 0)
|
||||
@@ -320,7 +310,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
.Match(d => d.Day == key.Day)
|
||||
.Modify(b => b.Inc($"{key.Field}.{key.Key}", count))
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token));
|
||||
.ExecuteAsync(token), token);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
day = day.Date;
|
||||
if (_known.TryGetValue(day, out var cached))
|
||||
return cached;
|
||||
InteractionSalt salt;
|
||||
var salt = default(InteractionSalt);
|
||||
if (day == DateTime.UtcNow.Date)
|
||||
salt = await DB.Default.UpdateAndGet<InteractionSalt>()
|
||||
.Match(s => s.Day == day)
|
||||
|
||||
@@ -11,13 +11,11 @@ using System.Globalization;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Statistics
|
||||
{
|
||||
public class RollupJob : IJobHandler
|
||||
public class RollupJob(
|
||||
IJobQueue queue) : IJobHandler
|
||||
{
|
||||
public static readonly TimeSpan After = TimeSpan.FromMinutes(30);
|
||||
|
||||
readonly IJobQueue _queue;
|
||||
|
||||
public RollupJob(IJobQueue queue) => _queue = queue;
|
||||
|
||||
public JobKind Kind => JobKind.RollupDay;
|
||||
public int Concurrency => 1;
|
||||
@@ -47,7 +45,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
await DB.Default.DeleteAsync<InteractionSalt>(s => s.Day == day);
|
||||
var next = end;
|
||||
if (next < DateTime.UtcNow.Date.AddDays(1))
|
||||
await _queue.EnqueueMany(new[] { For(next) }, token);
|
||||
await queue.EnqueueMany(new[] { For(next) }, token);
|
||||
return JobOutcome.Done;
|
||||
}
|
||||
|
||||
@@ -95,21 +93,14 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
}
|
||||
}
|
||||
|
||||
public class StatisticsSchedule : BackgroundService
|
||||
public class StatisticsSchedule(
|
||||
IJobQueue queue,
|
||||
ILogger<StatisticsSchedule> logger,
|
||||
IOptionsMonitor<StatisticsOptions> options = default) : BackgroundService
|
||||
{
|
||||
static readonly TimeSpan Interval = TimeSpan.FromHours(1);
|
||||
const int CatchUpDays = 7;
|
||||
|
||||
readonly IJobQueue _queue;
|
||||
readonly ILogger<StatisticsSchedule> _logger;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
|
||||
public StatisticsSchedule(IJobQueue queue, ILogger<StatisticsSchedule> logger, IOptionsMonitor<StatisticsOptions> options = default)
|
||||
{
|
||||
_queue = queue;
|
||||
_logger = logger;
|
||||
_options = options;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
@@ -121,7 +112,7 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Statistics rollups could not be planned");
|
||||
logger.LogWarning(ex, "Statistics rollups could not be planned");
|
||||
}
|
||||
try
|
||||
{
|
||||
@@ -146,12 +137,12 @@ namespace PrivaPub.Infrastructure.Statistics
|
||||
if (!rolled && await DB.Default.Find<InteractionEvent>().Match(e => e.At >= day && e.At < end).ExecuteAnyAsync(token))
|
||||
jobs.Add(RollupJob.For(day));
|
||||
}
|
||||
if (_options?.CurrentValue.Crawler.Enabled == true)
|
||||
if (options?.CurrentValue.Crawler.Enabled == true)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
jobs.Add(Federation.Crawler.CrawlPlanner.For(new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Utc)));
|
||||
}
|
||||
await _queue.EnqueueMany(jobs, token);
|
||||
await queue.EnqueueMany(jobs, token);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user