House rules: primary constructors, a cancellation token on every async call, and the rules fail the build
Build / Build (push) Successful in 30s
Build / Build (push) Successful in 30s
- Services, controllers and jobs take their dependencies through primary constructors and use them directly; where a method already named a local after a dependency, the dependency takes its type's name. - Every async method takes a CancellationToken last and passes it down to MongoDB, HTTP and the job queue. The setup claim is still given back when a request is cancelled. - The federation and media options are read through IOptionsMonitor. - The Mastodon entities keep sending "" for empty strings, and every model field where null means something says so. - The inbox's 503 with Retry-After, which other servers retry, is the one server error left, and says why. - Arasaka.HouseRules.Analyzers checks all of this in every build, here and in PrivaPub.ClientModels as decePubClient builds it. Checked against GoToSocial in the pasture: 64 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
This commit is contained in:
1 parent
f0a14c737f
commit
0f73f69239
176 files changed
+2789
-3974
No files matched your search
@@ -44,17 +44,17 @@ namespace PrivaPub.Domain.Media
|
||||
// - what is too big to cache is remembered for an hour (it is streamed instead), what failed for five minutes;
|
||||
// - the cache's size is counted as it grows, and trimmed as soon as it passes the cap;
|
||||
// - nothing of a suspended server, or of one whose media are rejected, is proxied, and blocking one purges its files.
|
||||
public class MediaProxy : IMediaProxy
|
||||
public class MediaProxy(
|
||||
ILocalActorService localActors,
|
||||
IFederationHttp http,
|
||||
IMediaService mediaService,
|
||||
IOptionsMonitor<MediaOptions> options,
|
||||
Federation.Moderation.IDomainBlocks domainBlocks = default) : IMediaProxy
|
||||
{
|
||||
static readonly TimeSpan TooBigFor = TimeSpan.FromHours(1);
|
||||
static readonly TimeSpan FailedFor = TimeSpan.FromMinutes(5);
|
||||
const int Downloads = 8;//remote files fetched at once, however many clients ask
|
||||
|
||||
readonly ILocalActorService _localActors;
|
||||
readonly IFederationHttp _http;
|
||||
readonly IMediaService _media;
|
||||
readonly IOptionsMonitor<MediaOptions> _options;
|
||||
readonly Federation.Moderation.IDomainBlocks _domainBlocks;
|
||||
// Lazy: GetOrAdd may run its factory more than once when asked at the same time, but only one Lazy is ever evaluated
|
||||
readonly ConcurrentDictionary<string, Lazy<Task<(ProxyOutcome, string, string)>>> _inFlight = new();
|
||||
readonly ConcurrentDictionary<string, (ProxyOutcome Outcome, DateTime Until)> _refused = new();
|
||||
@@ -64,16 +64,6 @@ namespace PrivaPub.Domain.Media
|
||||
long _bytes = -1;//what the cache holds, counted once from the disk and then as it changes
|
||||
int _trimming;
|
||||
|
||||
public MediaProxy(ILocalActorService localActors, IFederationHttp http, IMediaService media, IOptionsMonitor<MediaOptions> options,
|
||||
Federation.Moderation.IDomainBlocks domainBlocks = default)
|
||||
{
|
||||
_localActors = localActors;
|
||||
_http = http;
|
||||
_media = media;
|
||||
_options = options;
|
||||
_domainBlocks = domainBlocks;
|
||||
}
|
||||
|
||||
byte[] Key
|
||||
{
|
||||
get
|
||||
@@ -87,15 +77,15 @@ namespace PrivaPub.Domain.Media
|
||||
|
||||
public string Wrap(string remoteUrl)
|
||||
{
|
||||
if (string.IsNullOrEmpty(remoteUrl) || remoteUrl.StartsWith(_localActors.BaseAddress + "/", StringComparison.OrdinalIgnoreCase))
|
||||
if (string.IsNullOrEmpty(remoteUrl) || remoteUrl.StartsWith(localActors.BaseAddress + "/", StringComparison.OrdinalIgnoreCase))
|
||||
return remoteUrl;
|
||||
var encoded = Base64Url(Encoding.UTF8.GetBytes(remoteUrl));
|
||||
return $"{_localActors.BaseAddress}/media/proxy/{Sign(remoteUrl)}/{encoded}";
|
||||
return $"{localActors.BaseAddress}/media/proxy/{Sign(remoteUrl)}/{encoded}";
|
||||
}
|
||||
|
||||
public string Verified(string signature, string encodedUrl)
|
||||
{
|
||||
string url;
|
||||
var url = default(string);
|
||||
try
|
||||
{
|
||||
url = Encoding.UTF8.GetString(FromBase64Url(encodedUrl));
|
||||
@@ -109,9 +99,9 @@ namespace PrivaPub.Domain.Media
|
||||
|
||||
public bool Refuses(string url)
|
||||
{
|
||||
if (_domainBlocks == default || !Uri.TryCreate(url, UriKind.Absolute, out var target))
|
||||
if (domainBlocks == default || !Uri.TryCreate(url, UriKind.Absolute, out var target))
|
||||
return false;
|
||||
var block = _domainBlocks.Find(target.Host);
|
||||
var block = domainBlocks.Find(target.Host);
|
||||
return block is { Severity: DomainBlockSeverity.Suspend } or { RejectMedia: true };
|
||||
}
|
||||
|
||||
@@ -132,7 +122,7 @@ namespace PrivaPub.Domain.Media
|
||||
}
|
||||
|
||||
public Task<HttpResponseMessage> Open(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token) =>
|
||||
_http.OpenMedia(url, range, token);
|
||||
http.OpenMedia(url, range, token);
|
||||
|
||||
public async Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token)
|
||||
{
|
||||
@@ -166,10 +156,10 @@ namespace PrivaPub.Domain.Media
|
||||
}
|
||||
}
|
||||
|
||||
async Task<(ProxyOutcome, string, string)> DownloadOnce(string url)
|
||||
async Task<(ProxyOutcome, string, string)> DownloadOnce(string url, CancellationToken token = default)
|
||||
{
|
||||
await Task.Yield();
|
||||
if (!await _downloads.WaitAsync(TimeSpan.FromSeconds(30)))
|
||||
if (!await _downloads.WaitAsync(TimeSpan.FromSeconds(30), token))
|
||||
return (ProxyOutcome.Failed, default, default);//too busy: the client tries again
|
||||
// cached by a download that ended just before this one began
|
||||
if (Cached(url) is { Path: not null } done)
|
||||
@@ -182,16 +172,17 @@ namespace PrivaPub.Domain.Media
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)!);
|
||||
string contentType, refusal;
|
||||
var contentType = default(string);
|
||||
var refusal = default(string);
|
||||
await using (var file = new FileStream(part, FileMode.CreateNew, FileAccess.Write, FileShare.None, 64 * 1024, useAsync: true))
|
||||
(contentType, refusal) = await _http.DownloadMedia(url, _options.CurrentValue.MaxProxiedBytes, file, CancellationToken.None);
|
||||
(contentType, refusal) = await http.DownloadMedia(url, options.CurrentValue.MaxProxiedBytes, file, CancellationToken.None);
|
||||
if (contentType == default)
|
||||
{
|
||||
var outcome = refusal == "too-large" ? ProxyOutcome.TooBig : ProxyOutcome.Failed;
|
||||
_refused[url] = (outcome, DateTime.UtcNow + (outcome == ProxyOutcome.TooBig ? TooBigFor : FailedFor));
|
||||
return (outcome, default, default);
|
||||
}
|
||||
await File.WriteAllTextAsync(typePath, contentType + "\n" + new Uri(url).Host);
|
||||
await File.WriteAllTextAsync(typePath, contentType + "\n" + new Uri(url).Host, token);
|
||||
var size = new FileInfo(part).Length;
|
||||
File.Move(part, path, overwrite: true);
|
||||
Grew(size);
|
||||
@@ -209,7 +200,7 @@ namespace PrivaPub.Domain.Media
|
||||
{
|
||||
if (Interlocked.Read(ref _bytes) < 0)
|
||||
Interlocked.CompareExchange(ref _bytes, Measure(), -1);
|
||||
if (Interlocked.Add(ref _bytes, bytes) > _options.CurrentValue.ProxyCacheBytes)
|
||||
if (Interlocked.Add(ref _bytes, bytes) > options.CurrentValue.ProxyCacheBytes)
|
||||
_ = Task.Run(Trim);
|
||||
}
|
||||
|
||||
@@ -219,7 +210,7 @@ namespace PrivaPub.Domain.Media
|
||||
return;
|
||||
try
|
||||
{
|
||||
Interlocked.Exchange(ref _bytes, TrimDirectory(_media.ProxyRoot, _options.CurrentValue.ProxyCacheBytes));
|
||||
Interlocked.Exchange(ref _bytes, TrimDirectory(mediaService.ProxyRoot, options.CurrentValue.ProxyCacheBytes));
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -229,7 +220,7 @@ namespace PrivaPub.Domain.Media
|
||||
|
||||
public int Purge(string domain)
|
||||
{
|
||||
var directory = new DirectoryInfo(_media.ProxyRoot);
|
||||
var directory = new DirectoryInfo(mediaService.ProxyRoot);
|
||||
if (string.IsNullOrEmpty(domain) || !directory.Exists)
|
||||
return 0;
|
||||
var purged = 0;
|
||||
@@ -256,7 +247,7 @@ namespace PrivaPub.Domain.Media
|
||||
|
||||
long Measure()
|
||||
{
|
||||
var directory = new DirectoryInfo(_media.ProxyRoot);
|
||||
var directory = new DirectoryInfo(mediaService.ProxyRoot);
|
||||
return directory.Exists ? directory.EnumerateFiles("*", SearchOption.AllDirectories).Where(f => f.Extension is not (".type" or ".part")).Sum(f => f.Length) : 0;
|
||||
}
|
||||
|
||||
@@ -290,7 +281,7 @@ namespace PrivaPub.Domain.Media
|
||||
(string Path, string TypePath) CachePaths(string url)
|
||||
{
|
||||
var name = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(url)));
|
||||
var path = System.IO.Path.Combine(_media.ProxyRoot, name[..2], name);
|
||||
var path = System.IO.Path.Combine(mediaService.ProxyRoot, name[..2], name);
|
||||
return (path, path + ".type");
|
||||
}
|
||||
|
||||
@@ -317,7 +308,10 @@ namespace PrivaPub.Domain.Media
|
||||
}
|
||||
}
|
||||
|
||||
public class MediaJanitor : BackgroundService
|
||||
public class MediaJanitor(
|
||||
IMediaService mediaService,
|
||||
IOptionsMonitor<MediaOptions> options,
|
||||
ILogger<MediaJanitor> logger) : BackgroundService
|
||||
{
|
||||
static readonly TimeSpan FirstPass = TimeSpan.FromMinutes(5);
|
||||
static readonly TimeSpan Interval = TimeSpan.FromHours(1);
|
||||
@@ -325,16 +319,6 @@ namespace PrivaPub.Domain.Media
|
||||
// a trashed file waits this long before it is deleted, already out of what /media/files serves
|
||||
public static readonly TimeSpan TrashGrace = TimeSpan.FromDays(1);
|
||||
|
||||
readonly IMediaService _media;
|
||||
readonly IOptionsMonitor<MediaOptions> _options;
|
||||
readonly ILogger<MediaJanitor> _logger;
|
||||
|
||||
public MediaJanitor(IMediaService media, IOptionsMonitor<MediaOptions> options, ILogger<MediaJanitor> logger)
|
||||
{
|
||||
_media = media;
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
@@ -353,7 +337,7 @@ namespace PrivaPub.Domain.Media
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "{Service} pass failed", nameof(MediaJanitor));
|
||||
logger.LogWarning(ex, "{Service} pass failed", nameof(MediaJanitor));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -368,7 +352,7 @@ namespace PrivaPub.Domain.Media
|
||||
public async Task Sweep(CancellationToken token)
|
||||
{
|
||||
var cutoff = DateTime.UtcNow - UnattachedLifetime;
|
||||
var unattached = await _media.Trash(m => m.PostId == null && m.ScheduledStatusId == null && m.ProfileOfAvatarId == null && m.ServerUse == null
|
||||
var unattached = await mediaService.Trash(m => m.PostId == null && m.ScheduledStatusId == null && m.ProfileOfAvatarId == null && m.ServerUse == null
|
||||
&& m.CreatedAt < cutoff, "never posted", token);
|
||||
|
||||
var trashed = await DB.Default.Find<MediaAttachment>().Match(m => m.TrashedAt != null).Limit(2000).ExecuteAsync(token);
|
||||
@@ -379,18 +363,18 @@ namespace PrivaPub.Domain.Media
|
||||
{
|
||||
if (row.TrashedAt < purgeBefore && !maintenance)
|
||||
{
|
||||
await _media.Purge(row, token);
|
||||
await mediaService.Purge(row, token);
|
||||
purged++;
|
||||
}
|
||||
else
|
||||
_media.Hide(row);
|
||||
mediaService.Hide(row);
|
||||
}
|
||||
if (unattached > 0 || purged > 0)
|
||||
_logger.LogInformation("{Service}: {Unattached} uploads never posted trashed, {Purged} trashed files deleted",
|
||||
logger.LogInformation("{Service}: {Unattached} uploads never posted trashed, {Purged} trashed files deleted",
|
||||
nameof(MediaJanitor), unattached, purged);
|
||||
TrimProxyCache();
|
||||
}
|
||||
|
||||
void TrimProxyCache() => MediaProxy.TrimDirectory(_media.ProxyRoot, _options.CurrentValue.ProxyCacheBytes);
|
||||
void TrimProxyCache() => MediaProxy.TrimDirectory(mediaService.ProxyRoot, options.CurrentValue.ProxyCacheBytes);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user