House rules: primary constructors, a cancellation token on every async call, and the rules fail the build
Build / Build (push) Successful in 30s

- Services, controllers and jobs take their dependencies through primary
  constructors and use them directly; where a method already named a local
  after a dependency, the dependency takes its type's name.
- Every async method takes a CancellationToken last and passes it down to
  MongoDB, HTTP and the job queue. The setup claim is still given back when
  a request is cancelled.
- The federation and media options are read through IOptionsMonitor.
- The Mastodon entities keep sending "" for empty strings, and every model
  field where null means something says so.
- The inbox's 503 with Retry-After, which other servers retry, is the one
  server error left, and says why.
- Arasaka.HouseRules.Analyzers checks all of this in every build, here and in
  PrivaPub.ClientModels as decePubClient builds it.

Checked against GoToSocial in the pasture: 64 passed, 0 failed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
This commit is contained in:
thepraandClaude Opus 5.5 committed 2026-11-10 01:24:43 +01:00
1 parent f0a14c737f
commit 0f73f69239
176 files changed
+2789 -3974

No files matched your search

+28 -49
View File
@@ -23,21 +23,12 @@ namespace PrivaPub.Federation.Crawler
}
//each hour, the servers due for a visit, spread across the hour so the crawler reaches one at a time
public class CrawlPlanner : IJobHandler
public class CrawlPlanner(
IJobQueue queue,
IDomainBlocks blocks,
IOptionsMonitor<StatisticsOptions> options,
IOptionsMonitor<FederationOptions> federation = default) : IJobHandler
{
readonly IJobQueue _queue;
readonly IDomainBlocks _blocks;
readonly IOptionsMonitor<StatisticsOptions> _options;
readonly IOptionsMonitor<FederationOptions> _federation;
public CrawlPlanner(IJobQueue queue, IDomainBlocks blocks, IOptionsMonitor<StatisticsOptions> options, IOptionsMonitor<FederationOptions> federation = default)
{
_queue = queue;
_blocks = blocks;
_options = options;
_federation = federation;
}
public JobKind Kind => JobKind.CrawlPlan;
public int Concurrency => 1;
public int MaxAttempts => 1;
@@ -53,11 +44,11 @@ namespace PrivaPub.Federation.Crawler
public async Task<JobOutcome> Handle(Job job, CancellationToken token)
{
var crawler = _options.CurrentValue.Crawler;
var crawler = options.CurrentValue.Crawler;
if (!crawler.Enabled)
return JobOutcome.Done;
var now = DateTime.UtcNow;
foreach (var seed in PeerHosts.Filter(crawler.Seeds ?? new(), host => !_blocks.IsSuspended(host), _federation?.CurrentValue.AllowPrivateNetworks == true))
foreach (var seed in PeerHosts.Filter(crawler.Seeds ?? new(), host => !blocks.IsSuspended(host), federation?.CurrentValue.AllowPrivateNetworks == true))
await DB.Default.Update<RemoteInstance>()
.Match(i => i.Host == seed)
.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
@@ -73,7 +64,7 @@ namespace PrivaPub.Federation.Crawler
.Limit(perHour * 3)
.ExecuteAsync(token);
var week = InstanceDescriber.Week(now);
var jobs = candidates.Where(i => _blocks.Find(i.Host) == default && PeerHosts.Normalise(i.Host) != default)
var jobs = candidates.Where(i => blocks.Find(i.Host) == default && PeerHosts.Normalise(i.Host) != default)
.Take(perHour)
.Select((instance, index) => new Job
{
@@ -83,36 +74,24 @@ namespace PrivaPub.Federation.Crawler
DedupeKey = $"crawl|{instance.Host}|{week}",
RunAt = now.AddSeconds(index * 3600.0 / perHour)
});
await _queue.EnqueueMany(jobs, token);
await queue.EnqueueMany(jobs, token);
return JobOutcome.Done;
}
}
//one server: its robots.txt, a description if only crawling ever found it, then the servers its peers list names
public class InstanceCrawler : IJobHandler
public class InstanceCrawler(
IFederationHttp http,
InstanceDescriber describer,
IDomainBlocks blocks,
IOptionsMonitor<StatisticsOptions> options,
IOptionsMonitor<AppConfiguration> app,
IOptionsMonitor<FederationOptions> federation,
IInteractionLedger ledger = default) : IJobHandler
{
const int MaxPeers = 20_000;
const int MaxRobotsBytes = 64 * 1024;
readonly IFederationHttp _http;
readonly InstanceDescriber _describer;
readonly IDomainBlocks _blocks;
readonly IOptionsMonitor<StatisticsOptions> _options;
readonly IOptionsMonitor<AppConfiguration> _app;
readonly IOptionsMonitor<FederationOptions> _federation;
readonly IInteractionLedger _ledger;
public InstanceCrawler(IFederationHttp http, InstanceDescriber describer, IDomainBlocks blocks, IOptionsMonitor<StatisticsOptions> options,
IOptionsMonitor<AppConfiguration> app, IOptionsMonitor<FederationOptions> federation, IInteractionLedger ledger = default)
{
_http = http;
_describer = describer;
_blocks = blocks;
_options = options;
_app = app;
_federation = federation;
_ledger = ledger;
}
public JobKind Kind => JobKind.CrawlInstance;
public int Concurrency => 1;
@@ -123,16 +102,16 @@ namespace PrivaPub.Federation.Crawler
public async Task<JobOutcome> Handle(Job job, CancellationToken token)
{
var crawler = _options.CurrentValue.Crawler;
var crawler = options.CurrentValue.Crawler;
var host = job.Payload;
if (!crawler.Enabled || _blocks.Find(host) != default)
if (!crawler.Enabled || blocks.Find(host) != default)
return JobOutcome.Done;
using var crawl = HttpScope.Crawl(Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress));
using var crawl = HttpScope.Crawl(Stargazer.UserAgent(app.CurrentValue.BackendBaseAddress));
RobotsTxt robots;
var robots = default(RobotsTxt);
using (HttpScope.For("robots"))
{
var (status, text) = await _http.GetText(Address($"https://{host}/robots.txt"), MaxRobotsBytes, token);
var (status, text) = await http.GetText(Address($"https://{host}/robots.txt"), MaxRobotsBytes, token);
robots = RobotsTxt.From(status, text, Stargazer.Token, "PrivaPub");
}
var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
@@ -146,14 +125,14 @@ namespace PrivaPub.Federation.Crawler
var reachable = true;
if (instance?.Seen != "touched")
reachable = await _describer.Describe(host, crawled: true, robots.Allows, token);
reachable = await describer.Describe(host, crawled: true, robots.Allows, token);
int? peersCount = default;
if (robots.Allows("/api/v1/instance/peers"))
{
IReadOnlyList<string> peers;
var peers = default(IReadOnlyList<string>);
using (HttpScope.For("peers"))
peers = await _http.GetStringArray(Address($"https://{host}/api/v1/instance/peers"), MaxPeers, token);
peers = await http.GetStringArray(Address($"https://{host}/api/v1/instance/peers"), MaxPeers, token);
peersCount = peers.Count;
await Discover(peers, crawler, token);
}
@@ -167,8 +146,8 @@ namespace PrivaPub.Federation.Crawler
var room = Math.Min(crawler.MaxNewHostsPerCrawl, crawler.MaxHosts - (int)await DB.Default.CountAsync<RemoteInstance>(i => i.Seen == "crawled", token));
if (room <= 0)
return;
var candidates = PeerHosts.Filter(peers, host => _http.IsAllowed(new Uri($"https://{host}/")) && _blocks.Find(host) == default,
_federation.CurrentValue.AllowPrivateNetworks).ToList();
var candidates = PeerHosts.Filter(peers, host => http.IsAllowed(new Uri($"https://{host}/")) && blocks.Find(host) == default,
federation.CurrentValue.AllowPrivateNetworks).ToList();
var now = DateTime.UtcNow;
foreach (var batch in candidates.Chunk(500))
{
@@ -203,6 +182,6 @@ namespace PrivaPub.Federation.Crawler
}
void Record(string host, string outcome, string reason) =>
_ledger?.Record(new InteractionEvent { Channel = Interactions.Crawl, Host = host, Outcome = outcome, Reason = reason, Crawl = true });
ledger?.Record(new InteractionEvent { Channel = Interactions.Crawl, Host = host, Outcome = outcome, Reason = reason, Crawl = true });
}
}