House rules: primary constructors, a cancellation token on every async call, and the rules fail the build
Build / Build (push) Successful in 30s
Build / Build (push) Successful in 30s
- Services, controllers and jobs take their dependencies through primary constructors and use them directly; where a method already named a local after a dependency, the dependency takes its type's name. - Every async method takes a CancellationToken last and passes it down to MongoDB, HTTP and the job queue. The setup claim is still given back when a request is cancelled. - The federation and media options are read through IOptionsMonitor. - The Mastodon entities keep sending "" for empty strings, and every model field where null means something says so. - The inbox's 503 with Retry-After, which other servers retry, is the one server error left, and says why. - Arasaka.HouseRules.Analyzers checks all of this in every build, here and in PrivaPub.ClientModels as decePubClient builds it. Checked against GoToSocial in the pasture: 64 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
This commit is contained in:
1 parent
f0a14c737f
commit
0f73f69239
176 files changed
+2789
-3974
No files matched your search
@@ -23,21 +23,12 @@ namespace PrivaPub.Federation.Crawler
|
||||
}
|
||||
|
||||
//each hour, the servers due for a visit, spread across the hour so the crawler reaches one at a time
|
||||
public class CrawlPlanner : IJobHandler
|
||||
public class CrawlPlanner(
|
||||
IJobQueue queue,
|
||||
IDomainBlocks blocks,
|
||||
IOptionsMonitor<StatisticsOptions> options,
|
||||
IOptionsMonitor<FederationOptions> federation = default) : IJobHandler
|
||||
{
|
||||
readonly IJobQueue _queue;
|
||||
readonly IDomainBlocks _blocks;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly IOptionsMonitor<FederationOptions> _federation;
|
||||
|
||||
public CrawlPlanner(IJobQueue queue, IDomainBlocks blocks, IOptionsMonitor<StatisticsOptions> options, IOptionsMonitor<FederationOptions> federation = default)
|
||||
{
|
||||
_queue = queue;
|
||||
_blocks = blocks;
|
||||
_options = options;
|
||||
_federation = federation;
|
||||
}
|
||||
|
||||
public JobKind Kind => JobKind.CrawlPlan;
|
||||
public int Concurrency => 1;
|
||||
public int MaxAttempts => 1;
|
||||
@@ -53,11 +44,11 @@ namespace PrivaPub.Federation.Crawler
|
||||
|
||||
public async Task<JobOutcome> Handle(Job job, CancellationToken token)
|
||||
{
|
||||
var crawler = _options.CurrentValue.Crawler;
|
||||
var crawler = options.CurrentValue.Crawler;
|
||||
if (!crawler.Enabled)
|
||||
return JobOutcome.Done;
|
||||
var now = DateTime.UtcNow;
|
||||
foreach (var seed in PeerHosts.Filter(crawler.Seeds ?? new(), host => !_blocks.IsSuspended(host), _federation?.CurrentValue.AllowPrivateNetworks == true))
|
||||
foreach (var seed in PeerHosts.Filter(crawler.Seeds ?? new(), host => !blocks.IsSuspended(host), federation?.CurrentValue.AllowPrivateNetworks == true))
|
||||
await DB.Default.Update<RemoteInstance>()
|
||||
.Match(i => i.Host == seed)
|
||||
.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
|
||||
@@ -73,7 +64,7 @@ namespace PrivaPub.Federation.Crawler
|
||||
.Limit(perHour * 3)
|
||||
.ExecuteAsync(token);
|
||||
var week = InstanceDescriber.Week(now);
|
||||
var jobs = candidates.Where(i => _blocks.Find(i.Host) == default && PeerHosts.Normalise(i.Host) != default)
|
||||
var jobs = candidates.Where(i => blocks.Find(i.Host) == default && PeerHosts.Normalise(i.Host) != default)
|
||||
.Take(perHour)
|
||||
.Select((instance, index) => new Job
|
||||
{
|
||||
@@ -83,36 +74,24 @@ namespace PrivaPub.Federation.Crawler
|
||||
DedupeKey = $"crawl|{instance.Host}|{week}",
|
||||
RunAt = now.AddSeconds(index * 3600.0 / perHour)
|
||||
});
|
||||
await _queue.EnqueueMany(jobs, token);
|
||||
await queue.EnqueueMany(jobs, token);
|
||||
return JobOutcome.Done;
|
||||
}
|
||||
}
|
||||
|
||||
//one server: its robots.txt, a description if only crawling ever found it, then the servers its peers list names
|
||||
public class InstanceCrawler : IJobHandler
|
||||
public class InstanceCrawler(
|
||||
IFederationHttp http,
|
||||
InstanceDescriber describer,
|
||||
IDomainBlocks blocks,
|
||||
IOptionsMonitor<StatisticsOptions> options,
|
||||
IOptionsMonitor<AppConfiguration> app,
|
||||
IOptionsMonitor<FederationOptions> federation,
|
||||
IInteractionLedger ledger = default) : IJobHandler
|
||||
{
|
||||
const int MaxPeers = 20_000;
|
||||
const int MaxRobotsBytes = 64 * 1024;
|
||||
|
||||
readonly IFederationHttp _http;
|
||||
readonly InstanceDescriber _describer;
|
||||
readonly IDomainBlocks _blocks;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly IOptionsMonitor<AppConfiguration> _app;
|
||||
readonly IOptionsMonitor<FederationOptions> _federation;
|
||||
readonly IInteractionLedger _ledger;
|
||||
|
||||
public InstanceCrawler(IFederationHttp http, InstanceDescriber describer, IDomainBlocks blocks, IOptionsMonitor<StatisticsOptions> options,
|
||||
IOptionsMonitor<AppConfiguration> app, IOptionsMonitor<FederationOptions> federation, IInteractionLedger ledger = default)
|
||||
{
|
||||
_http = http;
|
||||
_describer = describer;
|
||||
_blocks = blocks;
|
||||
_options = options;
|
||||
_app = app;
|
||||
_federation = federation;
|
||||
_ledger = ledger;
|
||||
}
|
||||
|
||||
public JobKind Kind => JobKind.CrawlInstance;
|
||||
public int Concurrency => 1;
|
||||
@@ -123,16 +102,16 @@ namespace PrivaPub.Federation.Crawler
|
||||
|
||||
public async Task<JobOutcome> Handle(Job job, CancellationToken token)
|
||||
{
|
||||
var crawler = _options.CurrentValue.Crawler;
|
||||
var crawler = options.CurrentValue.Crawler;
|
||||
var host = job.Payload;
|
||||
if (!crawler.Enabled || _blocks.Find(host) != default)
|
||||
if (!crawler.Enabled || blocks.Find(host) != default)
|
||||
return JobOutcome.Done;
|
||||
using var crawl = HttpScope.Crawl(Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress));
|
||||
using var crawl = HttpScope.Crawl(Stargazer.UserAgent(app.CurrentValue.BackendBaseAddress));
|
||||
|
||||
RobotsTxt robots;
|
||||
var robots = default(RobotsTxt);
|
||||
using (HttpScope.For("robots"))
|
||||
{
|
||||
var (status, text) = await _http.GetText(Address($"https://{host}/robots.txt"), MaxRobotsBytes, token);
|
||||
var (status, text) = await http.GetText(Address($"https://{host}/robots.txt"), MaxRobotsBytes, token);
|
||||
robots = RobotsTxt.From(status, text, Stargazer.Token, "PrivaPub");
|
||||
}
|
||||
var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
|
||||
@@ -146,14 +125,14 @@ namespace PrivaPub.Federation.Crawler
|
||||
|
||||
var reachable = true;
|
||||
if (instance?.Seen != "touched")
|
||||
reachable = await _describer.Describe(host, crawled: true, robots.Allows, token);
|
||||
reachable = await describer.Describe(host, crawled: true, robots.Allows, token);
|
||||
|
||||
int? peersCount = default;
|
||||
if (robots.Allows("/api/v1/instance/peers"))
|
||||
{
|
||||
IReadOnlyList<string> peers;
|
||||
var peers = default(IReadOnlyList<string>);
|
||||
using (HttpScope.For("peers"))
|
||||
peers = await _http.GetStringArray(Address($"https://{host}/api/v1/instance/peers"), MaxPeers, token);
|
||||
peers = await http.GetStringArray(Address($"https://{host}/api/v1/instance/peers"), MaxPeers, token);
|
||||
peersCount = peers.Count;
|
||||
await Discover(peers, crawler, token);
|
||||
}
|
||||
@@ -167,8 +146,8 @@ namespace PrivaPub.Federation.Crawler
|
||||
var room = Math.Min(crawler.MaxNewHostsPerCrawl, crawler.MaxHosts - (int)await DB.Default.CountAsync<RemoteInstance>(i => i.Seen == "crawled", token));
|
||||
if (room <= 0)
|
||||
return;
|
||||
var candidates = PeerHosts.Filter(peers, host => _http.IsAllowed(new Uri($"https://{host}/")) && _blocks.Find(host) == default,
|
||||
_federation.CurrentValue.AllowPrivateNetworks).ToList();
|
||||
var candidates = PeerHosts.Filter(peers, host => http.IsAllowed(new Uri($"https://{host}/")) && blocks.Find(host) == default,
|
||||
federation.CurrentValue.AllowPrivateNetworks).ToList();
|
||||
var now = DateTime.UtcNow;
|
||||
foreach (var batch in candidates.Chunk(500))
|
||||
{
|
||||
@@ -203,6 +182,6 @@ namespace PrivaPub.Federation.Crawler
|
||||
}
|
||||
|
||||
void Record(string host, string outcome, string reason) =>
|
||||
_ledger?.Record(new InteractionEvent { Channel = Interactions.Crawl, Host = host, Outcome = outcome, Reason = reason, Crawl = true });
|
||||
ledger?.Record(new InteractionEvent { Channel = Interactions.Crawl, Host = host, Outcome = outcome, Reason = reason, Crawl = true });
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user