Files
SocialPub/PrivaPub/Federation/Crawler/Crawler.cs
T
thepraandClaude Opus 5.5 0f73f69239
Build / Build (push) Successful in 30s
House rules: primary constructors, a cancellation token on every async call, and the rules fail the build
- Services, controllers and jobs take their dependencies through primary
  constructors and use them directly; where a method already named a local
  after a dependency, the dependency takes its type's name.
- Every async method takes a CancellationToken last and passes it down to
  MongoDB, HTTP and the job queue. The setup claim is still given back when
  a request is cancelled.
- The federation and media options are read through IOptionsMonitor.
- The Mastodon entities keep sending "" for empty strings, and every model
  field where null means something says so.
- The inbox's 503 with Retry-After, which other servers retry, is the one
  server error left, and says why.
- Arasaka.HouseRules.Analyzers checks all of this in every build, here and in
  PrivaPub.ClientModels as decePubClient builds it.

Checked against GoToSocial in the pasture: 64 passed, 0 failed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
2026-11-10 01:24:43 +01:00

188 lines
7.1 KiB
C#

using Microsoft.Extensions.Options;
using MongoDB.Entities;
using PrivaPub.Federation.Moderation;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Infrastructure.Statistics;
using PrivaPub.Models;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Statistics;
using PrivaPub.StaticServices;
namespace PrivaPub.Federation.Crawler
{
public static class Stargazer
{
public const string Token = "PrivaPub-Stargazer";
public const string Page = "/stargazing";
public static string UserAgent(string baseAddress) => $"{Token}/{BuildInfo.Ref} (+{baseAddress?.TrimEnd('/')}{Page})";
}
//each hour, the servers due for a visit, spread across the hour so the crawler reaches one at a time
public class CrawlPlanner(
IJobQueue queue,
IDomainBlocks blocks,
IOptionsMonitor<StatisticsOptions> options,
IOptionsMonitor<FederationOptions> federation = default) : IJobHandler
{
public JobKind Kind => JobKind.CrawlPlan;
public int Concurrency => 1;
public int MaxAttempts => 1;
public int PerHostLimit => 1;
public static Job For(DateTime hour) => new()
{
Kind = JobKind.CrawlPlan,
Payload = hour.ToString("yyyy-MM-ddTHH", System.Globalization.CultureInfo.InvariantCulture),
DedupeKey = "crawl-plan|" + hour.ToString("yyyy-MM-ddTHH", System.Globalization.CultureInfo.InvariantCulture),
RunAt = hour
};
public async Task<JobOutcome> Handle(Job job, CancellationToken token)
{
var crawler = options.CurrentValue.Crawler;
if (!crawler.Enabled)
return JobOutcome.Done;
var now = DateTime.UtcNow;
foreach (var seed in PeerHosts.Filter(crawler.Seeds ?? new(), host => !blocks.IsSuspended(host), federation?.CurrentValue.AllowPrivateNetworks == true))
await DB.Default.Update<RemoteInstance>()
.Match(i => i.Host == seed)
.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
.Option(o => o.IsUpsert = true)
.ExecuteAsync(token);
var due = now.AddDays(-Math.Max(crawler.RevisitDays, 1));
var perHour = Math.Clamp(crawler.HostsPerHour, 1, 600);
var candidates = await DB.Default.Find<RemoteInstance>()
.Match(i => (i.LastCrawledAt == null || i.LastCrawledAt < due) && (i.UnavailableUntil == null || i.UnavailableUntil < now))
.Sort(i => i.LastCrawledAt, Order.Ascending)
.Limit(perHour * 3)
.ExecuteAsync(token);
var week = InstanceDescriber.Week(now);
var jobs = candidates.Where(i => blocks.Find(i.Host) == default && PeerHosts.Normalise(i.Host) != default)
.Take(perHour)
.Select((instance, index) => new Job
{
Kind = JobKind.CrawlInstance,
Payload = instance.Host,
Host = instance.Host,
DedupeKey = $"crawl|{instance.Host}|{week}",
RunAt = now.AddSeconds(index * 3600.0 / perHour)
});
await queue.EnqueueMany(jobs, token);
return JobOutcome.Done;
}
}
//one server: its robots.txt, a description if only crawling ever found it, then the servers its peers list names
public class InstanceCrawler(
IFederationHttp http,
InstanceDescriber describer,
IDomainBlocks blocks,
IOptionsMonitor<StatisticsOptions> options,
IOptionsMonitor<AppConfiguration> app,
IOptionsMonitor<FederationOptions> federation,
IInteractionLedger ledger = default) : IJobHandler
{
const int MaxPeers = 20_000;
const int MaxRobotsBytes = 64 * 1024;
public JobKind Kind => JobKind.CrawlInstance;
public int Concurrency => 1;
public int MaxAttempts => 1;
public int PerHostLimit => 1;
protected virtual string Address(string url) => url;
public async Task<JobOutcome> Handle(Job job, CancellationToken token)
{
var crawler = options.CurrentValue.Crawler;
var host = job.Payload;
if (!crawler.Enabled || blocks.Find(host) != default)
return JobOutcome.Done;
using var crawl = HttpScope.Crawl(Stargazer.UserAgent(app.CurrentValue.BackendBaseAddress));
var robots = default(RobotsTxt);
using (HttpScope.For("robots"))
{
var (status, text) = await http.GetText(Address($"https://{host}/robots.txt"), MaxRobotsBytes, token);
robots = RobotsTxt.From(status, text, Stargazer.Token, "PrivaPub");
}
var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
var now = DateTime.UtcNow;
if (!robots.Allows("/.well-known/nodeinfo") && !robots.Allows("/api/v1/instance/peers"))
{
await Visited(host, robotsDisallowed: true, peers: default, "robots.txt keeps us out", token);
Record(host, Interactions.Refused, "robots-disallowed");
return JobOutcome.Done;
}
var reachable = true;
if (instance?.Seen != "touched")
reachable = await describer.Describe(host, crawled: true, robots.Allows, token);
int? peersCount = default;
if (robots.Allows("/api/v1/instance/peers"))
{
var peers = default(IReadOnlyList<string>);
using (HttpScope.For("peers"))
peers = await http.GetStringArray(Address($"https://{host}/api/v1/instance/peers"), MaxPeers, token);
peersCount = peers.Count;
await Discover(peers, crawler, token);
}
await Visited(host, robotsDisallowed: false, peersCount, reachable ? default : "no NodeInfo", token);
Record(host, reachable ? Interactions.Ok : Interactions.Failed, reachable ? default : "unreachable");
return JobOutcome.Done;
}
async Task Discover(IReadOnlyList<string> peers, CrawlerOptions crawler, CancellationToken token)
{
var room = Math.Min(crawler.MaxNewHostsPerCrawl, crawler.MaxHosts - (int)await DB.Default.CountAsync<RemoteInstance>(i => i.Seen == "crawled", token));
if (room <= 0)
return;
var candidates = PeerHosts.Filter(peers, host => http.IsAllowed(new Uri($"https://{host}/")) && blocks.Find(host) == default,
federation.CurrentValue.AllowPrivateNetworks).ToList();
var now = DateTime.UtcNow;
foreach (var batch in candidates.Chunk(500))
{
if (room <= 0)
return;
var known = (await DB.Default.Find<RemoteInstance, string>().Match(i => batch.Contains(i.Host)).Project(i => i.Host).ExecuteAsync(token)).ToHashSet();
foreach (var host in batch.Where(h => !known.Contains(h)).Take(room))
{
await DB.Default.Update<RemoteInstance>()
.Match(i => i.Host == host)
.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
.Option(o => o.IsUpsert = true)
.ExecuteAsync(token);
room--;
}
}
}
static async Task Visited(string host, bool robotsDisallowed, int? peers, string error, CancellationToken token)
{
var update = DB.Default.Update<RemoteInstance>()
.Match(i => i.Host == host)
.Modify(i => i.LastCrawledAt, DateTime.UtcNow)
.Modify(i => i.RobotsDisallowed, robotsDisallowed)
.Modify(i => i.CrawlError, error)
.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
.Option(o => o.IsUpsert = true);
if (peers is { } count)
update = update.Modify(i => i.PeersCount, count);
await update.ExecuteAsync(token);
}
void Record(string host, string outcome, string reason) =>
ledger?.Record(new InteractionEvent { Channel = Interactions.Crawl, Host = host, Outcome = outcome, Reason = reason, Crawl = true });
}
}