Build / Build (push) Successful in 30s
- Services, controllers and jobs take their dependencies through primary constructors and use them directly; where a method already named a local after a dependency, the dependency takes its type's name. - Every async method takes a CancellationToken last and passes it down to MongoDB, HTTP and the job queue. The setup claim is still given back when a request is cancelled. - The federation and media options are read through IOptionsMonitor. - The Mastodon entities keep sending "" for empty strings, and every model field where null means something says so. - The inbox's 503 with Retry-After, which other servers retry, is the one server error left, and says why. - Arasaka.HouseRules.Analyzers checks all of this in every build, here and in PrivaPub.ClientModels as decePubClient builds it. Checked against GoToSocial in the pasture: 64 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
214 lines
8.9 KiB
C#
214 lines
8.9 KiB
C#
using MongoDB.Entities;
|
|
|
|
using PrivaPub.Infrastructure.Geo;
|
|
using PrivaPub.Infrastructure.Http;
|
|
using PrivaPub.Infrastructure.Jobs;
|
|
using PrivaPub.Models.Jobs;
|
|
|
|
using System.Globalization;
|
|
using System.Net.Sockets;
|
|
using System.Text.Json;
|
|
|
|
namespace PrivaPub.Federation.Objects
|
|
{
|
|
public class InstanceDescriber(
|
|
IFederationHttp federationHttp,
|
|
IGeoLocator geoLocator = default,
|
|
IConnectedAddresses addresses = default,
|
|
ICdnRanges cdnRanges = default,
|
|
IEdgeHints edgeHints = default) : IJobHandler
|
|
{
|
|
const int MaxRawChars = 64 * 1024;
|
|
|
|
|
|
public JobKind Kind => JobKind.DescribeInstance;
|
|
public int Concurrency => 1;
|
|
public int MaxAttempts => 2;
|
|
public int PerHostLimit => 1;
|
|
|
|
[System.Diagnostics.CodeAnalysis.SuppressMessage("HouseRules", "HR0011", Justification = "An ISO week key such as 2026-W05, not a number shown to a reader.")]
|
|
public static string Week(DateTime at) => $"{ISOWeek.GetYear(at)}-W{ISOWeek.GetWeekOfYear(at):00}";
|
|
|
|
public static string DedupeKey(string host, DateTime at) => $"describe|{host}|{ISOWeek.GetYear(at)}-{ISOWeek.GetWeekOfYear(at)}";
|
|
|
|
public async Task<JobOutcome> Handle(Job job, CancellationToken token)
|
|
{
|
|
// a description that failed (Smithereen serves no NodeInfo until its admin writes a description) frees its
|
|
// week's key, so the server's next arrival, an hour on at the soonest, asks again
|
|
if (!await Describe(job.Payload, crawled: false, default, token) && !string.IsNullOrEmpty(job.DedupeKey))
|
|
await DB.Default.Update<Job>().MatchID(job.ID).Modify(j => j.DedupeKey, $"{job.DedupeKey}|failed|{job.ID}").ExecuteAsync(token);
|
|
return JobOutcome.Done;
|
|
}
|
|
|
|
//NodeInfo, the Mastodon instance API where the software has one, and where the address we reached sits; unsigned,
|
|
//because none of these are ActivityPub documents. `allowed` is the crawler's robots.txt.
|
|
public async Task<bool> Describe(string host, bool crawled, Func<string, bool> allowed, CancellationToken token)
|
|
{
|
|
allowed ??= _ => true;
|
|
var facts = default(NodeInfoFacts);
|
|
var rawNodeInfo = default(string);
|
|
var error = default(string);
|
|
using (HttpScope.For("nodeinfo"))
|
|
{
|
|
error = allowed("/.well-known/nodeinfo") ? "no NodeInfo" : "robots.txt disallows NodeInfo";
|
|
using var links = allowed("/.well-known/nodeinfo")
|
|
? await federationHttp.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token)
|
|
: default;
|
|
var (href, version) = links == default ? default : InstanceDocuments.NodeInfoLink(links.Root);
|
|
if (href != default && allowed(new Uri(href).AbsolutePath))
|
|
{
|
|
using var nodeInfo = await federationHttp.GetJson(Address(href), "application/json", sign: default, token);
|
|
if (nodeInfo != default)
|
|
{
|
|
facts = InstanceDocuments.NodeInfo(nodeInfo.Root, version);
|
|
rawNodeInfo = nodeInfo.Root.GetRawText();
|
|
error = default;
|
|
}
|
|
else
|
|
error = "NodeInfo unreadable";
|
|
}
|
|
// no NodeInfo: NodeInfo2, Castopod's
|
|
if (facts == default && allowed("/.well-known/x-nodeinfo2"))
|
|
{
|
|
using var nodeInfo2 = await federationHttp.GetJson(Address($"https://{host}/.well-known/x-nodeinfo2"), "application/json", sign: default, token);
|
|
if (nodeInfo2 != default && InstanceDocuments.NodeInfo2(nodeInfo2.Root) is { } second)
|
|
{
|
|
facts = second;
|
|
rawNodeInfo = nodeInfo2.Root.GetRawText();
|
|
error = default;
|
|
}
|
|
}
|
|
}
|
|
|
|
InstanceApiFacts api = default;
|
|
string apiKind = default, rawApi = default;
|
|
if (InstanceDocuments.SpeaksMastodonApi(facts?.Software))
|
|
using (HttpScope.For("instance-api"))
|
|
foreach (var (path, kind) in new[] { ("/api/v2/instance", "mastodon-v2"), ("/api/v1/instance", "mastodon-v1") })
|
|
{
|
|
if (!allowed(path))
|
|
continue;
|
|
using var document = await federationHttp.GetJson(Address($"https://{host}{path}"), "application/json", sign: default, token);
|
|
if (document == default || document.Root.ValueKind != JsonValueKind.Object)
|
|
continue;
|
|
api = InstanceDocuments.InstanceApi(document.Root);
|
|
apiKind = kind;
|
|
rawApi = document.Root.GetRawText();
|
|
break;
|
|
}
|
|
|
|
var geo = Locate(host);
|
|
var now = DateTime.UtcNow;
|
|
var update = DB.Default.Update<RemoteInstance>()
|
|
.Match(i => i.Host == host)
|
|
.Modify(i => i.DescribedAt, now)
|
|
.Modify(i => i.DescriptionError, error)
|
|
.Option(o => o.IsUpsert = true);
|
|
if (crawled)
|
|
update = update
|
|
.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
|
|
.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now));
|
|
if (facts != default)
|
|
update = update
|
|
.Modify(i => i.Software, facts.Software)
|
|
.Modify(i => i.SoftwareVersion, facts.SoftwareVersion)
|
|
.Modify(i => i.NodeName, facts.NodeName)
|
|
.Modify(i => i.NodeDescription, facts.NodeDescription)
|
|
.Modify(i => i.NodeInfoVersion, facts.Version)
|
|
.Modify(i => i.Protocols, facts.Protocols)
|
|
.Modify(i => i.OpenRegistrations, facts.OpenRegistrations)
|
|
.Modify(i => i.UsersTotal, facts.UsersTotal)
|
|
.Modify(i => i.UsersActiveMonth, facts.UsersActiveMonth)
|
|
.Modify(i => i.UsersActiveHalfyear, facts.UsersActiveHalfyear)
|
|
.Modify(i => i.LocalPosts, facts.LocalPosts)
|
|
.Modify(i => i.LocalComments, facts.LocalComments)
|
|
.Modify(i => i.NodeInfo, rawNodeInfo.Length <= MaxRawChars ? rawNodeInfo : default);
|
|
if (api != default)
|
|
update = update
|
|
.Modify(i => i.InstanceApi, apiKind)
|
|
.Modify(i => i.InstanceApiRaw, rawApi.Length <= MaxRawChars ? rawApi : default)
|
|
.Modify(i => i.Title, api.Title)
|
|
.Modify(i => i.Languages, api.Languages ?? facts?.Languages)
|
|
.Modify(i => i.RegistrationMode, api.RegistrationMode)
|
|
.Modify(i => i.MaxPostChars, api.MaxPostChars)
|
|
.Modify(i => i.MastodonApiVersion, api.ApiVersion)
|
|
.Modify(i => i.SourceUrl, api.SourceUrl);
|
|
else if (facts?.Languages != default)
|
|
update = update.Modify(i => i.Languages, facts.Languages);
|
|
if (geo != default)
|
|
update = update.Modify(i => i.Geo, geo);
|
|
await update.ExecuteAsync(token);
|
|
var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
|
|
|
|
var week = Week(now);
|
|
await DB.Default.Update<RemoteInstanceSnapshot>()
|
|
.Match(s => s.Host == host && s.Week == week)
|
|
.Modify(s => s.TakenAt, now)
|
|
.Modify(s => s.Seen, instance?.Seen)
|
|
.Modify(s => s.Reachable, facts != default || api != default)
|
|
.Modify(s => s.Software, instance?.Software)
|
|
.Modify(s => s.SoftwareVersion, instance?.SoftwareVersion)
|
|
.Modify(s => s.UsersTotal, instance?.UsersTotal)
|
|
.Modify(s => s.UsersActiveMonth, instance?.UsersActiveMonth)
|
|
.Modify(s => s.UsersActiveHalfyear, instance?.UsersActiveHalfyear)
|
|
.Modify(s => s.LocalPosts, instance?.LocalPosts)
|
|
.Modify(s => s.OpenRegistrations, instance?.OpenRegistrations)
|
|
.Modify(s => s.RegistrationMode, instance?.RegistrationMode)
|
|
.Modify(s => s.Languages, instance?.Languages)
|
|
.Modify(s => s.Country, instance?.Geo?.Country)
|
|
.Modify(s => s.City, instance?.Geo?.City)
|
|
.Modify(s => s.Latitude, instance?.Geo?.Latitude)
|
|
.Modify(s => s.Longitude, instance?.Geo?.Longitude)
|
|
.Modify(s => s.Asn, instance?.Geo?.Asn)
|
|
.Modify(s => s.Cdn, instance?.Geo?.Cdn)
|
|
.Modify(s => s.CdnDomain, instance?.Geo?.CdnDomain)
|
|
.Modify(s => s.CdnSource, instance?.Geo?.CdnSource)
|
|
.Modify(s => s.Ipv6, instance?.Geo?.Ipv6)
|
|
.Option(o => o.IsUpsert = true)
|
|
.ExecuteAsync(token);
|
|
return facts != default;
|
|
}
|
|
|
|
// Where the address we reached sits, and the CDN in front of it, found by its published ranges, then by the
|
|
// fingerprint in the responses we just got, then by a network that carries only a CDN.
|
|
InstanceGeo Locate(string host)
|
|
{
|
|
var reachedHost = new Uri(Address($"https://{host}/")).Host;
|
|
var reached = addresses?.Of(reachedHost);
|
|
var fix = reached == default ? default : geoLocator?.Locate(reached);
|
|
var (cdn, cdnSource) = Cdn(reached, reachedHost, fix?.Asn, cdnRanges, edgeHints);
|
|
if (fix == default && cdn == default)
|
|
return default;
|
|
return new InstanceGeo
|
|
{
|
|
Country = fix?.Country,
|
|
City = fix?.City,
|
|
Latitude = fix?.Latitude,
|
|
Longitude = fix?.Longitude,
|
|
Asn = fix?.Asn,
|
|
AsnOrg = fix?.AsnOrg,
|
|
Cdn = cdn?.Name,
|
|
CdnDomain = cdn?.Domain,
|
|
CdnSource = cdnSource,
|
|
Ipv6 = reached?.AddressFamily == AddressFamily.InterNetworkV6,
|
|
Source = geoLocator?.Source,
|
|
LocatedAt = DateTime.UtcNow
|
|
};
|
|
}
|
|
|
|
public static (CdnProvider Cdn, string Source) Cdn(System.Net.IPAddress address, string host, int? asn, ICdnRanges ranges, IEdgeHints edges)
|
|
{
|
|
if (ranges?.Of(address) is { } listed)
|
|
return (listed, "ranges");
|
|
if (edges?.Of(host) is { } hinted)
|
|
return (hinted, "headers");
|
|
if (CdnCatalog.OfAsn(asn) is { } network)
|
|
return (network, "asn");
|
|
return default;
|
|
}
|
|
|
|
//where an https address is fetched from: itself, except in tests, which point it at a plain-http peer
|
|
protected virtual string Address(string url) => url;
|
|
}
|
|
}
|