2026-10-04 11:33:39 +02:00
using MongoDB.Entities ;
2026-10-01 17:49:22 +02:00
2026-10-03 12:07:57 +02:00
using PrivaPub.Infrastructure.Geo ;
2026-10-01 17:49:22 +02:00
using PrivaPub.Infrastructure.Http ;
using PrivaPub.Infrastructure.Jobs ;
using PrivaPub.Models.Jobs ;
2026-10-03 12:07:57 +02:00
using System.Globalization ;
using System.Net.Sockets ;
2026-10-01 17:49:22 +02:00
using System.Text.Json ;
namespace PrivaPub.Federation.Objects
{
2026-11-10 01:24:43 +01:00
public class InstanceDescriber (
IFederationHttp federationHttp ,
IGeoLocator geoLocator = default ,
IConnectedAddresses addresses = default ,
ICdnRanges cdnRanges = default ,
IEdgeHints edgeHints = default ) : IJobHandler
2026-10-01 17:49:22 +02:00
{
2026-10-03 12:07:57 +02:00
const int MaxRawChars = 64 * 1024 ;
2026-10-01 17:49:22 +02:00
public JobKind Kind => JobKind . DescribeInstance ;
public int Concurrency => 1 ;
public int MaxAttempts => 2 ;
public int PerHostLimit => 1 ;
2026-11-10 01:24:43 +01:00
[System.Diagnostics.CodeAnalysis.SuppressMessage("HouseRules", "HR0011", Justification = "An ISO week key such as 2026-W05, not a number shown to a reader.")]
2026-10-03 12:07:57 +02:00
public static string Week ( DateTime at ) => $"{ISOWeek.GetYear(at)}-W{ISOWeek.GetWeekOfYear(at):00}" ;
public static string DedupeKey ( string host , DateTime at ) => $"describe|{host}|{ISOWeek.GetYear(at)}-{ISOWeek.GetWeekOfYear(at)}" ;
2026-10-01 17:49:22 +02:00
public async Task < JobOutcome > Handle ( Job job , CancellationToken token )
{
2026-10-05 22:03:35 +02:00
// a description that failed (Smithereen serves no NodeInfo until its admin writes a description) frees its
// week's key, so the server's next arrival, an hour on at the soonest, asks again
if (! await Describe ( job . Payload , crawled : false , default , token ) && ! string . IsNullOrEmpty ( job . DedupeKey ))
await DB . Default . Update < Job >(). MatchID ( job . ID ). Modify ( j => j . DedupeKey , $"{job.DedupeKey}|failed|{job.ID}" ). ExecuteAsync ( token );
2026-10-03 12:07:57 +02:00
return JobOutcome . Done ;
2026-10-01 17:49:22 +02:00
}
2026-10-03 12:07:57 +02:00
//NodeInfo, the Mastodon instance API where the software has one, and where the address we reached sits; unsigned,
//because none of these are ActivityPub documents. `allowed` is the crawler's robots.txt.
public async Task < bool > Describe ( string host , bool crawled , Func < string , bool > allowed , CancellationToken token )
2026-10-01 17:49:22 +02:00
{
2026-10-03 12:07:57 +02:00
allowed ??= _ => true ;
2026-11-10 01:24:43 +01:00
var facts = default ( NodeInfoFacts );
var rawNodeInfo = default ( string );
var error = default ( string );
2026-10-03 12:07:57 +02:00
using ( HttpScope . For ( "nodeinfo" ))
{
error = allowed ( "/.well-known/nodeinfo" ) ? "no NodeInfo" : "robots.txt disallows NodeInfo" ;
using var links = allowed ( "/.well-known/nodeinfo" )
2026-11-10 01:24:43 +01:00
? await federationHttp . GetJson ( Address ( $"https://{host}/.well-known/nodeinfo" ), "application/json" , sign : default , token )
2026-10-03 12:07:57 +02:00
: default ;
var ( href , version ) = links == default ? default : InstanceDocuments . NodeInfoLink ( links . Root );
if ( href != default && allowed ( new Uri ( href ). AbsolutePath ))
{
2026-11-10 01:24:43 +01:00
using var nodeInfo = await federationHttp . GetJson ( Address ( href ), "application/json" , sign : default , token );
2026-10-03 12:07:57 +02:00
if ( nodeInfo != default )
{
facts = InstanceDocuments . NodeInfo ( nodeInfo . Root , version );
rawNodeInfo = nodeInfo . Root . GetRawText ();
error = default ;
}
else
error = "NodeInfo unreadable" ;
}
2026-10-06 11:12:05 +02:00
// no NodeInfo: NodeInfo2, Castopod's
if ( facts == default && allowed ( "/.well-known/x-nodeinfo2" ))
{
2026-11-10 01:24:43 +01:00
using var nodeInfo2 = await federationHttp . GetJson ( Address ( $"https://{host}/.well-known/x-nodeinfo2" ), "application/json" , sign : default , token );
2026-10-06 11:12:05 +02:00
if ( nodeInfo2 != default && InstanceDocuments . NodeInfo2 ( nodeInfo2 . Root ) is { } second )
{
facts = second ;
rawNodeInfo = nodeInfo2 . Root . GetRawText ();
error = default ;
}
}
2026-10-03 12:07:57 +02:00
}
InstanceApiFacts api = default ;
string apiKind = default , rawApi = default ;
if ( InstanceDocuments . SpeaksMastodonApi ( facts ?. Software ))
using ( HttpScope . For ( "instance-api" ))
foreach ( var ( path , kind ) in new [] { ( "/api/v2/instance" , "mastodon-v2" ), ( "/api/v1/instance" , "mastodon-v1" ) })
{
if (! allowed ( path ))
continue ;
2026-11-10 01:24:43 +01:00
using var document = await federationHttp . GetJson ( Address ( $"https://{host}{path}" ), "application/json" , sign : default , token );
2026-10-03 12:07:57 +02:00
if ( document == default || document . Root . ValueKind != JsonValueKind . Object )
continue ;
api = InstanceDocuments . InstanceApi ( document . Root );
apiKind = kind ;
rawApi = document . Root . GetRawText ();
break ;
}
var geo = Locate ( host );
var now = DateTime . UtcNow ;
2026-10-01 17:49:22 +02:00
var update = DB . Default . Update < RemoteInstance >()
. Match ( i => i . Host == host )
2026-10-03 12:07:57 +02:00
. Modify ( i => i . DescribedAt , now )
2026-10-01 17:49:22 +02:00
. Modify ( i => i . DescriptionError , error )
. Option ( o => o . IsUpsert = true );
2026-10-03 12:07:57 +02:00
if ( crawled )
2026-10-01 17:49:22 +02:00
update = update
2026-10-03 12:07:57 +02:00
. Modify ( b => b . SetOnInsert ( i => i . Seen , "crawled" ))
. Modify ( b => b . SetOnInsert ( i => i . FirstSeenAt , now ));
if ( facts != default )
update = update
. Modify ( i => i . Software , facts . Software )
. Modify ( i => i . SoftwareVersion , facts . SoftwareVersion )
. Modify ( i => i . NodeName , facts . NodeName )
. Modify ( i => i . NodeDescription , facts . NodeDescription )
. Modify ( i => i . NodeInfoVersion , facts . Version )
. Modify ( i => i . Protocols , facts . Protocols )
. Modify ( i => i . OpenRegistrations , facts . OpenRegistrations )
. Modify ( i => i . UsersTotal , facts . UsersTotal )
. Modify ( i => i . UsersActiveMonth , facts . UsersActiveMonth )
. Modify ( i => i . UsersActiveHalfyear , facts . UsersActiveHalfyear )
. Modify ( i => i . LocalPosts , facts . LocalPosts )
. Modify ( i => i . LocalComments , facts . LocalComments )
. Modify ( i => i . NodeInfo , rawNodeInfo . Length <= MaxRawChars ? rawNodeInfo : default );
if ( api != default )
update = update
. Modify ( i => i . InstanceApi , apiKind )
. Modify ( i => i . InstanceApiRaw , rawApi . Length <= MaxRawChars ? rawApi : default )
. Modify ( i => i . Title , api . Title )
. Modify ( i => i . Languages , api . Languages ?? facts ?. Languages )
. Modify ( i => i . RegistrationMode , api . RegistrationMode )
. Modify ( i => i . MaxPostChars , api . MaxPostChars )
. Modify ( i => i . MastodonApiVersion , api . ApiVersion )
. Modify ( i => i . SourceUrl , api . SourceUrl );
else if ( facts ?. Languages != default )
update = update . Modify ( i => i . Languages , facts . Languages );
if ( geo != default )
update = update . Modify ( i => i . Geo , geo );
2026-10-01 17:49:22 +02:00
await update . ExecuteAsync ( token );
2026-10-03 12:07:57 +02:00
var instance = await DB . Default . Find < RemoteInstance >(). Match ( i => i . Host == host ). ExecuteFirstAsync ( token );
2026-10-01 17:49:22 +02:00
2026-10-03 12:07:57 +02:00
var week = Week ( now );
await DB . Default . Update < RemoteInstanceSnapshot >()
. Match ( s => s . Host == host && s . Week == week )
. Modify ( s => s . TakenAt , now )
. Modify ( s => s . Seen , instance ?. Seen )
. Modify ( s => s . Reachable , facts != default || api != default )
. Modify ( s => s . Software , instance ?. Software )
. Modify ( s => s . SoftwareVersion , instance ?. SoftwareVersion )
. Modify ( s => s . UsersTotal , instance ?. UsersTotal )
. Modify ( s => s . UsersActiveMonth , instance ?. UsersActiveMonth )
. Modify ( s => s . UsersActiveHalfyear , instance ?. UsersActiveHalfyear )
. Modify ( s => s . LocalPosts , instance ?. LocalPosts )
. Modify ( s => s . OpenRegistrations , instance ?. OpenRegistrations )
. Modify ( s => s . RegistrationMode , instance ?. RegistrationMode )
. Modify ( s => s . Languages , instance ?. Languages )
. Modify ( s => s . Country , instance ?. Geo ?. Country )
2026-10-04 11:33:39 +02:00
. Modify ( s => s . City , instance ?. Geo ?. City )
. Modify ( s => s . Latitude , instance ?. Geo ?. Latitude )
. Modify ( s => s . Longitude , instance ?. Geo ?. Longitude )
2026-10-03 12:07:57 +02:00
. Modify ( s => s . Asn , instance ?. Geo ?. Asn )
. Modify ( s => s . Cdn , instance ?. Geo ?. Cdn )
2026-10-04 11:33:39 +02:00
. Modify ( s => s . CdnDomain , instance ?. Geo ?. CdnDomain )
. Modify ( s => s . CdnSource , instance ?. Geo ?. CdnSource )
2026-10-03 12:07:57 +02:00
. Modify ( s => s . Ipv6 , instance ?. Geo ?. Ipv6 )
. Option ( o => o . IsUpsert = true )
. ExecuteAsync ( token );
return facts != default ;
}
2026-10-01 17:49:22 +02:00
2026-10-04 11:33:39 +02:00
// Where the address we reached sits, and the CDN in front of it, found by its published ranges, then by the
// fingerprint in the responses we just got, then by a network that carries only a CDN.
2026-10-03 12:07:57 +02:00
InstanceGeo Locate ( string host )
2026-10-01 17:49:22 +02:00
{
2026-10-04 11:33:39 +02:00
var reachedHost = new Uri ( Address ( $"https://{host}/" )). Host ;
2026-11-10 01:24:43 +01:00
var reached = addresses ?. Of ( reachedHost );
var fix = reached == default ? default : geoLocator ?. Locate ( reached );
var ( cdn , cdnSource ) = Cdn ( reached , reachedHost , fix ?. Asn , cdnRanges , edgeHints );
2026-10-04 11:33:39 +02:00
if ( fix == default && cdn == default )
2026-10-01 17:49:22 +02:00
return default ;
2026-10-03 12:07:57 +02:00
return new InstanceGeo
{
2026-10-04 11:33:39 +02:00
Country = fix ?. Country ,
City = fix ?. City ,
Latitude = fix ?. Latitude ,
Longitude = fix ?. Longitude ,
Asn = fix ?. Asn ,
AsnOrg = fix ?. AsnOrg ,
Cdn = cdn ?. Name ,
CdnDomain = cdn ?. Domain ,
CdnSource = cdnSource ,
Ipv6 = reached ?. AddressFamily == AddressFamily . InterNetworkV6 ,
2026-11-10 01:24:43 +01:00
Source = geoLocator ?. Source ,
2026-10-03 12:07:57 +02:00
LocatedAt = DateTime . UtcNow
};
2026-10-01 17:49:22 +02:00
}
2026-10-03 12:07:57 +02:00
2026-10-04 11:33:39 +02:00
public static ( CdnProvider Cdn , string Source ) Cdn ( System . Net . IPAddress address , string host , int? asn , ICdnRanges ranges , IEdgeHints edges )
{
if ( ranges ?. Of ( address ) is { } listed )
return ( listed , "ranges" );
if ( edges ?. Of ( host ) is { } hinted )
return ( hinted , "headers" );
if ( CdnCatalog . OfAsn ( asn ) is { } network )
return ( network , "asn" );
return default ;
}
2026-10-03 12:07:57 +02:00
//where an https address is fetched from: itself, except in tests, which point it at a plain-http peer
protected virtual string Address ( string url ) => url ;
2026-10-01 17:49:22 +02:00
}
}