Files
SocialPub/PrivaPub/Domain/Statistics/ServerPlaces.cs
T
thepraandClaude Opus 5.5 0f73f69239
Build / Build (push) Successful in 30s
House rules: primary constructors, a cancellation token on every async call, and the rules fail the build
- Services, controllers and jobs take their dependencies through primary
  constructors and use them directly; where a method already named a local
  after a dependency, the dependency takes its type's name.
- Every async method takes a CancellationToken last and passes it down to
  MongoDB, HTTP and the job queue. The setup claim is still given back when
  a request is cancelled.
- The federation and media options are read through IOptionsMonitor.
- The Mastodon entities keep sending "" for empty strings, and every model
  field where null means something says so.
- The inbox's 503 with Retry-After, which other servers retry, is the one
  server error left, and says why.
- Arasaka.HouseRules.Analyzers checks all of this in every build, here and in
  PrivaPub.ClientModels as decePubClient builds it.

Checked against GoToSocial in the pasture: 64 passed, 0 failed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
2026-11-10 01:24:43 +01:00

158 lines
7.0 KiB
C#

using Microsoft.Extensions.Caching.Memory;
using MongoDB.Entities;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Models.Jobs;
namespace PrivaPub.Domain.Statistics
{
// Where a server was before it went behind a CDN, as its weekly snapshots last saw it.
public sealed record PlaceBeforeCdn(string Week, string Country, string City, double? Latitude, double? Longitude);
// A CDN as a group of servers: those behind it now, and week by week how many were behind it, who joined and who left.
public sealed record CdnGroup(string Key, string Name, string Domain, int Servers, int Ranges, DateTime? RangesUpdatedAt, List<string> DetectedBy);
public sealed record CdnWeek(string Week, int Servers, List<string> Joined, List<string> Left);
public sealed record CdnHistory(CdnGroup Group, List<string> Hosts, List<CdnWeek> Weeks);
public interface IServerPlaces
{
Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token);
Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token);
Task<List<CdnGroup>> Cdns(CancellationToken token);
Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token);
}
// Servers through time (owner decision 2026-10-04): each server's weekly snapshots (place, CDN, size), and the servers
// grouped by the CDN in front of them, named by the CDN's own domain. Read from the weekly snapshots InstanceDescriber
// writes; the CDN pages are kept for half an hour, because a large CDN reads many snapshots.
public class ServerPlaces(
ICdnRanges cdnRanges,
IMemoryCache cache) : IServerPlaces
{
public const int MaxHosts = 500;
const int MaxChanges = 50;
static readonly TimeSpan CacheFor = TimeSpan.FromMinutes(30);
// the CDN key a server's place belongs to: the CDN's domain, or its name for a CDN that has none
public static string KeyOf(InstanceGeo geo) =>
geo?.Cdn == default ? default : geo.CdnDomain ?? CdnCatalog.ByKey(geo.Cdn)?.Key ?? geo.Cdn.ToLowerInvariant();
static string KeyOf(RemoteInstanceSnapshot snapshot) =>
snapshot.Cdn == default ? default : snapshot.CdnDomain ?? CdnCatalog.ByKey(snapshot.Cdn)?.Key ?? snapshot.Cdn.ToLowerInvariant();
public async Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token)
{
var wanted = hosts.Where(h => h != default).Distinct().ToList();
if (wanted.Count == 0)
return new();
var located = await DB.Default.Find<RemoteInstanceSnapshot>()
.Match(s => wanted.Contains(s.Host) && s.Cdn == null && (s.Country != null || s.Latitude != null))
.Sort(s => s.Week, Order.Descending)
.ExecuteAsync(token);
return located.GroupBy(s => s.Host)
.Select(g => g.First())
.ToDictionary(s => s.Host, s => new PlaceBeforeCdn(s.Week, s.Country, s.City, s.Latitude, s.Longitude));
}
public async Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token) =>
await DB.Default.Find<RemoteInstanceSnapshot>()
.Match(s => s.Host == host)
.Sort(s => s.Week, Order.Descending)
.Limit(weeks)
.ExecuteAsync(token);
public async Task<List<CdnGroup>> Cdns(CancellationToken token)
{
var fronted = await Fronted(token);
var status = cdnRanges.Status;
var keys = CdnCatalog.All.Select(p => p.Key).Concat(fronted.Select(i => KeyOf(i.Geo))).Distinct(StringComparer.OrdinalIgnoreCase);
return keys.Select(key => Group(key, fronted, status)).OrderByDescending(g => g.Servers).ThenBy(g => g.Name).ToList();
}
public async Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token)
{
key = key?.Trim().ToLowerInvariant();
if (string.IsNullOrEmpty(key) || key.Length > 100)
return default;
return await cache.GetOrCreateAsync($"cdn-history|{key}|{weeks}", async entry =>
{
entry.AbsoluteExpirationRelativeToNow = CacheFor;
var fronted = await Fronted(token);
var provider = CdnCatalog.ByKey(key);
if (provider == default && fronted.All(i => KeyOf(i.Geo) != key))
return default;
var group = Group(provider?.Key ?? key, fronted, cdnRanges.Status);
var hosts = fronted.Where(i => KeyOf(i.Geo) == group.Key).Select(i => i.Host).OrderBy(h => h).Take(MaxHosts).ToList();
return new CdnHistory(group, hosts, await Weeks(group, weeks, token));
});
}
async Task<List<CdnWeek>> Weeks(CdnGroup group, int weeks, CancellationToken token)
{
var since = InstanceDescriber.Week(DateTime.UtcNow.AddDays(-7 * weeks));
var everBehind = (await DB.Default.Find<RemoteInstanceSnapshot>()
.Match(f => f.Gte(s => s.Week, since) & (f.Eq(s => s.CdnDomain, group.Key) | f.Eq(s => s.Cdn, group.Name)))
.Project(s => new RemoteInstanceSnapshot { Host = s.Host })
.ExecuteAsync(token))
.Select(s => s.Host).Distinct().ToList();
if (everBehind.Count == 0)
return new();
var snapshots = await DB.Default.Find<RemoteInstanceSnapshot>()
.Match(f => f.In(s => s.Host, everBehind) & f.Gte(s => s.Week, since))
.Project(s => new RemoteInstanceSnapshot { Host = s.Host, Week = s.Week, Cdn = s.Cdn, CdnDomain = s.CdnDomain })
.ExecuteAsync(token);
// a server is behind the CDN from the week a snapshot shows it there until one shows it elsewhere
var behind = new Dictionary<string, bool>();
var history = new List<CdnWeek>();
foreach (var week in snapshots.GroupBy(s => s.Week).OrderBy(w => w.Key, StringComparer.Ordinal))
{
var joined = new List<string>();
var left = new List<string>();
foreach (var snapshot in week)
{
var now = KeyOf(snapshot) == group.Key;
var before = behind.GetValueOrDefault(snapshot.Host);
if (now && !before)
joined.Add(snapshot.Host);
else if (!now && before)
left.Add(snapshot.Host);
behind[snapshot.Host] = now;
}
history.Add(new CdnWeek(week.Key, behind.Count(b => b.Value), joined.OrderBy(h => h).Take(MaxChanges).ToList(),
left.OrderBy(h => h).Take(MaxChanges).ToList()));
}
history.Reverse();
return history;
}
static async Task<List<RemoteInstance>> Fronted(CancellationToken token) =>
await DB.Default.Find<RemoteInstance>()
.Match(i => i.Geo.Cdn != null)
.Project(i => new RemoteInstance { Host = i.Host, Geo = i.Geo })
.ExecuteAsync(token);
static CdnGroup Group(string key, List<RemoteInstance> fronted, IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> status)
{
var provider = CdnCatalog.ByKey(key);
var servers = fronted.Where(i => KeyOf(i.Geo) == key).ToList();
var detectedBy = new List<string>();
if (provider?.Lists.Length > 0)
detectedBy.Add("ranges");
if (provider != default)
detectedBy.Add("headers");
if (provider?.Asns.Length > 0)
detectedBy.Add("asn");
var ranges = status.TryGetValue(key, out var known) ? known : default;
return new CdnGroup(key, provider?.Name ?? servers.Select(i => i.Geo.Cdn).FirstOrDefault() ?? key, provider?.Domain, servers.Count, ranges.Ranges,
ranges.Ranges > 0 ? ranges.FetchedAt : null, detectedBy);
}
}
}