Build / Build (push) Successful in 30s
- Services, controllers and jobs take their dependencies through primary constructors and use them directly; where a method already named a local after a dependency, the dependency takes its type's name. - Every async method takes a CancellationToken last and passes it down to MongoDB, HTTP and the job queue. The setup claim is still given back when a request is cancelled. - The federation and media options are read through IOptionsMonitor. - The Mastodon entities keep sending "" for empty strings, and every model field where null means something says so. - The inbox's 503 with Retry-After, which other servers retry, is the one server error left, and says why. - Arasaka.HouseRules.Analyzers checks all of this in every build, here and in PrivaPub.ClientModels as decePubClient builds it. Checked against GoToSocial in the pasture: 64 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W4YtEJrgxG12Pc39Fai8Bd
165 lines
6.4 KiB
C#
165 lines
6.4 KiB
C#
using AngleSharp.Html.Parser;
|
|
|
|
using Microsoft.Extensions.Options;
|
|
|
|
using MongoDB.Entities;
|
|
|
|
using PrivaPub.Infrastructure.Statistics;
|
|
using PrivaPub.Models.Statistics;
|
|
using PrivaPub.Domain.Privacy;
|
|
using PrivaPub.Federation.Actors;
|
|
using PrivaPub.Federation.Objects;
|
|
using PrivaPub.Infrastructure.Http;
|
|
using PrivaPub.Infrastructure.Jobs;
|
|
using PrivaPub.Models.Jobs;
|
|
using PrivaPub.Models.Post;
|
|
using PrivaPub.StaticServices;
|
|
|
|
using System.Security.Cryptography;
|
|
|
|
using PostEntity = PrivaPub.Models.Post.Post;
|
|
|
|
namespace PrivaPub.Domain.Content
|
|
{
|
|
public class LinkPreview : Entity
|
|
{
|
|
public string Url { get; set; }
|
|
public string Title { get; set; }
|
|
public string Description { get; set; }
|
|
public string ImageURL { get; set; }
|
|
public string SiteName { get; set; }
|
|
public bool Failed { get; set; }
|
|
public DateTime FetchedAt { get; set; } = DateTime.UtcNow;
|
|
}
|
|
|
|
public interface ILinkPreviews
|
|
{
|
|
Task Wanted(PostEntity post, CancellationToken token);
|
|
}
|
|
|
|
public class LinkPreviews(
|
|
DbEntities dbEntities,
|
|
ILocalActorService localActors,
|
|
IFederationHttp federationHttp,
|
|
IJobQueue queue,
|
|
IOptionsMonitor<FederationOptions> options,
|
|
IInteractionLedger ledger = default) : ILinkPreviews, IJobHandler
|
|
{
|
|
const int MaxJitterSeconds = 60;
|
|
static readonly TimeSpan Freshness = TimeSpan.FromDays(7);
|
|
static readonly HtmlParser Parser = new();
|
|
|
|
|
|
|
|
public JobKind Kind => JobKind.FetchPreview;
|
|
public int Concurrency => 2;
|
|
public int MaxAttempts => 1;
|
|
public int PerHostLimit => 1;
|
|
|
|
public async Task Wanted(PostEntity post, CancellationToken token)
|
|
{
|
|
if (!options.CurrentValue.FetchLinkPreviews || !Previewable(post) || post.Link == default && post.Media.Count > 0)
|
|
return;
|
|
var url = post.Link?.Href ?? FirstLink(post, localActors.BaseAddress);
|
|
if (url == default)
|
|
return;
|
|
await queue.EnqueueMany(new[]
|
|
{
|
|
new Job
|
|
{
|
|
Kind = JobKind.FetchPreview, Payload = post.ID, Host = new Uri(url).Host.ToLowerInvariant(), DedupeKey = $"preview|{post.ID}",
|
|
RunAt = DateTime.UtcNow.AddSeconds(RandomNumberGenerator.GetInt32(0, MaxJitterSeconds + 1))
|
|
}
|
|
}, token);
|
|
}
|
|
|
|
public async Task<JobOutcome> Handle(Job job, CancellationToken token)
|
|
{
|
|
var post = await dbEntities.Posts.MatchID(job.Payload).ExecuteFirstAsync(token);
|
|
if (!VisibilityPolicy.Shown(post) || !Previewable(post))
|
|
return JobOutcome.Done;
|
|
var url = post.Link?.Href ?? FirstLink(post, localActors.BaseAddress);
|
|
if (url == default)
|
|
return JobOutcome.Done;
|
|
|
|
var preview = await DB.Default.Find<LinkPreview>().Match(p => p.Url == url).ExecuteFirstAsync(token);
|
|
if (preview == default || preview.FetchedAt < DateTime.UtcNow - Freshness)
|
|
preview = await Fetch(url, token);
|
|
if (preview == default || preview.Failed)
|
|
return JobOutcome.Done;
|
|
|
|
var link = post.Link ?? new PostLink { Href = url };
|
|
link.Title ??= preview.Title;
|
|
link.Description ??= preview.Description;
|
|
link.ImageURL ??= preview.ImageURL;
|
|
await DB.Default.Update<PostEntity>().MatchID(post.ID).Modify(p => p.Link, link).ExecuteAsync(token);
|
|
return JobOutcome.Done;
|
|
}
|
|
|
|
async Task<LinkPreview> Fetch(string url, CancellationToken token)
|
|
{
|
|
var started = System.Diagnostics.Stopwatch.GetTimestamp();
|
|
var (finalUri, html) = await federationHttp.GetPage(url, token);
|
|
var preview = html == default ? new LinkPreview { Url = url, Failed = true } : Read(url, finalUri, html);
|
|
ledger?.Record(new InteractionEvent
|
|
{
|
|
Channel = Interactions.Preview,
|
|
Host = Interactions.HostOf(url),
|
|
Outcome = html == default ? Interactions.Failed : Interactions.Ok,
|
|
Reason = html == default ? default : preview.Title == default && preview.ImageURL == default ? "no-card" : "card",
|
|
LatencyMs = (int)System.Diagnostics.Stopwatch.GetElapsedTime(started).TotalMilliseconds,
|
|
Bytes = html == default ? default : System.Text.Encoding.UTF8.GetByteCount(html),
|
|
Redirects = finalUri == default || finalUri.AbsoluteUri == url ? 0 : 1
|
|
});
|
|
await DB.Default.Update<LinkPreview>()
|
|
.Match(p => p.Url == url)
|
|
.Modify(p => p.Title, preview.Title)
|
|
.Modify(p => p.Description, preview.Description)
|
|
.Modify(p => p.ImageURL, preview.ImageURL)
|
|
.Modify(p => p.SiteName, preview.SiteName)
|
|
.Modify(p => p.Failed, preview.Failed)
|
|
.Modify(p => p.FetchedAt, DateTime.UtcNow)
|
|
.Option(o => o.IsUpsert = true)
|
|
.ExecuteAsync(token);
|
|
return preview;
|
|
}
|
|
|
|
//only public posts (owner decision 1), checked again when the job runs; never a boost or a post whose card is complete
|
|
static bool Previewable(PostEntity post) => post.Visibility is (PostVisibility.Public or PostVisibility.Unlisted) && !post.IsLocalOnly
|
|
&& post.ReblogOfPostId == default && post.Link?.Title == default;
|
|
|
|
public static LinkPreview Read(string url, Uri finalUri, string html)
|
|
{
|
|
var document = Parser.ParseDocument(html);
|
|
string Meta(params string[] names) => names
|
|
.Select(name => document.QuerySelector($"meta[property='{name}'], meta[name='{name}']")?.GetAttribute("content"))
|
|
.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value));
|
|
var image = Meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image", "twitter:image:src");
|
|
var imageUri = image != default && Uri.TryCreate(finalUri, image, out var resolved) && resolved.Scheme is "https" or "http" ? resolved.AbsoluteUri : default;
|
|
var preview = new LinkPreview
|
|
{
|
|
Url = url,
|
|
Title = ObjectShapes.Text(Meta("og:title", "twitter:title") ?? document.Title, 300),
|
|
Description = ObjectShapes.Text(Meta("og:description", "twitter:description", "description"), 1000),
|
|
SiteName = ObjectShapes.Text(Meta("og:site_name"), 200),
|
|
ImageURL = imageUri
|
|
};
|
|
preview.Failed = preview.Title == default && preview.Description == default && preview.ImageURL == default;
|
|
return preview;
|
|
}
|
|
|
|
public static string FirstLink(PostEntity post, string ownBase)
|
|
{
|
|
if (string.IsNullOrEmpty(post.ContentHtml))
|
|
return default;
|
|
var document = Parser.ParseDocument(post.ContentHtml);
|
|
return document.QuerySelectorAll("a[href]")
|
|
.Where(a => !(a.ClassList.Contains("mention") || a.ClassList.Contains("hashtag") || a.GetAttribute("rel")?.Contains("tag") == true
|
|
|| a.Closest(".quote-inline") != default))
|
|
.Select(a => a.GetAttribute("href"))
|
|
.FirstOrDefault(href => Uri.TryCreate(href, UriKind.Absolute, out var uri) && uri.Scheme is "https" or "http"
|
|
&& href != post.Url && href != post.ObjectURI && !href.StartsWith(ownBase + "/", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
}
|
|
}
|