SocialPub/PrivaPub/Federation/Actors/OutboxBackfill.cs

using MongoDB.Entities;

using PrivaPub.Federation.Inbox;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Models.Jobs;
using PrivaPub.StaticServices;

using System.Text.Json;

namespace PrivaPub.Federation.Actors
{
	// The latest public posts of an account a persona has just followed, read from the first page of its outbox (at most
	// twenty, once a day) and kept as any fetched post: its profile shows them and threads find them at once, instead of
	// only what it posts from then on. Homes still get only what arrives from now on, as on Mastodon.
	public class OutboxBackfill : IJobHandler
	{
		public const int MaxPosts = 20;

		readonly DbEntities _dbEntities;
		readonly IRemoteActorService _remoteActors;
		readonly IRemotePosts _remotePosts;

		public OutboxBackfill(DbEntities dbEntities, IRemoteActorService remoteActors, IRemotePosts remotePosts)
		{
			_dbEntities = dbEntities;
			_remoteActors = remoteActors;
			_remotePosts = remotePosts;
		}

		public JobKind Kind => JobKind.BackfillOutbox;
		public int Concurrency => 1;
		public int MaxAttempts => 2;
		public int PerHostLimit => 1;

		public static string DedupeKey(string actorUri, DateTime day) => $"backfill|{actorUri}|{day:yyyyMMdd}";

		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
		{
			var actor = await _dbEntities.ForeignAvatars.Match(a => a.ActorURI == job.Payload && !a.DeletionAt.HasValue).ExecuteFirstAsync(token);
			if (actor == default || string.IsNullOrEmpty(actor.OutboxURL) || !Origin.Same(actor.OutboxURL, actor.ActorURI))
				return JobOutcome.Done;
			foreach (var objectUri in (await Items(actor, token)).Take(MaxPosts))
				await _remotePosts.StoreContext(objectUri, 0, token);
			return JobOutcome.Done;
		}

		// the posts the outbox's first page creates, newest first: Create items name or hold them; Announces are left out,
		// being others' posts
		async Task<List<string>> Items(Models.User.ForeignAvatar actor, CancellationToken token)
		{
			using var scope = HttpScope.For("outbox");
			using var outbox = await _remoteActors.FetchObject(actor.OutboxURL, token);
			if (outbox == default || outbox.Root.ValueKind != JsonValueKind.Object)
				return new();
			var page = ItemsOf(outbox.Root, actor);
			if (page.Count > 0 || !outbox.Root.TryGetProperty("first", out var first))
				return page;
			if (first.ValueKind == JsonValueKind.Object)
				return ItemsOf(first, actor);
			var pageUri = first.ValueKind == JsonValueKind.String ? first.GetString() : default;
			if (pageUri == default || !Origin.Same(pageUri, actor.ActorURI))
				return page;
			using var fetched = await _remoteActors.FetchObject(pageUri, token);
			return fetched == default || fetched.Root.ValueKind != JsonValueKind.Object ? page : ItemsOf(fetched.Root, actor);
		}

		static List<string> ItemsOf(JsonElement collection, Models.User.ForeignAvatar actor)
		{
			var posts = new List<string>();
			foreach (var name in new[] { "orderedItems", "items" })
			{
				if (!collection.TryGetProperty(name, out var list) || list.ValueKind != JsonValueKind.Array)
					continue;
				foreach (var item in list.EnumerateArray())
				{
					var objectUri = item.ValueKind switch
					{
						JsonValueKind.Object when Text(item, "type") == "Create" && item.TryGetProperty("object", out var created) =>
							created.ValueKind == JsonValueKind.String ? created.GetString() : created.ValueKind == JsonValueKind.Object ? Text(created, "id") : default,
						_ => default
					};
					if (objectUri != default && Origin.Same(objectUri, actor.ActorURI))
						posts.Add(objectUri);
				}
			}
			return posts;
		}

		static string Text(JsonElement element, string property) =>
			element.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : default;
	}
}