SocialPub/PrivaPub/Domain/Portability/ExportArchiveJob.cs

using MongoDB.Entities;

using PrivaPub.Domain.Media;
using PrivaPub.Domain.Social;
using PrivaPub.Federation.Actors;
using PrivaPub.Federation.Rendering;
using PrivaPub.Infrastructure.Backup;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Models.Federation;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Media;
using PrivaPub.Models.Post;
using PrivaPub.Models.Social;
using PrivaPub.Models.User;

using System.IO.Compression;
using System.Text;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Nodes;

using PostEntity = PrivaPub.Models.Post.Post;

namespace PrivaPub.Domain.Portability
{
	// A persona's archive, in Mastodon's account-archive layout so other servers' importers read it:
	// - actor.json (as other servers see the persona: its public key, never the private one), avatar and header;
	// - outbox.json: every post of its own as a Create (any visibility but circles' and located posts, addressed as it was
	//   delivered) and its boosts as Announces; the media its posts hold under media_attachments/files/, which the
	//   attachments' urls name;
	// - likes.json, bookmarks.json, and the CSV files Mastodon exports: follows, lists, blocks, mutes, blocked servers,
	//   bookmarks;
	// - privapub/: what only PrivaPub has: filters, followed hashtags, the notification policy, pinned posts, scheduled and
	//   located posts, and archive.json saying whose it is.
	// Nothing of its root, its siblings, its keys or anyone's tokens. Written whole to a .partial file, then renamed; kept a
	// week (PersonaArchives.Forget).
	public class ExportArchiveJob(ILocalActorService localActors, IMediaService media, Backups backups, ILogger<ExportArchiveJob> logger) : IJobHandler
	{
		public const int Format = 1;
		static readonly JsonSerializerOptions Options = new() { WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping };
		static readonly JsonWriterOptions Writer = new() { Indented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping };
		const string MediaFolder = "media_attachments/files/";

		public JobKind Kind => JobKind.ExportArchive;
		public int Concurrency => 1;
		public int MaxAttempts => 2;
		public int PerHostLimit => 1;

		public static Job JobFor(string avatarId, string host) => new()
		{
			Kind = JobKind.ExportArchive,
			Payload = avatarId,
			Host = host,
			DedupeKey = $"export|{avatarId}|{DateTime.UtcNow.Ticks}"
		};

		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
		{
			var avatarId = job.Payload;
			var actor = await localActors.FindById(LocalActorKind.Person, avatarId, token);
			if (actor == default)
				return JobOutcome.Done;
			await DB.Default.Update<PersonaArchive>().MatchID(avatarId).Modify(a => a.ExportState, "running").Modify(a => a.ExportError, null).ExecuteAsync(token);
			var folder = PersonaArchives.Folder(backups.Root, avatarId);
			ServerBackup.MakeDirectory(Path.GetDirectoryName(folder)!);
			ServerBackup.MakeDirectory(folder);
			var name = $"{actor.UserName}-{DateTime.UtcNow:yyyyMMdd-HHmmss}.zip";
			var partial = Path.Combine(folder, name + ServerBackup.PartialSuffix);
			try
			{
				await using (var file = new FileStream(partial, FileMode.Create, FileAccess.Write))
				using (var zip = new ZipArchive(file, ZipArchiveMode.Create))
					await Write(zip, actor, token);
				var final = Path.Combine(folder, name);
				File.Move(partial, final);
				foreach (var old in Directory.EnumerateFiles(folder, "*.zip").Where(f => f != final))
					File.Delete(old);
				await DB.Default.Update<PersonaArchive>().MatchID(avatarId)
					.Modify(a => a.ExportState, "ready").Modify(a => a.ExportReadyAt, DateTime.UtcNow)
					.Modify(a => a.ExportFile, name).Modify(a => a.ExportBytes, new FileInfo(final).Length).ExecuteAsync(token);
				return JobOutcome.Done;
			}
			catch (Exception ex) when (ex is not OperationCanceledException)
			{
				logger.LogError(ex, "The archive of {Persona} could not be made", actor.Handle);
				if (File.Exists(partial))
					File.Delete(partial);
				await DB.Default.Update<PersonaArchive>().MatchID(avatarId).Modify(a => a.ExportState, "failed")
					.Modify(a => a.ExportError, ex.Message).ExecuteAsync(CancellationToken.None);
				return JobOutcome.Done;
			}
		}

		async Task Write(ZipArchive zip, LocalActor actor, CancellationToken token)
		{
			var avatar = await DB.Default.Find<Avatar>().MatchID(actor.Id).ExecuteFirstAsync(token);
			var files = new SortedSet<string>(StringComparer.Ordinal);
			var prefix = media.Url(string.Empty);

			var document = ActivityPubRenderer.Actor(actor);
			foreach (var (field, kind) in new[] { ("icon", "avatar"), ("image", "header") })
				if (document[field] is JsonObject image && image["url"]?.GetValue<string>() is { } url && url.StartsWith(prefix, StringComparison.Ordinal))
				{
					var relative = url[prefix.Length..];
					var local = Path.Combine(media.Root, relative);
					if (!File.Exists(local))
						continue;
					var entry = kind + Path.GetExtension(relative);
					await Copy(zip, local, entry, token);
					image["url"] = entry;
				}
			await Text(zip, "actor.json", document.ToJsonString(Options), token);

			var counts = await Outbox(zip, actor, prefix, files, token);
			foreach (var relative in files)
			{
				var local = Path.Combine(media.Root, relative);
				if (File.Exists(local))
					await Copy(zip, local, MediaFolder + relative, token);
			}

			var handles = new Handles(localActors);
			await Collection(zip, "likes.json", await Uris(DB.Default.Find<Favourite, string>().Match(f => f.AccountId == actor.Id).Project(f => f.PostId), token), token);
			var bookmarks = await Uris(DB.Default.Find<Bookmark, string>().Match(b => b.AvatarId == actor.Id).Project(b => b.PostId), token);
			await Collection(zip, "bookmarks.json", bookmarks, token);
			await Text(zip, "bookmarks.csv", string.Concat(bookmarks.Select(b => b + "\n")), token);

			var follows = await DB.Default.Find<Following>().Match(f => f.AvatarId == actor.Id).ExecuteAsync(token);
			var csv = new StringBuilder("Account address,Show boosts,Notify on new posts,Languages\n");
			foreach (var follow in follows)
				csv.Append($"{Csv(await handles.Of(follow.TargetActorURI, token))},{(follow.ShowReblogs ? "true" : "false")},false,\n");
			await Text(zip, "following_accounts.csv", csv.ToString(), token);

			csv.Clear();
			var lists = await DB.Default.Find<PersonaList>().Match(l => l.AvatarId == actor.Id).ExecuteAsync(token);
			foreach (var list in lists)
				foreach (var member in await DB.Default.Find<PersonaListMember>().Match(m => m.ListId == list.ID).ExecuteAsync(token))
					if (follows.FirstOrDefault(f => f.TargetAccountId == member.AccountId) is { } followed)
						csv.Append($"{Csv(list.Title)},{Csv(await handles.Of(followed.TargetActorURI, token))}\n");
			await Text(zip, "lists.csv", csv.ToString(), token);

			csv.Clear();
			foreach (var block in await DB.Default.Find<Block>().Match(b => b.AvatarId == actor.Id).ExecuteAsync(token))
				csv.Append(Csv(await handles.Of(block.TargetActorURI, token)) + "\n");
			await Text(zip, "blocked_accounts.csv", csv.ToString(), token);

			csv.Clear().Append("Account address,Hide notifications\n");
			foreach (var mute in await DB.Default.Find<Mute>().Match(m => m.AvatarId == actor.Id).ExecuteAsync(token))
				csv.Append($"{Csv(await handles.Of(mute.TargetActorURI, token))},{(mute.HideNotifications ? "true" : "false")}\n");
			await Text(zip, "muted_accounts.csv", csv.ToString(), token);

			csv.Clear();
			foreach (var domain in await DB.Default.Find<AccountDomainBlock, string>().Match(d => d.AvatarId == actor.Id).Project(d => d.Domain).ExecuteAsync(token))
				csv.Append(domain + "\n");
			await Text(zip, "blocked_domains.csv", csv.ToString(), token);

			await PrivaPubParts(zip, actor, avatar, prefix, counts, token);
		}

		// the outbox, a post at a time, newest first as Mastodon writes it: what was published and boosted
		async Task<(int Posts, int Boosts, int Located)> Outbox(ZipArchive zip, LocalActor actor, string prefix, ISet<string> files, CancellationToken token)
		{
			var entry = zip.CreateEntry("outbox.json", CompressionLevel.Optimal);
			await using var stream = entry.Open();
			await using var writer = new Utf8JsonWriter(stream, Writer);
			writer.WriteStartObject();
			writer.WriteString("@context", "https://www.w3.org/ns/activitystreams");
			writer.WriteString("id", "outbox.json");
			writer.WriteString("type", "OrderedCollection");
			writer.WritePropertyName("orderedItems");
			writer.WriteStartArray();
			var (posts, boosts) = (0, 0);
			var located = 0;
			var before = default(string);
			while (true)
			{
				var find = DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == actor.Id && !p.IsFederatedCopy && !p.DeletedAt.HasValue);
				if (before != default)
					find = find.Match(f => f.Lt(p => p.ID, before));
				var batch = await find.Sort(p => p.ID, Order.Descending).Limit(200).ExecuteAsync(token);
				if (batch.Count == 0)
					break;
				before = batch[^1].ID;
				var originals = (await DB.Default.Find<PostEntity>().Match(p => batch.Select(b => b.ReblogOfPostId).Contains(p.ID)).ExecuteAsync(token))
					.ToDictionary(p => p.ID);
				foreach (var post in batch)
				{
					if (post.Visibility == PostVisibility.LocalGeo)
					{
						located++;
						continue;
					}
					if (post.Visibility == PostVisibility.Circle)
						continue;
					if (post.ReblogOfPostId != default)
					{
						if (!originals.TryGetValue(post.ReblogOfPostId, out var original) || string.IsNullOrEmpty(original.ObjectURI))
							continue;
						new JsonObject
						{
							["id"] = actor.ActivityUri($"announce-{post.ID}"),
							["type"] = "Announce",
							["actor"] = actor.Uri,
							["published"] = ActivityPubRenderer.Timestamp(post.CreationDate),
							["to"] = new JsonArray("https://www.w3.org/ns/activitystreams#Public"),
							["cc"] = new JsonArray(actor.Followers),
							["object"] = original.ObjectURI
						}.WriteTo(writer);
						boosts++;
						continue;
					}
					var group = post.GroupId == default ? default : await localActors.FindById(LocalActorKind.Group, post.GroupId, token);
					var note = ActivityPubRenderer.Note(post, actor, group, post.InReplyToURI);
					if (post.Visibility is PostVisibility.Direct or PostVisibility.FollowersOnly && (post.To.Count > 0 || post.Cc.Count > 0))
					{
						note["to"] = new JsonArray(post.To.Select(t => (JsonNode)t).ToArray());
						note["cc"] = new JsonArray(post.Cc.Select(c => (JsonNode)c).ToArray());
					}
					if (note["attachment"] is JsonArray attachments)
						foreach (var attachment in attachments.OfType<JsonObject>())
							if (attachment["url"]?.GetValue<string>() is { } url && url.StartsWith(prefix, StringComparison.Ordinal))
							{
								var relative = url[prefix.Length..];
								files.Add(relative);
								attachment["url"] = "/" + MediaFolder + relative;
							}
					var create = ActivityPubRenderer.Create(actor, note, $"create-{post.ID}");
					create["to"] = note["to"]?.DeepClone();
					create["cc"] = note["cc"]?.DeepClone();
					create.WriteTo(writer);
					posts++;
				}
				await writer.FlushAsync(token);
			}
			writer.WriteEndArray();
			writer.WriteNumber("totalItems", posts + boosts);
			writer.WriteEndObject();
			return (posts, boosts, located);
		}

		async Task PrivaPubParts(ZipArchive zip, LocalActor actor, Avatar avatar, string prefix, (int Posts, int Boosts, int Located) counts, CancellationToken token)
		{
			var filters = await DB.Default.Find<PersonaFilter>().Match(f => f.AvatarId == actor.Id).ExecuteAsync(token);
			var filtered = await UriMap(filters.SelectMany(f => f.Statuses.Select(s => s.PostId)), token);
			await Json(zip, "privapub/filters.json", filters.Select(f => new
			{
				title = f.Title,
				context = f.Context,
				expiresAt = f.ExpiresAt,
				action = f.Action,
				keywords = f.Keywords.Select(k => new { keyword = k.Keyword, wholeWord = k.WholeWord }),
				statuses = f.Statuses.Select(s => filtered.GetValueOrDefault(s.PostId)).Where(u => u != default)
			}), token);
			await Json(zip, "privapub/followed_tags.json",
				await DB.Default.Find<FollowedTag, string>().Match(t => t.AvatarId == actor.Id).Project(t => t.Name).ExecuteAsync(token), token);
			var policy = await NotificationPolicies.Of(actor.Id, token);
			await Json(zip, "privapub/notification_policy.json", new
			{
				forNotFollowing = policy.ForNotFollowing.ToString(),
				forNotFollowers = policy.ForNotFollowers.ToString(),
				forNewAccounts = policy.ForNewAccounts.ToString(),
				forPrivateMentions = policy.ForPrivateMentions.ToString(),
				forLimitedAccounts = policy.ForLimitedAccounts.ToString()
			}, token);
			await Json(zip, "privapub/pins.json",
				await Uris(DB.Default.Find<Pin, string>().Match(p => p.AvatarId == actor.Id).Project(p => p.PostId), token), token);

			var scheduled = await DB.Default.Find<ScheduledStatus>().Match(s => s.AvatarId == actor.Id).ExecuteAsync(token);
			var scheduledMedia = await MediaFiles(scheduled.SelectMany(s => s.Params.MediaIds), token);
			foreach (var file in scheduledMedia.Values)
				await CopyMedia(zip, file.Path, token);
			await Json(zip, "privapub/scheduled.json", scheduled.Select(s => new
			{
				scheduledAt = s.ScheduledAt,
				text = s.Params.Text,
				spoilerText = s.Params.SpoilerText,
				sensitive = s.Params.Sensitive,
				visibility = s.Params.Visibility,
				language = s.Params.Language,
				poll = s.Params.Poll,
				media = s.Params.MediaIds.Where(scheduledMedia.ContainsKey).Select(id => new { url = MediaFolder + scheduledMedia[id].Path, description = scheduledMedia[id].Description })
			}), token);

			var located = await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == actor.Id && !p.IsFederatedCopy && !p.DeletedAt.HasValue
				&& p.Visibility == PostVisibility.LocalGeo).ExecuteAsync(token);
			foreach (var post in located)
				foreach (var item in post.Media.Where(m => m.URL?.StartsWith(prefix, StringComparison.Ordinal) == true))
					await CopyMedia(zip, item.URL[prefix.Length..], token);
			await Json(zip, "privapub/located.json", located.Select(p => new
			{
				id = p.ObjectURI,
				published = p.CreationDate,
				title = p.Title,
				text = p.Text,
				spoilerText = p.SpoilerText,
				latitude = p.Geo?.Coordinates[1],
				longitude = p.Geo?.Coordinates[0],
				rangeKm = p.RangeKm,
				media = p.Media.Where(m => m.URL?.StartsWith(prefix, StringComparison.Ordinal) == true)
					.Select(m => new { url = MediaFolder + m.URL[prefix.Length..], description = m.Description })
			}), token);

			await Json(zip, "privapub/archive.json", new
			{
				format = Format,
				software = "privapub",
				actor = actor.Uri,
				username = avatar?.UserName ?? actor.UserName,
				exportedAt = DateTime.UtcNow,
				posts = counts.Posts,
				boosts = counts.Boosts,
				located = counts.Located,
				scheduled = scheduled.Count
			}, token);
		}

		async Task CopyMedia(ZipArchive zip, string relative, CancellationToken token)
		{
			var local = Path.Combine(media.Root, relative);
			if (File.Exists(local) && zip.GetEntry(MediaFolder + relative) == default)
				await Copy(zip, local, MediaFolder + relative, token);
		}

		static async Task<Dictionary<string, (string Path, string Description)>> MediaFiles(IEnumerable<string> ids, CancellationToken token)
		{
			var wanted = ids.Distinct().ToList();
			if (wanted.Count == 0)
				return [];
			return (await DB.Default.Find<MediaAttachment>().Match(m => wanted.Contains(m.ID) && m.TrashedAt == null).ExecuteAsync(token))
				.ToDictionary(m => m.ID, m => (m.FilePath, m.Description));
		}

		static async Task<Dictionary<string, string>> UriMap(IEnumerable<string> postIds, CancellationToken token)
		{
			var ids = postIds.Where(id => id != default).Distinct().ToList();
			if (ids.Count == 0)
				return [];
			return (await DB.Default.Find<PostEntity>().Match(p => ids.Contains(p.ID)).Project(p => new PostEntity { ID = p.ID, ObjectURI = p.ObjectURI }).ExecuteAsync(token))
				.Where(p => !string.IsNullOrEmpty(p.ObjectURI)).ToDictionary(p => p.ID, p => p.ObjectURI);
		}

		static async Task<List<string>> Uris<T>(Find<T, string> postIds, CancellationToken token) where T : IEntity
		{
			var ids = await postIds.ExecuteAsync(token);
			var map = await UriMap(ids, token);
			return ids.Where(map.ContainsKey).Select(id => map[id]).ToList();
		}

		static async Task Collection(ZipArchive zip, string name, List<string> items, CancellationToken token) =>
			await Text(zip, name, new JsonObject
			{
				["@context"] = "https://www.w3.org/ns/activitystreams",
				["id"] = name,
				["type"] = "OrderedCollection",
				["totalItems"] = items.Count,
				["orderedItems"] = new JsonArray(items.Select(i => (JsonNode)i).ToArray())
			}.ToJsonString(Options), token);

		static Task Json<T>(ZipArchive zip, string name, T value, CancellationToken token) =>
			Text(zip, name, JsonSerializer.Serialize(value, Options), token);

		static async Task Text(ZipArchive zip, string name, string text, CancellationToken token)
		{
			await using var stream = zip.CreateEntry(name, CompressionLevel.Optimal).Open();
			await stream.WriteAsync(Encoding.UTF8.GetBytes(text), token);
		}

		static async Task Copy(ZipArchive zip, string path, string name, CancellationToken token)
		{
			await using var source = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, FileOptions.SequentialScan);
			await using var target = zip.CreateEntry(name, CompressionLevel.NoCompression).Open();
			await source.CopyToAsync(target, token);
		}

		// a CSV cell, quoted when it must be
		static string Csv(string value) =>
			value.IndexOfAny([',', '"', '\n', '\r']) < 0 ? value : "\"" + value.Replace("\"", "\"\"") + "\"";

		// an account as Mastodon's CSV files name it: user@host, or its address when the handle is not known
		sealed class Handles(ILocalActorService localActors)
		{
			readonly Dictionary<string, string> _known = new(StringComparer.Ordinal);

			public async Task<string> Of(string actorUri, CancellationToken token)
			{
				if (_known.TryGetValue(actorUri, out var handle))
					return handle;
				if (await localActors.FindByUri(actorUri, token) is { } local)
					handle = local.Handle;
				else if (await DB.Default.Find<ForeignAvatar>().Match(a => a.ActorURI == actorUri).ExecuteFirstAsync(token) is { UserName.Length: > 0 } remote
					&& Uri.TryCreate(actorUri, UriKind.Absolute, out var parsed))
					handle = $"{remote.UserName}@{(string.IsNullOrEmpty(remote.Domain) ? parsed.Authority : remote.Domain)}";
				else
					handle = actorUri;
				return _known[actorUri] = handle;
			}
		}
	}
}