SocialPub/PrivaPub/Domain/Media/MediaAudit.cs

using MongoDB.Entities;

using PrivaPub.Models.Media;
using PrivaPub.Models.User;

using PostEntity = PrivaPub.Models.Post.Post;

namespace PrivaPub.Domain.Media
{
	// What the media directory holds against what the database says holds it, and, with fix, the two made to agree:
	// - a persona's current avatar or header with no row (they had none before) becomes a row of its own;
	// - media of a deleted post or of a deleted persona go to the trash, and a picture of the server's own (ServerUse) that
	//   its settings no longer name (its thumbnail) or no custom emoji holds;
	// - rows whose files are gone go to the trash;
	// - files nothing holds (deleted posts' media and replaced pictures, from before media were trashed) get a trashed row
	//   each.
	// Trashed files leave what /media/files serves at once, and the janitor deletes them after its grace. Run it without
	// fix first; with fix it must run as the user that owns the media (www-data).
	public static class MediaAudit
	{
		public sealed record Report(int Files, int Held, int Adopted, int OfDeleted, int MissingFiles, int Unheld, IReadOnlyList<string> Examples, bool Fixed);

		public static async Task<Report> Run(IMediaService media, bool fix, CancellationToken token)
		{
			var files = Served(media.Root);
			var rows = await DB.Default.Find<MediaAttachment>().Match(m => m.TrashedAt == null).ExecuteAsync(token);
			var held = rows.SelectMany(r => new[] { r.FilePath, r.PreviewPath }).Where(p => !string.IsNullOrEmpty(p)).ToHashSet();
			var examples = new List<string>();

			// pictures personas show today, kept as rows from now on
			var prefix = media.Url(string.Empty);
			var adopted = 0;
			foreach (var avatar in await DB.Default.Find<Avatar>().Match(a => !a.DeletionAt.HasValue && (a.PictureURL != null || a.ThumbnailURL != null)).ExecuteAsync(token))
				foreach (var (url, kind) in new[] { (avatar.PictureURL, "avatar"), (avatar.ThumbnailURL, "header") })
				{
					if (url?.StartsWith(prefix, StringComparison.Ordinal) != true)
						continue;
					var relative = url[prefix.Length..];
					if (held.Contains(relative) || !files.Contains(relative))
						continue;
					adopted++;
					held.Add(relative);
					if (fix)
						await DB.Default.SaveAsync(new MediaAttachment
						{
							OwnerAvatarId = avatar.ID,
							ProfileOfAvatarId = avatar.ID,
							Kind = kind,
							ContentType = "image/jpeg",
							FilePath = relative,
							Size = new FileInfo(Path.Combine(media.Root, relative)).Length,
							AttachedAt = DateTime.UtcNow
						}, token);
				}

			// held by something that is gone: a deleted post, a deleted persona
			var postIds = rows.Where(r => r.PostId != null).Select(r => r.PostId).Distinct().ToList();
			var livePosts = (await DB.Default.Find<PostEntity>().Match(p => postIds.Contains(p.ID) && !p.DeletedAt.HasValue).Project(p => p.Include(x => x.ID)).ExecuteAsync(token))
				.Select(p => p.ID).ToHashSet();
			var ownerIds = rows.Select(r => r.OwnerAvatarId).Where(id => id != null).Distinct().ToList();
			var liveOwners = (await DB.Default.Find<Avatar>().Match(a => ownerIds.Contains(a.ID) && !a.DeletionAt.HasValue).Project(a => a.Include(x => x.ID)).ExecuteAsync(token))
				.Select(a => a.ID).ToHashSet();
			// (the server's own pictures belong to nobody: they are held while the settings name them, or a custom emoji that is
			// not deleted, disabled ones too)
			var settings = await DB.Default.Find<Models.ServerSettings>().Match(x => x.ID == Models.ServerSettings.Name).ExecuteFirstAsync(token);
			var serverHeld = new HashSet<string> { settings?.ThumbnailMediaId };
			serverHeld.UnionWith((await DB.Default.Find<Models.Post.LocalEmoji>().Match(e => e.DeletedAt == null).Project(e => e.Include(x => x.MediaId))
				.ExecuteAsync(token)).Select(e => e.MediaId).Where(id => id != null));
			var ofDeleted = rows.Where(r => r.ServerUse != null ? !serverHeld.Contains(r.ID)
				: (r.PostId != null && !livePosts.Contains(r.PostId)) || (r.OwnerAvatarId != null && !liveOwners.Contains(r.OwnerAvatarId))).ToList();
			var missing = rows.Except(ofDeleted).Where(r => !string.IsNullOrEmpty(r.FilePath) && !files.Contains(r.FilePath)).ToList();
			if (fix)
				foreach (var row in ofDeleted.Concat(missing))
					await media.Trash(m => m.ID == row.ID, ofDeleted.Contains(row) ? "audit: its holder is deleted" : "audit: its file is missing", token);
			foreach (var row in ofDeleted)
				held.Remove(row.FilePath);

			// files nothing holds
			var unheld = files.Where(f => !held.Contains(f)).OrderBy(f => f, StringComparer.Ordinal).ToList();
			examples.AddRange(unheld.Take(10));
			if (fix)
				foreach (var relative in unheld)
				{
					var orphan = new MediaAttachment
					{
						Kind = "orphan",
						FilePath = relative,
						Size = new FileInfo(Path.Combine(media.Root, relative)).Length,
						TrashedAt = DateTime.UtcNow,
						TrashReason = "audit: nothing holds it"
					};
					await DB.Default.SaveAsync(orphan, token);
					media.Hide(orphan);
				}

			return new Report(files.Count, rows.Count, adopted, ofDeleted.Count, missing.Count, unheld.Count, examples, fix);
		}

		// every file /media/files serves: everything under the root
		static HashSet<string> Served(string root)
		{
			if (!Directory.Exists(root))
				return new HashSet<string>();
			return Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)
				.Select(f => Path.GetRelativePath(root, f).Replace('\\', '/'))
				.ToHashSet();
		}
	}
}