SocialPub/PrivaPub/Infrastructure/Backup/ServerRestore.cs

using MongoDB.Bson;
using MongoDB.Driver;

using System.IO.Compression;

namespace PrivaPub.Infrastructure.Backup
{
	public enum RestoreOutcome
	{
		None,//nothing was asked
		Restored,
		Abandoned,//refused before anything changed: the server boots as it was
		Failed,//failed midway: the next boot tries again from the start
		GaveUp//failed MaxAttempts times: the server stays down until someone looks
	}

	// A backup restored (owner decision 2026-10-07: from the CLI or the administrator's page). Asking writes restore.json
	// and the running service stops at once (RestoreWatcher); the restore itself runs at the next boot, before migrations,
	// indexes and every hosted service, with nothing else touching the database:
	// 1. a pre-restore backup P is taken, once (a retry reuses it);
	// 2. every collection the backup holds is dropped and imported raw, with its indexes; every other one is dropped,
	//    except what a backup never holds (ServerBackup.Excluded), which stays as it is;
	// 3. media files the live directory lacks come back from the backup (or the trash); a restore deletes no file;
	// 4. ProtectiveMerge brings back from P every protective act made since the backup;
	// 5. every session ends, every server's circuit closes, and a RestoreRecord tells what happened.
	// Each attempt redoes everything, so one that dies midway converges on the next.
	public static class ServerRestore
	{
		static readonly TimeSpan LockWait = TimeSpan.FromMinutes(10);

		/// <summary>Why a backup can't be restored here: none when it can.</summary>
		public static async Task<List<string>> Check(BackupContext context, string id, CancellationToken token)
		{
			var backup = ServerBackup.Find(context.BackupsRoot, id);
			if (backup?.Manifest == default)
				return ["no such backup"];
			var manifest = backup.Manifest;
			var problems = new List<string>();
			if (manifest.Format != ArchiveManifest.CurrentFormat)
				problems.Add($"its format is {manifest.Format}, this build reads {ArchiveManifest.CurrentFormat}");
			if (!string.Equals(manifest.Host?.TrimEnd('/'), context.Host, StringComparison.OrdinalIgnoreCase))
				problems.Add($"it was made by {manifest.Host}, not {context.Host}: every id and address in it names its host");
			var code = ServerBackup.CodeMigration();
			if (manifest.MigrationNumber > code)
				problems.Add($"it was made by a newer build (migration {manifest.MigrationNumber}; this one knows {code})");
			problems.AddRange(await ServerBackup.Verify(context.BackupsRoot, id, token));
			return problems;
		}

		/// <summary>Asks for a backup to be restored at the next boot: why not, or null once asked.</summary>
		public static async Task<string> Request(BackupContext context, string id, string requestedBy, CancellationToken token)
		{
			var problems = await Check(context, id, token);
			if (problems.Count > 0)
				return string.Join("; ", problems.Take(5)) + (problems.Count > 5 ? $"; and {problems.Count - 5} more" : string.Empty);
			var waiting = RestoreMarker.Read(context.BackupsRoot);
			if (waiting != default && waiting.Attempts < RestoreMarker.MaxAttempts)
				return $"the restore of {waiting.Backup} is already {waiting.State}";
			if (await MaintenanceLock.Current(token) is { } held)
				return $"a {held.What} is running";
			new RestoreMarker { Backup = id, RequestedBy = requestedBy }.Write(context.BackupsRoot);
			return default;
		}

		/// <summary>The restore asked for, carried out (at boot, before migrations): what came of it.</summary>
		public static async Task<RestoreOutcome> ApplyPending(BackupContext context, ILogger logger, CancellationToken token)
		{
			var marker = RestoreMarker.Read(context.BackupsRoot);
			if (marker == default)
				return RestoreOutcome.None;
			if (marker.Attempts >= RestoreMarker.MaxAttempts)
			{
				logger.LogCritical("The restore of {Backup} failed {Attempts} times ({Error}). The database may be half restored; {PreRestore} holds it as it was before. Restore that backup or another (PrivaPub admin restore <id>), or delete {Marker} to boot as it is",
					marker.Backup, marker.Attempts, marker.Error, marker.PreRestore ?? "no backup", RestoreMarker.PathIn(context.BackupsRoot));
				return RestoreOutcome.GaveUp;
			}

			await using var held = await TakeLock(token);
			if (held == default)
				return await Abandon(context, marker, "a backup kept the maintenance lock for ten minutes", logger, token);

			var problems = await Check(context, marker.Backup, token);
			if (problems.Count > 0 && marker.PreRestore == default)
				return await Abandon(context, marker, string.Join("; ", problems), logger, token);

			if (marker.PreRestore == default)
			{
				var (taken, error) = await ServerBackup.CreateHeld(context, "pre-restore", dbOnly: false, token);
				if (taken == default)
					return await Abandon(context, marker, $"the pre-restore backup could not be made: {error}", logger, token);
				marker.PreRestore = taken.Id;
			}
			marker.State = "running";
			marker.Attempts++;
			marker.Error = default;
			marker.Write(context.BackupsRoot);
			logger.LogWarning("Restoring {Backup} (attempt {Attempt}); the server as it was is {PreRestore}", marker.Backup, marker.Attempts, marker.PreRestore);

			try
			{
				if (problems.Count > 0)
					throw new InvalidOperationException(string.Join("; ", problems));
				var backup = ServerBackup.Find(context.BackupsRoot, marker.Backup);
				var report = new RestoreReport();
				await Import(context, backup, report, token);
				Media(context, backup, report);
				await ProtectiveMerge.Apply(context.Database, Path.Combine(context.BackupsRoot, marker.PreRestore, "db"), report, token);
				await Record(context, new RestoreRecord
				{
					Backup = backup.Id,
					BackupCreatedAt = backup.CreatedAt,
					PreRestore = marker.PreRestore,
					RequestedBy = marker.RequestedBy,
					RequestedAt = marker.RequestedAt,
					Attempts = marker.Attempts,
					Report = report
				}, token);
				RestoreRecord.Forget();
				RestoreMarker.Clear(context.BackupsRoot);
				logger.LogWarning("Restored {Backup}: {Documents} documents in {Collections} collections, {Media} media files brought back, {Tombstoned} accounts made since deleted",
					backup.Id, report.Documents, report.Collections, report.MediaRestored, report.RootsTombstoned.Count + report.PersonasTombstoned.Count + report.GroupsTombstoned.Count);
				return RestoreOutcome.Restored;
			}
			catch (Exception ex) when (ex is not OperationCanceledException)
			{
				marker.Error = ex.Message;
				marker.Write(context.BackupsRoot);
				logger.LogError(ex, "The restore of {Backup} failed (attempt {Attempt} of {Max})", marker.Backup, marker.Attempts, RestoreMarker.MaxAttempts);
				return marker.Attempts >= RestoreMarker.MaxAttempts ? RestoreOutcome.GaveUp : RestoreOutcome.Failed;
			}
		}

		static async Task<MaintenanceLock.Held> TakeLock(CancellationToken token)
		{
			var until = DateTime.UtcNow + LockWait;
			while (true)
			{
				var held = await MaintenanceLock.Take("restore", token);
				if (held != default || DateTime.UtcNow > until)
					return held;
				await Task.Delay(TimeSpan.FromSeconds(5), token);
			}
		}

		// refused before anything changed: recorded for the administrator's page, and the server boots as it was
		static async Task<RestoreOutcome> Abandon(BackupContext context, RestoreMarker marker, string why, ILogger logger, CancellationToken token)
		{
			logger.LogError("The restore of {Backup} was abandoned, nothing changed: {Why}", marker.Backup, why);
			await Record(context, new RestoreRecord
			{
				Backup = marker.Backup,
				RequestedBy = marker.RequestedBy,
				RequestedAt = marker.RequestedAt,
				Attempts = marker.Attempts,
				Abandoned = true,
				Error = why
			}, token);
			RestoreMarker.Clear(context.BackupsRoot);
			return RestoreOutcome.Abandoned;
		}

		// in the database restored (a backup never holds these records: they outlive what they restore)
		static async Task Record(BackupContext context, RestoreRecord record, CancellationToken token)
		{
			record.ID = ObjectId.GenerateNewId().ToString();
			await context.Database.GetCollection<RestoreRecord>(nameof(RestoreRecord)).InsertOneAsync(record, cancellationToken: token);
		}

		// each collection dropped and imported as the backup holds it, its indexes made again; the others dropped
		static async Task Import(BackupContext context, BackupInfo backup, RestoreReport report, CancellationToken token)
		{
			var database = context.Database;
			var directory = Path.Combine(context.BackupsRoot, backup.Id, "db");
			foreach (var entry in backup.Manifest.Collections)
			{
				await database.DropCollectionAsync(entry.Name, token);
				await database.CreateCollectionAsync(entry.Name, cancellationToken: token);
				report.Documents += await Insert(database.GetCollection<BsonDocument>(entry.Name), Read(Path.Combine(directory, entry.Name + ".jsonl.gz")), token);
				var indexes = entry.Indexes.Select(BsonDocument.Parse).Where(i => i.GetValue("name", "").AsString != "_id_").ToList();
				foreach (var index in indexes)
					index.Remove("ns");
				if (indexes.Count > 0)
					await database.RunCommandAsync<BsonDocument>(new BsonDocument { { "createIndexes", entry.Name }, { "indexes", new BsonArray(indexes) } }, cancellationToken: token);
				report.Collections++;
			}
			var held = backup.Manifest.Collections.Select(c => c.Name).ToHashSet(StringComparer.Ordinal);
			foreach (var name in await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token))
			{
				if (held.Contains(name) || ServerBackup.Excluded.ContainsKey(name) || name.StartsWith("system.", StringComparison.Ordinal))
					continue;
				await database.DropCollectionAsync(name, token);
				report.CollectionsDropped++;
			}
		}

		/// <summary>Documents inserted in batches, unordered and unvalidated, as they were: how many.</summary>
		public static async Task<long> Insert(IMongoCollection<BsonDocument> collection, IEnumerable<BsonDocument> documents, CancellationToken token)
		{
			var count = 0L;
			foreach (var batch in documents.Chunk(1000))
			{
				await collection.InsertManyAsync(batch, new InsertManyOptions { IsOrdered = false, BypassDocumentValidation = true }, token);
				count += batch.Length;
			}
			return count;
		}

		/// <summary>A collection's documents in a backup, one by one; none when it has no file.</summary>
		public static IEnumerable<BsonDocument> Read(string file)
		{
			if (!File.Exists(file))
				yield break;
			using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
			using var reader = new StreamReader(gzip);
			while (reader.ReadLine() is { } line)
				if (line.Length > 0)
					yield return BsonDocument.Parse(line);
		}

		// the backup's media files the live directory lacks: linked from the backup, or moved back from the trash
		static void Media(BackupContext context, BackupInfo backup, RestoreReport report)
		{
			var kept = Path.Combine(context.BackupsRoot, backup.Id, "media");
			foreach (var relative in backup.Manifest.Media.List)
			{
				var live = ServerBackup.Inside(context.MediaRoot, relative);
				if (File.Exists(live))
					continue;
				var fromBackup = ServerBackup.Inside(kept, relative);
				var fromTrash = ServerBackup.Inside(context.TrashRoot, relative);
				if (File.Exists(fromBackup))
					HardLink.LinkOrCopy(fromBackup, live);
				else if (File.Exists(fromTrash))
				{
					Directory.CreateDirectory(Path.GetDirectoryName(live)!);
					File.Move(fromTrash, live);
				}
				else
				{
					report.MediaMissing++;
					continue;
				}
				report.MediaRestored++;
			}
		}
	}
}