using collAnon.Client.Services; using System.Text; using decePubClient.Extensions; using decePubClient.Models; using decePubClient.Models.Mastodon; using decePubClient.Models.Types; using PrivaPub.ClientModels; namespace decePubClient.Services { /// What a persona carries from one server to another, each in the CSV file Mastodon exports and imports. public enum PortableData { Follows = 1, Lists, Blocks, Mutes, ServerBlocks, Bookmarks } /// How an import went: how many rows were done, and the rows that could not be, with why. public sealed class ImportReport { public int Total { get; set; } public int Done { get; set; } public List Failed { get; set; } = []; } /// /// The persona's follows, lists, blocks, mutes, blocked servers and bookmarks as Mastodon's CSV exports, written in /// the browser from the Mastodon API, and the same files imported: each row resolved through PrivaPub (a remote /// account or post is fetched by PrivaPub, never by the browser) and acted on one at a time. Nothing throws; an import /// reports the rows it could not do. /// public class PortabilityService(AccountService accounts, ListService lists, StatusService statuses, TimelineService timelines, InstanceService instances, CoalescingStringLocalizer localizer) { /// The most rows read from an export or an import file. public const int MaxRows = 5000; const string AccountHeader = "Account address"; public static string FileName(PortableData data) => data switch { PortableData.Follows => "following_accounts.csv", PortableData.Lists => "lists.csv", PortableData.Blocks => "blocked_accounts.csv", PortableData.Mutes => "muted_accounts.csv", PortableData.ServerBlocks => "blocked_domains.csv", _ => "bookmarks.csv" }; /// The file Mastodon would export, as UTF-8 bytes. // a page that could not be read stops the export, which says so, instead of a file silently missing the rest sealed class Stop { public WebResult Failed { get; set; } } /// The file, as Data (bytes); invalid when a page of it could not be read. public async Task Export(PortableData data, string personaId, CancellationToken cancellationToken = default) { var stop = new Stop(); var rows = data switch { PortableData.Follows => await Follows(personaId, stop, cancellationToken), PortableData.Lists => await Lists(cancellationToken), PortableData.Blocks => [.. (await Pages(accounts.Blocks, stop, cancellationToken)).Select(a => Csv(Full(a.Acct)))], PortableData.Mutes => await Mutes(stop, cancellationToken), PortableData.ServerBlocks => [.. (await accounts.DomainBlocks(cancellationToken)).Select(d => Csv(d))], _ => await Bookmarks(personaId, stop, cancellationToken) }; if (stop.Failed is { } failed) return failed; return new WebResult { Data = Encoding.UTF8.GetBytes(string.Concat(rows.Select(r => r + "\n"))) }; } async Task> Follows(string personaId, Stop stop, CancellationToken cancellationToken) { var followed = await Pages((maxId, token) => accounts.Following(personaId, maxId, token), stop, cancellationToken); var relationships = (await accounts.Relationships([.. followed.Select(a => a.Id)], cancellationToken)).ToDictionary(r => r.Id); return [ Csv(AccountHeader, "Show boosts", "Notify on new posts", "Languages"), .. followed.Select(a => relationships.GetValueOrDefault(a.Id) is { } r ? Csv(Full(a.Acct), Flag(r.ShowingReblogs), Flag(r.Notifying), null) : Csv(Full(a.Acct), "true", "false", null)) ]; } async Task> Mutes(Stop stop, CancellationToken cancellationToken) { var muted = await Pages(accounts.Mutes, stop, cancellationToken); var relationships = (await accounts.Relationships([.. muted.Select(a => a.Id)], cancellationToken)).ToDictionary(r => r.Id); return [ Csv(AccountHeader, "Hide notifications"), .. muted.Select(a => Csv(Full(a.Acct), Flag(relationships.GetValueOrDefault(a.Id)?.MutingNotifications ?? true))) ]; } async Task> Lists(CancellationToken cancellationToken) { var rows = new List(); foreach (var list in await lists.All(cancellationToken)) rows.AddRange((await lists.Members(list.Id, cancellationToken)).Select(member => Csv(list.Title, Full(member.Acct)))); return rows; } async Task> Bookmarks(string personaId, Stop stop, CancellationToken cancellationToken) { var rows = new List(); var maxId = default(string); do { var result = await timelines.Load(new FeedQuery(TimelineType.Bookmarks, personaId), maxId, cancellationToken: cancellationToken); if (result.IsInvalid || result.Data is not FeedPage page) { stop.Failed = result.IsInvalid ? result : new WebResult().Invalidate(localizer["The server sent an invalid response."]); return rows; } rows.AddRange(page.Posts.Select(p => Csv(p.Status.Uri))); maxId = page.Posts.Count == 0 ? null : page.MaxId; } while (maxId is not null && rows.Count < MaxRows); return rows; } // every page of an account list, up to MaxRows async Task> Pages(Func> page, Stop stop, CancellationToken cancellationToken) { var all = new List(); var maxId = default(string); do { var result = await page(maxId, cancellationToken); if (result.IsInvalid || result.Data is not AccountPage accountPage) { stop.Failed = result.IsInvalid ? result : new WebResult().Invalidate(localizer["The server sent an invalid response."]); return all; } all.AddRange(accountPage.Accounts); maxId = accountPage.MaxId; } while (maxId is not null && all.Count < MaxRows); return all; } /// Does what each row of an exported file says, one row at a time; progress counts the rows done. public async Task Import(PortableData data, string csv, IProgress progress = default, CancellationToken cancellationToken = default) { var rows = Parse(csv.TrimStart('\uFEFF')).Where(r => r.Count > 0 && !string.IsNullOrWhiteSpace(r[0])).ToList(); if (rows.Count > 0 && rows[0][0].Equals(AccountHeader, StringComparison.OrdinalIgnoreCase)) rows.RemoveAt(0); var report = new ImportReport { Total = Math.Min(rows.Count, MaxRows) }; var known = data == PortableData.Lists ? await lists.All(cancellationToken) : []; foreach (var row in rows.Take(MaxRows)) { if (cancellationToken.IsCancellationRequested) break; var failure = data switch { PortableData.Follows => await FollowRow(row, cancellationToken), PortableData.Lists => await ListRow(row, known, cancellationToken), PortableData.Blocks => await AccountRow(row[0], AccountAction.Block, cancellationToken), PortableData.Mutes => await AccountRow(row[0], AccountAction.Mute, cancellationToken), PortableData.ServerBlocks => Failure(row[0], await accounts.BlockDomain(row[0], cancellationToken)), _ => await BookmarkRow(row[0].Trim(), cancellationToken) }; if (failure is null) report.Done++; else report.Failed.Add(failure); progress?.Report(report); } return report; } async Task FollowRow(List row, CancellationToken cancellationToken) { if (await accounts.Lookup(row[0], cancellationToken) is not { } account) return Unknown(row[0]); var showReblogs = row.Count < 2 || !row[1].Trim().Equals("false", StringComparison.OrdinalIgnoreCase); return Failure(row[0], await accounts.Follow(account.Id, showReblogs, cancellationToken)); } async Task AccountRow(string acct, AccountAction action, CancellationToken cancellationToken) { if (await accounts.Lookup(acct, cancellationToken) is not { } account) return Unknown(acct); return Failure(acct, await accounts.Act(account.Id, action, cancellationToken)); } // a list row names the list and an account; the list is made when missing, and an account is followed first, since // a list holds followed accounts only async Task ListRow(List row, List known, CancellationToken cancellationToken) { if (row.Count < 2 || string.IsNullOrWhiteSpace(row[1])) return string.Join(',', row); var title = row[0].Trim(); var list = known.FirstOrDefault(l => string.Equals(l.Title, title, StringComparison.Ordinal)); if (list is null) { var made = await lists.Save(new MastodonList { Title = title }, cancellationToken); if (made.IsInvalid || made.Data is not MastodonList created) return Failure(title, made) ?? title; known.Add(list = created); } if (await accounts.Lookup(row[1], cancellationToken) is not { } account) return Unknown(row[1]); var relationship = await accounts.Relationship(account.Id, cancellationToken); if (relationship is not { Following: true }) { var followed = await accounts.Follow(account.Id, true, cancellationToken); if (followed.IsInvalid) return Failure(row[1], followed); } return Failure($"{title}: {row[1]}", await lists.SetMember(list.Id, account.Id, true, cancellationToken)); } async Task BookmarkRow(string uri, CancellationToken cancellationToken) { var found = await accounts.Search(uri, resolve: true, cancellationToken); if ((found.Statuses.FirstOrDefault(s => s.Uri == uri || s.Url == uri) ?? found.Statuses.FirstOrDefault()) is not { } status) return Unknown(uri); return Failure(uri, await statuses.Act(status.Id, StatusAction.Bookmark, cancellationToken)); } static string Unknown(string what) => $"{what}: ?"; static string Failure(string what, WebResult result) => result.IsInvalid ? $"{what}: {result.ErrorMessage}" : null; // a local account's acct has no host; an export names every account in full string Full(string acct) => acct.Contains('@') ? acct : $"{acct}@{instances.SelfHost}"; static string Flag(bool value) => value ? "true" : "false"; static string Csv(params string[] cells) => string.Join(',', cells.Select(c => c is null ? "" : c.IndexOfAny([',', '"', '\n', '\r']) < 0 ? c : $"\"{c.Replace("\"", "\"\"")}\"")); // RFC 4180: quoted cells may hold commas, quotes (doubled) and line breaks static IEnumerable> Parse(string csv) { var row = new List(); var cell = new StringBuilder(); var quoted = false; for (var i = 0; i < csv.Length; i++) { var c = csv[i]; if (quoted) { if (c == '"' && i + 1 < csv.Length && csv[i + 1] == '"') { cell.Append('"'); i++; } else if (c == '"') quoted = false; else cell.Append(c); continue; } switch (c) { case '"': quoted = true; break; case ',': row.Add(cell.ToString()); cell.Clear(); break; case '\r': break; case '\n': row.Add(cell.ToString()); cell.Clear(); yield return row; row = []; break; default: cell.Append(c); break; } } if (cell.Length > 0 || row.Count > 0) { row.Add(cell.ToString()); yield return row; } } } }