using collAnon.Client.Services;
using System.Text;
using decePubClient.Extensions;
using decePubClient.Models;
using decePubClient.Models.Mastodon;
using decePubClient.Models.Types;
using PrivaPub.ClientModels;
namespace decePubClient.Services
{
/// What a persona carries from one server to another, each in the CSV file Mastodon exports and imports.
public enum PortableData
{
Follows = 1,
Lists,
Blocks,
Mutes,
ServerBlocks,
Bookmarks
}
/// How an import went: how many rows were done, and the rows that could not be, with why.
public sealed class ImportReport
{
public int Total { get; set; }
public int Done { get; set; }
public List Failed { get; set; } = [];
}
///
/// The persona's follows, lists, blocks, mutes, blocked servers and bookmarks as Mastodon's CSV exports, written in
/// the browser from the Mastodon API, and the same files imported: each row resolved through PrivaPub (a remote
/// account or post is fetched by PrivaPub, never by the browser) and acted on one at a time. Nothing throws; an import
/// reports the rows it could not do.
///
public class PortabilityService(AccountService accounts, ListService lists, StatusService statuses, TimelineService timelines,
InstanceService instances, CoalescingStringLocalizer localizer)
{
/// The most rows read from an export or an import file.
public const int MaxRows = 5000;
const string AccountHeader = "Account address";
public static string FileName(PortableData data) => data switch
{
PortableData.Follows => "following_accounts.csv",
PortableData.Lists => "lists.csv",
PortableData.Blocks => "blocked_accounts.csv",
PortableData.Mutes => "muted_accounts.csv",
PortableData.ServerBlocks => "blocked_domains.csv",
_ => "bookmarks.csv"
};
/// The file Mastodon would export, as UTF-8 bytes.
// a page that could not be read stops the export, which says so, instead of a file silently missing the rest
sealed class Stop
{
public WebResult Failed { get; set; }
}
/// The file, as Data (bytes); invalid when a page of it could not be read.
public async Task Export(PortableData data, string personaId, CancellationToken cancellationToken = default)
{
var stop = new Stop();
var rows = data switch
{
PortableData.Follows => await Follows(personaId, stop, cancellationToken),
PortableData.Lists => await Lists(cancellationToken),
PortableData.Blocks => [.. (await Pages(accounts.Blocks, stop, cancellationToken)).Select(a => Csv(Full(a.Acct)))],
PortableData.Mutes => await Mutes(stop, cancellationToken),
PortableData.ServerBlocks => [.. (await accounts.DomainBlocks(cancellationToken)).Select(d => Csv(d))],
_ => await Bookmarks(personaId, stop, cancellationToken)
};
if (stop.Failed is { } failed)
return failed;
return new WebResult { Data = Encoding.UTF8.GetBytes(string.Concat(rows.Select(r => r + "\n"))) };
}
async Task> Follows(string personaId, Stop stop, CancellationToken cancellationToken)
{
var followed = await Pages((maxId, token) => accounts.Following(personaId, maxId, token), stop, cancellationToken);
var relationships = (await accounts.Relationships([.. followed.Select(a => a.Id)], cancellationToken)).ToDictionary(r => r.Id);
return
[
Csv(AccountHeader, "Show boosts", "Notify on new posts", "Languages"),
.. followed.Select(a => relationships.GetValueOrDefault(a.Id) is { } r
? Csv(Full(a.Acct), Flag(r.ShowingReblogs), Flag(r.Notifying), null)
: Csv(Full(a.Acct), "true", "false", null))
];
}
async Task> Mutes(Stop stop, CancellationToken cancellationToken)
{
var muted = await Pages(accounts.Mutes, stop, cancellationToken);
var relationships = (await accounts.Relationships([.. muted.Select(a => a.Id)], cancellationToken)).ToDictionary(r => r.Id);
return
[
Csv(AccountHeader, "Hide notifications"),
.. muted.Select(a => Csv(Full(a.Acct), Flag(relationships.GetValueOrDefault(a.Id)?.MutingNotifications ?? true)))
];
}
async Task> Lists(CancellationToken cancellationToken)
{
var rows = new List();
foreach (var list in await lists.All(cancellationToken))
rows.AddRange((await lists.Members(list.Id, cancellationToken)).Select(member => Csv(list.Title, Full(member.Acct))));
return rows;
}
async Task> Bookmarks(string personaId, Stop stop, CancellationToken cancellationToken)
{
var rows = new List();
var maxId = default(string);
do
{
var result = await timelines.Load(new FeedQuery(TimelineType.Bookmarks, personaId), maxId, cancellationToken: cancellationToken);
if (result.IsInvalid || result.Data is not FeedPage page)
{
stop.Failed = result.IsInvalid ? result : new WebResult().Invalidate(localizer["The server sent an invalid response."]);
return rows;
}
rows.AddRange(page.Posts.Select(p => Csv(p.Status.Uri)));
maxId = page.Posts.Count == 0 ? null : page.MaxId;
}
while (maxId is not null && rows.Count < MaxRows);
return rows;
}
// every page of an account list, up to MaxRows
async Task> Pages(Func> page, Stop stop, CancellationToken cancellationToken)
{
var all = new List();
var maxId = default(string);
do
{
var result = await page(maxId, cancellationToken);
if (result.IsInvalid || result.Data is not AccountPage accountPage)
{
stop.Failed = result.IsInvalid ? result : new WebResult().Invalidate(localizer["The server sent an invalid response."]);
return all;
}
all.AddRange(accountPage.Accounts);
maxId = accountPage.MaxId;
}
while (maxId is not null && all.Count < MaxRows);
return all;
}
/// Does what each row of an exported file says, one row at a time; progress counts the rows done.
public async Task Import(PortableData data, string csv, IProgress progress = default,
CancellationToken cancellationToken = default)
{
var rows = Parse(csv.TrimStart('\uFEFF')).Where(r => r.Count > 0 && !string.IsNullOrWhiteSpace(r[0])).ToList();
if (rows.Count > 0 && rows[0][0].Equals(AccountHeader, StringComparison.OrdinalIgnoreCase))
rows.RemoveAt(0);
var report = new ImportReport { Total = Math.Min(rows.Count, MaxRows) };
var known = data == PortableData.Lists ? await lists.All(cancellationToken) : [];
foreach (var row in rows.Take(MaxRows))
{
if (cancellationToken.IsCancellationRequested)
break;
var failure = data switch
{
PortableData.Follows => await FollowRow(row, cancellationToken),
PortableData.Lists => await ListRow(row, known, cancellationToken),
PortableData.Blocks => await AccountRow(row[0], AccountAction.Block, cancellationToken),
PortableData.Mutes => await AccountRow(row[0], AccountAction.Mute, cancellationToken),
PortableData.ServerBlocks => Failure(row[0], await accounts.BlockDomain(row[0], cancellationToken)),
_ => await BookmarkRow(row[0].Trim(), cancellationToken)
};
if (failure is null)
report.Done++;
else
report.Failed.Add(failure);
progress?.Report(report);
}
return report;
}
async Task FollowRow(List row, CancellationToken cancellationToken)
{
if (await accounts.Lookup(row[0], cancellationToken) is not { } account)
return Unknown(row[0]);
var showReblogs = row.Count < 2 || !row[1].Trim().Equals("false", StringComparison.OrdinalIgnoreCase);
return Failure(row[0], await accounts.Follow(account.Id, showReblogs, cancellationToken));
}
async Task AccountRow(string acct, AccountAction action, CancellationToken cancellationToken)
{
if (await accounts.Lookup(acct, cancellationToken) is not { } account)
return Unknown(acct);
return Failure(acct, await accounts.Act(account.Id, action, cancellationToken));
}
// a list row names the list and an account; the list is made when missing, and an account is followed first, since
// a list holds followed accounts only
async Task ListRow(List row, List known, CancellationToken cancellationToken)
{
if (row.Count < 2 || string.IsNullOrWhiteSpace(row[1]))
return string.Join(',', row);
var title = row[0].Trim();
var list = known.FirstOrDefault(l => string.Equals(l.Title, title, StringComparison.Ordinal));
if (list is null)
{
var made = await lists.Save(new MastodonList { Title = title }, cancellationToken);
if (made.IsInvalid || made.Data is not MastodonList created)
return Failure(title, made) ?? title;
known.Add(list = created);
}
if (await accounts.Lookup(row[1], cancellationToken) is not { } account)
return Unknown(row[1]);
var relationship = await accounts.Relationship(account.Id, cancellationToken);
if (relationship is not { Following: true })
{
var followed = await accounts.Follow(account.Id, true, cancellationToken);
if (followed.IsInvalid)
return Failure(row[1], followed);
}
return Failure($"{title}: {row[1]}", await lists.SetMember(list.Id, account.Id, true, cancellationToken));
}
async Task BookmarkRow(string uri, CancellationToken cancellationToken)
{
var found = await accounts.Search(uri, resolve: true, cancellationToken);
if ((found.Statuses.FirstOrDefault(s => s.Uri == uri || s.Url == uri) ?? found.Statuses.FirstOrDefault()) is not { } status)
return Unknown(uri);
return Failure(uri, await statuses.Act(status.Id, StatusAction.Bookmark, cancellationToken));
}
static string Unknown(string what) => $"{what}: ?";
static string Failure(string what, WebResult result) => result.IsInvalid ? $"{what}: {result.ErrorMessage}" : null;
// a local account's acct has no host; an export names every account in full
string Full(string acct) => acct.Contains('@') ? acct : $"{acct}@{instances.SelfHost}";
static string Flag(bool value) => value ? "true" : "false";
static string Csv(params string[] cells) => string.Join(',', cells.Select(c => c is null ? "" :
c.IndexOfAny([',', '"', '\n', '\r']) < 0 ? c : $"\"{c.Replace("\"", "\"\"")}\""));
// RFC 4180: quoted cells may hold commas, quotes (doubled) and line breaks
static IEnumerable> Parse(string csv)
{
var row = new List();
var cell = new StringBuilder();
var quoted = false;
for (var i = 0; i < csv.Length; i++)
{
var c = csv[i];
if (quoted)
{
if (c == '"' && i + 1 < csv.Length && csv[i + 1] == '"')
{
cell.Append('"');
i++;
}
else if (c == '"')
quoted = false;
else
cell.Append(c);
continue;
}
switch (c)
{
case '"':
quoted = true;
break;
case ',':
row.Add(cell.ToString());
cell.Clear();
break;
case '\r':
break;
case '\n':
row.Add(cell.ToString());
cell.Clear();
yield return row;
row = [];
break;
default:
cell.Append(c);
break;
}
}
if (cell.Length > 0 || row.Count > 0)
{
row.Add(cell.ToString());
yield return row;
}
}
}
}