270 lines
8.9 KiB
C#
270 lines
8.9 KiB
C#
 
|
using System.IO.Compression;
|
||
|
|
using System.Text.Json;
|
||
|
|
using System.Text.Json.Nodes;
|
||
|
|
|
||
|
|
namespace PrivaPub.Domain.Portability
|
||
|
|
{
|
||
|
|
// An archive someone uploads, opened only if nothing in it can harm the server: no link, no path leaving it (.. or
|
||
|
|
// absolute), no name twice, at most MaxEntries files and MaxBytes in all (and what the disk has), and no file
|
||
|
|
// compressed more than MaxRatio to one. Every file is read through a stream that stops at its stated size, and the
|
||
|
|
// outbox one item at a time, none larger than MaxItemBytes, so a large archive never sits in memory.
|
||
|
|
public sealed class SafeArchive : IDisposable
|
||
|
|
{
|
||
|
|
public const int MaxEntries = 200_000;
|
||
|
|
public const long MaxBytes = 64L * 1024 * 1024 * 1024;
|
||
|
|
public const int MaxRatio = 100;
|
||
|
|
public const int MaxItemBytes = 1024 * 1024;
|
||
|
|
public const int MaxJsonBytes = 64 * 1024 * 1024;
|
||
|
|
|
||
|
|
readonly ZipArchive _zip;
|
||
|
|
readonly Dictionary<string, ZipArchiveEntry> _entries;
|
||
|
|
readonly string _root;//the folder the archive's files sit in, when it wraps them in one
|
||
|
|
|
||
|
|
SafeArchive(ZipArchive zip, Dictionary<string, ZipArchiveEntry> entries, string root)
|
||
|
|
{
|
||
|
|
_zip = zip;
|
||
|
|
_entries = entries;
|
||
|
|
_root = root;
|
||
|
|
}
|
||
|
|
|
||
|
|
public static (SafeArchive Archive, string Error) Open(string path)
|
||
|
|
{
|
||
|
|
ZipArchive zip;
|
||
|
|
try
|
||
|
|
{
|
||
|
|
zip = ZipFile.OpenRead(path);
|
||
|
|
}
|
||
|
|
catch (InvalidDataException)
|
||
|
|
{
|
||
|
|
return (default, "not a zip archive");
|
||
|
|
}
|
||
|
|
var entries = new Dictionary<string, ZipArchiveEntry>(StringComparer.Ordinal);
|
||
|
|
var total = 0L;
|
||
|
|
var free = new DriveInfo(Path.GetFullPath(path)).AvailableFreeSpace;
|
||
|
|
string Refuse(string why)
|
||
|
|
{
|
||
|
|
zip.Dispose();
|
||
|
|
return why;
|
||
|
|
}
|
||
|
|
if (zip.Entries.Count > MaxEntries)
|
||
|
|
return (default, Refuse("too many files"));
|
||
|
|
foreach (var entry in zip.Entries)
|
||
|
|
{
|
||
|
|
var name = entry.FullName;
|
||
|
|
if (name.Contains('\\') || name.StartsWith('/') || name.Contains(':') || name.Split('/').Any(part => part is ".." or "."))
|
||
|
|
return (default, Refuse($"{name}: a path that leaves the archive"));
|
||
|
|
if (((entry.ExternalAttributes >> 16) & 0xF000) == 0xA000)
|
||
|
|
return (default, Refuse($"{name}: a link"));
|
||
|
|
if (name.EndsWith('/'))
|
||
|
|
continue;
|
||
|
|
if (!entries.TryAdd(name, entry))
|
||
|
|
return (default, Refuse($"{name}: twice"));
|
||
|
|
total += entry.Length;
|
||
|
|
if (total > MaxBytes || total > free)
|
||
|
|
return (default, Refuse("larger than the server can take"));
|
||
|
|
if (entry.Length > 1024 * 1024 && entry.Length > (long)MaxRatio * Math.Max(1, entry.CompressedLength))
|
||
|
|
return (default, Refuse($"{name}: compressed more than {MaxRatio} to one"));
|
||
|
|
}
|
||
|
|
// some archivers wrap everything in one folder
|
||
|
|
var root = string.Empty;
|
||
|
|
if (!entries.ContainsKey("actor.json") && entries.Keys.FirstOrDefault(k => k.EndsWith("/actor.json", StringComparison.Ordinal)) is { } nested
|
||
|
|
&& nested.Count(c => c == '/') == 1)
|
||
|
|
root = nested[..(nested.IndexOf('/') + 1)];
|
||
|
|
return (new SafeArchive(zip, entries, root), default);
|
||
|
|
}
|
||
|
|
|
||
|
|
public bool Has(string name) => _entries.ContainsKey(_root + name);
|
||
|
|
|
||
|
|
public long Length(string name) => _entries.TryGetValue(_root + name, out var entry) ? entry.Length : -1;
|
||
|
|
|
||
|
|
/// <summary>A file's contents, never more than its stated size.</summary>
|
||
|
|
public Stream Read(string name) =>
|
||
|
|
_entries.TryGetValue(_root + name, out var entry) ? new Bounded(entry.Open(), entry.Length) : default;
|
||
|
|
|
||
|
|
/// <summary>A JSON file, or null when it is missing, too large or not JSON.</summary>
|
||
|
|
public JsonNode Json(string name)
|
||
|
|
{
|
||
|
|
if (!_entries.TryGetValue(_root + name, out var entry) || entry.Length > MaxJsonBytes)
|
||
|
|
return default;
|
||
|
|
try
|
||
|
|
{
|
||
|
|
using var stream = Read(name);
|
||
|
|
return JsonNode.Parse(stream);
|
||
|
|
}
|
||
|
|
catch (JsonException)
|
||
|
|
{
|
||
|
|
return default;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/// <summary>The lines of a text file (a CSV), none when it is missing or too large.</summary>
|
||
|
|
public List<string> Lines(string name)
|
||
|
|
{
|
||
|
|
if (!_entries.TryGetValue(_root + name, out var entry) || entry.Length > MaxJsonBytes)
|
||
|
|
return [];
|
||
|
|
using var reader = new StreamReader(Read(name));
|
||
|
|
var lines = new List<string>();
|
||
|
|
while (reader.ReadLine() is { } line)
|
||
|
|
if (line.Length > 0)
|
||
|
|
lines.Add(line);
|
||
|
|
return lines;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// <summary>outbox.json's orderedItems, one object at a time; an item larger than MaxItemBytes stops it.</summary>
|
||
|
|
public async IAsyncEnumerable<JsonObject> Outbox([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
|
||
|
|
{
|
||
|
|
await using var stream = Read("outbox.json");
|
||
|
|
if (stream == default)
|
||
|
|
yield break;
|
||
|
|
var buffer = new byte[64 * 1024];
|
||
|
|
var filled = 0;
|
||
|
|
var final = false;
|
||
|
|
var state = new JsonReaderState();
|
||
|
|
var phase = Phase.Seeking;
|
||
|
|
while (phase != Phase.Done)
|
||
|
|
{
|
||
|
|
if (!final)
|
||
|
|
{
|
||
|
|
if (filled == buffer.Length)
|
||
|
|
{
|
||
|
|
if (buffer.Length > MaxItemBytes + 64 * 1024)
|
||
|
|
throw new InvalidDataException($"an outbox item larger than {MaxItemBytes / 1024} KiB");
|
||
|
|
Array.Resize(ref buffer, buffer.Length * 2);
|
||
|
|
}
|
||
|
|
var read = await stream.ReadAsync(buffer.AsMemory(filled), token);
|
||
|
|
filled += read;
|
||
|
|
final = read == 0;
|
||
|
|
}
|
||
|
|
var (items, consumed, next, after) = Scan(buffer.AsSpan(0, filled), final, state, phase);
|
||
|
|
foreach (var item in items)
|
||
|
|
yield return item;
|
||
|
|
(state, phase) = (next, after);
|
||
|
|
Buffer.BlockCopy(buffer, consumed, buffer, 0, filled - consumed);
|
||
|
|
filled -= consumed;
|
||
|
|
if (final && consumed == 0 && items.Count == 0)
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
enum Phase { Seeking, Array, Done }
|
||
|
|
|
||
|
|
static (List<JsonObject> Items, int Consumed, JsonReaderState State, Phase Phase) Scan(ReadOnlySpan<byte> data, bool final, JsonReaderState state, Phase phase)
|
||
|
|
{
|
||
|
|
var items = new List<JsonObject>();
|
||
|
|
var reader = new Utf8JsonReader(data, final, state);
|
||
|
|
while (phase != Phase.Done)
|
||
|
|
{
|
||
|
|
var mark = reader;
|
||
|
|
if (!reader.Read())
|
||
|
|
{
|
||
|
|
reader = mark;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
if (phase == Phase.Seeking)
|
||
|
|
{
|
||
|
|
if (reader.TokenType == JsonTokenType.PropertyName && reader.CurrentDepth == 1 && reader.ValueTextEquals("orderedItems"))
|
||
|
|
{
|
||
|
|
var before = reader;
|
||
|
|
if (!reader.Read())
|
||
|
|
{
|
||
|
|
reader = mark;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
if (reader.TokenType == JsonTokenType.StartArray)
|
||
|
|
phase = Phase.Array;
|
||
|
|
else
|
||
|
|
{
|
||
|
|
reader = before;
|
||
|
|
if (!reader.TrySkip())
|
||
|
|
{
|
||
|
|
reader = mark;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray && reader.CurrentDepth >= 1 && !reader.TrySkip())
|
||
|
|
{
|
||
|
|
reader = mark;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (reader.TokenType == JsonTokenType.EndArray)
|
||
|
|
{
|
||
|
|
phase = Phase.Done;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
if (reader.TokenType != JsonTokenType.StartObject)
|
||
|
|
{
|
||
|
|
if (reader.TokenType == JsonTokenType.StartArray && !reader.TrySkip())
|
||
|
|
{
|
||
|
|
reader = mark;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
var start = (int)reader.TokenStartIndex;
|
||
|
|
if (!reader.TrySkip())
|
||
|
|
{
|
||
|
|
reader = mark;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
var length = (int)reader.BytesConsumed - start;
|
||
|
|
if (length > MaxItemBytes)
|
||
|
|
throw new InvalidDataException($"an outbox item larger than {MaxItemBytes / 1024} KiB");
|
||
|
|
if (JsonNode.Parse(data.Slice(start, length)) is JsonObject item)
|
||
|
|
items.Add(item);
|
||
|
|
}
|
||
|
|
return (items, (int)reader.BytesConsumed, reader.CurrentState, phase);
|
||
|
|
}
|
||
|
|
|
||
|
|
public void Dispose() => _zip.Dispose();
|
||
|
|
|
||
|
|
// a stream that ends at the size the archive states, whatever the compressed data says
|
||
|
|
sealed class Bounded(Stream inner, long length) : Stream
|
||
|
|
{
|
||
|
|
long _left = length;
|
||
|
|
|
||
|
|
public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
|
||
|
|
|
||
|
|
public override int Read(Span<byte> buffer)
|
||
|
|
{
|
||
|
|
if (_left <= 0)
|
||
|
|
return 0;
|
||
|
|
var read = inner.Read(buffer[..(int)Math.Min(buffer.Length, _left)]);
|
||
|
|
_left -= read;
|
||
|
|
return read;
|
||
|
|
}
|
||
|
|
|
||
|
|
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken token = default)
|
||
|
|
{
|
||
|
|
if (_left <= 0)
|
||
|
|
return 0;
|
||
|
|
var read = await inner.ReadAsync(buffer[..(int)Math.Min(buffer.Length, _left)], token);
|
||
|
|
_left -= read;
|
||
|
|
return read;
|
||
|
|
}
|
||
|
|
|
||
|
|
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken token) =>
|
||
|
|
ReadAsync(buffer.AsMemory(offset, count), token).AsTask();
|
||
|
|
|
||
|
|
public override bool CanRead => true;
|
||
|
|
public override bool CanSeek => false;
|
||
|
|
public override bool CanWrite => false;
|
||
|
|
public override long Length => length;
|
||
|
|
public override long Position { get => length - _left; set => throw new NotSupportedException(); }
|
||
|
|
public override void Flush() { }
|
||
|
|
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||
|
|
public override void SetLength(long value) => throw new NotSupportedException();
|
||
|
|
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||
|
|
|
||
|
|
protected override void Dispose(bool disposing)
|
||
|
|
{
|
||
|
|
if (disposing)
|
||
|
|
inner.Dispose();
|
||
|
|
base.Dispose(disposing);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|