thepraandClaude Opus 5.5
d08fbabaa5
LocalEmoji, a new collection (unique shortcode whatever its case, never renamed: the shortcode is its id elsewhere),
managed on /clientapi/admin/emoji/list|insert|update|delete and kept in the staff's log (emoji.insert, .update,
.delete). Its picture goes through IMediaService.EmojiImage: PNG, GIF, WebP or JPEG up to 256 KB, a PNG within 256 px
with alpha kept (an animated GIF or WebP stays an animated GIF, with a still PNG), keep=none, shrunk until it fits
GoToSocial's 100 KiB, held by ServerUse "emoji:<shortcode>" so the janitor and the audit leave it. LocalEmojis keeps the
offered ones in memory, reloaded on every change: GET /api/v1/custom_emojis is real; local posts (words, title, warning,
poll options, each revision) and profiles keep the emoji they name; Notes, Questions, actors and EmojiReacts carry
Mastodon's Emoji tag with its id, served as an Emoji at /emojis/{shortcode} (410 once deleted; the path awaits the
owner's confirmation); a persona reacts with one (Pleroma's reactions, announcements). A restore keeps the live emoji
as it keeps the settings, and trashes server pictures nothing holds. tools/pasture/scenarios/emoji.sh joins batch A.
Verified: built, and run locally against a scratch database (uploads of every type, animation kept, metadata gone,
refusals, rendering, reactions, history, announcements, delete and revive, backup and restore, media audit). The
pasture scenario and the sweep were not run.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WpVCoDeM1qUTRp4fXQdfof
711 lines
33 KiB
C#
711 lines
33 KiB
C#
using FFMpegCore;
|
||
|
||
using Microsoft.Extensions.Options;
|
||
|
||
using MongoDB.Entities;
|
||
|
||
using NetVips;
|
||
|
||
using PrivaPub.Federation.Actors;
|
||
using PrivaPub.Models.Media;
|
||
|
||
using System.Globalization;
|
||
using System.Security.Cryptography;
|
||
|
||
namespace PrivaPub.Domain.Media
|
||
{
|
||
public sealed record MediaOutcome(MediaAttachment Attachment, int Status = StatusCodes.Status200OK, string Error = default)
|
||
{
|
||
public bool Ok => Error == default;
|
||
public static MediaOutcome Fail(int status, string error) => new(default, status, error);
|
||
}
|
||
|
||
public sealed record ProcessedImage(byte[] Bytes, string Extension, string ContentType, int Width, int Height, byte[] Preview, string Blurhash);
|
||
|
||
// what libvips makes of an upload's header alone, before anything is decoded
|
||
public sealed record ImageHeader(string Loader, int Width, int PageHeight, int Pages)
|
||
{
|
||
public bool Animated => Pages > 1 && Loader?.StartsWith("gifload", StringComparison.Ordinal) == true;
|
||
public long Pixels => (long)Width * PageHeight * (Animated ? Pages : 1);
|
||
}
|
||
|
||
public interface IMediaService
|
||
{
|
||
string Root { get; }
|
||
string ProxyRoot { get; }
|
||
string Url(string relativePath);
|
||
/// <summary>Where trashed files wait for the janitor: the sibling of Root, on the same disk (a move is a rename) and
|
||
/// outside what /media/files serves. (Not a dot-prefixed folder inside Root: the file provider only hides a file whose
|
||
/// own name starts with a dot.)</summary>
|
||
string TrashRoot { get; }
|
||
/// <summary>Audio and video waiting for their processing: beside the media root, never served.</summary>
|
||
string IncomingRoot { get; }
|
||
/// <summary>An upload; with <paramref name="later"/>, audio and video are only stored, "pending" for ProcessMedia.</summary>
|
||
Task<MediaOutcome> Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token);
|
||
/// <summary>Processes a pending upload's audio or video; false when it can't be (the row then says why).</summary>
|
||
Task<bool> ProcessPending(MediaAttachment pending, CancellationToken token);
|
||
/// <summary>A persona's new avatar or header ("avatar" or "header"), cropped to its size, as a row of its own.</summary>
|
||
Task<MediaOutcome> ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token);
|
||
/// <summary>A picture of the server's own (its thumbnail, "thumbnail"), fitted within 1200×630, as a row nobody owns,
|
||
/// held by its <see cref="MediaAttachment.ServerUse"/>.</summary>
|
||
Task<MediaOutcome> ServerImage(string use, IFormFile file, CancellationToken token);
|
||
/// <summary>A custom emoji's picture (R7): a PNG at most 256 px on its longer side, alpha kept, or an animated GIF
|
||
/// that stays one, small enough for every peer (EmojiMaxBytes); a row nobody owns, held by ServerUse
|
||
/// "emoji:<shortcode>".</summary>
|
||
Task<MediaOutcome> EmojiImage(string shortcode, IFormFile file, CancellationToken token);
|
||
/// <summary>Trashes the media <paramref name="which"/> matches (and that isn't trashed yet): each row is marked in one
|
||
/// conditional update, so a row attached meanwhile is left alone, and its files stop being served at once.</summary>
|
||
Task<long> Trash(System.Linq.Expressions.Expression<Func<MediaAttachment, bool>> which, string reason, CancellationToken token);
|
||
/// <summary>Moves a trashed row's files out of what is served, if they are still there (a crash between the mark and the move).</summary>
|
||
void Hide(MediaAttachment trashed);
|
||
/// <summary>Deletes a trashed row's files and the row.</summary>
|
||
Task Purge(MediaAttachment trashed, CancellationToken token);
|
||
}
|
||
|
||
public class MediaService : IMediaService
|
||
{
|
||
// HEIC and HEIF are not among them: the libvips bundled here decodes AVIF but has no HEVC decoder
|
||
static readonly string[] ImageTypes = { "image/jpeg", "image/png", "image/gif", "image/webp", "image/avif" };
|
||
// the only loaders an upload ever reaches: everything else libvips could read (SVG, PDF, TIFF, ImageMagick, ...) is
|
||
// blocked, whatever type the upload claims
|
||
static readonly string[] Loaders = { "VipsForeignLoadJpeg", "VipsForeignLoadPng", "VipsForeignLoadNsgif", "VipsForeignLoadWebp", "VipsForeignLoadHeif" };
|
||
static readonly Dictionary<string, string> AvTypes = new()
|
||
{
|
||
["video/mp4"] = "mp4", ["video/quicktime"] = "mp4", ["video/webm"] = "webm",
|
||
["audio/mpeg"] = "mp3", ["audio/mp3"] = "mp3", ["audio/ogg"] = "ogg", ["audio/wav"] = "wav", ["audio/x-wav"] = "wav",
|
||
["audio/mp4"] = "m4a", ["audio/x-m4a"] = "m4a", ["audio/flac"] = "flac", ["audio/webm"] = "webm"
|
||
};
|
||
|
||
// what /media/files says each file is: ASP.NET's map, which has no entry for FLAC
|
||
public static Microsoft.AspNetCore.StaticFiles.FileExtensionContentTypeProvider ContentTypes { get; } = new()
|
||
{
|
||
Mappings = { [".flac"] = "audio/flac" }
|
||
};
|
||
|
||
// what an upload may be, as the instance API advertises it
|
||
public static IReadOnlyList<string> SupportedTypes => ImageTypes.Concat(AvTypes.Keys).ToList();
|
||
|
||
static MediaService()
|
||
{
|
||
NetVips.NetVips.BlockUntrusted = true;
|
||
Operation.Block("VipsForeignLoad", true);
|
||
foreach (var loader in Loaders)
|
||
Operation.Block(loader, false);
|
||
Cache.Max = 0;//an upload is decoded once: nothing to keep between operations
|
||
}
|
||
|
||
readonly IOptionsMonitor<MediaOptions> _options;
|
||
readonly ILocalActorService _localActors;
|
||
readonly SemaphoreSlim _processing;
|
||
readonly IWebHostEnvironment _environment;
|
||
readonly ILogger<MediaService> _logger;
|
||
|
||
public MediaService(IOptionsMonitor<MediaOptions> options, ILocalActorService localActors, IWebHostEnvironment environment, ILogger<MediaService> logger)
|
||
{
|
||
_options = options;
|
||
_localActors = localActors;
|
||
_processing = new SemaphoreSlim(Math.Max(1, options.CurrentValue.Concurrency));
|
||
_environment = environment;
|
||
_logger = logger;
|
||
}
|
||
|
||
public string Root => Path.GetFullPath(_options.CurrentValue.Root ?? Path.Combine(_environment.ContentRootPath, "media-store"));
|
||
|
||
public string ProxyRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-proxy";
|
||
|
||
public string TrashRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-trash";
|
||
|
||
public string IncomingRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-incoming";
|
||
|
||
string Incoming(string id) => Path.Combine(IncomingRoot, id + ".in");
|
||
|
||
public string Url(string relativePath) => relativePath == default ? default : $"{_localActors.BaseAddress}/media/files/{relativePath.Replace('\\', '/')}";
|
||
|
||
public async Task<MediaOutcome> Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token)
|
||
{
|
||
if (file == default || file.Length == 0)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank");
|
||
var options = _options.CurrentValue;
|
||
var contentType = file.ContentType?.Split(';')[0].Trim().ToLowerInvariant();
|
||
if (await OverQuota(owner.Id, file.Length, token) is { } full)
|
||
return full;
|
||
|
||
var attachment = new MediaAttachment
|
||
{
|
||
OwnerAvatarId = owner.Id,
|
||
Description = Clean(description, 1500),
|
||
Focus = FocalPoint.Parse(focus)
|
||
};
|
||
if (ImageTypes.Contains(contentType))
|
||
{
|
||
if (file.Length > options.MaxImageBytes)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big");
|
||
var bytes = await Read(file, token);
|
||
if (Refusal(bytes, options) is { } refused)
|
||
return refused;
|
||
await _processing.WaitAsync(token);
|
||
try
|
||
{
|
||
var header = Inspect(bytes);
|
||
if (header.Animated)
|
||
{
|
||
var animated = await ProcessAnimation(bytes, header, attachment, token);
|
||
if (!animated.Ok)
|
||
return animated;
|
||
}
|
||
else
|
||
{
|
||
var processed = ProcessImage(bytes, options.MaxImageSide, options.PreviewSide);
|
||
attachment.Kind = "image";
|
||
attachment.ContentType = processed.ContentType;
|
||
attachment.FilePath = await Save(processed.Bytes, processed.Extension, token);
|
||
attachment.PreviewPath = await Save(processed.Preview, "jpg", token);
|
||
attachment.Width = processed.Width;
|
||
attachment.Height = processed.Height;
|
||
attachment.Blurhash = processed.Blurhash;
|
||
attachment.Size = processed.Bytes.Length;
|
||
}
|
||
}
|
||
catch (VipsException ex)
|
||
{
|
||
_logger.LogInformation("Refused an image upload: {Error}", ex.Message);
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
|
||
}
|
||
finally
|
||
{
|
||
_processing.Release();
|
||
}
|
||
}
|
||
else if (contentType != default && AvTypes.ContainsKey(contentType))
|
||
{
|
||
if (file.Length > options.MaxVideoBytes)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big");
|
||
// stored as it came, outside what is served, then processed: by a job when asked later (v2), here otherwise (v1)
|
||
attachment.ID = (string)attachment.GenerateNewID();
|
||
attachment.Kind = contentType.StartsWith("video/", StringComparison.Ordinal) ? "video" : "audio";
|
||
attachment.ContentType = contentType;
|
||
Directory.CreateDirectory(IncomingRoot);
|
||
await using (var stored = File.Create(Incoming(attachment.ID)))
|
||
await file.CopyToAsync(stored, token);
|
||
if (later)
|
||
attachment.ProcessingState = "pending";
|
||
else
|
||
{
|
||
var outcome = await ProcessIncoming(attachment, token);
|
||
if (!outcome.Ok)
|
||
return outcome;
|
||
}
|
||
}
|
||
else
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported");
|
||
|
||
await DB.Default.SaveAsync(attachment, token);
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
|
||
public async Task<MediaOutcome> ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token)
|
||
{
|
||
var contentType = file?.ContentType?.Split(';')[0].Trim().ToLowerInvariant();
|
||
if (file == default || file.Length == 0)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank");
|
||
if (!ImageTypes.Contains(contentType))
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported");
|
||
if (file.Length > _options.CurrentValue.MaxImageBytes)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big");
|
||
if (await OverQuota(avatarId, file.Length, token) is { } full)
|
||
return full;
|
||
var input = await Read(file, token);
|
||
if (Refusal(input, _options.CurrentValue) is { } refused)
|
||
return refused;
|
||
byte[] bytes;
|
||
int outWidth, outHeight;
|
||
await _processing.WaitAsync(token);
|
||
try
|
||
{
|
||
using var image = Image.ThumbnailBuffer(input, width, height: height, crop: Enums.Interesting.Centre, size: Enums.Size.Down);
|
||
using var flat = Flatten(image);
|
||
bytes = flat.WriteToBuffer(".jpg[Q=85,keep=none]");
|
||
(outWidth, outHeight) = (flat.Width, flat.Height);
|
||
}
|
||
catch (VipsException ex)
|
||
{
|
||
_logger.LogInformation("Refused a profile picture: {Error}", ex.Message);
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
|
||
}
|
||
finally
|
||
{
|
||
_processing.Release();
|
||
}
|
||
var attachment = new MediaAttachment
|
||
{
|
||
OwnerAvatarId = avatarId,
|
||
ProfileOfAvatarId = avatarId,
|
||
Kind = kind,
|
||
ContentType = "image/jpeg",
|
||
FilePath = await Save(bytes, "jpg", token),
|
||
Size = bytes.Length,
|
||
Width = outWidth,
|
||
Height = outHeight,
|
||
AttachedAt = DateTime.UtcNow
|
||
};
|
||
await DB.Default.SaveAsync(attachment, token);
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
|
||
public const int ServerImageWidth = 1200;
|
||
public const int ServerImageHeight = 630;
|
||
|
||
// as safe as a profile picture: checked before it is decoded, decoded by the loaders allowed alone, written without its
|
||
// metadata; fitted (not cropped) within the size link previews use
|
||
public async Task<MediaOutcome> ServerImage(string use, IFormFile file, CancellationToken token)
|
||
{
|
||
var contentType = file?.ContentType?.Split(';')[0].Trim().ToLowerInvariant();
|
||
if (file == default || file.Length == 0)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank");
|
||
if (!ImageTypes.Contains(contentType))
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported");
|
||
if (file.Length > _options.CurrentValue.MaxImageBytes)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big");
|
||
var input = await Read(file, token);
|
||
if (Refusal(input, _options.CurrentValue) is { } refused)
|
||
return refused;
|
||
byte[] bytes;
|
||
int outWidth, outHeight;
|
||
await _processing.WaitAsync(token);
|
||
try
|
||
{
|
||
using var image = Image.ThumbnailBuffer(input, ServerImageWidth, height: ServerImageHeight, size: Enums.Size.Down);
|
||
using var flat = Flatten(image);
|
||
bytes = flat.WriteToBuffer(".jpg[Q=85,keep=none]");
|
||
(outWidth, outHeight) = (flat.Width, flat.Height);
|
||
}
|
||
catch (VipsException ex)
|
||
{
|
||
_logger.LogInformation("Refused a server picture: {Error}", ex.Message);
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
|
||
}
|
||
finally
|
||
{
|
||
_processing.Release();
|
||
}
|
||
var attachment = new MediaAttachment
|
||
{
|
||
Kind = "image",
|
||
ServerUse = use,
|
||
ContentType = "image/jpeg",
|
||
FilePath = await Save(bytes, "jpg", token),
|
||
Size = bytes.Length,
|
||
Width = outWidth,
|
||
Height = outHeight,
|
||
AttachedAt = DateTime.UtcNow
|
||
};
|
||
await DB.Default.SaveAsync(attachment, token);
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
|
||
// what an emoji's picture may be sent as, and how big (Mastodon's limit for one, 256 KB)
|
||
static readonly string[] EmojiTypes = { "image/png", "image/gif", "image/webp", "image/jpeg" };
|
||
public const long EmojiMaxUploadBytes = 256 * 1024;
|
||
// what it may be once made: GoToSocial's default for another server's emoji (media-emoji-remote-max-size, 100 KiB) is
|
||
// the smallest any peer takes; Mastodon's is 256 KB
|
||
public const long EmojiMaxBytes = 100 * 1024;
|
||
public const int EmojiSide = 256;
|
||
// tried in turn until the picture fits EmojiMaxBytes: 256 px as it is, then with a palette, then fewer pixels
|
||
static readonly (int Side, bool Palette)[] EmojiAttempts = { (256, false), (256, true), (192, true), (128, true), (96, true), (64, true) };
|
||
|
||
// As safe as any upload (checked before it is decoded, the allowed loaders alone, written without its metadata).
|
||
// A still picture becomes a PNG, its transparency kept; an animated GIF or WebP becomes an animated GIF, since a PNG
|
||
// would lose the animation and GIF is the animated picture every peer shows as an emoji, with a still PNG of its first
|
||
// frame for static_url.
|
||
public async Task<MediaOutcome> EmojiImage(string shortcode, IFormFile file, CancellationToken token)
|
||
{
|
||
var contentType = file?.ContentType?.Split(';')[0].Trim().ToLowerInvariant();
|
||
if (file == default || file.Length == 0)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: Image can't be blank");
|
||
if (!EmojiTypes.Contains(contentType))
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: An emoji's image is a PNG, GIF, WebP or JPEG");
|
||
if (file.Length > EmojiMaxUploadBytes)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: An emoji's image is at most 256 KB");
|
||
var input = await Read(file, token);
|
||
if (Refusal(input, _options.CurrentValue) is { } refused)
|
||
return refused;
|
||
byte[] bytes, still = default;
|
||
int width, height;
|
||
bool animated;
|
||
await _processing.WaitAsync(token);
|
||
try
|
||
{
|
||
var header = Inspect(input);
|
||
animated = header.Pages > 1 && (header.Loader?.StartsWith("gifload", StringComparison.Ordinal) == true
|
||
|| header.Loader?.StartsWith("webpload", StringComparison.Ordinal) == true);
|
||
(bytes, still, width, height) = animated ? EmojiAnimation(input) : EmojiStill(input);
|
||
}
|
||
catch (VipsException ex)
|
||
{
|
||
_logger.LogInformation("Refused an emoji's picture: {Error}", ex.Message);
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
|
||
}
|
||
finally
|
||
{
|
||
_processing.Release();
|
||
}
|
||
if (bytes == default)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The image can't be made small enough for an emoji (100 KB)");
|
||
var attachment = new MediaAttachment
|
||
{
|
||
Kind = "emoji",
|
||
ServerUse = Content.LocalEmojis.ServerUse(shortcode),
|
||
ContentType = animated ? "image/gif" : "image/png",
|
||
FilePath = await Save(bytes, animated ? "gif" : "png", token),
|
||
PreviewPath = still == default ? default : await Save(still, "png", token),
|
||
Size = bytes.Length,
|
||
Width = width,
|
||
Height = height,
|
||
AttachedAt = DateTime.UtcNow
|
||
};
|
||
await DB.Default.SaveAsync(attachment, token);
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
|
||
static (byte[] Bytes, byte[] Still, int Width, int Height) EmojiStill(byte[] input)
|
||
{
|
||
foreach (var (side, palette) in EmojiAttempts)
|
||
{
|
||
using var image = Image.ThumbnailBuffer(input, side, height: side, size: Enums.Size.Down);
|
||
using var pixels = EmojiPixels(image);
|
||
var bytes = pixels.WriteToBuffer(palette ? ".png[keep=none,compression=9,palette=true]" : ".png[keep=none,compression=9]");
|
||
if (bytes.Length <= EmojiMaxBytes)
|
||
return (bytes, default, pixels.Width, pixels.Height);
|
||
}
|
||
return default;
|
||
}
|
||
|
||
// every frame shrunk to the side (thumbnail with n=-1 resizes each page), written as a GIF; its first frame as a PNG
|
||
static (byte[] Bytes, byte[] Still, int Width, int Height) EmojiAnimation(byte[] input)
|
||
{
|
||
foreach (var side in EmojiAttempts.Select(a => a.Side).Distinct())
|
||
{
|
||
using var frames = Image.ThumbnailBuffer(input, side, height: side, optionString: "n=-1", size: Enums.Size.Down);
|
||
using var pixels = EmojiPixels(frames);
|
||
var bytes = pixels.WriteToBuffer(".gif[keep=none]");
|
||
if (bytes.Length > EmojiMaxBytes)
|
||
continue;
|
||
using var first = Image.ThumbnailBuffer(input, side, height: side, size: Enums.Size.Down);
|
||
using var still = EmojiPixels(first);
|
||
var height = pixels.Contains("page-height") ? (int)pixels.Get("page-height") : pixels.Height;
|
||
return (bytes, still.WriteToBuffer(".png[keep=none,compression=9,palette=true]"), pixels.Width, height);
|
||
}
|
||
return default;
|
||
}
|
||
|
||
// 8-bit sRGB, its alpha kept (thumbnail already brought its colours into sRGB; a 16-bit or grey picture is made so)
|
||
static Image EmojiPixels(Image image)
|
||
{
|
||
using var srgb = image.Interpretation == Enums.Interpretation.Srgb ? image.Copy() : image.Colourspace(Enums.Interpretation.Srgb);
|
||
return srgb.Format == Enums.BandFormat.Uchar ? srgb.Copy() : srgb.Cast(Enums.BandFormat.Uchar);
|
||
}
|
||
|
||
// a login over its quota (Media:QuotaBytesPerRoot; none when 0) uploads nothing more
|
||
async Task<MediaOutcome> OverQuota(string avatarId, long incoming, CancellationToken token)
|
||
{
|
||
var quota = _options.CurrentValue.QuotaBytesPerRoot;
|
||
if (quota <= 0)
|
||
return default;
|
||
var rootId = (await DB.Default.Find<Models.User.RootToAvatar>().Match(r => r.AvatarId == avatarId).ExecuteFirstAsync(token))?.RootId;
|
||
if (rootId == default)
|
||
return default;
|
||
var (bytes, _) = await Privacy.Counted.MediaOfRoot(rootId, token);
|
||
return bytes + incoming > quota ? MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: Your storage is full") : default;
|
||
}
|
||
|
||
public async Task<long> Trash(System.Linq.Expressions.Expression<Func<MediaAttachment, bool>> which, string reason, CancellationToken token)
|
||
{
|
||
var trashed = 0L;
|
||
foreach (var candidate in await DB.Default.Find<MediaAttachment>().Match(which).Match(m => m.TrashedAt == null).ExecuteAsync(token))
|
||
{
|
||
var marked = await DB.Default.Update<MediaAttachment>()
|
||
.Match(which).Match(m => m.ID == candidate.ID && m.TrashedAt == null)
|
||
.Modify(m => m.TrashedAt, DateTime.UtcNow)
|
||
.Modify(m => m.TrashReason, reason)
|
||
.ExecuteAsync(token);
|
||
if (marked.ModifiedCount == 0)
|
||
continue;
|
||
Hide(candidate);
|
||
trashed++;
|
||
}
|
||
return trashed;
|
||
}
|
||
|
||
public void Hide(MediaAttachment trashed)
|
||
{
|
||
foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p)))
|
||
{
|
||
var served = Path.Combine(Root, relative);
|
||
if (!File.Exists(served))
|
||
continue;
|
||
var hidden = Path.Combine(TrashRoot, relative);
|
||
Directory.CreateDirectory(Path.GetDirectoryName(hidden)!);
|
||
File.Move(served, hidden, overwrite: true);
|
||
}
|
||
}
|
||
|
||
public async Task Purge(MediaAttachment trashed, CancellationToken token)
|
||
{
|
||
foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p)))
|
||
foreach (var full in new[] { Path.Combine(TrashRoot, relative), Path.Combine(Root, relative) })
|
||
if (File.Exists(full))
|
||
File.Delete(full);
|
||
if (File.Exists(Incoming(trashed.ID)))
|
||
File.Delete(Incoming(trashed.ID));
|
||
await DB.Default.DeleteAsync<MediaAttachment>(m => m.ID == trashed.ID && m.TrashedAt != null);
|
||
}
|
||
|
||
// reads only the header: which loader takes the bytes and how big they would decode (throws VipsException when none)
|
||
public static ImageHeader Inspect(byte[] input)
|
||
{
|
||
using var image = Image.NewFromBuffer(input, access: Enums.Access.Sequential);
|
||
var pages = image.Contains("n-pages") ? Math.Max(1, (int)image.Get("n-pages")) : 1;
|
||
var pageHeight = image.Contains("page-height") ? (int)image.Get("page-height") : image.Height;
|
||
return new ImageHeader(image.Contains("vips-loader") ? (string)image.Get("vips-loader") : default, image.Width, pageHeight, pages);
|
||
}
|
||
|
||
// an upload refused before anything is decoded: no loader takes it, or it would decode to too many pixels or frames
|
||
static MediaOutcome Refusal(byte[] input, MediaOptions options)
|
||
{
|
||
ImageHeader header;
|
||
try
|
||
{
|
||
header = Inspect(input);
|
||
}
|
||
catch (VipsException)
|
||
{
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
|
||
}
|
||
if (header.Pixels > options.MaxPixels || header.Pages > options.MaxFrames)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The image is too large");
|
||
return default;
|
||
}
|
||
|
||
// a still image: shrunk on load to the largest side allowed, turned by its orientation, its colours brought into sRGB
|
||
// (thumbnail does all three), then written without its metadata, with a preview and a blurhash
|
||
public static ProcessedImage ProcessImage(byte[] input, int maxSide, int previewSide)
|
||
{
|
||
// rendered into memory once (it is read three times below, and a shrunk-on-load image can be read only in order)
|
||
using var shrunk = Image.ThumbnailBuffer(input, maxSide, height: maxSide, size: Enums.Size.Down);
|
||
using var resized = shrunk.CopyMemory();
|
||
var keepsAlpha = resized.HasAlpha();
|
||
var bytes = keepsAlpha ? resized.WriteToBuffer(".png[keep=none]") : resized.WriteToBuffer(".jpg[Q=88,keep=none]");
|
||
var (extension, contentType) = keepsAlpha ? ("png", "image/png") : ("jpg", "image/jpeg");
|
||
|
||
using var preview = resized.ThumbnailImage(previewSide, height: previewSide, size: Enums.Size.Down);
|
||
using var previewFlat = Flatten(preview);
|
||
var previewBytes = previewFlat.WriteToBuffer(".jpg[Q=80,keep=none]");
|
||
|
||
using var tiny = resized.ThumbnailImage(32, height: 32);
|
||
using var tinyFlat = Flatten(tiny);
|
||
var pixels = tinyFlat.WriteToMemory();
|
||
return new ProcessedImage(bytes, extension, contentType, resized.Width, resized.Height, previewBytes, Blurhash.Encode(pixels, tinyFlat.Width, tinyFlat.Height));
|
||
}
|
||
|
||
// an animated GIF becomes what Mastodon makes of one: a silent, looping H.264 mp4 (a gifv) at most MaxGifSide wide,
|
||
// its first frame the poster
|
||
async Task<MediaOutcome> ProcessAnimation(byte[] input, ImageHeader header, MediaAttachment attachment, CancellationToken token)
|
||
{
|
||
var options = _options.CurrentValue;
|
||
var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}");
|
||
var gif = temp + ".gif";
|
||
var mp4 = temp + ".mp4";
|
||
try
|
||
{
|
||
await File.WriteAllBytesAsync(gif, input, token);
|
||
var width = Math.Min(header.Width, options.MaxGifSide) / 2 * 2;
|
||
var height = Math.Max(2, (int)Math.Round(header.PageHeight * (double)width / header.Width / 2) * 2);
|
||
await FFMpegArguments.FromFileInput(gif, true, o => o.ForceFormat("gif").WithCustomArgument("-protocol_whitelist file"))
|
||
.OutputToFile(mp4, true, o => o.WithCustomArgument(
|
||
$"-an -map_metadata -1 -map_chapters -1 -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p -movflags +faststart -vf scale={width}:{height}"))
|
||
.CancellableThrough(token)
|
||
.ProcessAsynchronously();
|
||
var poster = ProcessImage(input, options.PreviewSide, options.PreviewSide);
|
||
attachment.Kind = "gifv";
|
||
attachment.ContentType = "video/mp4";
|
||
attachment.Width = width;
|
||
attachment.Height = height;
|
||
attachment.Size = new FileInfo(mp4).Length;
|
||
attachment.Blurhash = poster.Blurhash;
|
||
attachment.PreviewPath = await Save(poster.Preview, "jpg", token);
|
||
attachment.FilePath = SaveFile(mp4, "mp4");
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||
{
|
||
_logger.LogInformation(ex, "Refused a GIF upload");
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed");
|
||
}
|
||
finally
|
||
{
|
||
foreach (var path in new[] { gif, mp4 })
|
||
if (File.Exists(path))
|
||
File.Delete(path);
|
||
}
|
||
}
|
||
|
||
static async Task<byte[]> Read(IFormFile file, CancellationToken token)
|
||
{
|
||
await using var stream = file.OpenReadStream();
|
||
using var buffer = new MemoryStream((int)Math.Min(file.Length, int.MaxValue));
|
||
await stream.CopyToAsync(buffer, token);
|
||
return buffer.ToArray();
|
||
}
|
||
|
||
static Image Flatten(Image image)
|
||
{
|
||
var srgb = image.Interpretation == Enums.Interpretation.Srgb ? image.Copy() : image.Colourspace(Enums.Interpretation.Srgb);
|
||
var flat = srgb.HasAlpha() ? srgb.Flatten(background: new double[] { 255, 255, 255 }) : srgb.Copy();
|
||
srgb.Dispose();
|
||
var bands = flat.Bands > 3 ? flat.ExtractBand(0, n: 3) : flat.Bands == 1 ? flat.Bandjoin(flat, flat) : flat.Copy();
|
||
flat.Dispose();
|
||
using var cast = bands;
|
||
return bands.Format == Enums.BandFormat.Uchar ? bands.Copy() : bands.Cast(Enums.BandFormat.Uchar);
|
||
}
|
||
|
||
public async Task<bool> ProcessPending(MediaAttachment pending, CancellationToken token)
|
||
{
|
||
var outcome = await ProcessIncoming(pending, token);
|
||
if (outcome.Ok)
|
||
{
|
||
await DB.Default.Update<MediaAttachment>().Match(m => m.ID == pending.ID && m.TrashedAt == null)
|
||
.Modify(m => m.Kind, pending.Kind)
|
||
.Modify(m => m.ContentType, pending.ContentType)
|
||
.Modify(m => m.FilePath, pending.FilePath)
|
||
.Modify(m => m.PreviewPath, pending.PreviewPath)
|
||
.Modify(m => m.Size, pending.Size)
|
||
.Modify(m => m.Width, pending.Width)
|
||
.Modify(m => m.Height, pending.Height)
|
||
.Modify(m => m.Blurhash, pending.Blurhash)
|
||
.Modify(m => m.DurationSeconds, pending.DurationSeconds)
|
||
.Modify(m => m.ProcessingState, null)
|
||
.ExecuteAsync(token);
|
||
return true;
|
||
}
|
||
await DB.Default.Update<MediaAttachment>().MatchID(pending.ID)
|
||
.Modify(m => m.ProcessingState, "failed").Modify(m => m.ProcessingError, outcome.Error).ExecuteAsync(token);
|
||
return false;
|
||
}
|
||
|
||
// the stored upload, made playable: probed, then remuxed without its metadata when browsers play it as it is (H.264,
|
||
// VP8, VP9 or AV1 within MaxVideoPixels and MaxFrameRate), or else transcoded to H.264 that fits; a frame becomes the
|
||
// poster. ffmpeg only ever reads the local file (no protocol but file, its format forced from the probe). Nothing is
|
||
// saved until everything succeeded, and every temporary file goes, the stored upload too.
|
||
async Task<MediaOutcome> ProcessIncoming(MediaAttachment attachment, CancellationToken token)
|
||
{
|
||
var options = _options.CurrentValue;
|
||
var input = Incoming(attachment.ID);
|
||
var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}");
|
||
var frame = temp + ".png";
|
||
var output = default(string);
|
||
await _processing.WaitAsync(token);
|
||
try
|
||
{
|
||
var probe = await FFProbe.AnalyseAsync(input, default, token, "-protocol_whitelist file");
|
||
if (probe.Duration > TimeSpan.FromSeconds(options.MaxSeconds))
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is too long");
|
||
var video = attachment.ContentType.StartsWith("video/", StringComparison.Ordinal) ? probe.PrimaryVideoStream : default;
|
||
var audio = probe.PrimaryAudioStream;
|
||
if (video == default && audio == default)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed");
|
||
var extension = AvTypes[attachment.ContentType];
|
||
var format = probe.Format.FormatName.Split(',')[0];
|
||
string arguments;
|
||
if (video == default)
|
||
arguments = "-map 0:a:0 -vn -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "m4a" ? " -movflags +faststart" : string.Empty);
|
||
else if (video.CodecName is "h264" or "vp8" or "vp9" or "av1" && (long)video.Width * video.Height <= options.MaxVideoPixels
|
||
&& video.FrameRate <= options.MaxFrameRate + 0.5)
|
||
arguments = "-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "mp4" ? " -movflags +faststart" : string.Empty);
|
||
else
|
||
{
|
||
var scale = Math.Min(1, Math.Sqrt(options.MaxVideoPixels / (double)((long)video.Width * video.Height)));
|
||
var width = Math.Max(2, (int)(video.Width * scale) / 2 * 2);
|
||
var height = Math.Max(2, (int)(video.Height * scale) / 2 * 2);
|
||
extension = "mp4";
|
||
arguments = $"-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p "
|
||
+ $"-vf scale={width}:{height} -fpsmax {options.MaxFrameRate.ToString(CultureInfo.InvariantCulture)} -c:a aac -b:a 128k -movflags +faststart";
|
||
}
|
||
output = temp + "." + extension;
|
||
await FFMpegArguments.FromFileInput(input, true, o => o.ForceFormat(format).WithCustomArgument("-protocol_whitelist file"))
|
||
.OutputToFile(output, true, o => o.WithCustomArgument(arguments))
|
||
.CancellableThrough(token)
|
||
.ProcessAsynchronously();
|
||
|
||
var made = await FFProbe.AnalyseAsync(output, default, token, "-protocol_whitelist file");
|
||
byte[] poster = default;
|
||
if (video != default)
|
||
{
|
||
attachment.Width = made.PrimaryVideoStream?.Width;
|
||
attachment.Height = made.PrimaryVideoStream?.Height;
|
||
await FFMpeg.SnapshotAsync(output, frame, captureTime: TimeSpan.FromSeconds(Math.Min(1, made.Duration.TotalSeconds / 2)));
|
||
var still = ProcessImage(await File.ReadAllBytesAsync(frame, token), 640, options.PreviewSide);
|
||
poster = still.Preview;
|
||
attachment.Blurhash = still.Blurhash;
|
||
}
|
||
attachment.Kind = video != default ? "video" : "audio";
|
||
attachment.ContentType = extension switch
|
||
{
|
||
"mp4" => "video/mp4",
|
||
"webm" => video != default ? "video/webm" : "audio/webm",
|
||
"m4a" => "audio/mp4",
|
||
"mp3" => "audio/mpeg",
|
||
"ogg" => "audio/ogg",
|
||
"wav" => "audio/wav",
|
||
"flac" => "audio/flac",
|
||
_ => attachment.ContentType
|
||
};
|
||
attachment.DurationSeconds = Math.Round(made.Duration.TotalSeconds, 2);
|
||
attachment.Size = new FileInfo(output).Length;
|
||
attachment.FilePath = SaveFile(output, extension);
|
||
if (poster != default)
|
||
attachment.PreviewPath = await Save(poster, "jpg", token);
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||
{
|
||
_logger.LogInformation(ex, "Refused an audio or video upload");
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed");
|
||
}
|
||
finally
|
||
{
|
||
_processing.Release();
|
||
foreach (var path in new[] { input, output, frame }.Where(p => p != default))
|
||
if (File.Exists(path))
|
||
File.Delete(path);
|
||
}
|
||
}
|
||
|
||
string SaveFile(string source, string extension)
|
||
{
|
||
var (relative, full) = NewPath(extension);
|
||
File.Move(source, full);
|
||
return relative;
|
||
}
|
||
|
||
(string Relative, string Full) NewPath(string extension)
|
||
{
|
||
var now = DateTime.UtcNow;
|
||
var relative = Path.Combine(now.ToString("yyyy", CultureInfo.InvariantCulture), now.ToString("MM", CultureInfo.InvariantCulture),
|
||
$"{Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16))}.{extension}");
|
||
var full = Path.Combine(Root, relative);
|
||
Directory.CreateDirectory(Path.GetDirectoryName(full)!);
|
||
return (relative.Replace('\\', '/'), full);
|
||
}
|
||
|
||
async Task<string> Save(byte[] bytes, string extension, CancellationToken token)
|
||
{
|
||
var (relative, full) = NewPath(extension);
|
||
await File.WriteAllBytesAsync(full, bytes, token);
|
||
return relative;
|
||
}
|
||
|
||
static string Clean(string value, int max) =>
|
||
string.IsNullOrWhiteSpace(value) ? default : value.Trim().Length <= max ? value.Trim() : value.Trim()[..max];
|
||
}
|
||
}
|