thepraandClaude Opus 5.5
70d1ffe9a2
Build / Build (push) Successful in 1m3s
A singleton ServerSettings (title, descriptions, contact, thumbnail, privacy policy, terms, rules, relays, media overrides, PublishActivity), cached by InstanceSettings and read by the instance API, NodeInfo, the instance actor, the web pages and the mailer; its defaults are the old literals, so nothing changes until an administrator edits. /clientapi/admin/settings (and /thumbnail), /announcements, /relays for administrators; /clientapi/moderator/log, the staff's log (AdminAction, written by domain blocks, backups, settings and the CLI; no IP, never a root paired with its personas). The Mastodon API's rules, extended description, privacy policy, terms of service, domain blocks (Published, Obfuscate), announcements (dismiss, reactions, streaming) and activity (only when published). Reports carry rule ids. Media overrides clamped to hard caps. Relays named by the settings reconcile next to the configured ones. Verified: interop.sh instance accounts relay with mastodon, relay, aoderelay and mailpit, 119 passed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WpVCoDeM1qUTRp4fXQdfof
605 lines
27 KiB
C#
605 lines
27 KiB
C#
using FFMpegCore;
|
||
|
||
using Microsoft.Extensions.Options;
|
||
|
||
using MongoDB.Entities;
|
||
|
||
using NetVips;
|
||
|
||
using PrivaPub.Federation.Actors;
|
||
using PrivaPub.Models.Media;
|
||
|
||
using System.Globalization;
|
||
using System.Security.Cryptography;
|
||
|
||
namespace PrivaPub.Domain.Media
|
||
{
|
||
public sealed record MediaOutcome(MediaAttachment Attachment, int Status = StatusCodes.Status200OK, string Error = default)
|
||
{
|
||
public bool Ok => Error == default;
|
||
public static MediaOutcome Fail(int status, string error) => new(default, status, error);
|
||
}
|
||
|
||
public sealed record ProcessedImage(byte[] Bytes, string Extension, string ContentType, int Width, int Height, byte[] Preview, string Blurhash);
|
||
|
||
// what libvips makes of an upload's header alone, before anything is decoded
|
||
public sealed record ImageHeader(string Loader, int Width, int PageHeight, int Pages)
|
||
{
|
||
public bool Animated => Pages > 1 && Loader?.StartsWith("gifload", StringComparison.Ordinal) == true;
|
||
public long Pixels => (long)Width * PageHeight * (Animated ? Pages : 1);
|
||
}
|
||
|
||
public interface IMediaService
|
||
{
|
||
string Root { get; }
|
||
string ProxyRoot { get; }
|
||
string Url(string relativePath);
|
||
/// <summary>Where trashed files wait for the janitor: the sibling of Root, on the same disk (a move is a rename) and
|
||
/// outside what /media/files serves. (Not a dot-prefixed folder inside Root: the file provider only hides a file whose
|
||
/// own name starts with a dot.)</summary>
|
||
string TrashRoot { get; }
|
||
/// <summary>Audio and video waiting for their processing: beside the media root, never served.</summary>
|
||
string IncomingRoot { get; }
|
||
/// <summary>An upload; with <paramref name="later"/>, audio and video are only stored, "pending" for ProcessMedia.</summary>
|
||
Task<MediaOutcome> Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token);
|
||
/// <summary>Processes a pending upload's audio or video; false when it can't be (the row then says why).</summary>
|
||
Task<bool> ProcessPending(MediaAttachment pending, CancellationToken token);
|
||
/// <summary>A persona's new avatar or header ("avatar" or "header"), cropped to its size, as a row of its own.</summary>
|
||
Task<MediaOutcome> ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token);
|
||
/// <summary>A picture of the server's own (its thumbnail, "thumbnail"), fitted within 1200×630, as a row nobody owns,
|
||
/// held by its <see cref="MediaAttachment.ServerUse"/>.</summary>
|
||
Task<MediaOutcome> ServerImage(string use, IFormFile file, CancellationToken token);
|
||
/// <summary>Trashes the media <paramref name="which"/> matches (and that isn't trashed yet): each row is marked in one
|
||
/// conditional update, so a row attached meanwhile is left alone, and its files stop being served at once.</summary>
|
||
Task<long> Trash(System.Linq.Expressions.Expression<Func<MediaAttachment, bool>> which, string reason, CancellationToken token);
|
||
/// <summary>Moves a trashed row's files out of what is served, if they are still there (a crash between the mark and the move).</summary>
|
||
void Hide(MediaAttachment trashed);
|
||
/// <summary>Deletes a trashed row's files and the row.</summary>
|
||
Task Purge(MediaAttachment trashed, CancellationToken token);
|
||
}
|
||
|
||
public class MediaService : IMediaService
|
||
{
|
||
// HEIC and HEIF are not among them: the libvips bundled here decodes AVIF but has no HEVC decoder
|
||
static readonly string[] ImageTypes = { "image/jpeg", "image/png", "image/gif", "image/webp", "image/avif" };
|
||
// the only loaders an upload ever reaches: everything else libvips could read (SVG, PDF, TIFF, ImageMagick, ...) is
|
||
// blocked, whatever type the upload claims
|
||
static readonly string[] Loaders = { "VipsForeignLoadJpeg", "VipsForeignLoadPng", "VipsForeignLoadNsgif", "VipsForeignLoadWebp", "VipsForeignLoadHeif" };
|
||
static readonly Dictionary<string, string> AvTypes = new()
|
||
{
|
||
["video/mp4"] = "mp4", ["video/quicktime"] = "mp4", ["video/webm"] = "webm",
|
||
["audio/mpeg"] = "mp3", ["audio/mp3"] = "mp3", ["audio/ogg"] = "ogg", ["audio/wav"] = "wav", ["audio/x-wav"] = "wav",
|
||
["audio/mp4"] = "m4a", ["audio/x-m4a"] = "m4a", ["audio/flac"] = "flac", ["audio/webm"] = "webm"
|
||
};
|
||
|
||
// what /media/files says each file is: ASP.NET's map, which has no entry for FLAC
|
||
public static Microsoft.AspNetCore.StaticFiles.FileExtensionContentTypeProvider ContentTypes { get; } = new()
|
||
{
|
||
Mappings = { [".flac"] = "audio/flac" }
|
||
};
|
||
|
||
// what an upload may be, as the instance API advertises it
|
||
public static IReadOnlyList<string> SupportedTypes => ImageTypes.Concat(AvTypes.Keys).ToList();
|
||
|
||
static MediaService()
|
||
{
|
||
NetVips.NetVips.BlockUntrusted = true;
|
||
Operation.Block("VipsForeignLoad", true);
|
||
foreach (var loader in Loaders)
|
||
Operation.Block(loader, false);
|
||
Cache.Max = 0;//an upload is decoded once: nothing to keep between operations
|
||
}
|
||
|
||
readonly IOptionsMonitor<MediaOptions> _options;
|
||
readonly ILocalActorService _localActors;
|
||
readonly SemaphoreSlim _processing;
|
||
readonly IWebHostEnvironment _environment;
|
||
readonly ILogger<MediaService> _logger;
|
||
|
||
public MediaService(IOptionsMonitor<MediaOptions> options, ILocalActorService localActors, IWebHostEnvironment environment, ILogger<MediaService> logger)
|
||
{
|
||
_options = options;
|
||
_localActors = localActors;
|
||
_processing = new SemaphoreSlim(Math.Max(1, options.CurrentValue.Concurrency));
|
||
_environment = environment;
|
||
_logger = logger;
|
||
}
|
||
|
||
public string Root => Path.GetFullPath(_options.CurrentValue.Root ?? Path.Combine(_environment.ContentRootPath, "media-store"));
|
||
|
||
public string ProxyRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-proxy";
|
||
|
||
public string TrashRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-trash";
|
||
|
||
public string IncomingRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-incoming";
|
||
|
||
string Incoming(string id) => Path.Combine(IncomingRoot, id + ".in");
|
||
|
||
public string Url(string relativePath) => relativePath == default ? default : $"{_localActors.BaseAddress}/media/files/{relativePath.Replace('\\', '/')}";
|
||
|
||
public async Task<MediaOutcome> Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token)
|
||
{
|
||
if (file == default || file.Length == 0)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank");
|
||
var options = _options.CurrentValue;
|
||
var contentType = file.ContentType?.Split(';')[0].Trim().ToLowerInvariant();
|
||
if (await OverQuota(owner.Id, file.Length, token) is { } full)
|
||
return full;
|
||
|
||
var attachment = new MediaAttachment
|
||
{
|
||
OwnerAvatarId = owner.Id,
|
||
Description = Clean(description, 1500),
|
||
Focus = FocalPoint.Parse(focus)
|
||
};
|
||
if (ImageTypes.Contains(contentType))
|
||
{
|
||
if (file.Length > options.MaxImageBytes)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big");
|
||
var bytes = await Read(file, token);
|
||
if (Refusal(bytes, options) is { } refused)
|
||
return refused;
|
||
await _processing.WaitAsync(token);
|
||
try
|
||
{
|
||
var header = Inspect(bytes);
|
||
if (header.Animated)
|
||
{
|
||
var animated = await ProcessAnimation(bytes, header, attachment, token);
|
||
if (!animated.Ok)
|
||
return animated;
|
||
}
|
||
else
|
||
{
|
||
var processed = ProcessImage(bytes, options.MaxImageSide, options.PreviewSide);
|
||
attachment.Kind = "image";
|
||
attachment.ContentType = processed.ContentType;
|
||
attachment.FilePath = await Save(processed.Bytes, processed.Extension, token);
|
||
attachment.PreviewPath = await Save(processed.Preview, "jpg", token);
|
||
attachment.Width = processed.Width;
|
||
attachment.Height = processed.Height;
|
||
attachment.Blurhash = processed.Blurhash;
|
||
attachment.Size = processed.Bytes.Length;
|
||
}
|
||
}
|
||
catch (VipsException ex)
|
||
{
|
||
_logger.LogInformation("Refused an image upload: {Error}", ex.Message);
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
|
||
}
|
||
finally
|
||
{
|
||
_processing.Release();
|
||
}
|
||
}
|
||
else if (contentType != default && AvTypes.ContainsKey(contentType))
|
||
{
|
||
if (file.Length > options.MaxVideoBytes)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big");
|
||
// stored as it came, outside what is served, then processed: by a job when asked later (v2), here otherwise (v1)
|
||
attachment.ID = (string)attachment.GenerateNewID();
|
||
attachment.Kind = contentType.StartsWith("video/", StringComparison.Ordinal) ? "video" : "audio";
|
||
attachment.ContentType = contentType;
|
||
Directory.CreateDirectory(IncomingRoot);
|
||
await using (var stored = File.Create(Incoming(attachment.ID)))
|
||
await file.CopyToAsync(stored, token);
|
||
if (later)
|
||
attachment.ProcessingState = "pending";
|
||
else
|
||
{
|
||
var outcome = await ProcessIncoming(attachment, token);
|
||
if (!outcome.Ok)
|
||
return outcome;
|
||
}
|
||
}
|
||
else
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported");
|
||
|
||
await DB.Default.SaveAsync(attachment, token);
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
|
||
public async Task<MediaOutcome> ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token)
|
||
{
|
||
var contentType = file?.ContentType?.Split(';')[0].Trim().ToLowerInvariant();
|
||
if (file == default || file.Length == 0)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank");
|
||
if (!ImageTypes.Contains(contentType))
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported");
|
||
if (file.Length > _options.CurrentValue.MaxImageBytes)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big");
|
||
if (await OverQuota(avatarId, file.Length, token) is { } full)
|
||
return full;
|
||
var input = await Read(file, token);
|
||
if (Refusal(input, _options.CurrentValue) is { } refused)
|
||
return refused;
|
||
byte[] bytes;
|
||
int outWidth, outHeight;
|
||
await _processing.WaitAsync(token);
|
||
try
|
||
{
|
||
using var image = Image.ThumbnailBuffer(input, width, height: height, crop: Enums.Interesting.Centre, size: Enums.Size.Down);
|
||
using var flat = Flatten(image);
|
||
bytes = flat.WriteToBuffer(".jpg[Q=85,keep=none]");
|
||
(outWidth, outHeight) = (flat.Width, flat.Height);
|
||
}
|
||
catch (VipsException ex)
|
||
{
|
||
_logger.LogInformation("Refused a profile picture: {Error}", ex.Message);
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
|
||
}
|
||
finally
|
||
{
|
||
_processing.Release();
|
||
}
|
||
var attachment = new MediaAttachment
|
||
{
|
||
OwnerAvatarId = avatarId,
|
||
ProfileOfAvatarId = avatarId,
|
||
Kind = kind,
|
||
ContentType = "image/jpeg",
|
||
FilePath = await Save(bytes, "jpg", token),
|
||
Size = bytes.Length,
|
||
Width = outWidth,
|
||
Height = outHeight,
|
||
AttachedAt = DateTime.UtcNow
|
||
};
|
||
await DB.Default.SaveAsync(attachment, token);
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
|
||
public const int ServerImageWidth = 1200;
|
||
public const int ServerImageHeight = 630;
|
||
|
||
// as safe as a profile picture: checked before it is decoded, decoded by the loaders allowed alone, written without its
|
||
// metadata; fitted (not cropped) within the size link previews use
|
||
public async Task<MediaOutcome> ServerImage(string use, IFormFile file, CancellationToken token)
|
||
{
|
||
var contentType = file?.ContentType?.Split(';')[0].Trim().ToLowerInvariant();
|
||
if (file == default || file.Length == 0)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank");
|
||
if (!ImageTypes.Contains(contentType))
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported");
|
||
if (file.Length > _options.CurrentValue.MaxImageBytes)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big");
|
||
var input = await Read(file, token);
|
||
if (Refusal(input, _options.CurrentValue) is { } refused)
|
||
return refused;
|
||
byte[] bytes;
|
||
int outWidth, outHeight;
|
||
await _processing.WaitAsync(token);
|
||
try
|
||
{
|
||
using var image = Image.ThumbnailBuffer(input, ServerImageWidth, height: ServerImageHeight, size: Enums.Size.Down);
|
||
using var flat = Flatten(image);
|
||
bytes = flat.WriteToBuffer(".jpg[Q=85,keep=none]");
|
||
(outWidth, outHeight) = (flat.Width, flat.Height);
|
||
}
|
||
catch (VipsException ex)
|
||
{
|
||
_logger.LogInformation("Refused a server picture: {Error}", ex.Message);
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
|
||
}
|
||
finally
|
||
{
|
||
_processing.Release();
|
||
}
|
||
var attachment = new MediaAttachment
|
||
{
|
||
Kind = "image",
|
||
ServerUse = use,
|
||
ContentType = "image/jpeg",
|
||
FilePath = await Save(bytes, "jpg", token),
|
||
Size = bytes.Length,
|
||
Width = outWidth,
|
||
Height = outHeight,
|
||
AttachedAt = DateTime.UtcNow
|
||
};
|
||
await DB.Default.SaveAsync(attachment, token);
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
|
||
// a login over its quota (Media:QuotaBytesPerRoot; none when 0) uploads nothing more
|
||
async Task<MediaOutcome> OverQuota(string avatarId, long incoming, CancellationToken token)
|
||
{
|
||
var quota = _options.CurrentValue.QuotaBytesPerRoot;
|
||
if (quota <= 0)
|
||
return default;
|
||
var rootId = (await DB.Default.Find<Models.User.RootToAvatar>().Match(r => r.AvatarId == avatarId).ExecuteFirstAsync(token))?.RootId;
|
||
if (rootId == default)
|
||
return default;
|
||
var (bytes, _) = await Privacy.Counted.MediaOfRoot(rootId, token);
|
||
return bytes + incoming > quota ? MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: Your storage is full") : default;
|
||
}
|
||
|
||
public async Task<long> Trash(System.Linq.Expressions.Expression<Func<MediaAttachment, bool>> which, string reason, CancellationToken token)
|
||
{
|
||
var trashed = 0L;
|
||
foreach (var candidate in await DB.Default.Find<MediaAttachment>().Match(which).Match(m => m.TrashedAt == null).ExecuteAsync(token))
|
||
{
|
||
var marked = await DB.Default.Update<MediaAttachment>()
|
||
.Match(which).Match(m => m.ID == candidate.ID && m.TrashedAt == null)
|
||
.Modify(m => m.TrashedAt, DateTime.UtcNow)
|
||
.Modify(m => m.TrashReason, reason)
|
||
.ExecuteAsync(token);
|
||
if (marked.ModifiedCount == 0)
|
||
continue;
|
||
Hide(candidate);
|
||
trashed++;
|
||
}
|
||
return trashed;
|
||
}
|
||
|
||
public void Hide(MediaAttachment trashed)
|
||
{
|
||
foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p)))
|
||
{
|
||
var served = Path.Combine(Root, relative);
|
||
if (!File.Exists(served))
|
||
continue;
|
||
var hidden = Path.Combine(TrashRoot, relative);
|
||
Directory.CreateDirectory(Path.GetDirectoryName(hidden)!);
|
||
File.Move(served, hidden, overwrite: true);
|
||
}
|
||
}
|
||
|
||
public async Task Purge(MediaAttachment trashed, CancellationToken token)
|
||
{
|
||
foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p)))
|
||
foreach (var full in new[] { Path.Combine(TrashRoot, relative), Path.Combine(Root, relative) })
|
||
if (File.Exists(full))
|
||
File.Delete(full);
|
||
if (File.Exists(Incoming(trashed.ID)))
|
||
File.Delete(Incoming(trashed.ID));
|
||
await DB.Default.DeleteAsync<MediaAttachment>(m => m.ID == trashed.ID && m.TrashedAt != null);
|
||
}
|
||
|
||
// reads only the header: which loader takes the bytes and how big they would decode (throws VipsException when none)
|
||
public static ImageHeader Inspect(byte[] input)
|
||
{
|
||
using var image = Image.NewFromBuffer(input, access: Enums.Access.Sequential);
|
||
var pages = image.Contains("n-pages") ? Math.Max(1, (int)image.Get("n-pages")) : 1;
|
||
var pageHeight = image.Contains("page-height") ? (int)image.Get("page-height") : image.Height;
|
||
return new ImageHeader(image.Contains("vips-loader") ? (string)image.Get("vips-loader") : default, image.Width, pageHeight, pages);
|
||
}
|
||
|
||
// an upload refused before anything is decoded: no loader takes it, or it would decode to too many pixels or frames
|
||
static MediaOutcome Refusal(byte[] input, MediaOptions options)
|
||
{
|
||
ImageHeader header;
|
||
try
|
||
{
|
||
header = Inspect(input);
|
||
}
|
||
catch (VipsException)
|
||
{
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
|
||
}
|
||
if (header.Pixels > options.MaxPixels || header.Pages > options.MaxFrames)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The image is too large");
|
||
return default;
|
||
}
|
||
|
||
// a still image: shrunk on load to the largest side allowed, turned by its orientation, its colours brought into sRGB
|
||
// (thumbnail does all three), then written without its metadata, with a preview and a blurhash
|
||
public static ProcessedImage ProcessImage(byte[] input, int maxSide, int previewSide)
|
||
{
|
||
// rendered into memory once (it is read three times below, and a shrunk-on-load image can be read only in order)
|
||
using var shrunk = Image.ThumbnailBuffer(input, maxSide, height: maxSide, size: Enums.Size.Down);
|
||
using var resized = shrunk.CopyMemory();
|
||
var keepsAlpha = resized.HasAlpha();
|
||
var bytes = keepsAlpha ? resized.WriteToBuffer(".png[keep=none]") : resized.WriteToBuffer(".jpg[Q=88,keep=none]");
|
||
var (extension, contentType) = keepsAlpha ? ("png", "image/png") : ("jpg", "image/jpeg");
|
||
|
||
using var preview = resized.ThumbnailImage(previewSide, height: previewSide, size: Enums.Size.Down);
|
||
using var previewFlat = Flatten(preview);
|
||
var previewBytes = previewFlat.WriteToBuffer(".jpg[Q=80,keep=none]");
|
||
|
||
using var tiny = resized.ThumbnailImage(32, height: 32);
|
||
using var tinyFlat = Flatten(tiny);
|
||
var pixels = tinyFlat.WriteToMemory();
|
||
return new ProcessedImage(bytes, extension, contentType, resized.Width, resized.Height, previewBytes, Blurhash.Encode(pixels, tinyFlat.Width, tinyFlat.Height));
|
||
}
|
||
|
||
// an animated GIF becomes what Mastodon makes of one: a silent, looping H.264 mp4 (a gifv) at most MaxGifSide wide,
|
||
// its first frame the poster
|
||
async Task<MediaOutcome> ProcessAnimation(byte[] input, ImageHeader header, MediaAttachment attachment, CancellationToken token)
|
||
{
|
||
var options = _options.CurrentValue;
|
||
var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}");
|
||
var gif = temp + ".gif";
|
||
var mp4 = temp + ".mp4";
|
||
try
|
||
{
|
||
await File.WriteAllBytesAsync(gif, input, token);
|
||
var width = Math.Min(header.Width, options.MaxGifSide) / 2 * 2;
|
||
var height = Math.Max(2, (int)Math.Round(header.PageHeight * (double)width / header.Width / 2) * 2);
|
||
await FFMpegArguments.FromFileInput(gif, true, o => o.ForceFormat("gif").WithCustomArgument("-protocol_whitelist file"))
|
||
.OutputToFile(mp4, true, o => o.WithCustomArgument(
|
||
$"-an -map_metadata -1 -map_chapters -1 -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p -movflags +faststart -vf scale={width}:{height}"))
|
||
.CancellableThrough(token)
|
||
.ProcessAsynchronously();
|
||
var poster = ProcessImage(input, options.PreviewSide, options.PreviewSide);
|
||
attachment.Kind = "gifv";
|
||
attachment.ContentType = "video/mp4";
|
||
attachment.Width = width;
|
||
attachment.Height = height;
|
||
attachment.Size = new FileInfo(mp4).Length;
|
||
attachment.Blurhash = poster.Blurhash;
|
||
attachment.PreviewPath = await Save(poster.Preview, "jpg", token);
|
||
attachment.FilePath = SaveFile(mp4, "mp4");
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||
{
|
||
_logger.LogInformation(ex, "Refused a GIF upload");
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed");
|
||
}
|
||
finally
|
||
{
|
||
foreach (var path in new[] { gif, mp4 })
|
||
if (File.Exists(path))
|
||
File.Delete(path);
|
||
}
|
||
}
|
||
|
||
static async Task<byte[]> Read(IFormFile file, CancellationToken token)
|
||
{
|
||
await using var stream = file.OpenReadStream();
|
||
using var buffer = new MemoryStream((int)Math.Min(file.Length, int.MaxValue));
|
||
await stream.CopyToAsync(buffer, token);
|
||
return buffer.ToArray();
|
||
}
|
||
|
||
static Image Flatten(Image image)
|
||
{
|
||
var srgb = image.Interpretation == Enums.Interpretation.Srgb ? image.Copy() : image.Colourspace(Enums.Interpretation.Srgb);
|
||
var flat = srgb.HasAlpha() ? srgb.Flatten(background: new double[] { 255, 255, 255 }) : srgb.Copy();
|
||
srgb.Dispose();
|
||
var bands = flat.Bands > 3 ? flat.ExtractBand(0, n: 3) : flat.Bands == 1 ? flat.Bandjoin(flat, flat) : flat.Copy();
|
||
flat.Dispose();
|
||
using var cast = bands;
|
||
return bands.Format == Enums.BandFormat.Uchar ? bands.Copy() : bands.Cast(Enums.BandFormat.Uchar);
|
||
}
|
||
|
||
public async Task<bool> ProcessPending(MediaAttachment pending, CancellationToken token)
|
||
{
|
||
var outcome = await ProcessIncoming(pending, token);
|
||
if (outcome.Ok)
|
||
{
|
||
await DB.Default.Update<MediaAttachment>().Match(m => m.ID == pending.ID && m.TrashedAt == null)
|
||
.Modify(m => m.Kind, pending.Kind)
|
||
.Modify(m => m.ContentType, pending.ContentType)
|
||
.Modify(m => m.FilePath, pending.FilePath)
|
||
.Modify(m => m.PreviewPath, pending.PreviewPath)
|
||
.Modify(m => m.Size, pending.Size)
|
||
.Modify(m => m.Width, pending.Width)
|
||
.Modify(m => m.Height, pending.Height)
|
||
.Modify(m => m.Blurhash, pending.Blurhash)
|
||
.Modify(m => m.DurationSeconds, pending.DurationSeconds)
|
||
.Modify(m => m.ProcessingState, null)
|
||
.ExecuteAsync(token);
|
||
return true;
|
||
}
|
||
await DB.Default.Update<MediaAttachment>().MatchID(pending.ID)
|
||
.Modify(m => m.ProcessingState, "failed").Modify(m => m.ProcessingError, outcome.Error).ExecuteAsync(token);
|
||
return false;
|
||
}
|
||
|
||
// the stored upload, made playable: probed, then remuxed without its metadata when browsers play it as it is (H.264,
|
||
// VP8, VP9 or AV1 within MaxVideoPixels and MaxFrameRate), or else transcoded to H.264 that fits; a frame becomes the
|
||
// poster. ffmpeg only ever reads the local file (no protocol but file, its format forced from the probe). Nothing is
|
||
// saved until everything succeeded, and every temporary file goes, the stored upload too.
|
||
async Task<MediaOutcome> ProcessIncoming(MediaAttachment attachment, CancellationToken token)
|
||
{
|
||
var options = _options.CurrentValue;
|
||
var input = Incoming(attachment.ID);
|
||
var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}");
|
||
var frame = temp + ".png";
|
||
var output = default(string);
|
||
await _processing.WaitAsync(token);
|
||
try
|
||
{
|
||
var probe = await FFProbe.AnalyseAsync(input, default, token, "-protocol_whitelist file");
|
||
if (probe.Duration > TimeSpan.FromSeconds(options.MaxSeconds))
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is too long");
|
||
var video = attachment.ContentType.StartsWith("video/", StringComparison.Ordinal) ? probe.PrimaryVideoStream : default;
|
||
var audio = probe.PrimaryAudioStream;
|
||
if (video == default && audio == default)
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed");
|
||
var extension = AvTypes[attachment.ContentType];
|
||
var format = probe.Format.FormatName.Split(',')[0];
|
||
string arguments;
|
||
if (video == default)
|
||
arguments = "-map 0:a:0 -vn -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "m4a" ? " -movflags +faststart" : string.Empty);
|
||
else if (video.CodecName is "h264" or "vp8" or "vp9" or "av1" && (long)video.Width * video.Height <= options.MaxVideoPixels
|
||
&& video.FrameRate <= options.MaxFrameRate + 0.5)
|
||
arguments = "-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "mp4" ? " -movflags +faststart" : string.Empty);
|
||
else
|
||
{
|
||
var scale = Math.Min(1, Math.Sqrt(options.MaxVideoPixels / (double)((long)video.Width * video.Height)));
|
||
var width = Math.Max(2, (int)(video.Width * scale) / 2 * 2);
|
||
var height = Math.Max(2, (int)(video.Height * scale) / 2 * 2);
|
||
extension = "mp4";
|
||
arguments = $"-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p "
|
||
+ $"-vf scale={width}:{height} -fpsmax {options.MaxFrameRate.ToString(CultureInfo.InvariantCulture)} -c:a aac -b:a 128k -movflags +faststart";
|
||
}
|
||
output = temp + "." + extension;
|
||
await FFMpegArguments.FromFileInput(input, true, o => o.ForceFormat(format).WithCustomArgument("-protocol_whitelist file"))
|
||
.OutputToFile(output, true, o => o.WithCustomArgument(arguments))
|
||
.CancellableThrough(token)
|
||
.ProcessAsynchronously();
|
||
|
||
var made = await FFProbe.AnalyseAsync(output, default, token, "-protocol_whitelist file");
|
||
byte[] poster = default;
|
||
if (video != default)
|
||
{
|
||
attachment.Width = made.PrimaryVideoStream?.Width;
|
||
attachment.Height = made.PrimaryVideoStream?.Height;
|
||
await FFMpeg.SnapshotAsync(output, frame, captureTime: TimeSpan.FromSeconds(Math.Min(1, made.Duration.TotalSeconds / 2)));
|
||
var still = ProcessImage(await File.ReadAllBytesAsync(frame, token), 640, options.PreviewSide);
|
||
poster = still.Preview;
|
||
attachment.Blurhash = still.Blurhash;
|
||
}
|
||
attachment.Kind = video != default ? "video" : "audio";
|
||
attachment.ContentType = extension switch
|
||
{
|
||
"mp4" => "video/mp4",
|
||
"webm" => video != default ? "video/webm" : "audio/webm",
|
||
"m4a" => "audio/mp4",
|
||
"mp3" => "audio/mpeg",
|
||
"ogg" => "audio/ogg",
|
||
"wav" => "audio/wav",
|
||
"flac" => "audio/flac",
|
||
_ => attachment.ContentType
|
||
};
|
||
attachment.DurationSeconds = Math.Round(made.Duration.TotalSeconds, 2);
|
||
attachment.Size = new FileInfo(output).Length;
|
||
attachment.FilePath = SaveFile(output, extension);
|
||
if (poster != default)
|
||
attachment.PreviewPath = await Save(poster, "jpg", token);
|
||
return new MediaOutcome(attachment);
|
||
}
|
||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||
{
|
||
_logger.LogInformation(ex, "Refused an audio or video upload");
|
||
return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed");
|
||
}
|
||
finally
|
||
{
|
||
_processing.Release();
|
||
foreach (var path in new[] { input, output, frame }.Where(p => p != default))
|
||
if (File.Exists(path))
|
||
File.Delete(path);
|
||
}
|
||
}
|
||
|
||
string SaveFile(string source, string extension)
|
||
{
|
||
var (relative, full) = NewPath(extension);
|
||
File.Move(source, full);
|
||
return relative;
|
||
}
|
||
|
||
(string Relative, string Full) NewPath(string extension)
|
||
{
|
||
var now = DateTime.UtcNow;
|
||
var relative = Path.Combine(now.ToString("yyyy", CultureInfo.InvariantCulture), now.ToString("MM", CultureInfo.InvariantCulture),
|
||
$"{Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16))}.{extension}");
|
||
var full = Path.Combine(Root, relative);
|
||
Directory.CreateDirectory(Path.GetDirectoryName(full)!);
|
||
return (relative.Replace('\\', '/'), full);
|
||
}
|
||
|
||
async Task<string> Save(byte[] bytes, string extension, CancellationToken token)
|
||
{
|
||
var (relative, full) = NewPath(extension);
|
||
await File.WriteAllBytesAsync(full, bytes, token);
|
||
return relative;
|
||
}
|
||
|
||
static string Clean(string value, int max) =>
|
||
string.IsNullOrWhiteSpace(value) ? default : value.Trim().Length <= max ? value.Trim() : value.Trim()[..max];
|
||
}
|
||
}
|