thepraandClaude Opus 5.5
d08fbabaa5
LocalEmoji, a new collection (unique shortcode whatever its case, never renamed: the shortcode is its id elsewhere),
managed on /clientapi/admin/emoji/list|insert|update|delete and kept in the staff's log (emoji.insert, .update,
.delete). Its picture goes through IMediaService.EmojiImage: PNG, GIF, WebP or JPEG up to 256 KB, a PNG within 256 px
with alpha kept (an animated GIF or WebP stays an animated GIF, with a still PNG), keep=none, shrunk until it fits
GoToSocial's 100 KiB, held by ServerUse "emoji:<shortcode>" so the janitor and the audit leave it. LocalEmojis keeps the
offered ones in memory, reloaded on every change: GET /api/v1/custom_emojis is real; local posts (words, title, warning,
poll options, each revision) and profiles keep the emoji they name; Notes, Questions, actors and EmojiReacts carry
Mastodon's Emoji tag with its id, served as an Emoji at /emojis/{shortcode} (410 once deleted; the path awaits the
owner's confirmation); a persona reacts with one (Pleroma's reactions, announcements). A restore keeps the live emoji
as it keeps the settings, and trashes server pictures nothing holds. tools/pasture/scenarios/emoji.sh joins batch A.
Verified: built, and run locally against a scratch database (uploads of every type, animation kept, metadata gone,
refusals, rendering, reactions, history, announcements, delete and revive, backup and restore, media audit). The
pasture scenario and the sweep were not run.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WpVCoDeM1qUTRp4fXQdfof
251 lines
16 KiB
C#
251 lines
16 KiB
C#
using MongoDB.Bson;
|
|
using MongoDB.Driver;
|
|
using MongoDB.Entities;
|
|
|
|
using PrivaPub.Models.Federation;
|
|
using PrivaPub.Models.Group;
|
|
using PrivaPub.Models.Jobs;
|
|
using PrivaPub.Models.Post;
|
|
using PrivaPub.Models.Social;
|
|
using PrivaPub.Models.Statistics;
|
|
using PrivaPub.Models.User;
|
|
|
|
namespace PrivaPub.Infrastructure.Data
|
|
{
|
|
public static class Indexes
|
|
{
|
|
public static readonly TimeSpan TombstoneLifetime = TimeSpan.FromDays(90);
|
|
public static readonly TimeSpan ThreadCollectionLifetime = TimeSpan.FromDays(30);
|
|
|
|
public static async Task Create(CancellationToken token = default)
|
|
{
|
|
await Unique<Post>(p => p.ObjectURI, Builders<Post>.Filter.Gt(p => p.ObjectURI, ""), token);
|
|
await Plain<Post>(token, p => p.GroupUserId, p => p.ID);
|
|
await Plain<Post>(token, p => p.GroupId, p => p.ID);
|
|
await Plain<Post>(token, p => p.ActorURI);
|
|
await Plain<Post>(token, p => p.WallURI, p => p.ID);//a persona's wall (FEP-400e)
|
|
await Plain<Post>(token, p => p.ConversationId, p => p.ID);
|
|
await Plain<Post>(token, p => p.InReplyToURI);
|
|
await Plain<Post>(token, p => p.AudienceURI, p => p.ID);//a feed's timeline: the posts of its communities
|
|
// who boosted what: a persona's boosts of the posts a page shows were a scan of every post under load
|
|
// (tools/pasture/load.sh), and a post's boosts and replies are read for its counts and context
|
|
await Plain<Post>(token, p => p.AuthorAccountId, p => p.ReblogOfPostId);
|
|
await Plain<Post>(token, p => p.ReblogOfPostId);
|
|
await Plain<Post>(token, p => p.AnsweringToPostId);
|
|
await DB.Default.Index<Post>().Key(p => p.Geo, KeyType.Geo2DSphere).CreateAsync(token);
|
|
// a post imported from another actor's archive, once per persona: importing it twice changes nothing
|
|
await DB.Default.Index<Post>().Key(p => p.GroupUserId, KeyType.Ascending).Key(p => p.ImportedFromURI, KeyType.Ascending)
|
|
.Option(o =>
|
|
{
|
|
o.Unique = true;
|
|
o.PartialFilterExpression = Builders<Post>.Filter.Gt(p => p.ImportedFromURI, "");
|
|
})
|
|
.CreateAsync(token);
|
|
|
|
// media: a post's, a persona's, what a scheduled post holds, the janitor's never-posted uploads and its trash
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.PostId, m => m.ScheduledStatusId, m => m.CreatedAt);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.OwnerAvatarId);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.ScheduledStatusId);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.ProfileOfAvatarId);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.TrashedAt);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.ServerUse);//the server's own pictures (its thumbnail)
|
|
|
|
// the administrators' announcements: each persona's dismissal and reactions, once each (new collections, R5)
|
|
await DB.Default.Index<AnnouncementRead>().Key(r => r.AnnouncementId, KeyType.Ascending).Key(r => r.AvatarId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<AnnouncementRead>(token, r => r.AvatarId);
|
|
await DB.Default.Index<AnnouncementReaction>().Key(r => r.AnnouncementId, KeyType.Ascending).Key(r => r.AvatarId, KeyType.Ascending)
|
|
.Key(r => r.Name, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<AnnouncementReaction>(token, r => r.AvatarId);
|
|
// the server's custom emoji, one per shortcode whatever its case (a new collection, R7)
|
|
await Unique<LocalEmoji>(e => e.Key, Builders<LocalEmoji>.Filter.Gt(e => e.Key, ""), token);
|
|
// the staff's log, read newest first
|
|
await DB.Default.Index<Models.AdminAction>().Key(a => a.At, KeyType.Descending).CreateAsync(token);
|
|
|
|
await Unique<ForeignAvatar>(a => a.ActorURI, Builders<ForeignAvatar>.Filter.Gt(a => a.ActorURI, ""), token);
|
|
await Plain<ForeignAvatar>(token, a => a.PublicKeyId);
|
|
// the staff's moderation (R6): the moderated accounts, here and elsewhere, and the posts taken down
|
|
await Plain<ForeignAvatar>(token, a => a.SuspendedAt);
|
|
await Plain<ForeignAvatar>(token, a => a.SilencedAt);
|
|
await Plain<Avatar>(token, a => a.SuspendedAt);
|
|
await Plain<Avatar>(token, a => a.SilencedAt);
|
|
await DB.Default.Index<Post>().Key(p => p.ModeratedAt, KeyType.Ascending)
|
|
.Option(o => o.PartialFilterExpression = Builders<Post>.Filter.Type(p => p.ModeratedAt, MongoDB.Bson.BsonType.DateTime))
|
|
.CreateAsync(token);
|
|
|
|
await DB.Default.Index<Follower>()
|
|
.Key(f => f.LocalActorId, KeyType.Ascending)
|
|
.Key(f => f.LocalActorKind, KeyType.Ascending)
|
|
.Key(f => f.ActorURI, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Follower>(token, f => f.ActorURI);
|
|
|
|
await DB.Default.Index<RootToAvatar>()
|
|
.Key(r => r.RootId, KeyType.Ascending)
|
|
.Key(r => r.AvatarId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<RootToAvatar>(token, r => r.AvatarId);
|
|
|
|
await Unique<ReservedName>(r => r.Name, Builders<ReservedName>.Filter.Gt(r => r.Name, ""), token);
|
|
await Unique<RootUser>(u => u.UserName, Builders<RootUser>.Filter.Gt(u => u.UserName, ""), token);
|
|
await Plain<Avatar>(token, a => a.UserName);
|
|
await Plain<Group>(token, g => g.UserName);
|
|
await Plain<Group>(token, g => g.InvitationCode);
|
|
|
|
await Plain<DmGroup>(token, g => g.ParticipantsKey);
|
|
await Plain<DmGroup>(token, g => g.LastPostId);
|
|
await DB.Default.Index<ConversationState>()
|
|
.Key(s => s.AvatarId, KeyType.Ascending)
|
|
.Key(s => s.ConversationId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Unique<GroupAnnouncement>(a => a.ActivityURI, Builders<GroupAnnouncement>.Filter.Gt(a => a.ActivityURI, ""), token);
|
|
|
|
await Plain<Delivery>(token, d => d.DeliveredAt, d => d.AbandonedAt, d => d.NextAttemptAt);
|
|
|
|
await Plain<Job>(token, j => j.Kind, j => j.State, j => j.RunAt);
|
|
await Plain<Job>(token, j => j.State, j => j.LeasedUntil);
|
|
await Unique<Job>(j => j.DedupeKey, Builders<Job>.Filter.Gt(j => j.DedupeKey, ""), token);
|
|
await DB.Default.Index<Job>()
|
|
.Key(j => j.FinishedAt, KeyType.Ascending)
|
|
.Option(o => o.ExpireAfter = TimeSpan.FromDays(7))
|
|
.CreateAsync(token);
|
|
await DB.Default.Index<Following>()
|
|
.Key(f => f.AvatarId, KeyType.Ascending)
|
|
.Key(f => f.TargetActorURI, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Following>(token, f => f.TargetActorURI, f => f.State);
|
|
await Plain<Following>(token, f => f.TargetAccountId, f => f.State);
|
|
await Plain<Following>(token, f => f.TargetActorURI, f => f.State);//who here follows the author of what arrives
|
|
await Plain<Following>(token, f => f.FollowActivityURI);
|
|
await DB.Default.Index<TimelineEntry>()
|
|
.Key(e => e.AvatarId, KeyType.Ascending)
|
|
.Key(e => e.PostId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<TimelineEntry>(token, e => e.PostId);
|
|
await Plain<TimelineEntry>(token, e => e.AuthorAccountId);
|
|
await Unique<Notification>(n => n.DedupeKey, Builders<Notification>.Filter.Gt(n => n.DedupeKey, ""), token);
|
|
await Plain<Notification>(token, n => n.AvatarId, n => n.ID);
|
|
await DB.Default.Index<Favourite>()
|
|
.Key(f => f.AccountId, KeyType.Ascending)
|
|
.Key(f => f.PostId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Favourite>(token, f => f.ActivityURI);
|
|
await Plain<Favourite>(token, f => f.PostId);
|
|
await DB.Default.Index<Participation>()
|
|
.Key(p => p.AvatarId, KeyType.Ascending)
|
|
.Key(p => p.PostId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Participation>(token, p => p.ActivityURI);
|
|
await DB.Default.Index<Marker>()
|
|
.Key(m => m.AvatarId, KeyType.Ascending)
|
|
.Key(m => m.Timeline, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
foreach (var pair in new (Func<Task>, string)[]
|
|
{
|
|
(() => DB.Default.Index<Block>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "block"),
|
|
(() => DB.Default.Index<FollowedTag>().Key(t => t.AvatarId, KeyType.Ascending).Key(t => t.Name, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "followed tag"),
|
|
(() => DB.Default.Index<PersonaListMember>().Key(m => m.ListId, KeyType.Ascending).Key(m => m.AccountId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "list member"),
|
|
(() => DB.Default.Index<BlockedBy>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "blocked by"),
|
|
(() => DB.Default.Index<Mute>().Key(m => m.AvatarId, KeyType.Ascending).Key(m => m.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "mute"),
|
|
(() => DB.Default.Index<AccountDomainBlock>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.Domain, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "domain block"),
|
|
(() => DB.Default.Index<Bookmark>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "bookmark"),
|
|
(() => DB.Default.Index<Pin>().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "pin"),
|
|
(() => DB.Default.Index<Models.Federation.RelaySubscription>().Key(r => r.Configured, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "relay"),
|
|
(() => DB.Default.Index<RemoteFeatured>().Key(f => f.ActorURI, KeyType.Ascending).Key(f => f.ObjectURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "remote-featured"),
|
|
(() => DB.Default.Index<NotificationPolicy>().Key(p => p.AvatarId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification policy"),
|
|
(() => DB.Default.Index<NotificationRequest>().Key(r => r.AvatarId, KeyType.Ascending).Key(r => r.FromActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification request"),
|
|
(() => DB.Default.Index<NotificationPermission>().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.FromActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification permission")
|
|
})
|
|
await pair.Item1();
|
|
await Plain<Block>(token, b => b.TargetActorURI);
|
|
await Plain<Pin>(token, p => p.PostId);
|
|
await Plain<BlockedBy>(token, b => b.ActorURI);
|
|
await Plain<PersonaList>(token, l => l.AvatarId);
|
|
await Plain<PersonaFilter>(token, f => f.AvatarId);
|
|
await Plain<FollowedTag>(token, t => t.Name);
|
|
await Plain<Post>(token, p => p.Tags);
|
|
//the words status search reads, in every language alike (no stemming, no stop words); Language is not the
|
|
//index's language field
|
|
await DB.Default.Index<Post>().Key(p => p.Text, KeyType.Text).Key(p => p.ContentHtml, KeyType.Text).Key(p => p.SpoilerText, KeyType.Text)
|
|
.Key(p => p.Title, KeyType.Text).Option(o => { o.DefaultLanguage = "none"; o.LanguageOverride = "textSearchLanguage"; o.Name = "words"; })
|
|
.CreateAsync(token);
|
|
await Plain<PersonaListMember>(token, m => m.AvatarId);
|
|
await Plain<Mute>(token, m => m.TargetActorURI);
|
|
await Plain<Report>(token, r => r.IsResolved, r => r.ID);
|
|
await Unique<DomainBlock>(b => b.Domain, Builders<DomainBlock>.Filter.Gt(b => b.Domain, ""), token);
|
|
await Unique<RemoteInstance>(i => i.Host, Builders<RemoteInstance>.Filter.Gt(i => i.Host, ""), token);
|
|
await Unique<ObjectRecord>(r => r.ObjectURI, Builders<ObjectRecord>.Filter.Gt(r => r.ObjectURI, ""), token);
|
|
await Plain<ObjectRecord>(token, r => r.PostId);
|
|
await Unique<DeletedObject>(d => d.ObjectURI, Builders<DeletedObject>.Filter.Gt(d => d.ObjectURI, ""), token);
|
|
await DB.Default.Index<DeletedObject>().Key(d => d.DeletedAt, KeyType.Ascending).Option(o => o.ExpireAfter = TombstoneLifetime).CreateAsync(token);
|
|
await DB.Default.Index<FeedSubscription>().Key(s => s.AvatarId, KeyType.Ascending).Key(s => s.FeedActorURI, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<FeedSubscription>(token, s => s.FeedActorURI);
|
|
await Unique<ThreadCollection>(c => c.URI, Builders<ThreadCollection>.Filter.Gt(c => c.URI, ""), token);
|
|
await DB.Default.Index<ThreadCollection>().Key(c => c.ReadAt, KeyType.Ascending).Option(o => o.ExpireAfter = ThreadCollectionLifetime).CreateAsync(token);
|
|
await DB.Default.Index<Downvote>().Key(d => d.PostId, KeyType.Ascending).Key(d => d.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<Downvote>(token, d => d.ActivityURI);
|
|
await DB.Default.Index<PollVote>().Key(v => v.PostId, KeyType.Ascending).Key(v => v.ActorURI, KeyType.Ascending).Key(v => v.Choice, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await DB.Default.Index<PollVote>().Key(v => v.PostId, KeyType.Ascending).Key(v => v.VoterAccountId, KeyType.Ascending).CreateAsync(token);
|
|
await DB.Default.Index<Reaction>().Key(r => r.PostId, KeyType.Ascending).Key(r => r.ActorURI, KeyType.Ascending).Key(r => r.Emoji, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<Reaction>(token, r => r.ActivityURI);
|
|
await DB.Default.Index<QuoteLicence>().Key(l => l.PostId, KeyType.Ascending).Key(l => l.QuotingObjectURI, KeyType.Ascending).CreateAsync(token);
|
|
await Unique<Domain.Content.LinkPreview>(p => p.Url, Builders<Domain.Content.LinkPreview>.Filter.Gt(p => p.Url, ""), token);
|
|
|
|
await DB.Default.Index<InteractionEvent>().Key(e => e.At, KeyType.Ascending).Option(o => o.ExpireAfter = InteractionEvent.Retention).CreateAsync(token);
|
|
await Plain<InteractionEvent>(token, e => e.Host, e => e.At);
|
|
await DB.Default.Index<InteractionSalt>().Key(s => s.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await DB.Default.Index<InteractionSalt>().Key(s => s.ExpiresAt, KeyType.Ascending).Option(o => o.ExpireAfter = TimeSpan.Zero).CreateAsync(token);
|
|
// codes live an hour: the TTL monitor removes what expired, nothing else ever has to
|
|
await DB.Default.Index<EmailRecovery>().Key(r => r.ExpiresAt, KeyType.Ascending).Option(o => o.ExpireAfter = TimeSpan.Zero).CreateAsync(token);
|
|
await Plain<EmailRecovery>(token, r => r.CodeHash);
|
|
await DB.Default.Index<EmailVerification>().Key(v => v.ExpiresAt, KeyType.Ascending).Option(o => o.ExpireAfter = TimeSpan.Zero).CreateAsync(token);
|
|
await Plain<EmailVerification>(token, v => v.RootUserId);
|
|
await DB.Default.Index<InstanceDay>().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<InstanceDay>(token, d => d.Host, d => d.Day);
|
|
await DB.Default.Index<ServerDay>().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
|
|
await Plain<RemoteInstanceSnapshot>(token, s => s.CdnDomain, s => s.Week);
|
|
await Plain<RemoteInstanceSnapshot>(token, s => s.Cdn, s => s.Week);
|
|
await Plain<CdnRangeSet>(token, s => s.Key);
|
|
await Plain<RemoteInstance>(token, i => i.Seen);
|
|
await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
|
|
}
|
|
|
|
// unique among the documents that have the key: the partial filter is `$gt: ""` (any non-empty string) rather than
|
|
// `$type: "string"`, because MongoDB uses a partial index for a lookup only when the lookup implies its filter, and an
|
|
// equality on a string implies the first, never the second: every lookup by these keys scanned the collection
|
|
// (tools/pasture/load.sh, 280 ms a lookup at 30 000 posts)
|
|
static async Task Unique<T>(System.Linq.Expressions.Expression<Func<T, object>> key, FilterDefinition<T> partial,
|
|
CancellationToken token) where T : IEntity =>
|
|
await DB.Default.Index<T>()
|
|
.Key(key, KeyType.Ascending)
|
|
.Option(o =>
|
|
{
|
|
o.Unique = true;
|
|
o.PartialFilterExpression = partial;
|
|
})
|
|
.CreateAsync(token);
|
|
|
|
static async Task Plain<T>(CancellationToken token, params System.Linq.Expressions.Expression<Func<T, object>>[] keys) where T : IEntity
|
|
{
|
|
var index = DB.Default.Index<T>();
|
|
foreach (var key in keys)
|
|
index.Key(key, KeyType.Ascending);
|
|
await index.CreateAsync(token);
|
|
}
|
|
}
|
|
}
|