SocialPub/PrivaPub.Tests/Statistics/RollupTests.cs

using MongoDB.Entities;

using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Infrastructure.Statistics;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Statistics;
using PrivaPub.Tests.Support;

namespace PrivaPub.Tests.Statistics
{
	public class RollupFoldTests
	{
		static IEnumerable<InteractionEvent> Day(string host) => new[]
		{
			new InteractionEvent { Host = host, Channel = "recv", Activity = "Create", Status = 202, Reason = "queued", Signature = "cavage:rsa-sha256", LatencyMs = 40, Bytes = 900 },
			new InteractionEvent { Host = host, Channel = "recv", Activity = "Create", Status = 401, Reason = "no-signature" },
			new InteractionEvent { Host = host, Channel = "in", Activity = "Create", Object = "Note", Outcome = "accepted", Reason = "stored", Audience = "public", LocalKind = "person", ActorHash = "a", Features = new() { "fep-044f-quote", "source:text/markdown" }, WaitMs = 1500 },
			new InteractionEvent { Host = host, Channel = "in", Activity = "Create", Object = "Note", Outcome = "accepted", Reason = "stored", Audience = "private", ActorHash = "b", Features = new() { "interaction-policy" } },
			new InteractionEvent { Host = host, Channel = "in", Activity = "Flag", Outcome = "accepted", Reason = "reported", Audience = "none", ActorHash = "a" },
			new InteractionEvent { Host = host, Channel = "in", Activity = "Like", Object = "Note", Outcome = "dropped", Reason = "unknown-object", Audience = "public" },
			new InteractionEvent { Host = host, Channel = "out", Activity = "Create", Object = "Note", Outcome = "ok", Status = 202, Audience = "public", LocalKind = "person", LatencyMs = 120, Bytes = 2000 },
			new InteractionEvent { Host = host, Channel = "out", Activity = "Block", Outcome = "ok", Status = 202, Audience = "none" },
			new InteractionEvent { Host = host, Channel = "out", Activity = "Create", Object = "Note", Outcome = "retry", Reason = "503", Audience = "private" },
			new InteractionEvent { Host = host, Channel = "http", Purpose = "actor", Trigger = "verify", Outcome = "ok", LatencyMs = 300, Bytes = 1400 },
			new InteractionEvent { Host = host, Channel = "http", Purpose = "object", Trigger = "fetchancestors", Outcome = "refused", Reason = "410" }
		};

		[Fact]
		public void Every_event_counts_for_the_admin_with_its_reason()
		{
			var fold = Rollup.Fold(Day("social.example")).Hosts["social.example"];

			Assert.Equal(1, fold.Counters["recv:Create:202:queued"]);
			Assert.Equal(1, fold.Counters["recv:Create:401:no-signature"]);
			Assert.Equal(1, fold.Counters["sig:cavage:rsa-sha256"]);
			Assert.Equal(2, fold.Counters["in:Create:Note:accepted:stored"]);
			Assert.Equal(1, fold.Counters["in:Like:Note:dropped:unknown-object"]);
			Assert.Equal(1, fold.Counters["out:Create:Note:retry:503"]);
			Assert.Equal(1, fold.Counters["out:Block:-:ok"]);
			Assert.Equal(1, fold.Counters["http:object:refused:410"]);
			Assert.Equal(1, fold.Counters["aud:in:private"]);
			Assert.Equal(1, fold.Counters["kind:out:person"]);
			Assert.Equal(1, fold.Counters["feat:source:text/markdown"]);
			Assert.Equal(1, fold.Latency["out:le250ms"]);
			Assert.Equal(1, fold.Latency["http:actor:le500ms"]);
			Assert.Equal(1, fold.Waits["in:le10s"]);
			Assert.Equal(2000, fold.Bytes["out"]);
			Assert.Equal(1400, fold.Bytes["http:actor"]);
			Assert.Equal(2, fold.Accounts.Count);
		}

		[Fact]
		public void The_public_counters_keep_only_public_kinds_of_traffic_without_reasons()
		{
			var fold = Rollup.Fold(Day("social.example")).Hosts["social.example"];

			Assert.Equal(new Dictionary<string, long>
			{
				["in:Create:Note:accepted"] = 1,
				["in:Like:Note:dropped"] = 1,
				["out:Create:Note:delivered"] = 1,
				["feat:fep-044f-quote"] = 1,
				["feat:source:text/markdown"] = 1,
				["health:ok"] = 4,
				["health:failed"] = 1
			}, fold.PublicCounters);
			Assert.DoesNotContain(fold.PublicCounters.Keys, k => k.Contains("Flag") || k.Contains("Block") || k.Contains("recv") || k.Contains("interaction-policy"));
		}

		[Fact]
		public void Keys_are_safe_field_names_and_folding_is_repeatable()
		{
			Assert.Equal("in:-:Note:accepted", Rollup.Key("in", null, "Note", "accepted", null));
			Assert.Equal("feat:content:text/x_mfm", Rollup.Key("feat", "content:text/x.mfm"));
			var first = Rollup.Fold(Day("a.example").Concat(Day("b.example")));
			var second = Rollup.Fold(Day("a.example").Concat(Day("b.example")));

			Assert.Equal(first.Hosts["a.example"].Counters, second.Hosts["a.example"].Counters);
			Assert.Equal(2, first.Accounts.Count);
			Assert.Equal(2, first.Hosts.Count);
		}
	}

	[Trait("Category", "Integration")]
	public sealed class RollupJobTests : IAsyncLifetime
	{
		static readonly DateTime Day = new(2001, 2, 3, 0, 0, 0, DateTimeKind.Utc);
		readonly string _host = $"rollup{Guid.NewGuid():N}.example";

		public ValueTask InitializeAsync()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			return ValueTask.CompletedTask;
		}

		public ValueTask DisposeAsync() => ValueTask.CompletedTask;

		[Fact]
		public async Task A_past_day_folds_into_server_rows_replaces_itself_keeps_reads_and_forgets_its_salt()
		{
			var token = TestContext.Current.CancellationToken;
			await DB.Default.SaveAsync(new[]
			{
				new InteractionEvent { At = Day.AddHours(3), Host = _host, Channel = "in", Activity = "Create", Object = "Note", Outcome = "accepted", Audience = "public", ActorHash = "x" },
				new InteractionEvent { At = Day.AddHours(5), Host = _host, Channel = "out", Activity = "Create", Object = "Note", Outcome = "ok", Audience = "public" },
				new InteractionEvent { At = Day.AddDays(1).AddMinutes(1), Host = _host, Channel = "in", Activity = "Like", Outcome = "accepted" }
			}, token);
			await DB.Default.SaveAsync(new InstanceDay { Day = Day, Host = _host, Reads = new() { ["media:hit"] = 7 } }, token);
			await DB.Default.SaveAsync(new InteractionSalt { Day = Day, Key = "AAAA", ExpiresAt = DateTime.UtcNow.AddHours(1) }, token);
			var job = new RollupJob(new JobQueue(j => j.DedupeKey == "never"));

			Assert.Equal(JobResult.Done, (await job.Handle(RollupJob.For(Day, $"test|{Guid.NewGuid():N}"), token)).Result);
			var first = await DB.Default.Find<InstanceDay>().Match(d => d.Day == Day && d.Host == _host).ExecuteSingleAsync(token);
			Assert.Equal(JobResult.Done, (await job.Handle(RollupJob.For(Day, $"test|{Guid.NewGuid():N}"), token)).Result);
			var second = await DB.Default.Find<InstanceDay>().Match(d => d.Day == Day && d.Host == _host).ExecuteSingleAsync(token);

			Assert.Equal(1, first.Counters["in:Create:Note:accepted"]);
			Assert.False(first.Counters.ContainsKey("in:Like:-:accepted"));
			Assert.Equal(1, first.PublicCounters["out:Create:Note:delivered"]);
			Assert.Equal(1, first.Accounts);
			Assert.Equal(7, first.Reads["media:hit"]);
			Assert.Equal(first.Counters, second.Counters);
			Assert.Equal(7, second.Reads["media:hit"]);
			Assert.NotNull(second.RolledUpAt);
			Assert.False(await DB.Default.Find<InteractionSalt>().Match(s => s.Day == Day).ExecuteAnyAsync(token));
			Assert.True(await DB.Default.Find<Job>().Match(j => j.DedupeKey == "rollup|2001-02-04" && j.Kind == JobKind.RollupDay).ExecuteAnyAsync(token));
		}

		[Fact]
		public async Task Today_is_not_folded_before_it_ends()
		{
			var outcome = await new RollupJob(new JobQueue(j => j.DedupeKey == "never")).Handle(RollupJob.For(DateTime.UtcNow.Date, "x"), TestContext.Current.CancellationToken);

			Assert.Equal(JobResult.Defer, outcome.Result);
			Assert.Equal(DateTime.UtcNow.Date.AddDays(1) + RollupJob.After, outcome.RetryAt);
		}
	
		[Fact]
		public async Task The_schedule_plans_today_and_catches_up_a_missed_day()
		{
			var token = TestContext.Current.CancellationToken;
			var missed = DateTime.UtcNow.Date.AddDays(-3);
			await DB.Default.SaveAsync(new InteractionEvent { At = missed.AddHours(12), Host = _host, Channel = "in", Activity = "Create", Outcome = "accepted" }, token);

			await new StatisticsSchedule(new JobQueue(j => j.DedupeKey == "never"), Microsoft.Extensions.Logging.Abstractions.NullLogger<StatisticsSchedule>.Instance).Plan(token);

			var today = await DB.Default.Find<Job>().Match(j => j.DedupeKey == $"rollup|{RollupJob.DayKey(DateTime.UtcNow.Date)}").ExecuteFirstAsync(token);
			Assert.Equal(DateTime.UtcNow.Date.AddDays(1) + RollupJob.After, today.RunAt);
			Assert.True(await DB.Default.Find<Job>().Match(j => j.DedupeKey == $"rollup|{RollupJob.DayKey(missed)}").ExecuteAnyAsync(token));
		}
	}
}