SocialPub/PrivaPub.Tests/Statistics/CrawlerTests.cs

using Microsoft.Extensions.Logging.Abstractions;

using MongoDB.Entities;

using PrivaPub.Federation.Crawler;
using PrivaPub.Federation.Moderation;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Infrastructure.Statistics;
using PrivaPub.Models;
using PrivaPub.Models.Federation;
using PrivaPub.Models.Jobs;
using PrivaPub.Tests.Support;
using PrivaPub.Tests.Support.Host;

using System.Net;
using System.Text;

namespace PrivaPub.Tests.Statistics
{
	public class RobotsTxtTests
	{
		const string Robots = """
			# comments are ignored
			User-agent: *
			Disallow: /private
			Allow: /private/ok

			User-agent: PrivaPub-Stargazer
			User-agent: other
			Disallow: /api/
			Allow: /api/v1/instance$

			User-agent: greedy
			Disallow: /*.json$
			""";

		[Fact]
		public void The_crawlers_own_group_wins_and_the_longest_rule_decides()
		{
			var mine = RobotsTxt.Parse(Robots, "PrivaPub-Stargazer");
			var anyone = RobotsTxt.Parse(Robots, "SomethingElse");
			var greedy = RobotsTxt.Parse(Robots, "greedy");

			Assert.False(mine.Allows("/api/v1/instance/peers"));
			Assert.True(mine.Allows("/api/v1/instance"));
			Assert.True(mine.Allows("/private"));
			Assert.True(mine.Allows("/.well-known/nodeinfo"));
			Assert.False(anyone.Allows("/private/nope"));
			Assert.True(anyone.Allows("/private/ok/yes"));
			Assert.True(anyone.Allows("/api/v1/instance/peers"));
			Assert.False(greedy.Allows("/nodeinfo/2.1.json"));
			Assert.True(greedy.Allows("/nodeinfo/2.1.json?x=1"));
		}

		[Fact]
		public void An_error_keeps_the_crawler_out_and_a_missing_file_lets_it_in()
		{
			Assert.True(RobotsTxt.From(404, default, Stargazer.Token).Allows("/anything"));
			Assert.False(RobotsTxt.From(503, default, Stargazer.Token).Allows("/anything"));
			Assert.False(RobotsTxt.From(0, default, Stargazer.Token).Allows("/anything"));
			Assert.False(RobotsTxt.From(200, "User-agent: *\nDisallow: /", Stargazer.Token).Allows("/robots.txt"));
			Assert.True(RobotsTxt.From(200, "User-agent: *\nDisallow:", Stargazer.Token).Allows("/x"));
		}

		[Fact]
		public void Peer_lists_yield_only_names_we_could_ever_reach()
		{
			var names = new[] { "Social.Example.org.", "10.0.0.1", "host.example.org:8443", "x.onion", "printer.local", "bücher.example.org", "a.test", "noDot", "social.example.org" };

			Assert.Equal(new[] { "social.example.org", "xn--bcher-kva.example.org" }, PeerHosts.Filter(names, _ => true));
			Assert.Contains("a.test", PeerHosts.Filter(names, _ => true, testNetwork: true));
			Assert.Empty(PeerHosts.Filter(names, _ => false));
		}

		[Fact]
		public void A_truncated_peers_list_still_gives_what_was_read()
		{
			var items = new List<string>();
			FederationHttp.ReadStrings(Encoding.UTF8.GetBytes("[\"a.example\", 3, {\"x\":1}, \"b.example\", \"c.exa"), 10, items);
			var capped = new List<string>();
			FederationHttp.ReadStrings(Encoding.UTF8.GetBytes("[\"a\",\"b\",\"c\"]"), 2, capped);

			Assert.Equal(new[] { "a.example", "b.example" }, items);
			Assert.Equal(new[] { "a", "b" }, capped);
		}
	}

	[Trait("Category", "Integration")]
	public sealed class CrawlerTests : IAsyncLifetime
	{
		Peer _peer;
		readonly string _run = Guid.NewGuid().ToString("N")[..10];

		public async ValueTask InitializeAsync()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			_peer = await Peer.Start();
		}

		public async ValueTask DisposeAsync()
		{
			if (_peer != default)
				await _peer.DisposeAsync();
		}

		sealed class PeerDescriber : InstanceDescriber
		{
			readonly string _peer;

			public PeerDescriber(IFederationHttp http, string peer) : base(http) => _peer = peer;

			protected override string Address(string url) => $"{_peer}/{new Uri(url).Host}{new Uri(url).PathAndQuery}";
		}

		sealed class PeerCrawler : InstanceCrawler
		{
			readonly string _peer;

			public PeerCrawler(IFederationHttp http, InstanceDescriber describer, IDomainBlocks blocks, CrawlerOptions crawler, string peer, MemoryLedger ledger)
				: base(http, describer, blocks, new StaticOptions<StatisticsOptions>(new StatisticsOptions { Crawler = crawler }),
					new StaticOptions<AppConfiguration>(new AppConfiguration { BackendBaseAddress = "https://privapub.test" }),
					new StaticOptions<FederationOptions>(new FederationOptions { AllowPrivateNetworks = true }), ledger) => _peer = peer;

			protected override string Address(string url) => $"{_peer}/{new Uri(url).Host}{new Uri(url).PathAndQuery}";
		}

		(PeerCrawler Crawler, MemoryLedger Ledger) Crawler(CrawlerOptions options, IDomainBlocks blocks = default)
		{
			var http = Peer.Http();
			var ledger = new MemoryLedger();
			return (new PeerCrawler(http, new PeerDescriber(http, _peer.A), blocks ?? new NoBlocks(), options, _peer.A, ledger), ledger);
		}

		string Host(string name) => $"{name}{_run}.example.org";

		void Serve(string host, string robots = default, int robotsStatus = 200, params string[] peers)
		{
			if (robots != default)
				_peer.ServeText($"/{host}/robots.txt", robots, "text/plain");
			else
				_peer.Answer($"/{host}/robots.txt", robotsStatus);
			_peer.Serve($"/{host}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://{{host}}/nodeinfo/2.0"}]}""");
			_peer.Serve($"/{host}/nodeinfo/2.0", """{"software":{"name":"lemmy","version":"0.19.9"},"usage":{"users":{"total":3}}}""");
			_peer.Serve($"/{host}/api/v1/instance/peers", "[" + string.Join(",", peers.Select(p => $"\"{p}\"")) + "]");
		}

		[Fact]
		public async Task A_crawl_describes_a_crawled_server_and_learns_its_peers_politely()
		{
			var token = TestContext.Current.CancellationToken;
			var host = Host("seed");
			var blocked = Host("blocked");
			Serve(host, "User-agent: *\nDisallow: /private", 200, Host("new1"), Host("new2"), blocked, "10.0.0.1", Host("new1"));
			var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
			blocks.Load(new[] { new DomainBlock { Domain = blocked, Severity = DomainBlockSeverity.Silence } });
			await DB.Default.SaveAsync(new RemoteInstance { Host = host, Seen = "crawled" }, token);
			var (crawler, ledger) = Crawler(new CrawlerOptions { Enabled = true }, blocks);

			await crawler.Handle(new Job { Kind = JobKind.CrawlInstance, Payload = host }, token);

			var seed = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteSingleAsync(token);
			Assert.Equal(("lemmy", 5, false), (seed.Software, seed.PeersCount!.Value, seed.RobotsDisallowed!.Value));
			Assert.NotNull(seed.LastCrawledAt);
			var found = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == Host("new1") || i.Host == Host("new2") || i.Host == blocked).ExecuteAsync(token);
			Assert.Equal(new[] { Host("new1"), Host("new2") }, found.Select(i => i.Host).OrderBy(h => h));
			Assert.All(found, i => Assert.Equal("crawled", i.Seen));
			var visited = _peer.Requests.Where(r => r.Path.StartsWith("/" + host + "/")).ToList();
			Assert.All(visited, r => Assert.Contains(r.Path[(host.Length + 1)..], new[] { "/robots.txt", "/.well-known/nodeinfo", "/nodeinfo/2.0", "/api/v1/instance/peers" }));
			Assert.All(visited, r => Assert.StartsWith("PrivaPub-Stargazer/", r.Headers["User-Agent"]));
			Assert.All(visited, r => Assert.Contains("/stargazing)", r.Headers["User-Agent"]));
			Assert.Equal(("crawl", "ok"), (Assert.Single(ledger.Events).Event.Channel, ledger.Events.Single().Event.Outcome));
		}

		[Fact]
		public async Task Robots_txt_and_its_absence_are_obeyed_and_a_touched_server_is_not_described_again()
		{
			var token = TestContext.Current.CancellationToken;
			var shy = Host("shy");
			var broken = Host("broken");
			var touched = Host("touched");
			Serve(shy, "User-agent: PrivaPub-Stargazer\nDisallow: /api/", 200, Host("hidden"));
			Serve(broken, default, 503, Host("never"));
			Serve(touched, "", 200);
			await DB.Default.SaveAsync(new[] { new RemoteInstance { Host = shy, Seen = "crawled" }, new RemoteInstance { Host = touched, Seen = "touched" } }, token);
			var (crawler, ledger) = Crawler(new CrawlerOptions { Enabled = true });

			await crawler.Handle(new Job { Payload = shy }, token);
			await crawler.Handle(new Job { Payload = broken }, token);
			await crawler.Handle(new Job { Payload = touched }, token);

			Assert.DoesNotContain(_peer.Requests, r => r.Path == $"/{shy}/api/v1/instance/peers");
			Assert.Contains(_peer.Requests, r => r.Path == $"/{shy}/nodeinfo/2.0");
			Assert.Equal(new[] { $"/{broken}/robots.txt" }, _peer.Requests.Where(r => r.Path.StartsWith($"/{broken}/")).Select(r => r.Path));
			Assert.True((await DB.Default.Find<RemoteInstance>().Match(i => i.Host == broken).ExecuteSingleAsync(token)).RobotsDisallowed);
			Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith($"/{touched}/.well-known/"));
			Assert.Contains(ledger.Events, e => e.Event.Host == broken && e.Event.Reason == "robots-disallowed");
			Assert.False(await DB.Default.Find<RemoteInstance>().Match(i => i.Host == Host("hidden") || i.Host == Host("never")).ExecuteAnyAsync(token));
		}

		[Fact]
		public async Task A_disabled_crawler_does_nothing_and_the_caps_hold()
		{
			var token = TestContext.Current.CancellationToken;
			var host = Host("capped");
			Serve(host, "", 200, Enumerable.Range(0, 20).Select(i => Host($"many{i}-")).ToArray());
			await DB.Default.SaveAsync(new RemoteInstance { Host = host, Seen = "touched" }, token);

			await Crawler(new CrawlerOptions { Enabled = false }).Crawler.Handle(new Job { Payload = host }, token);
			Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith($"/{host}/"));

			await Crawler(new CrawlerOptions { Enabled = true, MaxNewHostsPerCrawl = 3 }).Crawler.Handle(new Job { Payload = host }, token);
			Assert.Equal(3, await DB.Default.CountAsync<RemoteInstance>(i => i.Host.StartsWith("many") && i.Host.EndsWith($"{_run}.example.org"), cancellation: token));
		}

		[Fact]
		public async Task The_planner_spreads_due_servers_across_the_hour_and_skips_fresh_and_blocked_ones()
		{
			var token = TestContext.Current.CancellationToken;
			var due = Host("due");
			var fresh = Host("fresh");
			var blocked = Host("plannedblock");
			await DB.Default.SaveAsync(new[]
			{
				new RemoteInstance { Host = due, Seen = "crawled", LastCrawledAt = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc) },
				new RemoteInstance { Host = fresh, Seen = "crawled", LastCrawledAt = DateTime.UtcNow },
				new RemoteInstance { Host = blocked, Seen = "crawled", LastCrawledAt = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc) }
			}, token);
			var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
			blocks.Load(new[] { new DomainBlock { Domain = blocked, Severity = DomainBlockSeverity.Suspend } });
			var options = new StatisticsOptions { Crawler = new CrawlerOptions { Enabled = true, HostsPerHour = 600, Seeds = new() { Host("seedling"), "10.1.1.1" } } };

			await new CrawlPlanner(new JobQueue(j => j.DedupeKey == "never"), blocks, new StaticOptions<StatisticsOptions>(options)).Handle(new Job(), token);
			await new CrawlPlanner(new JobQueue(j => j.DedupeKey == "never"), blocks, new StaticOptions<StatisticsOptions>(new StatisticsOptions())).Handle(new Job(), token);

			var planned = (await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.CrawlInstance && j.Payload.EndsWith($"{_run}.example.org")).ExecuteAsync(token)).Select(j => j.Payload).ToList();
			Assert.Contains(due, planned);
			Assert.Contains(Host("seedling"), planned);
			Assert.DoesNotContain(fresh, planned);
			Assert.DoesNotContain(blocked, planned);
			Assert.Equal("crawled", (await DB.Default.Find<RemoteInstance>().Match(i => i.Host == Host("seedling")).ExecuteSingleAsync(token)).Seen);
		}
	}

	[Trait("Category", "Integration")]
	public sealed class StargazingPageTests : IAsyncLifetime
	{
		PrivaPubHost _host;

		public async ValueTask InitializeAsync()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			_host = await PrivaPubHost.Shared();
		}

		public ValueTask DisposeAsync() => ValueTask.CompletedTask;

		[Fact]
		public async Task The_page_names_the_crawler_says_it_is_off_and_how_to_keep_it_out()
		{
			using var client = _host.Client();

			using var response = await client.GetAsync("/stargazing", TestContext.Current.CancellationToken);
			var page = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken);

			Assert.Equal(HttpStatusCode.OK, response.StatusCode);
			Assert.Contains("PrivaPub-Stargazer/", page);
			Assert.Contains("https://privapub.test/stargazing", page);
			Assert.Contains("<strong>off</strong>", page);
			Assert.Contains("Disallow: /", page);
			Assert.Contains("DB-IP", page);
			Assert.StartsWith("default-src 'none'", response.Headers.GetValues("Content-Security-Policy").Single());
		}
	}
}