MeWeb/tools/knowledge.cs

#:project ../src/MeWeb.Core/MeWeb.Core.csproj
#:project ../src/MeWeb.Shared/MeWeb.Shared.csproj
using System.Net.Http.Json;
using System.Security.Cryptography;

using MeWeb.Core.Knowledge;
using MeWeb.Shared;

const string Fallback = "https://iptoasn.com/data/ip2asn-combined.tsv.gz";
var root = FindRoot();
var output = Path.Combine(root, "artifacts", "knowledge");
var cache = Path.Combine(root, "tools", "cache");
Directory.CreateDirectory(output);
Directory.CreateDirectory(cache);
var server = Environment.GetEnvironmentVariable("MEWEB_KNOWLEDGE_URL") ?? Endpoints.Base;

using var http = new HttpClient { Timeout = TimeSpan.FromMinutes(2) };
http.DefaultRequestHeaders.UserAgent.ParseAdd("me-web-knowledge/2 (+https://git.thepra.dev/Arasaka/MeWeb)");

if (!args.Contains("--local"))
{
  try
  {
    var manifest = await http.GetFromJsonAsync(server + Endpoints.Knowledge.GET_Manifest, MeWebJsonContext.Default.ViewKnowledge);
    if (manifest is { Files.Count: > 0 })
    {
      foreach (var file in manifest.Files)
      {
        var bytes = await http.GetByteArrayAsync(server + Endpoints.Knowledge.GET_M_File(file.Name));
        var hash = Convert.ToHexStringLower(SHA256.HashData(bytes));
        if (hash != file.Sha256) throw new InvalidDataException($"{file.Name}: sha256 {hash}, the manifest says {file.Sha256}");
        File.WriteAllBytes(Path.Combine(output, file.Name), bytes);
        Console.WriteLine($"knowledge: {file.Name} {bytes.Length / 1024} KiB from {server}");
      }
      File.Move(Path.Combine(output, "SOURCES.txt"), Path.Combine(output, "SOURCE.txt"), overwrite: true);
      Report();
      return 0;
    }
  }
  catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or InvalidDataException or System.Text.Json.JsonException)
  {
    Console.Error.WriteLine($"knowledge: the server's files could not be used ({ex.Message}); building the provider map alone");
  }
}

var download = Path.Combine(cache, "ip2asn-combined.tsv.gz");
var sourceDate = File.Exists(download) ? new DateTimeOffset(File.GetLastWriteTimeUtc(download), TimeSpan.Zero) : DateTimeOffset.UtcNow;
if (!File.Exists(download) || File.GetLastWriteTimeUtc(download) < DateTime.UtcNow.AddHours(-12) || args.Contains("--refresh"))
{
  using var response = await http.GetAsync(Fallback, HttpCompletionOption.ResponseHeadersRead);
  response.EnsureSuccessStatusCode();
  sourceDate = response.Content.Headers.LastModified ?? DateTimeOffset.UtcNow;
  await using (var file = File.Create(download + ".tmp"))
    await response.Content.CopyToAsync(file);
  File.Move(download + ".tmp", download, overwrite: true);
  File.SetLastWriteTimeUtc(download, sourceDate.UtcDateTime);
}
using (var input = File.OpenRead(download))
using (var file = File.Create(Path.Combine(output, "ip2asn.bin")))
  AsnTable.Write(file, sourceDate, Sources.IpToAsn(input));
foreach (var stale in new[] { "networks.bin", "carriers.json", "registry.bin" }) File.Delete(Path.Combine(output, stale));
string sha;
using (var input = File.OpenRead(download)) sha = Convert.ToHexStringLower(SHA256.HashData(input));
File.WriteAllText(Path.Combine(output, "SOURCE.txt"), $"iptoasn.com, Public Domain Dedication and License (PDDL) 1.0\n{Fallback}\nlast-modified {sourceDate:O}\nsha256 {sha}\n");
Report();
return 0;

void Report()
{
  using var stream = File.OpenRead(Path.Combine(output, "ip2asn.bin"));
  var table = AsnTable.Read(stream);
  var path = Path.Combine(output, "networks.bin");
  NetworkTable networks = null;
  if (File.Exists(path))
    using (var input = File.OpenRead(path)) networks = NetworkTable.Read(input);
  path = Path.Combine(output, "registry.bin");
  using var registry = File.Exists(path) ? RegistryTable.Open(File.OpenRead(path)) : null;
  Console.WriteLine($"knowledge: {table.Count} networks in the provider map (source {table.SourceDate:yyyy-MM-dd}), {networks?.Count ?? 0} with names or upstreams, {registry?.Count ?? 0} address blocks");
}

static string FindRoot()
{
  var dir = new DirectoryInfo(Directory.GetCurrentDirectory());
  while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "MeWeb.slnx"))) dir = dir.Parent;
  return dir?.FullName ?? throw new InvalidOperationException("run from inside the repository (MeWeb.slnx not found)");
}