#:project ../src/MeWeb.Core/MeWeb.Core.csproj #:project ../src/MeWeb.Shared/MeWeb.Shared.csproj using System.Net.Http.Json; using System.Security.Cryptography; using MeWeb.Core.Knowledge; using MeWeb.Shared; const string Fallback = "https://iptoasn.com/data/ip2asn-combined.tsv.gz"; var root = FindRoot(); var output = Path.Combine(root, "artifacts", "knowledge"); var cache = Path.Combine(root, "tools", "cache"); Directory.CreateDirectory(output); Directory.CreateDirectory(cache); var server = Environment.GetEnvironmentVariable("MEWEB_KNOWLEDGE_URL") ?? Endpoints.Base; using var http = new HttpClient { Timeout = TimeSpan.FromMinutes(2) }; http.DefaultRequestHeaders.UserAgent.ParseAdd("me-web-knowledge/2 (+https://git.thepra.dev/Arasaka/MeWeb)"); if (!args.Contains("--local")) { try { var manifest = await http.GetFromJsonAsync(server + Endpoints.Knowledge.GET_Manifest, MeWebJsonContext.Default.ViewKnowledge); if (manifest is { Files.Count: > 0 }) { foreach (var file in manifest.Files) { var bytes = await http.GetByteArrayAsync(server + Endpoints.Knowledge.GET_M_File(file.Name)); var hash = Convert.ToHexStringLower(SHA256.HashData(bytes)); if (hash != file.Sha256) throw new InvalidDataException($"{file.Name}: sha256 {hash}, the manifest says {file.Sha256}"); File.WriteAllBytes(Path.Combine(output, file.Name), bytes); Console.WriteLine($"knowledge: {file.Name} {bytes.Length / 1024} KiB from {server}"); } File.Move(Path.Combine(output, "SOURCES.txt"), Path.Combine(output, "SOURCE.txt"), overwrite: true); Report(); return 0; } } catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or InvalidDataException or System.Text.Json.JsonException) { Console.Error.WriteLine($"knowledge: the server's files could not be used ({ex.Message}); building the provider map alone"); } } var download = Path.Combine(cache, "ip2asn-combined.tsv.gz"); var sourceDate = File.Exists(download) ? new DateTimeOffset(File.GetLastWriteTimeUtc(download), TimeSpan.Zero) : DateTimeOffset.UtcNow; if (!File.Exists(download) || File.GetLastWriteTimeUtc(download) < DateTime.UtcNow.AddHours(-12) || args.Contains("--refresh")) { using var response = await http.GetAsync(Fallback, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); sourceDate = response.Content.Headers.LastModified ?? DateTimeOffset.UtcNow; await using (var file = File.Create(download + ".tmp")) await response.Content.CopyToAsync(file); File.Move(download + ".tmp", download, overwrite: true); File.SetLastWriteTimeUtc(download, sourceDate.UtcDateTime); } using (var input = File.OpenRead(download)) using (var file = File.Create(Path.Combine(output, "ip2asn.bin"))) AsnTable.Write(file, sourceDate, Sources.IpToAsn(input)); foreach (var stale in new[] { "networks.bin", "carriers.json", "registry.bin" }) File.Delete(Path.Combine(output, stale)); string sha; using (var input = File.OpenRead(download)) sha = Convert.ToHexStringLower(SHA256.HashData(input)); File.WriteAllText(Path.Combine(output, "SOURCE.txt"), $"iptoasn.com, Public Domain Dedication and License (PDDL) 1.0\n{Fallback}\nlast-modified {sourceDate:O}\nsha256 {sha}\n"); Report(); return 0; void Report() { using var stream = File.OpenRead(Path.Combine(output, "ip2asn.bin")); var table = AsnTable.Read(stream); var path = Path.Combine(output, "networks.bin"); NetworkTable networks = null; if (File.Exists(path)) using (var input = File.OpenRead(path)) networks = NetworkTable.Read(input); path = Path.Combine(output, "registry.bin"); using var registry = File.Exists(path) ? RegistryTable.Open(File.OpenRead(path)) : null; Console.WriteLine($"knowledge: {table.Count} networks in the provider map (source {table.SourceDate:yyyy-MM-dd}), {networks?.Count ?? 0} with names or upstreams, {registry?.Count ?? 0} named address blocks"); } static string FindRoot() { var dir = new DirectoryInfo(Directory.GetCurrentDirectory()); while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "MeWeb.slnx"))) dir = dir.Parent; return dir?.FullName ?? throw new InvalidOperationException("run from inside the repository (MeWeb.slnx not found)"); }